@dreamlake/dreamlake-cli 0.117.0 → 0.118.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/api/agent.d.ts +1 -0
  2. package/dist/api/file-multipart.d.ts +6 -0
  3. package/dist/api/file-multipart.js +34 -0
  4. package/dist/api/file-source.d.ts +30 -0
  5. package/dist/api/file-source.js +165 -0
  6. package/dist/api/file-workspace.d.ts +79 -0
  7. package/dist/api/file-workspace.js +142 -0
  8. package/dist/api/index.d.ts +60 -1
  9. package/dist/api/index.js +8 -2
  10. package/dist/api/native-experiments.d.ts +25 -0
  11. package/dist/api/native-experiments.js +49 -4
  12. package/dist/api/sessions.d.ts +6 -0
  13. package/dist/cli/agents/chat.d.ts +21 -0
  14. package/dist/cli/agents/chat.js +126 -0
  15. package/dist/cli/agents/index.js +3 -1
  16. package/dist/cli/experiments/index.js +4 -0
  17. package/dist/cli/experiments/management.d.ts +16 -0
  18. package/dist/cli/experiments/management.js +82 -0
  19. package/dist/cli/experiments/transfer/adapter.d.ts +86 -0
  20. package/dist/cli/experiments/transfer/adapter.js +637 -0
  21. package/dist/cli/experiments/transfer/context.d.ts +13 -0
  22. package/dist/cli/experiments/transfer/context.js +20 -0
  23. package/dist/cli/experiments/transfer/download.d.ts +24 -0
  24. package/dist/cli/experiments/transfer/download.js +516 -0
  25. package/dist/cli/experiments/transfer/index.d.ts +2 -0
  26. package/dist/cli/experiments/transfer/index.js +38 -0
  27. package/dist/cli/experiments/transfer/local/safe-path.d.ts +29 -0
  28. package/dist/cli/experiments/transfer/local/safe-path.js +106 -0
  29. package/dist/cli/experiments/transfer/local/storage.d.ts +125 -0
  30. package/dist/cli/experiments/transfer/local/storage.js +432 -0
  31. package/dist/cli/experiments/transfer/transport.d.ts +7 -0
  32. package/dist/cli/experiments/transfer/transport.js +31 -0
  33. package/dist/cli/experiments/transfer/types.d.ts +48 -0
  34. package/dist/cli/experiments/transfer/types.js +1 -0
  35. package/dist/cli/experiments/transfer/upload.d.ts +58 -0
  36. package/dist/cli/experiments/transfer/upload.js +665 -0
  37. package/dist/cli/experiments/transfer/util/ansi.d.ts +24 -0
  38. package/dist/cli/experiments/transfer/util/ansi.js +70 -0
  39. package/dist/cli/experiments/transfer/util/glob.d.ts +4 -0
  40. package/dist/cli/experiments/transfer/util/glob.js +39 -0
  41. package/dist/cli/experiments/transfer/util/json.d.ts +6 -0
  42. package/dist/cli/experiments/transfer/util/json.js +58 -0
  43. package/dist/cli/experiments/transfer/util/pool.d.ts +5 -0
  44. package/dist/cli/experiments/transfer/util/pool.js +21 -0
  45. package/dist/cli/help-examples.generated.js +17 -0
  46. package/dist/cli/library/discover.d.ts +4 -2
  47. package/dist/cli/library/discover.js +5 -3
  48. package/dist/cli/sessions/control.js +2 -28
  49. package/dist/cli/sessions/runner/native-result.d.ts +34 -0
  50. package/dist/cli/sessions/runner/native-result.js +101 -0
  51. package/dist/cli/sessions/runner/serve.js +45 -2
  52. package/dist/cli/source/files.d.ts +2 -0
  53. package/dist/cli/source/files.js +39 -0
  54. package/dist/cli/source/index.js +2 -0
  55. package/dist/cli/version.d.ts +1 -1
  56. package/dist/cli/version.js +1 -1
  57. package/package.json +9 -9
@@ -0,0 +1,106 @@
1
+ /** .dash format compatibility, ported from the MIT-licensed ml-dash-cli. */
2
+ import { lstatSync } from "node:fs";
3
+ import path from "node:path";
4
+ export class UnsafePathError extends Error {
5
+ constructor(label, value, reason) {
6
+ super(`unsafe ${label} ${JSON.stringify(value)}: ${reason}`);
7
+ this.name = "UnsafePathError";
8
+ }
9
+ }
10
+ const WINDOWS_DRIVE = /^[A-Za-z]:/;
11
+ /** Rules every component shares, whatever its shape. */
12
+ function checkComponent(label, value) {
13
+ if (typeof value !== "string") {
14
+ throw new UnsafePathError(label, String(value), "is not a string");
15
+ }
16
+ if (value.includes("\0")) {
17
+ throw new UnsafePathError(label, value, "contains a NUL byte");
18
+ }
19
+ if (value.includes("\\")) {
20
+ // A backslash is a separator on Windows and a legal filename character on
21
+ // POSIX; accepting it would make the same metadata escape on one platform
22
+ // and not the other.
23
+ throw new UnsafePathError(label, value, "contains a backslash separator");
24
+ }
25
+ if (WINDOWS_DRIVE.test(value)) {
26
+ throw new UnsafePathError(label, value, "names a Windows drive");
27
+ }
28
+ if (value.startsWith("/") || path.isAbsolute(value)) {
29
+ throw new UnsafePathError(label, value, "is an absolute path");
30
+ }
31
+ }
32
+ /** One path segment: a filename, or a directory name that may not nest. */
33
+ export function safeSegment(label, value) {
34
+ checkComponent(label, value);
35
+ if (value === "")
36
+ throw new UnsafePathError(label, value, "is empty");
37
+ if (value.includes("/")) {
38
+ throw new UnsafePathError(label, value, "must be a single path segment");
39
+ }
40
+ if (value === "." || value === "..") {
41
+ throw new UnsafePathError(label, value, "is a relative path component");
42
+ }
43
+ return value;
44
+ }
45
+ /** A relative path that may nest: `owner/project/exp`, `train/loss`. */
46
+ export function safeRelativeSegments(label, value) {
47
+ checkComponent(label, value);
48
+ const segments = value.split("/");
49
+ for (const segment of segments) {
50
+ if (segment === "." || segment === "..") {
51
+ throw new UnsafePathError(label, value, `contains a '${segment}' segment`);
52
+ }
53
+ }
54
+ const kept = segments.filter((s) => s !== "");
55
+ if (kept.length === 0)
56
+ throw new UnsafePathError(label, value, "is empty");
57
+ return kept;
58
+ }
59
+ /** A literal component this code chose, not the server: `files`, `logs.jsonl`. */
60
+ export const literal = (value) => ({ label: "path component", value });
61
+ const contains = (root, target) => {
62
+ const rel = path.relative(root, target);
63
+ return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
64
+ };
65
+ /**
66
+ * Build a path under `root` from its parts, or throw.
67
+ *
68
+ * `mode: "write"` additionally refuses a target any part of which is a symlink
69
+ * — dangling or not — because a link inside the tree redirects the write out
70
+ * of it, and because following one to delete or overwrite would destroy a file
71
+ * that is not ours. `mode: "read"` keeps the lexical rules only, so a user who
72
+ * has symlinked their own metric data can still upload it.
73
+ */
74
+ export function resolveUnderRoot(root, parts, mode = "write") {
75
+ const segments = [];
76
+ for (const part of parts) {
77
+ if (part.nested)
78
+ segments.push(...safeRelativeSegments(part.label, part.value));
79
+ else
80
+ segments.push(safeSegment(part.label, part.value));
81
+ }
82
+ const rootAbs = path.resolve(root);
83
+ const target = path.resolve(rootAbs, ...segments);
84
+ if (!contains(rootAbs, target)) {
85
+ throw new UnsafePathError(parts.map((p) => p.value).join("/"), target, "resolves outside the root");
86
+ }
87
+ if (mode === "write") {
88
+ let current = rootAbs;
89
+ for (const segment of segments) {
90
+ current = path.join(current, segment);
91
+ let stats;
92
+ try {
93
+ stats = lstatSync(current);
94
+ }
95
+ catch {
96
+ // Does not exist: nothing here to redirect the write, and nothing
97
+ // below it can exist either.
98
+ break;
99
+ }
100
+ if (stats.isSymbolicLink()) {
101
+ throw new UnsafePathError(parts.map((p) => p.value).join("/"), current, "is reached through a symbolic link inside the root");
102
+ }
103
+ }
104
+ }
105
+ return target;
106
+ }
@@ -0,0 +1,125 @@
1
+ export declare const FILES_METADATA_FILENAME = ".files_metadata.json";
2
+ export declare const utcNowIso: () => string;
3
+ /** Local-mode ID, matching the Python `(ms << 12) | rand12` shape. */
4
+ export declare function generateSnowflakeId(): string;
5
+ export declare function sha256File(filePath: string): Promise<string>;
6
+ export interface LocalFileRecord {
7
+ id: string;
8
+ experimentId: string;
9
+ path: string;
10
+ filename: string;
11
+ description: string | null;
12
+ tags: string[];
13
+ bindrs: string[];
14
+ contentType: string;
15
+ sizeBytes: number;
16
+ checksum: string;
17
+ metadata: Record<string, unknown> | null;
18
+ uploadedAt: string;
19
+ updatedAt: string;
20
+ deletedAt: string | null;
21
+ }
22
+ /** One metric found on disk, in whichever layout it was written. */
23
+ export interface LocalMetric {
24
+ name: string;
25
+ layout: "directory" | "flat";
26
+ dataFile: string;
27
+ }
28
+ export declare class LocalStorage {
29
+ readonly rootPath: string;
30
+ constructor(rootPath: string);
31
+ /**
32
+ * The single boundary: every path this class touches is built here, from
33
+ * the root down, with the server-supplied parts and the literal ones
34
+ * checked alike. A derived directory is never treated as a new root.
35
+ */
36
+ private target;
37
+ private static prefixPart;
38
+ /**
39
+ * Experiment directory for a prefix of the form owner/project/folders…/name.
40
+ *
41
+ * Read mode: callers outside this class use it to look, and `upload` has to
42
+ * keep working on a tree where the user symlinked their own data. Every
43
+ * write below goes through `target(…, "write")` instead.
44
+ */
45
+ experimentDir(prefix: string): string;
46
+ /** Same directory, as a write target. */
47
+ private experimentWriteDir;
48
+ /**
49
+ * Create or merge experiment.json. Existing files are merged, never
50
+ * clobbered, so a download into a populated tree does not discard fields the
51
+ * server did not return.
52
+ */
53
+ createExperiment(args: {
54
+ project: string;
55
+ prefix: string;
56
+ description?: string | null;
57
+ tags?: string[] | null;
58
+ bindrs?: string[] | null;
59
+ writeProtected?: boolean;
60
+ metadata?: Record<string, unknown> | null;
61
+ }): string;
62
+ readExperiment(prefix: string): Record<string, any> | null;
63
+ /** Merge into parameters.json, bumping `version` the way the SDK does. */
64
+ writeParameters(prefix: string, data: Record<string, unknown>): void;
65
+ /** Read parameters, accepting both the versioned and the bare-dict shapes. */
66
+ readParameters(prefix: string): Record<string, unknown> | null;
67
+ /**
68
+ * Append log entries, continuing the `.log_sequence` counter so downloaded
69
+ * logs interleave correctly with anything already recorded locally.
70
+ */
71
+ appendLogs(prefix: string, entries: {
72
+ message: string;
73
+ level?: string;
74
+ timestamp?: string;
75
+ metadata?: unknown;
76
+ }[]): number;
77
+ readLogs(prefix: string): Record<string, any>[];
78
+ /**
79
+ * Every metric in the experiment, from both layouts.
80
+ *
81
+ * A name present in both is reported once, from the directory layout, which
82
+ * is the one the current SDK writes and the one that carries an index.
83
+ */
84
+ listMetrics(prefix: string): LocalMetric[];
85
+ /**
86
+ * Read one metric's data points as the values the server wants.
87
+ *
88
+ * Both layouts store one JSON object per line with the point under `data`;
89
+ * the directory layout adds `index` and `createdAt`. Lines without `data`
90
+ * are skipped and counted, never uploaded as `undefined`.
91
+ */
92
+ readMetricPoints(metric: LocalMetric): {
93
+ points: unknown[];
94
+ skipped: number;
95
+ bytes: number;
96
+ };
97
+ /** Append points in the directory layout, maintaining index and counts. */
98
+ appendBatchToMetric(prefix: string, metricName: string | null, dataPoints: unknown[]): void;
99
+ private filesDir;
100
+ /** The parts of a stored file's path, from the root down. */
101
+ private fileParts;
102
+ loadFilesMetadata(prefix: string): {
103
+ files: LocalFileRecord[];
104
+ };
105
+ private saveFilesMetadata;
106
+ /** Non-deleted file records for an experiment. */
107
+ listFiles(prefix: string, pathPrefix?: string, tags?: string[]): LocalFileRecord[];
108
+ /** Absolute on-disk path of a recorded file. */
109
+ filePath(prefix: string, record: Pick<LocalFileRecord, "id" | "path" | "filename">): string;
110
+ /** Copy a file in and record it, replacing any record with the same path+filename. */
111
+ writeFile(args: {
112
+ prefix: string;
113
+ project: string;
114
+ sourcePath: string;
115
+ path?: string;
116
+ filename: string;
117
+ description?: string | null;
118
+ tags?: string[] | null;
119
+ bindrs?: string[] | null;
120
+ metadata?: Record<string, unknown> | null;
121
+ checksum?: string;
122
+ contentType?: string;
123
+ sizeBytes?: number;
124
+ }): Promise<LocalFileRecord>;
125
+ }
@@ -0,0 +1,432 @@
1
+ /** .dash format compatibility, ported from the MIT-licensed ml-dash-cli. */
2
+ import { createHash } from "node:crypto";
3
+ import { appendFileSync, createReadStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
4
+ import { copyFile } from "node:fs/promises";
5
+ import path from "node:path";
6
+ import { literal, resolveUnderRoot } from "./safe-path.js";
7
+ export const FILES_METADATA_FILENAME = ".files_metadata.json";
8
+ export const utcNowIso = () => new Date().toISOString().replace(/Z$/, "Z");
9
+ /** Local-mode ID, matching the Python `(ms << 12) | rand12` shape. */
10
+ export function generateSnowflakeId() {
11
+ const ms = BigInt(Date.now());
12
+ const rand = BigInt(Math.floor(Math.random() * 4096));
13
+ return ((ms << 12n) | rand).toString();
14
+ }
15
+ export function sha256File(filePath) {
16
+ return new Promise((resolve, reject) => {
17
+ const hash = createHash("sha256");
18
+ createReadStream(filePath)
19
+ .on("error", reject)
20
+ .on("data", (c) => hash.update(c))
21
+ .on("end", () => resolve(hash.digest("hex")));
22
+ });
23
+ }
24
+ export class LocalStorage {
25
+ rootPath;
26
+ constructor(rootPath) {
27
+ const resolved = path.resolve(rootPath);
28
+ mkdirSync(resolved, { recursive: true });
29
+ // The root is the one path the user picked deliberately, so following its
30
+ // own links is what they asked for; everything below it is checked
31
+ // against this canonical form.
32
+ this.rootPath = realpathSync(resolved);
33
+ }
34
+ /**
35
+ * The single boundary: every path this class touches is built here, from
36
+ * the root down, with the server-supplied parts and the literal ones
37
+ * checked alike. A derived directory is never treated as a new root.
38
+ */
39
+ target(parts, mode = "write") {
40
+ return resolveUnderRoot(this.rootPath, parts, mode);
41
+ }
42
+ static prefixPart = (prefix) => ({
43
+ label: "experiment prefix",
44
+ value: prefix,
45
+ nested: true,
46
+ });
47
+ /**
48
+ * Experiment directory for a prefix of the form owner/project/folders…/name.
49
+ *
50
+ * Read mode: callers outside this class use it to look, and `upload` has to
51
+ * keep working on a tree where the user symlinked their own data. Every
52
+ * write below goes through `target(…, "write")` instead.
53
+ */
54
+ experimentDir(prefix) {
55
+ return this.target([LocalStorage.prefixPart(prefix)], "read");
56
+ }
57
+ /** Same directory, as a write target. */
58
+ experimentWriteDir(prefix) {
59
+ return this.target([LocalStorage.prefixPart(prefix)]);
60
+ }
61
+ // ── experiment metadata ────────────────────────────────────────────────────
62
+ /**
63
+ * Create or merge experiment.json. Existing files are merged, never
64
+ * clobbered, so a download into a populated tree does not discard fields the
65
+ * server did not return.
66
+ */
67
+ createExperiment(args) {
68
+ const prefixClean = args.prefix.replace(/\/+$/, "");
69
+ const dir = this.experimentWriteDir(prefixClean);
70
+ mkdirSync(dir, { recursive: true });
71
+ for (const sub of ["logs", "metrics", "files"]) {
72
+ mkdirSync(this.target([LocalStorage.prefixPart(prefixClean), literal(sub)]), { recursive: true });
73
+ }
74
+ const name = prefixClean.split("/").pop();
75
+ const file = this.target([LocalStorage.prefixPart(prefixClean), literal("experiment.json")]);
76
+ if (!existsSync(file)) {
77
+ writeFileSync(file, JSON.stringify({
78
+ name,
79
+ project: args.project,
80
+ description: args.description ?? null,
81
+ tags: args.tags ?? [],
82
+ bindrs: args.bindrs ?? [],
83
+ prefix: args.prefix,
84
+ metadata: args.metadata ?? null,
85
+ created_at: utcNowIso(),
86
+ write_protected: args.writeProtected ?? false,
87
+ }));
88
+ return dir;
89
+ }
90
+ let existing;
91
+ try {
92
+ existing = JSON.parse(readFileSync(file, "utf8"));
93
+ }
94
+ catch {
95
+ // Corrupt or truncated — rewrite rather than fail the whole download.
96
+ writeFileSync(file, JSON.stringify({
97
+ name,
98
+ project: args.project,
99
+ description: args.description ?? null,
100
+ tags: args.tags ?? [],
101
+ bindrs: args.bindrs ?? [],
102
+ prefix: args.prefix,
103
+ metadata: args.metadata ?? null,
104
+ created_at: utcNowIso(),
105
+ write_protected: args.writeProtected ?? false,
106
+ }));
107
+ return dir;
108
+ }
109
+ if (args.description != null)
110
+ existing.description = args.description;
111
+ if (args.tags != null)
112
+ existing.tags = args.tags;
113
+ if (args.bindrs != null)
114
+ existing.bindrs = args.bindrs;
115
+ if (args.writeProtected !== undefined)
116
+ existing.write_protected = args.writeProtected;
117
+ if (args.prefix != null)
118
+ existing.prefix = args.prefix;
119
+ if (args.metadata != null)
120
+ existing.metadata = args.metadata;
121
+ existing.updated_at = utcNowIso();
122
+ writeFileSync(file, JSON.stringify(existing));
123
+ return dir;
124
+ }
125
+ readExperiment(prefix) {
126
+ const file = this.target([LocalStorage.prefixPart(prefix), literal("experiment.json")], "read");
127
+ if (!existsSync(file))
128
+ return null;
129
+ try {
130
+ return JSON.parse(readFileSync(file, "utf8"));
131
+ }
132
+ catch {
133
+ return null;
134
+ }
135
+ }
136
+ // ── parameters ─────────────────────────────────────────────────────────────
137
+ /** Merge into parameters.json, bumping `version` the way the SDK does. */
138
+ writeParameters(prefix, data) {
139
+ mkdirSync(this.experimentWriteDir(prefix), { recursive: true });
140
+ const file = this.target([LocalStorage.prefixPart(prefix), literal("parameters.json")]);
141
+ let merged = {};
142
+ let version = 1;
143
+ if (existsSync(file)) {
144
+ try {
145
+ const doc = JSON.parse(readFileSync(file, "utf8"));
146
+ if (doc && typeof doc === "object") {
147
+ merged = (doc.data && typeof doc.data === "object" ? doc.data : doc);
148
+ version = typeof doc.version === "number" ? doc.version + 1 : 2;
149
+ }
150
+ }
151
+ catch {
152
+ merged = {};
153
+ }
154
+ }
155
+ writeFileSync(file, JSON.stringify({ version, data: { ...merged, ...data }, updatedAt: utcNowIso() }));
156
+ }
157
+ /** Read parameters, accepting both the versioned and the bare-dict shapes. */
158
+ readParameters(prefix) {
159
+ const file = this.target([LocalStorage.prefixPart(prefix), literal("parameters.json")], "read");
160
+ if (!existsSync(file))
161
+ return null;
162
+ try {
163
+ const doc = JSON.parse(readFileSync(file, "utf8"));
164
+ if (!doc || typeof doc !== "object" || Array.isArray(doc))
165
+ return null;
166
+ if ("data" in doc) {
167
+ return doc.data && typeof doc.data === "object" ? doc.data : null;
168
+ }
169
+ return doc;
170
+ }
171
+ catch {
172
+ return null;
173
+ }
174
+ }
175
+ // ── logs ───────────────────────────────────────────────────────────────────
176
+ /**
177
+ * Append log entries, continuing the `.log_sequence` counter so downloaded
178
+ * logs interleave correctly with anything already recorded locally.
179
+ */
180
+ appendLogs(prefix, entries) {
181
+ if (entries.length === 0)
182
+ return 0;
183
+ const logsDir = this.target([LocalStorage.prefixPart(prefix), literal("logs")]);
184
+ mkdirSync(logsDir, { recursive: true });
185
+ const logsFile = this.target([LocalStorage.prefixPart(prefix), literal("logs"), literal("logs.jsonl")]);
186
+ const seqFile = this.target([LocalStorage.prefixPart(prefix), literal("logs"), literal(".log_sequence")]);
187
+ let sequence = 0;
188
+ if (existsSync(seqFile)) {
189
+ const parsed = Number.parseInt(readFileSync(seqFile, "utf8").trim(), 10);
190
+ if (Number.isFinite(parsed))
191
+ sequence = parsed;
192
+ }
193
+ let out = "";
194
+ for (const e of entries) {
195
+ const record = {
196
+ sequenceNumber: sequence++,
197
+ timestamp: e.timestamp ?? "",
198
+ level: e.level ?? "info",
199
+ message: e.message,
200
+ };
201
+ if (e.metadata)
202
+ record.metadata = e.metadata;
203
+ out += JSON.stringify(record) + "\n";
204
+ }
205
+ appendFileSync(logsFile, out);
206
+ writeFileSync(seqFile, String(sequence));
207
+ return entries.length;
208
+ }
209
+ readLogs(prefix) {
210
+ const file = this.target([LocalStorage.prefixPart(prefix), literal("logs"), literal("logs.jsonl")], "read");
211
+ if (!existsSync(file))
212
+ return [];
213
+ const out = [];
214
+ for (const line of readFileSync(file, "utf8").split("\n")) {
215
+ if (!line.trim())
216
+ continue;
217
+ try {
218
+ out.push(JSON.parse(line));
219
+ }
220
+ catch {
221
+ /* skipped, as the Python uploader skipped unparseable lines */
222
+ }
223
+ }
224
+ return out;
225
+ }
226
+ // ── metrics ────────────────────────────────────────────────────────────────
227
+ /**
228
+ * Every metric in the experiment, from both layouts.
229
+ *
230
+ * A name present in both is reported once, from the directory layout, which
231
+ * is the one the current SDK writes and the one that carries an index.
232
+ */
233
+ listMetrics(prefix) {
234
+ const metricsDir = this.target([LocalStorage.prefixPart(prefix), literal("metrics")], "read");
235
+ if (!existsSync(metricsDir))
236
+ return [];
237
+ const found = new Map();
238
+ for (const entry of readdirSync(metricsDir, { withFileTypes: true })) {
239
+ if (entry.isDirectory()) {
240
+ const dataFile = path.join(metricsDir, entry.name, "data.jsonl");
241
+ if (existsSync(dataFile)) {
242
+ found.set(entry.name, { name: entry.name, layout: "directory", dataFile });
243
+ }
244
+ }
245
+ }
246
+ for (const entry of readdirSync(metricsDir, { withFileTypes: true })) {
247
+ if (entry.isFile() && entry.name.endsWith(".jsonl")) {
248
+ const name = entry.name.slice(0, -".jsonl".length);
249
+ if (!found.has(name)) {
250
+ found.set(name, { name, layout: "flat", dataFile: path.join(metricsDir, entry.name) });
251
+ }
252
+ }
253
+ }
254
+ return [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
255
+ }
256
+ /**
257
+ * Read one metric's data points as the values the server wants.
258
+ *
259
+ * Both layouts store one JSON object per line with the point under `data`;
260
+ * the directory layout adds `index` and `createdAt`. Lines without `data`
261
+ * are skipped and counted, never uploaded as `undefined`.
262
+ */
263
+ readMetricPoints(metric) {
264
+ const points = [];
265
+ let skipped = 0;
266
+ let bytes = 0;
267
+ if (!existsSync(metric.dataFile))
268
+ return { points, skipped, bytes };
269
+ for (const line of readFileSync(metric.dataFile, "utf8").split("\n")) {
270
+ if (!line.trim())
271
+ continue;
272
+ bytes += Buffer.byteLength(line, "utf8");
273
+ try {
274
+ const parsed = JSON.parse(line);
275
+ if (parsed === null || typeof parsed !== "object" || !("data" in parsed)) {
276
+ skipped++;
277
+ continue;
278
+ }
279
+ points.push(parsed.data);
280
+ }
281
+ catch {
282
+ skipped++;
283
+ }
284
+ }
285
+ return { points, skipped, bytes };
286
+ }
287
+ /** Append points in the directory layout, maintaining index and counts. */
288
+ appendBatchToMetric(prefix, metricName, dataPoints) {
289
+ const dirName = metricName === null ? "None" : String(metricName);
290
+ // A nested name ('train/loss') keeps nesting; '..' in one does not. Every
291
+ // part, including 'metrics' and the two filenames, is checked from the
292
+ // root — `metrics` being a link out of the tree is the case a check
293
+ // anchored at the metrics directory could not see.
294
+ const metricParts = [
295
+ LocalStorage.prefixPart(prefix),
296
+ literal("metrics"),
297
+ { label: "metric name", value: dirName, nested: true },
298
+ ];
299
+ const metricDir = this.target(metricParts);
300
+ mkdirSync(metricDir, { recursive: true });
301
+ const dataFile = this.target([...metricParts, literal("data.jsonl")]);
302
+ const metadataFile = this.target([...metricParts, literal("metadata.json")]);
303
+ let meta = {
304
+ metricId: `local-metric-${metricName}`,
305
+ name: metricName,
306
+ description: null,
307
+ tags: [],
308
+ metadata: null,
309
+ totalDataPoints: 0,
310
+ nextIndex: 0,
311
+ createdAt: utcNowIso(),
312
+ };
313
+ if (existsSync(metadataFile)) {
314
+ try {
315
+ meta = JSON.parse(readFileSync(metadataFile, "utf8"));
316
+ }
317
+ catch {
318
+ /* reinitialise from the default above */
319
+ }
320
+ }
321
+ const startIndex = meta.nextIndex ?? 0;
322
+ const batchTs = utcNowIso();
323
+ let out = "";
324
+ dataPoints.forEach((data, i) => {
325
+ out += JSON.stringify({ index: startIndex + i, data, createdAt: batchTs }) + "\n";
326
+ });
327
+ if (out)
328
+ appendFileSync(dataFile, out);
329
+ meta.nextIndex = startIndex + dataPoints.length;
330
+ meta.totalDataPoints = (meta.totalDataPoints ?? 0) + dataPoints.length;
331
+ meta.updatedAt = batchTs;
332
+ writeFileSync(metadataFile, JSON.stringify(meta));
333
+ }
334
+ // ── files ──────────────────────────────────────────────────────────────────
335
+ filesDir(prefix, mode = "write") {
336
+ return this.target([LocalStorage.prefixPart(prefix), literal("files")], mode);
337
+ }
338
+ /** The parts of a stored file's path, from the root down. */
339
+ fileParts(prefix, sub, ...rest) {
340
+ const parts = [LocalStorage.prefixPart(prefix), literal("files")];
341
+ // Python's storage lstrips a leading separator here; anything else in the
342
+ // path is checked rather than repaired.
343
+ const normalized = (sub ?? "").replace(/^\/+/, "");
344
+ if (normalized)
345
+ parts.push({ label: "file path", value: normalized, nested: true });
346
+ return [...parts, ...rest];
347
+ }
348
+ loadFilesMetadata(prefix) {
349
+ const file = this.target([LocalStorage.prefixPart(prefix), literal("files"), literal(FILES_METADATA_FILENAME)], "read");
350
+ if (!existsSync(file))
351
+ return { files: [] };
352
+ try {
353
+ const parsed = JSON.parse(readFileSync(file, "utf8"));
354
+ return Array.isArray(parsed?.files) ? parsed : { files: [] };
355
+ }
356
+ catch {
357
+ return { files: [] };
358
+ }
359
+ }
360
+ saveFilesMetadata(prefix, data) {
361
+ mkdirSync(this.filesDir(prefix), { recursive: true });
362
+ const file = this.target([LocalStorage.prefixPart(prefix), literal("files"), literal(FILES_METADATA_FILENAME)]);
363
+ // Write-then-rename: a crash mid-write would otherwise leave the manifest
364
+ // unparseable and orphan every file it listed. The temporary name is a
365
+ // target of its own, so a planted link cannot catch the write either.
366
+ const tmp = this.target([
367
+ LocalStorage.prefixPart(prefix),
368
+ literal("files"),
369
+ literal(`${FILES_METADATA_FILENAME}.tmp-${process.pid}`),
370
+ ]);
371
+ writeFileSync(tmp, JSON.stringify(data));
372
+ renameSync(tmp, file);
373
+ }
374
+ /** Non-deleted file records for an experiment. */
375
+ listFiles(prefix, pathPrefix, tags) {
376
+ let files = this.loadFilesMetadata(prefix).files.filter((f) => f.deletedAt === null);
377
+ if (pathPrefix)
378
+ files = files.filter((f) => (f.path ?? "").startsWith(pathPrefix));
379
+ if (tags?.length) {
380
+ const wanted = new Set(tags);
381
+ files = files.filter((f) => (f.tags ?? []).some((t) => wanted.has(t)));
382
+ }
383
+ return files;
384
+ }
385
+ /** Absolute on-disk path of a recorded file. */
386
+ filePath(prefix, record) {
387
+ return this.target(this.fileParts(prefix, record.path ?? "", { label: "file id", value: String(record.id) }, { label: "filename", value: record.filename }), "read");
388
+ }
389
+ /** Copy a file in and record it, replacing any record with the same path+filename. */
390
+ async writeFile(args) {
391
+ const fileId = generateSnowflakeId();
392
+ const idPart = { label: "file id", value: fileId };
393
+ const namePart = { label: "filename", value: args.filename };
394
+ const fileDir = this.target(this.fileParts(args.prefix, args.path ?? "", idPart));
395
+ const destination = this.target(this.fileParts(args.prefix, args.path ?? "", idPart, namePart));
396
+ const filename = args.filename;
397
+ mkdirSync(fileDir, { recursive: true });
398
+ await copyFile(args.sourcePath, destination);
399
+ const now = utcNowIso();
400
+ const record = {
401
+ id: fileId,
402
+ experimentId: `${args.project}/${args.prefix}`,
403
+ path: args.path ?? "",
404
+ filename,
405
+ description: args.description ?? null,
406
+ tags: args.tags ?? [],
407
+ bindrs: args.bindrs ?? [],
408
+ contentType: args.contentType ?? "",
409
+ sizeBytes: args.sizeBytes ?? statSync(destination).size,
410
+ checksum: args.checksum ?? "",
411
+ metadata: args.metadata ?? null,
412
+ uploadedAt: now,
413
+ updatedAt: now,
414
+ deletedAt: null,
415
+ };
416
+ const meta = this.loadFilesMetadata(args.prefix);
417
+ const existingIndex = meta.files.findIndex((f) => f.path === record.path && f.filename === record.filename && f.deletedAt === null);
418
+ if (existingIndex >= 0) {
419
+ const old = meta.files[existingIndex];
420
+ // Only the directory this manifest created for that record, and only
421
+ // if every component of it is a real directory under the root: never a
422
+ // link the record could point at something else through.
423
+ rmSync(this.target(this.fileParts(args.prefix, old.path ?? "", { label: "file id", value: String(old.id) })), { recursive: true, force: true });
424
+ meta.files[existingIndex] = record;
425
+ }
426
+ else {
427
+ meta.files.push(record);
428
+ }
429
+ this.saveFilesMetadata(args.prefix, meta);
430
+ return record;
431
+ }
432
+ }
@@ -0,0 +1,7 @@
1
+ import { ManagementError, type Context } from './context.js';
2
+ import type { FileMultipart } from '../../../api/file-multipart.js';
3
+ export declare const isNativeMetricName: (v: string) => boolean;
4
+ export declare const validName: (v: string) => boolean;
5
+ export declare const nativeError: (code: string, message: string, remediation: string, retryable?: boolean) => ManagementError;
6
+ export declare function call(c: Context, route: string, method?: string, body?: unknown, form?: FileMultipart): Promise<Response>;
7
+ export declare function json(c: Context, route: string, method?: string, body?: unknown, form?: FileMultipart): Promise<Record<string, any>>;