@dreamlake/dreamlake-cli 0.105.0 → 0.106.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,826 @@
1
+ // Run data of DreamLake-native Experiments: show/status, parameters, metrics, logs and files.
2
+ // Every request goes through the SDK client to the central DreamLake API under the selected
3
+ // namespace/project; run, metric and file identifiers are validated before any request.
4
+ import { createHash, randomUUID } from "node:crypto";
5
+ import { createWriteStream, existsSync, openAsBlob, statSync, readFileSync } from "node:fs";
6
+ import { link, unlink } from "node:fs/promises";
7
+ import { Readable, Transform } from "node:stream";
8
+ import { pipeline } from "node:stream/promises";
9
+ import path from "node:path";
10
+ import { ApiError, createTransport } from "../../api/client.js";
11
+ import { RequestNotSent } from "../../api/http.js";
12
+ import { nativeExperimentsApi, } from "../../api/native-experiments.js";
13
+ import { resolveRemote, resolveToken } from "../config.js";
14
+ import { emitJson, fail } from "../helpers.js";
15
+ /** A refusal with its own stable code, decided before any write (e.g. a capability the server does not list). */
16
+ class Refused extends Error {
17
+ code;
18
+ constructor(code, message) {
19
+ super(message);
20
+ this.code = code;
21
+ }
22
+ }
23
+ class Usage extends Error {
24
+ }
25
+ const RUN_ID = /^[0-9a-f]{24}$/;
26
+ const FILE_ID = /^[0-9a-f]{24}$/;
27
+ /**
28
+ * Metric and log appends (the native routes, as ml-dash): up to 1000 entries at any size, more when the
29
+ * entries serialize to at most 1 MiB, the bound ml-dash's request had.
30
+ */
31
+ const BATCH_COUNT = 1000;
32
+ const BATCH_BYTES = 1024 * 1024;
33
+ const batchSizeOk = (entries) => entries.length <= BATCH_COUNT || Buffer.byteLength(JSON.stringify(entries)) <= BATCH_BYTES;
34
+ const BATCH_RULE = "up to 1000 entries of any size, or more when they serialize to at most 1 MiB";
35
+ const LEVELS = ["info", "warn", "error", "debug", "fatal"];
36
+ const STATUSES = ["RUNNING", "COMPLETED", "FAILED", "CANCELLED"];
37
+ /** Long enough for a large upload or download over the gateway; reads keep the 30 s default. */
38
+ const TRANSFER_TIMEOUT_MS = 30 * 60 * 1000;
39
+ function runId(value) {
40
+ if (!RUN_ID.test(value))
41
+ throw new Usage("Run ID must be a DreamLake native Experiment ID.");
42
+ return value;
43
+ }
44
+ function folderId(value) {
45
+ if (!FILE_ID.test(value))
46
+ throw new Usage("Folder and operation IDs are DreamLake native IDs (24 hex characters).");
47
+ return value;
48
+ }
49
+ function fileId(value) {
50
+ if (!FILE_ID.test(value))
51
+ throw new Usage("File ID must be a DreamLake native Experiment file ID.");
52
+ return value;
53
+ }
54
+ /** A metric name as the native routes take it (ml-dash's): 1-256 characters, any but `/`, not `.` or `..`. Sent percent-encoded. */
55
+ function metricName(value) {
56
+ if (!value.length || value.includes("/") || value === "." || value === ".." || [...value].length > 256)
57
+ throw new Usage("Metric name must be 1-256 characters, without '/', and not '.' or '..'.");
58
+ return value;
59
+ }
60
+ function limit(value, max) {
61
+ if (value === undefined)
62
+ return undefined;
63
+ if (!/^[1-9][0-9]*$/.test(value) || Number(value) > max)
64
+ throw new Usage(`--limit must be an integer from 1 to ${max}.`);
65
+ return Number(value);
66
+ }
67
+ /** Offset cursors of logs and metric rows are the server's `nextCursor`, a nonnegative integer. */
68
+ function offsetCursor(value) {
69
+ if (value !== undefined && !/^(0|[1-9][0-9]*)$/.test(value))
70
+ throw new Usage("--cursor must be the nextCursor of the previous page.");
71
+ return value;
72
+ }
73
+ function tags(value) {
74
+ return value.split(",").map(tag => tag.trim()).filter(Boolean);
75
+ }
76
+ /** A JSON document from a file or `-` (stdin). Never a credential. */
77
+ function loadJson(source) {
78
+ let text;
79
+ try {
80
+ text = source === "-" ? readFileSync(0, "utf8") : readFileSync(source, "utf8");
81
+ }
82
+ catch {
83
+ throw new Usage(`Cannot read JSON from ${source === "-" ? "stdin" : source}.`);
84
+ }
85
+ try {
86
+ return JSON.parse(text);
87
+ }
88
+ catch {
89
+ throw new Usage(`${source === "-" ? "stdin" : source} is not valid JSON.`);
90
+ }
91
+ }
92
+ function isObject(value) {
93
+ return !!value && typeof value === "object" && !Array.isArray(value);
94
+ }
95
+ function requireData(opts) {
96
+ if (opts.data === undefined)
97
+ throw new Usage("--data <file|-> is required.");
98
+ return loadJson(opts.data);
99
+ }
100
+ /** `--description` ("" clears), `--tags` ("" clears) and `--metadata <file|->` (a JSON object or null). */
101
+ function metadataPatch(opts) {
102
+ const patch = {};
103
+ if (opts.description !== undefined)
104
+ patch.description = opts.description === "" ? null : opts.description;
105
+ if (opts.tags !== undefined)
106
+ patch.tags = tags(opts.tags);
107
+ if (opts.metadata !== undefined) {
108
+ const value = loadJson(opts.metadata);
109
+ if (value !== null && !isObject(value))
110
+ throw new Usage("--metadata must contain a JSON object or null.");
111
+ patch.metadata = value;
112
+ }
113
+ if (!Object.keys(patch).length)
114
+ throw new Usage("Give at least one of --description, --tags or --metadata.");
115
+ return patch;
116
+ }
117
+ function client(opts, timeoutMs) {
118
+ const token = resolveToken();
119
+ if (!token)
120
+ throw new Usage("DreamLake login is required; run dreamlake login.");
121
+ if (!opts.namespace || !opts.project)
122
+ throw new Usage("--namespace and --project are required.");
123
+ const transport = createTransport({ remote: resolveRemote(), token, ...(timeoutMs ? { timeoutMs } : {}),
124
+ fetch: (input, init) => globalThis.fetch(input, init) });
125
+ return nativeExperimentsApi(transport, opts.namespace, opts.project);
126
+ }
127
+ async function download(opts, run, file) {
128
+ const api = client(opts, TRANSFER_TIMEOUT_MS);
129
+ return verifiedDownload(opts, () => api.files.get(run, file), signal => api.files.download(run, file, { signal }), { runId: run, fileId: file });
130
+ }
131
+ /** A file's bytes to a new --out path, linked into place only once they match the file row's checksum. */
132
+ async function verifiedDownload(opts, row, body, ids) {
133
+ if (!opts.out)
134
+ throw new Usage("--out <path> is required.");
135
+ const out = path.resolve(opts.out);
136
+ if (existsSync(out))
137
+ throw new Usage("Download destination already exists; nothing was overwritten.");
138
+ // The file row gives the checksum the bytes are verified against before the destination appears.
139
+ const meta = (await row()).file;
140
+ if (!meta || typeof meta.checksum !== "string" || !/^[a-f0-9]{64}$/i.test(meta.checksum))
141
+ throw new Error("invalid file response");
142
+ const response = await body(AbortSignal.timeout(TRANSFER_TIMEOUT_MS));
143
+ if (!response.body)
144
+ throw new Error("empty download body");
145
+ // Bytes land in a private temp file beside the destination; only verified bytes are linked into place.
146
+ const temp = path.join(path.dirname(out), `.dreamlake-download-${process.pid}-${randomUUID()}`);
147
+ const hash = createHash("sha256");
148
+ let bytes = 0;
149
+ try {
150
+ await pipeline(Readable.fromWeb(response.body), new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); bytes += chunk.length; done(null, chunk); } }), createWriteStream(temp, { flags: "wx", mode: 0o600 }));
151
+ const checksum = hash.digest("hex");
152
+ if (checksum !== meta.checksum.toLowerCase() || statSync(temp).size !== bytes ||
153
+ typeof meta.sizeBytes === "number" && meta.sizeBytes !== bytes)
154
+ throw new Usage("Downloaded bytes do not match the file size or checksum; nothing was written to the destination.");
155
+ // link() fails when the destination exists, so a concurrent file is never replaced.
156
+ try {
157
+ await link(temp, out);
158
+ }
159
+ catch (error) {
160
+ throw error.code === "EEXIST" ? new Usage("Download destination already exists; nothing was overwritten.") : error;
161
+ }
162
+ return { ok: true, ...ids, path: out, sizeBytes: bytes, checksum };
163
+ }
164
+ finally {
165
+ await unlink(temp).catch(() => { });
166
+ }
167
+ }
168
+ /** A track topic: `/`-separated non-empty segments, up to 256 characters, no control characters. */
169
+ function topicName(value) {
170
+ if (!value || value.length > 256 || /[\u0000-\u001f\u007f]/.test(value) || value.split("/").some(part => !part))
171
+ throw new Usage("Topic must be 1-256 characters of '/'-separated, non-empty segments.");
172
+ return value;
173
+ }
174
+ function bound(value, name) {
175
+ if (value === undefined)
176
+ return undefined;
177
+ if (!Number.isFinite(Number(value)) || value.trim() === "")
178
+ throw new Usage(`--${name} must be a number (track timestamp).`);
179
+ return Number(value);
180
+ }
181
+ /** JSON prints to stdout; jsonl, parquet (Arrow IPC file) and mcap stream to a new --out file. */
182
+ async function readTrack(opts, run, raw) {
183
+ const format = opts.format ?? "json";
184
+ if (!["json", "jsonl", "parquet", "mcap"].includes(format))
185
+ throw new Usage("--format must be json, jsonl, parquet or mcap.");
186
+ // mcap alone accepts a trailing '/': every track under that prefix.
187
+ const topic = format === "mcap" && raw.endsWith("/") && raw.length > 1 ? `${topicName(raw.slice(0, -1))}/` : topicName(raw);
188
+ const query = { startTimestamp: bound(opts.start, "start"), endTimestamp: bound(opts.end, "end"),
189
+ columns: opts.columns === undefined ? undefined : tags(opts.columns) };
190
+ if (format === "json" && !opts.out)
191
+ return client(opts).tracks.read(run, topic, query);
192
+ if (!opts.out)
193
+ throw new Usage(`--out <path> is required for --format ${format}.`);
194
+ const out = path.resolve(opts.out);
195
+ if (existsSync(out))
196
+ throw new Usage("Output already exists; nothing was overwritten.");
197
+ const response = await client(opts, TRANSFER_TIMEOUT_MS).tracks.export(run, topic, { ...query, format: format }, { signal: AbortSignal.timeout(TRANSFER_TIMEOUT_MS) });
198
+ if (!response.body)
199
+ throw new Error("empty track body");
200
+ const temp = path.join(path.dirname(out), `.dreamlake-track-${process.pid}-${randomUUID()}`);
201
+ const hash = createHash("sha256");
202
+ let bytes = 0;
203
+ try {
204
+ await pipeline(Readable.fromWeb(response.body), new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); bytes += chunk.length; done(null, chunk); } }), createWriteStream(temp, { flags: "wx", mode: 0o600 }));
205
+ if (statSync(temp).size !== bytes)
206
+ throw new Usage("The track export was not fully written; nothing was written to the destination.");
207
+ try {
208
+ await link(temp, out);
209
+ }
210
+ catch (error) {
211
+ throw error.code === "EEXIST" ? new Usage("Output already exists; nothing was overwritten.") : error;
212
+ }
213
+ return { ok: true, runId: run, topic, format, path: out, sizeBytes: bytes, sha256: hash.digest("hex") };
214
+ }
215
+ finally {
216
+ await unlink(temp).catch(() => { });
217
+ }
218
+ }
219
+ /** Run one action; success is always JSON on stdout, errors follow `--json`. */
220
+ export async function runNativeRunData(action, opts, args) {
221
+ const write = !["show", "params-get", "metrics-list", "metrics-rows", "metrics-stats", "logs-read", "files-list", "files-get", "tracks-list", "tracks-read",
222
+ "files-media", "dashboard", "bindrs-list"].includes(action)
223
+ && !(action === "status" && args[1] === undefined);
224
+ try {
225
+ const run = runId(args[0] ?? "");
226
+ let result;
227
+ switch (action) {
228
+ case "show":
229
+ result = await client(opts).get(run);
230
+ break;
231
+ case "update": {
232
+ const { description, tags: list, metadata } = metadataPatch(opts);
233
+ result = await client(opts).update(run, { ...(description !== undefined ? { description } : {}), ...(list ? { tags: list } : {}),
234
+ ...(metadata !== undefined ? { metadata: metadata } : {}) });
235
+ break;
236
+ }
237
+ case "status": {
238
+ if (args[1] === undefined) {
239
+ const { experiment } = await client(opts).get(run);
240
+ result = { id: experiment.id, name: experiment.name, status: experiment.status, endedAt: experiment.endedAt ?? null };
241
+ }
242
+ else {
243
+ const status = args[1].toUpperCase();
244
+ if (!STATUSES.includes(status))
245
+ throw new Usage(`Status must be one of ${STATUSES.join(", ")}.`);
246
+ result = await client(opts).setStatus(run, status);
247
+ }
248
+ break;
249
+ }
250
+ case "params-get":
251
+ result = await client(opts).parameters.get(run);
252
+ break;
253
+ case "params-set": {
254
+ const data = requireData(opts);
255
+ if (!isObject(data) || !Object.keys(data).length)
256
+ throw new Usage("Parameters must be a non-empty JSON object.");
257
+ result = await client(opts).parameters.set(run, data);
258
+ break;
259
+ }
260
+ case "params-delete":
261
+ result = await client(opts).parameters.delete(run);
262
+ break;
263
+ case "metrics-list":
264
+ result = await client(opts).metrics.list(run);
265
+ break;
266
+ case "metrics-rows":
267
+ result = await client(opts).metrics.rows(run, metricName(args[1] ?? ""), { limit: limit(opts.limit, 1000), cursor: offsetCursor(opts.cursor) });
268
+ break;
269
+ case "metrics-append": {
270
+ const name = metricName(args[1] ?? "");
271
+ const points = requireData(opts);
272
+ if (!Array.isArray(points) || !points.length || !points.every(isObject) || !batchSizeOk(points))
273
+ throw new Usage(`Metric data must be a JSON array of objects: ${BATCH_RULE}.`);
274
+ const described = opts.description !== undefined || opts.tags !== undefined || opts.metadata !== undefined;
275
+ result = await client(opts).metrics.append(run, name, points, described ? metadataPatch(opts) : {});
276
+ break;
277
+ }
278
+ case "metrics-stats":
279
+ result = await client(opts).metrics.stats(run, metricName(args[1] ?? ""));
280
+ break;
281
+ case "metrics-metadata": {
282
+ const name = metricName(args[1] ?? "");
283
+ result = await client(opts).metrics.updateMetadata(run, name, metadataPatch(opts));
284
+ break;
285
+ }
286
+ case "logs-read": {
287
+ const query = { limit: limit(opts.limit, 1000), cursor: offsetCursor(opts.cursor) };
288
+ if (opts.level !== undefined) {
289
+ const levels = tags(opts.level);
290
+ if (!levels.length || levels.some(level => !LEVELS.includes(level)))
291
+ throw new Usage(`--level takes a comma list of ${LEVELS.join(", ")}.`);
292
+ query.level = levels;
293
+ }
294
+ for (const key of ["startTime", "endTime"]) {
295
+ const value = opts[key];
296
+ if (value !== undefined && !Number.isFinite(Date.parse(value)))
297
+ throw new Usage(`--${key === "startTime" ? "start" : "end"}-time must be an ISO timestamp.`);
298
+ if (value !== undefined)
299
+ query[key] = value;
300
+ }
301
+ if (opts.search !== undefined) {
302
+ if (!opts.search.length || opts.search.length > 1000)
303
+ throw new Usage("--search must be 1-1000 characters.");
304
+ query.search = opts.search;
305
+ }
306
+ if (opts.orderBy !== undefined) {
307
+ if (opts.orderBy !== "sequenceNumber" && opts.orderBy !== "timestamp")
308
+ throw new Usage("--order-by must be sequenceNumber or timestamp.");
309
+ query.orderBy = opts.orderBy;
310
+ }
311
+ if (opts.order !== undefined) {
312
+ if (opts.order !== "asc" && opts.order !== "desc")
313
+ throw new Usage("--order must be asc or desc.");
314
+ query.order = opts.order;
315
+ }
316
+ result = await client(opts).logs.list(run, query);
317
+ break;
318
+ }
319
+ case "logs-append": {
320
+ const entries = requireData(opts);
321
+ if (!Array.isArray(entries) || !entries.length || !batchSizeOk(entries) || !entries.every(entry => isObject(entry) && typeof entry.message === "string" && typeof entry.timestamp === "string" &&
322
+ Number.isFinite(Date.parse(entry.timestamp)) && LEVELS.includes(entry.level)))
323
+ throw new Usage(`Log data must be a JSON array of {timestamp, level, message} entries: ${BATCH_RULE}.`);
324
+ result = await client(opts).logs.append(run, entries);
325
+ break;
326
+ }
327
+ case "files-list":
328
+ result = await client(opts).files.list(run, { prefix: opts.prefix, tags: opts.tags === undefined ? undefined : tags(opts.tags) });
329
+ break;
330
+ case "files-get":
331
+ result = await client(opts).files.get(run, fileId(args[1] ?? ""));
332
+ break;
333
+ case "files-upload":
334
+ result = await client(opts, TRANSFER_TIMEOUT_MS).files.upload(run, { ...await localUpload(opts, args[1] ?? ""), prefix: opts.prefix });
335
+ break;
336
+ case "files-media":
337
+ result = await client(opts).files.media(run, fileId(args[1] ?? ""));
338
+ break;
339
+ case "dashboard": {
340
+ const api = client(opts);
341
+ if (opts.charts !== undefined) {
342
+ if (!["image", "line", "video"].includes(opts.charts))
343
+ throw new Usage("--charts must be image, line or video.");
344
+ result = await api.dashboard.charts(run, opts.charts);
345
+ break;
346
+ }
347
+ const sections = args[1] === undefined ? ["header", "metrics", "charts", "metadata"] : [args[1]];
348
+ if (!sections.every(field => ["header", "metrics", "charts", "metadata"].includes(field)))
349
+ throw new Usage("Dashboard section must be header, metrics, charts or metadata.");
350
+ const read = await Promise.all(sections.map(field => api.dashboard.section(run, field)));
351
+ result = args[1] === undefined ? Object.fromEntries(sections.map((field, i) => [field, read[i]])) : read[0];
352
+ break;
353
+ }
354
+ case "bindrs-list":
355
+ result = await client(opts).bindrs.list(run, runItem(opts));
356
+ break;
357
+ case "bindrs-attach":
358
+ case "bindrs-detach": {
359
+ if ((opts.bindr === undefined) === (opts.bindrId === undefined))
360
+ throw new Usage("Give exactly one of --bindr <name> or --bindr-id <id>.");
361
+ if (opts.bindrId !== undefined && !FILE_ID.test(opts.bindrId))
362
+ throw new Usage("--bindr-id must be a bindr ID (24 hex characters).");
363
+ if (opts.bindr !== undefined && (!opts.bindr.trim() || opts.bindr.trim().length > 120))
364
+ throw new Usage("--bindr must be a name of 1-120 characters.");
365
+ const item = runItem(opts);
366
+ const ref = { ...(opts.bindrId !== undefined ? { bindrId: opts.bindrId } : { name: opts.bindr.trim() }), ...(item ? { item } : {}) };
367
+ result = action === "bindrs-attach" ? await client(opts).bindrs.attach(run, ref) : await client(opts).bindrs.detach(run, ref);
368
+ break;
369
+ }
370
+ case "files-download":
371
+ result = await download(opts, run, fileId(args[1] ?? ""));
372
+ break;
373
+ case "files-update": {
374
+ const file = fileId(args[1] ?? "");
375
+ result = await client(opts).files.update(run, file, metadataPatch(opts));
376
+ break;
377
+ }
378
+ case "files-delete":
379
+ result = await client(opts).files.delete(run, fileId(args[1] ?? ""));
380
+ break;
381
+ case "tracks-list":
382
+ result = await client(opts).tracks.list(run, opts.topic === undefined ? {} : { topic: opts.topic });
383
+ break;
384
+ case "tracks-create": {
385
+ const described = opts.description !== undefined || opts.tags !== undefined || opts.metadata !== undefined;
386
+ result = await client(opts).tracks.create(run, topicName(args[1] ?? ""), described ? metadataPatch(opts) : {});
387
+ break;
388
+ }
389
+ case "tracks-append": {
390
+ const topic = topicName(args[1] ?? "");
391
+ const entries = requireData(opts);
392
+ if (!Array.isArray(entries) || !entries.length || entries.length > 10_000 ||
393
+ !entries.every(entry => isObject(entry) && typeof entry.timestamp === "number" && Number.isFinite(entry.timestamp)))
394
+ throw new Usage("Track data must be a JSON array of 1-10000 objects with a numeric timestamp.");
395
+ result = await client(opts).tracks.append(run, topic, entries);
396
+ break;
397
+ }
398
+ case "tracks-read":
399
+ result = await readTrack(opts, run, args[1] ?? "");
400
+ break;
401
+ }
402
+ emitJson(result);
403
+ return 0;
404
+ }
405
+ catch (error) {
406
+ return failure(error, opts, write);
407
+ }
408
+ }
409
+ /** A local file as an upload (streamed from disk), with the command's description, tags and metadata. */
410
+ async function localUpload(opts, local) {
411
+ let size;
412
+ try {
413
+ const stat = statSync(local);
414
+ if (!stat.isFile())
415
+ throw new Error();
416
+ size = stat.size;
417
+ }
418
+ catch {
419
+ throw new Usage(`${local} is not a readable file.`);
420
+ }
421
+ let metadata;
422
+ if (opts.metadata !== undefined) {
423
+ const value = loadJson(opts.metadata);
424
+ if (!isObject(value))
425
+ throw new Usage("--metadata must contain a JSON object.");
426
+ metadata = value;
427
+ }
428
+ const contentType = opts.contentType ?? "application/octet-stream";
429
+ const data = await openAsBlob(local, { type: contentType });
430
+ if (data.size !== size)
431
+ throw new Usage(`${local} changed while it was opened.`);
432
+ return { data, filename: opts.name ?? path.basename(local), contentType, description: opts.description,
433
+ tags: opts.tags === undefined ? undefined : tags(opts.tags), metadata };
434
+ }
435
+ /** `--item-kind`/`--item-id`: one item of the run, by the item's own id (from its list or read), or none. */
436
+ function runItem(opts) {
437
+ if (opts.itemKind === undefined && opts.itemId === undefined)
438
+ return undefined;
439
+ if (!["metric", "file", "parameters"].includes(opts.itemKind ?? "") || !FILE_ID.test(opts.itemId ?? ""))
440
+ throw new Usage("--item-kind (metric, file or parameters) and --item-id (the item's 24-hex ID) go together.");
441
+ return { kind: opts.itemKind, id: opts.itemId };
442
+ }
443
+ /** Writes that need a newer server check its advertised capabilities first: an older one is refused before any write. */
444
+ async function requireCapability(opts, capability) {
445
+ const status = await client(opts).status();
446
+ if (!Array.isArray(status.capabilities) || !status.capabilities.includes(capability))
447
+ throw new Refused("NATIVE_CAPABILITY_MISSING", `This DreamLake server does not offer '${capability}'; nothing was written.`);
448
+ }
449
+ /** Errors as JSON with `--json`, else a line on stderr; only the server's stable code is reflected, never its body. */
450
+ function failure(error, opts, write) {
451
+ let code = "NATIVE_REQUEST_FAILED";
452
+ let message;
453
+ if (error instanceof Usage) {
454
+ code = "NATIVE_INVALID_REQUEST";
455
+ message = error.message;
456
+ }
457
+ else if (error instanceof Refused) {
458
+ code = error.code;
459
+ message = error.message;
460
+ }
461
+ else if (error instanceof ApiError) {
462
+ const candidate = error.body?.error?.code;
463
+ code = typeof candidate === "string" && /^NATIVE_[A-Z0-9_]+$/.test(candidate) ? candidate : `NATIVE_HTTP_${error.status}`;
464
+ message = `Experiment request failed (HTTP ${error.status}).${write ? " Inspect the run before retrying a write." : ""}`;
465
+ }
466
+ else if (error instanceof RequestNotSent)
467
+ message = "The request was not sent.";
468
+ else
469
+ message = write ? "The request did not complete; its outcome is unknown. Inspect the run before retrying." : "The request did not complete.";
470
+ if (opts.json)
471
+ emitJson({ ok: false, error: { code, message } });
472
+ else
473
+ fail(`${code}: ${message}`);
474
+ return 1;
475
+ }
476
+ /** A folder or run name as the native routes take it: not blank, not `.`/`..`, no `/` or control character, <= 255 characters. */
477
+ function pathName(value) {
478
+ return value.trim() !== "" && value !== "." && value !== ".." && !value.includes("/") && !/[\u0000-\u001f\u007f]/.test(value) &&
479
+ [...value].length <= 255;
480
+ }
481
+ /** `a/b/c` as segments (at most 31), each a valid name. */
482
+ function pathSegments(value, what) {
483
+ const segments = value.split("/");
484
+ if (!value || segments.length > 31 || !segments.every(pathName))
485
+ throw new Usage(`${what} must be '/'-separated names (not blank, '.' or '..', no control characters, up to 255 characters each).`);
486
+ return segments;
487
+ }
488
+ const RUN_KEY = /^[A-Za-z0-9_-]{1,128}$/;
489
+ /** A project file scope: "root" or a folder ID. */
490
+ function scope(value) {
491
+ if (value !== "root" && !FILE_ID.test(value))
492
+ throw new Usage("--folder must be a folder ID or root.");
493
+ return value;
494
+ }
495
+ const FOLDER_OPERATIONS = ["trash", "restore", "protect", "unprotect", "purge"];
496
+ /**
497
+ * Runs and folders of the project (protocol 6): create-or-get by key or path, with the run's description, tags,
498
+ * metadata and protection written with it; a person's move/rename, protection and purge; folders and their
499
+ * operations; batch trash/purge. The DreamLake login is the person's: the server refuses people-only actions
500
+ * to any other session, and the CLI has no other.
501
+ */
502
+ export async function runNativeProject(action, opts, args) {
503
+ const write = !["folders-list", "folders-show", "folders-resolve", "folders-operation", "pfiles-list", "pfiles-show",
504
+ "pfiles-text", "pfiles-download", "pfiles-media"].includes(action);
505
+ // The capability a write needs from the server (listed by `GET status`); reads need none.
506
+ const needs = { move: "move", purge: "purge", "folders-create": "folders",
507
+ "folders-move": "folders", "folders-operate": "folders", batch: "folders", "pfiles-upload": "project-files",
508
+ "pfiles-replace": "project-files", "pfiles-update": "project-files", "pfiles-delete": "project-files" };
509
+ try {
510
+ let result;
511
+ // After the input is validated, right before the write: invalid input sends nothing at all.
512
+ const gate = async () => { if (needs[action])
513
+ await requireCapability(opts, needs[action]); };
514
+ switch (action) {
515
+ case "create": {
516
+ const segments = pathSegments(args[0] ?? "", "Run key or path");
517
+ const body = {};
518
+ if (opts.description !== undefined)
519
+ body.description = opts.description === "" ? null : opts.description;
520
+ if (opts.tags !== undefined)
521
+ body.tags = tags(opts.tags);
522
+ if (opts.metadata !== undefined) {
523
+ const value = loadJson(opts.metadata);
524
+ if (value !== null && !isObject(value))
525
+ throw new Usage("--metadata must contain a JSON object or null.");
526
+ body.metadata = value;
527
+ }
528
+ // Born protected, with its metadata, in the create's own transaction (ml-dash `write_protected`).
529
+ if (opts.writeProtected)
530
+ body.writeProtected = true;
531
+ if (opts.bindrs !== undefined) {
532
+ const names = [...new Set(tags(opts.bindrs))];
533
+ if (!names.length || names.length > 32 || names.some(name => name.length > 120))
534
+ throw new Usage("--bindrs takes 1-32 names of up to 120 characters.");
535
+ // With --write-protected too: the run is born protected and labelled in one create (the server refuses a
536
+ // new label on a run that is already protected).
537
+ body.bindrs = names;
538
+ }
539
+ const sent = Object.keys(body).length ? body : undefined;
540
+ // A root ASCII key keeps `PUT runs/:key`; any other name or a path is a run path (needs the run lifecycle).
541
+ const root = segments.length === 1 && RUN_KEY.test(segments[0]);
542
+ if (!root)
543
+ await requireCapability(opts, "run-paths");
544
+ result = root ? await client(opts).create(segments[0], sent) : await client(opts).createAt(segments, sent);
545
+ break;
546
+ }
547
+ case "move": {
548
+ const run = runId(args[0] ?? "");
549
+ if (opts.name === undefined && opts.folder === undefined)
550
+ throw new Usage("Give --name and/or --folder.");
551
+ if (opts.name !== undefined && !pathName(opts.name))
552
+ throw new Usage("--name must be a valid run name.");
553
+ if (opts.folder !== undefined && opts.folder !== "")
554
+ pathSegments(opts.folder, "--folder");
555
+ await gate();
556
+ result = await client(opts).update(run, { ...(opts.name !== undefined ? { name: opts.name } : {}),
557
+ ...(opts.folder !== undefined ? { folderPath: opts.folder } : {}) });
558
+ break;
559
+ }
560
+ case "protect":
561
+ case "unprotect":
562
+ result = await client(opts).update(runId(args[0] ?? ""), { writeProtected: action === "protect" });
563
+ break;
564
+ case "purge": {
565
+ const id = runId(args[0] ?? "");
566
+ await gate();
567
+ result = await client(opts).purge(id);
568
+ break;
569
+ }
570
+ case "folders-list":
571
+ result = await client(opts).folders.list({ ...(opts.parent !== undefined ? { parentId: folderId(opts.parent) } : {}),
572
+ limit: limit(opts.limit, 200), ...(opts.cursor !== undefined ? { cursor: folderId(opts.cursor) } : {}) });
573
+ break;
574
+ case "folders-create": {
575
+ const folder = pathSegments(args[0] ?? "", "Folder path").join("/");
576
+ await gate();
577
+ result = await client(opts).folders.create({ path: folder });
578
+ break;
579
+ }
580
+ case "folders-show":
581
+ result = await client(opts).folders.get(folderId(args[0] ?? ""));
582
+ break;
583
+ case "folders-move": {
584
+ const id = folderId(args[0] ?? "");
585
+ if (opts.name === undefined && opts.parent === undefined)
586
+ throw new Usage("Give --name and/or --parent.");
587
+ if (opts.name !== undefined && !pathName(opts.name))
588
+ throw new Usage("--name must be a valid folder name.");
589
+ const parentId = opts.parent === undefined ? undefined : opts.parent === "root" ? null : folderId(opts.parent);
590
+ await gate();
591
+ result = await client(opts).folders.update(id, { ...(opts.name !== undefined ? { name: opts.name } : {}),
592
+ ...(parentId !== undefined ? { parentId } : {}) });
593
+ break;
594
+ }
595
+ case "folders-resolve":
596
+ result = await client(opts).folders.byPath(pathSegments(args[0] ?? "", "Path"));
597
+ break;
598
+ case "folders-operate": {
599
+ const kind = args[0];
600
+ if (!FOLDER_OPERATIONS.includes(kind))
601
+ throw new Usage(`Folder operation must be ${FOLDER_OPERATIONS.join(", ")}.`);
602
+ const target = folderId(args[1] ?? "");
603
+ await gate();
604
+ result = await client(opts).folders.operate(target, kind);
605
+ break;
606
+ }
607
+ case "folders-operation":
608
+ result = await client(opts).folders.operation(folderId(args[0] ?? ""));
609
+ break;
610
+ case "pfiles-list":
611
+ result = await client(opts).projectFiles.list(opts.folder !== undefined ? { folderId: scope(opts.folder) } : {});
612
+ break;
613
+ case "pfiles-upload": {
614
+ const upload = await localUpload(opts, args[0] ?? "");
615
+ const folder = opts.folder !== undefined && scope(opts.folder) !== "root" ? { folderId: opts.folder } : {};
616
+ await gate();
617
+ result = await client(opts, TRANSFER_TIMEOUT_MS).projectFiles.upload({ ...upload, ...folder });
618
+ break;
619
+ }
620
+ case "pfiles-show":
621
+ result = await client(opts).projectFiles.get(fileId(args[0] ?? ""));
622
+ break;
623
+ case "pfiles-text":
624
+ result = await client(opts).projectFiles.text(fileId(args[0] ?? ""));
625
+ break;
626
+ case "pfiles-media":
627
+ result = await client(opts).projectFiles.media(fileId(args[0] ?? ""));
628
+ break;
629
+ case "pfiles-download": {
630
+ const id = fileId(args[0] ?? "");
631
+ const api = client(opts, TRANSFER_TIMEOUT_MS);
632
+ result = await verifiedDownload(opts, () => api.projectFiles.get(id), signal => api.projectFiles.download(id, { signal }), { fileId: id });
633
+ break;
634
+ }
635
+ case "pfiles-replace": {
636
+ const id = fileId(args[0] ?? "");
637
+ const upload = await localUpload(opts, args[1] ?? "");
638
+ await gate();
639
+ result = await client(opts, TRANSFER_TIMEOUT_MS).projectFiles.replace(id, { data: upload.data, filename: upload.filename, contentType: upload.contentType });
640
+ break;
641
+ }
642
+ case "pfiles-update": {
643
+ const id = fileId(args[0] ?? "");
644
+ const patch = opts.description !== undefined || opts.tags !== undefined || opts.metadata !== undefined ? metadataPatch(opts) : {};
645
+ if (opts.name !== undefined) {
646
+ if (!pathName(opts.name))
647
+ throw new Usage("--name must be a valid file name.");
648
+ patch.filename = opts.name;
649
+ }
650
+ if (opts.folder !== undefined)
651
+ patch.folderId = scope(opts.folder) === "root" ? null : opts.folder;
652
+ if (!Object.keys(patch).length)
653
+ throw new Usage("Give --name, --folder, --description, --tags or --metadata.");
654
+ await gate();
655
+ result = await client(opts).projectFiles.update(id, patch);
656
+ break;
657
+ }
658
+ case "pfiles-delete": {
659
+ const id = fileId(args[0] ?? "");
660
+ await gate();
661
+ result = await client(opts).projectFiles.delete(id);
662
+ break;
663
+ }
664
+ case "batch": {
665
+ const kind = args[0];
666
+ if (kind !== "trash" && kind !== "purge")
667
+ throw new Usage("Batch action must be trash or purge.");
668
+ const items = [...(opts.runs ? tags(opts.runs).map(id => ({ type: "run", id: runId(id) })) : []),
669
+ ...(opts.folders ? tags(opts.folders).map(id => ({ type: "folder", id: folderId(id) })) : [])];
670
+ if (!items.length || items.length > 100)
671
+ throw new Usage("Give 1-100 items with --runs and/or --folders.");
672
+ await gate();
673
+ result = await client(opts).batch(kind, items);
674
+ break;
675
+ }
676
+ }
677
+ emitJson(result);
678
+ return 0;
679
+ }
680
+ catch (error) {
681
+ return failure(error, opts, write);
682
+ }
683
+ }
684
+ /** `experiments runs show|status|params|metrics|logs|files`. */
685
+ export function registerNativeRunDataCommands(runs, common) {
686
+ const act = (action) => async (...input) => {
687
+ const opts = input.find(value => value && typeof value === "object" && !Array.isArray(value) && !value.commands);
688
+ const args = input.filter((value) => typeof value === "string");
689
+ process.exitCode = await runNativeRunData(action, opts, args);
690
+ };
691
+ const page = (command, max) => command
692
+ .option("--limit <count>", `page size, 1-${max}`)
693
+ .option("--cursor <cursor>", "nextCursor from the previous page");
694
+ const describe = (command) => command
695
+ .option("--description <text>", "description; an empty string clears it")
696
+ .option("--tags <list>", "comma-separated tags, replacing the set; an empty string clears them")
697
+ .option("--metadata <file|->", "JSON object (or null to clear) from a file or stdin");
698
+ common(runs.command("show").description("show one native run").argument("<run-id>")).action(act("show"));
699
+ describe(common(runs.command("update").description("change a native run's description, tags or metadata").argument("<run-id>")))
700
+ .action(act("update"));
701
+ const project = (action) => async (...input) => {
702
+ const opts = input.find(value => value && typeof value === "object" && !Array.isArray(value) && !value.commands);
703
+ process.exitCode = await runNativeProject(action, opts, input.filter((value) => typeof value === "string"));
704
+ };
705
+ common(runs.command("move").description("rename a native run and/or move it to a folder path").argument("<run-id>"))
706
+ .option("--name <name>", "new run name").option("--folder <path>", "folder path ('' for the project root)").action(project("move"));
707
+ common(runs.command("protect").description("write-protect a native run").argument("<run-id>")).action(project("protect"));
708
+ common(runs.command("unprotect").description("clear a native run's write protection").argument("<run-id>")).action(project("unprotect"));
709
+ common(runs.command("purge").description("permanently delete a trashed native run (project admin)").argument("<run-id>")).action(project("purge"));
710
+ const batch = runs.command("batch").description("trash or purge up to 100 runs and folders, one receipt each");
711
+ for (const kind of ["trash", "purge"])
712
+ common(batch.command(kind).description(`${kind} the given runs and folders`))
713
+ .option("--runs <ids>", "comma-separated run IDs").option("--folders <ids>", "comma-separated folder IDs")
714
+ .action((opts) => runNativeProject("batch", opts, [kind]).then(code => { process.exitCode = code; }));
715
+ common(runs.command("dashboard").description("show a native run's dashboard (its .dashrc, else the default), read-only")
716
+ .argument("<run-id>").argument("[section]", "header, metrics, charts or metadata (default: all four)"))
717
+ .option("--charts <type>", "only the charts of one type: image, line or video").action(act("dashboard"));
718
+ const item = (command) => command
719
+ .option("--item-kind <kind>", "a run item instead of the run: metric, file or parameters")
720
+ .option("--item-id <id>", "the item's own ID (from its list or read)");
721
+ const bindrs = runs.command("bindrs").description("bindrs of a native run or one of its items");
722
+ item(common(bindrs.command("list").description("list the bindrs a run (or item) is in").argument("<run-id>"))).action(act("bindrs-list"));
723
+ for (const [kind, text] of [["attach", "attach a run (or item) to a bindr; a new name creates the bindr"], ["detach", "detach a run (or item) from a bindr"]])
724
+ item(common(bindrs.command(kind).description(text).argument("<run-id>")))
725
+ .option("--bindr <name>", "bindr name").option("--bindr-id <id>", "bindr ID").action(act(`bindrs-${kind}`));
726
+ const pfiles = runs.command("project-files").description("project and folder files (README.md, .dashrc), not a run's");
727
+ common(pfiles.command("list").description("list the files of the project root or a folder"))
728
+ .option("--folder <folder-id|root>", "folder (default: the project root)").action(project("pfiles-list"));
729
+ describe(common(pfiles.command("upload").description("upload a local file to the project root or a folder").argument("<local-path>")))
730
+ .option("--folder <folder-id|root>", "folder (default: the project root)").option("--name <filename>", "stored name (default: the local name)")
731
+ .option("--content-type <type>", "MIME type").action(project("pfiles-upload"));
732
+ common(pfiles.command("show").description("show one project file").argument("<file-id>")).action(project("pfiles-show"));
733
+ common(pfiles.command("text").description("print a project file as UTF-8 text (up to 512 KiB)").argument("<file-id>")).action(project("pfiles-text"));
734
+ common(pfiles.command("download").description("download a project file to a new path, checksum-verified").argument("<file-id>"))
735
+ .requiredOption("--out <path>", "destination (must not exist)").action(project("pfiles-download"));
736
+ common(pfiles.command("media").description("a short-lived read URL for an image or video file").argument("<file-id>")).action(project("pfiles-media"));
737
+ common(pfiles.command("replace").description("replace a project file's bytes; its ID, name and metadata stay").argument("<file-id>").argument("<local-path>"))
738
+ .option("--content-type <type>", "MIME type").action(project("pfiles-replace"));
739
+ describe(common(pfiles.command("update").description("rename, move or change a project file's description, tags or metadata").argument("<file-id>")))
740
+ .option("--name <filename>", "new name").option("--folder <folder-id|root>", "new folder").action(project("pfiles-update"));
741
+ common(pfiles.command("delete").description("delete a project file").argument("<file-id>")).action(project("pfiles-delete"));
742
+ const folders = runs.command("folders").description("native run folders (ml-dash prefixes)");
743
+ page(common(folders.command("list").description("list child folders of a folder or the root")), 200)
744
+ .option("--parent <folder-id>", "parent folder (default: the project root)").action(project("folders-list"));
745
+ common(folders.command("create").description("create a folder path, with any missing parents").argument("<path>")).action(project("folders-create"));
746
+ common(folders.command("show").description("show one folder").argument("<folder-id>")).action(project("folders-show"));
747
+ common(folders.command("move").description("rename a folder and/or move it under another").argument("<folder-id>"))
748
+ .option("--name <name>", "new folder name").option("--parent <folder-id|root>", "new parent folder").action(project("folders-move"));
749
+ common(folders.command("resolve").description("resolve a path to its folder or run").argument("<path>")).action(project("folders-resolve"));
750
+ for (const kind of FOLDER_OPERATIONS)
751
+ common(folders.command(kind).description(`${kind} every run below a folder (asynchronous; returns an operation ID)`).argument("<folder-id>"))
752
+ .action((id, opts) => runNativeProject("folders-operate", opts, [kind, id]).then(code => { process.exitCode = code; }));
753
+ common(folders.command("operation").description("show a folder operation's state and counts").argument("<operation-id>"))
754
+ .action(project("folders-operation"));
755
+ common(runs.command("status").description("show a native run's status, or set RUNNING, COMPLETED, FAILED or CANCELLED")
756
+ .argument("<run-id>").argument("[status]")).action(act("status"));
757
+ const params = runs.command("params").description("native run parameters");
758
+ common(params.command("get").description("read a run's parameters").argument("<run-id>")).action(act("params-get"));
759
+ common(params.command("set").description("merge a JSON object into a run's parameters").argument("<run-id>"))
760
+ .requiredOption("--data <file|->", "JSON object from a file or stdin").action(act("params-set"));
761
+ common(params.command("delete").description("delete all of a run's parameters").argument("<run-id>")).action(act("params-delete"));
762
+ const metrics = runs.command("metrics").description("native run metrics");
763
+ common(metrics.command("list").description("list a run's metrics").argument("<run-id>")).action(act("metrics-list"));
764
+ page(common(metrics.command("rows").description("read one page of a metric's rows").argument("<run-id>").argument("<metric>")), 1000)
765
+ .action(act("metrics-rows"));
766
+ common(metrics.command("append").description("append data points to a metric (up to 1000, or more within 1 MiB)").argument("<run-id>").argument("<metric>"))
767
+ .requiredOption("--data <file|->", "JSON array of point objects from a file or stdin")
768
+ .option("--description <text>", "description, stored only when this append creates the metric")
769
+ .option("--tags <list>", "comma-separated tags, stored only when this append creates the metric")
770
+ .option("--metadata <file|->", "JSON object, stored only when this append creates the metric")
771
+ .action(act("metrics-append"));
772
+ common(metrics.command("stats").description("show a metric's point count and metadata").argument("<run-id>").argument("<metric>"))
773
+ .action(act("metrics-stats"));
774
+ describe(common(metrics.command("metadata").description("update a metric's description, tags or metadata")
775
+ .argument("<run-id>").argument("<metric>"))).action(act("metrics-metadata"));
776
+ const logs = runs.command("logs").description("native run logs");
777
+ page(common(logs.command("read").description("read one page of a run's logs, optionally filtered").argument("<run-id>")), 1000)
778
+ .option("--level <list>", "comma list of info, warn, error, debug, fatal")
779
+ .option("--start-time <iso>", "inclusive lower timestamp bound")
780
+ .option("--end-time <iso>", "inclusive upper timestamp bound")
781
+ .option("--search <text>", "case-sensitive substring of the message")
782
+ .option("--order-by <field>", "sequenceNumber or timestamp")
783
+ .option("--order <direction>", "asc or desc")
784
+ .action(act("logs-read"));
785
+ common(logs.command("append").description("append log entries (up to 1000, or more within 1 MiB)").argument("<run-id>"))
786
+ .requiredOption("--data <file|->", "JSON array of {timestamp, level, message, metadata?} from a file or stdin")
787
+ .action(act("logs-append"));
788
+ const tracks = runs.command("tracks").description("native run tracks (time-indexed entries on Dash-homed runs)");
789
+ common(tracks.command("list").description("list a run's tracks").argument("<run-id>"))
790
+ .option("--topic <topic>", "only this topic").action(act("tracks-list"));
791
+ describe(common(tracks.command("create").description("create a track, or return the existing one").argument("<run-id>").argument("<topic>")))
792
+ .action(act("tracks-create"));
793
+ common(tracks.command("append").description("append 1-10000 entries to a track in one commit").argument("<run-id>").argument("<topic>"))
794
+ .requiredOption("--data <file|->", "JSON array of {timestamp, ...data} from a file or stdin").action(act("tracks-append"));
795
+ common(tracks.command("read").description("read or export a track").argument("<run-id>").argument("<topic>"))
796
+ .option("--format <format>", "json (default), jsonl, parquet (Arrow IPC file) or mcap")
797
+ .option("--start <timestamp>", "inclusive lower timestamp bound")
798
+ .option("--end <timestamp>", "inclusive upper timestamp bound")
799
+ .option("--columns <list>", "comma-separated data columns (timestamp is always included)")
800
+ .option("--out <path>", "write to a new file (required for jsonl, parquet and mcap)")
801
+ .action(act("tracks-read"));
802
+ const files = runs.command("files").description("native run files");
803
+ common(files.command("list").description("list a run's files").argument("<run-id>"))
804
+ .option("--prefix <path>", "only files under this prefix")
805
+ .option("--tags <list>", "only files with these comma-separated tags")
806
+ .action(act("files-list"));
807
+ common(files.command("get").description("show one file's metadata").argument("<run-id>").argument("<file-id>")).action(act("files-get"));
808
+ common(files.command("upload").description("upload a local file to a run").argument("<run-id>").argument("<path>"))
809
+ .option("--name <filename>", "stored filename (default: the local basename)")
810
+ .option("--prefix <path>", "destination prefix in the run, default /")
811
+ .option("--content-type <mime>", "MIME type, default application/octet-stream")
812
+ .option("--description <text>", "file description")
813
+ .option("--tags <list>", "comma-separated tags")
814
+ .option("--metadata <file|->", "JSON object from a file or stdin")
815
+ .action(act("files-upload"));
816
+ common(files.command("download").description("download a file to a new local path, verifying its checksum")
817
+ .argument("<run-id>").argument("<file-id>"))
818
+ .requiredOption("--out <path>", "destination; an existing file is never overwritten")
819
+ .action(act("files-download"));
820
+ describe(common(files.command("update").description("update a file's description, tags or metadata")
821
+ .argument("<run-id>").argument("<file-id>"))).action(act("files-update"));
822
+ common(files.command("delete").description("delete a file from a run").argument("<run-id>").argument("<file-id>"))
823
+ .action(act("files-delete"));
824
+ common(files.command("media").description("a short-lived read URL for a run's image or video file").argument("<run-id>").argument("<file-id>"))
825
+ .action(act("files-media"));
826
+ }