@dreamlake/dreamlake-cli 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/file-multipart.d.ts +6 -0
- package/dist/api/file-multipart.js +11 -0
- package/dist/api/index.d.ts +44 -0
- package/dist/api/native-experiments.d.ts +25 -0
- package/dist/api/native-experiments.js +49 -3
- package/dist/cli/experiments/index.js +4 -0
- package/dist/cli/experiments/management.d.ts +16 -0
- package/dist/cli/experiments/management.js +82 -0
- package/dist/cli/experiments/transfer/adapter.d.ts +86 -0
- package/dist/cli/experiments/transfer/adapter.js +637 -0
- package/dist/cli/experiments/transfer/context.d.ts +13 -0
- package/dist/cli/experiments/transfer/context.js +20 -0
- package/dist/cli/experiments/transfer/download.d.ts +24 -0
- package/dist/cli/experiments/transfer/download.js +516 -0
- package/dist/cli/experiments/transfer/index.d.ts +2 -0
- package/dist/cli/experiments/transfer/index.js +38 -0
- package/dist/cli/experiments/transfer/local/safe-path.d.ts +29 -0
- package/dist/cli/experiments/transfer/local/safe-path.js +106 -0
- package/dist/cli/experiments/transfer/local/storage.d.ts +125 -0
- package/dist/cli/experiments/transfer/local/storage.js +432 -0
- package/dist/cli/experiments/transfer/transport.d.ts +7 -0
- package/dist/cli/experiments/transfer/transport.js +31 -0
- package/dist/cli/experiments/transfer/types.d.ts +48 -0
- package/dist/cli/experiments/transfer/types.js +1 -0
- package/dist/cli/experiments/transfer/upload.d.ts +58 -0
- package/dist/cli/experiments/transfer/upload.js +665 -0
- package/dist/cli/experiments/transfer/util/ansi.d.ts +24 -0
- package/dist/cli/experiments/transfer/util/ansi.js +70 -0
- package/dist/cli/experiments/transfer/util/glob.d.ts +4 -0
- package/dist/cli/experiments/transfer/util/glob.js +39 -0
- package/dist/cli/experiments/transfer/util/json.d.ts +6 -0
- package/dist/cli/experiments/transfer/util/json.js +58 -0
- package/dist/cli/experiments/transfer/util/pool.d.ts +5 -0
- package/dist/cli/experiments/transfer/util/pool.js +21 -0
- package/dist/cli/help-examples.generated.js +12 -0
- package/dist/cli/version.d.ts +1 -1
- package/dist/cli/version.js +1 -1
- package/package.json +9 -9
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { createTransport } from '../../../api/client.js';
|
|
2
|
+
import { resolveRemote, resolveToken } from '../../config.js';
|
|
3
|
+
export class ManagementError extends Error {
|
|
4
|
+
code;
|
|
5
|
+
remediation;
|
|
6
|
+
retryable;
|
|
7
|
+
constructor(code, message, remediation, retryable = false) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.code = code;
|
|
10
|
+
this.remediation = remediation;
|
|
11
|
+
this.retryable = retryable;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export function context(_args) {
|
|
15
|
+
const token = resolveToken();
|
|
16
|
+
if (!token)
|
|
17
|
+
throw new ManagementError('NATIVE_LOGIN_REQUIRED', 'DreamLake login is required.', 'Run dl login for the intended environment.');
|
|
18
|
+
const url = resolveRemote();
|
|
19
|
+
return { url, transport: createTransport({ remote: url, token }) };
|
|
20
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { ParsedArgs, RemoteClient } from "./types.js";
|
|
2
|
+
export interface RemoteExperiment {
|
|
3
|
+
project: string;
|
|
4
|
+
experiment: string;
|
|
5
|
+
experimentId: string;
|
|
6
|
+
owner?: string;
|
|
7
|
+
prefix?: string;
|
|
8
|
+
description?: string | null;
|
|
9
|
+
tags: string[];
|
|
10
|
+
metricNames: string[];
|
|
11
|
+
fileCount: number;
|
|
12
|
+
status: string;
|
|
13
|
+
/** Native discovery lists runs only; metric and file counts are learned during the download. */
|
|
14
|
+
countsUnknown?: boolean;
|
|
15
|
+
/** Native run metadata, recorded in experiment.json. */
|
|
16
|
+
metadata?: Record<string, unknown> | null;
|
|
17
|
+
bindrs?: string[];
|
|
18
|
+
writeProtected?: boolean;
|
|
19
|
+
}
|
|
20
|
+
/** Where an experiment lands under the .dash root. */
|
|
21
|
+
export declare function localPrefixFor(exp: RemoteExperiment): string;
|
|
22
|
+
/** What one experiment's download needs from a server: the standalone RemoteClient, or the native adapter. */
|
|
23
|
+
export type ExperimentDownloadClient = Pick<RemoteClient, "getParameters" | "queryLogs" | "listMetrics" | "getMetricStats" | "downloadMetricChunk" | "getMetricData" | "listFiles" | "downloadFileStreaming">;
|
|
24
|
+
export declare function run(args: ParsedArgs): Promise<number>;
|
|
@@ -0,0 +1,516 @@
|
|
|
1
|
+
/** Native experiment transfer using the interoperable .dash directory format. */
|
|
2
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync, renameSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { randomUUID } from "node:crypto";
|
|
6
|
+
import { LocalStorage, sha256File } from "./local/storage.js";
|
|
7
|
+
import { resolveUnderRoot } from "./local/safe-path.js";
|
|
8
|
+
import { bold, cyan, dim, formatBytes, green, red, renderTable, yellow } from "./util/ansi.js";
|
|
9
|
+
import { asId } from "./util/json.js";
|
|
10
|
+
import { runPool } from "./util/pool.js";
|
|
11
|
+
/** Where an experiment lands under the .dash root. */
|
|
12
|
+
export function localPrefixFor(exp) {
|
|
13
|
+
if (exp.prefix)
|
|
14
|
+
return exp.prefix.replace(/^\/+|\/+$/g, "");
|
|
15
|
+
if (exp.owner)
|
|
16
|
+
return `${exp.owner}/${localPrefixFor(exp)}`;
|
|
17
|
+
return `${localPrefixFor(exp)}`;
|
|
18
|
+
}
|
|
19
|
+
const loadState = (file) => {
|
|
20
|
+
if (!existsSync(file))
|
|
21
|
+
return null;
|
|
22
|
+
try {
|
|
23
|
+
const data = JSON.parse(readFileSync(file, "utf8"));
|
|
24
|
+
if (typeof data?.remote_url !== "string")
|
|
25
|
+
return null;
|
|
26
|
+
return {
|
|
27
|
+
remote_url: data.remote_url,
|
|
28
|
+
local_path: data.local_path ?? "",
|
|
29
|
+
completed_experiments: data.completed_experiments ?? [],
|
|
30
|
+
failed_experiments: data.failed_experiments ?? [],
|
|
31
|
+
in_progress_experiment: data.in_progress_experiment ?? null,
|
|
32
|
+
timestamp: data.timestamp ?? null,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
catch (e) {
|
|
36
|
+
console.log(yellow(`Warning: Could not load state file: ${e.message}`));
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
const saveState = (file, state) => {
|
|
41
|
+
state.timestamp = new Date().toISOString();
|
|
42
|
+
writeFileSync(file, JSON.stringify(state, null, 2));
|
|
43
|
+
};
|
|
44
|
+
async function downloadExperiment(storage, client, exp, opts) {
|
|
45
|
+
const scratch = mkdtempSync(path.join(storage.rootPath, ".dl-transfer-"));
|
|
46
|
+
const staged = new LocalStorage(scratch);
|
|
47
|
+
const prefix = localPrefixFor(exp);
|
|
48
|
+
const backup = path.join(scratch, ".previous-" + randomUUID());
|
|
49
|
+
let target;
|
|
50
|
+
let backedUp = false;
|
|
51
|
+
try {
|
|
52
|
+
const result = await downloadExperimentInto(staged, client, exp, opts);
|
|
53
|
+
if (!result.success)
|
|
54
|
+
return result;
|
|
55
|
+
target = resolveUnderRoot(storage.rootPath, [{ label: "experiment prefix", value: prefix, nested: true }], "write");
|
|
56
|
+
if (existsSync(target)) {
|
|
57
|
+
if (!opts.overwrite)
|
|
58
|
+
throw new Error("Experiment appeared locally during download; use --overwrite explicitly.");
|
|
59
|
+
renameSync(target, backup);
|
|
60
|
+
backedUp = true;
|
|
61
|
+
}
|
|
62
|
+
mkdirSync(path.dirname(target), { recursive: true });
|
|
63
|
+
renameSync(staged.experimentDir(prefix), target);
|
|
64
|
+
backedUp = false;
|
|
65
|
+
return result;
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
if (backedUp && target) {
|
|
69
|
+
try {
|
|
70
|
+
renameSync(backup, target);
|
|
71
|
+
backedUp = false;
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
throw new Error(`Previous experiment is preserved at ${backup}; restore it before retrying.`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return { experiment: prefix, success: false, downloaded: {}, failed: {}, bytesDownloaded: 0,
|
|
78
|
+
errors: [error.message] };
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
if (!backedUp)
|
|
82
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
async function downloadExperimentInto(storage, client, exp, opts) {
|
|
86
|
+
const result = {
|
|
87
|
+
experiment: `${localPrefixFor(exp)}`,
|
|
88
|
+
success: false,
|
|
89
|
+
downloaded: {},
|
|
90
|
+
failed: {},
|
|
91
|
+
errors: [],
|
|
92
|
+
bytesDownloaded: 0,
|
|
93
|
+
};
|
|
94
|
+
const prefix = localPrefixFor(exp);
|
|
95
|
+
const note = (line) => {
|
|
96
|
+
if (opts.verbose)
|
|
97
|
+
console.log(line);
|
|
98
|
+
};
|
|
99
|
+
try {
|
|
100
|
+
note(dim(` Downloading ${localPrefixFor(exp)}`));
|
|
101
|
+
storage.createExperiment({
|
|
102
|
+
project: exp.project,
|
|
103
|
+
prefix,
|
|
104
|
+
description: exp.description,
|
|
105
|
+
tags: exp.tags,
|
|
106
|
+
bindrs: exp.bindrs ?? [],
|
|
107
|
+
writeProtected: exp.writeProtected,
|
|
108
|
+
metadata: exp.metadata ?? null,
|
|
109
|
+
});
|
|
110
|
+
if (!opts.skipParams) {
|
|
111
|
+
try {
|
|
112
|
+
const params = await client.getParameters(exp.experimentId);
|
|
113
|
+
if (params && Object.keys(params).length > 0) {
|
|
114
|
+
storage.writeParameters(prefix, params);
|
|
115
|
+
result.downloaded.parameters = Object.keys(params).length;
|
|
116
|
+
result.bytesDownloaded += Buffer.byteLength(JSON.stringify(params), "utf8");
|
|
117
|
+
note(` ${green("✓")} ${result.downloaded.parameters} parameters`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
catch (e) {
|
|
121
|
+
(result.failed.parameters ??= []).push(e.message);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (!opts.skipLogs) {
|
|
125
|
+
try {
|
|
126
|
+
let offset = 0;
|
|
127
|
+
let total = 0;
|
|
128
|
+
for (;;) {
|
|
129
|
+
const page = await client.queryLogs(exp.experimentId, {
|
|
130
|
+
limit: opts.batchSize,
|
|
131
|
+
offset,
|
|
132
|
+
orderBy: "sequenceNumber",
|
|
133
|
+
order: "asc",
|
|
134
|
+
});
|
|
135
|
+
const logs = page?.logs ?? [];
|
|
136
|
+
if (logs.length === 0)
|
|
137
|
+
break;
|
|
138
|
+
storage.appendLogs(prefix, logs.map((log) => ({
|
|
139
|
+
message: log.message,
|
|
140
|
+
level: log.level,
|
|
141
|
+
timestamp: log.timestamp,
|
|
142
|
+
metadata: log.metadata,
|
|
143
|
+
})));
|
|
144
|
+
total += logs.length;
|
|
145
|
+
result.bytesDownloaded += logs.reduce((n, log) => n + Buffer.byteLength(JSON.stringify(log), "utf8"), 0);
|
|
146
|
+
if (!page?.hasMore)
|
|
147
|
+
break;
|
|
148
|
+
offset += logs.length;
|
|
149
|
+
}
|
|
150
|
+
result.downloaded.logs = total;
|
|
151
|
+
note(` ${green("✓")} ${total} log entries`);
|
|
152
|
+
}
|
|
153
|
+
catch (e) {
|
|
154
|
+
(result.failed.logs ??= []).push(e.message);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (!opts.skipMetrics) {
|
|
158
|
+
let names = exp.metricNames;
|
|
159
|
+
if (names.length === 0) {
|
|
160
|
+
// The list query only carries names when the server filled them in;
|
|
161
|
+
// asking directly is what keeps a metric from being skipped silently.
|
|
162
|
+
try {
|
|
163
|
+
names = (await client.listMetrics(exp.experimentId))
|
|
164
|
+
.map((m) => m?.name)
|
|
165
|
+
.filter((n) => typeof n === "string");
|
|
166
|
+
}
|
|
167
|
+
catch (e) {
|
|
168
|
+
(result.failed.metrics ??= []).push(e.message);
|
|
169
|
+
names = [];
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if (names.length > 0) {
|
|
173
|
+
const outcomes = await runPool(names, opts.maxConcurrentMetrics, (name) => downloadMetric(storage, client, exp.experimentId, prefix, name, opts.batchSize));
|
|
174
|
+
let ok = 0;
|
|
175
|
+
outcomes.forEach((outcome, i) => {
|
|
176
|
+
if (outcome.error) {
|
|
177
|
+
(result.failed.metrics ??= []).push(`${names[i]}: ${outcome.error.message}`);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
ok++;
|
|
181
|
+
result.bytesDownloaded += outcome.value.bytes;
|
|
182
|
+
note(` ${green("✓")} metric '${names[i]}': ${outcome.value.points} points`);
|
|
183
|
+
});
|
|
184
|
+
result.downloaded.metrics = ok;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (!opts.skipFiles) {
|
|
188
|
+
await downloadFiles(storage, client, exp, prefix, result, opts);
|
|
189
|
+
}
|
|
190
|
+
const failedSections = Object.keys(result.failed);
|
|
191
|
+
if (failedSections.length > 0) {
|
|
192
|
+
result.success = false;
|
|
193
|
+
result.errors.push(`${failedSections.join(", ")} failed: ` +
|
|
194
|
+
failedSections.flatMap((s) => result.failed[s]).slice(0, 3).join("; "));
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
result.success = true;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
catch (e) {
|
|
201
|
+
result.success = false;
|
|
202
|
+
result.errors.push(e.message);
|
|
203
|
+
}
|
|
204
|
+
return result;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* One metric, chunks first.
|
|
208
|
+
*
|
|
209
|
+
* Sealed chunks come down in parallel and the still-open buffer is fetched
|
|
210
|
+
* separately; if any of that fails the whole metric falls back to index
|
|
211
|
+
* pagination, which every server supports.
|
|
212
|
+
*/
|
|
213
|
+
async function downloadMetric(storage, client, experimentId, prefix, metricName, batchSize) {
|
|
214
|
+
try {
|
|
215
|
+
const stats = await client.getMetricStats(experimentId, metricName);
|
|
216
|
+
const totalChunks = Number(stats?.totalChunks ?? 0);
|
|
217
|
+
const buffered = Number(stats?.bufferedDataPoints ?? 0);
|
|
218
|
+
const all = [];
|
|
219
|
+
if (totalChunks > 0) {
|
|
220
|
+
const indices = Array.from({ length: totalChunks }, (_, i) => i);
|
|
221
|
+
const outcomes = await runPool(indices, Math.min(10, totalChunks), async (i) => {
|
|
222
|
+
const chunk = await client.downloadMetricChunk(experimentId, metricName, i);
|
|
223
|
+
return chunk?.data ?? [];
|
|
224
|
+
});
|
|
225
|
+
for (const outcome of outcomes) {
|
|
226
|
+
if (outcome.error)
|
|
227
|
+
throw outcome.error;
|
|
228
|
+
all.push(...outcome.value);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (buffered > 0) {
|
|
232
|
+
const response = await client.getMetricData(experimentId, metricName, { bufferOnly: true });
|
|
233
|
+
all.push(...(response?.data ?? []));
|
|
234
|
+
}
|
|
235
|
+
if (totalChunks === 0 && buffered === 0) {
|
|
236
|
+
throw new Error("no chunk or buffer counts reported");
|
|
237
|
+
}
|
|
238
|
+
all.sort((a, b) => Number(a?.index ?? 0) - Number(b?.index ?? 0));
|
|
239
|
+
let bytes = 0;
|
|
240
|
+
for (let i = 0; i < all.length; i += 10_000) {
|
|
241
|
+
const batch = all.slice(i, i + 10_000);
|
|
242
|
+
storage.appendBatchToMetric(prefix, metricName, batch.map((d) => d.data));
|
|
243
|
+
bytes += batch.reduce((n, d) => n + Buffer.byteLength(JSON.stringify(d), "utf8"), 0);
|
|
244
|
+
}
|
|
245
|
+
return { points: all.length, bytes };
|
|
246
|
+
}
|
|
247
|
+
catch {
|
|
248
|
+
let startIndex = 0;
|
|
249
|
+
let points = 0;
|
|
250
|
+
let bytes = 0;
|
|
251
|
+
for (;;) {
|
|
252
|
+
const response = await client.getMetricData(experimentId, metricName, {
|
|
253
|
+
startIndex,
|
|
254
|
+
limit: batchSize,
|
|
255
|
+
});
|
|
256
|
+
const data = response?.data ?? [];
|
|
257
|
+
if (data.length === 0)
|
|
258
|
+
break;
|
|
259
|
+
storage.appendBatchToMetric(prefix, metricName, data.map((d) => d.data));
|
|
260
|
+
points += data.length;
|
|
261
|
+
bytes += data.reduce((n, d) => n + Buffer.byteLength(JSON.stringify(d), "utf8"), 0);
|
|
262
|
+
if (!response?.hasMore)
|
|
263
|
+
break;
|
|
264
|
+
startIndex += data.length;
|
|
265
|
+
}
|
|
266
|
+
return { points, bytes };
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/** Flatten a GraphQL file node into the fields local storage records. */
|
|
270
|
+
function normalizeFile(node) {
|
|
271
|
+
const physical = node?.physicalFile ?? {};
|
|
272
|
+
return {
|
|
273
|
+
id: asId(node?.id),
|
|
274
|
+
filename: physical.filename ?? node?.name ?? "unnamed",
|
|
275
|
+
filePath: typeof node?.pPath === "string" ? node.pPath.replace(/^\/+/, "") : "",
|
|
276
|
+
description: node?.description ?? null,
|
|
277
|
+
tags: node?.tags ?? [],
|
|
278
|
+
bindrs: node?.bindrs ?? [],
|
|
279
|
+
metadata: node?.metadata ?? null,
|
|
280
|
+
checksum: physical.checksum ?? "",
|
|
281
|
+
contentType: physical.contentType ?? "",
|
|
282
|
+
sizeBytes: physical.sizeBytes != null ? Number(physical.sizeBytes) : 0,
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
async function downloadFiles(storage, client, exp, prefix, result, opts) {
|
|
286
|
+
const limit = 500;
|
|
287
|
+
let offset = 0;
|
|
288
|
+
for (;;) {
|
|
289
|
+
let page;
|
|
290
|
+
try {
|
|
291
|
+
page = await client.listFiles(exp.experimentId, limit, offset);
|
|
292
|
+
}
|
|
293
|
+
catch (e) {
|
|
294
|
+
(result.failed.files ??= []).push(`List files failed: ${e.message}`);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const nodes = page?.files ?? [];
|
|
298
|
+
if (nodes.length === 0)
|
|
299
|
+
break;
|
|
300
|
+
const files = nodes.map(normalizeFile);
|
|
301
|
+
const outcomes = await runPool(files, opts.maxConcurrentFiles, (file) => downloadSingleFile(storage, client, prefix, exp.project, file));
|
|
302
|
+
outcomes.forEach((outcome, i) => {
|
|
303
|
+
if (outcome.error) {
|
|
304
|
+
(result.failed.files ??= []).push(`${files[i].filename}: ${outcome.error.message}`);
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
result.downloaded.files = (result.downloaded.files ?? 0) + 1;
|
|
308
|
+
result.bytesDownloaded += outcome.value;
|
|
309
|
+
if (opts.verbose)
|
|
310
|
+
console.log(` ${green("✓")} ${files[i].filename}`);
|
|
311
|
+
});
|
|
312
|
+
if (!page?.hasMore)
|
|
313
|
+
break;
|
|
314
|
+
offset += limit;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
async function downloadSingleFile(storage, client, prefix, project, file) {
|
|
318
|
+
const scratch = mkdtempSync(path.join(tmpdir(), "ml-dash-download-"));
|
|
319
|
+
// A fixed local name: the server's filename decides where the file lands in
|
|
320
|
+
// the .dash tree (checked there), never where the unverified bytes land
|
|
321
|
+
// here, so a '../' in it cannot write over anything outside the scratch dir.
|
|
322
|
+
const temp = path.join(scratch, "payload");
|
|
323
|
+
try {
|
|
324
|
+
await client.downloadFileStreaming(file.id, temp);
|
|
325
|
+
if (file.checksum) {
|
|
326
|
+
const actual = await sha256File(temp);
|
|
327
|
+
if (actual !== file.checksum) {
|
|
328
|
+
// Deleted, not stored: a corrupt archive entry that looks complete is
|
|
329
|
+
// worse than a download that says it failed.
|
|
330
|
+
throw new Error(`checksum mismatch (expected ${file.checksum}, got ${actual})`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
await storage.writeFile({
|
|
334
|
+
prefix,
|
|
335
|
+
project,
|
|
336
|
+
sourcePath: temp,
|
|
337
|
+
path: file.filePath,
|
|
338
|
+
filename: file.filename,
|
|
339
|
+
description: file.description,
|
|
340
|
+
tags: file.tags,
|
|
341
|
+
bindrs: file.bindrs,
|
|
342
|
+
metadata: file.metadata,
|
|
343
|
+
checksum: file.checksum,
|
|
344
|
+
contentType: file.contentType,
|
|
345
|
+
sizeBytes: file.sizeBytes,
|
|
346
|
+
});
|
|
347
|
+
return file.sizeBytes;
|
|
348
|
+
}
|
|
349
|
+
finally {
|
|
350
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
// ── track download ───────────────────────────────────────────────────────────
|
|
354
|
+
// ── entry point ──────────────────────────────────────────────────────────────
|
|
355
|
+
const positiveInt = (value, fallback) => {
|
|
356
|
+
if (value === undefined)
|
|
357
|
+
return fallback;
|
|
358
|
+
const n = Number.parseInt(String(value), 10);
|
|
359
|
+
return Number.isFinite(n) && n > 0 ? n : Number.NaN;
|
|
360
|
+
};
|
|
361
|
+
export async function run(args) {
|
|
362
|
+
const native = await import("./adapter.js");
|
|
363
|
+
if (native && args.tracks)
|
|
364
|
+
return native.runNativeTransfer(() => native.downloadTrackNative(args));
|
|
365
|
+
const nativeSource = native.nativeDownloadSource(args);
|
|
366
|
+
const ctx = { remoteUrl: nativeSource.stateIdentity };
|
|
367
|
+
const batchSize = positiveInt(args.batch_size, 1000);
|
|
368
|
+
const maxMetrics = positiveInt(args.max_concurrent_metrics, 5);
|
|
369
|
+
const maxFiles = positiveInt(args.max_concurrent_files, 3);
|
|
370
|
+
for (const [flag, value] of [
|
|
371
|
+
["--batch-size", batchSize],
|
|
372
|
+
["--max-concurrent-metrics", maxMetrics],
|
|
373
|
+
["--max-concurrent-files", maxFiles],
|
|
374
|
+
]) {
|
|
375
|
+
if (!Number.isFinite(value)) {
|
|
376
|
+
console.error(`${red("Error:")} ${flag} must be a positive integer`);
|
|
377
|
+
return 1;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
const projectFilter = typeof args.project === "string" ? args.project : undefined;
|
|
381
|
+
const namespace = projectFilter?.replace(/^\/+|\/+$/g, "").split("/")[0];
|
|
382
|
+
if (!projectFilter || projectFilter.replace(/^\/+|\/+$/g, "").split("/").length < 2) {
|
|
383
|
+
console.error(`${red("Error:")} --project must be in format 'namespace/project' or 'namespace/project/exp'`);
|
|
384
|
+
console.error("Example: dl experiments download --project alice/my-project");
|
|
385
|
+
return 1;
|
|
386
|
+
}
|
|
387
|
+
const localPath = path.resolve(String(args.path ?? "./.dash"));
|
|
388
|
+
const client = nativeSource.client;
|
|
389
|
+
const storage = new LocalStorage(localPath);
|
|
390
|
+
const stateFile = path.resolve(String(args.state_file ?? ".dash-download-state.json"));
|
|
391
|
+
let state = args.resume ? loadState(stateFile) : null;
|
|
392
|
+
// Completed experiments of one backend are never skipped on another (native state is tagged `native:`).
|
|
393
|
+
if (state && (state.remote_url !== ctx.remoteUrl || state.local_path !== localPath)) {
|
|
394
|
+
console.log(`${yellow("Warning:")} State file belongs to a different backend or server. Starting fresh download.`);
|
|
395
|
+
state = null;
|
|
396
|
+
}
|
|
397
|
+
if (args.resume) {
|
|
398
|
+
if (state) {
|
|
399
|
+
console.log(cyan(`Resuming from previous download (${state.completed_experiments.length} completed)`));
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
console.log(yellow("No previous state found, starting fresh"));
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
state ??= {
|
|
406
|
+
remote_url: ctx.remoteUrl,
|
|
407
|
+
local_path: localPath,
|
|
408
|
+
completed_experiments: [],
|
|
409
|
+
failed_experiments: [],
|
|
410
|
+
in_progress_experiment: null,
|
|
411
|
+
timestamp: null,
|
|
412
|
+
};
|
|
413
|
+
console.log(bold("Discovering experiments on remote server..."));
|
|
414
|
+
let experiments;
|
|
415
|
+
try {
|
|
416
|
+
experiments = await nativeSource.discover(typeof args.experiment === "string" ? args.experiment : undefined);
|
|
417
|
+
}
|
|
418
|
+
catch (e) {
|
|
419
|
+
console.error(red(`Failed to discover experiments: ${e.message}`));
|
|
420
|
+
return 1;
|
|
421
|
+
}
|
|
422
|
+
if (experiments.length === 0) {
|
|
423
|
+
console.log(yellow("No experiments found"));
|
|
424
|
+
return 0;
|
|
425
|
+
}
|
|
426
|
+
console.log(`Found ${experiments.length} experiment(s)`);
|
|
427
|
+
const queued = [];
|
|
428
|
+
for (const exp of experiments) {
|
|
429
|
+
const key = `${localPrefixFor(exp)}`;
|
|
430
|
+
if (state.completed_experiments.includes(key) && !args.overwrite) {
|
|
431
|
+
console.log(dim(` Skipping ${key} (already completed)`));
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
// A prefix that cannot be resolved under the root is not "already here":
|
|
435
|
+
// it is queued so the per-experiment path reports it as the failure it is,
|
|
436
|
+
// rather than aborting the whole run from inside a skip check.
|
|
437
|
+
let expJson = null;
|
|
438
|
+
try {
|
|
439
|
+
expJson = path.join(storage.experimentDir(localPrefixFor(exp)), "experiment.json");
|
|
440
|
+
}
|
|
441
|
+
catch {
|
|
442
|
+
expJson = null;
|
|
443
|
+
}
|
|
444
|
+
if (expJson && existsSync(expJson) && !args.overwrite) {
|
|
445
|
+
console.log(yellow(` Skipping ${key} (already exists locally)`));
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
queued.push(exp);
|
|
449
|
+
}
|
|
450
|
+
if (queued.length === 0) {
|
|
451
|
+
console.log(green("All experiments already downloaded"));
|
|
452
|
+
return 0;
|
|
453
|
+
}
|
|
454
|
+
if (args.dry_run) {
|
|
455
|
+
console.log(`\n${bold("Dry run - would download:")}`);
|
|
456
|
+
for (const exp of queued) {
|
|
457
|
+
console.log(` • ${localPrefixFor(exp)}`);
|
|
458
|
+
console.log(exp.countsUnknown ? " Logs: unknown, Metrics: unknown, Files: unknown"
|
|
459
|
+
: ` Logs: unknown, Metrics: ${exp.metricNames.length}, Files: ${exp.fileCount}`);
|
|
460
|
+
}
|
|
461
|
+
return 0;
|
|
462
|
+
}
|
|
463
|
+
const opts = {
|
|
464
|
+
batchSize,
|
|
465
|
+
skipLogs: args.skip_logs === true,
|
|
466
|
+
skipMetrics: args.skip_metrics === true,
|
|
467
|
+
skipFiles: args.skip_files === true,
|
|
468
|
+
skipParams: args.skip_params === true,
|
|
469
|
+
verbose: args.verbose === true,
|
|
470
|
+
maxConcurrentMetrics: maxMetrics,
|
|
471
|
+
maxConcurrentFiles: maxFiles,
|
|
472
|
+
overwrite: args.overwrite === true,
|
|
473
|
+
};
|
|
474
|
+
console.log(`\n${bold(`Downloading ${queued.length} experiment(s)...`)}`);
|
|
475
|
+
const started = Date.now();
|
|
476
|
+
const results = [];
|
|
477
|
+
for (let i = 0; i < queued.length; i++) {
|
|
478
|
+
const exp = queued[i];
|
|
479
|
+
const key = `${localPrefixFor(exp)}`;
|
|
480
|
+
console.log(`\n${cyan(`[${i + 1}/${queued.length}] ${key}`)}`);
|
|
481
|
+
state.in_progress_experiment = key;
|
|
482
|
+
saveState(stateFile, state);
|
|
483
|
+
const result = await downloadExperiment(storage, client, exp, opts);
|
|
484
|
+
results.push(result);
|
|
485
|
+
if (result.success) {
|
|
486
|
+
state.completed_experiments.push(key);
|
|
487
|
+
console.log(` ${green("✓ Downloaded successfully")}`);
|
|
488
|
+
}
|
|
489
|
+
else {
|
|
490
|
+
state.failed_experiments.push(key);
|
|
491
|
+
console.log(` ${red(`✗ Failed: ${result.errors.join(", ")}`)}`);
|
|
492
|
+
}
|
|
493
|
+
state.in_progress_experiment = null;
|
|
494
|
+
saveState(stateFile, state);
|
|
495
|
+
}
|
|
496
|
+
const elapsed = (Date.now() - started) / 1000;
|
|
497
|
+
const totalBytes = results.reduce((n, r) => n + r.bytesDownloaded, 0);
|
|
498
|
+
const successful = results.filter((r) => r.success).length;
|
|
499
|
+
const rows = [
|
|
500
|
+
["Total Experiments", String(results.length)],
|
|
501
|
+
["Successful", String(successful)],
|
|
502
|
+
["Failed", String(results.length - successful)],
|
|
503
|
+
["Total Data", formatBytes(totalBytes)],
|
|
504
|
+
["Total Time", `${elapsed.toFixed(2)}s`],
|
|
505
|
+
];
|
|
506
|
+
if (elapsed > 0)
|
|
507
|
+
rows.push(["Avg Speed", `${formatBytes(totalBytes / elapsed)}/s`]);
|
|
508
|
+
console.log(`\n${bold("Download Summary")}`);
|
|
509
|
+
console.log(renderTable([{ header: "Metric" }, { header: "Value" }], rows));
|
|
510
|
+
if (successful === results.length) {
|
|
511
|
+
if (existsSync(stateFile))
|
|
512
|
+
unlinkSync(stateFile);
|
|
513
|
+
return 0;
|
|
514
|
+
}
|
|
515
|
+
return 1;
|
|
516
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { reportNativeError } from './adapter.js';
|
|
2
|
+
import { run as upload } from './upload.js';
|
|
3
|
+
import { run as download } from './download.js';
|
|
4
|
+
export function registerExperimentTransferCommands(parent) {
|
|
5
|
+
for (const action of ['upload', 'download']) {
|
|
6
|
+
const command = parent.command(action).description(`${action} complete experiment runs in .dash directory format`)
|
|
7
|
+
.argument('[path]', 'local .dash directory', './.dash')
|
|
8
|
+
.option('-p, --project <selector>', 'namespace/project[/folder or glob] selector')
|
|
9
|
+
.option('--experiment <name>', 'only this run name')
|
|
10
|
+
.option('--skip-logs', 'skip log entries').option('--skip-metrics', 'skip metrics')
|
|
11
|
+
.option('--skip-files', 'skip run files').option('--skip-params', 'skip parameters')
|
|
12
|
+
.option('--dry-run', 'preview without transferring data').option('--resume', 'resume a previous transfer')
|
|
13
|
+
.option('--state-file <path>', 'resume state file (removed on success)')
|
|
14
|
+
.option('--batch-size <count>', 'log/metric batch size')
|
|
15
|
+
.option('-v, --verbose', 'show transfer details')
|
|
16
|
+
.option('--tracks <path>', action === 'upload' ? 'JSONL track source file' : 'namespace/project/run/topic to export');
|
|
17
|
+
if (action === 'upload')
|
|
18
|
+
command.option('-t, --target <prefix>', 'target namespace/project[/folder]; appends each run name')
|
|
19
|
+
.option('--remote-path <path>', 'track destination namespace/project/run/topic').option('--strict', 'fail on invalid local data');
|
|
20
|
+
else
|
|
21
|
+
command.option('--overwrite', 'replace existing local experiment data')
|
|
22
|
+
.option('-f, --format <format>', 'track export: json, jsonl, parquet (Arrow IPC), mcap', 'jsonl')
|
|
23
|
+
.option('-o, --output <path>', 'track export destination')
|
|
24
|
+
.option('--max-concurrent-metrics <count>', 'parallel metric reads (default 5)')
|
|
25
|
+
.option('--max-concurrent-files <count>', 'parallel file downloads (default 3)');
|
|
26
|
+
command.action(async (path, options) => {
|
|
27
|
+
const args = { path };
|
|
28
|
+
for (const [key, value] of Object.entries(options))
|
|
29
|
+
args[key.replace(/[A-Z]/g, c => '_' + c.toLowerCase())] = value;
|
|
30
|
+
try {
|
|
31
|
+
process.exitCode = await (action === 'upload' ? upload : download)(args);
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
process.exitCode = reportNativeError(error);
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export declare class UnsafePathError extends Error {
|
|
2
|
+
constructor(label: string, value: string, reason: string);
|
|
3
|
+
}
|
|
4
|
+
/** One path segment: a filename, or a directory name that may not nest. */
|
|
5
|
+
export declare function safeSegment(label: string, value: string): string;
|
|
6
|
+
/** A relative path that may nest: `owner/project/exp`, `train/loss`. */
|
|
7
|
+
export declare function safeRelativeSegments(label: string, value: string): string[];
|
|
8
|
+
/**
|
|
9
|
+
* One piece of a target path. `nested` marks the pieces that may legally carry
|
|
10
|
+
* separators — a prefix, a metric name, a file's path — from the ones that are
|
|
11
|
+
* a single name.
|
|
12
|
+
*/
|
|
13
|
+
export interface PathPart {
|
|
14
|
+
label: string;
|
|
15
|
+
value: string;
|
|
16
|
+
nested?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** A literal component this code chose, not the server: `files`, `logs.jsonl`. */
|
|
19
|
+
export declare const literal: (value: string) => PathPart;
|
|
20
|
+
/**
|
|
21
|
+
* Build a path under `root` from its parts, or throw.
|
|
22
|
+
*
|
|
23
|
+
* `mode: "write"` additionally refuses a target any part of which is a symlink
|
|
24
|
+
* — dangling or not — because a link inside the tree redirects the write out
|
|
25
|
+
* of it, and because following one to delete or overwrite would destroy a file
|
|
26
|
+
* that is not ours. `mode: "read"` keeps the lexical rules only, so a user who
|
|
27
|
+
* has symlinked their own metric data can still upload it.
|
|
28
|
+
*/
|
|
29
|
+
export declare function resolveUnderRoot(root: string, parts: PathPart[], mode?: "read" | "write"): string;
|