@dreamlake/dreamlake-cli 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/file-multipart.d.ts +6 -0
- package/dist/api/file-multipart.js +11 -0
- package/dist/api/index.d.ts +44 -0
- package/dist/api/native-experiments.d.ts +25 -0
- package/dist/api/native-experiments.js +49 -3
- package/dist/cli/experiments/index.js +4 -0
- package/dist/cli/experiments/management.d.ts +16 -0
- package/dist/cli/experiments/management.js +82 -0
- package/dist/cli/experiments/transfer/adapter.d.ts +86 -0
- package/dist/cli/experiments/transfer/adapter.js +637 -0
- package/dist/cli/experiments/transfer/context.d.ts +13 -0
- package/dist/cli/experiments/transfer/context.js +20 -0
- package/dist/cli/experiments/transfer/download.d.ts +24 -0
- package/dist/cli/experiments/transfer/download.js +516 -0
- package/dist/cli/experiments/transfer/index.d.ts +2 -0
- package/dist/cli/experiments/transfer/index.js +38 -0
- package/dist/cli/experiments/transfer/local/safe-path.d.ts +29 -0
- package/dist/cli/experiments/transfer/local/safe-path.js +106 -0
- package/dist/cli/experiments/transfer/local/storage.d.ts +125 -0
- package/dist/cli/experiments/transfer/local/storage.js +432 -0
- package/dist/cli/experiments/transfer/transport.d.ts +7 -0
- package/dist/cli/experiments/transfer/transport.js +31 -0
- package/dist/cli/experiments/transfer/types.d.ts +48 -0
- package/dist/cli/experiments/transfer/types.js +1 -0
- package/dist/cli/experiments/transfer/upload.d.ts +58 -0
- package/dist/cli/experiments/transfer/upload.js +665 -0
- package/dist/cli/experiments/transfer/util/ansi.d.ts +24 -0
- package/dist/cli/experiments/transfer/util/ansi.js +70 -0
- package/dist/cli/experiments/transfer/util/glob.d.ts +4 -0
- package/dist/cli/experiments/transfer/util/glob.js +39 -0
- package/dist/cli/experiments/transfer/util/json.d.ts +6 -0
- package/dist/cli/experiments/transfer/util/json.js +58 -0
- package/dist/cli/experiments/transfer/util/pool.d.ts +5 -0
- package/dist/cli/experiments/transfer/util/pool.js +21 -0
- package/dist/cli/help-examples.generated.js +12 -0
- package/dist/cli/version.d.ts +1 -1
- package/dist/cli/version.js +1 -1
- package/package.json +9 -9
|
@@ -0,0 +1,665 @@
|
|
|
1
|
+
/** Native experiment transfer using the interoperable .dash directory format. */
|
|
2
|
+
import { existsSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { LocalStorage } from "./local/storage.js";
|
|
5
|
+
import { bold, cyan, dim, formatBytes, green, red, renderTable, yellow } from "./util/ansi.js";
|
|
6
|
+
import { fnmatch, hasWildcard } from "./util/glob.js";
|
|
7
|
+
import { parseJson } from "./util/json.js";
|
|
8
|
+
import { runPool } from "./util/pool.js";
|
|
9
|
+
/** Metric uploads run in parallel, as `ThreadPoolExecutor(max_workers=5)` did. */
|
|
10
|
+
const METRIC_WORKERS = 5;
|
|
11
|
+
/** Every directory under `root` that contains an experiment.json. */
|
|
12
|
+
function findExperimentDirs(root) {
|
|
13
|
+
const out = [];
|
|
14
|
+
const walk = (dir, depth) => {
|
|
15
|
+
if (depth > 24)
|
|
16
|
+
return;
|
|
17
|
+
let entries;
|
|
18
|
+
try {
|
|
19
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
for (const entry of entries) {
|
|
25
|
+
if (!entry.isDirectory())
|
|
26
|
+
continue;
|
|
27
|
+
const child = path.join(dir, entry.name);
|
|
28
|
+
if (existsSync(path.join(child, "experiment.json")))
|
|
29
|
+
out.push(child);
|
|
30
|
+
// Descend regardless: `.dash/owner/project/folder/exp` nests experiments
|
|
31
|
+
// under plain folders, and an experiment may sit below another one.
|
|
32
|
+
walk(child, depth + 1);
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
walk(root, 0);
|
|
36
|
+
return out.sort();
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Match the three filter modes argparse's `-p` carried in the Python CLI:
|
|
40
|
+
* a glob is fnmatched against the path under the root, a value with a slash is
|
|
41
|
+
* a path prefix, and a bare word is an exact project-name match.
|
|
42
|
+
*/
|
|
43
|
+
export function matchesFilter(relPath, projectName, filter) {
|
|
44
|
+
if (hasWildcard(filter))
|
|
45
|
+
return fnmatch(relPath, filter);
|
|
46
|
+
if (filter.includes("/"))
|
|
47
|
+
return relPath === filter || relPath.startsWith(`${filter}/`);
|
|
48
|
+
return projectName === filter;
|
|
49
|
+
}
|
|
50
|
+
export function discoverExperiments(storage, projectFilter, experimentFilter) {
|
|
51
|
+
const root = storage.rootPath;
|
|
52
|
+
const found = [];
|
|
53
|
+
for (const dir of findExperimentDirs(root)) {
|
|
54
|
+
const relPath = path.relative(root, dir).split(path.sep).join("/");
|
|
55
|
+
if (!relPath || relPath.startsWith(".."))
|
|
56
|
+
continue;
|
|
57
|
+
let declaredPrefix;
|
|
58
|
+
try {
|
|
59
|
+
const meta = parseJson(readFileSync(path.join(dir, "experiment.json"), "utf8"));
|
|
60
|
+
if (typeof meta?.prefix === "string" && meta.prefix.trim())
|
|
61
|
+
declaredPrefix = meta.prefix;
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
// Unreadable metadata is a validation error, not a discovery one.
|
|
65
|
+
}
|
|
66
|
+
const relParts = relPath.split("/");
|
|
67
|
+
let project;
|
|
68
|
+
let experiment;
|
|
69
|
+
if (declaredPrefix) {
|
|
70
|
+
const parts = declaredPrefix.replace(/^\/+|\/+$/g, "").split("/");
|
|
71
|
+
if (parts.length < 3)
|
|
72
|
+
continue; // needs at least owner/project/experiment
|
|
73
|
+
project = parts[1];
|
|
74
|
+
experiment = parts[parts.length - 1];
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
if (relParts.length < 2)
|
|
78
|
+
continue;
|
|
79
|
+
experiment = relParts[relParts.length - 1];
|
|
80
|
+
project = relParts[relParts.length - 2];
|
|
81
|
+
}
|
|
82
|
+
if (projectFilter && !matchesFilter(relPath, project, projectFilter))
|
|
83
|
+
continue;
|
|
84
|
+
if (experimentFilter && experiment !== experimentFilter)
|
|
85
|
+
continue;
|
|
86
|
+
const info = {
|
|
87
|
+
project,
|
|
88
|
+
experiment,
|
|
89
|
+
dir,
|
|
90
|
+
storagePrefix: relPath,
|
|
91
|
+
declaredPrefix,
|
|
92
|
+
hasParams: existsSync(path.join(dir, "parameters.json")),
|
|
93
|
+
hasLogs: existsSync(path.join(dir, "logs", "logs.jsonl")),
|
|
94
|
+
metrics: storage.listMetrics(relPath),
|
|
95
|
+
fileCount: 0,
|
|
96
|
+
estimatedSize: 0,
|
|
97
|
+
};
|
|
98
|
+
const records = storage.listFiles(relPath);
|
|
99
|
+
info.fileCount = records.length;
|
|
100
|
+
for (const record of records) {
|
|
101
|
+
try {
|
|
102
|
+
info.estimatedSize += statSync(storage.filePath(relPath, record)).size;
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// Counted by validation as a missing file.
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
found.push(info);
|
|
109
|
+
}
|
|
110
|
+
return found;
|
|
111
|
+
}
|
|
112
|
+
export function validateExperiment(storage, info, strict) {
|
|
113
|
+
const result = { isValid: true, warnings: [], errors: [] };
|
|
114
|
+
const expJson = path.join(info.dir, "experiment.json");
|
|
115
|
+
if (!existsSync(expJson)) {
|
|
116
|
+
return { ...result, isValid: false, errors: ["Missing experiment.json"] };
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
const metadata = parseJson(readFileSync(expJson, "utf8"));
|
|
120
|
+
if (!metadata || typeof metadata !== "object" || !("name" in metadata) || !("project" in metadata)) {
|
|
121
|
+
return {
|
|
122
|
+
...result,
|
|
123
|
+
isValid: false,
|
|
124
|
+
errors: ["experiment.json missing required fields (name, project)"],
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
result.metadata = metadata;
|
|
128
|
+
}
|
|
129
|
+
catch (e) {
|
|
130
|
+
return { ...result, isValid: false, errors: [`Invalid JSON in experiment.json: ${e.message}`] };
|
|
131
|
+
}
|
|
132
|
+
if (info.hasParams) {
|
|
133
|
+
const params = storage.readParameters(info.storagePrefix);
|
|
134
|
+
if (params === null)
|
|
135
|
+
result.warnings.push("parameters.json is not a dict (will skip)");
|
|
136
|
+
else
|
|
137
|
+
result.parameters = params;
|
|
138
|
+
}
|
|
139
|
+
if (info.hasLogs) {
|
|
140
|
+
const invalid = countInvalidLines(path.join(info.dir, "logs", "logs.jsonl"), (o) => "message" in o);
|
|
141
|
+
if (invalid.count > 0) {
|
|
142
|
+
result.warnings.push(`logs.jsonl has ${invalid.count} invalid lines (e.g., [${invalid.preview.join(", ")}]...) - will skip these`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
for (const metric of info.metrics) {
|
|
146
|
+
const invalid = countInvalidLines(metric.dataFile, (o) => "data" in o);
|
|
147
|
+
if (invalid.count > 0) {
|
|
148
|
+
result.warnings.push(`metric '${metric.name}' has ${invalid.count} invalid lines (e.g., [${invalid.preview.join(", ")}]...) - will skip these`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const missing = storage
|
|
152
|
+
.listFiles(info.storagePrefix)
|
|
153
|
+
.filter((f) => !existsSync(storage.filePath(info.storagePrefix, f)))
|
|
154
|
+
.map((f) => f.filename);
|
|
155
|
+
if (missing.length > 0) {
|
|
156
|
+
result.warnings.push(`${missing.length} files referenced in metadata but missing on disk ` +
|
|
157
|
+
`(e.g., [${missing.slice(0, 3).join(", ")}]...) - will skip these`);
|
|
158
|
+
}
|
|
159
|
+
if (strict && result.warnings.length > 0) {
|
|
160
|
+
result.errors.push(...result.warnings);
|
|
161
|
+
result.warnings = [];
|
|
162
|
+
result.isValid = false;
|
|
163
|
+
}
|
|
164
|
+
return result;
|
|
165
|
+
}
|
|
166
|
+
function countInvalidLines(file, isValid) {
|
|
167
|
+
const bad = [];
|
|
168
|
+
if (!existsSync(file))
|
|
169
|
+
return { count: 0, preview: [] };
|
|
170
|
+
let text;
|
|
171
|
+
try {
|
|
172
|
+
text = readFileSync(file, "utf8");
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return { count: 0, preview: [] };
|
|
176
|
+
}
|
|
177
|
+
const lines = text.split("\n");
|
|
178
|
+
lines.forEach((line, i) => {
|
|
179
|
+
if (!line.trim())
|
|
180
|
+
return;
|
|
181
|
+
try {
|
|
182
|
+
const parsed = JSON.parse(line);
|
|
183
|
+
if (parsed === null || typeof parsed !== "object" || !isValid(parsed))
|
|
184
|
+
bad.push(i + 1);
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
bad.push(i + 1);
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
return { count: bad.length, preview: bad.slice(0, 5) };
|
|
191
|
+
}
|
|
192
|
+
const loadState = (file) => {
|
|
193
|
+
if (!existsSync(file))
|
|
194
|
+
return null;
|
|
195
|
+
try {
|
|
196
|
+
const data = JSON.parse(readFileSync(file, "utf8"));
|
|
197
|
+
if (typeof data?.dash_root !== "string" || typeof data?.remote_url !== "string")
|
|
198
|
+
return null;
|
|
199
|
+
return {
|
|
200
|
+
dash_root: data.dash_root,
|
|
201
|
+
remote_url: data.remote_url,
|
|
202
|
+
completed_experiments: data.completed_experiments ?? [],
|
|
203
|
+
failed_experiments: data.failed_experiments ?? [],
|
|
204
|
+
in_progress_experiment: data.in_progress_experiment ?? null,
|
|
205
|
+
timestamp: data.timestamp ?? null,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
};
|
|
212
|
+
const saveState = (file, state) => {
|
|
213
|
+
state.timestamp = new Date().toISOString();
|
|
214
|
+
writeFileSync(file, JSON.stringify(state, null, 2));
|
|
215
|
+
};
|
|
216
|
+
/**
|
|
217
|
+
* Where an experiment goes: `--target` acts like an scp destination directory (the
|
|
218
|
+
* experiment name is appended; its second segment names the project), else the prefix
|
|
219
|
+
* declared in experiment.json, else the storage location. Reading always uses storagePrefix.
|
|
220
|
+
*/
|
|
221
|
+
export function remotePrefixFor(info, targetPrefix) {
|
|
222
|
+
if (targetPrefix) {
|
|
223
|
+
const parts = targetPrefix.replace(/^\/+|\/+$/g, "").split("/");
|
|
224
|
+
return { fullPrefix: `${targetPrefix.replace(/\/+$/, "")}/${info.experiment}`, targetProject: parts.length >= 2 ? parts[1] : info.project };
|
|
225
|
+
}
|
|
226
|
+
return { fullPrefix: info.declaredPrefix ?? info.storagePrefix, targetProject: info.project };
|
|
227
|
+
}
|
|
228
|
+
async function uploadExperiment(storage, client, info, validation, opts) {
|
|
229
|
+
const result = {
|
|
230
|
+
experiment: `${info.project}/${info.experiment}`,
|
|
231
|
+
success: false,
|
|
232
|
+
uploaded: {},
|
|
233
|
+
failed: {},
|
|
234
|
+
errors: [],
|
|
235
|
+
bytesUploaded: 0,
|
|
236
|
+
};
|
|
237
|
+
const note = (line) => {
|
|
238
|
+
if (opts.verbose)
|
|
239
|
+
console.log(line);
|
|
240
|
+
};
|
|
241
|
+
try {
|
|
242
|
+
const meta = validation.metadata ?? {};
|
|
243
|
+
const { fullPrefix, targetProject } = remotePrefixFor(info, opts.targetPrefix);
|
|
244
|
+
note(dim(" Creating experiment..."));
|
|
245
|
+
const response = await client.createOrUpdateExperiment({
|
|
246
|
+
project: targetProject,
|
|
247
|
+
name: info.experiment,
|
|
248
|
+
description: meta.description ?? null,
|
|
249
|
+
tags: meta.tags ?? null,
|
|
250
|
+
bindrs: meta.bindrs ?? null,
|
|
251
|
+
prefix: fullPrefix,
|
|
252
|
+
writeProtected: meta.write_protected === true,
|
|
253
|
+
metadata: meta.metadata ?? null,
|
|
254
|
+
});
|
|
255
|
+
const experimentId = String(response?.experiment?.id ?? response?.id ?? "");
|
|
256
|
+
if (!experimentId)
|
|
257
|
+
throw new Error("Server did not return an experiment id");
|
|
258
|
+
note(` ${green("✓")} Created experiment (id: ${experimentId})`);
|
|
259
|
+
if (!opts.skipParams && validation.parameters) {
|
|
260
|
+
const params = validation.parameters;
|
|
261
|
+
await client.setParameters(experimentId, params);
|
|
262
|
+
result.uploaded.params = Object.keys(params).length;
|
|
263
|
+
result.bytesUploaded += Buffer.byteLength(JSON.stringify(params), "utf8");
|
|
264
|
+
note(` ${green("✓")} Uploaded ${result.uploaded.params} parameters`);
|
|
265
|
+
}
|
|
266
|
+
if (!opts.skipLogs && info.hasLogs) {
|
|
267
|
+
try {
|
|
268
|
+
const { uploaded, skipped, bytes } = await uploadLogs(storage, client, experimentId, info, opts);
|
|
269
|
+
result.uploaded.logs = uploaded;
|
|
270
|
+
result.bytesUploaded += bytes;
|
|
271
|
+
note(` ${green("✓")} Uploaded ${uploaded} log entries` +
|
|
272
|
+
(skipped > 0 ? ` (skipped ${skipped} invalid)` : ""));
|
|
273
|
+
}
|
|
274
|
+
catch (e) {
|
|
275
|
+
(result.failed.logs ??= []).push(e.message);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (!opts.skipMetrics && info.metrics.length > 0) {
|
|
279
|
+
const outcomes = await runPool(info.metrics, METRIC_WORKERS, async (metric) => {
|
|
280
|
+
const { points, skipped, bytes } = storage.readMetricPoints(metric);
|
|
281
|
+
for (let i = 0; i < points.length; i += opts.batchSize) {
|
|
282
|
+
await client.appendBatchToMetric(experimentId, metric.name, points.slice(i, i + opts.batchSize));
|
|
283
|
+
}
|
|
284
|
+
return { uploaded: points.length, skipped, bytes };
|
|
285
|
+
});
|
|
286
|
+
let ok = 0;
|
|
287
|
+
outcomes.forEach((outcome, i) => {
|
|
288
|
+
const metric = info.metrics[i];
|
|
289
|
+
if (outcome.error) {
|
|
290
|
+
(result.failed.metrics ??= []).push(`${metric.name}: ${outcome.error.message}`);
|
|
291
|
+
note(` ${red("✗")} Failed to upload '${metric.name}': ${outcome.error.message}`);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
ok++;
|
|
295
|
+
result.bytesUploaded += outcome.value.bytes;
|
|
296
|
+
note(` ${green("✓")} Uploaded ${outcome.value.uploaded} data points for '${metric.name}'` +
|
|
297
|
+
(outcome.value.skipped > 0 ? ` (skipped ${outcome.value.skipped} invalid)` : ""));
|
|
298
|
+
});
|
|
299
|
+
result.uploaded.metrics = ok;
|
|
300
|
+
}
|
|
301
|
+
if (!opts.skipFiles && info.fileCount > 0) {
|
|
302
|
+
const uploadedFiles = await uploadFiles(storage, client, experimentId, info, result, opts);
|
|
303
|
+
result.uploaded.files = uploadedFiles;
|
|
304
|
+
}
|
|
305
|
+
// A section that failed is a failed experiment: keeping success here is
|
|
306
|
+
// what let the Python CLI delete its resume state after losing metrics.
|
|
307
|
+
const failedSections = Object.keys(result.failed);
|
|
308
|
+
if (failedSections.length > 0) {
|
|
309
|
+
result.success = false;
|
|
310
|
+
result.errors.push(`${failedSections.join(", ")} failed: ` +
|
|
311
|
+
failedSections.flatMap((s) => result.failed[s]).slice(0, 3).join("; "));
|
|
312
|
+
}
|
|
313
|
+
else if (meta.write_protected === true && client.protectExperiment) {
|
|
314
|
+
// Only a complete run is protected; a failed part leaves it writable for --resume.
|
|
315
|
+
try {
|
|
316
|
+
await client.protectExperiment(experimentId);
|
|
317
|
+
note(` ${green("✓")} Write-protected`);
|
|
318
|
+
result.success = true;
|
|
319
|
+
}
|
|
320
|
+
catch (e) {
|
|
321
|
+
result.success = false;
|
|
322
|
+
result.errors.push(`all data was uploaded, but write protection failed: ${e.message}`);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
result.success = true;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
catch (e) {
|
|
330
|
+
result.success = false;
|
|
331
|
+
result.errors.push(e.message);
|
|
332
|
+
if (opts.verbose)
|
|
333
|
+
console.log(` ${red(`✗ Error: ${e.message}`)}`);
|
|
334
|
+
}
|
|
335
|
+
return result;
|
|
336
|
+
}
|
|
337
|
+
async function uploadLogs(storage, client, experimentId, info, opts) {
|
|
338
|
+
const file = path.join(info.dir, "logs", "logs.jsonl");
|
|
339
|
+
const lines = readFileSync(file, "utf8").split("\n");
|
|
340
|
+
let batch = [];
|
|
341
|
+
let uploaded = 0;
|
|
342
|
+
let skipped = 0;
|
|
343
|
+
let bytes = 0;
|
|
344
|
+
const flush = async () => {
|
|
345
|
+
if (batch.length === 0)
|
|
346
|
+
return;
|
|
347
|
+
await client.createLogEntries(experimentId, batch);
|
|
348
|
+
uploaded += batch.length;
|
|
349
|
+
batch = [];
|
|
350
|
+
};
|
|
351
|
+
for (const line of lines) {
|
|
352
|
+
if (!line.trim())
|
|
353
|
+
continue;
|
|
354
|
+
let entry;
|
|
355
|
+
try {
|
|
356
|
+
entry = JSON.parse(line);
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
skipped++;
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
if (entry === null || typeof entry !== "object" || !("message" in entry)) {
|
|
363
|
+
skipped++;
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
366
|
+
const apiLog = {
|
|
367
|
+
timestamp: entry.timestamp ?? null,
|
|
368
|
+
level: entry.level ?? "info",
|
|
369
|
+
message: entry.message,
|
|
370
|
+
};
|
|
371
|
+
if ("metadata" in entry)
|
|
372
|
+
apiLog.metadata = entry.metadata;
|
|
373
|
+
batch.push(apiLog);
|
|
374
|
+
bytes += Buffer.byteLength(line, "utf8");
|
|
375
|
+
if (batch.length >= opts.batchSize)
|
|
376
|
+
await flush();
|
|
377
|
+
}
|
|
378
|
+
await flush();
|
|
379
|
+
return { uploaded, skipped, bytes };
|
|
380
|
+
}
|
|
381
|
+
async function uploadFiles(storage, client, experimentId, info, result, opts) {
|
|
382
|
+
let uploaded = 0;
|
|
383
|
+
let records;
|
|
384
|
+
try {
|
|
385
|
+
records = storage.listFiles(info.storagePrefix);
|
|
386
|
+
}
|
|
387
|
+
catch (e) {
|
|
388
|
+
(result.failed.files ??= []).push(e.message);
|
|
389
|
+
return 0;
|
|
390
|
+
}
|
|
391
|
+
if (opts.verbose) {
|
|
392
|
+
console.log(dim(` Found ${records.length} files to upload from prefix: ${info.storagePrefix}`));
|
|
393
|
+
}
|
|
394
|
+
for (const record of records) {
|
|
395
|
+
const source = storage.filePath(info.storagePrefix, record);
|
|
396
|
+
try {
|
|
397
|
+
if (!existsSync(source))
|
|
398
|
+
throw new Error(`missing on disk: ${source}`);
|
|
399
|
+
await client.uploadFile({
|
|
400
|
+
experimentId,
|
|
401
|
+
filePath: source,
|
|
402
|
+
prefix: record.path ?? "",
|
|
403
|
+
filename: record.filename,
|
|
404
|
+
description: record.description,
|
|
405
|
+
tags: record.tags,
|
|
406
|
+
bindrs: record.bindrs,
|
|
407
|
+
metadata: record.metadata,
|
|
408
|
+
checksum: record.checksum,
|
|
409
|
+
contentType: record.contentType,
|
|
410
|
+
sizeBytes: record.sizeBytes,
|
|
411
|
+
});
|
|
412
|
+
uploaded++;
|
|
413
|
+
result.bytesUploaded += record.sizeBytes ?? 0;
|
|
414
|
+
if (opts.verbose) {
|
|
415
|
+
console.log(` ${green("✓")} ${record.filename} (${formatBytes(record.sizeBytes ?? 0)})`);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
catch (e) {
|
|
419
|
+
(result.failed.files ??= []).push(`${record.filename}: ${e.message}`);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
return uploaded;
|
|
423
|
+
}
|
|
424
|
+
// ── track upload ─────────────────────────────────────────────────────────────
|
|
425
|
+
// ── entry point ──────────────────────────────────────────────────────────────
|
|
426
|
+
export async function run(args) {
|
|
427
|
+
const native = await import("./adapter.js");
|
|
428
|
+
if (native && args.tracks)
|
|
429
|
+
return native.runNativeTransfer(() => native.uploadTrackNative(args));
|
|
430
|
+
const nativeTarget = native.nativeUploadTarget(args);
|
|
431
|
+
const ctx = { remoteUrl: nativeTarget.stateIdentity };
|
|
432
|
+
const batchSize = Number.parseInt(String(args.batch_size ?? "100"), 10);
|
|
433
|
+
if (!Number.isFinite(batchSize) || batchSize <= 0) {
|
|
434
|
+
console.error(`${red("Error:")} --batch-size must be a positive integer`);
|
|
435
|
+
return 1;
|
|
436
|
+
}
|
|
437
|
+
const dashRoot = path.resolve(String(args.path ?? "./.dash"));
|
|
438
|
+
if (!existsSync(dashRoot)) {
|
|
439
|
+
console.error(`${red("Error:")} Local storage path does not exist: ${args.path}`);
|
|
440
|
+
return 1;
|
|
441
|
+
}
|
|
442
|
+
const stateFile = path.resolve(String(args.state_file ?? ".dash-upload-state.json"));
|
|
443
|
+
let state = null;
|
|
444
|
+
if (args.resume) {
|
|
445
|
+
state = loadState(stateFile);
|
|
446
|
+
if (!state) {
|
|
447
|
+
console.log(yellow("No previous upload state found. Starting fresh upload."));
|
|
448
|
+
}
|
|
449
|
+
else if (state.dash_root !== dashRoot) {
|
|
450
|
+
console.log(`${yellow("Warning:")} State file local path doesn't match. Starting fresh upload.`);
|
|
451
|
+
state = null;
|
|
452
|
+
}
|
|
453
|
+
else if (state.remote_url !== ctx.remoteUrl) {
|
|
454
|
+
console.log(`${yellow("Warning:")} State file remote URL doesn't match. Starting fresh upload.`);
|
|
455
|
+
state = null;
|
|
456
|
+
}
|
|
457
|
+
else {
|
|
458
|
+
console.log(green(`Resuming previous upload from ${state.timestamp}`));
|
|
459
|
+
console.log(` Already completed: ${state.completed_experiments.length} experiments`);
|
|
460
|
+
console.log(` Failed: ${state.failed_experiments.length} experiments`);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
if (!state) {
|
|
464
|
+
state = {
|
|
465
|
+
dash_root: dashRoot,
|
|
466
|
+
remote_url: ctx.remoteUrl,
|
|
467
|
+
completed_experiments: [],
|
|
468
|
+
failed_experiments: [],
|
|
469
|
+
in_progress_experiment: null,
|
|
470
|
+
timestamp: null,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
const storage = new LocalStorage(dashRoot);
|
|
474
|
+
console.log(`${bold("Scanning local storage:")} ${dashRoot}`);
|
|
475
|
+
const projectFilter = typeof args.project === "string" ? args.project : undefined;
|
|
476
|
+
let experiments = discoverExperiments(storage, projectFilter);
|
|
477
|
+
if (experiments.length === 0) {
|
|
478
|
+
console.log(projectFilter
|
|
479
|
+
? `${yellow("No experiments found matching pattern:")} ${projectFilter}`
|
|
480
|
+
: yellow("No experiments found in local storage"));
|
|
481
|
+
return 1;
|
|
482
|
+
}
|
|
483
|
+
if (args.resume && state.completed_experiments.length > 0) {
|
|
484
|
+
const before = experiments.length;
|
|
485
|
+
const done = new Set(state.completed_experiments);
|
|
486
|
+
experiments = experiments.filter((e) => !done.has(`${e.project}/${e.experiment}`));
|
|
487
|
+
const skipped = before - experiments.length;
|
|
488
|
+
if (skipped > 0)
|
|
489
|
+
console.log(dim(`Skipping ${skipped} already completed experiment(s)`));
|
|
490
|
+
}
|
|
491
|
+
console.log(green(`Found ${experiments.length} experiment(s) to upload`));
|
|
492
|
+
if (args.verbose || args.dry_run) {
|
|
493
|
+
console.log(`\n${bold("Discovered experiments:")}`);
|
|
494
|
+
for (const exp of experiments) {
|
|
495
|
+
const parts = [];
|
|
496
|
+
if (exp.hasLogs)
|
|
497
|
+
parts.push("logs");
|
|
498
|
+
if (exp.hasParams)
|
|
499
|
+
parts.push("params");
|
|
500
|
+
if (exp.metrics.length)
|
|
501
|
+
parts.push(`${exp.metrics.length} metrics`);
|
|
502
|
+
if (exp.fileCount)
|
|
503
|
+
parts.push(`${exp.fileCount} files (${formatBytes(exp.estimatedSize)})`);
|
|
504
|
+
const details = parts.length ? parts.join(", ") : "metadata only";
|
|
505
|
+
console.log(` ${cyan("•")} ${exp.storagePrefix} ${dim(`(${details})`)}`);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
if (args.dry_run) {
|
|
509
|
+
console.log(`\n${yellow(bold("DRY RUN"))} - No data will be uploaded`);
|
|
510
|
+
console.log("Run without --dry-run to proceed with upload.");
|
|
511
|
+
return 0;
|
|
512
|
+
}
|
|
513
|
+
console.log(`\n${bold("Validating experiments...")}`);
|
|
514
|
+
const validations = new Map();
|
|
515
|
+
const valid = [];
|
|
516
|
+
let invalidCount = 0;
|
|
517
|
+
for (const exp of experiments) {
|
|
518
|
+
const key = `${exp.storagePrefix}`;
|
|
519
|
+
const validation = validateExperiment(storage, exp, args.strict === true);
|
|
520
|
+
validations.set(key, validation);
|
|
521
|
+
if (validation.isValid)
|
|
522
|
+
valid.push(exp);
|
|
523
|
+
else
|
|
524
|
+
invalidCount++;
|
|
525
|
+
if (validation.errors.length > 0) {
|
|
526
|
+
console.log(` ${red("✗")} ${key}:`);
|
|
527
|
+
for (const error of validation.errors)
|
|
528
|
+
console.log(` ${red(error)}`);
|
|
529
|
+
}
|
|
530
|
+
else if (args.verbose && validation.warnings.length > 0) {
|
|
531
|
+
console.log(` ${yellow("⚠")} ${key}:`);
|
|
532
|
+
for (const warning of validation.warnings)
|
|
533
|
+
console.log(` ${yellow(warning)}`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
if (invalidCount > 0) {
|
|
537
|
+
console.log(`\n${yellow(`${invalidCount} experiment(s) failed validation and will be skipped`)}`);
|
|
538
|
+
if (args.strict) {
|
|
539
|
+
console.error(red("Error: Validation failed in --strict mode"));
|
|
540
|
+
return 1;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
if (valid.length === 0) {
|
|
544
|
+
console.error(red("Error: No valid experiments to upload"));
|
|
545
|
+
return 1;
|
|
546
|
+
}
|
|
547
|
+
console.log(green(`${valid.length} experiment(s) ready to upload`));
|
|
548
|
+
const target = typeof args.target === "string" ? args.target : undefined;
|
|
549
|
+
let namespace;
|
|
550
|
+
if (target)
|
|
551
|
+
namespace = target.replace(/^\/+|\/+$/g, "").split("/")[0];
|
|
552
|
+
if (!namespace) {
|
|
553
|
+
const first = valid[0].declaredPrefix ?? valid[0].storagePrefix;
|
|
554
|
+
namespace = first.replace(/^\/+/, "").split("/")[0];
|
|
555
|
+
}
|
|
556
|
+
const client = nativeTarget.client;
|
|
557
|
+
if (nativeTarget) {
|
|
558
|
+
// Everything the native backend cannot store is refused here, before the first write of any experiment.
|
|
559
|
+
const problems = await nativeTarget.preflight(storage, valid.map(exp => ({ info: exp, validation: validations.get(`${exp.storagePrefix}`),
|
|
560
|
+
...remotePrefixFor(exp, target) })), args.skip_logs === true, args.skip_metrics === true);
|
|
561
|
+
if (problems.length) {
|
|
562
|
+
console.error(red(`Error: ${problems.length} experiment(s) cannot be uploaded to the native backend without losing data; nothing was uploaded.`));
|
|
563
|
+
for (const problem of problems)
|
|
564
|
+
console.error(` ${red("✗")} ${problem}`);
|
|
565
|
+
return 1;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
const opts = {
|
|
569
|
+
batchSize,
|
|
570
|
+
skipLogs: args.skip_logs === true,
|
|
571
|
+
skipMetrics: args.skip_metrics === true,
|
|
572
|
+
skipFiles: args.skip_files === true,
|
|
573
|
+
skipParams: args.skip_params === true,
|
|
574
|
+
verbose: args.verbose === true,
|
|
575
|
+
targetPrefix: target,
|
|
576
|
+
};
|
|
577
|
+
console.log(`\n${bold("Uploading to:")} ${nativeTarget ? `${nativeTarget.server} (native)` : ctx.remoteUrl}`);
|
|
578
|
+
if (target)
|
|
579
|
+
console.log(`${bold("Target prefix:")} ${target}`);
|
|
580
|
+
const started = Date.now();
|
|
581
|
+
const results = [];
|
|
582
|
+
for (let i = 0; i < valid.length; i++) {
|
|
583
|
+
const exp = valid[i];
|
|
584
|
+
const key = `${exp.storagePrefix}`;
|
|
585
|
+
console.log(`${cyan(`[${i + 1}/${valid.length}] ${key}`)}`);
|
|
586
|
+
state.in_progress_experiment = key;
|
|
587
|
+
saveState(stateFile, state);
|
|
588
|
+
const result = await uploadExperiment(storage, client, exp, validations.get(key), opts);
|
|
589
|
+
results.push(result);
|
|
590
|
+
state.in_progress_experiment = null;
|
|
591
|
+
if (result.success)
|
|
592
|
+
state.completed_experiments.push(key);
|
|
593
|
+
else
|
|
594
|
+
state.failed_experiments.push(key);
|
|
595
|
+
saveState(stateFile, state);
|
|
596
|
+
if (!args.verbose) {
|
|
597
|
+
if (result.success) {
|
|
598
|
+
const parts = [];
|
|
599
|
+
if (result.uploaded.params)
|
|
600
|
+
parts.push(`${result.uploaded.params} params`);
|
|
601
|
+
if (result.uploaded.logs)
|
|
602
|
+
parts.push(`${result.uploaded.logs} logs`);
|
|
603
|
+
if (result.uploaded.metrics)
|
|
604
|
+
parts.push(`${result.uploaded.metrics} metrics`);
|
|
605
|
+
if (result.uploaded.files)
|
|
606
|
+
parts.push(`${result.uploaded.files} files`);
|
|
607
|
+
console.log(` ${green("✓")} Uploaded (${parts.length ? parts.join(", ") : "metadata only"})`);
|
|
608
|
+
}
|
|
609
|
+
else {
|
|
610
|
+
console.log(` ${red("✗")} Failed`);
|
|
611
|
+
for (const error of result.errors.slice(0, 3))
|
|
612
|
+
console.log(` ${red(error)}`);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
const elapsed = (Date.now() - started) / 1000;
|
|
617
|
+
const totalBytes = results.reduce((n, r) => n + r.bytesUploaded, 0);
|
|
618
|
+
const successful = results.filter((r) => r.success);
|
|
619
|
+
const failed = results.filter((r) => !r.success);
|
|
620
|
+
const summary = [
|
|
621
|
+
["Successful", `${successful.length}/${results.length}`],
|
|
622
|
+
];
|
|
623
|
+
if (failed.length)
|
|
624
|
+
summary.push(["Failed", `${failed.length}/${results.length}`]);
|
|
625
|
+
summary.push(["Total Time", `${elapsed.toFixed(2)}s`]);
|
|
626
|
+
if (totalBytes > 0 && elapsed > 0) {
|
|
627
|
+
summary.push(["Avg Speed", `${formatBytes(totalBytes / elapsed)}/s`]);
|
|
628
|
+
}
|
|
629
|
+
console.log();
|
|
630
|
+
console.log(renderTable([{ header: "Status" }, { header: "Count", align: "right" }], summary, { title: "Upload Summary" }));
|
|
631
|
+
if (failed.length > 0) {
|
|
632
|
+
console.log(`\n${bold(red("Failed Experiments:"))}`);
|
|
633
|
+
for (const result of failed) {
|
|
634
|
+
console.log(` ${red("✗")} ${result.experiment}`);
|
|
635
|
+
for (const error of result.errors)
|
|
636
|
+
console.log(` ${dim(error)}`);
|
|
637
|
+
for (const [section, messages] of Object.entries(result.failed)) {
|
|
638
|
+
for (const message of messages)
|
|
639
|
+
console.log(` ${dim(`${section}: ${message}`)}`);
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
const totals = {
|
|
644
|
+
Logs: results.reduce((n, r) => n + (r.uploaded.logs ?? 0), 0),
|
|
645
|
+
Metrics: results.reduce((n, r) => n + (r.uploaded.metrics ?? 0), 0),
|
|
646
|
+
Files: results.reduce((n, r) => n + (r.uploaded.files ?? 0), 0),
|
|
647
|
+
};
|
|
648
|
+
const unit = { Logs: "entries", Metrics: "metrics", Files: "files" };
|
|
649
|
+
const dataRows = Object.entries(totals)
|
|
650
|
+
.filter(([, n]) => n > 0)
|
|
651
|
+
.map(([type, n]) => [type, `${n} ${unit[type]}`]);
|
|
652
|
+
if (dataRows.length > 0) {
|
|
653
|
+
console.log();
|
|
654
|
+
console.log(renderTable([{ header: "Type" }, { header: "Count", align: "right" }], dataRows, { title: "Data Uploaded" }));
|
|
655
|
+
}
|
|
656
|
+
if (failed.length === 0) {
|
|
657
|
+
if (existsSync(stateFile))
|
|
658
|
+
unlinkSync(stateFile);
|
|
659
|
+
console.log(`\n${dim("Upload complete. State file removed.")}`);
|
|
660
|
+
}
|
|
661
|
+
else {
|
|
662
|
+
console.log(`\n${yellow(`State saved to ${stateFile}. Use --resume to retry failed uploads.`)}`);
|
|
663
|
+
}
|
|
664
|
+
return failed.length === 0 ? 0 : 1;
|
|
665
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** .dash format compatibility, ported from the MIT-licensed ml-dash-cli. */
|
|
2
|
+
export declare const bold: (s: string) => string;
|
|
3
|
+
export declare const dim: (s: string) => string;
|
|
4
|
+
export declare const red: (s: string) => string;
|
|
5
|
+
export declare const green: (s: string) => string;
|
|
6
|
+
export declare const yellow: (s: string) => string;
|
|
7
|
+
export declare const blue: (s: string) => string;
|
|
8
|
+
export declare const cyan: (s: string) => string;
|
|
9
|
+
/** Visible width, ignoring the escape sequences the styles above insert. */
|
|
10
|
+
export declare const visibleWidth: (s: string) => number;
|
|
11
|
+
export interface Column {
|
|
12
|
+
header: string;
|
|
13
|
+
align?: "left" | "right" | "center";
|
|
14
|
+
}
|
|
15
|
+
/** Rounded-box table, the same shape `rich.box.ROUNDED` produced. */
|
|
16
|
+
export declare function renderTable(columns: Column[], rows: string[][], opts?: {
|
|
17
|
+
title?: string;
|
|
18
|
+
caption?: string;
|
|
19
|
+
}): string;
|
|
20
|
+
/** Bordered panel, the shape `rich.panel.Panel` produced. */
|
|
21
|
+
export declare function renderPanel(body: string, opts?: {
|
|
22
|
+
title?: string;
|
|
23
|
+
}): string;
|
|
24
|
+
export declare function formatBytes(n: number): string;
|