@dreamlake/ml-dash 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +98 -0
- package/bin/ml-dash.js +5 -0
- package/dist/auth/device-flow.js +131 -0
- package/dist/auth/device-secret.js +20 -0
- package/dist/auth/fernet.js +98 -0
- package/dist/auth/jwt.js +12 -0
- package/dist/auth/token-storage.js +217 -0
- package/dist/cli/context.js +34 -0
- package/dist/cli/parser.js +153 -0
- package/dist/client.js +588 -0
- package/dist/commands/api.js +89 -0
- package/dist/commands/create.js +82 -0
- package/dist/commands/download.js +661 -0
- package/dist/commands/list.js +341 -0
- package/dist/commands/login.js +128 -0
- package/dist/commands/logout.js +22 -0
- package/dist/commands/profile.js +143 -0
- package/dist/commands/remove.js +123 -0
- package/dist/commands/upload.js +786 -0
- package/dist/commands/version.js +10 -0
- package/dist/config.js +64 -0
- package/dist/index.js +74 -0
- package/dist/local/safe-path.js +137 -0
- package/dist/local/storage.js +447 -0
- package/dist/util/ansi.js +78 -0
- package/dist/util/glob.js +47 -0
- package/dist/util/json.js +70 -0
- package/dist/util/pool.js +30 -0
- package/dist/version.js +3 -0
- package/package.json +49 -0
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `.dash` directory on disk — the same tree the Python SDK writes.
|
|
3
|
+
*
|
|
4
|
+
* .dash/{owner}/{project}/{folders…}/{experiment}/
|
|
5
|
+
* experiment.json
|
|
6
|
+
* parameters.json
|
|
7
|
+
* logs/logs.jsonl + .log_sequence
|
|
8
|
+
* metrics/{name}/data.jsonl + metadata.json ← current layout
|
|
9
|
+
* metrics/{name}.jsonl ← legacy layout
|
|
10
|
+
* files/{path}/{file_id}/{filename}
|
|
11
|
+
* files/.files_metadata.json
|
|
12
|
+
*
|
|
13
|
+
* Both metric layouts are read. The Python SDK still writes the legacy flat
|
|
14
|
+
* form from `write_metric_data`, while its CLI's discovery only ever looked for
|
|
15
|
+
* the `metrics/<name>/data.jsonl` layout — so an experiment recorded through that path uploaded
|
|
16
|
+
* as "successful" with none of its metrics. Reading both is a deliberate
|
|
17
|
+
* divergence from the Python CLI, recorded in docs/PORTING.md.
|
|
18
|
+
*/
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
import { appendFileSync, createReadStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
|
|
21
|
+
import { copyFile } from "node:fs/promises";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
import { literal, resolveUnderRoot } from "./safe-path.js";
|
|
24
|
+
export const FILES_METADATA_FILENAME = ".files_metadata.json";
|
|
25
|
+
export const utcNowIso = () => new Date().toISOString().replace(/Z$/, "Z");
|
|
26
|
+
/** Local-mode ID, matching the Python `(ms << 12) | rand12` shape. */
|
|
27
|
+
export function generateSnowflakeId() {
|
|
28
|
+
const ms = BigInt(Date.now());
|
|
29
|
+
const rand = BigInt(Math.floor(Math.random() * 4096));
|
|
30
|
+
return ((ms << 12n) | rand).toString();
|
|
31
|
+
}
|
|
32
|
+
export function sha256File(filePath) {
|
|
33
|
+
return new Promise((resolve, reject) => {
|
|
34
|
+
const hash = createHash("sha256");
|
|
35
|
+
createReadStream(filePath)
|
|
36
|
+
.on("error", reject)
|
|
37
|
+
.on("data", (c) => hash.update(c))
|
|
38
|
+
.on("end", () => resolve(hash.digest("hex")));
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
export class LocalStorage {
|
|
42
|
+
rootPath;
|
|
43
|
+
constructor(rootPath) {
|
|
44
|
+
const resolved = path.resolve(rootPath);
|
|
45
|
+
mkdirSync(resolved, { recursive: true });
|
|
46
|
+
// The root is the one path the user picked deliberately, so following its
|
|
47
|
+
// own links is what they asked for; everything below it is checked
|
|
48
|
+
// against this canonical form.
|
|
49
|
+
this.rootPath = realpathSync(resolved);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The single boundary: every path this class touches is built here, from
|
|
53
|
+
* the root down, with the server-supplied parts and the literal ones
|
|
54
|
+
* checked alike. A derived directory is never treated as a new root.
|
|
55
|
+
*/
|
|
56
|
+
target(parts, mode = "write") {
|
|
57
|
+
return resolveUnderRoot(this.rootPath, parts, mode);
|
|
58
|
+
}
|
|
59
|
+
static prefixPart = (prefix) => ({
|
|
60
|
+
label: "experiment prefix",
|
|
61
|
+
value: prefix,
|
|
62
|
+
nested: true,
|
|
63
|
+
});
|
|
64
|
+
/**
|
|
65
|
+
* Experiment directory for a prefix of the form owner/project/folders…/name.
|
|
66
|
+
*
|
|
67
|
+
* Read mode: callers outside this class use it to look, and `upload` has to
|
|
68
|
+
* keep working on a tree where the user symlinked their own data. Every
|
|
69
|
+
* write below goes through `target(…, "write")` instead.
|
|
70
|
+
*/
|
|
71
|
+
experimentDir(prefix) {
|
|
72
|
+
return this.target([LocalStorage.prefixPart(prefix)], "read");
|
|
73
|
+
}
|
|
74
|
+
/** Same directory, as a write target. */
|
|
75
|
+
experimentWriteDir(prefix) {
|
|
76
|
+
return this.target([LocalStorage.prefixPart(prefix)]);
|
|
77
|
+
}
|
|
78
|
+
// ── experiment metadata ────────────────────────────────────────────────────
|
|
79
|
+
/**
|
|
80
|
+
* Create or merge experiment.json. Existing files are merged, never
|
|
81
|
+
* clobbered, so a download into a populated tree does not discard fields the
|
|
82
|
+
* server did not return.
|
|
83
|
+
*/
|
|
84
|
+
createExperiment(args) {
|
|
85
|
+
const prefixClean = args.prefix.replace(/\/+$/, "");
|
|
86
|
+
const dir = this.experimentWriteDir(prefixClean);
|
|
87
|
+
mkdirSync(dir, { recursive: true });
|
|
88
|
+
for (const sub of ["logs", "metrics", "files"]) {
|
|
89
|
+
mkdirSync(this.target([LocalStorage.prefixPart(prefixClean), literal(sub)]), { recursive: true });
|
|
90
|
+
}
|
|
91
|
+
const name = prefixClean.split("/").pop();
|
|
92
|
+
const file = this.target([LocalStorage.prefixPart(prefixClean), literal("experiment.json")]);
|
|
93
|
+
if (!existsSync(file)) {
|
|
94
|
+
writeFileSync(file, JSON.stringify({
|
|
95
|
+
name,
|
|
96
|
+
project: args.project,
|
|
97
|
+
description: args.description ?? null,
|
|
98
|
+
tags: args.tags ?? [],
|
|
99
|
+
bindrs: args.bindrs ?? [],
|
|
100
|
+
prefix: args.prefix,
|
|
101
|
+
metadata: args.metadata ?? null,
|
|
102
|
+
created_at: utcNowIso(),
|
|
103
|
+
write_protected: false,
|
|
104
|
+
}));
|
|
105
|
+
return dir;
|
|
106
|
+
}
|
|
107
|
+
let existing;
|
|
108
|
+
try {
|
|
109
|
+
existing = JSON.parse(readFileSync(file, "utf8"));
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// Corrupt or truncated — rewrite rather than fail the whole download.
|
|
113
|
+
writeFileSync(file, JSON.stringify({
|
|
114
|
+
name,
|
|
115
|
+
project: args.project,
|
|
116
|
+
description: args.description ?? null,
|
|
117
|
+
tags: args.tags ?? [],
|
|
118
|
+
bindrs: args.bindrs ?? [],
|
|
119
|
+
prefix: args.prefix,
|
|
120
|
+
metadata: args.metadata ?? null,
|
|
121
|
+
created_at: utcNowIso(),
|
|
122
|
+
write_protected: false,
|
|
123
|
+
}));
|
|
124
|
+
return dir;
|
|
125
|
+
}
|
|
126
|
+
if (args.description != null)
|
|
127
|
+
existing.description = args.description;
|
|
128
|
+
if (args.tags != null)
|
|
129
|
+
existing.tags = args.tags;
|
|
130
|
+
if (args.bindrs != null)
|
|
131
|
+
existing.bindrs = args.bindrs;
|
|
132
|
+
if (args.prefix != null)
|
|
133
|
+
existing.prefix = args.prefix;
|
|
134
|
+
if (args.metadata != null)
|
|
135
|
+
existing.metadata = args.metadata;
|
|
136
|
+
existing.updated_at = utcNowIso();
|
|
137
|
+
writeFileSync(file, JSON.stringify(existing));
|
|
138
|
+
return dir;
|
|
139
|
+
}
|
|
140
|
+
readExperiment(prefix) {
|
|
141
|
+
const file = this.target([LocalStorage.prefixPart(prefix), literal("experiment.json")], "read");
|
|
142
|
+
if (!existsSync(file))
|
|
143
|
+
return null;
|
|
144
|
+
try {
|
|
145
|
+
return JSON.parse(readFileSync(file, "utf8"));
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// ── parameters ─────────────────────────────────────────────────────────────
|
|
152
|
+
/** Merge into parameters.json, bumping `version` the way the SDK does. */
|
|
153
|
+
writeParameters(prefix, data) {
|
|
154
|
+
mkdirSync(this.experimentWriteDir(prefix), { recursive: true });
|
|
155
|
+
const file = this.target([LocalStorage.prefixPart(prefix), literal("parameters.json")]);
|
|
156
|
+
let merged = {};
|
|
157
|
+
let version = 1;
|
|
158
|
+
if (existsSync(file)) {
|
|
159
|
+
try {
|
|
160
|
+
const doc = JSON.parse(readFileSync(file, "utf8"));
|
|
161
|
+
if (doc && typeof doc === "object") {
|
|
162
|
+
merged = (doc.data && typeof doc.data === "object" ? doc.data : doc);
|
|
163
|
+
version = typeof doc.version === "number" ? doc.version + 1 : 2;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
merged = {};
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
writeFileSync(file, JSON.stringify({ version, data: { ...merged, ...data }, updatedAt: utcNowIso() }));
|
|
171
|
+
}
|
|
172
|
+
/** Read parameters, accepting both the versioned and the bare-dict shapes. */
|
|
173
|
+
readParameters(prefix) {
|
|
174
|
+
const file = this.target([LocalStorage.prefixPart(prefix), literal("parameters.json")], "read");
|
|
175
|
+
if (!existsSync(file))
|
|
176
|
+
return null;
|
|
177
|
+
try {
|
|
178
|
+
const doc = JSON.parse(readFileSync(file, "utf8"));
|
|
179
|
+
if (!doc || typeof doc !== "object" || Array.isArray(doc))
|
|
180
|
+
return null;
|
|
181
|
+
if ("data" in doc) {
|
|
182
|
+
return doc.data && typeof doc.data === "object" ? doc.data : null;
|
|
183
|
+
}
|
|
184
|
+
return doc;
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
// ── logs ───────────────────────────────────────────────────────────────────
|
|
191
|
+
/**
|
|
192
|
+
* Append log entries, continuing the `.log_sequence` counter so downloaded
|
|
193
|
+
* logs interleave correctly with anything already recorded locally.
|
|
194
|
+
*/
|
|
195
|
+
appendLogs(prefix, entries) {
|
|
196
|
+
if (entries.length === 0)
|
|
197
|
+
return 0;
|
|
198
|
+
const logsDir = this.target([LocalStorage.prefixPart(prefix), literal("logs")]);
|
|
199
|
+
mkdirSync(logsDir, { recursive: true });
|
|
200
|
+
const logsFile = this.target([LocalStorage.prefixPart(prefix), literal("logs"), literal("logs.jsonl")]);
|
|
201
|
+
const seqFile = this.target([LocalStorage.prefixPart(prefix), literal("logs"), literal(".log_sequence")]);
|
|
202
|
+
let sequence = 0;
|
|
203
|
+
if (existsSync(seqFile)) {
|
|
204
|
+
const parsed = Number.parseInt(readFileSync(seqFile, "utf8").trim(), 10);
|
|
205
|
+
if (Number.isFinite(parsed))
|
|
206
|
+
sequence = parsed;
|
|
207
|
+
}
|
|
208
|
+
let out = "";
|
|
209
|
+
for (const e of entries) {
|
|
210
|
+
const record = {
|
|
211
|
+
sequenceNumber: sequence++,
|
|
212
|
+
timestamp: e.timestamp ?? "",
|
|
213
|
+
level: e.level ?? "info",
|
|
214
|
+
message: e.message,
|
|
215
|
+
};
|
|
216
|
+
if (e.metadata)
|
|
217
|
+
record.metadata = e.metadata;
|
|
218
|
+
out += JSON.stringify(record) + "\n";
|
|
219
|
+
}
|
|
220
|
+
appendFileSync(logsFile, out);
|
|
221
|
+
writeFileSync(seqFile, String(sequence));
|
|
222
|
+
return entries.length;
|
|
223
|
+
}
|
|
224
|
+
readLogs(prefix) {
|
|
225
|
+
const file = this.target([LocalStorage.prefixPart(prefix), literal("logs"), literal("logs.jsonl")], "read");
|
|
226
|
+
if (!existsSync(file))
|
|
227
|
+
return [];
|
|
228
|
+
const out = [];
|
|
229
|
+
for (const line of readFileSync(file, "utf8").split("\n")) {
|
|
230
|
+
if (!line.trim())
|
|
231
|
+
continue;
|
|
232
|
+
try {
|
|
233
|
+
out.push(JSON.parse(line));
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
/* skipped, as the Python uploader skipped unparseable lines */
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return out;
|
|
240
|
+
}
|
|
241
|
+
// ── metrics ────────────────────────────────────────────────────────────────
|
|
242
|
+
/**
|
|
243
|
+
* Every metric in the experiment, from both layouts.
|
|
244
|
+
*
|
|
245
|
+
* A name present in both is reported once, from the directory layout, which
|
|
246
|
+
* is the one the current SDK writes and the one that carries an index.
|
|
247
|
+
*/
|
|
248
|
+
listMetrics(prefix) {
|
|
249
|
+
const metricsDir = this.target([LocalStorage.prefixPart(prefix), literal("metrics")], "read");
|
|
250
|
+
if (!existsSync(metricsDir))
|
|
251
|
+
return [];
|
|
252
|
+
const found = new Map();
|
|
253
|
+
for (const entry of readdirSync(metricsDir, { withFileTypes: true })) {
|
|
254
|
+
if (entry.isDirectory()) {
|
|
255
|
+
const dataFile = path.join(metricsDir, entry.name, "data.jsonl");
|
|
256
|
+
if (existsSync(dataFile)) {
|
|
257
|
+
found.set(entry.name, { name: entry.name, layout: "directory", dataFile });
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
for (const entry of readdirSync(metricsDir, { withFileTypes: true })) {
|
|
262
|
+
if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
263
|
+
const name = entry.name.slice(0, -".jsonl".length);
|
|
264
|
+
if (!found.has(name)) {
|
|
265
|
+
found.set(name, { name, layout: "flat", dataFile: path.join(metricsDir, entry.name) });
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Read one metric's data points as the values the server wants.
|
|
273
|
+
*
|
|
274
|
+
* Both layouts store one JSON object per line with the point under `data`;
|
|
275
|
+
* the directory layout adds `index` and `createdAt`. Lines without `data`
|
|
276
|
+
* are skipped and counted, never uploaded as `undefined`.
|
|
277
|
+
*/
|
|
278
|
+
readMetricPoints(metric) {
|
|
279
|
+
const points = [];
|
|
280
|
+
let skipped = 0;
|
|
281
|
+
let bytes = 0;
|
|
282
|
+
if (!existsSync(metric.dataFile))
|
|
283
|
+
return { points, skipped, bytes };
|
|
284
|
+
for (const line of readFileSync(metric.dataFile, "utf8").split("\n")) {
|
|
285
|
+
if (!line.trim())
|
|
286
|
+
continue;
|
|
287
|
+
bytes += Buffer.byteLength(line, "utf8");
|
|
288
|
+
try {
|
|
289
|
+
const parsed = JSON.parse(line);
|
|
290
|
+
if (parsed === null || typeof parsed !== "object" || !("data" in parsed)) {
|
|
291
|
+
skipped++;
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
points.push(parsed.data);
|
|
295
|
+
}
|
|
296
|
+
catch {
|
|
297
|
+
skipped++;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return { points, skipped, bytes };
|
|
301
|
+
}
|
|
302
|
+
/** Append points in the directory layout, maintaining index and counts. */
|
|
303
|
+
appendBatchToMetric(prefix, metricName, dataPoints) {
|
|
304
|
+
const dirName = metricName === null ? "None" : String(metricName);
|
|
305
|
+
// A nested name ('train/loss') keeps nesting; '..' in one does not. Every
|
|
306
|
+
// part, including 'metrics' and the two filenames, is checked from the
|
|
307
|
+
// root — `metrics` being a link out of the tree is the case a check
|
|
308
|
+
// anchored at the metrics directory could not see.
|
|
309
|
+
const metricParts = [
|
|
310
|
+
LocalStorage.prefixPart(prefix),
|
|
311
|
+
literal("metrics"),
|
|
312
|
+
{ label: "metric name", value: dirName, nested: true },
|
|
313
|
+
];
|
|
314
|
+
const metricDir = this.target(metricParts);
|
|
315
|
+
mkdirSync(metricDir, { recursive: true });
|
|
316
|
+
const dataFile = this.target([...metricParts, literal("data.jsonl")]);
|
|
317
|
+
const metadataFile = this.target([...metricParts, literal("metadata.json")]);
|
|
318
|
+
let meta = {
|
|
319
|
+
metricId: `local-metric-${metricName}`,
|
|
320
|
+
name: metricName,
|
|
321
|
+
description: null,
|
|
322
|
+
tags: [],
|
|
323
|
+
metadata: null,
|
|
324
|
+
totalDataPoints: 0,
|
|
325
|
+
nextIndex: 0,
|
|
326
|
+
createdAt: utcNowIso(),
|
|
327
|
+
};
|
|
328
|
+
if (existsSync(metadataFile)) {
|
|
329
|
+
try {
|
|
330
|
+
meta = JSON.parse(readFileSync(metadataFile, "utf8"));
|
|
331
|
+
}
|
|
332
|
+
catch {
|
|
333
|
+
/* reinitialise from the default above */
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
const startIndex = meta.nextIndex ?? 0;
|
|
337
|
+
const batchTs = utcNowIso();
|
|
338
|
+
let out = "";
|
|
339
|
+
dataPoints.forEach((data, i) => {
|
|
340
|
+
out += JSON.stringify({ index: startIndex + i, data, createdAt: batchTs }) + "\n";
|
|
341
|
+
});
|
|
342
|
+
if (out)
|
|
343
|
+
appendFileSync(dataFile, out);
|
|
344
|
+
meta.nextIndex = startIndex + dataPoints.length;
|
|
345
|
+
meta.totalDataPoints = (meta.totalDataPoints ?? 0) + dataPoints.length;
|
|
346
|
+
meta.updatedAt = batchTs;
|
|
347
|
+
writeFileSync(metadataFile, JSON.stringify(meta));
|
|
348
|
+
}
|
|
349
|
+
// ── files ──────────────────────────────────────────────────────────────────
|
|
350
|
+
filesDir(prefix, mode = "write") {
|
|
351
|
+
return this.target([LocalStorage.prefixPart(prefix), literal("files")], mode);
|
|
352
|
+
}
|
|
353
|
+
/** The parts of a stored file's path, from the root down. */
|
|
354
|
+
fileParts(prefix, sub, ...rest) {
|
|
355
|
+
const parts = [LocalStorage.prefixPart(prefix), literal("files")];
|
|
356
|
+
// Python's storage lstrips a leading separator here; anything else in the
|
|
357
|
+
// path is checked rather than repaired.
|
|
358
|
+
const normalized = (sub ?? "").replace(/^\/+/, "");
|
|
359
|
+
if (normalized)
|
|
360
|
+
parts.push({ label: "file path", value: normalized, nested: true });
|
|
361
|
+
return [...parts, ...rest];
|
|
362
|
+
}
|
|
363
|
+
loadFilesMetadata(prefix) {
|
|
364
|
+
const file = this.target([LocalStorage.prefixPart(prefix), literal("files"), literal(FILES_METADATA_FILENAME)], "read");
|
|
365
|
+
if (!existsSync(file))
|
|
366
|
+
return { files: [] };
|
|
367
|
+
try {
|
|
368
|
+
const parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
369
|
+
return Array.isArray(parsed?.files) ? parsed : { files: [] };
|
|
370
|
+
}
|
|
371
|
+
catch {
|
|
372
|
+
return { files: [] };
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
saveFilesMetadata(prefix, data) {
|
|
376
|
+
mkdirSync(this.filesDir(prefix), { recursive: true });
|
|
377
|
+
const file = this.target([LocalStorage.prefixPart(prefix), literal("files"), literal(FILES_METADATA_FILENAME)]);
|
|
378
|
+
// Write-then-rename: a crash mid-write would otherwise leave the manifest
|
|
379
|
+
// unparseable and orphan every file it listed. The temporary name is a
|
|
380
|
+
// target of its own, so a planted link cannot catch the write either.
|
|
381
|
+
const tmp = this.target([
|
|
382
|
+
LocalStorage.prefixPart(prefix),
|
|
383
|
+
literal("files"),
|
|
384
|
+
literal(`${FILES_METADATA_FILENAME}.tmp-${process.pid}`),
|
|
385
|
+
]);
|
|
386
|
+
writeFileSync(tmp, JSON.stringify(data));
|
|
387
|
+
renameSync(tmp, file);
|
|
388
|
+
}
|
|
389
|
+
/** Non-deleted file records for an experiment. */
|
|
390
|
+
listFiles(prefix, pathPrefix, tags) {
|
|
391
|
+
let files = this.loadFilesMetadata(prefix).files.filter((f) => f.deletedAt === null);
|
|
392
|
+
if (pathPrefix)
|
|
393
|
+
files = files.filter((f) => (f.path ?? "").startsWith(pathPrefix));
|
|
394
|
+
if (tags?.length) {
|
|
395
|
+
const wanted = new Set(tags);
|
|
396
|
+
files = files.filter((f) => (f.tags ?? []).some((t) => wanted.has(t)));
|
|
397
|
+
}
|
|
398
|
+
return files;
|
|
399
|
+
}
|
|
400
|
+
/** Absolute on-disk path of a recorded file. */
|
|
401
|
+
filePath(prefix, record) {
|
|
402
|
+
return this.target(this.fileParts(prefix, record.path ?? "", { label: "file id", value: String(record.id) }, { label: "filename", value: record.filename }), "read");
|
|
403
|
+
}
|
|
404
|
+
/** Copy a file in and record it, replacing any record with the same path+filename. */
|
|
405
|
+
async writeFile(args) {
|
|
406
|
+
const fileId = generateSnowflakeId();
|
|
407
|
+
const idPart = { label: "file id", value: fileId };
|
|
408
|
+
const namePart = { label: "filename", value: args.filename };
|
|
409
|
+
const fileDir = this.target(this.fileParts(args.prefix, args.path ?? "", idPart));
|
|
410
|
+
const destination = this.target(this.fileParts(args.prefix, args.path ?? "", idPart, namePart));
|
|
411
|
+
const filename = args.filename;
|
|
412
|
+
mkdirSync(fileDir, { recursive: true });
|
|
413
|
+
await copyFile(args.sourcePath, destination);
|
|
414
|
+
const now = utcNowIso();
|
|
415
|
+
const record = {
|
|
416
|
+
id: fileId,
|
|
417
|
+
experimentId: `${args.project}/${args.prefix}`,
|
|
418
|
+
path: args.path ?? "",
|
|
419
|
+
filename,
|
|
420
|
+
description: args.description ?? null,
|
|
421
|
+
tags: args.tags ?? [],
|
|
422
|
+
bindrs: args.bindrs ?? [],
|
|
423
|
+
contentType: args.contentType ?? "",
|
|
424
|
+
sizeBytes: args.sizeBytes ?? statSync(destination).size,
|
|
425
|
+
checksum: args.checksum ?? "",
|
|
426
|
+
metadata: args.metadata ?? null,
|
|
427
|
+
uploadedAt: now,
|
|
428
|
+
updatedAt: now,
|
|
429
|
+
deletedAt: null,
|
|
430
|
+
};
|
|
431
|
+
const meta = this.loadFilesMetadata(args.prefix);
|
|
432
|
+
const existingIndex = meta.files.findIndex((f) => f.path === record.path && f.filename === record.filename && f.deletedAt === null);
|
|
433
|
+
if (existingIndex >= 0) {
|
|
434
|
+
const old = meta.files[existingIndex];
|
|
435
|
+
// Only the directory this manifest created for that record, and only
|
|
436
|
+
// if every component of it is a real directory under the root: never a
|
|
437
|
+
// link the record could point at something else through.
|
|
438
|
+
rmSync(this.target(this.fileParts(args.prefix, old.path ?? "", { label: "file id", value: String(old.id) })), { recursive: true, force: true });
|
|
439
|
+
meta.files[existingIndex] = record;
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
meta.files.push(record);
|
|
443
|
+
}
|
|
444
|
+
this.saveFilesMetadata(args.prefix, meta);
|
|
445
|
+
return record;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal ANSI styling + table rendering.
|
|
3
|
+
*
|
|
4
|
+
* The Python CLI used `rich`. Porting `rich` wholesale would drag a rendering
|
|
5
|
+
* engine into a single-file binary for the sake of a handful of tables, so the
|
|
6
|
+
* subset that the CLI actually uses lives here instead: colours, a box table,
|
|
7
|
+
* and a panel. Colour is suppressed when stdout is not a TTY or NO_COLOR is
|
|
8
|
+
* set, so piping `ml-dash list` into a file yields plain text.
|
|
9
|
+
*/
|
|
10
|
+
const enabled = () => {
|
|
11
|
+
if (process.env.NO_COLOR !== undefined)
|
|
12
|
+
return false;
|
|
13
|
+
if (process.env.FORCE_COLOR !== undefined)
|
|
14
|
+
return true;
|
|
15
|
+
return process.stdout.isTTY === true;
|
|
16
|
+
};
|
|
17
|
+
const wrap = (code, close) => (s) => enabled() ? `\u001b[${code}m${s}\u001b[${close}m` : s;
|
|
18
|
+
export const bold = wrap(1, 22);
|
|
19
|
+
export const dim = wrap(2, 22);
|
|
20
|
+
export const red = wrap(31, 39);
|
|
21
|
+
export const green = wrap(32, 39);
|
|
22
|
+
export const yellow = wrap(33, 39);
|
|
23
|
+
export const blue = wrap(34, 39);
|
|
24
|
+
export const cyan = wrap(36, 39);
|
|
25
|
+
/** Visible width, ignoring the escape sequences the styles above insert. */
|
|
26
|
+
export const visibleWidth = (s) =>
|
|
27
|
+
// eslint-disable-next-line no-control-regex
|
|
28
|
+
s.replace(/\u001b\[[0-9;]*m/g, "").length;
|
|
29
|
+
const pad = (s, width, align) => {
|
|
30
|
+
const gap = Math.max(0, width - visibleWidth(s));
|
|
31
|
+
if (align === "right")
|
|
32
|
+
return " ".repeat(gap) + s;
|
|
33
|
+
if (align === "center") {
|
|
34
|
+
const l = Math.floor(gap / 2);
|
|
35
|
+
return " ".repeat(l) + s + " ".repeat(gap - l);
|
|
36
|
+
}
|
|
37
|
+
return s + " ".repeat(gap);
|
|
38
|
+
};
|
|
39
|
+
/** Rounded-box table, the same shape `rich.box.ROUNDED` produced. */
|
|
40
|
+
export function renderTable(columns, rows, opts = {}) {
|
|
41
|
+
const widths = columns.map((c, i) => Math.max(visibleWidth(c.header), ...rows.map((r) => visibleWidth(r[i] ?? ""))));
|
|
42
|
+
const line = (l, m, r) => l + widths.map((w) => "─".repeat(w + 2)).join(m) + r;
|
|
43
|
+
const out = [];
|
|
44
|
+
if (opts.title)
|
|
45
|
+
out.push(bold(opts.title));
|
|
46
|
+
out.push(line("╭", "┬", "╮"));
|
|
47
|
+
out.push("│ " + columns.map((c, i) => pad(bold(c.header), widths[i], c.align ?? "left")).join(" │ ") + " │");
|
|
48
|
+
out.push(line("├", "┼", "┤"));
|
|
49
|
+
for (const row of rows) {
|
|
50
|
+
out.push("│ " + columns.map((c, i) => pad(row[i] ?? "", widths[i], c.align ?? "left")).join(" │ ") + " │");
|
|
51
|
+
}
|
|
52
|
+
out.push(line("╰", "┴", "╯"));
|
|
53
|
+
if (opts.caption)
|
|
54
|
+
out.push(dim(opts.caption));
|
|
55
|
+
return out.join("\n");
|
|
56
|
+
}
|
|
57
|
+
/** Bordered panel, the shape `rich.panel.Panel` produced. */
|
|
58
|
+
export function renderPanel(body, opts = {}) {
|
|
59
|
+
const lines = body.split("\n");
|
|
60
|
+
const width = Math.max(...lines.map(visibleWidth), opts.title ? visibleWidth(opts.title) + 2 : 0);
|
|
61
|
+
const top = opts.title
|
|
62
|
+
? "╭─ " + opts.title + " " + "─".repeat(Math.max(0, width - visibleWidth(opts.title) - 1)) + "╮"
|
|
63
|
+
: "╭" + "─".repeat(width + 2) + "╮";
|
|
64
|
+
const out = [top];
|
|
65
|
+
for (const l of lines)
|
|
66
|
+
out.push("│ " + pad(l, width, "left") + " │");
|
|
67
|
+
out.push("╰" + "─".repeat(width + 2) + "╯");
|
|
68
|
+
return out.join("\n");
|
|
69
|
+
}
|
|
70
|
+
export function formatBytes(n) {
|
|
71
|
+
let v = n;
|
|
72
|
+
for (const unit of ["B", "KB", "MB", "GB"]) {
|
|
73
|
+
if (v < 1024)
|
|
74
|
+
return `${v.toFixed(2)} ${unit}`;
|
|
75
|
+
v /= 1024;
|
|
76
|
+
}
|
|
77
|
+
return `${v.toFixed(2)} TB`;
|
|
78
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* fnmatch-compatible globbing.
|
|
3
|
+
*
|
|
4
|
+
* The Python CLI matched local experiment paths with `fnmatch.fnmatch` and let
|
|
5
|
+
* the server match remote ones. Both accept `*`, `?` and `[...]` classes, and
|
|
6
|
+
* — unlike shell globbing — `*` crosses `/`. A path-aware matcher would quietly
|
|
7
|
+
* change which experiments a pattern like `tom/<any>/exp<any>` selects, so this deliberately
|
|
8
|
+
* reproduces fnmatch's flat semantics.
|
|
9
|
+
*/
|
|
10
|
+
export function fnmatch(name, pattern) {
|
|
11
|
+
return fnmatchToRegExp(pattern).test(name);
|
|
12
|
+
}
|
|
13
|
+
export function fnmatchToRegExp(pattern) {
|
|
14
|
+
let out = "";
|
|
15
|
+
let i = 0;
|
|
16
|
+
while (i < pattern.length) {
|
|
17
|
+
const c = pattern[i++];
|
|
18
|
+
if (c === "*")
|
|
19
|
+
out += ".*";
|
|
20
|
+
else if (c === "?")
|
|
21
|
+
out += ".";
|
|
22
|
+
else if (c === "[") {
|
|
23
|
+
let j = i;
|
|
24
|
+
if (j < pattern.length && (pattern[j] === "!" || pattern[j] === "^"))
|
|
25
|
+
j++;
|
|
26
|
+
if (j < pattern.length && pattern[j] === "]")
|
|
27
|
+
j++;
|
|
28
|
+
while (j < pattern.length && pattern[j] !== "]")
|
|
29
|
+
j++;
|
|
30
|
+
if (j >= pattern.length) {
|
|
31
|
+
out += "\\[";
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
let body = pattern.slice(i, j);
|
|
35
|
+
if (body.startsWith("!") || body.startsWith("^"))
|
|
36
|
+
body = "^" + body.slice(1);
|
|
37
|
+
out += "[" + body.replace(/\\/g, "\\\\") + "]";
|
|
38
|
+
i = j + 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
out += c.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return new RegExp(`^${out}$`, "s");
|
|
46
|
+
}
|
|
47
|
+
export const hasWildcard = (s) => /[*?[]/.test(s);
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON parsing that does not destroy Snowflake IDs.
|
|
3
|
+
*
|
|
4
|
+
* ml-dash node, experiment, project and file IDs are 64-bit Snowflakes. The
|
|
5
|
+
* server sends most of them as JSON strings, but not all — `upload_file`'s
|
|
6
|
+
* response and some GraphQL paths hand back bare numbers. `JSON.parse` turns
|
|
7
|
+
* those into IEEE-754 doubles, and a 19-digit Snowflake loses its last two or
|
|
8
|
+
* three digits in the process. Nothing throws: the ID simply comes back
|
|
9
|
+
* slightly wrong, and the 404 surfaces later at an unrelated call.
|
|
10
|
+
*
|
|
11
|
+
* So numbers that cannot survive a round trip are quoted before parsing, and
|
|
12
|
+
* every ID stays a string end to end.
|
|
13
|
+
*/
|
|
14
|
+
/** Rewrite unsafe integer literals as strings, leaving strings and floats alone. */
|
|
15
|
+
export function preserveBigIntegers(text) {
|
|
16
|
+
let out = "";
|
|
17
|
+
let i = 0;
|
|
18
|
+
const n = text.length;
|
|
19
|
+
while (i < n) {
|
|
20
|
+
const c = text[i];
|
|
21
|
+
if (c === '"') {
|
|
22
|
+
// Copy the string literal verbatim — digits inside it are not numbers.
|
|
23
|
+
const start = i++;
|
|
24
|
+
while (i < n) {
|
|
25
|
+
if (text[i] === "\\")
|
|
26
|
+
i += 2;
|
|
27
|
+
else if (text[i] === '"') {
|
|
28
|
+
i++;
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
else
|
|
32
|
+
i++;
|
|
33
|
+
}
|
|
34
|
+
out += text.slice(start, i);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (c === "-" || (c >= "0" && c <= "9")) {
|
|
38
|
+
const start = i;
|
|
39
|
+
if (text[i] === "-")
|
|
40
|
+
i++;
|
|
41
|
+
while (i < n && text[i] >= "0" && text[i] <= "9")
|
|
42
|
+
i++;
|
|
43
|
+
const isInteger = !(i < n && (text[i] === "." || text[i] === "e" || text[i] === "E"));
|
|
44
|
+
while (i < n && /[0-9.eE+-]/.test(text[i]))
|
|
45
|
+
i++;
|
|
46
|
+
const literal = text.slice(start, i);
|
|
47
|
+
// Only integers, and only those outside the exactly-representable range.
|
|
48
|
+
out += isInteger && !Number.isSafeInteger(Number(literal)) ? `"${literal}"` : literal;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
out += c;
|
|
52
|
+
i++;
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
export function parseJson(text) {
|
|
57
|
+
return JSON.parse(preserveBigIntegers(text));
|
|
58
|
+
}
|
|
59
|
+
/** Coerce whatever the server sent for an ID into a string, losslessly. */
|
|
60
|
+
export function asId(v) {
|
|
61
|
+
if (v === null || v === undefined)
|
|
62
|
+
return undefined;
|
|
63
|
+
if (typeof v === "string")
|
|
64
|
+
return v;
|
|
65
|
+
if (typeof v === "number")
|
|
66
|
+
return String(v);
|
|
67
|
+
if (typeof v === "bigint")
|
|
68
|
+
return v.toString();
|
|
69
|
+
return String(v);
|
|
70
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A bounded-concurrency task runner.
|
|
3
|
+
*
|
|
4
|
+
* The Python CLI used `ThreadPoolExecutor(max_workers=N)` for metrics, chunks
|
|
5
|
+
* and files. Node has no threads to pool, but the limit still matters: it is
|
|
6
|
+
* what stops an experiment with 400 metrics from opening 400 sockets at once
|
|
7
|
+
* and having the server start refusing them. Results come back in submission
|
|
8
|
+
* order so a caller can pair them with its inputs; failures are returned, not
|
|
9
|
+
* thrown, because one bad metric must not abandon the rest of the batch.
|
|
10
|
+
*/
|
|
11
|
+
export async function runPool(items, limit, worker) {
|
|
12
|
+
const results = new Array(items.length);
|
|
13
|
+
const width = Math.max(1, Math.min(limit, items.length));
|
|
14
|
+
let cursor = 0;
|
|
15
|
+
const runners = Array.from({ length: width }, async () => {
|
|
16
|
+
for (;;) {
|
|
17
|
+
const index = cursor++;
|
|
18
|
+
if (index >= items.length)
|
|
19
|
+
return;
|
|
20
|
+
try {
|
|
21
|
+
results[index] = { value: await worker(items[index], index) };
|
|
22
|
+
}
|
|
23
|
+
catch (e) {
|
|
24
|
+
results[index] = { error: e instanceof Error ? e : new Error(String(e)) };
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
await Promise.all(runners);
|
|
29
|
+
return results;
|
|
30
|
+
}
|
package/dist/version.js
ADDED