@dreamlake/ml-dash 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +98 -0
- package/bin/ml-dash.js +5 -0
- package/dist/auth/device-flow.js +131 -0
- package/dist/auth/device-secret.js +20 -0
- package/dist/auth/fernet.js +98 -0
- package/dist/auth/jwt.js +12 -0
- package/dist/auth/token-storage.js +217 -0
- package/dist/cli/context.js +34 -0
- package/dist/cli/parser.js +153 -0
- package/dist/client.js +588 -0
- package/dist/commands/api.js +89 -0
- package/dist/commands/create.js +82 -0
- package/dist/commands/download.js +661 -0
- package/dist/commands/list.js +341 -0
- package/dist/commands/login.js +128 -0
- package/dist/commands/logout.js +22 -0
- package/dist/commands/profile.js +143 -0
- package/dist/commands/remove.js +123 -0
- package/dist/commands/upload.js +786 -0
- package/dist/commands/version.js +10 -0
- package/dist/config.js +64 -0
- package/dist/index.js +74 -0
- package/dist/local/safe-path.js +137 -0
- package/dist/local/storage.js +447 -0
- package/dist/util/ansi.js +78 -0
- package/dist/util/glob.js +47 -0
- package/dist/util/json.js +70 -0
- package/dist/util/pool.js +30 -0
- package/dist/version.js +3 -0
- package/package.json +49 -0
package/dist/config.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ~/.dash/config.json — the same file the Python CLI reads and writes.
|
|
3
|
+
*
|
|
4
|
+
* Keys: remote_url, api_key, default_batch_size, auth_url, device_secret.
|
|
5
|
+
* A corrupt file is treated as empty, matching the Python behaviour, so a
|
|
6
|
+
* half-written config never blocks `ml-dash login`.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { homedir } from "node:os";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
export const DEFAULT_API_URL = "https://api.dash.ml";
|
|
12
|
+
export class Config {
|
|
13
|
+
configDir;
|
|
14
|
+
configPath;
|
|
15
|
+
data;
|
|
16
|
+
constructor(configDir) {
|
|
17
|
+
this.configDir = configDir ?? process.env.ML_DASH_CONFIG_DIR ?? path.join(homedir(), ".dash");
|
|
18
|
+
this.configPath = path.join(this.configDir, "config.json");
|
|
19
|
+
this.data = this.load();
|
|
20
|
+
}
|
|
21
|
+
load() {
|
|
22
|
+
if (!existsSync(this.configPath))
|
|
23
|
+
return {};
|
|
24
|
+
try {
|
|
25
|
+
const parsed = JSON.parse(readFileSync(this.configPath, "utf8"));
|
|
26
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return {};
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
save() {
|
|
33
|
+
mkdirSync(this.configDir, { recursive: true });
|
|
34
|
+
writeFileSync(this.configPath, JSON.stringify(this.data, null, 2));
|
|
35
|
+
}
|
|
36
|
+
get(key, fallback) {
|
|
37
|
+
return this.data[key] ?? fallback;
|
|
38
|
+
}
|
|
39
|
+
set(key, value) {
|
|
40
|
+
this.data[key] = value;
|
|
41
|
+
this.save();
|
|
42
|
+
}
|
|
43
|
+
delete(key) {
|
|
44
|
+
if (key in this.data) {
|
|
45
|
+
delete this.data[key];
|
|
46
|
+
this.save();
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
get remoteUrl() {
|
|
50
|
+
return this.get("remote_url", DEFAULT_API_URL);
|
|
51
|
+
}
|
|
52
|
+
get apiKey() {
|
|
53
|
+
return this.get("api_key", undefined);
|
|
54
|
+
}
|
|
55
|
+
get batchSize() {
|
|
56
|
+
return this.get("default_batch_size", 100);
|
|
57
|
+
}
|
|
58
|
+
get authUrl() {
|
|
59
|
+
return this.get("auth_url", undefined);
|
|
60
|
+
}
|
|
61
|
+
get deviceSecret() {
|
|
62
|
+
return this.get("device_secret", undefined);
|
|
63
|
+
}
|
|
64
|
+
}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* ml-dash CLI entry point.
|
|
4
|
+
*
|
|
5
|
+
* Commands are dispatched by name and their modules are imported lazily, so
|
|
6
|
+
* `ml-dash version` does not pay for the GraphQL client or the QR encoder.
|
|
7
|
+
*/
|
|
8
|
+
import { ParseError, parseArgs, renderCommandHelp, renderRootHelp, usageError } from "./cli/parser.js";
|
|
9
|
+
import { red, yellow } from "./util/ansi.js";
|
|
10
|
+
const loaders = {
|
|
11
|
+
version: () => import("./commands/version.js"),
|
|
12
|
+
login: () => import("./commands/login.js"),
|
|
13
|
+
logout: () => import("./commands/logout.js"),
|
|
14
|
+
profile: () => import("./commands/profile.js"),
|
|
15
|
+
api: () => import("./commands/api.js"),
|
|
16
|
+
create: () => import("./commands/create.js"),
|
|
17
|
+
remove: () => import("./commands/remove.js"),
|
|
18
|
+
list: () => import("./commands/list.js"),
|
|
19
|
+
upload: () => import("./commands/upload.js"),
|
|
20
|
+
download: () => import("./commands/download.js"),
|
|
21
|
+
};
|
|
22
|
+
export async function main(argv) {
|
|
23
|
+
const [command, ...rest] = argv;
|
|
24
|
+
if (!command || command === "--help" || command === "-h" || command === "help") {
|
|
25
|
+
const specs = await Promise.all(Object.values(loaders).map(async (l) => (await l()).spec));
|
|
26
|
+
console.log(renderRootHelp(specs, Object.keys(loaders)));
|
|
27
|
+
return 0;
|
|
28
|
+
}
|
|
29
|
+
if (command === "--version" || command === "-V") {
|
|
30
|
+
return (await loaders.version()).run({});
|
|
31
|
+
}
|
|
32
|
+
const loader = loaders[command];
|
|
33
|
+
if (!loader) {
|
|
34
|
+
console.error(`${red("error:")} unknown command '${command}'`);
|
|
35
|
+
console.error(`\nAvailable commands: ${Object.keys(loaders).join(", ")}`);
|
|
36
|
+
return 2;
|
|
37
|
+
}
|
|
38
|
+
const mod = await loader();
|
|
39
|
+
let args;
|
|
40
|
+
try {
|
|
41
|
+
args = parseArgs(mod.spec, rest);
|
|
42
|
+
}
|
|
43
|
+
catch (e) {
|
|
44
|
+
if (e instanceof ParseError) {
|
|
45
|
+
console.error(usageError(command, e.message));
|
|
46
|
+
return 2;
|
|
47
|
+
}
|
|
48
|
+
throw e;
|
|
49
|
+
}
|
|
50
|
+
if (args.help) {
|
|
51
|
+
console.log(renderCommandHelp(mod.spec));
|
|
52
|
+
return 0;
|
|
53
|
+
}
|
|
54
|
+
return mod.run(args);
|
|
55
|
+
}
|
|
56
|
+
const isEntryPoint =
|
|
57
|
+
// `import.meta.main` is set by Bun (including a compiled binary); Node needs
|
|
58
|
+
// the argv comparison, and the npm shim imports this module rather than
|
|
59
|
+
// running it directly, so neither check alone covers both channels.
|
|
60
|
+
import.meta.main === true ||
|
|
61
|
+
process.argv[1] === undefined ||
|
|
62
|
+
/ml-dash(\.js)?$|index\.ts$|index\.js$/.test(process.argv[1]);
|
|
63
|
+
if (isEntryPoint) {
|
|
64
|
+
main(process.argv.slice(2))
|
|
65
|
+
.then((code) => {
|
|
66
|
+
process.exitCode = code;
|
|
67
|
+
})
|
|
68
|
+
.catch((e) => {
|
|
69
|
+
console.error(`${red("✗ Unexpected error:")} ${e instanceof Error ? e.message : String(e)}`);
|
|
70
|
+
if (process.env.ML_DASH_DEBUG === "1" && e instanceof Error)
|
|
71
|
+
console.error(yellow(e.stack ?? ""));
|
|
72
|
+
process.exitCode = 1;
|
|
73
|
+
});
|
|
74
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a write is allowed to land.
|
|
3
|
+
*
|
|
4
|
+
* `download` claims it only ever writes into the `.dash` tree the user named
|
|
5
|
+
* and into its own scratch directory. Every path component that decides where
|
|
6
|
+
* a write goes — an experiment prefix, a metric name, a file's `pPath` and its
|
|
7
|
+
* filename — is server-supplied metadata, so a hostile or broken server could
|
|
8
|
+
* otherwise put `..` (or `C:`, or a backslash) in one of them and walk the
|
|
9
|
+
* write out of the tree.
|
|
10
|
+
*
|
|
11
|
+
* Two properties this file is responsible for, both learned from a review of
|
|
12
|
+
* the first version:
|
|
13
|
+
*
|
|
14
|
+
* 1. **One root, and it is the `.dash` root the user chose.** A check
|
|
15
|
+
* anchored at a derived directory — `…/metrics`, `…/files` — trusts that
|
|
16
|
+
* directory to be inside the tree, which is exactly what is in question
|
|
17
|
+
* when `files` is itself a link to somewhere else. Every target is built
|
|
18
|
+
* and checked from `rootPath` down, in one pass, including the literal
|
|
19
|
+
* components (`files`, `data.jsonl`, `.files_metadata.json`), which can be
|
|
20
|
+
* links too.
|
|
21
|
+
* 2. **Every component, not just the ones that exist.** Resolving the deepest
|
|
22
|
+
* *existing* ancestor with realpath misses a dangling symlink — realpath
|
|
23
|
+
* fails on it, which reads as "not there yet" — and misses a link at the
|
|
24
|
+
* leaf. Write targets are walked component by component with `lstat`,
|
|
25
|
+
* which sees a link whether or not it points at anything.
|
|
26
|
+
*
|
|
27
|
+
* The rule is reject, not repair: a component that would leave the root raises
|
|
28
|
+
* `UnsafePathError` and fails its section, rather than being rewritten into
|
|
29
|
+
* some other path that quietly stores the data in the wrong place. Legal
|
|
30
|
+
* nesting — `owner/project/folder/exp`, `train/loss`, `checkpoints/epoch-3` —
|
|
31
|
+
* still passes.
|
|
32
|
+
*/
|
|
33
|
+
import { lstatSync } from "node:fs";
|
|
34
|
+
import path from "node:path";
|
|
35
|
+
export class UnsafePathError extends Error {
|
|
36
|
+
constructor(label, value, reason) {
|
|
37
|
+
super(`unsafe ${label} ${JSON.stringify(value)}: ${reason}`);
|
|
38
|
+
this.name = "UnsafePathError";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const WINDOWS_DRIVE = /^[A-Za-z]:/;
|
|
42
|
+
/** Rules every component shares, whatever its shape. */
|
|
43
|
+
function checkComponent(label, value) {
|
|
44
|
+
if (typeof value !== "string") {
|
|
45
|
+
throw new UnsafePathError(label, String(value), "is not a string");
|
|
46
|
+
}
|
|
47
|
+
if (value.includes("\0")) {
|
|
48
|
+
throw new UnsafePathError(label, value, "contains a NUL byte");
|
|
49
|
+
}
|
|
50
|
+
if (value.includes("\\")) {
|
|
51
|
+
// A backslash is a separator on Windows and a legal filename character on
|
|
52
|
+
// POSIX; accepting it would make the same metadata escape on one platform
|
|
53
|
+
// and not the other.
|
|
54
|
+
throw new UnsafePathError(label, value, "contains a backslash separator");
|
|
55
|
+
}
|
|
56
|
+
if (WINDOWS_DRIVE.test(value)) {
|
|
57
|
+
throw new UnsafePathError(label, value, "names a Windows drive");
|
|
58
|
+
}
|
|
59
|
+
if (value.startsWith("/") || path.isAbsolute(value)) {
|
|
60
|
+
throw new UnsafePathError(label, value, "is an absolute path");
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/** One path segment: a filename, or a directory name that may not nest. */
|
|
64
|
+
export function safeSegment(label, value) {
|
|
65
|
+
checkComponent(label, value);
|
|
66
|
+
if (value === "")
|
|
67
|
+
throw new UnsafePathError(label, value, "is empty");
|
|
68
|
+
if (value.includes("/")) {
|
|
69
|
+
throw new UnsafePathError(label, value, "must be a single path segment");
|
|
70
|
+
}
|
|
71
|
+
if (value === "." || value === "..") {
|
|
72
|
+
throw new UnsafePathError(label, value, "is a relative path component");
|
|
73
|
+
}
|
|
74
|
+
return value;
|
|
75
|
+
}
|
|
76
|
+
/** A relative path that may nest: `owner/project/exp`, `train/loss`. */
|
|
77
|
+
export function safeRelativeSegments(label, value) {
|
|
78
|
+
checkComponent(label, value);
|
|
79
|
+
const segments = value.split("/");
|
|
80
|
+
for (const segment of segments) {
|
|
81
|
+
if (segment === "." || segment === "..") {
|
|
82
|
+
throw new UnsafePathError(label, value, `contains a '${segment}' segment`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const kept = segments.filter((s) => s !== "");
|
|
86
|
+
if (kept.length === 0)
|
|
87
|
+
throw new UnsafePathError(label, value, "is empty");
|
|
88
|
+
return kept;
|
|
89
|
+
}
|
|
90
|
+
/** A literal component this code chose, not the server: `files`, `logs.jsonl`. */
|
|
91
|
+
export const literal = (value) => ({ label: "path component", value });
|
|
92
|
+
const contains = (root, target) => {
|
|
93
|
+
const rel = path.relative(root, target);
|
|
94
|
+
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Build a path under `root` from its parts, or throw.
|
|
98
|
+
*
|
|
99
|
+
* `mode: "write"` additionally refuses a target any part of which is a symlink
|
|
100
|
+
* — dangling or not — because a link inside the tree redirects the write out
|
|
101
|
+
* of it, and because following one to delete or overwrite would destroy a file
|
|
102
|
+
* that is not ours. `mode: "read"` keeps the lexical rules only, so a user who
|
|
103
|
+
* has symlinked their own metric data can still upload it.
|
|
104
|
+
*/
|
|
105
|
+
export function resolveUnderRoot(root, parts, mode = "write") {
|
|
106
|
+
const segments = [];
|
|
107
|
+
for (const part of parts) {
|
|
108
|
+
if (part.nested)
|
|
109
|
+
segments.push(...safeRelativeSegments(part.label, part.value));
|
|
110
|
+
else
|
|
111
|
+
segments.push(safeSegment(part.label, part.value));
|
|
112
|
+
}
|
|
113
|
+
const rootAbs = path.resolve(root);
|
|
114
|
+
const target = path.resolve(rootAbs, ...segments);
|
|
115
|
+
if (!contains(rootAbs, target)) {
|
|
116
|
+
throw new UnsafePathError(parts.map((p) => p.value).join("/"), target, "resolves outside the root");
|
|
117
|
+
}
|
|
118
|
+
if (mode === "write") {
|
|
119
|
+
let current = rootAbs;
|
|
120
|
+
for (const segment of segments) {
|
|
121
|
+
current = path.join(current, segment);
|
|
122
|
+
let stats;
|
|
123
|
+
try {
|
|
124
|
+
stats = lstatSync(current);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
// Does not exist: nothing here to redirect the write, and nothing
|
|
128
|
+
// below it can exist either.
|
|
129
|
+
break;
|
|
130
|
+
}
|
|
131
|
+
if (stats.isSymbolicLink()) {
|
|
132
|
+
throw new UnsafePathError(parts.map((p) => p.value).join("/"), current, "is reached through a symbolic link inside the root");
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return target;
|
|
137
|
+
}
|