@dreamlake/dreamlake-cli 0.105.1 → 0.107.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/index.d.ts +255 -14
- package/dist/api/index.js +3 -0
- package/dist/api/lakeshore-connections.d.ts +30 -0
- package/dist/api/lakeshore-connections.js +29 -0
- package/dist/api/native-experiments.d.ts +218 -7
- package/dist/api/native-experiments.js +154 -9
- package/dist/cli/artifact/index.js +19 -3
- package/dist/cli/artifact/siteConfig.d.ts +12 -0
- package/dist/cli/artifact/siteConfig.js +17 -4
- package/dist/cli/auth/commands.d.ts +11 -1
- package/dist/cli/auth/commands.js +34 -10
- package/dist/cli/experiments/native-commands.d.ts +3 -1
- package/dist/cli/experiments/native-commands.js +21 -14
- package/dist/cli/experiments/native-run-data.d.ts +51 -0
- package/dist/cli/experiments/native-run-data.js +826 -0
- package/dist/cli/help-examples.generated.js +44 -31
- package/dist/cli/index.js +6 -3
- package/dist/cli/lakeshore/connections.d.ts +27 -0
- package/dist/cli/lakeshore/connections.js +268 -0
- package/dist/cli/memos/move-command.js +2 -2
- package/dist/cli/version.d.ts +1 -1
- package/dist/cli/version.js +1 -1
- package/package.json +9 -9
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Transport } from "./client.js";
|
|
2
|
+
import { type Query } from "./common.js";
|
|
2
3
|
export interface NativeExperimentRun {
|
|
3
4
|
id: string;
|
|
4
5
|
name: string;
|
|
@@ -9,6 +10,91 @@ export interface NativeExperimentRun {
|
|
|
9
10
|
createdAt: string;
|
|
10
11
|
updatedAt: string;
|
|
11
12
|
}
|
|
13
|
+
export type NativeRunStatus = "RUNNING" | "COMPLETED" | "FAILED" | "CANCELLED";
|
|
14
|
+
export type NativeLogLevel = "info" | "warn" | "error" | "debug" | "fatal";
|
|
15
|
+
/** Human metadata of a file or metric. Absent keys are unchanged; `tags` replaces the set; `null` clears. */
|
|
16
|
+
export interface NativeItemMetadataPatch {
|
|
17
|
+
description?: string | null;
|
|
18
|
+
tags?: string[];
|
|
19
|
+
metadata?: Record<string, unknown> | null;
|
|
20
|
+
}
|
|
21
|
+
/** Filters of `GET runs/:id/logs`. `cursor` is the server's opaque `nextCursor`. */
|
|
22
|
+
export interface NativeLogQuery {
|
|
23
|
+
limit?: number;
|
|
24
|
+
cursor?: string;
|
|
25
|
+
level?: NativeLogLevel[];
|
|
26
|
+
startTime?: string;
|
|
27
|
+
endTime?: string;
|
|
28
|
+
search?: string;
|
|
29
|
+
orderBy?: "sequenceNumber" | "timestamp";
|
|
30
|
+
order?: "asc" | "desc";
|
|
31
|
+
}
|
|
32
|
+
/** `GET runs/:id/tracks/:topic/data` bounds (inclusive) and column selection. */
|
|
33
|
+
export interface NativeTrackQuery {
|
|
34
|
+
startTimestamp?: number;
|
|
35
|
+
endTimestamp?: number;
|
|
36
|
+
columns?: string[];
|
|
37
|
+
format?: string;
|
|
38
|
+
}
|
|
39
|
+
export interface NativeFileUpload {
|
|
40
|
+
/** File bytes. A Blob (e.g. `fs.openAsBlob`) is sent without buffering it whole. */
|
|
41
|
+
data: Blob | Uint8Array;
|
|
42
|
+
filename: string;
|
|
43
|
+
/** Lowercase hex SHA-256 of `data`; computed when omitted (reads `data` once). */
|
|
44
|
+
checksum?: string;
|
|
45
|
+
contentType?: string;
|
|
46
|
+
prefix?: string;
|
|
47
|
+
description?: string;
|
|
48
|
+
tags?: string[];
|
|
49
|
+
metadata?: Record<string, unknown>;
|
|
50
|
+
/** Project files: the folder (absent: the project root). */
|
|
51
|
+
folderId?: string;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Every route of one DreamLake project's native Experiments, below the central API origin.
|
|
55
|
+
* Run, file and metric segments are encoded, so a run path can never leave its project.
|
|
56
|
+
*/
|
|
57
|
+
/** The body of a run create-or-get: written with the run in one transaction. `writeProtected` is a person's request. */
|
|
58
|
+
export interface NativeRunCreate {
|
|
59
|
+
description?: string | null;
|
|
60
|
+
tags?: string[];
|
|
61
|
+
metadata?: Record<string, unknown> | null;
|
|
62
|
+
writeProtected?: boolean;
|
|
63
|
+
/** Bindr names (found or created at the top level), attached at create. */
|
|
64
|
+
bindrs?: string[];
|
|
65
|
+
}
|
|
66
|
+
/** A run item a bindr can hold: its metric, file or parameters, by the item's own id. */
|
|
67
|
+
export interface NativeRunItem {
|
|
68
|
+
kind: "metric" | "file" | "parameters";
|
|
69
|
+
id: string;
|
|
70
|
+
}
|
|
71
|
+
export type NativeBindrRef = ({
|
|
72
|
+
bindrId: string;
|
|
73
|
+
} | {
|
|
74
|
+
name: string;
|
|
75
|
+
}) & {
|
|
76
|
+
item?: NativeRunItem;
|
|
77
|
+
};
|
|
78
|
+
export type NativeFolderOperation = "trash" | "restore" | "protect" | "unprotect" | "purge";
|
|
79
|
+
export declare function nativeExperimentPaths(namespace: string, project: string): {
|
|
80
|
+
base: string;
|
|
81
|
+
run: (id: string) => string;
|
|
82
|
+
parameters: (id: string) => string;
|
|
83
|
+
logs: (id: string) => string;
|
|
84
|
+
metrics: (id: string) => string;
|
|
85
|
+
metric: (id: string, name: string) => string;
|
|
86
|
+
metricStats: (id: string, name: string) => string;
|
|
87
|
+
files: (id: string) => string;
|
|
88
|
+
file: (id: string, fileId: string) => string;
|
|
89
|
+
fileDownload: (id: string, fileId: string) => string;
|
|
90
|
+
};
|
|
91
|
+
/** Query parameters of a log read, as the server names them (`level` is a comma list). */
|
|
92
|
+
export declare function nativeLogQuery(query?: NativeLogQuery): Query;
|
|
93
|
+
/**
|
|
94
|
+
* The multipart body of a file upload. Text fields precede the file part: the server reads
|
|
95
|
+
* only the fields received before the file, and verifies `sizeBytes` and `checksum` against it.
|
|
96
|
+
*/
|
|
97
|
+
export declare function nativeFileForm(upload: NativeFileUpload): Promise<FormData>;
|
|
12
98
|
export declare function nativeExperimentsApi(t: Transport, namespace: string, project: string): {
|
|
13
99
|
status: () => Promise<unknown>;
|
|
14
100
|
session: (mode: "read" | "write") => Promise<{
|
|
@@ -17,13 +103,56 @@ export declare function nativeExperimentsApi(t: Transport, namespace: string, pr
|
|
|
17
103
|
apiUrl: string;
|
|
18
104
|
provider: string;
|
|
19
105
|
}>;
|
|
20
|
-
create: (key: string) => Promise<{
|
|
106
|
+
create: (key: string, body?: NativeRunCreate) => Promise<{
|
|
21
107
|
experiment: NativeExperimentRun;
|
|
22
108
|
}>;
|
|
109
|
+
/** ml-dash's `Experiment(prefix="ns/project/a/b/name")`: create-or-get the run named last in folder `a/b`. */
|
|
110
|
+
createAt: (segments: string[], body?: NativeRunCreate) => Promise<{
|
|
111
|
+
experiment: NativeExperimentRun;
|
|
112
|
+
}>;
|
|
113
|
+
/** Purge a trashed run for good (ADMIN): 202, its data deleted in the background. */
|
|
114
|
+
purge: (id: string) => Promise<{
|
|
115
|
+
id: string;
|
|
116
|
+
deleted: boolean;
|
|
117
|
+
purged: boolean;
|
|
118
|
+
}>;
|
|
119
|
+
folders: {
|
|
120
|
+
/** `{ path }` (mkdir -p) or `{ parentId?, name }`: the folder, created if missing. */
|
|
121
|
+
create: (body: {
|
|
122
|
+
path: string;
|
|
123
|
+
} | {
|
|
124
|
+
parentId?: string | null;
|
|
125
|
+
name: string;
|
|
126
|
+
}) => Promise<unknown>;
|
|
127
|
+
list: (query?: {
|
|
128
|
+
parentId?: string;
|
|
129
|
+
limit?: number;
|
|
130
|
+
cursor?: string;
|
|
131
|
+
}) => Promise<unknown>;
|
|
132
|
+
get: (id: string) => Promise<unknown>;
|
|
133
|
+
/** Rename and/or move (`parentId` null: the project root). */
|
|
134
|
+
update: (id: string, patch: {
|
|
135
|
+
name?: string;
|
|
136
|
+
parentId?: string | null;
|
|
137
|
+
}) => Promise<unknown>;
|
|
138
|
+
byPath: (segments: string[]) => Promise<unknown>;
|
|
139
|
+
/** An operation on every run below: 202 `{ operationId, state }`, applied by the server's worker. */
|
|
140
|
+
operate: (id: string, kind: NativeFolderOperation) => Promise<{
|
|
141
|
+
operationId: string;
|
|
142
|
+
state: string;
|
|
143
|
+
}>;
|
|
144
|
+
operation: (operationId: string) => Promise<unknown>;
|
|
145
|
+
};
|
|
146
|
+
/** ml-dash's batch delete: up to 100 runs and folders, each on its own, one receipt per item. */
|
|
147
|
+
batch: (action: "trash" | "purge", items: Array<{
|
|
148
|
+
type: "run" | "folder";
|
|
149
|
+
id: string;
|
|
150
|
+
}>) => Promise<unknown>;
|
|
23
151
|
list: (query?: {
|
|
24
152
|
limit?: number;
|
|
25
153
|
cursor?: string;
|
|
26
154
|
includeArchived?: boolean;
|
|
155
|
+
folderId?: string;
|
|
27
156
|
}) => Promise<{
|
|
28
157
|
runs: NativeExperimentRun[];
|
|
29
158
|
hasMore: boolean;
|
|
@@ -49,15 +178,27 @@ export declare function nativeExperimentsApi(t: Transport, namespace: string, pr
|
|
|
49
178
|
archive: (id: string, archived: boolean) => Promise<{
|
|
50
179
|
experiment: NativeExperimentRun;
|
|
51
180
|
}>;
|
|
181
|
+
/**
|
|
182
|
+
* Run metadata: `description` (null clears), `tags` (replaces the set), ml-dash's `metadata` dict; a person's
|
|
183
|
+
* rename/move (`name`, `folderPath` "" for the root) and `writeProtected` (set or clear).
|
|
184
|
+
*/
|
|
185
|
+
update: (id: string, patch: {
|
|
186
|
+
description?: string | null;
|
|
187
|
+
tags?: string[];
|
|
188
|
+
metadata?: Record<string, unknown> | null;
|
|
189
|
+
name?: string;
|
|
190
|
+
folderPath?: string;
|
|
191
|
+
writeProtected?: boolean;
|
|
192
|
+
}) => Promise<{
|
|
193
|
+
experiment: NativeExperimentRun;
|
|
194
|
+
}>;
|
|
52
195
|
parameters: {
|
|
53
196
|
get: (id: string) => Promise<unknown>;
|
|
54
197
|
set: (id: string, data: Record<string, unknown>) => Promise<unknown>;
|
|
198
|
+
delete: (id: string) => Promise<unknown>;
|
|
55
199
|
};
|
|
56
200
|
logs: {
|
|
57
|
-
list: (id: string, query?:
|
|
58
|
-
limit?: number;
|
|
59
|
-
cursor?: string;
|
|
60
|
-
}) => Promise<unknown>;
|
|
201
|
+
list: (id: string, query?: NativeLogQuery) => Promise<unknown>;
|
|
61
202
|
append: (id: string, entries: Array<{
|
|
62
203
|
timestamp: string;
|
|
63
204
|
level: string;
|
|
@@ -71,7 +212,77 @@ export declare function nativeExperimentsApi(t: Transport, namespace: string, pr
|
|
|
71
212
|
limit?: number;
|
|
72
213
|
cursor?: string;
|
|
73
214
|
}) => Promise<unknown>;
|
|
74
|
-
|
|
215
|
+
/** `descriptor` is stored with the points, atomically, only when this append creates the metric. */
|
|
216
|
+
append: (id: string, name: string, dataPoints: Record<string, unknown>[], descriptor?: NativeItemMetadataPatch) => Promise<unknown>;
|
|
217
|
+
stats: (id: string, name: string) => Promise<unknown>;
|
|
218
|
+
updateMetadata: (id: string, name: string, patch: NativeItemMetadataPatch) => Promise<unknown>;
|
|
219
|
+
};
|
|
220
|
+
files: {
|
|
221
|
+
list: (id: string, query?: {
|
|
222
|
+
prefix?: string;
|
|
223
|
+
tags?: string[];
|
|
224
|
+
}) => Promise<unknown>;
|
|
225
|
+
get: (id: string, fileId: string) => Promise<unknown>;
|
|
226
|
+
upload: (id: string, upload: NativeFileUpload) => Promise<unknown>;
|
|
227
|
+
/** The file's bytes from the central API; redirects are refused, so the bearer never leaves it. */
|
|
228
|
+
download: (id: string, fileId: string, options?: {
|
|
229
|
+
signal?: AbortSignal;
|
|
230
|
+
}) => Promise<Response>;
|
|
231
|
+
update: (id: string, fileId: string, patch: NativeItemMetadataPatch) => Promise<unknown>;
|
|
232
|
+
delete: (id: string, fileId: string) => Promise<unknown>;
|
|
233
|
+
/** A signed read URL (images and video only, short-lived). */
|
|
234
|
+
media: (id: string, fileId: string) => Promise<unknown>;
|
|
235
|
+
};
|
|
236
|
+
/** Project and folder files (README.md, `.dashrc`): not a run's. `folderId` absent or "root": the project root. */
|
|
237
|
+
projectFiles: {
|
|
238
|
+
list: (query?: {
|
|
239
|
+
folderId?: string;
|
|
240
|
+
}) => Promise<unknown>;
|
|
241
|
+
upload: (upload: NativeFileUpload) => Promise<unknown>;
|
|
242
|
+
get: (fileId: string) => Promise<unknown>;
|
|
243
|
+
/** UTF-8 text, at most 512 KiB (larger: 413, never truncated). */
|
|
244
|
+
text: (fileId: string) => Promise<unknown>;
|
|
245
|
+
download: (fileId: string, options?: {
|
|
246
|
+
signal?: AbortSignal;
|
|
247
|
+
}) => Promise<Response>;
|
|
248
|
+
media: (fileId: string) => Promise<unknown>;
|
|
249
|
+
/** New bytes for the same file: its id, name and metadata stay. */
|
|
250
|
+
replace: (fileId: string, upload: NativeFileUpload) => Promise<unknown>;
|
|
251
|
+
/** description/tags/metadata, and/or `filename` and `folderId` (null: the root) to rename or move. */
|
|
252
|
+
update: (fileId: string, patch: NativeItemMetadataPatch & {
|
|
253
|
+
filename?: string;
|
|
254
|
+
folderId?: string | null;
|
|
255
|
+
}) => Promise<unknown>;
|
|
256
|
+
delete: (fileId: string) => Promise<unknown>;
|
|
257
|
+
};
|
|
258
|
+
/** The run's dashboard (its `.dashrc`, else ml-dash's default), read-only: one JSON section, or charts of one type. */
|
|
259
|
+
dashboard: {
|
|
260
|
+
section: (id: string, field: "header" | "metrics" | "charts" | "metadata") => Promise<unknown>;
|
|
261
|
+
charts: (id: string, type: "image" | "line" | "video") => Promise<unknown>;
|
|
262
|
+
};
|
|
263
|
+
/** The bindrs a run, or one item of it, is in; attach (by id, or by name: found or created) and detach. */
|
|
264
|
+
bindrs: {
|
|
265
|
+
list: (id: string, item?: NativeRunItem) => Promise<unknown>;
|
|
266
|
+
attach: (id: string, ref: NativeBindrRef) => Promise<unknown>;
|
|
267
|
+
detach: (id: string, ref: NativeBindrRef) => Promise<unknown>;
|
|
268
|
+
};
|
|
269
|
+
/** Tracks of a Dash-homed run; `topic` may contain `/` (encoded as one path segment). */
|
|
270
|
+
tracks: {
|
|
271
|
+
list: (id: string, query?: {
|
|
272
|
+
topic?: string;
|
|
273
|
+
}) => Promise<unknown>;
|
|
274
|
+
create: (id: string, topic: string, descriptor?: NativeItemMetadataPatch) => Promise<unknown>;
|
|
275
|
+
/** Entries `{ timestamp, ...data }`, at most 10000, appended in one commit. */
|
|
276
|
+
append: (id: string, topic: string, entries: Array<Record<string, unknown> & {
|
|
277
|
+
timestamp: number;
|
|
278
|
+
}>) => Promise<unknown>;
|
|
279
|
+
/** Track data: `json` returns `{ entries, count }`; use `export` for jsonl, parquet (Arrow IPC) or mcap bytes. */
|
|
280
|
+
read: (id: string, topic: string, query?: NativeTrackQuery) => Promise<unknown>;
|
|
281
|
+
export: (id: string, topic: string, query: NativeTrackQuery & {
|
|
282
|
+
format: "json" | "jsonl" | "parquet" | "mcap";
|
|
283
|
+
}, options?: {
|
|
284
|
+
signal?: AbortSignal;
|
|
285
|
+
}) => Promise<Response>;
|
|
75
286
|
};
|
|
76
|
-
setStatus: (id: string, status:
|
|
287
|
+
setStatus: (id: string, status: NativeRunStatus) => Promise<unknown>;
|
|
77
288
|
};
|
|
@@ -1,30 +1,175 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { seg } from "./client.js";
|
|
2
3
|
import { call } from "./common.js";
|
|
3
|
-
|
|
4
|
+
function trackQuery(q) {
|
|
5
|
+
return { start_timestamp: q.startTimestamp, end_timestamp: q.endTimestamp, columns: q.columns?.join(","), format: q.format };
|
|
6
|
+
}
|
|
7
|
+
export function nativeExperimentPaths(namespace, project) {
|
|
4
8
|
const base = `/native-experiments/ns/${seg(namespace)}/projects/${seg(project)}`;
|
|
5
9
|
const run = (id) => `${base}/runs/${seg(id)}`;
|
|
10
|
+
return {
|
|
11
|
+
base,
|
|
12
|
+
run,
|
|
13
|
+
parameters: (id) => `${run(id)}/parameters`,
|
|
14
|
+
logs: (id) => `${run(id)}/logs`,
|
|
15
|
+
metrics: (id) => `${run(id)}/metrics`,
|
|
16
|
+
metric: (id, name) => `${run(id)}/metrics/${seg(name)}`,
|
|
17
|
+
metricStats: (id, name) => `${run(id)}/metrics/${seg(name)}/stats`,
|
|
18
|
+
files: (id) => `${run(id)}/files`,
|
|
19
|
+
file: (id, fileId) => `${run(id)}/files/${seg(fileId)}`,
|
|
20
|
+
fileDownload: (id, fileId) => `${run(id)}/files/${seg(fileId)}/download`,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/** Query parameters of a log read, as the server names them (`level` is a comma list). */
|
|
24
|
+
export function nativeLogQuery(query = {}) {
|
|
25
|
+
const { level, ...rest } = query;
|
|
26
|
+
return { ...rest, ...(level ? { level: level.join(",") } : {}) };
|
|
27
|
+
}
|
|
28
|
+
async function sha256(data) {
|
|
29
|
+
const hash = createHash("sha256");
|
|
30
|
+
if (data instanceof Uint8Array)
|
|
31
|
+
return hash.update(data).digest("hex");
|
|
32
|
+
for await (const chunk of data.stream())
|
|
33
|
+
hash.update(chunk);
|
|
34
|
+
return hash.digest("hex");
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The multipart body of a file upload. Text fields precede the file part: the server reads
|
|
38
|
+
* only the fields received before the file, and verifies `sizeBytes` and `checksum` against it.
|
|
39
|
+
*/
|
|
40
|
+
export async function nativeFileForm(upload) {
|
|
41
|
+
const blob = upload.data instanceof Uint8Array
|
|
42
|
+
? new Blob([upload.data], { type: upload.contentType ?? "application/octet-stream" })
|
|
43
|
+
: upload.data;
|
|
44
|
+
const form = new FormData();
|
|
45
|
+
form.append("sizeBytes", String(blob.size));
|
|
46
|
+
form.append("checksum", upload.checksum ?? await sha256(upload.data));
|
|
47
|
+
if (upload.prefix !== undefined)
|
|
48
|
+
form.append("prefix", upload.prefix);
|
|
49
|
+
if (upload.description !== undefined)
|
|
50
|
+
form.append("description", upload.description);
|
|
51
|
+
if (upload.tags?.length)
|
|
52
|
+
form.append("tags", upload.tags.join(","));
|
|
53
|
+
if (upload.metadata !== undefined)
|
|
54
|
+
form.append("metadata", JSON.stringify(upload.metadata));
|
|
55
|
+
if (upload.folderId !== undefined)
|
|
56
|
+
form.append("folderId", upload.folderId);
|
|
57
|
+
const typed = upload.contentType && blob.type !== upload.contentType ? new Blob([blob], { type: upload.contentType }) : blob;
|
|
58
|
+
form.append("file", typed, upload.filename);
|
|
59
|
+
return form;
|
|
60
|
+
}
|
|
61
|
+
export function nativeExperimentsApi(t, namespace, project) {
|
|
62
|
+
const p = nativeExperimentPaths(namespace, project);
|
|
63
|
+
const { base, run } = p;
|
|
6
64
|
return {
|
|
7
65
|
status: () => call(t, "GET", `${base}/status`),
|
|
8
66
|
session: (mode) => call(t, "POST", `${base}/session`, { mode }),
|
|
9
|
-
create: (key) => call(t, "PUT", run(key)),
|
|
67
|
+
create: (key, body) => call(t, "PUT", run(key), body),
|
|
68
|
+
/** ml-dash's `Experiment(prefix="ns/project/a/b/name")`: create-or-get the run named last in folder `a/b`. */
|
|
69
|
+
createAt: (segments, body) => call(t, "PUT", `${base}/run-paths/${segments.map(seg).join("/")}`, body),
|
|
70
|
+
/** Purge a trashed run for good (ADMIN): 202, its data deleted in the background. */
|
|
71
|
+
purge: (id) => call(t, "POST", `${run(id)}/purge`),
|
|
72
|
+
folders: {
|
|
73
|
+
/** `{ path }` (mkdir -p) or `{ parentId?, name }`: the folder, created if missing. */
|
|
74
|
+
create: (body) => call(t, "POST", `${base}/folders`, body),
|
|
75
|
+
list: (query = {}) => call(t, "GET", `${base}/folders`, undefined, query),
|
|
76
|
+
get: (id) => call(t, "GET", `${base}/folders/${seg(id)}`),
|
|
77
|
+
/** Rename and/or move (`parentId` null: the project root). */
|
|
78
|
+
update: (id, patch) => call(t, "PATCH", `${base}/folders/${seg(id)}`, patch),
|
|
79
|
+
byPath: (segments) => call(t, "GET", `${base}/by-path/${segments.map(seg).join("/")}`),
|
|
80
|
+
/** An operation on every run below: 202 `{ operationId, state }`, applied by the server's worker. */
|
|
81
|
+
operate: (id, kind) => call(t, "POST", `${base}/folders/${seg(id)}/${kind}`),
|
|
82
|
+
operation: (operationId) => call(t, "GET", `${base}/folder-operations/${seg(operationId)}`),
|
|
83
|
+
},
|
|
84
|
+
/** ml-dash's batch delete: up to 100 runs and folders, each on its own, one receipt per item. */
|
|
85
|
+
batch: (action, items) => call(t, "POST", `${base}/batch/${action}`, { items }),
|
|
10
86
|
list: (query = {}) => call(t, "GET", `${base}/runs`, undefined, query),
|
|
11
87
|
get: (id) => call(t, "GET", run(id)),
|
|
12
88
|
trash: (query = {}) => call(t, "GET", `${base}/trash`, undefined, query),
|
|
13
89
|
delete: (id) => call(t, "DELETE", run(id)),
|
|
14
90
|
restoreDeleted: (id) => call(t, "POST", `${run(id)}/restore`),
|
|
15
91
|
archive: (id, archived) => call(t, "PATCH", run(id), { archived }),
|
|
92
|
+
/**
|
|
93
|
+
* Run metadata: `description` (null clears), `tags` (replaces the set), ml-dash's `metadata` dict; a person's
|
|
94
|
+
* rename/move (`name`, `folderPath` "" for the root) and `writeProtected` (set or clear).
|
|
95
|
+
*/
|
|
96
|
+
update: (id, patch) => call(t, "PATCH", run(id), patch),
|
|
16
97
|
parameters: {
|
|
17
|
-
get: (id) => call(t, "GET",
|
|
18
|
-
set: (id, data) => call(t, "POST",
|
|
98
|
+
get: (id) => call(t, "GET", p.parameters(id)),
|
|
99
|
+
set: (id, data) => call(t, "POST", p.parameters(id), { data }),
|
|
100
|
+
delete: (id) => call(t, "DELETE", p.parameters(id)),
|
|
19
101
|
},
|
|
20
102
|
logs: {
|
|
21
|
-
list: (id, query = {}) => call(t, "GET",
|
|
22
|
-
append: (id, entries) => call(t, "POST",
|
|
103
|
+
list: (id, query = {}) => call(t, "GET", p.logs(id), undefined, nativeLogQuery(query)),
|
|
104
|
+
append: (id, entries) => call(t, "POST", p.logs(id), { entries }),
|
|
23
105
|
},
|
|
24
106
|
metrics: {
|
|
25
|
-
list: (id) => call(t, "GET",
|
|
26
|
-
rows: (id, name, query = {}) => call(t, "GET",
|
|
27
|
-
|
|
107
|
+
list: (id) => call(t, "GET", p.metrics(id)),
|
|
108
|
+
rows: (id, name, query = {}) => call(t, "GET", p.metric(id, name), undefined, query),
|
|
109
|
+
/** `descriptor` is stored with the points, atomically, only when this append creates the metric. */
|
|
110
|
+
append: (id, name, dataPoints, descriptor = {}) => call(t, "POST", p.metric(id, name), { dataPoints, ...descriptor }),
|
|
111
|
+
stats: (id, name) => call(t, "GET", p.metricStats(id, name)),
|
|
112
|
+
updateMetadata: (id, name, patch) => call(t, "PATCH", p.metric(id, name), patch),
|
|
113
|
+
},
|
|
114
|
+
files: {
|
|
115
|
+
list: (id, query = {}) => call(t, "GET", p.files(id), undefined, { prefix: query.prefix, tags: query.tags?.join(",") }),
|
|
116
|
+
get: (id, fileId) => call(t, "GET", p.file(id, fileId)),
|
|
117
|
+
upload: async (id, upload) => t.request("POST", p.files(id), { body: await nativeFileForm(upload) }),
|
|
118
|
+
/** The file's bytes from the central API; redirects are refused, so the bearer never leaves it. */
|
|
119
|
+
download: (id, fileId, options = {}) => {
|
|
120
|
+
if (!t.raw)
|
|
121
|
+
throw new Error("This transport cannot stream file downloads.");
|
|
122
|
+
return t.raw("GET", p.fileDownload(id, fileId), { redirect: "error", ...(options.signal ? { signal: options.signal, timeoutMs: null } : {}) });
|
|
123
|
+
},
|
|
124
|
+
update: (id, fileId, patch) => call(t, "PATCH", p.file(id, fileId), patch),
|
|
125
|
+
delete: (id, fileId) => call(t, "DELETE", p.file(id, fileId)),
|
|
126
|
+
/** A signed read URL (images and video only, short-lived). */
|
|
127
|
+
media: (id, fileId) => call(t, "GET", `${p.file(id, fileId)}/media`),
|
|
128
|
+
},
|
|
129
|
+
/** Project and folder files (README.md, `.dashrc`): not a run's. `folderId` absent or "root": the project root. */
|
|
130
|
+
projectFiles: {
|
|
131
|
+
list: (query = {}) => call(t, "GET", `${base}/project-files`, undefined, query),
|
|
132
|
+
upload: async (upload) => t.request("POST", `${base}/project-files`, { body: await nativeFileForm(upload) }),
|
|
133
|
+
get: (fileId) => call(t, "GET", `${base}/project-files/${seg(fileId)}`),
|
|
134
|
+
/** UTF-8 text, at most 512 KiB (larger: 413, never truncated). */
|
|
135
|
+
text: (fileId) => call(t, "GET", `${base}/project-files/${seg(fileId)}/text`),
|
|
136
|
+
download: (fileId, options = {}) => {
|
|
137
|
+
if (!t.raw)
|
|
138
|
+
throw new Error("This transport cannot stream file downloads.");
|
|
139
|
+
return t.raw("GET", `${base}/project-files/${seg(fileId)}/download`, { redirect: "error", ...(options.signal ? { signal: options.signal, timeoutMs: null } : {}) });
|
|
140
|
+
},
|
|
141
|
+
media: (fileId) => call(t, "GET", `${base}/project-files/${seg(fileId)}/media`),
|
|
142
|
+
/** New bytes for the same file: its id, name and metadata stay. */
|
|
143
|
+
replace: async (fileId, upload) => t.request("PUT", `${base}/project-files/${seg(fileId)}/content`, { body: await nativeFileForm(upload) }),
|
|
144
|
+
/** description/tags/metadata, and/or `filename` and `folderId` (null: the root) to rename or move. */
|
|
145
|
+
update: (fileId, patch) => call(t, "PATCH", `${base}/project-files/${seg(fileId)}`, patch),
|
|
146
|
+
delete: (fileId) => call(t, "DELETE", `${base}/project-files/${seg(fileId)}`),
|
|
147
|
+
},
|
|
148
|
+
/** The run's dashboard (its `.dashrc`, else ml-dash's default), read-only: one JSON section, or charts of one type. */
|
|
149
|
+
dashboard: {
|
|
150
|
+
section: (id, field) => call(t, "GET", `${run(id)}/dashboard/${field}`),
|
|
151
|
+
charts: (id, type) => call(t, "GET", `${run(id)}/dashboard/charts/${type}`),
|
|
152
|
+
},
|
|
153
|
+
/** The bindrs a run, or one item of it, is in; attach (by id, or by name: found or created) and detach. */
|
|
154
|
+
bindrs: {
|
|
155
|
+
list: (id, item) => call(t, "GET", `${run(id)}/bindrs`, undefined, (item ? { itemKind: item.kind, itemId: item.id } : {})),
|
|
156
|
+
attach: (id, ref) => call(t, "POST", `${run(id)}/bindrs`, ref),
|
|
157
|
+
detach: (id, ref) => call(t, "DELETE", `${run(id)}/bindrs`, ref),
|
|
158
|
+
},
|
|
159
|
+
/** Tracks of a Dash-homed run; `topic` may contain `/` (encoded as one path segment). */
|
|
160
|
+
tracks: {
|
|
161
|
+
list: (id, query = {}) => call(t, "GET", `${run(id)}/tracks`, undefined, query),
|
|
162
|
+
create: (id, topic, descriptor = {}) => call(t, "POST", `${run(id)}/tracks`, { topic, ...descriptor }),
|
|
163
|
+
/** Entries `{ timestamp, ...data }`, at most 10000, appended in one commit. */
|
|
164
|
+
append: (id, topic, entries) => call(t, "POST", `${run(id)}/tracks/${seg(topic)}/append_batch`, { entries }),
|
|
165
|
+
/** Track data: `json` returns `{ entries, count }`; use `export` for jsonl, parquet (Arrow IPC) or mcap bytes. */
|
|
166
|
+
read: (id, topic, query = {}) => call(t, "GET", `${run(id)}/tracks/${seg(topic)}/data`, undefined, trackQuery({ ...query, format: "json" })),
|
|
167
|
+
export: (id, topic, query, options = {}) => {
|
|
168
|
+
if (!t.raw)
|
|
169
|
+
throw new Error("This transport cannot stream track exports.");
|
|
170
|
+
return t.raw("GET", `${run(id)}/tracks/${seg(topic)}/data`, { query: trackQuery(query), redirect: "error",
|
|
171
|
+
...(options.signal ? { signal: options.signal, timeoutMs: null } : {}) });
|
|
172
|
+
},
|
|
28
173
|
},
|
|
29
174
|
setStatus: (id, status) => call(t, "POST", `${run(id)}/status`, { status }),
|
|
30
175
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { registerArtifactRenameCommand } from "./rename.js";
|
|
2
|
-
import {
|
|
2
|
+
import { readSiteConfiguration } from './siteConfig.js';
|
|
3
3
|
import { collectionList } from "../collection-list.js";
|
|
4
4
|
import { registerShareCommands } from "../share.js";
|
|
5
5
|
// `dreamlake artifact push|list|delete|restore`.
|
|
@@ -91,6 +91,7 @@ export async function runDreamlakeArtifactPush(args, deps = {}) {
|
|
|
91
91
|
let content;
|
|
92
92
|
let site;
|
|
93
93
|
let directory = false;
|
|
94
|
+
let siteConfig;
|
|
94
95
|
try {
|
|
95
96
|
const stat = statSync(args.file);
|
|
96
97
|
directory = stat.isDirectory();
|
|
@@ -110,7 +111,8 @@ export async function runDreamlakeArtifactPush(args, deps = {}) {
|
|
|
110
111
|
fail("incremental artifacts are limited to 512 MiB");
|
|
111
112
|
return 1;
|
|
112
113
|
}
|
|
113
|
-
|
|
114
|
+
siteConfig = args.site ? readSiteConfiguration(args.siteConfig) : undefined;
|
|
115
|
+
site = args.site ? collectSite(args.file, args.entry, args.spa, siteConfig.publicDataOrigins) : undefined;
|
|
114
116
|
content = site ? site.content : directory ? packPages(args.file, args.entry) : readFileSync(args.file);
|
|
115
117
|
}
|
|
116
118
|
catch (error) {
|
|
@@ -156,7 +158,7 @@ export async function runDreamlakeArtifactPush(args, deps = {}) {
|
|
|
156
158
|
const capability = await jsonRequest('GET', `${ctx.remote}/artifacts/site-capabilities`, token);
|
|
157
159
|
const caps = capability.body;
|
|
158
160
|
const needsPublicData = site.manifest.schema === 'dreamlake-artifact-files/2';
|
|
159
|
-
if (capability.status !== 200 || caps?.schema !== 'dreamlake-artifact-files/1' || (needsPublicData && (!caps.supportedSchemas?.includes('dreamlake-artifact-files/2') || caps.features?.publicDataOrigins !== true))) {
|
|
161
|
+
if (capability.status !== 200 || caps?.schema !== 'dreamlake-artifact-files/1' || (siteConfig?.privateData && caps.features?.privateSourceCapabilities !== true) || (needsPublicData && (!caps.supportedSchemas?.includes('dreamlake-artifact-files/2') || caps.features?.publicDataOrigins !== true))) {
|
|
160
162
|
fail('Static artifact hosting is unavailable on this server; no files were uploaded.');
|
|
161
163
|
return 1;
|
|
162
164
|
}
|
|
@@ -238,6 +240,20 @@ export async function runDreamlakeArtifactPush(args, deps = {}) {
|
|
|
238
240
|
if (grant.status !== 200 || binding?.manifest !== site.hash || JSON.stringify(binding.origins) !== JSON.stringify(site.manifest.network?.publicDataOrigins))
|
|
239
241
|
throw new Error('Public data authorization failed; no artifact version was published. Owner or Project ADMIN access is required.');
|
|
240
242
|
}
|
|
243
|
+
if (siteConfig?.privateData) {
|
|
244
|
+
const request = { manifest: site.hash, ...siteConfig.privateData };
|
|
245
|
+
const grant = await jsonRequest('POST', `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/artifacts/${encodeURIComponent(artifactId)}/site-private-data-grants`, token, request);
|
|
246
|
+
const binding = grant.body;
|
|
247
|
+
let originValid = false;
|
|
248
|
+
try {
|
|
249
|
+
const origin = new URL(binding?.origin ?? '');
|
|
250
|
+
originValid = origin.protocol === 'https:' && origin.origin === binding.origin && !origin.username && !origin.password;
|
|
251
|
+
}
|
|
252
|
+
catch { }
|
|
253
|
+
if (grant.status !== 200 || binding?.manifest !== site.hash || binding.alias !== request.alias || binding.sourceId !== request.sourceId || binding.prefix !== request.prefix || !originValid) {
|
|
254
|
+
throw new Error('Private data authorization failed; no artifact version was published. Artifact administrator and Source administrator access are required.');
|
|
255
|
+
}
|
|
256
|
+
}
|
|
241
257
|
process.stdout.write(` site: ${site.manifest.files.length} files, ${result.uploadedFiles} new files, ${result.uploadedBytes} bytes uploaded\n`);
|
|
242
258
|
}
|
|
243
259
|
else if (incremental) {
|
|
@@ -1 +1,13 @@
|
|
|
1
|
+
export interface PrivateDataBinding {
|
|
2
|
+
alias: string;
|
|
3
|
+
sourceId: string;
|
|
4
|
+
namespaceId: string;
|
|
5
|
+
prefix: string;
|
|
6
|
+
}
|
|
7
|
+
export interface ArtifactSiteConfig {
|
|
8
|
+
publicDataOrigins: string[];
|
|
9
|
+
privateData?: PrivateDataBinding;
|
|
10
|
+
}
|
|
11
|
+
export declare function readSiteConfiguration(file?: string): ArtifactSiteConfig;
|
|
12
|
+
/** Backward-compatible public-origin reader for existing consumers. */
|
|
1
13
|
export declare function readSiteConfig(file?: string): string[];
|
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
import { existsSync, lstatSync, readFileSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { validatePublicDataOrigins } from './siteFormat.js';
|
|
4
|
-
export function
|
|
4
|
+
export function readSiteConfiguration(file) {
|
|
5
5
|
const target = file ?? path.resolve('artifact.config.json');
|
|
6
6
|
if (!file && !existsSync(target))
|
|
7
|
-
return [];
|
|
7
|
+
return { publicDataOrigins: [] };
|
|
8
8
|
const stat = lstatSync(target);
|
|
9
9
|
if (!stat.isFile() || stat.size > 16 * 1024)
|
|
10
10
|
throw new Error('Site configuration must be a regular file of at most 16 KiB');
|
|
11
11
|
const config = JSON.parse(readFileSync(target, 'utf8'));
|
|
12
|
-
if (config?.schema !== 'dreamlake-artifact-config/1' || Object.keys(config).some(key => !['schema', 'network'].includes(key)) ||
|
|
12
|
+
if (config?.schema !== 'dreamlake-artifact-config/1' || Object.keys(config).some(key => !['schema', 'network', 'privateData'].includes(key)) ||
|
|
13
|
+
(!config.network && !config.privateData) || (config.network && Object.keys(config.network).some(key => key !== 'publicDataOrigins')))
|
|
13
14
|
throw new Error('Invalid artifact site configuration');
|
|
14
|
-
|
|
15
|
+
const result = { publicDataOrigins: validatePublicDataOrigins(config.network?.publicDataOrigins ?? []) };
|
|
16
|
+
if (config.privateData !== undefined) {
|
|
17
|
+
const binding = config.privateData;
|
|
18
|
+
if (!binding || typeof binding !== 'object' || Object.keys(binding).sort().join(',') !== 'alias,namespaceId,prefix,sourceId' ||
|
|
19
|
+
!/^[a-z][a-z0-9-]{0,47}$/.test(binding.alias) || !/^[a-f0-9]{24}$/.test(binding.sourceId) || !/^[a-f0-9]{24}$/.test(binding.namespaceId) ||
|
|
20
|
+
typeof binding.prefix !== 'string' || !binding.prefix || binding.prefix.length > 2048 || /[\\%?#\x00-\x20\x7f]/.test(binding.prefix) || binding.prefix.startsWith('/') ||
|
|
21
|
+
binding.prefix.split('/').some((part) => !part || part === '.' || part === '..'))
|
|
22
|
+
throw new Error('Invalid private data binding');
|
|
23
|
+
result.privateData = { alias: binding.alias, sourceId: binding.sourceId, namespaceId: binding.namespaceId, prefix: binding.prefix };
|
|
24
|
+
}
|
|
25
|
+
return result;
|
|
15
26
|
}
|
|
27
|
+
/** Backward-compatible public-origin reader for existing consumers. */
|
|
28
|
+
export function readSiteConfig(file) { return readSiteConfiguration(file).publicDataOrigins; }
|
|
@@ -17,7 +17,17 @@ export declare const DEFAULT_LOGIN_ENV = "prod";
|
|
|
17
17
|
*/
|
|
18
18
|
export declare function resolveLoginEnvName(explicit?: string, filePath?: string): string;
|
|
19
19
|
export declare function runLogin(args: LoginArgs): Promise<number>;
|
|
20
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Shared-login logout (contract v1, docs: Environments → Shared login contract).
|
|
22
|
+
* Removes one saved environment (default: the active one). The projection
|
|
23
|
+
* (tokens.json + config.json) is replaced only when it still holds that
|
|
24
|
+
* environment's server and token: it then becomes the next saved environment,
|
|
25
|
+
* or is cleared. With no saved environment, a projection another client left
|
|
26
|
+
* behind is cleared, so a logout always stays logged out.
|
|
27
|
+
*/
|
|
28
|
+
export declare function runLogout(opts?: {
|
|
29
|
+
env?: string;
|
|
30
|
+
}): Promise<number>;
|
|
21
31
|
export declare function runEnvList(opts: {
|
|
22
32
|
json?: boolean;
|
|
23
33
|
}): Promise<number>;
|
|
@@ -16,7 +16,7 @@ import { spawn } from "node:child_process";
|
|
|
16
16
|
import { resolveRemote, resolveToken, decodeJwtPayload } from "../config.js";
|
|
17
17
|
import { emitJson, renderTable } from "../helpers.js";
|
|
18
18
|
import { authFilePath, builtinEnv, BUILTIN_ENVS, currentEnv, currentEnvName, listEnvs, removeEnv, saveEnv, useEnv, } from "./credentials.js";
|
|
19
|
-
import { clearProjection, dreamlakeDir, projectEnv } from "./materialized.js";
|
|
19
|
+
import { clearProjection, dreamlakeDir, projectEnv, readConfig, readToken } from "./materialized.js";
|
|
20
20
|
import { getOrCreateDeviceSecret } from "./device-secret.js";
|
|
21
21
|
import { AuthorizationDeniedError, DeviceCodeExpiredError, DeviceFlowClient, TokenExchangeError, } from "./device-flow.js";
|
|
22
22
|
async function fetchAuthMe(remote, token) {
|
|
@@ -178,21 +178,45 @@ export async function runLogin(args) {
|
|
|
178
178
|
return 0;
|
|
179
179
|
}
|
|
180
180
|
// ─── logout ──────────────────────────────────────────────────────────
|
|
181
|
-
|
|
182
|
-
|
|
181
|
+
/**
|
|
182
|
+
* Shared-login logout (contract v1, docs: Environments → Shared login contract).
|
|
183
|
+
* Removes one saved environment (default: the active one). The projection
|
|
184
|
+
* (tokens.json + config.json) is replaced only when it still holds that
|
|
185
|
+
* environment's server and token: it then becomes the next saved environment,
|
|
186
|
+
* or is cleared. With no saved environment, a projection another client left
|
|
187
|
+
* behind is cleared, so a logout always stays logged out.
|
|
188
|
+
*/
|
|
189
|
+
export async function runLogout(opts = {}) {
|
|
190
|
+
const name = opts.env ?? currentEnvName();
|
|
191
|
+
const projectedToken = readToken();
|
|
192
|
+
const projectedServer = readConfig().remote_url?.replace(/\/+$/, "");
|
|
183
193
|
if (!name) {
|
|
184
|
-
|
|
194
|
+
if (projectedToken) {
|
|
195
|
+
clearProjection();
|
|
196
|
+
process.stdout.write(`✓ Logged out of ${projectedServer ?? "the unsaved connection"} (it had no saved environment).\n`);
|
|
197
|
+
}
|
|
198
|
+
else
|
|
199
|
+
process.stdout.write(`No active environment (${authFilePath()})\n`);
|
|
185
200
|
return 0;
|
|
186
201
|
}
|
|
202
|
+
const removed = listEnvs().find(e => e.name === name)?.env;
|
|
203
|
+
if (!removed) {
|
|
204
|
+
process.stderr.write(`✗ No saved environment '${name}'. Saved: ${listEnvs().map(e => e.name).join(", ") || "(none)"}\n`);
|
|
205
|
+
return 1;
|
|
206
|
+
}
|
|
187
207
|
removeEnv(name);
|
|
188
|
-
const
|
|
189
|
-
if (
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
208
|
+
const wasProjected = projectedToken === removed.token && projectedServer === removed.server.replace(/\/+$/, "");
|
|
209
|
+
if (wasProjected) {
|
|
210
|
+
const next = currentEnv();
|
|
211
|
+
if (next)
|
|
212
|
+
projectEnv(next);
|
|
213
|
+
else
|
|
214
|
+
clearProjection();
|
|
215
|
+
}
|
|
193
216
|
const nextName = currentEnvName();
|
|
194
217
|
process.stdout.write(`✓ Logged out of '${name}'.` +
|
|
195
|
-
(
|
|
218
|
+
(!wasProjected ? " The active connection was a different environment and is unchanged.\n"
|
|
219
|
+
: nextName ? ` Active environment is now '${nextName}'.\n` : " No environments left.\n"));
|
|
196
220
|
return 0;
|
|
197
221
|
}
|
|
198
222
|
// ─── env (environment switching) ──────────────────────────────────────
|