dataiku-sdk 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -17
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +39 -4
- package/dist/packages/types/src/index.js +31 -2
- package/dist/src/cli/commands/api-service.js +62 -16
- package/dist/src/cli/commands/bundle.js +49 -15
- package/dist/src/cli/commands/code-env.js +210 -49
- package/dist/src/cli/commands/code.d.ts +19 -0
- package/dist/src/cli/commands/code.js +87 -15
- package/dist/src/cli/commands/dataset.js +8 -3
- package/dist/src/cli/commands/flow-zone.js +33 -5
- package/dist/src/cli/commands/notebook.js +72 -41
- package/dist/src/cli/commands/project-git.js +8 -5
- package/dist/src/cli/commands/project-library.d.ts +21 -0
- package/dist/src/cli/commands/project-library.js +121 -50
- package/dist/src/cli/commands/project.js +16 -2
- package/dist/src/cli/commands/recipe.js +37 -12
- package/dist/src/cli/commands/sql.d.ts +10 -0
- package/dist/src/cli/commands/sql.js +39 -17
- package/dist/src/cli/commands/webapp.js +67 -13
- package/dist/src/cli/contract.d.ts +15 -0
- package/dist/src/cli/contract.js +870 -107
- package/dist/src/cli/env.d.ts +9 -0
- package/dist/src/cli/env.js +20 -2
- package/dist/src/cli/flags.js +9 -0
- package/dist/src/cli/helpers/cleanup.js +104 -3
- package/dist/src/cli/helpers/flow-zone.d.ts +3 -0
- package/dist/src/cli/helpers/flow-zone.js +33 -3
- package/dist/src/cli/helpers/recipe.d.ts +20 -0
- package/dist/src/cli/helpers/recipe.js +158 -3
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +33 -6
- package/dist/src/cli/runtime.d.ts +35 -1
- package/dist/src/cli/runtime.js +125 -18
- package/dist/src/cli/types.d.ts +7 -0
- package/dist/src/cli.js +135 -28
- package/dist/src/client.d.ts +60 -0
- package/dist/src/client.js +237 -42
- package/dist/src/errors.d.ts +8 -2
- package/dist/src/errors.js +40 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +2 -1
- package/dist/src/resources/api-services.d.ts +48 -6
- package/dist/src/resources/api-services.js +54 -8
- package/dist/src/resources/bundles.d.ts +51 -6
- package/dist/src/resources/bundles.js +39 -9
- package/dist/src/resources/code-envs.d.ts +25 -3
- package/dist/src/resources/code-envs.js +104 -2
- package/dist/src/resources/datasets.d.ts +20 -0
- package/dist/src/resources/datasets.js +232 -52
- package/dist/src/resources/jobs.d.ts +35 -10
- package/dist/src/resources/jobs.js +92 -17
- package/dist/src/resources/notebooks.d.ts +77 -7
- package/dist/src/resources/notebooks.js +143 -19
- package/dist/src/resources/project-git.d.ts +12 -0
- package/dist/src/resources/project-git.js +39 -8
- package/dist/src/resources/project-library.d.ts +122 -7
- package/dist/src/resources/project-library.js +494 -25
- package/dist/src/resources/projects.d.ts +5 -2
- package/dist/src/resources/projects.js +18 -2
- package/dist/src/resources/scenarios.d.ts +30 -0
- package/dist/src/resources/scenarios.js +73 -27
- package/dist/src/resources/sql.d.ts +6 -0
- package/dist/src/resources/sql.js +22 -2
- package/dist/src/resources/webapps.d.ts +35 -4
- package/dist/src/resources/webapps.js +54 -5
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +9 -0
- package/dist/src/skill.js +37 -4
- package/dist/src/utils/flow-analysis.d.ts +53 -0
- package/dist/src/utils/flow-analysis.js +382 -0
- package/dist/src/utils/project-archive.d.ts +41 -2
- package/dist/src/utils/project-archive.js +159 -16
- package/package.json +4 -4
- package/packages/types/dist/index.d.ts +39 -4
- package/packages/types/dist/index.js +31 -2
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +74 -47
|
@@ -1,9 +1,16 @@
|
|
|
1
|
+
import { DataikuError, } from "../errors.js";
|
|
1
2
|
import { JobSummaryArraySchema, } from "../schemas.js";
|
|
2
3
|
import { BaseResource, } from "./base.js";
|
|
3
4
|
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
4
5
|
const MAX_POLL_INTERVAL_MS = 10_000;
|
|
5
6
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
6
7
|
const DEFAULT_MAX_LOG_LINES = 500;
|
|
8
|
+
/**
|
|
9
|
+
* Default bytes retained from a job log body (default 10 MiB). Logs larger
|
|
10
|
+
* than this are downloaded bounded: only the most recent `maxLogBytes` bytes
|
|
11
|
+
* are kept, so a collaborator-influenced log cannot exhaust client memory.
|
|
12
|
+
*/
|
|
13
|
+
const DEFAULT_MAX_LOG_BYTES = 10 * 1024 * 1024;
|
|
7
14
|
const TERMINAL_STATES = new Set([
|
|
8
15
|
"DONE",
|
|
9
16
|
"FAILED",
|
|
@@ -69,13 +76,23 @@ function jobBuildDefinition(targets, defaultProjectKey, opts) {
|
|
|
69
76
|
function jobLogLines(log) {
|
|
70
77
|
return log.split(/\r?\n/).map((line) => line.trimEnd());
|
|
71
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* DSS tags recipe subprocess output by reader thread: `[null-out-N]` carries
|
|
81
|
+
* the child's stdout and `[null-err-N]` its stderr. The legacy fallback is an
|
|
82
|
+
* explicit `stdout:` / `stderr:` record prefix (hand-written or non-DSS logs);
|
|
83
|
+
* a bare substring match would also select backend metadata such as the
|
|
84
|
+
* process wrapper's `pipes {"stdout": ...}` line and crowd out real output.
|
|
85
|
+
*/
|
|
86
|
+
const LEGACY_STDOUT_RECORD = /^\s*stdout:/i;
|
|
87
|
+
const LEGACY_STDERR_RECORD = /^\s*stderr:/i;
|
|
72
88
|
function lineMatchesLogFilter(line, filter) {
|
|
73
89
|
const normalized = line.toLowerCase();
|
|
74
90
|
switch (filter) {
|
|
75
91
|
case "stdout":
|
|
76
|
-
return normalized.includes("
|
|
92
|
+
return normalized.includes("[null-out-") || LEGACY_STDOUT_RECORD.test(line)
|
|
93
|
+
|| line.startsWith(">>> ");
|
|
77
94
|
case "stderr":
|
|
78
|
-
return normalized.includes("
|
|
95
|
+
return normalized.includes("[null-err-") || LEGACY_STDERR_RECORD.test(line);
|
|
79
96
|
case "errors":
|
|
80
97
|
return /\b(error|failed|failure|exception|traceback)\b/i.test(line);
|
|
81
98
|
case "user":
|
|
@@ -173,15 +190,22 @@ export class JobsResource extends BaseResource {
|
|
|
173
190
|
/**
|
|
174
191
|
* Retrieve job log text.
|
|
175
192
|
* Returns the last `maxLogLines` lines (default 500) from the tail.
|
|
176
|
-
* Use `0` or `-1` to return the
|
|
193
|
+
* Use `0` or `-1` to return the log without line truncation.
|
|
194
|
+
*
|
|
195
|
+
* The download is byte-bounded and deadline-covered: at most `maxLogBytes`
|
|
196
|
+
* (default 10 MiB) of the *most recent* log output is retained, so a
|
|
197
|
+
* collaborator-influenced oversized log cannot exhaust client memory while
|
|
198
|
+
* the line limit previously only applied after a full unbounded download.
|
|
199
|
+
* Logs up to the cap are returned in full.
|
|
177
200
|
*/
|
|
178
201
|
async log(jobId, opts) {
|
|
179
202
|
const jobEnc = encodeURIComponent(jobId);
|
|
180
203
|
const query = opts?.activity ? `?activity=${encodeURIComponent(opts.activity)}` : "";
|
|
181
204
|
// DSS cat-activity-log URLs require a browser session; API-key callers must use the public log endpoint.
|
|
182
205
|
const path = `/public/api/projects/${this.enc(opts?.projectKey)}/jobs/${jobEnc}/log/${query}`;
|
|
183
|
-
const
|
|
184
|
-
|
|
206
|
+
const maxLogBytes = Math.max(1, opts?.maxLogBytes ?? DEFAULT_MAX_LOG_BYTES);
|
|
207
|
+
const { text, } = await this.client.getTextTailLimited(path, maxLogBytes);
|
|
208
|
+
return limitJobLog(filterJobLog(text, opts?.logFilter), opts?.maxLogLines);
|
|
185
209
|
}
|
|
186
210
|
async logFromUrl(logUrl, opts) {
|
|
187
211
|
const parsed = new URL(logUrl, "http://dss.local");
|
|
@@ -191,7 +215,12 @@ export class JobsResource extends BaseResource {
|
|
|
191
215
|
if (!projectKey || !jobId || !activity) {
|
|
192
216
|
throw new Error("Log URL must include projectKey, jobId, and activityId query parameters.");
|
|
193
217
|
}
|
|
194
|
-
return this.log(jobId, {
|
|
218
|
+
return this.log(jobId, {
|
|
219
|
+
activity,
|
|
220
|
+
projectKey,
|
|
221
|
+
maxLogLines: opts?.maxLogLines,
|
|
222
|
+
maxLogBytes: opts?.maxLogBytes,
|
|
223
|
+
});
|
|
195
224
|
}
|
|
196
225
|
/**
|
|
197
226
|
* Start a build job for one or more dataset or managed-folder outputs.
|
|
@@ -250,6 +279,14 @@ export class JobsResource extends BaseResource {
|
|
|
250
279
|
* Adaptive polling doubles the interval every 3 polls when
|
|
251
280
|
* `pollIntervalMs` is not explicitly set.
|
|
252
281
|
*
|
|
282
|
+
* A terminal state observed from the status endpoint is authoritative:
|
|
283
|
+
* when log retrieval fails afterwards (e.g. `not_found` because logs were
|
|
284
|
+
* purged or the job was removed), `wait` still returns the terminal
|
|
285
|
+
* outcome instead of throwing, with `logUnavailable` holding the machine-
|
|
286
|
+
* readable reason and `removed` set once the job itself is probed and
|
|
287
|
+
* confirmed gone. Explicit `includeLogs`/`summary` requests stay honest:
|
|
288
|
+
* `log`/`logSummary` are only present when logs were actually retrieved.
|
|
289
|
+
*
|
|
253
290
|
* On timeout, returns `{ success: false, ... }` rather than throwing.
|
|
254
291
|
*/
|
|
255
292
|
async wait(jobId, opts) {
|
|
@@ -272,18 +309,38 @@ export class JobsResource extends BaseResource {
|
|
|
272
309
|
const success = isSuccessfulTerminalState(state);
|
|
273
310
|
let log;
|
|
274
311
|
let logSummary;
|
|
312
|
+
let logUnavailable;
|
|
313
|
+
let removed;
|
|
275
314
|
if (opts?.includeLogs || opts?.summary) {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
315
|
+
try {
|
|
316
|
+
// Fetch unlimited lines (still byte-bounded by log()) so the
|
|
317
|
+
// filter runs over the whole retained log; the line limit is
|
|
318
|
+
// applied to the filtered result below.
|
|
319
|
+
const rawLog = await this.log(jobId, {
|
|
320
|
+
activity: opts.activity,
|
|
321
|
+
maxLogLines: 0,
|
|
322
|
+
logId: opts.logId,
|
|
323
|
+
projectKey: opts.projectKey,
|
|
324
|
+
});
|
|
325
|
+
const filteredLog = filterJobLog(rawLog, opts.logFilter);
|
|
326
|
+
if (opts.includeLogs)
|
|
327
|
+
log = limitJobLog(filteredLog, opts.maxLogLines);
|
|
328
|
+
if (opts.summary) {
|
|
329
|
+
logSummary = summarizeJobLog(state, filteredLog, opts.maxLogLines ?? 20, elapsedMs);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
if (error instanceof DataikuError) {
|
|
334
|
+
// A terminal outcome observed from the status endpoint is
|
|
335
|
+
// authoritative; failed log retrieval must not fail the wait.
|
|
336
|
+
logUnavailable = error.category === "not_found" ? "not_found" : "error";
|
|
337
|
+
if (logUnavailable === "not_found") {
|
|
338
|
+
removed = await this.probeJobRemoved(projectEnc, jobEnc);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
else {
|
|
342
|
+
throw error;
|
|
343
|
+
}
|
|
287
344
|
}
|
|
288
345
|
}
|
|
289
346
|
return {
|
|
@@ -301,6 +358,8 @@ export class JobsResource extends BaseResource {
|
|
|
301
358
|
},
|
|
302
359
|
...(log !== undefined ? { log, } : {}),
|
|
303
360
|
...(logSummary !== undefined ? { logSummary, } : {}),
|
|
361
|
+
...(logUnavailable !== undefined ? { logUnavailable, } : {}),
|
|
362
|
+
...(removed !== undefined ? { removed, } : {}),
|
|
304
363
|
};
|
|
305
364
|
}
|
|
306
365
|
// Timeout — return failure result, don't throw
|
|
@@ -329,6 +388,22 @@ export class JobsResource extends BaseResource {
|
|
|
329
388
|
await sleep(Math.min(nextDelayMs, timeout - elapsedMs));
|
|
330
389
|
}
|
|
331
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* Probe whether a job still exists. Returns `true` when the server reports
|
|
393
|
+
* it as not found, `false` when it is still visible, and `undefined` when
|
|
394
|
+
* the probe outcome is ambiguous.
|
|
395
|
+
*/
|
|
396
|
+
async probeJobRemoved(projectEnc, jobEnc) {
|
|
397
|
+
try {
|
|
398
|
+
await this.client.get(`/public/api/projects/${projectEnc}/jobs/${jobEnc}/`);
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
catch (error) {
|
|
402
|
+
if (error instanceof DataikuError && error.category === "not_found")
|
|
403
|
+
return true;
|
|
404
|
+
return undefined;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
332
407
|
/** Request a job abort. */
|
|
333
408
|
async abort(jobId, projectKey) {
|
|
334
409
|
const jobEnc = encodeURIComponent(jobId);
|
|
@@ -1,21 +1,63 @@
|
|
|
1
|
-
import type { JupyterNotebookContent, JupyterNotebookSummary, NotebookSession, SqlNotebookContent, SqlNotebookSummary } from "../schemas.js";
|
|
1
|
+
import type { JupyterNotebookContent, JupyterNotebookSummary, NotebookSession, SqlNotebookContent, SqlNotebookHistory, SqlNotebookSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
|
+
/** Result of a save that lands through POST (create) or PUT (update). */
|
|
4
|
+
export interface NotebookSaveResult {
|
|
5
|
+
/** True when this call created the notebook, false when it overwrote one. */
|
|
6
|
+
created: boolean;
|
|
7
|
+
/** SHA-256 (hex) of the persisted content after a confirming re-read. */
|
|
8
|
+
hash: string;
|
|
9
|
+
}
|
|
10
|
+
export interface SaveNotebookOptions {
|
|
11
|
+
/** Reject the save when the stored content hash no longer matches. */
|
|
12
|
+
expectHash?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface JupyterUnloadAllResult {
|
|
15
|
+
name: string;
|
|
16
|
+
unloadedSessionIds: string[];
|
|
17
|
+
}
|
|
3
18
|
export declare class NotebooksResource extends BaseResource {
|
|
4
|
-
/**
|
|
5
|
-
|
|
19
|
+
/**
|
|
20
|
+
* List Jupyter notebooks in a project.
|
|
21
|
+
*
|
|
22
|
+
* Pass `opts.active` to use the official `?active=` query filter: `true`
|
|
23
|
+
* returns only currently running notebooks, `false` only non-running ones
|
|
24
|
+
* (verified against `list_jupyter_notebooks` in the official Python API
|
|
25
|
+
* client). Omit it to list without the filter.
|
|
26
|
+
*/
|
|
27
|
+
listJupyter(projectKey?: string, opts?: {
|
|
28
|
+
active?: boolean;
|
|
29
|
+
}): Promise<JupyterNotebookSummary[]>;
|
|
6
30
|
/** Get the full content of a Jupyter notebook. */
|
|
7
31
|
getJupyter(name: string, projectKey?: string): Promise<JupyterNotebookContent>;
|
|
8
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Create a Jupyter notebook.
|
|
34
|
+
*
|
|
35
|
+
* The API takes the notebook name in the path; the body is the content.
|
|
36
|
+
*/
|
|
9
37
|
createJupyter(name: string, content: JupyterNotebookContent, projectKey?: string): Promise<void>;
|
|
10
38
|
/** Save (overwrite) a Jupyter notebook's content. */
|
|
11
39
|
saveJupyter(name: string, content: JupyterNotebookContent, projectKey?: string): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Save a Jupyter notebook, creating it when missing.
|
|
42
|
+
*
|
|
43
|
+
* `created` reflects what actually happened: a 404 on the fresh read makes
|
|
44
|
+
* the follow-up POST a creation; an existing notebook is overwritten by
|
|
45
|
+
* PUT. The returned `hash` is always the hash of the persisted content
|
|
46
|
+
* from a confirming re-read, so it can be passed straight to
|
|
47
|
+
* `--expect-hash` on a later save. The PUT/POST themselves stay
|
|
48
|
+
* unconditional: DSS has no conditional write, so a writer committing
|
|
49
|
+
* between the read and the write is not detected.
|
|
50
|
+
*/
|
|
51
|
+
saveOrCreateJupyter(name: string, content: JupyterNotebookContent, projectKey?: string, opts?: SaveNotebookOptions): Promise<NotebookSaveResult>;
|
|
12
52
|
/** Delete a Jupyter notebook. */
|
|
13
53
|
deleteJupyter(name: string, projectKey?: string): Promise<void>;
|
|
14
54
|
/**
|
|
15
55
|
* Clear all cell outputs from a Jupyter notebook.
|
|
16
56
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
57
|
+
* Uses the official `DELETE .../outputs` endpoint (verified against
|
|
58
|
+
* `DSSJupyterNotebook.clear_outputs` in the official Python API client),
|
|
59
|
+
* so the clear is a single atomic server-side operation instead of a
|
|
60
|
+
* non-atomic GET-strip-PUT round trip.
|
|
19
61
|
*/
|
|
20
62
|
clearJupyterOutputs(name: string, projectKey?: string): Promise<void>;
|
|
21
63
|
/** List running kernel sessions for a Jupyter notebook. */
|
|
@@ -28,6 +70,16 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
28
70
|
* requires an externally-started disposable session fixture.
|
|
29
71
|
*/
|
|
30
72
|
unloadJupyter(name: string, sessionId: string, projectKey?: string): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* Unload every session of every currently running Jupyter notebook.
|
|
75
|
+
*
|
|
76
|
+
* DSS exposes no single unload-all endpoint (verified against the official
|
|
77
|
+
* Python API client, which only ever issues per-session DELETEs), so this
|
|
78
|
+
* composes the verified endpoints: list active notebooks, list each
|
|
79
|
+
* notebook's sessions, and DELETE each session. Sessions that vanish
|
|
80
|
+
* between listing and unloading count as already unloaded.
|
|
81
|
+
*/
|
|
82
|
+
unloadJupyterAll(projectKey?: string): Promise<JupyterUnloadAllResult[]>;
|
|
31
83
|
/** List all SQL notebooks in a project. */
|
|
32
84
|
listSql(projectKey?: string): Promise<SqlNotebookSummary[]>;
|
|
33
85
|
/** Get the full content of a SQL notebook. */
|
|
@@ -36,14 +88,32 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
36
88
|
createSql(id: string, content: SqlNotebookContent, projectKey?: string): Promise<void>;
|
|
37
89
|
/** Save (overwrite) a SQL notebook's content. */
|
|
38
90
|
saveSql(id: string, content: SqlNotebookContent, projectKey?: string): Promise<void>;
|
|
91
|
+
/**
|
|
92
|
+
* Save a SQL notebook, creating it when missing. Same contract as
|
|
93
|
+
* `saveOrCreateJupyter`: `created` reflects the fresh read, and `hash` is
|
|
94
|
+
* the hash of the persisted content from a confirming re-read.
|
|
95
|
+
*/
|
|
96
|
+
saveOrCreateSql(id: string, content: SqlNotebookContent, projectKey?: string, opts?: SaveNotebookOptions): Promise<NotebookSaveResult>;
|
|
39
97
|
/** Delete a SQL notebook. */
|
|
40
98
|
deleteSql(id: string, projectKey?: string): Promise<void>;
|
|
41
99
|
/** Get execution history for a SQL notebook (keyed by cell ID). */
|
|
42
|
-
getSqlHistory(id: string, projectKey?: string): Promise<
|
|
100
|
+
getSqlHistory(id: string, projectKey?: string): Promise<SqlNotebookHistory>;
|
|
43
101
|
/** Clear execution history for a SQL notebook. */
|
|
44
102
|
clearSqlHistory(id: string, opts?: {
|
|
45
103
|
cellId?: string;
|
|
46
104
|
numRunsToRetain?: number;
|
|
47
105
|
projectKey?: string;
|
|
48
106
|
}): Promise<void>;
|
|
107
|
+
/**
|
|
108
|
+
* Probe the stored notebook before a save. A missing notebook resolves to
|
|
109
|
+
* `undefined` (the caller will create it); anything else propagates. When
|
|
110
|
+
* `expectHash` is armed, the fresh read doubles as the stale-read guard:
|
|
111
|
+
* the hash of the stored content must match, or the save is rejected
|
|
112
|
+
* before any write. Like every DSS write, the PUT/POST that follows stays
|
|
113
|
+
* unconditional — the guard cannot detect a writer that commits between
|
|
114
|
+
* this read and the write.
|
|
115
|
+
*/
|
|
116
|
+
private prepareSave;
|
|
117
|
+
/** Validate the armed `--expect-hash` value into a comparable hex digest. */
|
|
118
|
+
private validatedExpectHash;
|
|
49
119
|
}
|
|
@@ -1,10 +1,23 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
+
import { JupyterNotebookContentSchema, JupyterNotebookSummaryArraySchema, NotebookSessionArraySchema, SqlNotebookContentSchema, SqlNotebookHistorySchema, SqlNotebookSummaryArraySchema, } from "../schemas.js";
|
|
3
|
+
import { stableHash, } from "../utils/stable-hash.js";
|
|
2
4
|
import { BaseResource, } from "./base.js";
|
|
5
|
+
const EXPECT_HASH_PATTERN = /^[0-9a-fA-F]{64}$/;
|
|
3
6
|
export class NotebooksResource extends BaseResource {
|
|
4
7
|
// ── Jupyter Notebooks ──────────────────────────────────────────────
|
|
5
|
-
/**
|
|
6
|
-
|
|
7
|
-
|
|
8
|
+
/**
|
|
9
|
+
* List Jupyter notebooks in a project.
|
|
10
|
+
*
|
|
11
|
+
* Pass `opts.active` to use the official `?active=` query filter: `true`
|
|
12
|
+
* returns only currently running notebooks, `false` only non-running ones
|
|
13
|
+
* (verified against `list_jupyter_notebooks` in the official Python API
|
|
14
|
+
* client). Omit it to list without the filter.
|
|
15
|
+
*/
|
|
16
|
+
async listJupyter(projectKey, opts) {
|
|
17
|
+
const suffix = opts?.active === undefined
|
|
18
|
+
? "/"
|
|
19
|
+
: `/?active=${String(opts.active)}`;
|
|
20
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks${suffix}`);
|
|
8
21
|
return this.client.safeParse(JupyterNotebookSummaryArraySchema, raw, "notebooks.listJupyter");
|
|
9
22
|
}
|
|
10
23
|
/** Get the full content of a Jupyter notebook. */
|
|
@@ -13,7 +26,11 @@ export class NotebooksResource extends BaseResource {
|
|
|
13
26
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`);
|
|
14
27
|
return this.client.safeParse(JupyterNotebookContentSchema, raw, "notebooks.getJupyter");
|
|
15
28
|
}
|
|
16
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Create a Jupyter notebook.
|
|
31
|
+
*
|
|
32
|
+
* The API takes the notebook name in the path; the body is the content.
|
|
33
|
+
*/
|
|
17
34
|
async createJupyter(name, content, projectKey) {
|
|
18
35
|
const nameEnc = encodeURIComponent(name);
|
|
19
36
|
await this.client.post(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`, content);
|
|
@@ -23,6 +40,29 @@ export class NotebooksResource extends BaseResource {
|
|
|
23
40
|
const nameEnc = encodeURIComponent(name);
|
|
24
41
|
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}`, content);
|
|
25
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Save a Jupyter notebook, creating it when missing.
|
|
45
|
+
*
|
|
46
|
+
* `created` reflects what actually happened: a 404 on the fresh read makes
|
|
47
|
+
* the follow-up POST a creation; an existing notebook is overwritten by
|
|
48
|
+
* PUT. The returned `hash` is always the hash of the persisted content
|
|
49
|
+
* from a confirming re-read, so it can be passed straight to
|
|
50
|
+
* `--expect-hash` on a later save. The PUT/POST themselves stay
|
|
51
|
+
* unconditional: DSS has no conditional write, so a writer committing
|
|
52
|
+
* between the read and the write is not detected.
|
|
53
|
+
*/
|
|
54
|
+
async saveOrCreateJupyter(name, content, projectKey, opts) {
|
|
55
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
56
|
+
const before = await this.prepareSave(() => this.getJupyter(name, pk), opts?.expectHash, pk, "Jupyter notebook", name);
|
|
57
|
+
if (before === undefined) {
|
|
58
|
+
await this.createJupyter(name, content, pk);
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
await this.saveJupyter(name, content, pk);
|
|
62
|
+
}
|
|
63
|
+
const persisted = await this.getJupyter(name, pk);
|
|
64
|
+
return { created: before === undefined, hash: stableHash(persisted), };
|
|
65
|
+
}
|
|
26
66
|
/** Delete a Jupyter notebook. */
|
|
27
67
|
async deleteJupyter(name, projectKey) {
|
|
28
68
|
const nameEnc = encodeURIComponent(name);
|
|
@@ -31,21 +71,14 @@ export class NotebooksResource extends BaseResource {
|
|
|
31
71
|
/**
|
|
32
72
|
* Clear all cell outputs from a Jupyter notebook.
|
|
33
73
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
74
|
+
* Uses the official `DELETE .../outputs` endpoint (verified against
|
|
75
|
+
* `DSSJupyterNotebook.clear_outputs` in the official Python API client),
|
|
76
|
+
* so the clear is a single atomic server-side operation instead of a
|
|
77
|
+
* non-atomic GET-strip-PUT round trip.
|
|
36
78
|
*/
|
|
37
79
|
async clearJupyterOutputs(name, projectKey) {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
if (cell.cell_type !== "code")
|
|
41
|
-
return cell;
|
|
42
|
-
return {
|
|
43
|
-
...cell,
|
|
44
|
-
outputs: [],
|
|
45
|
-
execution_count: 0,
|
|
46
|
-
};
|
|
47
|
-
});
|
|
48
|
-
await this.saveJupyter(name, { ...notebook, cells: clearedCells, }, projectKey);
|
|
80
|
+
const nameEnc = encodeURIComponent(name);
|
|
81
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}/outputs`);
|
|
49
82
|
}
|
|
50
83
|
/** List running kernel sessions for a Jupyter notebook. */
|
|
51
84
|
async listJupyterSessions(name, projectKey) {
|
|
@@ -65,6 +98,35 @@ export class NotebooksResource extends BaseResource {
|
|
|
65
98
|
const sidEnc = encodeURIComponent(sessionId);
|
|
66
99
|
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/jupyter-notebooks/${nameEnc}/sessions/${sidEnc}`);
|
|
67
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Unload every session of every currently running Jupyter notebook.
|
|
103
|
+
*
|
|
104
|
+
* DSS exposes no single unload-all endpoint (verified against the official
|
|
105
|
+
* Python API client, which only ever issues per-session DELETEs), so this
|
|
106
|
+
* composes the verified endpoints: list active notebooks, list each
|
|
107
|
+
* notebook's sessions, and DELETE each session. Sessions that vanish
|
|
108
|
+
* between listing and unloading count as already unloaded.
|
|
109
|
+
*/
|
|
110
|
+
async unloadJupyterAll(projectKey) {
|
|
111
|
+
const active = await this.listJupyter(projectKey, { active: true, });
|
|
112
|
+
const results = [];
|
|
113
|
+
for (const summary of active) {
|
|
114
|
+
const sessions = await this.listJupyterSessions(summary.name, projectKey);
|
|
115
|
+
const unloadedSessionIds = [];
|
|
116
|
+
for (const session of sessions) {
|
|
117
|
+
try {
|
|
118
|
+
await this.unloadJupyter(summary.name, session.sessionId, projectKey);
|
|
119
|
+
unloadedSessionIds.push(session.sessionId);
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
if (!(error instanceof DataikuError && error.category === "not_found"))
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
results.push({ name: summary.name, unloadedSessionIds, });
|
|
127
|
+
}
|
|
128
|
+
return results;
|
|
129
|
+
}
|
|
68
130
|
// ── SQL Notebooks ──────────────────────────────────────────────────
|
|
69
131
|
/** List all SQL notebooks in a project. */
|
|
70
132
|
async listSql(projectKey) {
|
|
@@ -86,6 +148,23 @@ export class NotebooksResource extends BaseResource {
|
|
|
86
148
|
const idEnc = encodeURIComponent(id);
|
|
87
149
|
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`, content);
|
|
88
150
|
}
|
|
151
|
+
/**
|
|
152
|
+
* Save a SQL notebook, creating it when missing. Same contract as
|
|
153
|
+
* `saveOrCreateJupyter`: `created` reflects the fresh read, and `hash` is
|
|
154
|
+
* the hash of the persisted content from a confirming re-read.
|
|
155
|
+
*/
|
|
156
|
+
async saveOrCreateSql(id, content, projectKey, opts) {
|
|
157
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
158
|
+
const before = await this.prepareSave(() => this.getSql(id, pk), opts?.expectHash, pk, "SQL notebook", id);
|
|
159
|
+
if (before === undefined) {
|
|
160
|
+
await this.createSql(id, content, pk);
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
await this.saveSql(id, content, pk);
|
|
164
|
+
}
|
|
165
|
+
const persisted = await this.getSql(id, pk);
|
|
166
|
+
return { created: before === undefined, hash: stableHash(persisted), };
|
|
167
|
+
}
|
|
89
168
|
/** Delete a SQL notebook. */
|
|
90
169
|
async deleteSql(id, projectKey) {
|
|
91
170
|
const idEnc = encodeURIComponent(id);
|
|
@@ -94,11 +173,56 @@ export class NotebooksResource extends BaseResource {
|
|
|
94
173
|
/** Get execution history for a SQL notebook (keyed by cell ID). */
|
|
95
174
|
async getSqlHistory(id, projectKey) {
|
|
96
175
|
const idEnc = encodeURIComponent(id);
|
|
97
|
-
|
|
176
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}/history`);
|
|
177
|
+
return this.client.safeParse(SqlNotebookHistorySchema, raw, "notebooks.getSqlHistory");
|
|
98
178
|
}
|
|
99
179
|
/** Clear execution history for a SQL notebook. */
|
|
100
180
|
async clearSqlHistory(id, opts) {
|
|
101
181
|
const idEnc = encodeURIComponent(id);
|
|
102
182
|
await this.client.post(`/public/api/projects/${this.enc(opts?.projectKey)}/sql-notebooks/${idEnc}/history/clear`, { cellId: opts?.cellId, numRunsToRetain: opts?.numRunsToRetain, });
|
|
103
183
|
}
|
|
184
|
+
// ── Shared save helpers ────────────────────────────────────────────
|
|
185
|
+
/**
|
|
186
|
+
* Probe the stored notebook before a save. A missing notebook resolves to
|
|
187
|
+
* `undefined` (the caller will create it); anything else propagates. When
|
|
188
|
+
* `expectHash` is armed, the fresh read doubles as the stale-read guard:
|
|
189
|
+
* the hash of the stored content must match, or the save is rejected
|
|
190
|
+
* before any write. Like every DSS write, the PUT/POST that follows stays
|
|
191
|
+
* unconditional — the guard cannot detect a writer that commits between
|
|
192
|
+
* this read and the write.
|
|
193
|
+
*/
|
|
194
|
+
async prepareSave(read, expectHash, pk, resourceLabel, id) {
|
|
195
|
+
// Shape validation runs before any request so an armed but malformed
|
|
196
|
+
// guard rejects without touching DSS.
|
|
197
|
+
const expected = expectHash === undefined
|
|
198
|
+
? undefined
|
|
199
|
+
: this.validatedExpectHash(expectHash, pk, resourceLabel, id);
|
|
200
|
+
let current;
|
|
201
|
+
try {
|
|
202
|
+
current = await read();
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
if (!(error instanceof DataikuError && error.category === "not_found"))
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
if (expected === undefined)
|
|
209
|
+
return current;
|
|
210
|
+
const actual = current === undefined ? undefined : stableHash(current);
|
|
211
|
+
if (actual !== expected) {
|
|
212
|
+
throw new ClientValidationError(`The ${resourceLabel} ${JSON.stringify(id)} changed since it was read.`, "validation_failed", "Re-read the notebook and retry with the current hash value.", {
|
|
213
|
+
projectKey: pk,
|
|
214
|
+
id,
|
|
215
|
+
expectedHash: expected,
|
|
216
|
+
...(actual !== undefined ? { actualHash: actual, } : { absent: true, }),
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
return current;
|
|
220
|
+
}
|
|
221
|
+
/** Validate the armed `--expect-hash` value into a comparable hex digest. */
|
|
222
|
+
validatedExpectHash(expectHash, pk, resourceLabel, id) {
|
|
223
|
+
if (!EXPECT_HASH_PATTERN.test(expectHash)) {
|
|
224
|
+
throw new ClientValidationError(`Expected ${resourceLabel} hash must be a 64-character SHA-256 hex digest.`, "validation_failed", "Use the hash value returned by a previous save or read.", { projectKey: pk, id, });
|
|
225
|
+
}
|
|
226
|
+
return expectHash.toLowerCase();
|
|
227
|
+
}
|
|
104
228
|
}
|
|
@@ -67,6 +67,18 @@ export interface ProjectGitFutureWaitOptions {
|
|
|
67
67
|
pollIntervalMs?: number;
|
|
68
68
|
timeoutMs?: number;
|
|
69
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Validate and canonicalize a Git library reference path. This is the single
|
|
72
|
+
* validator used by both SDK dispatch and CLI mutation planning.
|
|
73
|
+
*
|
|
74
|
+
* Strip only the permitted outer '/' separators, then require nonempty
|
|
75
|
+
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
76
|
+
* path therefore cannot change which API resource a dispatcher's URL
|
|
77
|
+
* normalization would otherwise select.
|
|
78
|
+
*/
|
|
79
|
+
export declare function validateGitReferencePath(path: string): string;
|
|
80
|
+
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
81
|
+
export declare function encodeGitReferencePath(path: string): string;
|
|
70
82
|
export declare class ProjectGitResource extends BaseResource {
|
|
71
83
|
/** Working-copy status: current branch, remotes, tracking counts, changes. */
|
|
72
84
|
status(projectKey: string): Promise<ProjectGitStatus>;
|
|
@@ -17,11 +17,39 @@ function sleep(ms) {
|
|
|
17
17
|
setTimeout(resolve, ms);
|
|
18
18
|
return promise;
|
|
19
19
|
}
|
|
20
|
-
function
|
|
20
|
+
function gitReferencePathError(path, message) {
|
|
21
|
+
return new ClientValidationError(message, "validation_failed", "Use a Git library reference path relative to the project library root: '/' -separated segment names, with no '.' or '..' segments.", { path, });
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Validate and canonicalize a Git library reference path. This is the single
|
|
25
|
+
* validator used by both SDK dispatch and CLI mutation planning.
|
|
26
|
+
*
|
|
27
|
+
* Strip only the permitted outer '/' separators, then require nonempty
|
|
28
|
+
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
29
|
+
* path therefore cannot change which API resource a dispatcher's URL
|
|
30
|
+
* normalization would otherwise select.
|
|
31
|
+
*/
|
|
32
|
+
export function validateGitReferencePath(path) {
|
|
33
|
+
if (typeof path !== "string" || path.trim() === "") {
|
|
34
|
+
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
35
|
+
}
|
|
21
36
|
const normalized = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
22
|
-
if (
|
|
23
|
-
throw
|
|
24
|
-
|
|
37
|
+
if (normalized === "") {
|
|
38
|
+
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
39
|
+
}
|
|
40
|
+
for (const segment of normalized.split("/")) {
|
|
41
|
+
if (segment === "") {
|
|
42
|
+
throw gitReferencePathError(path, "A Git library reference path must not contain empty segments (consecutive or repeated '/').");
|
|
43
|
+
}
|
|
44
|
+
if (segment === "." || segment === "..") {
|
|
45
|
+
throw gitReferencePathError(path, "A Git library reference path must not contain '.' or '..' segments.");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return normalized;
|
|
49
|
+
}
|
|
50
|
+
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
51
|
+
export function encodeGitReferencePath(path) {
|
|
52
|
+
return validateGitReferencePath(path)
|
|
25
53
|
.split("/")
|
|
26
54
|
.map((segment) => encodeURIComponent(segment))
|
|
27
55
|
.join("/");
|
|
@@ -280,7 +308,8 @@ export class ProjectGitResource extends BaseResource {
|
|
|
280
308
|
async setLibrary(projectKey, gitReferencePath, options) {
|
|
281
309
|
const repository = validatedRepository(options.repository, "Git library repository URL");
|
|
282
310
|
const secrets = [options.password,].filter((secret) => secret !== undefined && secret !== "");
|
|
283
|
-
const
|
|
311
|
+
const encodedPath = encodeGitReferencePath(gitReferencePath);
|
|
312
|
+
const raw = await this.request("PUT", `${this.libraryPath(projectKey)}/${encodedPath}`, {
|
|
284
313
|
repository,
|
|
285
314
|
login: options.login ?? null,
|
|
286
315
|
password: options.password ?? null,
|
|
@@ -291,19 +320,21 @@ export class ProjectGitResource extends BaseResource {
|
|
|
291
320
|
}
|
|
292
321
|
/** Detach a library, optionally deleting its local directory. */
|
|
293
322
|
async removeLibrary(projectKey, gitReferencePath, deleteDirectory = false) {
|
|
294
|
-
await this.request("DELETE", `${this.libraryPath(projectKey)}/${
|
|
323
|
+
await this.request("DELETE", `${this.libraryPath(projectKey)}/${encodeGitReferencePath(gitReferencePath)}${this.query({ deleteDirectory, })}`);
|
|
295
324
|
}
|
|
296
325
|
/** Reset one library to its remote state. Runs as a DSS future. */
|
|
297
326
|
async resetLibrary(projectKey, gitReferencePath) {
|
|
327
|
+
const gitRef = validateGitReferencePath(gitReferencePath);
|
|
298
328
|
const raw = await this.request("POST", `${this.libraryPath(projectKey)}/action/reset`, {
|
|
299
|
-
gitRef
|
|
329
|
+
gitRef,
|
|
300
330
|
});
|
|
301
331
|
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.resetLibrary");
|
|
302
332
|
}
|
|
303
333
|
/** Commit and push one library. Runs as a DSS future. */
|
|
304
334
|
async pushLibrary(projectKey, gitReferencePath, commitMessage) {
|
|
335
|
+
const gitRef = validateGitReferencePath(gitReferencePath);
|
|
305
336
|
const raw = await this.request("POST", `${this.libraryPath(projectKey)}/action/push`, {
|
|
306
|
-
gitRef
|
|
337
|
+
gitRef,
|
|
307
338
|
commitMessage,
|
|
308
339
|
});
|
|
309
340
|
return this.client.safeParse(ProjectGitFutureResponseSchema, raw ?? {}, "projectGit.pushLibrary");
|