dataiku-sdk 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +253 -38
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +102 -0
- package/dist/packages/types/src/index.js +76 -0
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +5 -3
- package/dist/src/cli/commands/app.js +61 -18
- package/dist/src/cli/commands/bundle.js +3 -2
- package/dist/src/cli/commands/code-env.js +23 -16
- package/dist/src/cli/commands/connection.d.ts +9 -0
- package/dist/src/cli/commands/connection.js +267 -1
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-collection.d.ts +2 -0
- package/dist/src/cli/commands/data-collection.js +164 -0
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +170 -40
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +9 -8
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +4 -3
- package/dist/src/cli/commands/group.d.ts +2 -0
- package/dist/src/cli/commands/group.js +109 -0
- package/dist/src/cli/commands/index.js +151 -78
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
- package/dist/src/cli/commands/knowledge-bank.js +80 -0
- package/dist/src/cli/commands/llm.d.ts +2 -0
- package/dist/src/cli/commands/llm.js +93 -0
- package/dist/src/cli/commands/macro.d.ts +2 -0
- package/dist/src/cli/commands/macro.js +221 -0
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +21 -8
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +17 -12
- package/dist/src/cli/commands/plugin.d.ts +2 -0
- package/dist/src/cli/commands/plugin.js +799 -0
- package/dist/src/cli/commands/project-folder.d.ts +2 -0
- package/dist/src/cli/commands/project-folder.js +197 -0
- package/dist/src/cli/commands/project-git.js +14 -54
- package/dist/src/cli/commands/project-library.js +7 -6
- package/dist/src/cli/commands/project.js +55 -8
- package/dist/src/cli/commands/recipe.js +50 -14
- package/dist/src/cli/commands/saved-model.d.ts +5 -0
- package/dist/src/cli/commands/saved-model.js +392 -4
- package/dist/src/cli/commands/scenario.js +147 -9
- package/dist/src/cli/commands/sql.js +3 -2
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/user.d.ts +8 -0
- package/dist/src/cli/commands/user.js +296 -0
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +2 -1
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +2 -3
- package/dist/src/cli/contract.js +1167 -163
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +50 -2
- package/dist/src/cli/helpers/app-successor.d.ts +19 -4
- package/dist/src/cli/helpers/app-successor.js +90 -30
- package/dist/src/cli/helpers/cleanup.js +3 -1
- package/dist/src/cli/helpers/flow-zone.js +1 -2
- package/dist/src/cli/output.d.ts +1 -1
- package/dist/src/cli/output.js +23 -10
- package/dist/src/cli/usage.d.ts +1 -0
- package/dist/src/cli/usage.js +47 -1
- package/dist/src/cli.js +225 -95
- package/dist/src/client.d.ts +86 -6
- package/dist/src/client.js +263 -50
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +4 -0
- package/dist/src/index.d.ts +16 -7
- package/dist/src/index.js +11 -2
- package/dist/src/resources/analyses.js +1 -8
- package/dist/src/resources/applications.d.ts +8 -6
- package/dist/src/resources/applications.js +21 -21
- package/dist/src/resources/base.d.ts +4 -0
- package/dist/src/resources/base.js +15 -0
- package/dist/src/resources/bundles.d.ts +9 -4
- package/dist/src/resources/bundles.js +10 -5
- package/dist/src/resources/connections.d.ts +78 -1
- package/dist/src/resources/connections.js +132 -1
- package/dist/src/resources/data-collections.d.ts +118 -0
- package/dist/src/resources/data-collections.js +49 -0
- package/dist/src/resources/datasets.d.ts +69 -0
- package/dist/src/resources/datasets.js +99 -1
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/futures.d.ts +4 -1
- package/dist/src/resources/futures.js +38 -6
- package/dist/src/resources/groups.d.ts +47 -0
- package/dist/src/resources/groups.js +56 -0
- package/dist/src/resources/jobs.d.ts +8 -13
- package/dist/src/resources/jobs.js +41 -16
- package/dist/src/resources/knowledge-banks.d.ts +58 -0
- package/dist/src/resources/knowledge-banks.js +61 -0
- package/dist/src/resources/llms.d.ts +92 -0
- package/dist/src/resources/llms.js +62 -0
- package/dist/src/resources/macros.d.ts +121 -0
- package/dist/src/resources/macros.js +231 -0
- package/dist/src/resources/ml-tasks.d.ts +37 -2
- package/dist/src/resources/ml-tasks.js +100 -20
- package/dist/src/resources/model-evaluation-stores.js +1 -8
- package/dist/src/resources/notebooks.d.ts +43 -7
- package/dist/src/resources/notebooks.js +53 -13
- package/dist/src/resources/plugins.d.ts +265 -0
- package/dist/src/resources/plugins.js +458 -0
- package/dist/src/resources/project-folders.d.ts +71 -0
- package/dist/src/resources/project-folders.js +57 -0
- package/dist/src/resources/project-git.d.ts +6 -0
- package/dist/src/resources/project-git.js +69 -42
- package/dist/src/resources/project-library.d.ts +0 -1
- package/dist/src/resources/project-library.js +1 -17
- package/dist/src/resources/projects.d.ts +19 -3
- package/dist/src/resources/projects.js +30 -18
- package/dist/src/resources/recipes.d.ts +15 -1
- package/dist/src/resources/recipes.js +73 -6
- package/dist/src/resources/saved-models.d.ts +134 -1
- package/dist/src/resources/saved-models.js +214 -6
- package/dist/src/resources/scenarios.d.ts +105 -0
- package/dist/src/resources/scenarios.js +232 -106
- package/dist/src/resources/sql.js +5 -1
- package/dist/src/resources/users.d.ts +129 -0
- package/dist/src/resources/users.js +128 -0
- package/dist/src/resources/webapps.d.ts +15 -2
- package/dist/src/resources/webapps.js +18 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +13 -4
- package/dist/src/skill.js +75 -24
- package/dist/src/utils/deep-merge.d.ts +1 -1
- package/dist/src/utils/polling.d.ts +20 -0
- package/dist/src/utils/polling.js +27 -0
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/dist/src/utils/secret-sanitize.d.ts +54 -0
- package/dist/src/utils/secret-sanitize.js +101 -0
- package/package.json +8 -5
- package/packages/types/dist/index.d.ts +102 -0
- package/packages/types/dist/index.js +76 -0
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +47 -225
- package/skills/dataiku-dss/references/app-releases.md +28 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
+
import { MacroDefinitionSchema, MacroRunStartSchema, MacroStateSchema, MacroSummaryArraySchema, MacroWaitResultSchema, } from "../schemas.js";
|
|
3
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
4
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
5
|
+
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
6
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
7
|
+
/** Hard floor for the result download cap; keeps a caller typo from hanging the read. */
|
|
8
|
+
const MIN_RESULT_MAX_BYTES = 1;
|
|
9
|
+
const MAX_RESULT_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
|
10
|
+
/**
|
|
11
|
+
* Normalize a byte cap coming from user input (CLI `--max-bytes`) into the
|
|
12
|
+
* inclusive range [1, 2 GiB], rejecting non-integers before any DSS request.
|
|
13
|
+
*/
|
|
14
|
+
export function macroResultMaxBytes(maxBytes) {
|
|
15
|
+
const raw = maxBytes ?? 8 * 1024 * 1024;
|
|
16
|
+
if (!Number.isInteger(raw) || raw < MIN_RESULT_MAX_BYTES) {
|
|
17
|
+
throw new ClientValidationError(`maxBytes must be a positive integer >= ${String(MIN_RESULT_MAX_BYTES)} (got ${String(raw)},)`);
|
|
18
|
+
}
|
|
19
|
+
return Math.min(raw, MAX_RESULT_MAX_BYTES);
|
|
20
|
+
}
|
|
21
|
+
function isRecord(value) {
|
|
22
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Build the run POST body from params/adminParams. DSS macro runs accept
|
|
26
|
+
* `params` keyed by the macro's declared param names and `adminParams` keyed
|
|
27
|
+
* by its declared adminParams names; both are optional.
|
|
28
|
+
*/
|
|
29
|
+
function macroRunBody(params, adminParams) {
|
|
30
|
+
const body = {};
|
|
31
|
+
if (params !== undefined)
|
|
32
|
+
body["params"] = params;
|
|
33
|
+
if (adminParams !== undefined)
|
|
34
|
+
body["adminParams"] = adminParams;
|
|
35
|
+
return body;
|
|
36
|
+
}
|
|
37
|
+
function macroRunFailure(state) {
|
|
38
|
+
if (state["resultError"] !== undefined && state["resultError"] !== null) {
|
|
39
|
+
return { source: "resultError", details: state["resultError"], };
|
|
40
|
+
}
|
|
41
|
+
if (state["storedError"] !== undefined && state["storedError"] !== null) {
|
|
42
|
+
return { source: "storedError", details: state["storedError"], };
|
|
43
|
+
}
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* DSS Macros — the plugin "runnable" surface under
|
|
48
|
+
* `/public/api/projects/{pk}/runnables`.
|
|
49
|
+
*
|
|
50
|
+
* A macro run executes arbitrary plugin code on the DSS backend with
|
|
51
|
+
* side effects that DSS cannot predict; every method here that starts or
|
|
52
|
+
* stops a run is a write. Result bodies may be plain text or HTML rather
|
|
53
|
+
* than JSON, so {@link MacrosResource.result} returns the raw body.
|
|
54
|
+
*/
|
|
55
|
+
export class MacrosResource extends BaseResource {
|
|
56
|
+
/**
|
|
57
|
+
* List the macros (runnables) available in a project. The official client
|
|
58
|
+
* lists at `/runnables/` (trailing slash): the slashless path 404s on live
|
|
59
|
+
* DSS 15, while get/run/state/result stay slashless.
|
|
60
|
+
*/
|
|
61
|
+
async list(projectKey) {
|
|
62
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/runnables/`);
|
|
63
|
+
return this.client.safeParse(MacroSummaryArraySchema, raw, "macros.list");
|
|
64
|
+
}
|
|
65
|
+
/** Get the definition of a macro: params, result type, owning plugin. */
|
|
66
|
+
async definition(runnableType, opts) {
|
|
67
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
68
|
+
const raw = await this.client.get(`${this.base(runnableType, opts?.projectKey)}`);
|
|
69
|
+
return this.client.safeParse(MacroDefinitionSchema, raw, "macros.definition");
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Start a macro run. Executes arbitrary plugin code: classify as
|
|
73
|
+
* destructive. With `wait: true` the server call blocks until completion
|
|
74
|
+
* and the resolved state is attached to the returned run handle.
|
|
75
|
+
*/
|
|
76
|
+
async run(runnableType, opts) {
|
|
77
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
78
|
+
const wait = opts?.wait === true;
|
|
79
|
+
const query = wait ? "?wait=true" : "?wait=false";
|
|
80
|
+
const raw = await this.client.post(`${this.base(runnableType, opts?.projectKey)}${query}`, macroRunBody(opts?.params, opts?.adminParams));
|
|
81
|
+
const run = this.client.safeParse(MacroRunStartSchema, raw, "macros.run");
|
|
82
|
+
if (typeof run?.runId !== "string" || run.runId.trim().length === 0) {
|
|
83
|
+
throw new DataikuError(200, "Unexpected Response", "macros.run: DSS did not return a run identifier.");
|
|
84
|
+
}
|
|
85
|
+
if (!wait)
|
|
86
|
+
return run;
|
|
87
|
+
try {
|
|
88
|
+
const state = await this.state(runnableType, run.runId, {
|
|
89
|
+
projectKey: opts?.projectKey,
|
|
90
|
+
});
|
|
91
|
+
return { ...run, state, };
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// The run started; a failed post-wait state probe must not lose the runId.
|
|
95
|
+
return run;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** Request abort of a running macro run (204 response). */
|
|
99
|
+
async abort(runnableType, runId, projectKey) {
|
|
100
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
101
|
+
requireNonEmpty(runId, "runId");
|
|
102
|
+
await this.client.post(`${this.base(runnableType, projectKey)}/abort/${encodeURIComponent(runId)}`, {});
|
|
103
|
+
}
|
|
104
|
+
/** Get the poll state of a macro run (running flag, progress, errors). */
|
|
105
|
+
async state(runnableType, runId, opts) {
|
|
106
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
107
|
+
requireNonEmpty(runId, "runId");
|
|
108
|
+
const remainingMs = opts?.timeoutMs;
|
|
109
|
+
const raw = await this.client.get(`${this.base(runnableType, opts?.projectKey)}/state/${encodeURIComponent(runId)}`, ...(remainingMs !== undefined ? [{ timeoutMs: remainingMs, },] : []));
|
|
110
|
+
return this.client.safeParse(MacroStateSchema, raw, "macros.state");
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Download the result of a macro run. Macro results are documents of the
|
|
114
|
+
* macro's declared `resultType` (HTML, text, JSON, …) and are frequently
|
|
115
|
+
* NOT JSON: the raw body is returned verbatim. JSON bodies are returned
|
|
116
|
+
* as parsed values so programmatic callers can read fields directly.
|
|
117
|
+
*/
|
|
118
|
+
async result(runnableType, runId, opts) {
|
|
119
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
120
|
+
requireNonEmpty(runId, "runId");
|
|
121
|
+
const maxBytes = macroResultMaxBytes(opts?.maxBytes);
|
|
122
|
+
const raw = await this.client.getTextLimited(`${this.base(runnableType, opts?.projectKey)}/result/${encodeURIComponent(runId)}`, maxBytes);
|
|
123
|
+
if (raw.truncated) {
|
|
124
|
+
throw new Error(`Macro result exceeded ${String(maxBytes)} bytes; retry with a larger maxBytes.`);
|
|
125
|
+
}
|
|
126
|
+
const text = raw.text;
|
|
127
|
+
const trimmed = text.trim();
|
|
128
|
+
if (trimmed.startsWith("{") && trimmed.endsWith("}")
|
|
129
|
+
|| trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
|
130
|
+
try {
|
|
131
|
+
return JSON.parse(text);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// Not JSON after all — fall through and return the text.
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return text;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Run a macro and poll its state until it finishes or times out, using a
|
|
141
|
+
* bounded client-side loop on the state endpoint. Returns
|
|
142
|
+
* `{ success: false, timedOut: true }` on timeout rather than throwing.
|
|
143
|
+
*/
|
|
144
|
+
async runAndWait(runnableType, opts) {
|
|
145
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
146
|
+
// Validate the wait knobs BEFORE the run POST: an invalid budget must be
|
|
147
|
+
// a caller error, not a mutation whose wait behavior is undefined.
|
|
148
|
+
if (opts?.pollIntervalMs !== undefined) {
|
|
149
|
+
if (!Number.isFinite(opts.pollIntervalMs) || opts.pollIntervalMs < 1) {
|
|
150
|
+
throw new ClientValidationError("pollIntervalMs must be a finite number >= 1.");
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (opts?.timeoutMs !== undefined) {
|
|
154
|
+
if (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs < 0) {
|
|
155
|
+
throw new ClientValidationError("timeoutMs must be a finite number >= 0.");
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const baseIntervalMs = Math.max(1, opts?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
|
|
159
|
+
const adaptivePolling = opts?.pollIntervalMs === undefined;
|
|
160
|
+
// One state observation always happens regardless of budget (poll
|
|
161
|
+
// precedes deadline check), but never round a budget up to a full poll.
|
|
162
|
+
const timeoutMs = Math.max(0, opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
163
|
+
const startedAt = Date.now();
|
|
164
|
+
const run = await this.run(runnableType, {
|
|
165
|
+
wait: false,
|
|
166
|
+
projectKey: opts?.projectKey,
|
|
167
|
+
params: opts?.params,
|
|
168
|
+
adminParams: opts?.adminParams,
|
|
169
|
+
});
|
|
170
|
+
const runId = run.runId;
|
|
171
|
+
let pollCount = 0;
|
|
172
|
+
let lastState;
|
|
173
|
+
const timedOutResult = (elapsedMs, state) => this.client.safeParse(MacroWaitResultSchema, {
|
|
174
|
+
runnableType,
|
|
175
|
+
runId,
|
|
176
|
+
running: true,
|
|
177
|
+
success: false,
|
|
178
|
+
timedOut: true,
|
|
179
|
+
elapsedMs,
|
|
180
|
+
pollCount,
|
|
181
|
+
...(state ? { state, } : {}),
|
|
182
|
+
}, "macros.runAndWait");
|
|
183
|
+
while (true) {
|
|
184
|
+
pollCount += 1;
|
|
185
|
+
try {
|
|
186
|
+
lastState = await this.state(runnableType, runId, {
|
|
187
|
+
projectKey: opts?.projectKey,
|
|
188
|
+
timeoutMs: Math.max(1, timeoutMs - (Date.now() - startedAt)),
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
if (!isRequestDeadlineError(error, startedAt + timeoutMs))
|
|
193
|
+
throw error;
|
|
194
|
+
return timedOutResult(Date.now() - startedAt, lastState);
|
|
195
|
+
}
|
|
196
|
+
const running = lastState.running === true;
|
|
197
|
+
if (running !== true) {
|
|
198
|
+
return this.client.safeParse(MacroWaitResultSchema, {
|
|
199
|
+
runnableType,
|
|
200
|
+
runId,
|
|
201
|
+
running: false,
|
|
202
|
+
success: macroRunFailure(lastState) === undefined
|
|
203
|
+
&& lastState.exists !== false,
|
|
204
|
+
elapsedMs: Date.now() - startedAt,
|
|
205
|
+
pollCount,
|
|
206
|
+
...(macroRunFailure(lastState)
|
|
207
|
+
? { failure: macroRunFailure(lastState), }
|
|
208
|
+
: {}),
|
|
209
|
+
state: lastState,
|
|
210
|
+
}, "macros.runAndWait");
|
|
211
|
+
}
|
|
212
|
+
const elapsedMs = Date.now() - startedAt;
|
|
213
|
+
if (elapsedMs >= timeoutMs) {
|
|
214
|
+
return timedOutResult(elapsedMs, lastState);
|
|
215
|
+
}
|
|
216
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
217
|
+
pollCount,
|
|
218
|
+
baseIntervalMs,
|
|
219
|
+
adaptiveEnabled: adaptivePolling,
|
|
220
|
+
});
|
|
221
|
+
const { promise, resolve, } = Promise.withResolvers();
|
|
222
|
+
setTimeout(resolve, Math.min(nextDelayMs, Math.max(1, timeoutMs - (Date.now() - startedAt))));
|
|
223
|
+
await promise;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
base(runnableType, projectKey) {
|
|
227
|
+
return `/public/api/projects/${this.enc(projectKey)}/runnables/${encodeURIComponent(runnableType)}`;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
// Re-exported for CLI/plan use: failure extraction from a poll state record.
|
|
231
|
+
export { isRecord as isMacroStateRecord, macroRunFailure, };
|
|
@@ -39,6 +39,19 @@ export interface MlTaskTrainOptions {
|
|
|
39
39
|
sessionDescription?: string;
|
|
40
40
|
runQueue?: boolean;
|
|
41
41
|
wait?: boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Deadline in milliseconds for the whole `wait` phase after DSS has
|
|
44
|
+
* accepted the training request. Validated before any request is sent:
|
|
45
|
+
* an invalid deadline must not start training work on the server.
|
|
46
|
+
* `timeoutMs: 0` disables waiting (single status check). Requires `wait`.
|
|
47
|
+
*/
|
|
48
|
+
timeoutMs?: number;
|
|
49
|
+
/**
|
|
50
|
+
* Fixed poll interval in milliseconds for the training status loop.
|
|
51
|
+
* When omitted the loop backs off adaptively (doubles every 3 polls, capped).
|
|
52
|
+
* Requires `wait`.
|
|
53
|
+
*/
|
|
54
|
+
pollIntervalMs?: number;
|
|
42
55
|
projectKey?: string;
|
|
43
56
|
}
|
|
44
57
|
export interface MlTaskTrainingSession extends Record<string, unknown> {
|
|
@@ -48,7 +61,17 @@ export interface MlTaskTrainCompletedResult extends Record<string, unknown> {
|
|
|
48
61
|
sessionId: string;
|
|
49
62
|
trainedModelIds: string[];
|
|
50
63
|
}
|
|
51
|
-
|
|
64
|
+
/** Result returned when the training wait budget was exhausted. */
|
|
65
|
+
export interface MlTaskTrainTimedOutResult extends Record<string, unknown> {
|
|
66
|
+
sessionId: string;
|
|
67
|
+
trainedModelIds: string[];
|
|
68
|
+
timedOut: true;
|
|
69
|
+
success: false;
|
|
70
|
+
state: "RUNNING";
|
|
71
|
+
elapsedMs: number;
|
|
72
|
+
pollCount: number;
|
|
73
|
+
}
|
|
74
|
+
export type MlTaskTrainResult = MlTaskTrainingSession | MlTaskTrainCompletedResult | MlTaskTrainTimedOutResult;
|
|
52
75
|
export interface MlTrainedModelDetails extends Record<string, unknown> {
|
|
53
76
|
id?: string;
|
|
54
77
|
algorithm?: string;
|
|
@@ -77,7 +100,19 @@ export declare class MlTasksResource extends BaseResource {
|
|
|
77
100
|
getSettings(analysisId: string, mlTaskId: string, projectKey?: string): Promise<MlTaskSettings>;
|
|
78
101
|
/** Save the full settings object of an ML task. */
|
|
79
102
|
saveSettings(analysisId: string, mlTaskId: string, settings: MlTaskSettings, projectKey?: string): Promise<MlTaskActionResult | undefined>;
|
|
80
|
-
/**
|
|
103
|
+
/**
|
|
104
|
+
* Start training, optionally waiting for the task to finish.
|
|
105
|
+
*
|
|
106
|
+
* `timeoutMs` is validated BEFORE the training POST: an invalid deadline
|
|
107
|
+
* never reaches the server, so bad input cannot start training work. Once
|
|
108
|
+
* DSS has accepted the training request, the deadline scopes only the
|
|
109
|
+
* wait phase — each status poll is issued with the remaining budget as a
|
|
110
|
+
* total request deadline, so a hung status request cannot outlast it.
|
|
111
|
+
*
|
|
112
|
+
* When the budget is exhausted the returned result keeps the documented
|
|
113
|
+
* success shape plus `timedOut: true` and `success: false` instead of
|
|
114
|
+
* throwing, so a caller can distinguish "still training" from "finished".
|
|
115
|
+
*/
|
|
81
116
|
train(opts: MlTaskTrainOptions): Promise<MlTaskTrainResult>;
|
|
82
117
|
/** List identifiers for every trained model currently present on the task. */
|
|
83
118
|
listTrainedModels(analysisId: string, mlTaskId: string, projectKey?: string): Promise<string[]>;
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ClientValidationError, } from "../errors.js";
|
|
2
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
3
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
3
4
|
const TRAIN_POLL_INTERVAL_MS = 5_000;
|
|
4
|
-
|
|
5
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
6
|
-
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
7
|
-
}
|
|
8
|
-
return value;
|
|
9
|
-
}
|
|
5
|
+
const TRAIN_DEFAULT_TIMEOUT_MS = 120_000;
|
|
10
6
|
function delay(ms) {
|
|
11
7
|
const { promise, resolve, } = Promise.withResolvers();
|
|
12
8
|
setTimeout(resolve, ms);
|
|
@@ -26,11 +22,11 @@ export class MlTasksResource extends BaseResource {
|
|
|
26
22
|
/** Create a prediction or clustering task in an existing visual analysis. */
|
|
27
23
|
async create(opts) {
|
|
28
24
|
if (opts.taskType !== "PREDICTION" && opts.taskType !== "CLUSTERING") {
|
|
29
|
-
throw new
|
|
25
|
+
throw new ClientValidationError("taskType must be PREDICTION or CLUSTERING.", "invalid_enum");
|
|
30
26
|
}
|
|
31
27
|
if (opts.taskType === "PREDICTION"
|
|
32
28
|
&& (typeof opts.targetVariable !== "string" || opts.targetVariable.trim().length === 0)) {
|
|
33
|
-
throw new
|
|
29
|
+
throw new ClientValidationError("targetVariable is required for PREDICTION ML tasks.", "missing_required_arg");
|
|
34
30
|
}
|
|
35
31
|
const analysisId = encodeURIComponent(requireNonEmpty(opts.analysisId, "analysisId"));
|
|
36
32
|
const backendType = opts.backendType ?? "PY_MEMORY";
|
|
@@ -65,17 +61,49 @@ export class MlTasksResource extends BaseResource {
|
|
|
65
61
|
/** Save the full settings object of an ML task. */
|
|
66
62
|
async saveSettings(analysisId, mlTaskId, settings, projectKey) {
|
|
67
63
|
if (settings === null || typeof settings !== "object" || Array.isArray(settings)) {
|
|
68
|
-
throw new
|
|
64
|
+
throw new ClientValidationError("settings must be an object.", "validation_failed");
|
|
69
65
|
}
|
|
70
66
|
return this.client.post(`${this.taskPath(analysisId, mlTaskId, projectKey)}/settings`, settings);
|
|
71
67
|
}
|
|
72
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Start training, optionally waiting for the task to finish.
|
|
70
|
+
*
|
|
71
|
+
* `timeoutMs` is validated BEFORE the training POST: an invalid deadline
|
|
72
|
+
* never reaches the server, so bad input cannot start training work. Once
|
|
73
|
+
* DSS has accepted the training request, the deadline scopes only the
|
|
74
|
+
* wait phase — each status poll is issued with the remaining budget as a
|
|
75
|
+
* total request deadline, so a hung status request cannot outlast it.
|
|
76
|
+
*
|
|
77
|
+
* When the budget is exhausted the returned result keeps the documented
|
|
78
|
+
* success shape plus `timedOut: true` and `success: false` instead of
|
|
79
|
+
* throwing, so a caller can distinguish "still training" from "finished".
|
|
80
|
+
*/
|
|
73
81
|
async train(opts) {
|
|
74
82
|
if (opts.sessionName !== undefined)
|
|
75
83
|
requireNonEmpty(opts.sessionName, "sessionName");
|
|
76
84
|
if (opts.sessionDescription !== undefined) {
|
|
77
85
|
requireNonEmpty(opts.sessionDescription, "sessionDescription");
|
|
78
86
|
}
|
|
87
|
+
// Deadline validation happens before any network request: a malformed
|
|
88
|
+
// budget must not start training work on the server.
|
|
89
|
+
const explicitIntervalMs = opts.pollIntervalMs;
|
|
90
|
+
if (explicitIntervalMs !== undefined && opts.wait !== true) {
|
|
91
|
+
throw new ClientValidationError("pollIntervalMs requires wait: true.", "validation_failed");
|
|
92
|
+
}
|
|
93
|
+
if (explicitIntervalMs !== undefined
|
|
94
|
+
&& (!Number.isFinite(explicitIntervalMs) || explicitIntervalMs <= 0)) {
|
|
95
|
+
throw new ClientValidationError("pollIntervalMs must be a finite positive number.", "validation_failed");
|
|
96
|
+
}
|
|
97
|
+
let timeoutMs;
|
|
98
|
+
if (opts.timeoutMs !== undefined) {
|
|
99
|
+
if (opts.wait !== true) {
|
|
100
|
+
throw new ClientValidationError("timeoutMs requires wait: true.", "validation_failed");
|
|
101
|
+
}
|
|
102
|
+
if (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs < 0) {
|
|
103
|
+
throw new ClientValidationError("timeoutMs must be a finite non-negative number.", "validation_failed");
|
|
104
|
+
}
|
|
105
|
+
timeoutMs = opts.timeoutMs;
|
|
106
|
+
}
|
|
79
107
|
const taskPath = this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
80
108
|
const session = await this.client.post(`${taskPath}/train`, {
|
|
81
109
|
sessionName: opts.sessionName,
|
|
@@ -84,15 +112,67 @@ export class MlTasksResource extends BaseResource {
|
|
|
84
112
|
});
|
|
85
113
|
if (opts.wait !== true)
|
|
86
114
|
return session;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
115
|
+
const deadlineMs = timeoutMs ?? TRAIN_DEFAULT_TIMEOUT_MS;
|
|
116
|
+
const baseIntervalMs = Math.max(1, explicitIntervalMs ?? TRAIN_POLL_INTERVAL_MS);
|
|
117
|
+
const adaptiveEnabled = explicitIntervalMs === undefined;
|
|
118
|
+
const startedAt = Date.now();
|
|
119
|
+
let pollCount = 0;
|
|
120
|
+
// One status observation always happens; the poll precedes the deadline
|
|
121
|
+
// check, so `timeoutMs: 0` still reports the freshly-started state once.
|
|
122
|
+
while (true) {
|
|
123
|
+
pollCount += 1;
|
|
124
|
+
// One status observation always happens, even when the budget is
|
|
125
|
+
// already spent (the poll precedes the deadline check); only the
|
|
126
|
+
// requests issued while budget remains are bounded by the remaining
|
|
127
|
+
// time, so a stalled status endpoint can never defeat the deadline.
|
|
128
|
+
const remainingMs = deadlineMs - (Date.now() - startedAt);
|
|
129
|
+
let status;
|
|
130
|
+
try {
|
|
131
|
+
status = await this.client.get(`${this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey)}/status`, remainingMs > 0
|
|
132
|
+
? { timeoutMs: remainingMs, }
|
|
133
|
+
: undefined);
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
// The poll budget ran out: report the structured timeout instead
|
|
137
|
+
// of letting the transport deadline error escape the loop.
|
|
138
|
+
if (!isRequestDeadlineError(error, startedAt + deadlineMs))
|
|
139
|
+
throw error;
|
|
140
|
+
return {
|
|
141
|
+
sessionId: session.sessionId,
|
|
142
|
+
trainedModelIds: [],
|
|
143
|
+
timedOut: true,
|
|
144
|
+
success: false,
|
|
145
|
+
state: "RUNNING",
|
|
146
|
+
elapsedMs: Date.now() - startedAt,
|
|
147
|
+
pollCount,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
const elapsedMs = Date.now() - startedAt;
|
|
151
|
+
if (status.training !== false) {
|
|
152
|
+
if (elapsedMs >= deadlineMs) {
|
|
153
|
+
return {
|
|
154
|
+
sessionId: session.sessionId,
|
|
155
|
+
trainedModelIds: modelIdsFromStatus(status),
|
|
156
|
+
timedOut: true,
|
|
157
|
+
success: false,
|
|
158
|
+
state: "RUNNING",
|
|
159
|
+
elapsedMs,
|
|
160
|
+
pollCount,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
164
|
+
pollCount,
|
|
165
|
+
baseIntervalMs,
|
|
166
|
+
adaptiveEnabled,
|
|
167
|
+
});
|
|
168
|
+
await delay(Math.min(nextDelayMs, deadlineMs - elapsedMs));
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
sessionId: session.sessionId,
|
|
173
|
+
trainedModelIds: modelIdsFromStatus(status),
|
|
174
|
+
};
|
|
91
175
|
}
|
|
92
|
-
return {
|
|
93
|
-
sessionId: session.sessionId,
|
|
94
|
-
trainedModelIds: modelIdsFromStatus(status),
|
|
95
|
-
};
|
|
96
176
|
}
|
|
97
177
|
/** List identifiers for every trained model currently present on the task. */
|
|
98
178
|
async listTrainedModels(analysisId, mlTaskId, projectKey) {
|
|
@@ -1,11 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { BaseResource, } from "./base.js";
|
|
3
|
-
function requireNonEmpty(value, name) {
|
|
4
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
5
|
-
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
6
|
-
}
|
|
7
|
-
return value;
|
|
8
|
-
}
|
|
1
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
9
2
|
export class ModelEvaluationStoresResource extends BaseResource {
|
|
10
3
|
/** List model evaluation stores in a project. */
|
|
11
4
|
async list(projectKey) {
|
|
@@ -7,6 +7,15 @@ export interface NotebookSaveResult {
|
|
|
7
7
|
/** SHA-256 (hex) of the persisted content after a confirming re-read. */
|
|
8
8
|
hash: string;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* SQL-notebook save result. `id` is the persisted notebook identifier: on a
|
|
12
|
+
* create DSS allocates it server-side (a request-supplied `id` is rejected),
|
|
13
|
+
* so every later read, update, and delete must use this id — the requested
|
|
14
|
+
* handle only becomes the notebook's display name.
|
|
15
|
+
*/
|
|
16
|
+
export interface SqlNotebookSaveResult extends NotebookSaveResult {
|
|
17
|
+
id: string;
|
|
18
|
+
}
|
|
10
19
|
export interface SaveNotebookOptions {
|
|
11
20
|
/** Reject the save when the stored content hash no longer matches. */
|
|
12
21
|
expectHash?: string;
|
|
@@ -84,16 +93,43 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
84
93
|
listSql(projectKey?: string): Promise<SqlNotebookSummary[]>;
|
|
85
94
|
/** Get the full content of a SQL notebook. */
|
|
86
95
|
getSql(id: string, projectKey?: string): Promise<SqlNotebookContent>;
|
|
87
|
-
/**
|
|
88
|
-
|
|
89
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Create a SQL notebook.
|
|
98
|
+
*
|
|
99
|
+
* DSS allocates the notebook id server-side and rejects a request-supplied
|
|
100
|
+
* `id` (`Field 'id' is specified, it must not be specified`); `name` is
|
|
101
|
+
* the display handle (the requested label) when provided. The returned id
|
|
102
|
+
* comes from the creation receipt and is the only handle later reads,
|
|
103
|
+
* updates, and deletes may use.
|
|
104
|
+
*/
|
|
105
|
+
createSql(content: SqlNotebookContent, projectKey?: string, opts?: {
|
|
106
|
+
name?: string;
|
|
107
|
+
}): Promise<{
|
|
108
|
+
id: string;
|
|
109
|
+
}>;
|
|
110
|
+
/**
|
|
111
|
+
* Save (overwrite) a SQL notebook's content.
|
|
112
|
+
*
|
|
113
|
+
* Identity is positional: the URL id and the resolved project key are
|
|
114
|
+
* injected into the body, overriding any `id`/`projectKey` the content may
|
|
115
|
+
* carry, so a stale or hand-built body can never retarget the write. PUT
|
|
116
|
+
* is a full replacement — fields absent from the content (e.g. `name`)
|
|
117
|
+
* are cleared server-side.
|
|
118
|
+
*/
|
|
90
119
|
saveSql(id: string, content: SqlNotebookContent, projectKey?: string): Promise<void>;
|
|
91
120
|
/**
|
|
92
|
-
* Save a SQL notebook, creating it when missing.
|
|
93
|
-
*
|
|
94
|
-
* the
|
|
121
|
+
* Save a SQL notebook, creating it when missing.
|
|
122
|
+
*
|
|
123
|
+
* Update path: the fresh read finds the notebook and a PUT overwrites it.
|
|
124
|
+
* Create path: DSS allocates the persisted id (the requested handle becomes
|
|
125
|
+
* the display name; there is no name-based re-lookup, so a foreign
|
|
126
|
+
* notebook can never be mistaken for the target), the id is returned, and
|
|
127
|
+
* every later read/update/delete must use that returned id. `created`
|
|
128
|
+
* reflects the fresh read; `hash` is the hash of the persisted content
|
|
129
|
+
* from a confirming re-read. The armed `--expect-hash` guard also refuses
|
|
130
|
+
* an absent target.
|
|
95
131
|
*/
|
|
96
|
-
saveOrCreateSql(id: string, content: SqlNotebookContent, projectKey?: string, opts?: SaveNotebookOptions): Promise<
|
|
132
|
+
saveOrCreateSql(id: string, content: SqlNotebookContent, projectKey?: string, opts?: SaveNotebookOptions): Promise<SqlNotebookSaveResult>;
|
|
97
133
|
/** Delete a SQL notebook. */
|
|
98
134
|
deleteSql(id: string, projectKey?: string): Promise<void>;
|
|
99
135
|
/** Get execution history for a SQL notebook (keyed by cell ID). */
|
|
@@ -139,31 +139,71 @@ export class NotebooksResource extends BaseResource {
|
|
|
139
139
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`);
|
|
140
140
|
return this.client.safeParse(SqlNotebookContentSchema, raw, "notebooks.getSql");
|
|
141
141
|
}
|
|
142
|
-
/**
|
|
143
|
-
|
|
144
|
-
|
|
142
|
+
/**
|
|
143
|
+
* Create a SQL notebook.
|
|
144
|
+
*
|
|
145
|
+
* DSS allocates the notebook id server-side and rejects a request-supplied
|
|
146
|
+
* `id` (`Field 'id' is specified, it must not be specified`); `name` is
|
|
147
|
+
* the display handle (the requested label) when provided. The returned id
|
|
148
|
+
* comes from the creation receipt and is the only handle later reads,
|
|
149
|
+
* updates, and deletes may use.
|
|
150
|
+
*/
|
|
151
|
+
async createSql(content, projectKey, opts) {
|
|
152
|
+
const body = {
|
|
153
|
+
...content,
|
|
154
|
+
projectKey: this.resolveProjectKey(projectKey),
|
|
155
|
+
};
|
|
156
|
+
// Positional identity only: a content-supplied id (e.g. a stale GET
|
|
157
|
+
// spread) is stripped before the request — DSS rejects it outright.
|
|
158
|
+
delete body["id"];
|
|
159
|
+
if (opts?.name !== undefined && body["name"] === undefined)
|
|
160
|
+
body["name"] = opts.name;
|
|
161
|
+
const raw = await this.client.post(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/`, body);
|
|
162
|
+
const record = raw !== null && typeof raw === "object" && !Array.isArray(raw)
|
|
163
|
+
? raw
|
|
164
|
+
: undefined;
|
|
165
|
+
const id = record?.["id"];
|
|
166
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
167
|
+
throw new DataikuError(200, "Unexpected Response", "notebooks.createSql: DSS did not return the generated notebook id.");
|
|
168
|
+
}
|
|
169
|
+
return { id, };
|
|
145
170
|
}
|
|
146
|
-
/**
|
|
171
|
+
/**
|
|
172
|
+
* Save (overwrite) a SQL notebook's content.
|
|
173
|
+
*
|
|
174
|
+
* Identity is positional: the URL id and the resolved project key are
|
|
175
|
+
* injected into the body, overriding any `id`/`projectKey` the content may
|
|
176
|
+
* carry, so a stale or hand-built body can never retarget the write. PUT
|
|
177
|
+
* is a full replacement — fields absent from the content (e.g. `name`)
|
|
178
|
+
* are cleared server-side.
|
|
179
|
+
*/
|
|
147
180
|
async saveSql(id, content, projectKey) {
|
|
148
181
|
const idEnc = encodeURIComponent(id);
|
|
149
|
-
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`, content);
|
|
182
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/sql-notebooks/${idEnc}`, { ...content, id, projectKey: this.resolveProjectKey(projectKey), });
|
|
150
183
|
}
|
|
151
184
|
/**
|
|
152
|
-
* Save a SQL notebook, creating it when missing.
|
|
153
|
-
*
|
|
154
|
-
* the
|
|
185
|
+
* Save a SQL notebook, creating it when missing.
|
|
186
|
+
*
|
|
187
|
+
* Update path: the fresh read finds the notebook and a PUT overwrites it.
|
|
188
|
+
* Create path: DSS allocates the persisted id (the requested handle becomes
|
|
189
|
+
* the display name; there is no name-based re-lookup, so a foreign
|
|
190
|
+
* notebook can never be mistaken for the target), the id is returned, and
|
|
191
|
+
* every later read/update/delete must use that returned id. `created`
|
|
192
|
+
* reflects the fresh read; `hash` is the hash of the persisted content
|
|
193
|
+
* from a confirming re-read. The armed `--expect-hash` guard also refuses
|
|
194
|
+
* an absent target.
|
|
155
195
|
*/
|
|
156
196
|
async saveOrCreateSql(id, content, projectKey, opts) {
|
|
157
197
|
const pk = this.resolveProjectKey(projectKey);
|
|
158
198
|
const before = await this.prepareSave(() => this.getSql(id, pk), opts?.expectHash, pk, "SQL notebook", id);
|
|
159
199
|
if (before === undefined) {
|
|
160
|
-
await this.createSql(
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
await this.saveSql(id, content, pk);
|
|
200
|
+
const created = await this.createSql(content, pk, { name: id, });
|
|
201
|
+
const persisted = await this.getSql(created.id, pk);
|
|
202
|
+
return { created: true, id: created.id, hash: stableHash(persisted), };
|
|
164
203
|
}
|
|
204
|
+
await this.saveSql(id, content, pk);
|
|
165
205
|
const persisted = await this.getSql(id, pk);
|
|
166
|
-
return { created:
|
|
206
|
+
return { created: false, id, hash: stableHash(persisted), };
|
|
167
207
|
}
|
|
168
208
|
/** Delete a SQL notebook. */
|
|
169
209
|
async deleteSql(id, projectKey) {
|