dataiku-sdk 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -46
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +102 -0
- package/dist/packages/types/src/index.js +76 -0
- package/dist/src/cli/commands/app.js +57 -15
- package/dist/src/cli/commands/bundle.js +1 -1
- package/dist/src/cli/commands/code-env.js +13 -7
- package/dist/src/cli/commands/connection.d.ts +9 -0
- package/dist/src/cli/commands/connection.js +267 -1
- package/dist/src/cli/commands/data-collection.d.ts +2 -0
- package/dist/src/cli/commands/data-collection.js +164 -0
- package/dist/src/cli/commands/dataset.js +161 -32
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/group.d.ts +2 -0
- package/dist/src/cli/commands/group.js +109 -0
- package/dist/src/cli/commands/index.js +151 -78
- package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
- package/dist/src/cli/commands/knowledge-bank.js +80 -0
- package/dist/src/cli/commands/llm.d.ts +2 -0
- package/dist/src/cli/commands/llm.js +93 -0
- package/dist/src/cli/commands/macro.d.ts +2 -0
- package/dist/src/cli/commands/macro.js +221 -0
- package/dist/src/cli/commands/ml-task.js +18 -6
- package/dist/src/cli/commands/notebook.js +6 -2
- package/dist/src/cli/commands/plugin.d.ts +2 -0
- package/dist/src/cli/commands/plugin.js +799 -0
- package/dist/src/cli/commands/project-folder.d.ts +2 -0
- package/dist/src/cli/commands/project-folder.js +197 -0
- package/dist/src/cli/commands/project-git.js +14 -54
- package/dist/src/cli/commands/project.js +52 -6
- package/dist/src/cli/commands/recipe.js +36 -1
- package/dist/src/cli/commands/saved-model.d.ts +5 -0
- package/dist/src/cli/commands/saved-model.js +388 -1
- package/dist/src/cli/commands/scenario.js +138 -1
- package/dist/src/cli/commands/user.d.ts +8 -0
- package/dist/src/cli/commands/user.js +296 -0
- package/dist/src/cli/contract.d.ts +1 -2
- package/dist/src/cli/contract.js +1089 -103
- package/dist/src/cli/flags.js +30 -1
- package/dist/src/cli/helpers/app-successor.d.ts +19 -4
- package/dist/src/cli/helpers/app-successor.js +90 -30
- package/dist/src/cli/helpers/cleanup.js +3 -1
- package/dist/src/cli/helpers/flow-zone.js +1 -2
- package/dist/src/cli/output.d.ts +1 -1
- package/dist/src/cli/output.js +23 -10
- package/dist/src/cli/usage.d.ts +1 -0
- package/dist/src/cli/usage.js +47 -1
- package/dist/src/cli.js +147 -41
- package/dist/src/client.d.ts +83 -5
- package/dist/src/client.js +185 -29
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +2 -0
- package/dist/src/index.d.ts +16 -7
- package/dist/src/index.js +11 -2
- package/dist/src/resources/analyses.js +1 -8
- package/dist/src/resources/applications.d.ts +8 -6
- package/dist/src/resources/applications.js +21 -21
- package/dist/src/resources/base.d.ts +4 -0
- package/dist/src/resources/base.js +15 -0
- package/dist/src/resources/bundles.d.ts +9 -4
- package/dist/src/resources/bundles.js +10 -5
- package/dist/src/resources/connections.d.ts +78 -1
- package/dist/src/resources/connections.js +132 -1
- package/dist/src/resources/data-collections.d.ts +118 -0
- package/dist/src/resources/data-collections.js +49 -0
- package/dist/src/resources/datasets.d.ts +69 -0
- package/dist/src/resources/datasets.js +99 -1
- package/dist/src/resources/futures.d.ts +4 -1
- package/dist/src/resources/futures.js +38 -6
- package/dist/src/resources/groups.d.ts +47 -0
- package/dist/src/resources/groups.js +56 -0
- package/dist/src/resources/jobs.d.ts +8 -13
- package/dist/src/resources/jobs.js +41 -16
- package/dist/src/resources/knowledge-banks.d.ts +58 -0
- package/dist/src/resources/knowledge-banks.js +61 -0
- package/dist/src/resources/llms.d.ts +92 -0
- package/dist/src/resources/llms.js +62 -0
- package/dist/src/resources/macros.d.ts +121 -0
- package/dist/src/resources/macros.js +231 -0
- package/dist/src/resources/ml-tasks.d.ts +37 -2
- package/dist/src/resources/ml-tasks.js +100 -20
- package/dist/src/resources/model-evaluation-stores.js +1 -8
- package/dist/src/resources/notebooks.d.ts +43 -7
- package/dist/src/resources/notebooks.js +53 -13
- package/dist/src/resources/plugins.d.ts +265 -0
- package/dist/src/resources/plugins.js +458 -0
- package/dist/src/resources/project-folders.d.ts +71 -0
- package/dist/src/resources/project-folders.js +57 -0
- package/dist/src/resources/project-git.d.ts +6 -0
- package/dist/src/resources/project-git.js +69 -42
- package/dist/src/resources/project-library.d.ts +0 -1
- package/dist/src/resources/project-library.js +1 -17
- package/dist/src/resources/projects.d.ts +19 -1
- package/dist/src/resources/projects.js +27 -1
- package/dist/src/resources/recipes.d.ts +15 -1
- package/dist/src/resources/recipes.js +73 -6
- package/dist/src/resources/saved-models.d.ts +134 -1
- package/dist/src/resources/saved-models.js +214 -6
- package/dist/src/resources/scenarios.d.ts +105 -0
- package/dist/src/resources/scenarios.js +232 -106
- package/dist/src/resources/sql.js +5 -1
- package/dist/src/resources/users.d.ts +129 -0
- package/dist/src/resources/users.js +128 -0
- package/dist/src/resources/webapps.d.ts +15 -2
- package/dist/src/resources/webapps.js +18 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/deep-merge.d.ts +1 -1
- package/dist/src/utils/polling.d.ts +20 -0
- package/dist/src/utils/polling.js +27 -0
- package/dist/src/utils/response-file.js +1 -1
- package/dist/src/utils/secret-sanitize.d.ts +54 -0
- package/dist/src/utils/secret-sanitize.js +101 -0
- package/package.json +6 -5
- package/packages/types/dist/index.d.ts +102 -0
- package/packages/types/dist/index.js +76 -0
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +7 -5
- package/skills/dataiku-dss/references/app-releases.md +8 -7
- package/skills/dataiku-dss/references/discovery.md +4 -4
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { ClientValidationError, } from "../errors.js";
|
|
2
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
3
|
+
export class LlmsResource extends BaseResource {
|
|
4
|
+
/**
|
|
5
|
+
* List the LLMs available in a project, including Retrieval-Augmented
|
|
6
|
+
* Generation ones. Requires READ_CONF.
|
|
7
|
+
*/
|
|
8
|
+
async list(options = {}) {
|
|
9
|
+
const params = new URLSearchParams();
|
|
10
|
+
if (options.purpose !== undefined) {
|
|
11
|
+
params.set("purpose", requireNonEmpty(options.purpose, "purpose"));
|
|
12
|
+
}
|
|
13
|
+
const query = params.size > 0 ? `?${params.toString()}` : "";
|
|
14
|
+
return this.client.get(`/public/api/projects/${this.enc(options.projectKey)}/llms/${query}`);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Perform completions on an LLM. This call is cost-bearing: it invokes an
|
|
18
|
+
* external LLM provider and may incur charges.
|
|
19
|
+
*/
|
|
20
|
+
async completions(body, projectKey) {
|
|
21
|
+
const payload = validateLlmInferenceBody(body, "completions");
|
|
22
|
+
return this.client.post(`/public/api/projects/${this.enc(projectKey)}/llms/completions`, payload);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Perform embeddings on an LLM. This call is cost-bearing: it invokes an
|
|
26
|
+
* external LLM provider and may incur charges.
|
|
27
|
+
*/
|
|
28
|
+
async embeddings(body, projectKey) {
|
|
29
|
+
const payload = validateLlmInferenceBody(body, "embeddings");
|
|
30
|
+
return this.client.post(`/public/api/projects/${this.enc(projectKey)}/llms/embeddings`, payload);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Validate a completions/embeddings request body on the client boundary:
|
|
35
|
+
* `llmId` must be a non-empty string and `queries` a non-empty array of
|
|
36
|
+
* objects. Unknown documented fields pass through untouched.
|
|
37
|
+
*/
|
|
38
|
+
function validateLlmInferenceBody(body, action) {
|
|
39
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
40
|
+
throw new ClientValidationError(`llms.${action} body must be an object with llmId and queries.`);
|
|
41
|
+
}
|
|
42
|
+
const llmId = requireNonEmpty(body.llmId, "llmId");
|
|
43
|
+
if (!Array.isArray(body.queries) || body.queries.length === 0) {
|
|
44
|
+
throw new ClientValidationError(`llms.${action} queries must be a non-empty array of query objects.`, "validation_failed", 'Pass at least one query, e.g. queries: [{ messages: [...] }] for completions or [{ text: "..." }] for embeddings.');
|
|
45
|
+
}
|
|
46
|
+
for (const [index, query,] of body.queries.entries()) {
|
|
47
|
+
if (query === null || typeof query !== "object" || Array.isArray(query)) {
|
|
48
|
+
throw new ClientValidationError(`llms.${action} queries[${index}] must be an object.`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const payload = {
|
|
52
|
+
llmId,
|
|
53
|
+
queries: body.queries,
|
|
54
|
+
};
|
|
55
|
+
if (body.settings !== undefined) {
|
|
56
|
+
if (body.settings === null || typeof body.settings !== "object" || Array.isArray(body.settings)) {
|
|
57
|
+
throw new ClientValidationError("llms.completions settings must be an object.");
|
|
58
|
+
}
|
|
59
|
+
payload["settings"] = body.settings;
|
|
60
|
+
}
|
|
61
|
+
return payload;
|
|
62
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import type { MacroDefinition, MacroRunStart, MacroState, MacroSummary, MacroWaitResult } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export interface MacroRunOptions {
|
|
4
|
+
/**
|
|
5
|
+
* Server-side wait: when true the POST call blocks until the run finishes
|
|
6
|
+
* (docs: `POST /runnables/{runnableType}?wait=true`). Default false.
|
|
7
|
+
*/
|
|
8
|
+
wait?: boolean;
|
|
9
|
+
projectKey?: string;
|
|
10
|
+
/**
|
|
11
|
+
* Macro parameter values keyed by the param names declared in the macro
|
|
12
|
+
* definition (`params` array). Sent as the POST body. Values must be
|
|
13
|
+
* JSON-serializable primitives or structures matching the declared types.
|
|
14
|
+
*/
|
|
15
|
+
params?: Record<string, unknown>;
|
|
16
|
+
/**
|
|
17
|
+
* Admin-only parameter values keyed by the adminParams names declared in
|
|
18
|
+
* the macro definition. Sent as the POST body's adminParams entry.
|
|
19
|
+
*/
|
|
20
|
+
adminParams?: Record<string, unknown>;
|
|
21
|
+
}
|
|
22
|
+
export interface MacroWaitOptions {
|
|
23
|
+
pollIntervalMs?: number;
|
|
24
|
+
timeoutMs?: number;
|
|
25
|
+
projectKey?: string;
|
|
26
|
+
/** Macro parameter values (see {@link MacroRunOptions.params}). */
|
|
27
|
+
params?: Record<string, unknown>;
|
|
28
|
+
/** Admin-only parameter values (see {@link MacroRunOptions.adminParams}). */
|
|
29
|
+
adminParams?: Record<string, unknown>;
|
|
30
|
+
}
|
|
31
|
+
export interface MacroResultOptions {
|
|
32
|
+
projectKey?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Cap on the number of bytes buffered from the result body. Macro results
|
|
35
|
+
* can be arbitrary documents (HTML, text, JSON); the default 8 MiB guards
|
|
36
|
+
* against oversized bodies while remaining generous for typical reports.
|
|
37
|
+
*/
|
|
38
|
+
maxBytes?: number;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Normalize a byte cap coming from user input (CLI `--max-bytes`) into the
|
|
42
|
+
* inclusive range [1, 2 GiB], rejecting non-integers before any DSS request.
|
|
43
|
+
*/
|
|
44
|
+
export declare function macroResultMaxBytes(maxBytes: number | undefined): number;
|
|
45
|
+
declare function isRecord(value: unknown): value is Record<string, unknown>;
|
|
46
|
+
/**
|
|
47
|
+
* Run result of {@link MacrosResource.run} when the caller opted into
|
|
48
|
+
* `wait: false` (the default): a run id for abort/state/result tracking.
|
|
49
|
+
*/
|
|
50
|
+
export type MacroRunHandle = MacroRunStart;
|
|
51
|
+
/**
|
|
52
|
+
* When `wait: true`, DSS blocks server-side until the run finishes and then
|
|
53
|
+
* returns the run id plus the terminal poll state (resultType, errors, …).
|
|
54
|
+
*/
|
|
55
|
+
export type MacroRunWaitedResult = MacroRunStart & {
|
|
56
|
+
state?: MacroState;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Error payload surfaced by DSS on a failed macro run, from either
|
|
60
|
+
* `resultError` or `storedError` in the poll state.
|
|
61
|
+
*/
|
|
62
|
+
export interface MacroRunFailure {
|
|
63
|
+
source: "resultError" | "storedError";
|
|
64
|
+
details: unknown;
|
|
65
|
+
}
|
|
66
|
+
declare function macroRunFailure(state: Record<string, unknown>): MacroRunFailure | undefined;
|
|
67
|
+
/**
|
|
68
|
+
* DSS Macros — the plugin "runnable" surface under
|
|
69
|
+
* `/public/api/projects/{pk}/runnables`.
|
|
70
|
+
*
|
|
71
|
+
* A macro run executes arbitrary plugin code on the DSS backend with
|
|
72
|
+
* side effects that DSS cannot predict; every method here that starts or
|
|
73
|
+
* stops a run is a write. Result bodies may be plain text or HTML rather
|
|
74
|
+
* than JSON, so {@link MacrosResource.result} returns the raw body.
|
|
75
|
+
*/
|
|
76
|
+
export declare class MacrosResource extends BaseResource {
|
|
77
|
+
/**
|
|
78
|
+
* List the macros (runnables) available in a project. The official client
|
|
79
|
+
* lists at `/runnables/` (trailing slash): the slashless path 404s on live
|
|
80
|
+
* DSS 15, while get/run/state/result stay slashless.
|
|
81
|
+
*/
|
|
82
|
+
list(projectKey?: string): Promise<MacroSummary[]>;
|
|
83
|
+
/** Get the definition of a macro: params, result type, owning plugin. */
|
|
84
|
+
definition(runnableType: string, opts?: {
|
|
85
|
+
projectKey?: string;
|
|
86
|
+
}): Promise<MacroDefinition>;
|
|
87
|
+
/**
|
|
88
|
+
* Start a macro run. Executes arbitrary plugin code: classify as
|
|
89
|
+
* destructive. With `wait: true` the server call blocks until completion
|
|
90
|
+
* and the resolved state is attached to the returned run handle.
|
|
91
|
+
*/
|
|
92
|
+
run(runnableType: string, opts?: MacroRunOptions): Promise<MacroRunHandle | MacroRunWaitedResult>;
|
|
93
|
+
/** Request abort of a running macro run (204 response). */
|
|
94
|
+
abort(runnableType: string, runId: string, projectKey?: string): Promise<void>;
|
|
95
|
+
/** Get the poll state of a macro run (running flag, progress, errors). */
|
|
96
|
+
state(runnableType: string, runId: string, opts?: {
|
|
97
|
+
projectKey?: string;
|
|
98
|
+
/**
|
|
99
|
+
* TOTAL duration budget in ms for this state GET, measured from
|
|
100
|
+
* call start (bounds fetch, retries, backoff, and body read).
|
|
101
|
+
* Polling loops pass their remaining wait budget so a stalled
|
|
102
|
+
* state endpoint cannot outlive the wait deadline.
|
|
103
|
+
*/
|
|
104
|
+
timeoutMs?: number;
|
|
105
|
+
}): Promise<MacroState>;
|
|
106
|
+
/**
|
|
107
|
+
* Download the result of a macro run. Macro results are documents of the
|
|
108
|
+
* macro's declared `resultType` (HTML, text, JSON, …) and are frequently
|
|
109
|
+
* NOT JSON: the raw body is returned verbatim. JSON bodies are returned
|
|
110
|
+
* as parsed values so programmatic callers can read fields directly.
|
|
111
|
+
*/
|
|
112
|
+
result(runnableType: string, runId: string, opts?: MacroResultOptions): Promise<unknown>;
|
|
113
|
+
/**
|
|
114
|
+
* Run a macro and poll its state until it finishes or times out, using a
|
|
115
|
+
* bounded client-side loop on the state endpoint. Returns
|
|
116
|
+
* `{ success: false, timedOut: true }` on timeout rather than throwing.
|
|
117
|
+
*/
|
|
118
|
+
runAndWait(runnableType: string, opts?: MacroWaitOptions): Promise<MacroWaitResult>;
|
|
119
|
+
private base;
|
|
120
|
+
}
|
|
121
|
+
export { isRecord as isMacroStateRecord, macroRunFailure, };
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
+
import { MacroDefinitionSchema, MacroRunStartSchema, MacroStateSchema, MacroSummaryArraySchema, MacroWaitResultSchema, } from "../schemas.js";
|
|
3
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
4
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
5
|
+
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
6
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
7
|
+
/** Hard floor for the result download cap; keeps a caller typo from hanging the read. */
|
|
8
|
+
const MIN_RESULT_MAX_BYTES = 1;
|
|
9
|
+
const MAX_RESULT_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
|
10
|
+
/**
|
|
11
|
+
* Normalize a byte cap coming from user input (CLI `--max-bytes`) into the
|
|
12
|
+
* inclusive range [1, 2 GiB], rejecting non-integers before any DSS request.
|
|
13
|
+
*/
|
|
14
|
+
export function macroResultMaxBytes(maxBytes) {
|
|
15
|
+
const raw = maxBytes ?? 8 * 1024 * 1024;
|
|
16
|
+
if (!Number.isInteger(raw) || raw < MIN_RESULT_MAX_BYTES) {
|
|
17
|
+
throw new ClientValidationError(`maxBytes must be a positive integer >= ${String(MIN_RESULT_MAX_BYTES)} (got ${String(raw)},)`);
|
|
18
|
+
}
|
|
19
|
+
return Math.min(raw, MAX_RESULT_MAX_BYTES);
|
|
20
|
+
}
|
|
21
|
+
function isRecord(value) {
|
|
22
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Build the run POST body from params/adminParams. DSS macro runs accept
|
|
26
|
+
* `params` keyed by the macro's declared param names and `adminParams` keyed
|
|
27
|
+
* by its declared adminParams names; both are optional.
|
|
28
|
+
*/
|
|
29
|
+
function macroRunBody(params, adminParams) {
|
|
30
|
+
const body = {};
|
|
31
|
+
if (params !== undefined)
|
|
32
|
+
body["params"] = params;
|
|
33
|
+
if (adminParams !== undefined)
|
|
34
|
+
body["adminParams"] = adminParams;
|
|
35
|
+
return body;
|
|
36
|
+
}
|
|
37
|
+
function macroRunFailure(state) {
|
|
38
|
+
if (state["resultError"] !== undefined && state["resultError"] !== null) {
|
|
39
|
+
return { source: "resultError", details: state["resultError"], };
|
|
40
|
+
}
|
|
41
|
+
if (state["storedError"] !== undefined && state["storedError"] !== null) {
|
|
42
|
+
return { source: "storedError", details: state["storedError"], };
|
|
43
|
+
}
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* DSS Macros — the plugin "runnable" surface under
|
|
48
|
+
* `/public/api/projects/{pk}/runnables`.
|
|
49
|
+
*
|
|
50
|
+
* A macro run executes arbitrary plugin code on the DSS backend with
|
|
51
|
+
* side effects that DSS cannot predict; every method here that starts or
|
|
52
|
+
* stops a run is a write. Result bodies may be plain text or HTML rather
|
|
53
|
+
* than JSON, so {@link MacrosResource.result} returns the raw body.
|
|
54
|
+
*/
|
|
55
|
+
export class MacrosResource extends BaseResource {
|
|
56
|
+
/**
|
|
57
|
+
* List the macros (runnables) available in a project. The official client
|
|
58
|
+
* lists at `/runnables/` (trailing slash): the slashless path 404s on live
|
|
59
|
+
* DSS 15, while get/run/state/result stay slashless.
|
|
60
|
+
*/
|
|
61
|
+
async list(projectKey) {
|
|
62
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/runnables/`);
|
|
63
|
+
return this.client.safeParse(MacroSummaryArraySchema, raw, "macros.list");
|
|
64
|
+
}
|
|
65
|
+
/** Get the definition of a macro: params, result type, owning plugin. */
|
|
66
|
+
async definition(runnableType, opts) {
|
|
67
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
68
|
+
const raw = await this.client.get(`${this.base(runnableType, opts?.projectKey)}`);
|
|
69
|
+
return this.client.safeParse(MacroDefinitionSchema, raw, "macros.definition");
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Start a macro run. Executes arbitrary plugin code: classify as
|
|
73
|
+
* destructive. With `wait: true` the server call blocks until completion
|
|
74
|
+
* and the resolved state is attached to the returned run handle.
|
|
75
|
+
*/
|
|
76
|
+
async run(runnableType, opts) {
|
|
77
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
78
|
+
const wait = opts?.wait === true;
|
|
79
|
+
const query = wait ? "?wait=true" : "?wait=false";
|
|
80
|
+
const raw = await this.client.post(`${this.base(runnableType, opts?.projectKey)}${query}`, macroRunBody(opts?.params, opts?.adminParams));
|
|
81
|
+
const run = this.client.safeParse(MacroRunStartSchema, raw, "macros.run");
|
|
82
|
+
if (typeof run?.runId !== "string" || run.runId.trim().length === 0) {
|
|
83
|
+
throw new DataikuError(200, "Unexpected Response", "macros.run: DSS did not return a run identifier.");
|
|
84
|
+
}
|
|
85
|
+
if (!wait)
|
|
86
|
+
return run;
|
|
87
|
+
try {
|
|
88
|
+
const state = await this.state(runnableType, run.runId, {
|
|
89
|
+
projectKey: opts?.projectKey,
|
|
90
|
+
});
|
|
91
|
+
return { ...run, state, };
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// The run started; a failed post-wait state probe must not lose the runId.
|
|
95
|
+
return run;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** Request abort of a running macro run (204 response). */
|
|
99
|
+
async abort(runnableType, runId, projectKey) {
|
|
100
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
101
|
+
requireNonEmpty(runId, "runId");
|
|
102
|
+
await this.client.post(`${this.base(runnableType, projectKey)}/abort/${encodeURIComponent(runId)}`, {});
|
|
103
|
+
}
|
|
104
|
+
/** Get the poll state of a macro run (running flag, progress, errors). */
|
|
105
|
+
async state(runnableType, runId, opts) {
|
|
106
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
107
|
+
requireNonEmpty(runId, "runId");
|
|
108
|
+
const remainingMs = opts?.timeoutMs;
|
|
109
|
+
const raw = await this.client.get(`${this.base(runnableType, opts?.projectKey)}/state/${encodeURIComponent(runId)}`, ...(remainingMs !== undefined ? [{ timeoutMs: remainingMs, },] : []));
|
|
110
|
+
return this.client.safeParse(MacroStateSchema, raw, "macros.state");
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Download the result of a macro run. Macro results are documents of the
|
|
114
|
+
* macro's declared `resultType` (HTML, text, JSON, …) and are frequently
|
|
115
|
+
* NOT JSON: the raw body is returned verbatim. JSON bodies are returned
|
|
116
|
+
* as parsed values so programmatic callers can read fields directly.
|
|
117
|
+
*/
|
|
118
|
+
async result(runnableType, runId, opts) {
|
|
119
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
120
|
+
requireNonEmpty(runId, "runId");
|
|
121
|
+
const maxBytes = macroResultMaxBytes(opts?.maxBytes);
|
|
122
|
+
const raw = await this.client.getTextLimited(`${this.base(runnableType, opts?.projectKey)}/result/${encodeURIComponent(runId)}`, maxBytes);
|
|
123
|
+
if (raw.truncated) {
|
|
124
|
+
throw new Error(`Macro result exceeded ${String(maxBytes)} bytes; retry with a larger maxBytes.`);
|
|
125
|
+
}
|
|
126
|
+
const text = raw.text;
|
|
127
|
+
const trimmed = text.trim();
|
|
128
|
+
if (trimmed.startsWith("{") && trimmed.endsWith("}")
|
|
129
|
+
|| trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
|
130
|
+
try {
|
|
131
|
+
return JSON.parse(text);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// Not JSON after all — fall through and return the text.
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return text;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Run a macro and poll its state until it finishes or times out, using a
|
|
141
|
+
* bounded client-side loop on the state endpoint. Returns
|
|
142
|
+
* `{ success: false, timedOut: true }` on timeout rather than throwing.
|
|
143
|
+
*/
|
|
144
|
+
async runAndWait(runnableType, opts) {
|
|
145
|
+
requireNonEmpty(runnableType, "runnableType");
|
|
146
|
+
// Validate the wait knobs BEFORE the run POST: an invalid budget must be
|
|
147
|
+
// a caller error, not a mutation whose wait behavior is undefined.
|
|
148
|
+
if (opts?.pollIntervalMs !== undefined) {
|
|
149
|
+
if (!Number.isFinite(opts.pollIntervalMs) || opts.pollIntervalMs < 1) {
|
|
150
|
+
throw new ClientValidationError("pollIntervalMs must be a finite number >= 1.");
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (opts?.timeoutMs !== undefined) {
|
|
154
|
+
if (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs < 0) {
|
|
155
|
+
throw new ClientValidationError("timeoutMs must be a finite number >= 0.");
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const baseIntervalMs = Math.max(1, opts?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
|
|
159
|
+
const adaptivePolling = opts?.pollIntervalMs === undefined;
|
|
160
|
+
// One state observation always happens regardless of budget (poll
|
|
161
|
+
// precedes deadline check), but never round a budget up to a full poll.
|
|
162
|
+
const timeoutMs = Math.max(0, opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
163
|
+
const startedAt = Date.now();
|
|
164
|
+
const run = await this.run(runnableType, {
|
|
165
|
+
wait: false,
|
|
166
|
+
projectKey: opts?.projectKey,
|
|
167
|
+
params: opts?.params,
|
|
168
|
+
adminParams: opts?.adminParams,
|
|
169
|
+
});
|
|
170
|
+
const runId = run.runId;
|
|
171
|
+
let pollCount = 0;
|
|
172
|
+
let lastState;
|
|
173
|
+
const timedOutResult = (elapsedMs, state) => this.client.safeParse(MacroWaitResultSchema, {
|
|
174
|
+
runnableType,
|
|
175
|
+
runId,
|
|
176
|
+
running: true,
|
|
177
|
+
success: false,
|
|
178
|
+
timedOut: true,
|
|
179
|
+
elapsedMs,
|
|
180
|
+
pollCount,
|
|
181
|
+
...(state ? { state, } : {}),
|
|
182
|
+
}, "macros.runAndWait");
|
|
183
|
+
while (true) {
|
|
184
|
+
pollCount += 1;
|
|
185
|
+
try {
|
|
186
|
+
lastState = await this.state(runnableType, runId, {
|
|
187
|
+
projectKey: opts?.projectKey,
|
|
188
|
+
timeoutMs: Math.max(1, timeoutMs - (Date.now() - startedAt)),
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
if (!isRequestDeadlineError(error, startedAt + timeoutMs))
|
|
193
|
+
throw error;
|
|
194
|
+
return timedOutResult(Date.now() - startedAt, lastState);
|
|
195
|
+
}
|
|
196
|
+
const running = lastState.running === true;
|
|
197
|
+
if (running !== true) {
|
|
198
|
+
return this.client.safeParse(MacroWaitResultSchema, {
|
|
199
|
+
runnableType,
|
|
200
|
+
runId,
|
|
201
|
+
running: false,
|
|
202
|
+
success: macroRunFailure(lastState) === undefined
|
|
203
|
+
&& lastState.exists !== false,
|
|
204
|
+
elapsedMs: Date.now() - startedAt,
|
|
205
|
+
pollCount,
|
|
206
|
+
...(macroRunFailure(lastState)
|
|
207
|
+
? { failure: macroRunFailure(lastState), }
|
|
208
|
+
: {}),
|
|
209
|
+
state: lastState,
|
|
210
|
+
}, "macros.runAndWait");
|
|
211
|
+
}
|
|
212
|
+
const elapsedMs = Date.now() - startedAt;
|
|
213
|
+
if (elapsedMs >= timeoutMs) {
|
|
214
|
+
return timedOutResult(elapsedMs, lastState);
|
|
215
|
+
}
|
|
216
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
217
|
+
pollCount,
|
|
218
|
+
baseIntervalMs,
|
|
219
|
+
adaptiveEnabled: adaptivePolling,
|
|
220
|
+
});
|
|
221
|
+
const { promise, resolve, } = Promise.withResolvers();
|
|
222
|
+
setTimeout(resolve, Math.min(nextDelayMs, Math.max(1, timeoutMs - (Date.now() - startedAt))));
|
|
223
|
+
await promise;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
base(runnableType, projectKey) {
|
|
227
|
+
return `/public/api/projects/${this.enc(projectKey)}/runnables/${encodeURIComponent(runnableType)}`;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
// Re-exported for CLI/plan use: failure extraction from a poll state record.
|
|
231
|
+
export { isRecord as isMacroStateRecord, macroRunFailure, };
|
|
@@ -39,6 +39,19 @@ export interface MlTaskTrainOptions {
|
|
|
39
39
|
sessionDescription?: string;
|
|
40
40
|
runQueue?: boolean;
|
|
41
41
|
wait?: boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Deadline in milliseconds for the whole `wait` phase after DSS has
|
|
44
|
+
* accepted the training request. Validated before any request is sent:
|
|
45
|
+
* an invalid deadline must not start training work on the server.
|
|
46
|
+
* `timeoutMs: 0` disables waiting (single status check). Requires `wait`.
|
|
47
|
+
*/
|
|
48
|
+
timeoutMs?: number;
|
|
49
|
+
/**
|
|
50
|
+
* Fixed poll interval in milliseconds for the training status loop.
|
|
51
|
+
* When omitted the loop backs off adaptively (doubles every 3 polls, capped).
|
|
52
|
+
* Requires `wait`.
|
|
53
|
+
*/
|
|
54
|
+
pollIntervalMs?: number;
|
|
42
55
|
projectKey?: string;
|
|
43
56
|
}
|
|
44
57
|
export interface MlTaskTrainingSession extends Record<string, unknown> {
|
|
@@ -48,7 +61,17 @@ export interface MlTaskTrainCompletedResult extends Record<string, unknown> {
|
|
|
48
61
|
sessionId: string;
|
|
49
62
|
trainedModelIds: string[];
|
|
50
63
|
}
|
|
51
|
-
|
|
64
|
+
/** Result returned when the training wait budget was exhausted. */
|
|
65
|
+
export interface MlTaskTrainTimedOutResult extends Record<string, unknown> {
|
|
66
|
+
sessionId: string;
|
|
67
|
+
trainedModelIds: string[];
|
|
68
|
+
timedOut: true;
|
|
69
|
+
success: false;
|
|
70
|
+
state: "RUNNING";
|
|
71
|
+
elapsedMs: number;
|
|
72
|
+
pollCount: number;
|
|
73
|
+
}
|
|
74
|
+
export type MlTaskTrainResult = MlTaskTrainingSession | MlTaskTrainCompletedResult | MlTaskTrainTimedOutResult;
|
|
52
75
|
export interface MlTrainedModelDetails extends Record<string, unknown> {
|
|
53
76
|
id?: string;
|
|
54
77
|
algorithm?: string;
|
|
@@ -77,7 +100,19 @@ export declare class MlTasksResource extends BaseResource {
|
|
|
77
100
|
getSettings(analysisId: string, mlTaskId: string, projectKey?: string): Promise<MlTaskSettings>;
|
|
78
101
|
/** Save the full settings object of an ML task. */
|
|
79
102
|
saveSettings(analysisId: string, mlTaskId: string, settings: MlTaskSettings, projectKey?: string): Promise<MlTaskActionResult | undefined>;
|
|
80
|
-
/**
|
|
103
|
+
/**
|
|
104
|
+
* Start training, optionally waiting for the task to finish.
|
|
105
|
+
*
|
|
106
|
+
* `timeoutMs` is validated BEFORE the training POST: an invalid deadline
|
|
107
|
+
* never reaches the server, so bad input cannot start training work. Once
|
|
108
|
+
* DSS has accepted the training request, the deadline scopes only the
|
|
109
|
+
* wait phase — each status poll is issued with the remaining budget as a
|
|
110
|
+
* total request deadline, so a hung status request cannot outlast it.
|
|
111
|
+
*
|
|
112
|
+
* When the budget is exhausted the returned result keeps the documented
|
|
113
|
+
* success shape plus `timedOut: true` and `success: false` instead of
|
|
114
|
+
* throwing, so a caller can distinguish "still training" from "finished".
|
|
115
|
+
*/
|
|
81
116
|
train(opts: MlTaskTrainOptions): Promise<MlTaskTrainResult>;
|
|
82
117
|
/** List identifiers for every trained model currently present on the task. */
|
|
83
118
|
listTrainedModels(analysisId: string, mlTaskId: string, projectKey?: string): Promise<string[]>;
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ClientValidationError, } from "../errors.js";
|
|
2
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
3
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
3
4
|
const TRAIN_POLL_INTERVAL_MS = 5_000;
|
|
4
|
-
|
|
5
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
6
|
-
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
7
|
-
}
|
|
8
|
-
return value;
|
|
9
|
-
}
|
|
5
|
+
const TRAIN_DEFAULT_TIMEOUT_MS = 120_000;
|
|
10
6
|
function delay(ms) {
|
|
11
7
|
const { promise, resolve, } = Promise.withResolvers();
|
|
12
8
|
setTimeout(resolve, ms);
|
|
@@ -26,11 +22,11 @@ export class MlTasksResource extends BaseResource {
|
|
|
26
22
|
/** Create a prediction or clustering task in an existing visual analysis. */
|
|
27
23
|
async create(opts) {
|
|
28
24
|
if (opts.taskType !== "PREDICTION" && opts.taskType !== "CLUSTERING") {
|
|
29
|
-
throw new
|
|
25
|
+
throw new ClientValidationError("taskType must be PREDICTION or CLUSTERING.", "invalid_enum");
|
|
30
26
|
}
|
|
31
27
|
if (opts.taskType === "PREDICTION"
|
|
32
28
|
&& (typeof opts.targetVariable !== "string" || opts.targetVariable.trim().length === 0)) {
|
|
33
|
-
throw new
|
|
29
|
+
throw new ClientValidationError("targetVariable is required for PREDICTION ML tasks.", "missing_required_arg");
|
|
34
30
|
}
|
|
35
31
|
const analysisId = encodeURIComponent(requireNonEmpty(opts.analysisId, "analysisId"));
|
|
36
32
|
const backendType = opts.backendType ?? "PY_MEMORY";
|
|
@@ -65,17 +61,49 @@ export class MlTasksResource extends BaseResource {
|
|
|
65
61
|
/** Save the full settings object of an ML task. */
|
|
66
62
|
async saveSettings(analysisId, mlTaskId, settings, projectKey) {
|
|
67
63
|
if (settings === null || typeof settings !== "object" || Array.isArray(settings)) {
|
|
68
|
-
throw new
|
|
64
|
+
throw new ClientValidationError("settings must be an object.", "validation_failed");
|
|
69
65
|
}
|
|
70
66
|
return this.client.post(`${this.taskPath(analysisId, mlTaskId, projectKey)}/settings`, settings);
|
|
71
67
|
}
|
|
72
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Start training, optionally waiting for the task to finish.
|
|
70
|
+
*
|
|
71
|
+
* `timeoutMs` is validated BEFORE the training POST: an invalid deadline
|
|
72
|
+
* never reaches the server, so bad input cannot start training work. Once
|
|
73
|
+
* DSS has accepted the training request, the deadline scopes only the
|
|
74
|
+
* wait phase — each status poll is issued with the remaining budget as a
|
|
75
|
+
* total request deadline, so a hung status request cannot outlast it.
|
|
76
|
+
*
|
|
77
|
+
* When the budget is exhausted the returned result keeps the documented
|
|
78
|
+
* success shape plus `timedOut: true` and `success: false` instead of
|
|
79
|
+
* throwing, so a caller can distinguish "still training" from "finished".
|
|
80
|
+
*/
|
|
73
81
|
async train(opts) {
|
|
74
82
|
if (opts.sessionName !== undefined)
|
|
75
83
|
requireNonEmpty(opts.sessionName, "sessionName");
|
|
76
84
|
if (opts.sessionDescription !== undefined) {
|
|
77
85
|
requireNonEmpty(opts.sessionDescription, "sessionDescription");
|
|
78
86
|
}
|
|
87
|
+
// Deadline validation happens before any network request: a malformed
|
|
88
|
+
// budget must not start training work on the server.
|
|
89
|
+
const explicitIntervalMs = opts.pollIntervalMs;
|
|
90
|
+
if (explicitIntervalMs !== undefined && opts.wait !== true) {
|
|
91
|
+
throw new ClientValidationError("pollIntervalMs requires wait: true.", "validation_failed");
|
|
92
|
+
}
|
|
93
|
+
if (explicitIntervalMs !== undefined
|
|
94
|
+
&& (!Number.isFinite(explicitIntervalMs) || explicitIntervalMs <= 0)) {
|
|
95
|
+
throw new ClientValidationError("pollIntervalMs must be a finite positive number.", "validation_failed");
|
|
96
|
+
}
|
|
97
|
+
let timeoutMs;
|
|
98
|
+
if (opts.timeoutMs !== undefined) {
|
|
99
|
+
if (opts.wait !== true) {
|
|
100
|
+
throw new ClientValidationError("timeoutMs requires wait: true.", "validation_failed");
|
|
101
|
+
}
|
|
102
|
+
if (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs < 0) {
|
|
103
|
+
throw new ClientValidationError("timeoutMs must be a finite non-negative number.", "validation_failed");
|
|
104
|
+
}
|
|
105
|
+
timeoutMs = opts.timeoutMs;
|
|
106
|
+
}
|
|
79
107
|
const taskPath = this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
80
108
|
const session = await this.client.post(`${taskPath}/train`, {
|
|
81
109
|
sessionName: opts.sessionName,
|
|
@@ -84,15 +112,67 @@ export class MlTasksResource extends BaseResource {
|
|
|
84
112
|
});
|
|
85
113
|
if (opts.wait !== true)
|
|
86
114
|
return session;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
115
|
+
const deadlineMs = timeoutMs ?? TRAIN_DEFAULT_TIMEOUT_MS;
|
|
116
|
+
const baseIntervalMs = Math.max(1, explicitIntervalMs ?? TRAIN_POLL_INTERVAL_MS);
|
|
117
|
+
const adaptiveEnabled = explicitIntervalMs === undefined;
|
|
118
|
+
const startedAt = Date.now();
|
|
119
|
+
let pollCount = 0;
|
|
120
|
+
// One status observation always happens; the poll precedes the deadline
|
|
121
|
+
// check, so `timeoutMs: 0` still reports the freshly-started state once.
|
|
122
|
+
while (true) {
|
|
123
|
+
pollCount += 1;
|
|
124
|
+
// One status observation always happens, even when the budget is
|
|
125
|
+
// already spent (the poll precedes the deadline check); only the
|
|
126
|
+
// requests issued while budget remains are bounded by the remaining
|
|
127
|
+
// time, so a stalled status endpoint can never defeat the deadline.
|
|
128
|
+
const remainingMs = deadlineMs - (Date.now() - startedAt);
|
|
129
|
+
let status;
|
|
130
|
+
try {
|
|
131
|
+
status = await this.client.get(`${this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey)}/status`, remainingMs > 0
|
|
132
|
+
? { timeoutMs: remainingMs, }
|
|
133
|
+
: undefined);
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
// The poll budget ran out: report the structured timeout instead
|
|
137
|
+
// of letting the transport deadline error escape the loop.
|
|
138
|
+
if (!isRequestDeadlineError(error, startedAt + deadlineMs))
|
|
139
|
+
throw error;
|
|
140
|
+
return {
|
|
141
|
+
sessionId: session.sessionId,
|
|
142
|
+
trainedModelIds: [],
|
|
143
|
+
timedOut: true,
|
|
144
|
+
success: false,
|
|
145
|
+
state: "RUNNING",
|
|
146
|
+
elapsedMs: Date.now() - startedAt,
|
|
147
|
+
pollCount,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
const elapsedMs = Date.now() - startedAt;
|
|
151
|
+
if (status.training !== false) {
|
|
152
|
+
if (elapsedMs >= deadlineMs) {
|
|
153
|
+
return {
|
|
154
|
+
sessionId: session.sessionId,
|
|
155
|
+
trainedModelIds: modelIdsFromStatus(status),
|
|
156
|
+
timedOut: true,
|
|
157
|
+
success: false,
|
|
158
|
+
state: "RUNNING",
|
|
159
|
+
elapsedMs,
|
|
160
|
+
pollCount,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
164
|
+
pollCount,
|
|
165
|
+
baseIntervalMs,
|
|
166
|
+
adaptiveEnabled,
|
|
167
|
+
});
|
|
168
|
+
await delay(Math.min(nextDelayMs, deadlineMs - elapsedMs));
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
sessionId: session.sessionId,
|
|
173
|
+
trainedModelIds: modelIdsFromStatus(status),
|
|
174
|
+
};
|
|
91
175
|
}
|
|
92
|
-
return {
|
|
93
|
-
sessionId: session.sessionId,
|
|
94
|
-
trainedModelIds: modelIdsFromStatus(status),
|
|
95
|
-
};
|
|
96
176
|
}
|
|
97
177
|
/** List identifiers for every trained model currently present on the task. */
|
|
98
178
|
async listTrainedModels(analysisId, mlTaskId, projectKey) {
|
|
@@ -1,11 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { BaseResource, } from "./base.js";
|
|
3
|
-
function requireNonEmpty(value, name) {
|
|
4
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
5
|
-
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
6
|
-
}
|
|
7
|
-
return value;
|
|
8
|
-
}
|
|
1
|
+
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
9
2
|
export class ModelEvaluationStoresResource extends BaseResource {
|
|
10
3
|
/** List model evaluation stores in a project. */
|
|
11
4
|
async list(projectKey) {
|