dataiku-sdk 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -46
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +102 -0
- package/dist/packages/types/src/index.js +76 -0
- package/dist/src/cli/commands/app.js +57 -15
- package/dist/src/cli/commands/bundle.js +1 -1
- package/dist/src/cli/commands/code-env.js +13 -7
- package/dist/src/cli/commands/code.js +1 -1
- package/dist/src/cli/commands/connection.d.ts +9 -0
- package/dist/src/cli/commands/connection.js +267 -1
- package/dist/src/cli/commands/data-collection.d.ts +2 -0
- package/dist/src/cli/commands/data-collection.js +164 -0
- package/dist/src/cli/commands/dataset.js +162 -33
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/group.d.ts +2 -0
- package/dist/src/cli/commands/group.js +109 -0
- package/dist/src/cli/commands/index.js +151 -78
- package/dist/src/cli/commands/job.js +4 -2
- package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
- package/dist/src/cli/commands/knowledge-bank.js +80 -0
- package/dist/src/cli/commands/llm.d.ts +2 -0
- package/dist/src/cli/commands/llm.js +93 -0
- package/dist/src/cli/commands/macro.d.ts +2 -0
- package/dist/src/cli/commands/macro.js +221 -0
- package/dist/src/cli/commands/ml-task.js +18 -6
- package/dist/src/cli/commands/notebook.js +6 -2
- package/dist/src/cli/commands/plugin.d.ts +2 -0
- package/dist/src/cli/commands/plugin.js +799 -0
- package/dist/src/cli/commands/project-folder.d.ts +2 -0
- package/dist/src/cli/commands/project-folder.js +197 -0
- package/dist/src/cli/commands/project-git.js +15 -55
- package/dist/src/cli/commands/project.js +52 -6
- package/dist/src/cli/commands/recipe.js +36 -1
- package/dist/src/cli/commands/saved-model.d.ts +5 -0
- package/dist/src/cli/commands/saved-model.js +388 -1
- package/dist/src/cli/commands/scenario.js +139 -2
- package/dist/src/cli/commands/user.d.ts +8 -0
- package/dist/src/cli/commands/user.js +296 -0
- package/dist/src/cli/contract.d.ts +1 -2
- package/dist/src/cli/contract.js +1098 -297
- package/dist/src/cli/flags.js +30 -1
- package/dist/src/cli/helpers/app-successor.d.ts +19 -4
- package/dist/src/cli/helpers/app-successor.js +90 -30
- package/dist/src/cli/helpers/cleanup.js +3 -1
- package/dist/src/cli/helpers/flow-zone.js +1 -2
- package/dist/src/cli/output-schemas.d.ts +24 -0
- package/dist/src/cli/output-schemas.js +233 -0
- package/dist/src/cli/output.d.ts +1 -1
- package/dist/src/cli/output.js +23 -10
- package/dist/src/cli/usage.d.ts +1 -0
- package/dist/src/cli/usage.js +47 -1
- package/dist/src/cli.js +147 -41
- package/dist/src/client.d.ts +92 -5
- package/dist/src/client.js +202 -31
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +8 -4
- package/dist/src/generated/action-output-schemas.json +1735 -0
- package/dist/src/index.d.ts +16 -7
- package/dist/src/index.js +11 -2
- package/dist/src/resources/analyses.js +1 -8
- package/dist/src/resources/applications.d.ts +8 -6
- package/dist/src/resources/applications.js +21 -21
- package/dist/src/resources/base.d.ts +4 -0
- package/dist/src/resources/base.js +15 -0
- package/dist/src/resources/bundles.d.ts +9 -4
- package/dist/src/resources/bundles.js +10 -5
- package/dist/src/resources/connections.d.ts +78 -1
- package/dist/src/resources/connections.js +132 -1
- package/dist/src/resources/data-collections.d.ts +118 -0
- package/dist/src/resources/data-collections.js +49 -0
- package/dist/src/resources/datasets.d.ts +69 -0
- package/dist/src/resources/datasets.js +136 -5
- package/dist/src/resources/flow-zones.d.ts +6 -2
- package/dist/src/resources/flow-zones.js +6 -3
- package/dist/src/resources/futures.d.ts +6 -1
- package/dist/src/resources/futures.js +68 -22
- package/dist/src/resources/groups.d.ts +47 -0
- package/dist/src/resources/groups.js +56 -0
- package/dist/src/resources/jobs.d.ts +8 -13
- package/dist/src/resources/jobs.js +71 -34
- package/dist/src/resources/knowledge-banks.d.ts +58 -0
- package/dist/src/resources/knowledge-banks.js +61 -0
- package/dist/src/resources/llms.d.ts +92 -0
- package/dist/src/resources/llms.js +62 -0
- package/dist/src/resources/macros.d.ts +121 -0
- package/dist/src/resources/macros.js +231 -0
- package/dist/src/resources/ml-tasks.d.ts +37 -2
- package/dist/src/resources/ml-tasks.js +109 -20
- package/dist/src/resources/model-evaluation-stores.js +1 -8
- package/dist/src/resources/notebooks.d.ts +43 -7
- package/dist/src/resources/notebooks.js +53 -13
- package/dist/src/resources/plugins.d.ts +263 -0
- package/dist/src/resources/plugins.js +455 -0
- package/dist/src/resources/project-folders.d.ts +71 -0
- package/dist/src/resources/project-folders.js +57 -0
- package/dist/src/resources/project-git.d.ts +12 -12
- package/dist/src/resources/project-git.js +84 -82
- package/dist/src/resources/project-library.d.ts +0 -1
- package/dist/src/resources/project-library.js +1 -17
- package/dist/src/resources/projects.d.ts +23 -3
- package/dist/src/resources/projects.js +60 -34
- package/dist/src/resources/recipes.d.ts +15 -1
- package/dist/src/resources/recipes.js +73 -6
- package/dist/src/resources/saved-models.d.ts +134 -1
- package/dist/src/resources/saved-models.js +214 -6
- package/dist/src/resources/scenarios.d.ts +105 -0
- package/dist/src/resources/scenarios.js +232 -106
- package/dist/src/resources/sql.js +5 -1
- package/dist/src/resources/users.d.ts +129 -0
- package/dist/src/resources/users.js +128 -0
- package/dist/src/resources/webapps.d.ts +15 -2
- package/dist/src/resources/webapps.js +18 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/deep-merge.d.ts +1 -1
- package/dist/src/utils/git-reference.d.ts +12 -0
- package/dist/src/utils/git-reference.js +38 -0
- package/dist/src/utils/polling.d.ts +20 -0
- package/dist/src/utils/polling.js +27 -0
- package/dist/src/utils/response-file.js +1 -1
- package/dist/src/utils/secret-sanitize.d.ts +54 -0
- package/dist/src/utils/secret-sanitize.js +101 -0
- package/package.json +9 -8
- package/packages/types/dist/index.d.ts +102 -0
- package/packages/types/dist/index.js +76 -0
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +11 -7
- package/skills/dataiku-dss/references/app-releases.md +8 -8
- package/skills/dataiku-dss/references/authentication.md +6 -6
- package/skills/dataiku-dss/references/discovery.md +4 -4
- package/skills/dataiku-dss/references/mutations.md +1 -1
- package/skills/dataiku-dss/references/troubleshooting.md +2 -2
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import { ClientValidationError,
|
|
1
|
+
import { ClientValidationError, } from "../errors.js";
|
|
2
2
|
import { ProjectGitActionResultSchema, ProjectGitDiffResultSchema, ProjectGitFutureResponseSchema, ProjectGitFutureStateSchema, ProjectGitLibrariesSchema, ProjectGitLogResultSchema, ProjectGitRemoteSchema, ProjectGitStatusSchema, ProjectGitTagsSchema, } from "../schemas.js";
|
|
3
|
+
import { encodeGitReferencePath, validateGitReferencePath, } from "../utils/git-reference.js";
|
|
4
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
5
|
+
import { sanitizeErrorSecrets, } from "../utils/secret-sanitize.js";
|
|
3
6
|
import { BaseResource, } from "./base.js";
|
|
4
7
|
/**
|
|
5
8
|
* The per-project Git routes exist only under the dataikuapi base, not under
|
|
@@ -17,43 +20,6 @@ function sleep(ms) {
|
|
|
17
20
|
setTimeout(resolve, ms);
|
|
18
21
|
return promise;
|
|
19
22
|
}
|
|
20
|
-
function gitReferencePathError(path, message) {
|
|
21
|
-
return new ClientValidationError(message, "validation_failed", "Use a Git library reference path relative to the project library root: '/' -separated segment names, with no '.' or '..' segments.", { path, });
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Validate and canonicalize a Git library reference path. This is the single
|
|
25
|
-
* validator used by both SDK dispatch and CLI mutation planning.
|
|
26
|
-
*
|
|
27
|
-
* Strip only the permitted outer '/' separators, then require nonempty
|
|
28
|
-
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
29
|
-
* path therefore cannot change which API resource a dispatcher's URL
|
|
30
|
-
* normalization would otherwise select.
|
|
31
|
-
*/
|
|
32
|
-
export function validateGitReferencePath(path) {
|
|
33
|
-
if (typeof path !== "string" || path.trim() === "") {
|
|
34
|
-
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
35
|
-
}
|
|
36
|
-
const normalized = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
37
|
-
if (normalized === "") {
|
|
38
|
-
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
39
|
-
}
|
|
40
|
-
for (const segment of normalized.split("/")) {
|
|
41
|
-
if (segment === "") {
|
|
42
|
-
throw gitReferencePathError(path, "A Git library reference path must not contain empty segments (consecutive or repeated '/').");
|
|
43
|
-
}
|
|
44
|
-
if (segment === "." || segment === "..") {
|
|
45
|
-
throw gitReferencePathError(path, "A Git library reference path must not contain '.' or '..' segments.");
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
return normalized;
|
|
49
|
-
}
|
|
50
|
-
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
51
|
-
export function encodeGitReferencePath(path) {
|
|
52
|
-
return validateGitReferencePath(path)
|
|
53
|
-
.split("/")
|
|
54
|
-
.map((segment) => encodeURIComponent(segment))
|
|
55
|
-
.join("/");
|
|
56
|
-
}
|
|
57
23
|
/**
|
|
58
24
|
* Reject `https://user:token@host/repo` before it reaches DSS: an embedded
|
|
59
25
|
* secret would be stored in the project's Git configuration and echoed back by
|
|
@@ -89,28 +55,10 @@ function validatedRepository(repository, field) {
|
|
|
89
55
|
return candidate;
|
|
90
56
|
}
|
|
91
57
|
/** Replace every exact occurrence of a secret so echoed errors cannot leak it. */
|
|
92
|
-
function redactSecrets(text, secrets) {
|
|
93
|
-
let result = text;
|
|
94
|
-
for (const secret of secrets) {
|
|
95
|
-
if (secret !== undefined && secret !== "") {
|
|
96
|
-
result = result.split(secret).join("[redacted]");
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
return result;
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* Strip a password that a server misconfiguration may have echoed from an
|
|
103
|
-
* error response. `DataikuError.message` was built at construction, so the
|
|
104
|
-
* raw body, the message, and the stack all need scrubbing.
|
|
105
|
-
*/
|
|
106
58
|
function scrubErrorSecrets(error, secrets) {
|
|
107
|
-
if (
|
|
59
|
+
if (secrets.length === 0)
|
|
108
60
|
return;
|
|
109
|
-
error
|
|
110
|
-
error.message = redactSecrets(error.message, secrets);
|
|
111
|
-
if (typeof error.stack === "string") {
|
|
112
|
-
error.stack = redactSecrets(error.stack, secrets);
|
|
113
|
-
}
|
|
61
|
+
sanitizeErrorSecrets(error, { sensitiveKeys: {}, secrets, });
|
|
114
62
|
}
|
|
115
63
|
function describeFutureFailure(error) {
|
|
116
64
|
if (error === undefined || error === null)
|
|
@@ -129,6 +77,31 @@ function describeFutureFailure(error) {
|
|
|
129
77
|
return String(error);
|
|
130
78
|
}
|
|
131
79
|
/* ------------------------------------------------------------------ */
|
|
80
|
+
/**
|
|
81
|
+
* DSS answers the lib-git-refs listing as an object map of attached external
|
|
82
|
+
* libraries keyed by target path alongside code-library metadata (observed
|
|
83
|
+
* raw on DSS 15: `{gitReferences, pythonPath, rsrcPath,
|
|
84
|
+
* importLibrariesFromProjects}`), while the official Python client and this
|
|
85
|
+
* resource's typed contract expose a plain array. Normalize the map into
|
|
86
|
+
* entries, preserving each key as the library's `localTargetPath`; a bare
|
|
87
|
+
* array passes through untouched.
|
|
88
|
+
*/
|
|
89
|
+
function normalizeLibraryList(raw) {
|
|
90
|
+
if (Array.isArray(raw) || raw === undefined || raw === null)
|
|
91
|
+
return raw ?? [];
|
|
92
|
+
if (typeof raw !== "object")
|
|
93
|
+
return raw;
|
|
94
|
+
const record = raw;
|
|
95
|
+
const refs = record["gitReferences"];
|
|
96
|
+
if (refs === undefined || refs === null || typeof refs !== "object" || Array.isArray(refs)) {
|
|
97
|
+
return raw;
|
|
98
|
+
}
|
|
99
|
+
return Object.entries(refs).map(([localTargetPath, spec,]) => ({
|
|
100
|
+
localTargetPath,
|
|
101
|
+
...(typeof spec === "object" && spec !== null ? spec : {}),
|
|
102
|
+
}));
|
|
103
|
+
}
|
|
104
|
+
/* ------------------------------------------------------------------ */
|
|
132
105
|
/* Resource */
|
|
133
106
|
/* ------------------------------------------------------------------ */
|
|
134
107
|
export class ProjectGitResource extends BaseResource {
|
|
@@ -287,7 +260,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
287
260
|
/* ---- external libraries (async futures) ---- */
|
|
288
261
|
async listLibraries(projectKey) {
|
|
289
262
|
const raw = await this.request("GET", `${this.libraryPath(projectKey)}/`);
|
|
290
|
-
return this.client.safeParse(ProjectGitLibrariesSchema, raw
|
|
263
|
+
return this.client.safeParse(ProjectGitLibrariesSchema, normalizeLibraryList(raw), "projectGit.listLibraries");
|
|
291
264
|
}
|
|
292
265
|
/** Attach an external Git library. Runs as a DSS future. */
|
|
293
266
|
async addLibrary(projectKey, options) {
|
|
@@ -355,7 +328,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
355
328
|
* what polling loops want; the default consumes it like the Python client.
|
|
356
329
|
*/
|
|
357
330
|
async getFutureState(jobId, options = {}) {
|
|
358
|
-
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}
|
|
331
|
+
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`, undefined, [], { timeoutMs: options.timeoutMs, noRetry: options.noRetry, });
|
|
359
332
|
return this.client.safeParse(ProjectGitFutureStateSchema, raw ?? {}, "projectGit.getFutureState");
|
|
360
333
|
}
|
|
361
334
|
/** Abort a running future. */
|
|
@@ -376,10 +349,46 @@ export class ProjectGitResource extends BaseResource {
|
|
|
376
349
|
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
377
350
|
throw new ClientValidationError("timeoutMs must be a finite non-negative number.");
|
|
378
351
|
}
|
|
352
|
+
// Adaptive backoff applies only when the caller did not pin an exact
|
|
353
|
+
// interval; an explicit pollIntervalMs is an exact contract.
|
|
354
|
+
const adaptiveEnabled = options.pollIntervalMs === undefined;
|
|
355
|
+
let pollCount = 0;
|
|
356
|
+
let lastState;
|
|
379
357
|
const startedAt = Date.now();
|
|
380
358
|
while (true) {
|
|
359
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
360
|
+
// The first observation always happens, even when the budget is
|
|
361
|
+
// already spent. A later poll is never started after the deadline:
|
|
362
|
+
// the loop reports the documented timeout from the last observed
|
|
363
|
+
// state instead of issuing a request the budget cannot cover.
|
|
364
|
+
if (lastState !== undefined && elapsedBeforeMs >= timeoutMs) {
|
|
365
|
+
throw new Error(`Timed out after ${String(elapsedBeforeMs)}ms waiting for Dataiku future ${jobId}`);
|
|
366
|
+
}
|
|
381
367
|
// Unlike peek state, the ordinary state includes the completed result.
|
|
382
|
-
|
|
368
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
369
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
370
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
371
|
+
// reaches the server instead of failing before any attempt.
|
|
372
|
+
const remainingMs = timeoutMs - elapsedBeforeMs;
|
|
373
|
+
pollCount += 1;
|
|
374
|
+
let state;
|
|
375
|
+
try {
|
|
376
|
+
state = await this.getFutureState(jobId, remainingMs > 0
|
|
377
|
+
? {
|
|
378
|
+
peek: false,
|
|
379
|
+
timeoutMs: remainingMs,
|
|
380
|
+
}
|
|
381
|
+
: { peek: false, noRetry: true, });
|
|
382
|
+
}
|
|
383
|
+
catch (error) {
|
|
384
|
+
// The poll budget ran out mid-request: report the documented
|
|
385
|
+
// timeout failure instead of letting the transport deadline
|
|
386
|
+
// error escape.
|
|
387
|
+
if (!isRequestDeadlineError(error, startedAt + timeoutMs))
|
|
388
|
+
throw error;
|
|
389
|
+
throw new Error(`Timed out after ${String(Date.now() - startedAt)}ms waiting for Dataiku future ${jobId}`, { cause: error, });
|
|
390
|
+
}
|
|
391
|
+
lastState = state;
|
|
383
392
|
const failure = describeFutureFailure(state.error);
|
|
384
393
|
if (failure !== undefined) {
|
|
385
394
|
throw new Error(`Dataiku future ${jobId} failed: ${failure}`);
|
|
@@ -395,13 +404,19 @@ export class ProjectGitResource extends BaseResource {
|
|
|
395
404
|
if (state.alive === false) {
|
|
396
405
|
throw new Error(`Dataiku future ${jobId} ended without producing a result`);
|
|
397
406
|
}
|
|
398
|
-
// Only the remaining budget is slept, so a caller's deadline is never
|
|
399
|
-
// overshot by a whole poll interval.
|
|
400
407
|
const elapsedMs = Date.now() - startedAt;
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
408
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
409
|
+
// timeout from this observation without issuing another request.
|
|
410
|
+
if (elapsedMs >= timeoutMs)
|
|
411
|
+
continue;
|
|
412
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
413
|
+
pollCount,
|
|
414
|
+
baseIntervalMs: pollIntervalMs,
|
|
415
|
+
adaptiveEnabled,
|
|
416
|
+
});
|
|
417
|
+
// Only the remaining budget is slept, so a caller's deadline is never
|
|
418
|
+
// overshot by a whole interval.
|
|
419
|
+
await sleep(Math.min(nextDelayMs, timeoutMs - elapsedMs));
|
|
405
420
|
}
|
|
406
421
|
}
|
|
407
422
|
/* ---- private ---- */
|
|
@@ -424,22 +439,9 @@ export class ProjectGitResource extends BaseResource {
|
|
|
424
439
|
const queryString = search.toString();
|
|
425
440
|
return queryString === "" ? "" : `?${queryString}`;
|
|
426
441
|
}
|
|
427
|
-
async request(method, path, body, secrets = []) {
|
|
428
|
-
const transport = this.client;
|
|
429
|
-
const init = {
|
|
430
|
-
method,
|
|
431
|
-
headers: {
|
|
432
|
-
// The Git routes are only reachable with dataikuapi-style Basic auth:
|
|
433
|
-
// API key as the username, empty password.
|
|
434
|
-
Authorization: `Basic ${Buffer.from(`${transport.apiKey}:`, "utf8").toString("base64")}`,
|
|
435
|
-
Accept: "application/json",
|
|
436
|
-
"Content-Type": "application/json",
|
|
437
|
-
},
|
|
438
|
-
};
|
|
439
|
-
if (body !== undefined)
|
|
440
|
-
init.body = JSON.stringify(body);
|
|
442
|
+
async request(method, path, body, secrets = [], opts = {}) {
|
|
441
443
|
try {
|
|
442
|
-
const res = await
|
|
444
|
+
const res = await this.client.requestGit(method, `${GIT_API_BASE}${path}`, body, opts.timeoutMs === undefined ? undefined : Date.now() + opts.timeoutMs, opts.noRetry === true);
|
|
443
445
|
const text = await res.text();
|
|
444
446
|
if (text.trim() === "")
|
|
445
447
|
return undefined;
|
|
@@ -147,7 +147,6 @@ export declare class ProjectLibraryResource extends BaseResource {
|
|
|
147
147
|
/** Move a project code-library file or folder into another library folder. */
|
|
148
148
|
move(path: string, destinationFolderPath: string, projectKey?: string): Promise<void>;
|
|
149
149
|
private contentsPath;
|
|
150
|
-
private postRawBody;
|
|
151
150
|
private alreadyExistsError;
|
|
152
151
|
private assertLibraryFileAbsent;
|
|
153
152
|
}
|
|
@@ -403,7 +403,7 @@ export class ProjectLibraryResource extends BaseResource {
|
|
|
403
403
|
}
|
|
404
404
|
const bytes = typeof content === "string" ? Buffer.from(content, "utf8") : content;
|
|
405
405
|
const sha256 = sha256Hex(bytes);
|
|
406
|
-
await this.postRawBody(this.contentsPath(valid, projectKey), content);
|
|
406
|
+
await this.client.postRawBody(this.contentsPath(valid, projectKey), content);
|
|
407
407
|
return {
|
|
408
408
|
path: valid,
|
|
409
409
|
bytes: bytes.length,
|
|
@@ -518,22 +518,6 @@ export class ProjectLibraryResource extends BaseResource {
|
|
|
518
518
|
const projectKeyPart = this.enc(projectKey);
|
|
519
519
|
return `/public/api/projects/${projectKeyPart}/libraries/contents/${encodeLibraryPath(path)}`;
|
|
520
520
|
}
|
|
521
|
-
async postRawBody(path, body) {
|
|
522
|
-
// DSS library writes use dataikuapi's raw_body; route through the client's
|
|
523
|
-
// transport so auth, retries, TLS options, and DSS error handling stay aligned.
|
|
524
|
-
// BodyInit rejects plain Uint8Array views, so non-Buffer bytes are copied
|
|
525
|
-
// into a Buffer (an ArrayBufferView BodyInit accepts).
|
|
526
|
-
const payload = body instanceof Uint8Array && !(body instanceof Buffer)
|
|
527
|
-
? Buffer.from(body.buffer, body.byteOffset, body.byteLength)
|
|
528
|
-
: body;
|
|
529
|
-
const rawClient = this.client;
|
|
530
|
-
const res = await rawClient.fetchWithRetry(`${rawClient.baseUrl}${path}`, {
|
|
531
|
-
method: "POST",
|
|
532
|
-
headers: rawClient.getAnyHeaders(),
|
|
533
|
-
body: payload,
|
|
534
|
-
});
|
|
535
|
-
await res.text();
|
|
536
|
-
}
|
|
537
521
|
alreadyExistsError(path) {
|
|
538
522
|
return new ClientValidationError(`Project library item "${path}" already exists.`, "validation_failed", "Refuse to overwrite an existing item; delete it first, or use the create-or-replace update to replace an existing file.", { path, });
|
|
539
523
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FlowMapOptions, ProjectDetails, ProjectMetadata, ProjectSummary } from "../schemas.js";
|
|
1
|
+
import type { FlowMapOptions, ProjectDetails, ProjectMetadata, ProjectSummary, ProjectTags } from "../schemas.js";
|
|
2
2
|
import type { AnalyzedFlowMap, FlowMapRendering } from "../utils/flow-analysis.js";
|
|
3
3
|
import { BaseResource } from "./base.js";
|
|
4
4
|
interface FlowMapTruncationSummary {
|
|
@@ -124,6 +124,24 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
124
124
|
get(projectKey?: string): Promise<ProjectDetails>;
|
|
125
125
|
/** Get metadata (tags, custom fields, checklists) for a project. */
|
|
126
126
|
metadata(projectKey?: string): Promise<ProjectMetadata>;
|
|
127
|
+
/**
|
|
128
|
+
* Replace project metadata with the supplied full object.
|
|
129
|
+
*
|
|
130
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
131
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
132
|
+
* absent from the payload are removed from the project.
|
|
133
|
+
*/
|
|
134
|
+
setMetadata(projectKey: string | undefined, metadata: ProjectMetadata): Promise<void>;
|
|
135
|
+
/** Get project-level tags (tag name → {color?}). */
|
|
136
|
+
tags(projectKey?: string): Promise<ProjectTags>;
|
|
137
|
+
/**
|
|
138
|
+
* Replace project-level tags with the supplied full object.
|
|
139
|
+
*
|
|
140
|
+
* DSS PUT semantics replace the whole tags dictionary: pass a tags object
|
|
141
|
+
* obtained from {@link tags}, edit it, and send it back — tags absent from
|
|
142
|
+
* the payload are removed from the project.
|
|
143
|
+
*/
|
|
144
|
+
setTags(projectKey: string | undefined, tags: ProjectTags): Promise<void>;
|
|
127
145
|
/** Get the raw flow graph for a project. */
|
|
128
146
|
flow(projectKey?: string): Promise<unknown>;
|
|
129
147
|
/** Compute a stable fingerprint of flow nodes and edges, excluding visual metadata. */
|
|
@@ -132,8 +150,10 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
132
150
|
* Build a normalized, optionally truncated flow map for a project.
|
|
133
151
|
*
|
|
134
152
|
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
135
|
-
* managed folders) in parallel.
|
|
136
|
-
*
|
|
153
|
+
* managed folders) in parallel. Each metadata fetch runs under a total
|
|
154
|
+
* `timeoutMs` budget: when the budget expires the in-flight request is
|
|
155
|
+
* aborted and the map is returned without that metadata, reported as a
|
|
156
|
+
* warning.
|
|
137
157
|
*/
|
|
138
158
|
map(opts?: FlowMapOptions & {
|
|
139
159
|
projectKey?: string;
|
|
@@ -1,39 +1,37 @@
|
|
|
1
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
+
import { isRequestDeadlineError, } from "../utils/polling.js";
|
|
2
3
|
import { writeResponseToFile, } from "../utils/response-file.js";
|
|
3
|
-
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, } from "../schemas.js";
|
|
4
|
+
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, ProjectTagsSchema, } from "../schemas.js";
|
|
4
5
|
import { analyzeFlowMap, flowTopologyFingerprint, renderFlowMap, } from "../utils/flow-analysis.js";
|
|
5
6
|
import { normalizeFlowGraph, } from "../utils/flow-map.js";
|
|
6
7
|
import { assertImportableProjectArchive, } from "../utils/project-archive.js";
|
|
7
8
|
import { projectIncarnationHash, } from "../utils/project-incarnation.js";
|
|
8
9
|
import { BaseResource, } from "./base.js";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Runs an optional metadata fetch under a total budget. The fetcher receives
|
|
12
|
+
* the client GET options and must pass `timeoutMs` through to its request, so
|
|
13
|
+
* the client owns the deadline: the in-flight request is aborted when the
|
|
14
|
+
* budget expires and no further retry is started afterwards. Expiry of the
|
|
15
|
+
* total budget resolves to a warning so the map still returns; an earlier
|
|
16
|
+
* per-request failure is reported as "unavailable" (classified with the same
|
|
17
|
+
* deadline check the wait loops use).
|
|
18
|
+
*/
|
|
19
|
+
async function fetchWithTimeout(label, timeoutMs, fetcher) {
|
|
20
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
21
|
+
try {
|
|
22
|
+
return { value: await fetcher({ timeoutMs, }), };
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
if (isRequestDeadlineError(error, deadlineAt)) {
|
|
26
|
+
return {
|
|
17
27
|
warning: `${label} metadata timed out after ${timeoutMs}ms; continuing without it.`,
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
resolve({ value, });
|
|
26
|
-
}, (error) => {
|
|
27
|
-
if (settled)
|
|
28
|
-
return;
|
|
29
|
-
settled = true;
|
|
30
|
-
clearTimeout(timer);
|
|
31
|
-
const detail = error instanceof Error ? error.message : String(error);
|
|
32
|
-
resolve({
|
|
33
|
-
warning: `${label} metadata unavailable: ${detail}`,
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
});
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
31
|
+
return {
|
|
32
|
+
warning: `${label} metadata unavailable: ${detail}`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
37
35
|
}
|
|
38
36
|
function computeRootsAndLeaves(nodes, edges) {
|
|
39
37
|
const inDegree = new Map();
|
|
@@ -302,6 +300,32 @@ export class ProjectsResource extends BaseResource {
|
|
|
302
300
|
const raw = await this.client.get(`/public/api/projects/${enc}/metadata`);
|
|
303
301
|
return this.client.safeParse(ProjectMetadataSchema, raw, "projects.metadata");
|
|
304
302
|
}
|
|
303
|
+
/**
|
|
304
|
+
* Replace project metadata with the supplied full object.
|
|
305
|
+
*
|
|
306
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
307
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
308
|
+
* absent from the payload are removed from the project.
|
|
309
|
+
*/
|
|
310
|
+
async setMetadata(projectKey, metadata) {
|
|
311
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/metadata`, metadata);
|
|
312
|
+
}
|
|
313
|
+
/** Get project-level tags (tag name → {color?}). */
|
|
314
|
+
async tags(projectKey) {
|
|
315
|
+
const enc = this.enc(projectKey);
|
|
316
|
+
const raw = await this.client.get(`/public/api/projects/${enc}/tags`);
|
|
317
|
+
return this.client.safeParse(ProjectTagsSchema, raw, "projects.tags");
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Replace project-level tags with the supplied full object.
|
|
321
|
+
*
|
|
322
|
+
* DSS PUT semantics replace the whole tags dictionary: pass a tags object
|
|
323
|
+
* obtained from {@link tags}, edit it, and send it back — tags absent from
|
|
324
|
+
* the payload are removed from the project.
|
|
325
|
+
*/
|
|
326
|
+
async setTags(projectKey, tags) {
|
|
327
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/tags`, tags);
|
|
328
|
+
}
|
|
305
329
|
/** Get the raw flow graph for a project. */
|
|
306
330
|
async flow(projectKey) {
|
|
307
331
|
const enc = this.enc(projectKey);
|
|
@@ -317,8 +341,10 @@ export class ProjectsResource extends BaseResource {
|
|
|
317
341
|
* Build a normalized, optionally truncated flow map for a project.
|
|
318
342
|
*
|
|
319
343
|
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
320
|
-
* managed folders) in parallel.
|
|
321
|
-
*
|
|
344
|
+
* managed folders) in parallel. Each metadata fetch runs under a total
|
|
345
|
+
* `timeoutMs` budget: when the budget expires the in-flight request is
|
|
346
|
+
* aborted and the map is returned without that metadata, reported as a
|
|
347
|
+
* warning.
|
|
322
348
|
*/
|
|
323
349
|
async map(opts) {
|
|
324
350
|
const enc = this.enc(opts?.projectKey);
|
|
@@ -326,10 +352,10 @@ export class ProjectsResource extends BaseResource {
|
|
|
326
352
|
const timeoutMs = DEFAULT_METADATA_TIMEOUT_MS;
|
|
327
353
|
const [rawGraph, foldersMeta, datasetsMeta, recipesMeta, zonesMeta,] = await Promise.all([
|
|
328
354
|
this.client.get(`/public/api/projects/${enc}/flow/graph/`),
|
|
329
|
-
fetchWithTimeout("Managed folders", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/managedfolders
|
|
330
|
-
fetchWithTimeout("Datasets", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/datasets
|
|
331
|
-
fetchWithTimeout("Recipes", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/recipes
|
|
332
|
-
fetchWithTimeout("Flow zones", timeoutMs, () => this.client.flowZones.list(pk)),
|
|
355
|
+
fetchWithTimeout("Managed folders", timeoutMs, (getOptions) => this.client.get(`/public/api/projects/${enc}/managedfolders/`, getOptions)),
|
|
356
|
+
fetchWithTimeout("Datasets", timeoutMs, (getOptions) => this.client.get(`/public/api/projects/${enc}/datasets/`, getOptions)),
|
|
357
|
+
fetchWithTimeout("Recipes", timeoutMs, (getOptions) => this.client.get(`/public/api/projects/${enc}/recipes/`, getOptions)),
|
|
358
|
+
fetchWithTimeout("Flow zones", timeoutMs, (getOptions) => this.client.flowZones.list(pk, getOptions)),
|
|
333
359
|
]);
|
|
334
360
|
// Build folder name lookup
|
|
335
361
|
const folderNamesById = {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BuildMode, JobWaitResult, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
1
|
+
import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
3
|
import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
|
|
4
4
|
export interface RecipeRunOutput extends JobBuildTarget {
|
|
@@ -68,6 +68,20 @@ export interface RecipeCloneResult {
|
|
|
68
68
|
export declare class RecipesResource extends BaseResource {
|
|
69
69
|
/** List all recipes in a project. */
|
|
70
70
|
list(projectKey?: string): Promise<RecipeSummary[]>;
|
|
71
|
+
/** Get recipe metadata (label, tags, description, custom fields). */
|
|
72
|
+
metadata(recipeName: string, opts?: {
|
|
73
|
+
projectKey?: string;
|
|
74
|
+
}): Promise<ProjectMetadata>;
|
|
75
|
+
/**
|
|
76
|
+
* Replace recipe metadata with the supplied full object.
|
|
77
|
+
*
|
|
78
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
79
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
80
|
+
* absent from the payload are removed from the recipe.
|
|
81
|
+
*/
|
|
82
|
+
setMetadata(recipeName: string, metadata: ProjectMetadata, opts?: {
|
|
83
|
+
projectKey?: string;
|
|
84
|
+
}): Promise<void>;
|
|
71
85
|
/**
|
|
72
86
|
* Get a recipe definition (and optionally its payload).
|
|
73
87
|
* Returns the raw API response shape: `{ recipe, payload }`.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { writeFile, } from "node:fs/promises";
|
|
2
2
|
import { resolve, } from "node:path";
|
|
3
3
|
import { DataikuError, } from "../errors.js";
|
|
4
|
-
import { RecipeSummaryArraySchema, } from "../schemas.js";
|
|
4
|
+
import { ProjectMetadataSchema, RecipeSummaryArraySchema, } from "../schemas.js";
|
|
5
5
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
6
6
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
7
7
|
import { BaseResource, } from "./base.js";
|
|
@@ -95,6 +95,8 @@ function normalizeRecipeOutputType(value) {
|
|
|
95
95
|
return "DATASET";
|
|
96
96
|
if (normalized === "MANAGED_FOLDER" || normalized === "FOLDER")
|
|
97
97
|
return "MANAGED_FOLDER";
|
|
98
|
+
if (normalized === "MODEL_EVALUATION_STORE")
|
|
99
|
+
return "MODEL_EVALUATION_STORE";
|
|
98
100
|
return undefined;
|
|
99
101
|
}
|
|
100
102
|
function recipeOutputItems(recipe) {
|
|
@@ -116,7 +118,12 @@ function recipeOutputItems(recipe) {
|
|
|
116
118
|
if (seen.has(seenKey))
|
|
117
119
|
continue;
|
|
118
120
|
seen.add(seenKey);
|
|
119
|
-
|
|
121
|
+
// The evaluation recipe's "evaluationStore" role targets a model
|
|
122
|
+
// evaluation store even when the item carries no explicit type
|
|
123
|
+
// (recipe.py EvaluationRecipeCreator.with_output_evaluation_store);
|
|
124
|
+
// the store is a valid job target (run object_type_map).
|
|
125
|
+
const type = normalizeRecipeOutputType(item?.type ?? item?.targetType ?? item?.objectType)
|
|
126
|
+
?? (role === "evaluationStore" ? "MODEL_EVALUATION_STORE" : undefined);
|
|
120
127
|
result.push({
|
|
121
128
|
ref,
|
|
122
129
|
role,
|
|
@@ -292,6 +299,25 @@ export class RecipesResource extends BaseResource {
|
|
|
292
299
|
const raw = await this.client.get(`/public/api/projects/${enc}/recipes/`);
|
|
293
300
|
return this.client.safeParse(RecipeSummaryArraySchema, raw, "recipes.list");
|
|
294
301
|
}
|
|
302
|
+
/** Get recipe metadata (label, tags, description, custom fields). */
|
|
303
|
+
async metadata(recipeName, opts) {
|
|
304
|
+
const enc = this.enc(opts?.projectKey);
|
|
305
|
+
const rnEnc = encodeURIComponent(recipeName);
|
|
306
|
+
const raw = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}/metadata`);
|
|
307
|
+
return this.client.safeParse(ProjectMetadataSchema, raw, "recipes.metadata");
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Replace recipe metadata with the supplied full object.
|
|
311
|
+
*
|
|
312
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
313
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
314
|
+
* absent from the payload are removed from the recipe.
|
|
315
|
+
*/
|
|
316
|
+
async setMetadata(recipeName, metadata, opts) {
|
|
317
|
+
const enc = this.enc(opts?.projectKey);
|
|
318
|
+
const rnEnc = encodeURIComponent(recipeName);
|
|
319
|
+
await this.client.putVoid(`/public/api/projects/${enc}/recipes/${rnEnc}/metadata`, metadata);
|
|
320
|
+
}
|
|
295
321
|
/**
|
|
296
322
|
* Get a recipe definition (and optionally its payload).
|
|
297
323
|
* Returns the raw API response shape: `{ recipe, payload }`.
|
|
@@ -320,11 +346,13 @@ export class RecipesResource extends BaseResource {
|
|
|
320
346
|
const { recipe, } = await this.get(recipeName, { projectKey: pk, });
|
|
321
347
|
const inputItems = recipeInputItems(recipe);
|
|
322
348
|
const outputItems = recipeOutputItems(recipe);
|
|
323
|
-
const [datasets, folders,] = await Promise.all([
|
|
349
|
+
const [datasets, folders, stores,] = await Promise.all([
|
|
324
350
|
this.client.datasets.list(pk),
|
|
325
351
|
this.client.folders.list(pk),
|
|
352
|
+
this.client.modelEvaluationStores.list(pk),
|
|
326
353
|
]);
|
|
327
354
|
const datasetNames = new Set(datasets.map((dataset) => dataset.name));
|
|
355
|
+
const storeIds = new Set(stores.map((store) => store.id));
|
|
328
356
|
const folderIdByRef = new Map();
|
|
329
357
|
for (const folder of folders) {
|
|
330
358
|
folderIdByRef.set(folder.id, folder.id);
|
|
@@ -337,6 +365,15 @@ export class RecipesResource extends BaseResource {
|
|
|
337
365
|
if (item.type === "DATASET") {
|
|
338
366
|
return { ref: item.ref, role: item.role, type: "DATASET", exists: isDataset, id: item.ref, };
|
|
339
367
|
}
|
|
368
|
+
if (item.type === "MODEL_EVALUATION_STORE") {
|
|
369
|
+
return {
|
|
370
|
+
ref: item.ref,
|
|
371
|
+
role: item.role,
|
|
372
|
+
type: "MODEL_EVALUATION_STORE",
|
|
373
|
+
exists: storeIds.has(item.ref),
|
|
374
|
+
id: item.ref,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
340
377
|
if (item.type === "MANAGED_FOLDER") {
|
|
341
378
|
return {
|
|
342
379
|
ref: item.ref,
|
|
@@ -420,6 +457,19 @@ export class RecipesResource extends BaseResource {
|
|
|
420
457
|
partition: opts?.partition,
|
|
421
458
|
};
|
|
422
459
|
}
|
|
460
|
+
if (item.type === "MODEL_EVALUATION_STORE") {
|
|
461
|
+
// The store id is the target id; evaluation builds into the store
|
|
462
|
+
// exactly like the official client (recipe.run maps
|
|
463
|
+
// COMPUTABLE_MODEL_EVALUATION_STORE -> MODEL_EVALUATION_STORE).
|
|
464
|
+
return {
|
|
465
|
+
ref: item.ref,
|
|
466
|
+
role: item.role,
|
|
467
|
+
id: item.ref,
|
|
468
|
+
type: "MODEL_EVALUATION_STORE",
|
|
469
|
+
projectKey: pk,
|
|
470
|
+
partition: opts?.partition,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
423
473
|
const folderId = folderIdByRef.get(item.ref);
|
|
424
474
|
if (item.type === "MANAGED_FOLDER") {
|
|
425
475
|
return {
|
|
@@ -632,10 +682,22 @@ export class RecipesResource extends BaseResource {
|
|
|
632
682
|
}
|
|
633
683
|
}
|
|
634
684
|
};
|
|
685
|
+
let finalRecipeName;
|
|
686
|
+
// DSS renames some recipe types server-side (e.g. prediction_scoring
|
|
687
|
+
// becomes "score_<inputDataset>") and documents the creation response as
|
|
688
|
+
// the final unique name: Body {"name": "recipe1"} ("Returns the final
|
|
689
|
+
// unique name of the recipe"). The response name is the contract —
|
|
690
|
+
// falling back to the requested name would mask a missing/invalid
|
|
691
|
+
// receipt, so a body without it is an error.
|
|
635
692
|
try {
|
|
636
693
|
if (rawConnection)
|
|
637
694
|
await provisionOutputDatasets();
|
|
638
|
-
await createRecipe();
|
|
695
|
+
const created = await createRecipe();
|
|
696
|
+
const receivedName = asString(created?.["name"]);
|
|
697
|
+
if (!receivedName) {
|
|
698
|
+
throw new Error('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
699
|
+
}
|
|
700
|
+
finalRecipeName = receivedName;
|
|
639
701
|
}
|
|
640
702
|
catch (error) {
|
|
641
703
|
if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
|
|
@@ -643,7 +705,12 @@ export class RecipesResource extends BaseResource {
|
|
|
643
705
|
}
|
|
644
706
|
usedOutputProvisioningFallback = true;
|
|
645
707
|
await provisionOutputDatasets();
|
|
646
|
-
await createRecipe();
|
|
708
|
+
const retryCreated = await createRecipe();
|
|
709
|
+
const retryName = asString(retryCreated?.["name"]);
|
|
710
|
+
if (!retryName) {
|
|
711
|
+
throw new Error('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).', { cause: error, });
|
|
712
|
+
}
|
|
713
|
+
finalRecipeName = retryName;
|
|
647
714
|
}
|
|
648
715
|
// For sync recipes (a 1:1 copy) DSS does not propagate schema on build, so a
|
|
649
716
|
// schemaless dataset output silently builds to an empty dataset. Copy the
|
|
@@ -781,7 +848,7 @@ export class RecipesResource extends BaseResource {
|
|
|
781
848
|
}
|
|
782
849
|
}
|
|
783
850
|
return {
|
|
784
|
-
recipeName:
|
|
851
|
+
recipeName: finalRecipeName,
|
|
785
852
|
type,
|
|
786
853
|
createdDatasets,
|
|
787
854
|
joinConfigured,
|