dataiku-sdk 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +253 -38
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +102 -0
- package/dist/packages/types/src/index.js +76 -0
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +5 -3
- package/dist/src/cli/commands/app.js +61 -18
- package/dist/src/cli/commands/bundle.js +3 -2
- package/dist/src/cli/commands/code-env.js +23 -16
- package/dist/src/cli/commands/connection.d.ts +9 -0
- package/dist/src/cli/commands/connection.js +267 -1
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-collection.d.ts +2 -0
- package/dist/src/cli/commands/data-collection.js +164 -0
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +170 -40
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +9 -8
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +4 -3
- package/dist/src/cli/commands/group.d.ts +2 -0
- package/dist/src/cli/commands/group.js +109 -0
- package/dist/src/cli/commands/index.js +151 -78
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
- package/dist/src/cli/commands/knowledge-bank.js +80 -0
- package/dist/src/cli/commands/llm.d.ts +2 -0
- package/dist/src/cli/commands/llm.js +93 -0
- package/dist/src/cli/commands/macro.d.ts +2 -0
- package/dist/src/cli/commands/macro.js +221 -0
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +21 -8
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +17 -12
- package/dist/src/cli/commands/plugin.d.ts +2 -0
- package/dist/src/cli/commands/plugin.js +799 -0
- package/dist/src/cli/commands/project-folder.d.ts +2 -0
- package/dist/src/cli/commands/project-folder.js +197 -0
- package/dist/src/cli/commands/project-git.js +14 -54
- package/dist/src/cli/commands/project-library.js +7 -6
- package/dist/src/cli/commands/project.js +55 -8
- package/dist/src/cli/commands/recipe.js +50 -14
- package/dist/src/cli/commands/saved-model.d.ts +5 -0
- package/dist/src/cli/commands/saved-model.js +392 -4
- package/dist/src/cli/commands/scenario.js +147 -9
- package/dist/src/cli/commands/sql.js +3 -2
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/user.d.ts +8 -0
- package/dist/src/cli/commands/user.js +296 -0
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +2 -1
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +2 -3
- package/dist/src/cli/contract.js +1167 -163
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +50 -2
- package/dist/src/cli/helpers/app-successor.d.ts +19 -4
- package/dist/src/cli/helpers/app-successor.js +90 -30
- package/dist/src/cli/helpers/cleanup.js +3 -1
- package/dist/src/cli/helpers/flow-zone.js +1 -2
- package/dist/src/cli/output.d.ts +1 -1
- package/dist/src/cli/output.js +23 -10
- package/dist/src/cli/usage.d.ts +1 -0
- package/dist/src/cli/usage.js +47 -1
- package/dist/src/cli.js +225 -95
- package/dist/src/client.d.ts +86 -6
- package/dist/src/client.js +263 -50
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +4 -0
- package/dist/src/index.d.ts +16 -7
- package/dist/src/index.js +11 -2
- package/dist/src/resources/analyses.js +1 -8
- package/dist/src/resources/applications.d.ts +8 -6
- package/dist/src/resources/applications.js +21 -21
- package/dist/src/resources/base.d.ts +4 -0
- package/dist/src/resources/base.js +15 -0
- package/dist/src/resources/bundles.d.ts +9 -4
- package/dist/src/resources/bundles.js +10 -5
- package/dist/src/resources/connections.d.ts +78 -1
- package/dist/src/resources/connections.js +132 -1
- package/dist/src/resources/data-collections.d.ts +118 -0
- package/dist/src/resources/data-collections.js +49 -0
- package/dist/src/resources/datasets.d.ts +69 -0
- package/dist/src/resources/datasets.js +99 -1
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/futures.d.ts +4 -1
- package/dist/src/resources/futures.js +38 -6
- package/dist/src/resources/groups.d.ts +47 -0
- package/dist/src/resources/groups.js +56 -0
- package/dist/src/resources/jobs.d.ts +8 -13
- package/dist/src/resources/jobs.js +41 -16
- package/dist/src/resources/knowledge-banks.d.ts +58 -0
- package/dist/src/resources/knowledge-banks.js +61 -0
- package/dist/src/resources/llms.d.ts +92 -0
- package/dist/src/resources/llms.js +62 -0
- package/dist/src/resources/macros.d.ts +121 -0
- package/dist/src/resources/macros.js +231 -0
- package/dist/src/resources/ml-tasks.d.ts +37 -2
- package/dist/src/resources/ml-tasks.js +100 -20
- package/dist/src/resources/model-evaluation-stores.js +1 -8
- package/dist/src/resources/notebooks.d.ts +43 -7
- package/dist/src/resources/notebooks.js +53 -13
- package/dist/src/resources/plugins.d.ts +265 -0
- package/dist/src/resources/plugins.js +458 -0
- package/dist/src/resources/project-folders.d.ts +71 -0
- package/dist/src/resources/project-folders.js +57 -0
- package/dist/src/resources/project-git.d.ts +6 -0
- package/dist/src/resources/project-git.js +69 -42
- package/dist/src/resources/project-library.d.ts +0 -1
- package/dist/src/resources/project-library.js +1 -17
- package/dist/src/resources/projects.d.ts +19 -3
- package/dist/src/resources/projects.js +30 -18
- package/dist/src/resources/recipes.d.ts +15 -1
- package/dist/src/resources/recipes.js +73 -6
- package/dist/src/resources/saved-models.d.ts +134 -1
- package/dist/src/resources/saved-models.js +214 -6
- package/dist/src/resources/scenarios.d.ts +105 -0
- package/dist/src/resources/scenarios.js +232 -106
- package/dist/src/resources/sql.js +5 -1
- package/dist/src/resources/users.d.ts +129 -0
- package/dist/src/resources/users.js +128 -0
- package/dist/src/resources/webapps.d.ts +15 -2
- package/dist/src/resources/webapps.js +18 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +13 -4
- package/dist/src/skill.js +75 -24
- package/dist/src/utils/deep-merge.d.ts +1 -1
- package/dist/src/utils/polling.d.ts +20 -0
- package/dist/src/utils/polling.js +27 -0
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/dist/src/utils/secret-sanitize.d.ts +54 -0
- package/dist/src/utils/secret-sanitize.js +101 -0
- package/package.json +8 -5
- package/packages/types/dist/index.d.ts +102 -0
- package/packages/types/dist/index.js +76 -0
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +47 -225
- package/skills/dataiku-dss/references/app-releases.md +28 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { ClientValidationError,
|
|
1
|
+
import { ClientValidationError, } from "../errors.js";
|
|
2
2
|
import { ProjectGitActionResultSchema, ProjectGitDiffResultSchema, ProjectGitFutureResponseSchema, ProjectGitFutureStateSchema, ProjectGitLibrariesSchema, ProjectGitLogResultSchema, ProjectGitRemoteSchema, ProjectGitStatusSchema, ProjectGitTagsSchema, } from "../schemas.js";
|
|
3
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
4
|
+
import { sanitizeErrorSecrets, } from "../utils/secret-sanitize.js";
|
|
3
5
|
import { BaseResource, } from "./base.js";
|
|
4
6
|
/**
|
|
5
7
|
* The per-project Git routes exist only under the dataikuapi base, not under
|
|
@@ -89,28 +91,10 @@ function validatedRepository(repository, field) {
|
|
|
89
91
|
return candidate;
|
|
90
92
|
}
|
|
91
93
|
/** Replace every exact occurrence of a secret so echoed errors cannot leak it. */
|
|
92
|
-
function redactSecrets(text, secrets) {
|
|
93
|
-
let result = text;
|
|
94
|
-
for (const secret of secrets) {
|
|
95
|
-
if (secret !== undefined && secret !== "") {
|
|
96
|
-
result = result.split(secret).join("[redacted]");
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
return result;
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* Strip a password that a server misconfiguration may have echoed from an
|
|
103
|
-
* error response. `DataikuError.message` was built at construction, so the
|
|
104
|
-
* raw body, the message, and the stack all need scrubbing.
|
|
105
|
-
*/
|
|
106
94
|
function scrubErrorSecrets(error, secrets) {
|
|
107
|
-
if (
|
|
95
|
+
if (secrets.length === 0)
|
|
108
96
|
return;
|
|
109
|
-
error
|
|
110
|
-
error.message = redactSecrets(error.message, secrets);
|
|
111
|
-
if (typeof error.stack === "string") {
|
|
112
|
-
error.stack = redactSecrets(error.stack, secrets);
|
|
113
|
-
}
|
|
97
|
+
sanitizeErrorSecrets(error, { sensitiveKeys: {}, secrets, });
|
|
114
98
|
}
|
|
115
99
|
function describeFutureFailure(error) {
|
|
116
100
|
if (error === undefined || error === null)
|
|
@@ -129,6 +113,31 @@ function describeFutureFailure(error) {
|
|
|
129
113
|
return String(error);
|
|
130
114
|
}
|
|
131
115
|
/* ------------------------------------------------------------------ */
|
|
116
|
+
/**
|
|
117
|
+
* DSS answers the lib-git-refs listing as an object map of attached external
|
|
118
|
+
* libraries keyed by target path alongside code-library metadata (observed
|
|
119
|
+
* raw on DSS 15: `{gitReferences, pythonPath, rsrcPath,
|
|
120
|
+
* importLibrariesFromProjects}`), while the official Python client and this
|
|
121
|
+
* resource's typed contract expose a plain array. Normalize the map into
|
|
122
|
+
* entries, preserving each key as the library's `localTargetPath`; a bare
|
|
123
|
+
* array passes through untouched.
|
|
124
|
+
*/
|
|
125
|
+
function normalizeLibraryList(raw) {
|
|
126
|
+
if (Array.isArray(raw) || raw === undefined || raw === null)
|
|
127
|
+
return raw ?? [];
|
|
128
|
+
if (typeof raw !== "object")
|
|
129
|
+
return raw;
|
|
130
|
+
const record = raw;
|
|
131
|
+
const refs = record["gitReferences"];
|
|
132
|
+
if (refs === undefined || refs === null || typeof refs !== "object" || Array.isArray(refs)) {
|
|
133
|
+
return raw;
|
|
134
|
+
}
|
|
135
|
+
return Object.entries(refs).map(([localTargetPath, spec,]) => ({
|
|
136
|
+
localTargetPath,
|
|
137
|
+
...(typeof spec === "object" && spec !== null ? spec : {}),
|
|
138
|
+
}));
|
|
139
|
+
}
|
|
140
|
+
/* ------------------------------------------------------------------ */
|
|
132
141
|
/* Resource */
|
|
133
142
|
/* ------------------------------------------------------------------ */
|
|
134
143
|
export class ProjectGitResource extends BaseResource {
|
|
@@ -287,7 +296,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
287
296
|
/* ---- external libraries (async futures) ---- */
|
|
288
297
|
async listLibraries(projectKey) {
|
|
289
298
|
const raw = await this.request("GET", `${this.libraryPath(projectKey)}/`);
|
|
290
|
-
return this.client.safeParse(ProjectGitLibrariesSchema, raw
|
|
299
|
+
return this.client.safeParse(ProjectGitLibrariesSchema, normalizeLibraryList(raw), "projectGit.listLibraries");
|
|
291
300
|
}
|
|
292
301
|
/** Attach an external Git library. Runs as a DSS future. */
|
|
293
302
|
async addLibrary(projectKey, options) {
|
|
@@ -355,7 +364,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
355
364
|
* what polling loops want; the default consumes it like the Python client.
|
|
356
365
|
*/
|
|
357
366
|
async getFutureState(jobId, options = {}) {
|
|
358
|
-
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}
|
|
367
|
+
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`, undefined, [], { timeoutMs: options.timeoutMs, });
|
|
359
368
|
return this.client.safeParse(ProjectGitFutureStateSchema, raw ?? {}, "projectGit.getFutureState");
|
|
360
369
|
}
|
|
361
370
|
/** Abort a running future. */
|
|
@@ -376,10 +385,36 @@ export class ProjectGitResource extends BaseResource {
|
|
|
376
385
|
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
377
386
|
throw new ClientValidationError("timeoutMs must be a finite non-negative number.");
|
|
378
387
|
}
|
|
388
|
+
// Adaptive backoff applies only when the caller did not pin an exact
|
|
389
|
+
// interval; an explicit pollIntervalMs is an exact contract.
|
|
390
|
+
const adaptiveEnabled = options.pollIntervalMs === undefined;
|
|
391
|
+
let pollCount = 0;
|
|
379
392
|
const startedAt = Date.now();
|
|
380
393
|
while (true) {
|
|
381
394
|
// Unlike peek state, the ordinary state includes the completed result.
|
|
382
|
-
|
|
395
|
+
// One state observation always happens, even when the budget is
|
|
396
|
+
// already spent (the poll precedes the deadline check); only the
|
|
397
|
+
// requests issued while budget remains are bounded by the remaining
|
|
398
|
+
// time, so a stalled endpoint can never defeat the caller's deadline.
|
|
399
|
+
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
400
|
+
pollCount += 1;
|
|
401
|
+
let state;
|
|
402
|
+
try {
|
|
403
|
+
state = await this.getFutureState(jobId, remainingMs > 0
|
|
404
|
+
? {
|
|
405
|
+
peek: false,
|
|
406
|
+
timeoutMs: remainingMs,
|
|
407
|
+
}
|
|
408
|
+
: { peek: false, });
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
// The poll budget ran out mid-request: report the documented
|
|
412
|
+
// timeout failure instead of letting the transport deadline
|
|
413
|
+
// error escape.
|
|
414
|
+
if (!isRequestDeadlineError(error, startedAt + timeoutMs))
|
|
415
|
+
throw error;
|
|
416
|
+
throw new Error(`Timed out after ${String(Date.now() - startedAt)}ms waiting for Dataiku future ${jobId}`, { cause: error, });
|
|
417
|
+
}
|
|
383
418
|
const failure = describeFutureFailure(state.error);
|
|
384
419
|
if (failure !== undefined) {
|
|
385
420
|
throw new Error(`Dataiku future ${jobId} failed: ${failure}`);
|
|
@@ -395,13 +430,18 @@ export class ProjectGitResource extends BaseResource {
|
|
|
395
430
|
if (state.alive === false) {
|
|
396
431
|
throw new Error(`Dataiku future ${jobId} ended without producing a result`);
|
|
397
432
|
}
|
|
398
|
-
// Only the remaining budget is slept, so a caller's deadline is never
|
|
399
|
-
// overshot by a whole poll interval.
|
|
400
433
|
const elapsedMs = Date.now() - startedAt;
|
|
401
434
|
if (elapsedMs >= timeoutMs) {
|
|
402
435
|
throw new Error(`Timed out after ${String(elapsedMs)}ms waiting for Dataiku future ${jobId}`);
|
|
403
436
|
}
|
|
404
|
-
|
|
437
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
438
|
+
pollCount,
|
|
439
|
+
baseIntervalMs: pollIntervalMs,
|
|
440
|
+
adaptiveEnabled,
|
|
441
|
+
});
|
|
442
|
+
// Only the remaining budget is slept, so a caller's deadline is never
|
|
443
|
+
// overshot by a whole interval.
|
|
444
|
+
await sleep(Math.min(nextDelayMs, timeoutMs - elapsedMs));
|
|
405
445
|
}
|
|
406
446
|
}
|
|
407
447
|
/* ---- private ---- */
|
|
@@ -424,22 +464,9 @@ export class ProjectGitResource extends BaseResource {
|
|
|
424
464
|
const queryString = search.toString();
|
|
425
465
|
return queryString === "" ? "" : `?${queryString}`;
|
|
426
466
|
}
|
|
427
|
-
async request(method, path, body, secrets = []) {
|
|
428
|
-
const transport = this.client;
|
|
429
|
-
const init = {
|
|
430
|
-
method,
|
|
431
|
-
headers: {
|
|
432
|
-
// The Git routes are only reachable with dataikuapi-style Basic auth:
|
|
433
|
-
// API key as the username, empty password.
|
|
434
|
-
Authorization: `Basic ${Buffer.from(`${transport.apiKey}:`, "utf8").toString("base64")}`,
|
|
435
|
-
Accept: "application/json",
|
|
436
|
-
"Content-Type": "application/json",
|
|
437
|
-
},
|
|
438
|
-
};
|
|
439
|
-
if (body !== undefined)
|
|
440
|
-
init.body = JSON.stringify(body);
|
|
467
|
+
async request(method, path, body, secrets = [], opts = {}) {
|
|
441
468
|
try {
|
|
442
|
-
const res = await
|
|
469
|
+
const res = await this.client.requestGit(method, `${GIT_API_BASE}${path}`, body, opts.timeoutMs === undefined ? undefined : Date.now() + opts.timeoutMs);
|
|
443
470
|
const text = await res.text();
|
|
444
471
|
if (text.trim() === "")
|
|
445
472
|
return undefined;
|
|
@@ -147,7 +147,6 @@ export declare class ProjectLibraryResource extends BaseResource {
|
|
|
147
147
|
/** Move a project code-library file or folder into another library folder. */
|
|
148
148
|
move(path: string, destinationFolderPath: string, projectKey?: string): Promise<void>;
|
|
149
149
|
private contentsPath;
|
|
150
|
-
private postRawBody;
|
|
151
150
|
private alreadyExistsError;
|
|
152
151
|
private assertLibraryFileAbsent;
|
|
153
152
|
}
|
|
@@ -403,7 +403,7 @@ export class ProjectLibraryResource extends BaseResource {
|
|
|
403
403
|
}
|
|
404
404
|
const bytes = typeof content === "string" ? Buffer.from(content, "utf8") : content;
|
|
405
405
|
const sha256 = sha256Hex(bytes);
|
|
406
|
-
await this.postRawBody(this.contentsPath(valid, projectKey), content);
|
|
406
|
+
await this.client.postRawBody(this.contentsPath(valid, projectKey), content);
|
|
407
407
|
return {
|
|
408
408
|
path: valid,
|
|
409
409
|
bytes: bytes.length,
|
|
@@ -518,22 +518,6 @@ export class ProjectLibraryResource extends BaseResource {
|
|
|
518
518
|
const projectKeyPart = this.enc(projectKey);
|
|
519
519
|
return `/public/api/projects/${projectKeyPart}/libraries/contents/${encodeLibraryPath(path)}`;
|
|
520
520
|
}
|
|
521
|
-
async postRawBody(path, body) {
|
|
522
|
-
// DSS library writes use dataikuapi's raw_body; route through the client's
|
|
523
|
-
// transport so auth, retries, TLS options, and DSS error handling stay aligned.
|
|
524
|
-
// BodyInit rejects plain Uint8Array views, so non-Buffer bytes are copied
|
|
525
|
-
// into a Buffer (an ArrayBufferView BodyInit accepts).
|
|
526
|
-
const payload = body instanceof Uint8Array && !(body instanceof Buffer)
|
|
527
|
-
? Buffer.from(body.buffer, body.byteOffset, body.byteLength)
|
|
528
|
-
: body;
|
|
529
|
-
const rawClient = this.client;
|
|
530
|
-
const res = await rawClient.fetchWithRetry(`${rawClient.baseUrl}${path}`, {
|
|
531
|
-
method: "POST",
|
|
532
|
-
headers: rawClient.getAnyHeaders(),
|
|
533
|
-
body: payload,
|
|
534
|
-
});
|
|
535
|
-
await res.text();
|
|
536
|
-
}
|
|
537
521
|
alreadyExistsError(path) {
|
|
538
522
|
return new ClientValidationError(`Project library item "${path}" already exists.`, "validation_failed", "Refuse to overwrite an existing item; delete it first, or use the create-or-replace update to replace an existing file.", { path, });
|
|
539
523
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { FlowMapOptions, ProjectDetails, ProjectMetadata, ProjectSummary } from "../schemas.js";
|
|
1
|
+
import type { FlowMapOptions, ProjectDetails, ProjectMetadata, ProjectSummary, ProjectTags } from "../schemas.js";
|
|
2
2
|
import type { AnalyzedFlowMap, FlowMapRendering } from "../utils/flow-analysis.js";
|
|
3
3
|
import { BaseResource } from "./base.js";
|
|
4
4
|
interface FlowMapTruncationSummary {
|
|
@@ -87,8 +87,6 @@ export interface ProjectImportResult extends ProjectImportProcessResult {
|
|
|
87
87
|
projectIncarnationHash?: string;
|
|
88
88
|
}
|
|
89
89
|
export declare class ProjectsResource extends BaseResource {
|
|
90
|
-
private httpInternals;
|
|
91
|
-
private postStream;
|
|
92
90
|
/** Create a new project. */
|
|
93
91
|
createProject(projectKey: string, name: string, ownerLogin: string, settings?: ProjectLifecycleSettings | null): Promise<Record<string, unknown>>;
|
|
94
92
|
/** Delete a project using the lifecycle endpoint. */
|
|
@@ -126,6 +124,24 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
126
124
|
get(projectKey?: string): Promise<ProjectDetails>;
|
|
127
125
|
/** Get metadata (tags, custom fields, checklists) for a project. */
|
|
128
126
|
metadata(projectKey?: string): Promise<ProjectMetadata>;
|
|
127
|
+
/**
|
|
128
|
+
* Replace project metadata with the supplied full object.
|
|
129
|
+
*
|
|
130
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
131
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
132
|
+
* absent from the payload are removed from the project.
|
|
133
|
+
*/
|
|
134
|
+
setMetadata(projectKey: string | undefined, metadata: ProjectMetadata): Promise<void>;
|
|
135
|
+
/** Get project-level tags (tag name → {color?}). */
|
|
136
|
+
tags(projectKey?: string): Promise<ProjectTags>;
|
|
137
|
+
/**
|
|
138
|
+
* Replace project-level tags with the supplied full object.
|
|
139
|
+
*
|
|
140
|
+
* DSS PUT semantics replace the whole tags dictionary: pass a tags object
|
|
141
|
+
* obtained from {@link tags}, edit it, and send it back — tags absent from
|
|
142
|
+
* the payload are removed from the project.
|
|
143
|
+
*/
|
|
144
|
+
setTags(projectKey: string | undefined, tags: ProjectTags): Promise<void>;
|
|
129
145
|
/** Get the raw flow graph for a project. */
|
|
130
146
|
flow(projectKey?: string): Promise<unknown>;
|
|
131
147
|
/** Compute a stable fingerprint of flow nodes and edges, excluding visual metadata. */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
-
import {
|
|
2
|
+
import { writeResponseToFile, } from "../utils/response-file.js";
|
|
3
|
+
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, ProjectTagsSchema, } from "../schemas.js";
|
|
3
4
|
import { analyzeFlowMap, flowTopologyFingerprint, renderFlowMap, } from "../utils/flow-analysis.js";
|
|
4
5
|
import { normalizeFlowGraph, } from "../utils/flow-map.js";
|
|
5
6
|
import { assertImportableProjectArchive, } from "../utils/project-archive.js";
|
|
@@ -104,21 +105,6 @@ const DEFAULT_MAX_EDGES = 600;
|
|
|
104
105
|
const DEFAULT_METADATA_TIMEOUT_MS = 1_500;
|
|
105
106
|
const PROJECT_IMPORT_AMBIGUOUS_REMEDIATION = "Import processing left DSS state indeterminate. Inspect the archive with `dss project inspect-archive <file>` and the live projects with `dss project list` before retrying the import.";
|
|
106
107
|
export class ProjectsResource extends BaseResource {
|
|
107
|
-
httpInternals() {
|
|
108
|
-
return this.client;
|
|
109
|
-
}
|
|
110
|
-
async postStream(path, body) {
|
|
111
|
-
const http = this.httpInternals();
|
|
112
|
-
return http.fetchWithRetry(`${http.baseUrl}${path}`, {
|
|
113
|
-
method: "POST",
|
|
114
|
-
headers: {
|
|
115
|
-
Authorization: `Bearer ${http.apiKey}`,
|
|
116
|
-
Accept: "*/*",
|
|
117
|
-
"Content-Type": "application/json",
|
|
118
|
-
},
|
|
119
|
-
body: JSON.stringify(body),
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
108
|
/** Create a new project. */
|
|
123
109
|
async createProject(projectKey, name, ownerLogin, settings) {
|
|
124
110
|
return this.client.post("/public/api/projects/", {
|
|
@@ -155,10 +141,10 @@ export class ProjectsResource extends BaseResource {
|
|
|
155
141
|
}
|
|
156
142
|
/** Export a project archive to a local file. */
|
|
157
143
|
async exportArchive(projectKey, filePath, options) {
|
|
158
|
-
const res = await this.postStream(`/public/api/projects/${encodeURIComponent(projectKey)}/export`, options ?? {});
|
|
144
|
+
const res = await this.client.postStream(`/public/api/projects/${encodeURIComponent(projectKey)}/export`, options ?? {});
|
|
159
145
|
if (!res.body)
|
|
160
146
|
throw new Error("projects.exportArchive response did not include a body");
|
|
161
|
-
await
|
|
147
|
+
await writeResponseToFile(filePath, res);
|
|
162
148
|
}
|
|
163
149
|
/** Upload a project archive and return its temporary import handle. */
|
|
164
150
|
async prepareProjectImport(filePath) {
|
|
@@ -316,6 +302,32 @@ export class ProjectsResource extends BaseResource {
|
|
|
316
302
|
const raw = await this.client.get(`/public/api/projects/${enc}/metadata`);
|
|
317
303
|
return this.client.safeParse(ProjectMetadataSchema, raw, "projects.metadata");
|
|
318
304
|
}
|
|
305
|
+
/**
|
|
306
|
+
* Replace project metadata with the supplied full object.
|
|
307
|
+
*
|
|
308
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
309
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
310
|
+
* absent from the payload are removed from the project.
|
|
311
|
+
*/
|
|
312
|
+
async setMetadata(projectKey, metadata) {
|
|
313
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/metadata`, metadata);
|
|
314
|
+
}
|
|
315
|
+
/** Get project-level tags (tag name → {color?}). */
|
|
316
|
+
async tags(projectKey) {
|
|
317
|
+
const enc = this.enc(projectKey);
|
|
318
|
+
const raw = await this.client.get(`/public/api/projects/${enc}/tags`);
|
|
319
|
+
return this.client.safeParse(ProjectTagsSchema, raw, "projects.tags");
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Replace project-level tags with the supplied full object.
|
|
323
|
+
*
|
|
324
|
+
* DSS PUT semantics replace the whole tags dictionary: pass a tags object
|
|
325
|
+
* obtained from {@link tags}, edit it, and send it back — tags absent from
|
|
326
|
+
* the payload are removed from the project.
|
|
327
|
+
*/
|
|
328
|
+
async setTags(projectKey, tags) {
|
|
329
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/tags`, tags);
|
|
330
|
+
}
|
|
319
331
|
/** Get the raw flow graph for a project. */
|
|
320
332
|
async flow(projectKey) {
|
|
321
333
|
const enc = this.enc(projectKey);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BuildMode, JobWaitResult, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
1
|
+
import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
3
|
import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
|
|
4
4
|
export interface RecipeRunOutput extends JobBuildTarget {
|
|
@@ -68,6 +68,20 @@ export interface RecipeCloneResult {
|
|
|
68
68
|
export declare class RecipesResource extends BaseResource {
|
|
69
69
|
/** List all recipes in a project. */
|
|
70
70
|
list(projectKey?: string): Promise<RecipeSummary[]>;
|
|
71
|
+
/** Get recipe metadata (label, tags, description, custom fields). */
|
|
72
|
+
metadata(recipeName: string, opts?: {
|
|
73
|
+
projectKey?: string;
|
|
74
|
+
}): Promise<ProjectMetadata>;
|
|
75
|
+
/**
|
|
76
|
+
* Replace recipe metadata with the supplied full object.
|
|
77
|
+
*
|
|
78
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
79
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
80
|
+
* absent from the payload are removed from the recipe.
|
|
81
|
+
*/
|
|
82
|
+
setMetadata(recipeName: string, metadata: ProjectMetadata, opts?: {
|
|
83
|
+
projectKey?: string;
|
|
84
|
+
}): Promise<void>;
|
|
71
85
|
/**
|
|
72
86
|
* Get a recipe definition (and optionally its payload).
|
|
73
87
|
* Returns the raw API response shape: `{ recipe, payload }`.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { writeFile, } from "node:fs/promises";
|
|
2
2
|
import { resolve, } from "node:path";
|
|
3
3
|
import { DataikuError, } from "../errors.js";
|
|
4
|
-
import { RecipeSummaryArraySchema, } from "../schemas.js";
|
|
4
|
+
import { ProjectMetadataSchema, RecipeSummaryArraySchema, } from "../schemas.js";
|
|
5
5
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
6
6
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
7
7
|
import { BaseResource, } from "./base.js";
|
|
@@ -95,6 +95,8 @@ function normalizeRecipeOutputType(value) {
|
|
|
95
95
|
return "DATASET";
|
|
96
96
|
if (normalized === "MANAGED_FOLDER" || normalized === "FOLDER")
|
|
97
97
|
return "MANAGED_FOLDER";
|
|
98
|
+
if (normalized === "MODEL_EVALUATION_STORE")
|
|
99
|
+
return "MODEL_EVALUATION_STORE";
|
|
98
100
|
return undefined;
|
|
99
101
|
}
|
|
100
102
|
function recipeOutputItems(recipe) {
|
|
@@ -116,7 +118,12 @@ function recipeOutputItems(recipe) {
|
|
|
116
118
|
if (seen.has(seenKey))
|
|
117
119
|
continue;
|
|
118
120
|
seen.add(seenKey);
|
|
119
|
-
|
|
121
|
+
// The evaluation recipe's "evaluationStore" role targets a model
|
|
122
|
+
// evaluation store even when the item carries no explicit type
|
|
123
|
+
// (recipe.py EvaluationRecipeCreator.with_output_evaluation_store);
|
|
124
|
+
// the store is a valid job target (run object_type_map).
|
|
125
|
+
const type = normalizeRecipeOutputType(item?.type ?? item?.targetType ?? item?.objectType)
|
|
126
|
+
?? (role === "evaluationStore" ? "MODEL_EVALUATION_STORE" : undefined);
|
|
120
127
|
result.push({
|
|
121
128
|
ref,
|
|
122
129
|
role,
|
|
@@ -292,6 +299,25 @@ export class RecipesResource extends BaseResource {
|
|
|
292
299
|
const raw = await this.client.get(`/public/api/projects/${enc}/recipes/`);
|
|
293
300
|
return this.client.safeParse(RecipeSummaryArraySchema, raw, "recipes.list");
|
|
294
301
|
}
|
|
302
|
+
/** Get recipe metadata (label, tags, description, custom fields). */
|
|
303
|
+
async metadata(recipeName, opts) {
|
|
304
|
+
const enc = this.enc(opts?.projectKey);
|
|
305
|
+
const rnEnc = encodeURIComponent(recipeName);
|
|
306
|
+
const raw = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}/metadata`);
|
|
307
|
+
return this.client.safeParse(ProjectMetadataSchema, raw, "recipes.metadata");
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Replace recipe metadata with the supplied full object.
|
|
311
|
+
*
|
|
312
|
+
* DSS PUT semantics replace the whole metadata object: pass a metadata
|
|
313
|
+
* object obtained from {@link metadata}, edit it, and send it back — fields
|
|
314
|
+
* absent from the payload are removed from the recipe.
|
|
315
|
+
*/
|
|
316
|
+
async setMetadata(recipeName, metadata, opts) {
|
|
317
|
+
const enc = this.enc(opts?.projectKey);
|
|
318
|
+
const rnEnc = encodeURIComponent(recipeName);
|
|
319
|
+
await this.client.putVoid(`/public/api/projects/${enc}/recipes/${rnEnc}/metadata`, metadata);
|
|
320
|
+
}
|
|
295
321
|
/**
|
|
296
322
|
* Get a recipe definition (and optionally its payload).
|
|
297
323
|
* Returns the raw API response shape: `{ recipe, payload }`.
|
|
@@ -320,11 +346,13 @@ export class RecipesResource extends BaseResource {
|
|
|
320
346
|
const { recipe, } = await this.get(recipeName, { projectKey: pk, });
|
|
321
347
|
const inputItems = recipeInputItems(recipe);
|
|
322
348
|
const outputItems = recipeOutputItems(recipe);
|
|
323
|
-
const [datasets, folders,] = await Promise.all([
|
|
349
|
+
const [datasets, folders, stores,] = await Promise.all([
|
|
324
350
|
this.client.datasets.list(pk),
|
|
325
351
|
this.client.folders.list(pk),
|
|
352
|
+
this.client.modelEvaluationStores.list(pk),
|
|
326
353
|
]);
|
|
327
354
|
const datasetNames = new Set(datasets.map((dataset) => dataset.name));
|
|
355
|
+
const storeIds = new Set(stores.map((store) => store.id));
|
|
328
356
|
const folderIdByRef = new Map();
|
|
329
357
|
for (const folder of folders) {
|
|
330
358
|
folderIdByRef.set(folder.id, folder.id);
|
|
@@ -337,6 +365,15 @@ export class RecipesResource extends BaseResource {
|
|
|
337
365
|
if (item.type === "DATASET") {
|
|
338
366
|
return { ref: item.ref, role: item.role, type: "DATASET", exists: isDataset, id: item.ref, };
|
|
339
367
|
}
|
|
368
|
+
if (item.type === "MODEL_EVALUATION_STORE") {
|
|
369
|
+
return {
|
|
370
|
+
ref: item.ref,
|
|
371
|
+
role: item.role,
|
|
372
|
+
type: "MODEL_EVALUATION_STORE",
|
|
373
|
+
exists: storeIds.has(item.ref),
|
|
374
|
+
id: item.ref,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
340
377
|
if (item.type === "MANAGED_FOLDER") {
|
|
341
378
|
return {
|
|
342
379
|
ref: item.ref,
|
|
@@ -420,6 +457,19 @@ export class RecipesResource extends BaseResource {
|
|
|
420
457
|
partition: opts?.partition,
|
|
421
458
|
};
|
|
422
459
|
}
|
|
460
|
+
if (item.type === "MODEL_EVALUATION_STORE") {
|
|
461
|
+
// The store id is the target id; evaluation builds into the store
|
|
462
|
+
// exactly like the official client (recipe.run maps
|
|
463
|
+
// COMPUTABLE_MODEL_EVALUATION_STORE -> MODEL_EVALUATION_STORE).
|
|
464
|
+
return {
|
|
465
|
+
ref: item.ref,
|
|
466
|
+
role: item.role,
|
|
467
|
+
id: item.ref,
|
|
468
|
+
type: "MODEL_EVALUATION_STORE",
|
|
469
|
+
projectKey: pk,
|
|
470
|
+
partition: opts?.partition,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
423
473
|
const folderId = folderIdByRef.get(item.ref);
|
|
424
474
|
if (item.type === "MANAGED_FOLDER") {
|
|
425
475
|
return {
|
|
@@ -632,10 +682,22 @@ export class RecipesResource extends BaseResource {
|
|
|
632
682
|
}
|
|
633
683
|
}
|
|
634
684
|
};
|
|
685
|
+
let finalRecipeName;
|
|
686
|
+
// DSS renames some recipe types server-side (e.g. prediction_scoring
|
|
687
|
+
// becomes "score_<inputDataset>") and documents the creation response as
|
|
688
|
+
// the final unique name: Body {"name": "recipe1"} ("Returns the final
|
|
689
|
+
// unique name of the recipe"). The response name is the contract —
|
|
690
|
+
// falling back to the requested name would mask a missing/invalid
|
|
691
|
+
// receipt, so a body without it is an error.
|
|
635
692
|
try {
|
|
636
693
|
if (rawConnection)
|
|
637
694
|
await provisionOutputDatasets();
|
|
638
|
-
await createRecipe();
|
|
695
|
+
const created = await createRecipe();
|
|
696
|
+
const receivedName = asString(created?.["name"]);
|
|
697
|
+
if (!receivedName) {
|
|
698
|
+
throw new Error('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
699
|
+
}
|
|
700
|
+
finalRecipeName = receivedName;
|
|
639
701
|
}
|
|
640
702
|
catch (error) {
|
|
641
703
|
if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
|
|
@@ -643,7 +705,12 @@ export class RecipesResource extends BaseResource {
|
|
|
643
705
|
}
|
|
644
706
|
usedOutputProvisioningFallback = true;
|
|
645
707
|
await provisionOutputDatasets();
|
|
646
|
-
await createRecipe();
|
|
708
|
+
const retryCreated = await createRecipe();
|
|
709
|
+
const retryName = asString(retryCreated?.["name"]);
|
|
710
|
+
if (!retryName) {
|
|
711
|
+
throw new Error('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).', { cause: error, });
|
|
712
|
+
}
|
|
713
|
+
finalRecipeName = retryName;
|
|
647
714
|
}
|
|
648
715
|
// For sync recipes (a 1:1 copy) DSS does not propagate schema on build, so a
|
|
649
716
|
// schemaless dataset output silently builds to an empty dataset. Copy the
|
|
@@ -781,7 +848,7 @@ export class RecipesResource extends BaseResource {
|
|
|
781
848
|
}
|
|
782
849
|
}
|
|
783
850
|
return {
|
|
784
|
-
recipeName:
|
|
851
|
+
recipeName: finalRecipeName,
|
|
785
852
|
type,
|
|
786
853
|
createdDatasets,
|
|
787
854
|
joinConfigured,
|