dataiku-sdk 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/build-metadata.json +1 -1
- package/dist/src/cli/command-syntax.json +14524 -0
- package/dist/src/cli/commands/analysis.js +6 -9
- package/dist/src/cli/commands/api-deployer.js +15 -33
- package/dist/src/cli/commands/api-service.js +17 -27
- package/dist/src/cli/commands/app.js +22 -39
- package/dist/src/cli/commands/bundle.js +15 -24
- package/dist/src/cli/commands/business-app.js +10 -17
- package/dist/src/cli/commands/code-env.js +23 -36
- package/dist/src/cli/commands/code.js +5 -5
- package/dist/src/cli/commands/connection.js +12 -22
- package/dist/src/cli/commands/continuous-activity.js +6 -9
- package/dist/src/cli/commands/dashboard.js +8 -13
- package/dist/src/cli/commands/data-collection.js +9 -16
- package/dist/src/cli/commands/data-quality.js +14 -26
- package/dist/src/cli/commands/dataset.js +28 -51
- package/dist/src/cli/commands/discussion.js +7 -10
- package/dist/src/cli/commands/doctor.js +3 -3
- package/dist/src/cli/commands/flow-zone.js +8 -17
- package/dist/src/cli/commands/folder.js +12 -20
- package/dist/src/cli/commands/future.js +7 -10
- package/dist/src/cli/commands/group.js +7 -11
- package/dist/src/cli/commands/insight.js +7 -11
- package/dist/src/cli/commands/job.js +17 -25
- package/dist/src/cli/commands/knowledge-bank.js +5 -6
- package/dist/src/cli/commands/llm.js +5 -8
- package/dist/src/cli/commands/macro.js +11 -18
- package/dist/src/cli/commands/meaning.js +7 -11
- package/dist/src/cli/commands/metrics.js +7 -10
- package/dist/src/cli/commands/ml-task.js +12 -20
- package/dist/src/cli/commands/model-evaluation-store.js +7 -11
- package/dist/src/cli/commands/notebook.js +15 -27
- package/dist/src/cli/commands/plugin.js +35 -68
- package/dist/src/cli/commands/project-deployer.js +10 -22
- package/dist/src/cli/commands/project-folder.js +10 -17
- package/dist/src/cli/commands/project-git.js +39 -71
- package/dist/src/cli/commands/project-library.js +15 -23
- package/dist/src/cli/commands/project.js +9 -26
- package/dist/src/cli/commands/recipe.d.ts +16 -0
- package/dist/src/cli/commands/recipe.js +83 -94
- package/dist/src/cli/commands/saved-model.js +23 -39
- package/dist/src/cli/commands/scenario.js +21 -35
- package/dist/src/cli/commands/sql.js +4 -4
- package/dist/src/cli/commands/statistics.js +11 -18
- package/dist/src/cli/commands/streaming-endpoint.js +7 -11
- package/dist/src/cli/commands/user.js +11 -22
- package/dist/src/cli/commands/variable.js +3 -4
- package/dist/src/cli/commands/webapp.js +10 -16
- package/dist/src/cli/commands/wiki.js +9 -13
- package/dist/src/cli/commands/workspace.js +8 -14
- package/dist/src/cli/contract.d.ts +5 -15
- package/dist/src/cli/contract.js +179 -461
- package/dist/src/cli/doctor.js +1 -0
- package/dist/src/cli/env.d.ts +2 -0
- package/dist/src/cli/env.js +15 -1
- package/dist/src/cli/helpers/app-permissions.js +2 -1
- package/dist/src/cli/helpers/flow-zone.js +2 -1
- package/dist/src/cli/helpers/recipe.js +2 -1
- package/dist/src/cli/output-schemas.d.ts +24 -0
- package/dist/src/cli/output-schemas.js +233 -0
- package/dist/src/cli/syntax.d.ts +83 -0
- package/dist/src/cli/syntax.js +217 -0
- package/dist/src/cli/types.d.ts +3 -0
- package/dist/src/cli/usage.d.ts +1 -1
- package/dist/src/cli/usage.js +4 -2
- package/dist/src/cli/version.js +2 -0
- package/dist/src/cli.js +37 -64
- package/dist/src/client.d.ts +10 -1
- package/dist/src/client.js +27 -11
- package/dist/src/errors.d.ts +12 -1
- package/dist/src/errors.js +34 -7
- package/dist/src/generated/action-output-schemas.json +1735 -0
- package/dist/src/index.d.ts +1 -3
- package/dist/src/index.js +2 -2
- package/dist/src/resources/api-services.js +2 -3
- package/dist/src/resources/applications.js +2 -4
- package/dist/src/resources/code-envs.js +2 -2
- package/dist/src/resources/data-quality.js +4 -3
- package/dist/src/resources/datasets.d.ts +13 -0
- package/dist/src/resources/datasets.js +42 -8
- package/dist/src/resources/flow-zones.d.ts +6 -2
- package/dist/src/resources/flow-zones.js +10 -5
- package/dist/src/resources/folders.js +4 -2
- package/dist/src/resources/futures.d.ts +2 -0
- package/dist/src/resources/futures.js +39 -27
- package/dist/src/resources/insights.js +3 -2
- package/dist/src/resources/jobs.js +41 -31
- package/dist/src/resources/macros.js +2 -4
- package/dist/src/resources/ml-tasks.js +28 -19
- package/dist/src/resources/notebooks.js +2 -3
- package/dist/src/resources/plugins.d.ts +0 -2
- package/dist/src/resources/plugins.js +0 -3
- package/dist/src/resources/project-git.d.ts +6 -12
- package/dist/src/resources/project-git.js +31 -58
- package/dist/src/resources/project-library.d.ts +0 -2
- package/dist/src/resources/project-library.js +2 -7
- package/dist/src/resources/projects.d.ts +4 -2
- package/dist/src/resources/projects.js +37 -36
- package/dist/src/resources/recipes.d.ts +28 -0
- package/dist/src/resources/recipes.js +116 -84
- package/dist/src/resources/sql.js +8 -3
- package/dist/src/resources/statistics.js +2 -1
- package/dist/src/resources/variables.js +2 -1
- package/dist/src/resources/webapps.js +2 -3
- package/dist/src/schemas.d.ts +1 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.js +6 -7
- package/dist/src/utils/cleanup-ledger.js +9 -8
- package/dist/src/utils/git-reference.d.ts +12 -0
- package/dist/src/utils/git-reference.js +38 -0
- package/dist/src/utils/polling.d.ts +3 -0
- package/dist/src/utils/polling.js +3 -0
- package/dist/src/utils/project-archive.js +2 -2
- package/dist/src/utils/stable-hash.d.ts +3 -1
- package/dist/src/utils/stable-hash.js +2 -0
- package/package.json +8 -8
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +3 -2
- package/skills/dataiku-dss/references/app-releases.md +2 -3
- package/skills/dataiku-dss/references/authentication.md +7 -7
- package/skills/dataiku-dss/references/coding.md +1 -1
- package/skills/dataiku-dss/references/mutations.md +1 -1
- package/skills/dataiku-dss/references/troubleshooting.md +2 -2
- package/dist/src/utils/pagination.d.ts +0 -8
- package/dist/src/utils/pagination.js +0 -23
|
@@ -117,20 +117,35 @@ export class MlTasksResource extends BaseResource {
|
|
|
117
117
|
const adaptiveEnabled = explicitIntervalMs === undefined;
|
|
118
118
|
const startedAt = Date.now();
|
|
119
119
|
let pollCount = 0;
|
|
120
|
-
|
|
121
|
-
// check, so `timeoutMs: 0` still reports the freshly-started state once.
|
|
120
|
+
let lastStatus;
|
|
122
121
|
while (true) {
|
|
122
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
123
|
+
// The first observation always happens, even when the budget is
|
|
124
|
+
// already spent. A later poll is never started after the deadline:
|
|
125
|
+
// the loop reports the structured timeout from the last observed
|
|
126
|
+
// status instead of issuing a request the budget cannot cover.
|
|
127
|
+
if (lastStatus !== undefined && elapsedBeforeMs >= deadlineMs) {
|
|
128
|
+
return {
|
|
129
|
+
sessionId: session.sessionId,
|
|
130
|
+
trainedModelIds: modelIdsFromStatus(lastStatus),
|
|
131
|
+
timedOut: true,
|
|
132
|
+
success: false,
|
|
133
|
+
state: "RUNNING",
|
|
134
|
+
elapsedMs: elapsedBeforeMs,
|
|
135
|
+
pollCount,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
123
138
|
pollCount += 1;
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
const remainingMs = deadlineMs -
|
|
139
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
140
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
141
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
142
|
+
// reaches the server instead of failing before any attempt.
|
|
143
|
+
const remainingMs = deadlineMs - elapsedBeforeMs;
|
|
129
144
|
let status;
|
|
130
145
|
try {
|
|
131
146
|
status = await this.client.get(`${this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey)}/status`, remainingMs > 0
|
|
132
147
|
? { timeoutMs: remainingMs, }
|
|
133
|
-
:
|
|
148
|
+
: { noRetry: true, });
|
|
134
149
|
}
|
|
135
150
|
catch (error) {
|
|
136
151
|
// The poll budget ran out: report the structured timeout instead
|
|
@@ -147,19 +162,13 @@ export class MlTasksResource extends BaseResource {
|
|
|
147
162
|
pollCount,
|
|
148
163
|
};
|
|
149
164
|
}
|
|
165
|
+
lastStatus = status;
|
|
150
166
|
const elapsedMs = Date.now() - startedAt;
|
|
151
167
|
if (status.training !== false) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
timedOut: true,
|
|
157
|
-
success: false,
|
|
158
|
-
state: "RUNNING",
|
|
159
|
-
elapsedMs,
|
|
160
|
-
pollCount,
|
|
161
|
-
};
|
|
162
|
-
}
|
|
168
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
169
|
+
// timeout from this status without issuing another request.
|
|
170
|
+
if (elapsedMs >= deadlineMs)
|
|
171
|
+
continue;
|
|
163
172
|
const nextDelayMs = computeNextPollDelayMs({
|
|
164
173
|
pollCount,
|
|
165
174
|
baseIntervalMs,
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
2
|
import { JupyterNotebookContentSchema, JupyterNotebookSummaryArraySchema, NotebookSessionArraySchema, SqlNotebookContentSchema, SqlNotebookHistorySchema, SqlNotebookSummaryArraySchema, } from "../schemas.js";
|
|
3
|
-
import { stableHash, } from "../utils/stable-hash.js";
|
|
3
|
+
import { SHA256_HEX_PATTERN, stableHash, } from "../utils/stable-hash.js";
|
|
4
4
|
import { BaseResource, } from "./base.js";
|
|
5
|
-
const EXPECT_HASH_PATTERN = /^[0-9a-fA-F]{64}$/;
|
|
6
5
|
export class NotebooksResource extends BaseResource {
|
|
7
6
|
// ── Jupyter Notebooks ──────────────────────────────────────────────
|
|
8
7
|
/**
|
|
@@ -260,7 +259,7 @@ export class NotebooksResource extends BaseResource {
|
|
|
260
259
|
}
|
|
261
260
|
/** Validate the armed `--expect-hash` value into a comparable hex digest. */
|
|
262
261
|
validatedExpectHash(expectHash, pk, resourceLabel, id) {
|
|
263
|
-
if (!
|
|
262
|
+
if (!SHA256_HEX_PATTERN.test(expectHash)) {
|
|
264
263
|
throw new ClientValidationError(`Expected ${resourceLabel} hash must be a 64-character SHA-256 hex digest.`, "validation_failed", "Use the hash value returned by a previous save or read.", { projectKey: pk, id, });
|
|
265
264
|
}
|
|
266
265
|
return expectHash.toLowerCase();
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { BaseResource } from "./base.js";
|
|
2
|
-
import { validateGitReferencePath } from "./project-git.js";
|
|
3
2
|
/** Summary entry of `/plugins/` as documented (id/version/isDev/meta). */
|
|
4
3
|
export interface PluginSummary {
|
|
5
4
|
id: string;
|
|
@@ -262,4 +261,3 @@ export declare function validatedPluginGitUrl(repository: string, field?: string
|
|
|
262
261
|
* CLI to decide which actions need a plugin id positional.
|
|
263
262
|
*/
|
|
264
263
|
export declare function isPluginIdCandidate(value: unknown): boolean;
|
|
265
|
-
export { validateGitReferencePath, };
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
2
|
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
3
|
-
import { validateGitReferencePath, } from "./project-git.js";
|
|
4
3
|
/* ------------------------------------------------------------------ */
|
|
5
4
|
/* Path helpers */
|
|
6
5
|
/* ------------------------------------------------------------------ */
|
|
@@ -454,5 +453,3 @@ export function validatedPluginGitUrl(repository, field = "Git repository URL")
|
|
|
454
453
|
export function isPluginIdCandidate(value) {
|
|
455
454
|
return typeof value === "string" && PLUGIN_ID_PATTERN.test(value);
|
|
456
455
|
}
|
|
457
|
-
// Re-exported for CLI consumers so a single import site covers path rules.
|
|
458
|
-
export { validateGitReferencePath, };
|
|
@@ -68,23 +68,17 @@ export interface ProjectGitFutureStateOptions {
|
|
|
68
68
|
* loops that must not hang on a stalled endpoint.
|
|
69
69
|
*/
|
|
70
70
|
timeoutMs?: number;
|
|
71
|
+
/**
|
|
72
|
+
* Issue exactly one transport attempt (no retries). Independent of
|
|
73
|
+
* `timeoutMs`: when both are supplied the total budget still applies. Used
|
|
74
|
+
* by the wait loop for the one observation whose wait budget is spent.
|
|
75
|
+
*/
|
|
76
|
+
noRetry?: boolean;
|
|
71
77
|
}
|
|
72
78
|
export interface ProjectGitFutureWaitOptions {
|
|
73
79
|
pollIntervalMs?: number;
|
|
74
80
|
timeoutMs?: number;
|
|
75
81
|
}
|
|
76
|
-
/**
|
|
77
|
-
* Validate and canonicalize a Git library reference path. This is the single
|
|
78
|
-
* validator used by both SDK dispatch and CLI mutation planning.
|
|
79
|
-
*
|
|
80
|
-
* Strip only the permitted outer '/' separators, then require nonempty
|
|
81
|
-
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
82
|
-
* path therefore cannot change which API resource a dispatcher's URL
|
|
83
|
-
* normalization would otherwise select.
|
|
84
|
-
*/
|
|
85
|
-
export declare function validateGitReferencePath(path: string): string;
|
|
86
|
-
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
87
|
-
export declare function encodeGitReferencePath(path: string): string;
|
|
88
82
|
export declare class ProjectGitResource extends BaseResource {
|
|
89
83
|
/** Working-copy status: current branch, remotes, tracking counts, changes. */
|
|
90
84
|
status(projectKey: string): Promise<ProjectGitStatus>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { ClientValidationError, } from "../errors.js";
|
|
1
|
+
import { ClientValidationError, DataikuError, nonJsonResponseBody, } from "../errors.js";
|
|
2
2
|
import { ProjectGitActionResultSchema, ProjectGitDiffResultSchema, ProjectGitFutureResponseSchema, ProjectGitFutureStateSchema, ProjectGitLibrariesSchema, ProjectGitLogResultSchema, ProjectGitRemoteSchema, ProjectGitStatusSchema, ProjectGitTagsSchema, } from "../schemas.js";
|
|
3
|
-
import {
|
|
3
|
+
import { encodeGitReferencePath, validateGitReferencePath, } from "../utils/git-reference.js";
|
|
4
|
+
import { computeNextPollDelayMs, DEFAULT_POLL_INTERVAL_MS, DEFAULT_TIMEOUT_MS, isRequestDeadlineError, } from "../utils/polling.js";
|
|
4
5
|
import { sanitizeErrorSecrets, } from "../utils/secret-sanitize.js";
|
|
5
6
|
import { BaseResource, } from "./base.js";
|
|
6
7
|
/**
|
|
@@ -12,50 +13,11 @@ const GIT_API_BASE = "/dip/publicapi";
|
|
|
12
13
|
const DEFAULT_REMOTE = "origin";
|
|
13
14
|
const DEFAULT_LOG_COUNT = 1_000;
|
|
14
15
|
const DEFAULT_TAG_REFERENCE = "HEAD";
|
|
15
|
-
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
16
|
-
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
17
16
|
function sleep(ms) {
|
|
18
17
|
const { promise, resolve, } = Promise.withResolvers();
|
|
19
18
|
setTimeout(resolve, ms);
|
|
20
19
|
return promise;
|
|
21
20
|
}
|
|
22
|
-
function gitReferencePathError(path, message) {
|
|
23
|
-
return new ClientValidationError(message, "validation_failed", "Use a Git library reference path relative to the project library root: '/' -separated segment names, with no '.' or '..' segments.", { path, });
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Validate and canonicalize a Git library reference path. This is the single
|
|
27
|
-
* validator used by both SDK dispatch and CLI mutation planning.
|
|
28
|
-
*
|
|
29
|
-
* Strip only the permitted outer '/' separators, then require nonempty
|
|
30
|
-
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
31
|
-
* path therefore cannot change which API resource a dispatcher's URL
|
|
32
|
-
* normalization would otherwise select.
|
|
33
|
-
*/
|
|
34
|
-
export function validateGitReferencePath(path) {
|
|
35
|
-
if (typeof path !== "string" || path.trim() === "") {
|
|
36
|
-
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
37
|
-
}
|
|
38
|
-
const normalized = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
39
|
-
if (normalized === "") {
|
|
40
|
-
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
41
|
-
}
|
|
42
|
-
for (const segment of normalized.split("/")) {
|
|
43
|
-
if (segment === "") {
|
|
44
|
-
throw gitReferencePathError(path, "A Git library reference path must not contain empty segments (consecutive or repeated '/').");
|
|
45
|
-
}
|
|
46
|
-
if (segment === "." || segment === "..") {
|
|
47
|
-
throw gitReferencePathError(path, "A Git library reference path must not contain '.' or '..' segments.");
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
return normalized;
|
|
51
|
-
}
|
|
52
|
-
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
53
|
-
export function encodeGitReferencePath(path) {
|
|
54
|
-
return validateGitReferencePath(path)
|
|
55
|
-
.split("/")
|
|
56
|
-
.map((segment) => encodeURIComponent(segment))
|
|
57
|
-
.join("/");
|
|
58
|
-
}
|
|
59
21
|
/**
|
|
60
22
|
* Reject `https://user:token@host/repo` before it reaches DSS: an embedded
|
|
61
23
|
* secret would be stored in the project's Git configuration and echoed back by
|
|
@@ -364,7 +326,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
364
326
|
* what polling loops want; the default consumes it like the Python client.
|
|
365
327
|
*/
|
|
366
328
|
async getFutureState(jobId, options = {}) {
|
|
367
|
-
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`, undefined, [], { timeoutMs: options.timeoutMs, });
|
|
329
|
+
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`, undefined, [], { timeoutMs: options.timeoutMs, noRetry: options.noRetry, });
|
|
368
330
|
return this.client.safeParse(ProjectGitFutureStateSchema, raw ?? {}, "projectGit.getFutureState");
|
|
369
331
|
}
|
|
370
332
|
/** Abort a running future. */
|
|
@@ -389,14 +351,23 @@ export class ProjectGitResource extends BaseResource {
|
|
|
389
351
|
// interval; an explicit pollIntervalMs is an exact contract.
|
|
390
352
|
const adaptiveEnabled = options.pollIntervalMs === undefined;
|
|
391
353
|
let pollCount = 0;
|
|
354
|
+
let lastState;
|
|
392
355
|
const startedAt = Date.now();
|
|
393
356
|
while (true) {
|
|
357
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
358
|
+
// The first observation always happens, even when the budget is
|
|
359
|
+
// already spent. A later poll is never started after the deadline:
|
|
360
|
+
// the loop reports the documented timeout from the last observed
|
|
361
|
+
// state instead of issuing a request the budget cannot cover.
|
|
362
|
+
if (lastState !== undefined && elapsedBeforeMs >= timeoutMs) {
|
|
363
|
+
throw new ClientValidationError(`Timed out after ${String(elapsedBeforeMs)}ms waiting for Dataiku future ${jobId}`, "long_running_failure");
|
|
364
|
+
}
|
|
394
365
|
// Unlike peek state, the ordinary state includes the completed result.
|
|
395
|
-
//
|
|
396
|
-
//
|
|
397
|
-
//
|
|
398
|
-
//
|
|
399
|
-
const remainingMs = timeoutMs -
|
|
366
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
367
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
368
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
369
|
+
// reaches the server instead of failing before any attempt.
|
|
370
|
+
const remainingMs = timeoutMs - elapsedBeforeMs;
|
|
400
371
|
pollCount += 1;
|
|
401
372
|
let state;
|
|
402
373
|
try {
|
|
@@ -405,7 +376,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
405
376
|
peek: false,
|
|
406
377
|
timeoutMs: remainingMs,
|
|
407
378
|
}
|
|
408
|
-
: { peek: false, });
|
|
379
|
+
: { peek: false, noRetry: true, });
|
|
409
380
|
}
|
|
410
381
|
catch (error) {
|
|
411
382
|
// The poll budget ran out mid-request: report the documented
|
|
@@ -413,27 +384,29 @@ export class ProjectGitResource extends BaseResource {
|
|
|
413
384
|
// error escape.
|
|
414
385
|
if (!isRequestDeadlineError(error, startedAt + timeoutMs))
|
|
415
386
|
throw error;
|
|
416
|
-
throw new
|
|
387
|
+
throw new ClientValidationError(`Timed out after ${String(Date.now() - startedAt)}ms waiting for Dataiku future ${jobId}`, "long_running_failure", undefined, undefined, { cause: error, });
|
|
417
388
|
}
|
|
389
|
+
lastState = state;
|
|
418
390
|
const failure = describeFutureFailure(state.error);
|
|
419
391
|
if (failure !== undefined) {
|
|
420
|
-
throw new
|
|
392
|
+
throw new ClientValidationError(`Dataiku future ${jobId} failed: ${failure}`, "long_running_failure");
|
|
421
393
|
}
|
|
422
394
|
if (state.aborted === true) {
|
|
423
|
-
throw new
|
|
395
|
+
throw new ClientValidationError(`Dataiku future ${jobId} was aborted`, "long_running_failure");
|
|
424
396
|
}
|
|
425
397
|
if (state.hasResult === true)
|
|
426
398
|
return state.result;
|
|
427
399
|
if (state.unknown === true) {
|
|
428
|
-
throw new
|
|
400
|
+
throw new ClientValidationError(`Dataiku future ${jobId} is unknown to the server`, "long_running_failure");
|
|
429
401
|
}
|
|
430
402
|
if (state.alive === false) {
|
|
431
|
-
throw new
|
|
403
|
+
throw new ClientValidationError(`Dataiku future ${jobId} ended without producing a result`, "long_running_failure");
|
|
432
404
|
}
|
|
433
405
|
const elapsedMs = Date.now() - startedAt;
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
406
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
407
|
+
// timeout from this observation without issuing another request.
|
|
408
|
+
if (elapsedMs >= timeoutMs)
|
|
409
|
+
continue;
|
|
437
410
|
const nextDelayMs = computeNextPollDelayMs({
|
|
438
411
|
pollCount,
|
|
439
412
|
baseIntervalMs: pollIntervalMs,
|
|
@@ -466,7 +439,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
466
439
|
}
|
|
467
440
|
async request(method, path, body, secrets = [], opts = {}) {
|
|
468
441
|
try {
|
|
469
|
-
const res = await this.client.requestGit(method, `${GIT_API_BASE}${path}`, body, opts.timeoutMs === undefined ? undefined : Date.now() + opts.timeoutMs);
|
|
442
|
+
const res = await this.client.requestGit(method, `${GIT_API_BASE}${path}`, body, opts.timeoutMs === undefined ? undefined : Date.now() + opts.timeoutMs, opts.noRetry === true);
|
|
470
443
|
const text = await res.text();
|
|
471
444
|
if (text.trim() === "")
|
|
472
445
|
return undefined;
|
|
@@ -474,7 +447,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
474
447
|
return JSON.parse(text);
|
|
475
448
|
}
|
|
476
449
|
catch {
|
|
477
|
-
throw new
|
|
450
|
+
throw new DataikuError(res.status, "Invalid JSON response", nonJsonResponseBody(text));
|
|
478
451
|
}
|
|
479
452
|
}
|
|
480
453
|
catch (error) {
|
|
@@ -75,8 +75,6 @@ export interface ProjectLibraryDiffResult {
|
|
|
75
75
|
* checked; it can never turn the subsequent POST into a conditional write.
|
|
76
76
|
*/
|
|
77
77
|
export declare const PROJECT_LIBRARY_CONCURRENCY_CONTROL = "client-side-non-atomic-stale-read-check";
|
|
78
|
-
/** Lowercase or uppercase SHA-256 hex digest. */
|
|
79
|
-
export declare const EXPECT_SHA256_PATTERN: RegExp;
|
|
80
78
|
/**
|
|
81
79
|
* Validate and canonicalize a project library path.
|
|
82
80
|
*
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createHash, } from "node:crypto";
|
|
2
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
|
+
import { SHA256_HEX_PATTERN, sha256Hex, } from "../utils/stable-hash.js";
|
|
3
3
|
import { BaseResource, } from "./base.js";
|
|
4
4
|
/**
|
|
5
5
|
* The only concurrency control the public project-library API permits. DSS
|
|
@@ -8,11 +8,6 @@ import { BaseResource, } from "./base.js";
|
|
|
8
8
|
* checked; it can never turn the subsequent POST into a conditional write.
|
|
9
9
|
*/
|
|
10
10
|
export const PROJECT_LIBRARY_CONCURRENCY_CONTROL = "client-side-non-atomic-stale-read-check";
|
|
11
|
-
/** Lowercase or uppercase SHA-256 hex digest. */
|
|
12
|
-
export const EXPECT_SHA256_PATTERN = /^[0-9a-fA-F]{64}$/;
|
|
13
|
-
function sha256Hex(value) {
|
|
14
|
-
return createHash("sha256").update(value).digest("hex");
|
|
15
|
-
}
|
|
16
11
|
const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/;
|
|
17
12
|
const BACKSLASH_RE = /\\/;
|
|
18
13
|
const TRAILING_AMBIGUOUS_RE = /[\s.]$/;
|
|
@@ -376,7 +371,7 @@ export class ProjectLibraryResource extends BaseResource {
|
|
|
376
371
|
}
|
|
377
372
|
const expectSha256 = options?.expectSha256;
|
|
378
373
|
if (expectSha256 !== undefined
|
|
379
|
-
&& (typeof expectSha256 !== "string" || !
|
|
374
|
+
&& (typeof expectSha256 !== "string" || !SHA256_HEX_PATTERN.test(expectSha256))) {
|
|
380
375
|
throw new ClientValidationError("expectSha256 must be a 64-character SHA-256 hex digest.", "validation_failed", "Use the sha256 value returned by a project library read or diff.", { path: valid, });
|
|
381
376
|
}
|
|
382
377
|
let beforeSha256;
|
|
@@ -150,8 +150,10 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
150
150
|
* Build a normalized, optionally truncated flow map for a project.
|
|
151
151
|
*
|
|
152
152
|
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
153
|
-
* managed folders) in parallel.
|
|
154
|
-
*
|
|
153
|
+
* managed folders) in parallel. Each metadata fetch runs under a total
|
|
154
|
+
* `timeoutMs` budget: when the budget expires the in-flight request is
|
|
155
|
+
* aborted and the map is returned without that metadata, reported as a
|
|
156
|
+
* warning.
|
|
155
157
|
*/
|
|
156
158
|
map(opts?: FlowMapOptions & {
|
|
157
159
|
projectKey?: string;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
1
|
+
import { ClientValidationError, DataikuError, unexpectedResponseError, } from "../errors.js";
|
|
2
|
+
import { isRequestDeadlineError, } from "../utils/polling.js";
|
|
2
3
|
import { writeResponseToFile, } from "../utils/response-file.js";
|
|
3
4
|
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, ProjectTagsSchema, } from "../schemas.js";
|
|
4
5
|
import { analyzeFlowMap, flowTopologyFingerprint, renderFlowMap, } from "../utils/flow-analysis.js";
|
|
@@ -6,34 +7,31 @@ import { normalizeFlowGraph, } from "../utils/flow-map.js";
|
|
|
6
7
|
import { assertImportableProjectArchive, } from "../utils/project-archive.js";
|
|
7
8
|
import { projectIncarnationHash, } from "../utils/project-incarnation.js";
|
|
8
9
|
import { BaseResource, } from "./base.js";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Runs an optional metadata fetch under a total budget. The fetcher receives
|
|
12
|
+
* the client GET options and must pass `timeoutMs` through to its request, so
|
|
13
|
+
* the client owns the deadline: the in-flight request is aborted when the
|
|
14
|
+
* budget expires and no further retry is started afterwards. Expiry of the
|
|
15
|
+
* total budget resolves to a warning so the map still returns; an earlier
|
|
16
|
+
* per-request failure is reported as "unavailable" (classified with the same
|
|
17
|
+
* deadline check the wait loops use).
|
|
18
|
+
*/
|
|
19
|
+
async function fetchWithTimeout(label, timeoutMs, fetcher) {
|
|
20
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
21
|
+
try {
|
|
22
|
+
return { value: await fetcher({ timeoutMs, }), };
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
if (isRequestDeadlineError(error, deadlineAt)) {
|
|
26
|
+
return {
|
|
17
27
|
warning: `${label} metadata timed out after ${timeoutMs}ms; continuing without it.`,
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
resolve({ value, });
|
|
26
|
-
}, (error) => {
|
|
27
|
-
if (settled)
|
|
28
|
-
return;
|
|
29
|
-
settled = true;
|
|
30
|
-
clearTimeout(timer);
|
|
31
|
-
const detail = error instanceof Error ? error.message : String(error);
|
|
32
|
-
resolve({
|
|
33
|
-
warning: `${label} metadata unavailable: ${detail}`,
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
});
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
31
|
+
return {
|
|
32
|
+
warning: `${label} metadata unavailable: ${detail}`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
37
35
|
}
|
|
38
36
|
function computeRootsAndLeaves(nodes, edges) {
|
|
39
37
|
const inDegree = new Map();
|
|
@@ -142,8 +140,9 @@ export class ProjectsResource extends BaseResource {
|
|
|
142
140
|
/** Export a project archive to a local file. */
|
|
143
141
|
async exportArchive(projectKey, filePath, options) {
|
|
144
142
|
const res = await this.client.postStream(`/public/api/projects/${encodeURIComponent(projectKey)}/export`, options ?? {});
|
|
145
|
-
if (!res.body)
|
|
146
|
-
throw
|
|
143
|
+
if (!res.body) {
|
|
144
|
+
throw unexpectedResponseError("projects.exportArchive response did not include a body");
|
|
145
|
+
}
|
|
147
146
|
await writeResponseToFile(filePath, res);
|
|
148
147
|
}
|
|
149
148
|
/** Upload a project archive and return its temporary import handle. */
|
|
@@ -343,8 +342,10 @@ export class ProjectsResource extends BaseResource {
|
|
|
343
342
|
* Build a normalized, optionally truncated flow map for a project.
|
|
344
343
|
*
|
|
345
344
|
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
346
|
-
* managed folders) in parallel.
|
|
347
|
-
*
|
|
345
|
+
* managed folders) in parallel. Each metadata fetch runs under a total
|
|
346
|
+
* `timeoutMs` budget: when the budget expires the in-flight request is
|
|
347
|
+
* aborted and the map is returned without that metadata, reported as a
|
|
348
|
+
* warning.
|
|
348
349
|
*/
|
|
349
350
|
async map(opts) {
|
|
350
351
|
const enc = this.enc(opts?.projectKey);
|
|
@@ -352,10 +353,10 @@ export class ProjectsResource extends BaseResource {
|
|
|
352
353
|
const timeoutMs = DEFAULT_METADATA_TIMEOUT_MS;
|
|
353
354
|
const [rawGraph, foldersMeta, datasetsMeta, recipesMeta, zonesMeta,] = await Promise.all([
|
|
354
355
|
this.client.get(`/public/api/projects/${enc}/flow/graph/`),
|
|
355
|
-
fetchWithTimeout("Managed folders", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/managedfolders
|
|
356
|
-
fetchWithTimeout("Datasets", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/datasets
|
|
357
|
-
fetchWithTimeout("Recipes", timeoutMs, () => this.client.get(`/public/api/projects/${enc}/recipes
|
|
358
|
-
fetchWithTimeout("Flow zones", timeoutMs, () => this.client.flowZones.list(pk)),
|
|
356
|
+
fetchWithTimeout("Managed folders", timeoutMs, (getOptions) => this.client.get(`/public/api/projects/${enc}/managedfolders/`, getOptions)),
|
|
357
|
+
fetchWithTimeout("Datasets", timeoutMs, (getOptions) => this.client.get(`/public/api/projects/${enc}/datasets/`, getOptions)),
|
|
358
|
+
fetchWithTimeout("Recipes", timeoutMs, (getOptions) => this.client.get(`/public/api/projects/${enc}/recipes/`, getOptions)),
|
|
359
|
+
fetchWithTimeout("Flow zones", timeoutMs, (getOptions) => this.client.flowZones.list(pk, getOptions)),
|
|
359
360
|
]);
|
|
360
361
|
// Build folder name lookup
|
|
361
362
|
const folderNamesById = {};
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
3
|
import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
|
|
4
|
+
type JoinType = "LEFT" | "INNER" | "RIGHT" | "FULL";
|
|
5
|
+
type JoinKeyPair = {
|
|
6
|
+
left: string;
|
|
7
|
+
right: string;
|
|
8
|
+
};
|
|
4
9
|
export interface RecipeRunOutput extends JobBuildTarget {
|
|
5
10
|
ref: string;
|
|
6
11
|
role: string;
|
|
@@ -65,6 +70,28 @@ export interface RecipeCloneResult {
|
|
|
65
70
|
payloadTextRewrites: Record<string, string>;
|
|
66
71
|
copiedOutputDatasets: string[];
|
|
67
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Pure request construction for POST /recipes/ (no DSS calls). Shared with
|
|
75
|
+
* `recipe create --plan` so the plan equals the request the command sends.
|
|
76
|
+
*/
|
|
77
|
+
export declare function buildRecipeCreateRequest(opts: RecipeCreateOptions, pk: string): {
|
|
78
|
+
creationSettings: Record<string, unknown>;
|
|
79
|
+
fuzzyDistance: string | undefined;
|
|
80
|
+
fuzzyKeys: JoinKeyPair[] | undefined;
|
|
81
|
+
fuzzyThreshold: number;
|
|
82
|
+
inputDatasets: string[] | undefined;
|
|
83
|
+
inputs: Record<string, unknown>;
|
|
84
|
+
joinKeys: JoinKeyPair[] | undefined;
|
|
85
|
+
name: string;
|
|
86
|
+
normalizedJoinType: JoinType | undefined;
|
|
87
|
+
outputFolder: string | undefined;
|
|
88
|
+
outputs: Record<string, unknown>;
|
|
89
|
+
payload: string | undefined;
|
|
90
|
+
rawConnection: string | undefined;
|
|
91
|
+
recipePrototype: Record<string, unknown>;
|
|
92
|
+
temporaryOutputDataset: string | undefined;
|
|
93
|
+
type: string;
|
|
94
|
+
};
|
|
68
95
|
export declare class RecipesResource extends BaseResource {
|
|
69
96
|
/** List all recipes in a project. */
|
|
70
97
|
list(projectKey?: string): Promise<RecipeSummary[]>;
|
|
@@ -144,3 +171,4 @@ export declare class RecipesResource extends BaseResource {
|
|
|
144
171
|
projectKey?: string;
|
|
145
172
|
}): Promise<string>;
|
|
146
173
|
}
|
|
174
|
+
export {};
|