dataiku-sdk 3.6.1 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/bin/bun-version.js +14 -0
- package/bin/dss.js +11 -15
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +114 -5
- package/dist/packages/types/src/index.js +75 -2
- package/dist/src/cli/coerce.d.ts +2 -0
- package/dist/src/cli/coerce.js +20 -16
- package/dist/src/cli/command-syntax.json +3369 -144
- package/dist/src/cli/commands/analysis.js +19 -1
- package/dist/src/cli/commands/bundle.js +22 -0
- package/dist/src/cli/commands/connection.js +26 -2
- package/dist/src/cli/commands/data-quality.js +33 -1
- package/dist/src/cli/commands/dataset.js +76 -9
- package/dist/src/cli/commands/discussion.js +15 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/flow-zone.js +241 -46
- package/dist/src/cli/commands/flow.d.ts +5 -0
- package/dist/src/cli/commands/flow.js +57 -0
- package/dist/src/cli/commands/future.js +8 -0
- package/dist/src/cli/commands/index.js +6 -0
- package/dist/src/cli/commands/job.js +44 -6
- package/dist/src/cli/commands/metrics.js +21 -1
- package/dist/src/cli/commands/ml-task.d.ts +5 -0
- package/dist/src/cli/commands/ml-task.js +86 -16
- package/dist/src/cli/commands/mlflow.d.ts +11 -0
- package/dist/src/cli/commands/mlflow.js +121 -0
- package/dist/src/cli/commands/plugin.js +5 -11
- package/dist/src/cli/commands/project-deployer.js +70 -0
- package/dist/src/cli/commands/project-library.js +2 -5
- package/dist/src/cli/commands/project.js +8 -0
- package/dist/src/cli/commands/recipe.d.ts +13 -0
- package/dist/src/cli/commands/recipe.js +130 -7
- package/dist/src/cli/commands/saved-model.d.ts +1 -1
- package/dist/src/cli/commands/saved-model.js +9 -76
- package/dist/src/cli/commands/streaming-endpoint.js +30 -1
- package/dist/src/cli/commands/webapp.js +8 -0
- package/dist/src/cli/commands/wiki.js +78 -0
- package/dist/src/cli/commands/workspace.js +8 -0
- package/dist/src/cli/contract.d.ts +16 -15
- package/dist/src/cli/contract.js +150 -3407
- package/dist/src/cli/doctor.d.ts +13 -0
- package/dist/src/cli/doctor.js +43 -1
- package/dist/src/cli/flags.d.ts +14 -1
- package/dist/src/cli/flags.js +106 -8
- package/dist/src/cli/helpers/app-successor.js +1 -2
- package/dist/src/cli/helpers/dataset.d.ts +12 -0
- package/dist/src/cli/helpers/dataset.js +36 -0
- package/dist/src/cli/helpers/documentation.d.ts +21 -0
- package/dist/src/cli/helpers/documentation.js +53 -0
- package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
- package/dist/src/cli/helpers/flow-zone.js +144 -4
- package/dist/src/cli/helpers/trained-model.d.ts +26 -0
- package/dist/src/cli/helpers/trained-model.js +144 -0
- package/dist/src/cli/list-output.d.ts +7 -0
- package/dist/src/cli/list-output.js +77 -0
- package/dist/src/cli/list-shapes.d.ts +21 -0
- package/dist/src/cli/list-shapes.js +111 -0
- package/dist/src/cli/output-schemas.d.ts +0 -14
- package/dist/src/cli/output-schemas.js +4 -7
- package/dist/src/cli/output.d.ts +0 -1
- package/dist/src/cli/output.js +33 -19
- package/dist/src/cli/plans.d.ts +14 -0
- package/dist/src/cli/plans.js +3740 -0
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +7 -0
- package/dist/src/cli/usage.js +20 -1
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +2 -2
- package/dist/src/cli.js +137 -115
- package/dist/src/client.d.ts +52 -43
- package/dist/src/client.js +267 -96
- package/dist/src/errors.d.ts +22 -2
- package/dist/src/errors.js +49 -50
- package/dist/src/generated/action-output-schemas.json +345 -89
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +3 -0
- package/dist/src/resources/analyses.d.ts +8 -0
- package/dist/src/resources/analyses.js +13 -1
- package/dist/src/resources/bundles.d.ts +34 -0
- package/dist/src/resources/bundles.js +67 -0
- package/dist/src/resources/connections.d.ts +9 -0
- package/dist/src/resources/connections.js +15 -0
- package/dist/src/resources/data-quality.d.ts +20 -0
- package/dist/src/resources/data-quality.js +30 -0
- package/dist/src/resources/dataset-create.d.ts +38 -0
- package/dist/src/resources/dataset-create.js +135 -0
- package/dist/src/resources/datasets.d.ts +46 -15
- package/dist/src/resources/datasets.js +120 -112
- package/dist/src/resources/discussions.d.ts +5 -0
- package/dist/src/resources/discussions.js +7 -0
- package/dist/src/resources/documentation.d.ts +25 -0
- package/dist/src/resources/documentation.js +16 -0
- package/dist/src/resources/flow-zones.d.ts +29 -3
- package/dist/src/resources/flow-zones.js +53 -3
- package/dist/src/resources/flow.d.ts +48 -0
- package/dist/src/resources/flow.js +65 -0
- package/dist/src/resources/futures.d.ts +23 -0
- package/dist/src/resources/futures.js +32 -0
- package/dist/src/resources/jobs.d.ts +34 -0
- package/dist/src/resources/jobs.js +170 -3
- package/dist/src/resources/metrics.d.ts +10 -0
- package/dist/src/resources/metrics.js +10 -0
- package/dist/src/resources/ml-tasks.d.ts +65 -2
- package/dist/src/resources/ml-tasks.js +102 -28
- package/dist/src/resources/mlflow-extension.d.ts +73 -0
- package/dist/src/resources/mlflow-extension.js +105 -0
- package/dist/src/resources/projects.d.ts +15 -0
- package/dist/src/resources/projects.js +18 -3
- package/dist/src/resources/recipe-create.d.ts +79 -0
- package/dist/src/resources/recipe-create.js +454 -0
- package/dist/src/resources/recipes.d.ts +28 -29
- package/dist/src/resources/recipes.js +201 -267
- package/dist/src/resources/saved-models.d.ts +10 -6
- package/dist/src/resources/saved-models.js +16 -14
- package/dist/src/resources/scenarios.js +8 -14
- package/dist/src/resources/sql.js +52 -13
- package/dist/src/resources/streaming-endpoints.d.ts +12 -0
- package/dist/src/resources/streaming-endpoints.js +15 -0
- package/dist/src/resources/trained-model.d.ts +60 -0
- package/dist/src/resources/trained-model.js +98 -0
- package/dist/src/resources/webapps.d.ts +8 -0
- package/dist/src/resources/webapps.js +10 -0
- package/dist/src/resources/wiki.d.ts +52 -1
- package/dist/src/resources/wiki.js +96 -0
- package/dist/src/resources/workspaces.d.ts +2 -0
- package/dist/src/resources/workspaces.js +4 -0
- package/dist/src/utils/flow-analysis.js +4 -0
- package/dist/src/utils/response-file.js +20 -9
- package/dist/src/utils/secret-sanitize.d.ts +7 -0
- package/dist/src/utils/secret-sanitize.js +46 -0
- package/package.json +25 -26
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
- package/skills/dataiku-dss/references/authentication.md +3 -1
- package/skills/dataiku-dss/references/coding.md +1 -0
- package/skills/dataiku-dss/references/discovery.md +4 -6
- package/skills/dataiku-dss/references/flow-maps.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +3 -2
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { ClientValidationError, } from "../errors.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
import { startDocumentation, } from "./documentation.js";
|
|
4
|
+
import { dssTask, } from "./futures.js";
|
|
5
|
+
/**
|
|
6
|
+
* Flow-wide tools: schema propagation from a dataset through every downstream
|
|
7
|
+
* recipe, and flow documentation export. Zones and the flow graph live on
|
|
8
|
+
* `flowZones`; per-recipe schema updates on `recipes.updateSchema`.
|
|
9
|
+
*/
|
|
10
|
+
export class FlowResource extends BaseResource {
|
|
11
|
+
flowPath(projectKey) {
|
|
12
|
+
return `/public/api/projects/${this.enc(projectKey)}/flow`;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Start propagating `datasetName`'s schema through the flow
|
|
16
|
+
* (POST /flow/tools/propagate-schema/, dataikuapi
|
|
17
|
+
* `new_schema_propagation(...).start()`). Returns the DSS future.
|
|
18
|
+
*/
|
|
19
|
+
async propagateSchema(datasetName, opts = {}) {
|
|
20
|
+
const name = datasetName.trim();
|
|
21
|
+
if (name.length === 0) {
|
|
22
|
+
throw new ClientValidationError("propagateSchema requires a dataset name.", "validation_failed");
|
|
23
|
+
}
|
|
24
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
25
|
+
const raw = await this.client.post(`${this.flowPath(pk)}/tools/propagate-schema/`, schemaPropagationBody(pk, name, opts));
|
|
26
|
+
return dssTask(raw, "Schema propagation");
|
|
27
|
+
}
|
|
28
|
+
async propagateSchemaAndWait(datasetName, opts = {}) {
|
|
29
|
+
return this.client.futures.waitTask(await this.propagateSchema(datasetName, opts), opts);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Start generating the flow documentation (docx) from the DSS default
|
|
33
|
+
* template, an uploaded docx template, or a docx in a managed folder.
|
|
34
|
+
* The finished future's result carries the `exportId` to download.
|
|
35
|
+
*/
|
|
36
|
+
async generateDocumentation(template = { kind: "default", }, projectKey) {
|
|
37
|
+
const base = `${this.flowPath(projectKey)}/documentation`;
|
|
38
|
+
return startDocumentation(this.client, {
|
|
39
|
+
default: `${base}/generate`,
|
|
40
|
+
file: `${base}/generate-with-template`,
|
|
41
|
+
folder: `${base}/generate-with-template-in-folder`,
|
|
42
|
+
}, template, "Flow documentation generation");
|
|
43
|
+
}
|
|
44
|
+
/** GET a generated flow documentation (docx) as a binary stream. */
|
|
45
|
+
async downloadDocumentation(exportId, projectKey) {
|
|
46
|
+
return this.client.stream(`${this.flowPath(projectKey)}/documentation/generated/${encodeURIComponent(exportId)}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/** Request body of POST /flow/tools/propagate-schema/ (pure; shared with `--plan`). */
|
|
50
|
+
export function schemaPropagationBody(projectKey, datasetName, opts) {
|
|
51
|
+
return {
|
|
52
|
+
options: {
|
|
53
|
+
recipeUpdateOptions: {
|
|
54
|
+
byType: opts.recipeUpdateOptions?.byType ?? {},
|
|
55
|
+
byName: opts.recipeUpdateOptions?.byName ?? {},
|
|
56
|
+
},
|
|
57
|
+
defaultPartitionValuesByDimension: opts.defaultPartitionValuesByDimension ?? {},
|
|
58
|
+
partitionsByComputable: opts.partitionsByComputable ?? {},
|
|
59
|
+
excludedRecipes: opts.excludedRecipes ?? [],
|
|
60
|
+
markAsOkRecipes: opts.markAsOkRecipes ?? [],
|
|
61
|
+
autoRebuild: opts.autoRebuild ?? true,
|
|
62
|
+
},
|
|
63
|
+
sources: [{ projectKey, id: datasetName, },],
|
|
64
|
+
};
|
|
65
|
+
}
|
|
@@ -4,7 +4,30 @@ export interface FutureWaitOptions {
|
|
|
4
4
|
pollIntervalMs?: number;
|
|
5
5
|
timeoutMs?: number;
|
|
6
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* A DSS long task as started (dataikuapi `DSSFuture.from_resp`): `jobId`
|
|
9
|
+
* addresses a running task in `futures.wait`; a task DSS finished inline comes
|
|
10
|
+
* back without one, as its final state (`hasResult`, `result`).
|
|
11
|
+
*/
|
|
12
|
+
export interface DssTask {
|
|
13
|
+
jobId?: string;
|
|
14
|
+
hasResult?: boolean;
|
|
15
|
+
result?: unknown;
|
|
16
|
+
[key: string]: unknown;
|
|
17
|
+
}
|
|
18
|
+
/** Validate a task-start response: an object whose `jobId`, when present, is a string. */
|
|
19
|
+
export declare function dssTask(raw: unknown, operation: string): DssTask;
|
|
7
20
|
export declare class FuturesResource extends BaseResource {
|
|
21
|
+
/**
|
|
22
|
+
* Long tasks in progress (GET /futures/): the caller's own by default,
|
|
23
|
+
* every user's with `allUsers`, running scenarios too with `withScenarios`.
|
|
24
|
+
*/
|
|
25
|
+
list(opts?: {
|
|
26
|
+
allUsers?: boolean;
|
|
27
|
+
withScenarios?: boolean;
|
|
28
|
+
}): Promise<FutureState[]>;
|
|
29
|
+
/** Wait for a started task; a task DSS already finished inline is returned as is. */
|
|
30
|
+
waitTask(task: DssTask, opts?: FutureWaitOptions): Promise<FutureWaitResult | DssTask>;
|
|
8
31
|
get(futureId: string, opts?: {
|
|
9
32
|
timeoutMs?: number;
|
|
10
33
|
noRetry?: boolean;
|
|
@@ -1,6 +1,18 @@
|
|
|
1
|
+
import { unexpectedResponseError, } from "../errors.js";
|
|
1
2
|
import { FutureStateSchema, FutureWaitResultSchema, } from "../schemas.js";
|
|
2
3
|
import { computeNextPollDelayMs, DEFAULT_POLL_INTERVAL_MS, DEFAULT_TIMEOUT_MS, isRequestDeadlineError, } from "../utils/polling.js";
|
|
3
4
|
import { BaseResource, } from "./base.js";
|
|
5
|
+
/** Validate a task-start response: an object whose `jobId`, when present, is a string. */
|
|
6
|
+
export function dssTask(raw, operation) {
|
|
7
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
8
|
+
throw unexpectedResponseError(`${operation} returned no task state.`);
|
|
9
|
+
}
|
|
10
|
+
const state = { ...raw, };
|
|
11
|
+
if (state.jobId !== undefined && typeof state.jobId !== "string") {
|
|
12
|
+
throw unexpectedResponseError(`${operation} returned a non-string jobId.`);
|
|
13
|
+
}
|
|
14
|
+
return state;
|
|
15
|
+
}
|
|
4
16
|
function sleep(ms) {
|
|
5
17
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
6
18
|
}
|
|
@@ -22,6 +34,26 @@ function waitState(state) {
|
|
|
22
34
|
return "RUNNING";
|
|
23
35
|
}
|
|
24
36
|
export class FuturesResource extends BaseResource {
|
|
37
|
+
/**
|
|
38
|
+
* Long tasks in progress (GET /futures/): the caller's own by default,
|
|
39
|
+
* every user's with `allUsers`, running scenarios too with `withScenarios`.
|
|
40
|
+
*/
|
|
41
|
+
async list(opts = {}) {
|
|
42
|
+
const query = new URLSearchParams({
|
|
43
|
+
allUsers: String(opts.allUsers === true),
|
|
44
|
+
withScenarios: String(opts.withScenarios === true),
|
|
45
|
+
});
|
|
46
|
+
return this.client.get(`/public/api/futures/?${query}`);
|
|
47
|
+
}
|
|
48
|
+
/** Wait for a started task; a task DSS already finished inline is returned as is. */
|
|
49
|
+
async waitTask(task, opts = {}) {
|
|
50
|
+
if (task.jobId === undefined)
|
|
51
|
+
return task;
|
|
52
|
+
return this.wait(task.jobId, {
|
|
53
|
+
pollIntervalMs: opts.pollIntervalMs,
|
|
54
|
+
timeoutMs: opts.timeoutMs,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
25
57
|
async get(futureId, opts = {}) {
|
|
26
58
|
return this.state(futureId, {
|
|
27
59
|
peek: false,
|
|
@@ -23,12 +23,30 @@ export interface JobLogSummary {
|
|
|
23
23
|
}
|
|
24
24
|
/** Why a requested job log could not be retrieved. */
|
|
25
25
|
export type JobLogUnavailableReason = "not_found" | "error";
|
|
26
|
+
/**
|
|
27
|
+
* A requested build target that no flow object can ever produce: a managed
|
|
28
|
+
* dataset with no recipe writing it.
|
|
29
|
+
*/
|
|
30
|
+
export interface JobUnbuildableTarget {
|
|
31
|
+
type: "DATASET";
|
|
32
|
+
id: string;
|
|
33
|
+
projectKey: string;
|
|
34
|
+
reason: "no_producing_recipe";
|
|
35
|
+
}
|
|
36
|
+
/** Why a job that ended DONE is reported as failed because it built nothing. */
|
|
37
|
+
export interface JobNothingToBuildFailure {
|
|
38
|
+
code: "nothing_to_build";
|
|
39
|
+
message: string;
|
|
40
|
+
targets: JobUnbuildableTarget[];
|
|
41
|
+
}
|
|
26
42
|
/** Result of a completed job wait, including log-unavailable metadata. */
|
|
27
43
|
export type JobWaitOutcome = JobWaitResult & {
|
|
28
44
|
logSummary?: JobLogSummary;
|
|
29
45
|
logUnavailable?: JobLogUnavailableReason;
|
|
30
46
|
/** True when the job itself no longer exists on the server. */
|
|
31
47
|
removed?: boolean;
|
|
48
|
+
/** Set (with `success: false`) when a DONE job ran no activity and a target has no producer. */
|
|
49
|
+
failure?: JobNothingToBuildFailure;
|
|
32
50
|
};
|
|
33
51
|
export interface JobBuildTarget {
|
|
34
52
|
id: string;
|
|
@@ -120,6 +138,12 @@ export declare class JobsResource extends BaseResource {
|
|
|
120
138
|
* confirmed gone. Explicit `includeLogs`/`summary` requests stay honest:
|
|
121
139
|
* `log`/`logSummary` are only present when logs were actually retrieved.
|
|
122
140
|
*
|
|
141
|
+
* A DONE job that ran no activity (`progress.total === 0`) is also what DSS
|
|
142
|
+
* reports for an already-up-to-date build, so it stays successful unless an
|
|
143
|
+
* output is a managed dataset that no recipe writes: then nothing can ever
|
|
144
|
+
* build it, and the outcome is `success: false` with
|
|
145
|
+
* `failure.code === "nothing_to_build"`.
|
|
146
|
+
*
|
|
123
147
|
* On timeout, returns `{ success: false, ... }` rather than throwing.
|
|
124
148
|
*/
|
|
125
149
|
wait(jobId: string, opts?: {
|
|
@@ -133,6 +157,16 @@ export declare class JobsResource extends BaseResource {
|
|
|
133
157
|
timeoutMs?: number;
|
|
134
158
|
projectKey?: string;
|
|
135
159
|
}): Promise<JobWaitOutcome>;
|
|
160
|
+
/**
|
|
161
|
+
* Dataset outputs of a job that ran no activity which nothing can produce.
|
|
162
|
+
* DSS ends such a job DONE exactly like an already-up-to-date build, so the
|
|
163
|
+
* job status cannot tell them apart; the flow graph can: a target is
|
|
164
|
+
* unbuildable when it is a managed dataset (DSS owns its storage, so a
|
|
165
|
+
* recipe is the only way to fill it) that no recipe writes. Unmanaged
|
|
166
|
+
* datasets (uploads, external tables) are sources and legitimately build
|
|
167
|
+
* nothing. Lookups that fail leave the DONE outcome untouched.
|
|
168
|
+
*/
|
|
169
|
+
private unbuildableTargets;
|
|
136
170
|
/**
|
|
137
171
|
* Probe whether a job still exists. Returns `true` when the server reports
|
|
138
172
|
* it as not found, `false` when it is still visible, and `undefined` when
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
2
|
import { JobSummaryArraySchema, } from "../schemas.js";
|
|
3
3
|
import { computeNextPollDelayMs, DEFAULT_POLL_INTERVAL_MS, DEFAULT_TIMEOUT_MS, isRequestDeadlineError, } from "../utils/polling.js";
|
|
4
|
+
import { asRecord, } from "../utils/records.js";
|
|
4
5
|
import { BaseResource, } from "./base.js";
|
|
5
6
|
const DEFAULT_MAX_LOG_LINES = 500;
|
|
6
7
|
/**
|
|
@@ -73,6 +74,36 @@ function jobLogLines(log) {
|
|
|
73
74
|
*/
|
|
74
75
|
const LEGACY_STDOUT_RECORD = /^\s*stdout:/i;
|
|
75
76
|
const LEGACY_STDERR_RECORD = /^\s*stderr:/i;
|
|
77
|
+
/**
|
|
78
|
+
* A DSS backend log record starts with a timestamped header line
|
|
79
|
+
* (`[YYYY/MM/DD-HH:MM:SS.mmm]`); every following non-timestamped line is a
|
|
80
|
+
* continuation of that record (exception chain, JVM frames, ...).
|
|
81
|
+
*/
|
|
82
|
+
const DSS_RECORD_HEADER = /^\[\d{4}\/\d{2}\/\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}\]/;
|
|
83
|
+
const ERROR_KEYWORDS = /\b(error|failed|failure|exception|traceback)\b/i;
|
|
84
|
+
/**
|
|
85
|
+
* Exception class names ending in the `Exception`/`Error` suffix (e.g.
|
|
86
|
+
* `java.lang.ClassCastException`, `ValueError`): no word boundary precedes the
|
|
87
|
+
* suffix inside a compound name, so the keyword regex alone misses them.
|
|
88
|
+
*/
|
|
89
|
+
const ERROR_CLASS_NAME = /\b\w*(?:Exception|Error)\b/;
|
|
90
|
+
const PY_TRACEBACK_HEAD = /^\s*Traceback \(most recent call last\):$/;
|
|
91
|
+
const PY_FILE_FRAME = /^\s*File "(?:[^"\\]|\\.)*", line \d+/;
|
|
92
|
+
const JVM_STACK_FRAME = /^\s+at\s/;
|
|
93
|
+
const JVM_MORE_FRAMES = /^\s*\.\.\. \d+ more\b/;
|
|
94
|
+
/**
|
|
95
|
+
* Message payload of a DSS record line: timestamp header and the
|
|
96
|
+
* `[thread] [LEVEL] [logger] -` prefix stripped, so continuation logic can be
|
|
97
|
+
* shared between timestamped records and raw log lines.
|
|
98
|
+
*/
|
|
99
|
+
function dssRecordMessage(line) {
|
|
100
|
+
return line
|
|
101
|
+
.replace(/^\[\d{4}\/\d{2}\/\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}\]\s?/, "")
|
|
102
|
+
.replace(/^\[[^\]]*\] \[[^\]]*\] \[[^\]]*\]\s+-\s?/, "");
|
|
103
|
+
}
|
|
104
|
+
function isErrorContent(line) {
|
|
105
|
+
return ERROR_KEYWORDS.test(line) || ERROR_CLASS_NAME.test(line) || PY_FILE_FRAME.test(line);
|
|
106
|
+
}
|
|
76
107
|
function lineMatchesLogFilter(line, filter) {
|
|
77
108
|
const normalized = line.toLowerCase();
|
|
78
109
|
switch (filter) {
|
|
@@ -82,17 +113,87 @@ function lineMatchesLogFilter(line, filter) {
|
|
|
82
113
|
case "stderr":
|
|
83
114
|
return normalized.includes("[null-err-") || LEGACY_STDERR_RECORD.test(line);
|
|
84
115
|
case "errors":
|
|
85
|
-
return
|
|
116
|
+
return isErrorContent(line);
|
|
86
117
|
case "user":
|
|
87
118
|
return !/^\d{4}[-/]\d{2}[-/]\d{2}/.test(line)
|
|
88
119
|
&& !normalized.includes("backend-log")
|
|
89
120
|
&& !normalized.includes("debug");
|
|
90
121
|
}
|
|
91
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* Errors filter over DSS records. A record's exception spans lines: the
|
|
125
|
+
* timestamped header carries the level/message, untimestamped continuations
|
|
126
|
+
* carry the exception chain. A record whose header matches the error keywords
|
|
127
|
+
* keeps its cause-bearing continuation lines (exception headers like
|
|
128
|
+
* `pkg.FooException: msg`, `Caused by:` chains, Python `Traceback` essentials)
|
|
129
|
+
* and drops bare JVM stack frames. A Python traceback body is tracked as state,
|
|
130
|
+
* so `File "..."` frames and the user-code source line survive even though they
|
|
131
|
+
* carry no error keyword — including when DSS re-logs each child-stderr line as
|
|
132
|
+
* its own timestamped record.
|
|
133
|
+
*/
|
|
134
|
+
function filterJobLogToErrors(lines) {
|
|
135
|
+
const kept = [];
|
|
136
|
+
let recordMatches = false;
|
|
137
|
+
let pyTraceback = false;
|
|
138
|
+
for (const line of lines) {
|
|
139
|
+
const isHeader = DSS_RECORD_HEADER.test(line);
|
|
140
|
+
const message = dssRecordMessage(line);
|
|
141
|
+
if (pyTraceback) {
|
|
142
|
+
if (JVM_STACK_FRAME.test(line) || JVM_MORE_FRAMES.test(line)) {
|
|
143
|
+
if (isHeader)
|
|
144
|
+
recordMatches = false;
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
if (/^\s+\S/.test(message)) {
|
|
148
|
+
kept.push(line); // `File "..."` frame or user-code source line
|
|
149
|
+
if (isHeader)
|
|
150
|
+
recordMatches = false;
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (ERROR_CLASS_NAME.test(message)) {
|
|
154
|
+
kept.push(line); // terminating exception line
|
|
155
|
+
pyTraceback = false;
|
|
156
|
+
if (isHeader)
|
|
157
|
+
recordMatches = false;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
pyTraceback = false;
|
|
161
|
+
}
|
|
162
|
+
if (isHeader) {
|
|
163
|
+
recordMatches = isErrorContent(line) || PY_FILE_FRAME.test(message);
|
|
164
|
+
if (recordMatches && (PY_TRACEBACK_HEAD.test(message) || PY_FILE_FRAME.test(message))) {
|
|
165
|
+
pyTraceback = true;
|
|
166
|
+
}
|
|
167
|
+
if (recordMatches)
|
|
168
|
+
kept.push(line);
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (PY_TRACEBACK_HEAD.test(message)) {
|
|
172
|
+
kept.push(line);
|
|
173
|
+
pyTraceback = true;
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (recordMatches
|
|
177
|
+
&& !JVM_STACK_FRAME.test(line)
|
|
178
|
+
&& !JVM_MORE_FRAMES.test(line)
|
|
179
|
+
&& isErrorContent(line))
|
|
180
|
+
kept.push(line);
|
|
181
|
+
}
|
|
182
|
+
return kept.join("\n");
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Filter a job log. DSS logs (any timestamped record) use the record-aware
|
|
186
|
+
* errors pass; untimestamped (non-DSS) logs keep the per-line behaviour, since
|
|
187
|
+
* no record structure can be inferred.
|
|
188
|
+
*/
|
|
92
189
|
function filterJobLog(log, filter) {
|
|
93
190
|
if (!filter)
|
|
94
191
|
return log;
|
|
95
|
-
|
|
192
|
+
const lines = jobLogLines(log);
|
|
193
|
+
if (filter !== "errors" || !lines.some((line) => DSS_RECORD_HEADER.test(line))) {
|
|
194
|
+
return lines.filter((line) => lineMatchesLogFilter(line, filter)).join("\n");
|
|
195
|
+
}
|
|
196
|
+
return filterJobLogToErrors(lines);
|
|
96
197
|
}
|
|
97
198
|
function limitJobLog(log, maxLines) {
|
|
98
199
|
if (!log)
|
|
@@ -163,6 +264,10 @@ function summarizeJobLog(state, log, maxLines, elapsedMs) {
|
|
|
163
264
|
...(progress ? { progress, } : {}),
|
|
164
265
|
};
|
|
165
266
|
}
|
|
267
|
+
function nothingToBuildMessage(targets) {
|
|
268
|
+
const names = targets.map((target) => `${target.projectKey}.${target.id}`).join(", ");
|
|
269
|
+
return `The job ran no activity: ${names} ${targets.length === 1 ? "is a managed dataset" : "are managed datasets"} that no recipe produces, so nothing was built. Create a recipe that writes ${targets.length === 1 ? "it" : "them"} or build the recipe's output instead.`;
|
|
270
|
+
}
|
|
166
271
|
export class JobsResource extends BaseResource {
|
|
167
272
|
/** List jobs in a project. */
|
|
168
273
|
async list(projectKey) {
|
|
@@ -275,6 +380,12 @@ export class JobsResource extends BaseResource {
|
|
|
275
380
|
* confirmed gone. Explicit `includeLogs`/`summary` requests stay honest:
|
|
276
381
|
* `log`/`logSummary` are only present when logs were actually retrieved.
|
|
277
382
|
*
|
|
383
|
+
* A DONE job that ran no activity (`progress.total === 0`) is also what DSS
|
|
384
|
+
* reports for an already-up-to-date build, so it stays successful unless an
|
|
385
|
+
* output is a managed dataset that no recipe writes: then nothing can ever
|
|
386
|
+
* build it, and the outcome is `success: false` with
|
|
387
|
+
* `failure.code === "nothing_to_build"`.
|
|
388
|
+
*
|
|
278
389
|
* On timeout, returns `{ success: false, ... }` rather than throwing.
|
|
279
390
|
*/
|
|
280
391
|
async wait(jobId, opts) {
|
|
@@ -392,8 +503,18 @@ export class JobsResource extends BaseResource {
|
|
|
392
503
|
}
|
|
393
504
|
}
|
|
394
505
|
}
|
|
506
|
+
const unbuildable = success && gs.total === 0
|
|
507
|
+
? await this.unbuildableTargets(def.outputs, def.projectKey ?? opts?.projectKey)
|
|
508
|
+
: [];
|
|
509
|
+
const failure = unbuildable.length > 0
|
|
510
|
+
? {
|
|
511
|
+
code: "nothing_to_build",
|
|
512
|
+
message: nothingToBuildMessage(unbuildable),
|
|
513
|
+
targets: unbuildable,
|
|
514
|
+
}
|
|
515
|
+
: undefined;
|
|
395
516
|
return {
|
|
396
|
-
success,
|
|
517
|
+
success: success && failure === undefined,
|
|
397
518
|
jobId: def.id ?? jobId,
|
|
398
519
|
state,
|
|
399
520
|
type: def.type ?? "unknown",
|
|
@@ -405,6 +526,7 @@ export class JobsResource extends BaseResource {
|
|
|
405
526
|
running: gs.running ?? 0,
|
|
406
527
|
total: gs.total ?? null,
|
|
407
528
|
},
|
|
529
|
+
...(failure !== undefined ? { failure, } : {}),
|
|
408
530
|
...(log !== undefined ? { log, } : {}),
|
|
409
531
|
...(logSummary !== undefined ? { logSummary, } : {}),
|
|
410
532
|
...(logUnavailable !== undefined ? { logUnavailable, } : {}),
|
|
@@ -423,6 +545,51 @@ export class JobsResource extends BaseResource {
|
|
|
423
545
|
await sleep(Math.min(nextDelayMs, timeout - elapsedMs));
|
|
424
546
|
}
|
|
425
547
|
}
|
|
548
|
+
/**
|
|
549
|
+
* Dataset outputs of a job that ran no activity which nothing can produce.
|
|
550
|
+
* DSS ends such a job DONE exactly like an already-up-to-date build, so the
|
|
551
|
+
* job status cannot tell them apart; the flow graph can: a target is
|
|
552
|
+
* unbuildable when it is a managed dataset (DSS owns its storage, so a
|
|
553
|
+
* recipe is the only way to fill it) that no recipe writes. Unmanaged
|
|
554
|
+
* datasets (uploads, external tables) are sources and legitimately build
|
|
555
|
+
* nothing. Lookups that fail leave the DONE outcome untouched.
|
|
556
|
+
*/
|
|
557
|
+
async unbuildableTargets(outputs, fallbackProjectKey) {
|
|
558
|
+
if (!Array.isArray(outputs))
|
|
559
|
+
return [];
|
|
560
|
+
const writtenByProject = new Map();
|
|
561
|
+
const unbuildable = [];
|
|
562
|
+
for (const output of outputs) {
|
|
563
|
+
const record = asRecord(output);
|
|
564
|
+
if (record?.type !== "DATASET" || typeof record.targetDataset !== "string")
|
|
565
|
+
continue;
|
|
566
|
+
const id = record.targetDataset;
|
|
567
|
+
const projectKey = typeof record.targetDatasetProjectKey === "string"
|
|
568
|
+
? record.targetDatasetProjectKey
|
|
569
|
+
: fallbackProjectKey;
|
|
570
|
+
if (projectKey === undefined)
|
|
571
|
+
continue;
|
|
572
|
+
try {
|
|
573
|
+
const details = await this.client.datasets.get(id, projectKey);
|
|
574
|
+
if (details.managed !== true)
|
|
575
|
+
continue;
|
|
576
|
+
let written = writtenByProject.get(projectKey);
|
|
577
|
+
if (written === undefined) {
|
|
578
|
+
const { graph, } = await this.client.projects.flowTopology(projectKey);
|
|
579
|
+
written = new Set(graph.edges.filter((edge) => edge.relation === "writes").map((edge) => edge.to));
|
|
580
|
+
writtenByProject.set(projectKey, written);
|
|
581
|
+
}
|
|
582
|
+
if (!written.has(id)) {
|
|
583
|
+
unbuildable.push({ type: "DATASET", id, projectKey, reason: "no_producing_recipe", });
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
catch (error) {
|
|
587
|
+
if (!(error instanceof DataikuError || error instanceof ClientValidationError))
|
|
588
|
+
throw error;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
return unbuildable;
|
|
592
|
+
}
|
|
426
593
|
/**
|
|
427
594
|
* Probe whether a job still exists. Returns `true` when the server reports
|
|
428
595
|
* it as not found, `false` when it is still visible, and `undefined` when
|
|
@@ -37,6 +37,16 @@ export declare class MetricsResource extends BaseResource {
|
|
|
37
37
|
getDatasetMetrics(datasetName: string, projectKey?: string): Promise<MetricValues>;
|
|
38
38
|
/** Compute the dataset metrics configured in DSS for the default partition. */
|
|
39
39
|
computeDatasetMetrics(datasetName: string, projectKey?: string): Promise<MetricComputationReport>;
|
|
40
|
+
/**
|
|
41
|
+
* Run the dataset's checks (POST /actions/runChecks/{?partitions}). Without
|
|
42
|
+
* `checks` DSS runs the checks configured on the dataset; with `checks` it
|
|
43
|
+
* runs those definitions instead (body `{"checks": [...]}`).
|
|
44
|
+
*/
|
|
45
|
+
runDatasetChecks(datasetName: string, opts?: {
|
|
46
|
+
projectKey?: string;
|
|
47
|
+
partitions?: string;
|
|
48
|
+
checks?: unknown[];
|
|
49
|
+
}): Promise<MetricComputationReport>;
|
|
40
50
|
/** Get the last metric values for a managed folder. */
|
|
41
51
|
getFolderMetrics(folderId: string, projectKey?: string): Promise<MetricValues>;
|
|
42
52
|
/** Get the metric history for a dataset's global partition. */
|
|
@@ -16,6 +16,16 @@ export class MetricsResource extends BaseResource {
|
|
|
16
16
|
const pk = this.enc(projectKey);
|
|
17
17
|
return this.client.post(`${datasetPath(pk, datasetName)}/actions/computeMetrics?partition=`);
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Run the dataset's checks (POST /actions/runChecks/{?partitions}). Without
|
|
21
|
+
* `checks` DSS runs the checks configured on the dataset; with `checks` it
|
|
22
|
+
* runs those definitions instead (body `{"checks": [...]}`).
|
|
23
|
+
*/
|
|
24
|
+
async runDatasetChecks(datasetName, opts = {}) {
|
|
25
|
+
const pk = this.enc(opts.projectKey);
|
|
26
|
+
const query = opts.partitions ? `?partitions=${encodeURIComponent(opts.partitions)}` : "";
|
|
27
|
+
return this.client.post(`${datasetPath(pk, datasetName)}/actions/runChecks/${query}`, opts.checks ? { checks: opts.checks, } : undefined);
|
|
28
|
+
}
|
|
19
29
|
/** Get the last metric values for a managed folder. */
|
|
20
30
|
async getFolderMetrics(folderId, projectKey) {
|
|
21
31
|
const pk = this.enc(projectKey);
|
|
@@ -1,14 +1,41 @@
|
|
|
1
1
|
import { BaseResource } from "./base.js";
|
|
2
|
+
import { TrainedModel } from "./trained-model.js";
|
|
2
3
|
export type MlTaskType = "PREDICTION" | "CLUSTERING";
|
|
3
|
-
|
|
4
|
-
|
|
4
|
+
/** ML task definition; time series forecasting is PREDICTION with predictionType TIMESERIES_FORECAST. */
|
|
5
|
+
export interface MlTaskCreateFields {
|
|
5
6
|
taskType: MlTaskType;
|
|
6
7
|
targetVariable?: string;
|
|
7
8
|
predictionType?: string;
|
|
9
|
+
/** Forecasting tasks: the parsed-date time column. */
|
|
10
|
+
timeVariable?: string;
|
|
11
|
+
/** Forecasting tasks: columns identifying each series of a multi-series dataset. */
|
|
12
|
+
timeseriesIdentifiers?: string[];
|
|
8
13
|
backendType?: string;
|
|
9
14
|
guessPolicy?: string;
|
|
10
15
|
projectKey?: string;
|
|
11
16
|
}
|
|
17
|
+
export interface MlTaskCreateOptions extends MlTaskCreateFields {
|
|
18
|
+
analysisId: string;
|
|
19
|
+
}
|
|
20
|
+
export interface MlTaskCreateForDatasetOptions extends MlTaskCreateFields {
|
|
21
|
+
inputDataset: string;
|
|
22
|
+
}
|
|
23
|
+
export interface MlTaskListItem extends Record<string, unknown> {
|
|
24
|
+
analysisId?: string;
|
|
25
|
+
mlTaskId?: string;
|
|
26
|
+
analysisName?: string;
|
|
27
|
+
mlTaskName?: string;
|
|
28
|
+
taskType?: string;
|
|
29
|
+
inputDataset?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface MlTaskReguessOptions {
|
|
32
|
+
predictionType?: string;
|
|
33
|
+
targetVariable?: string;
|
|
34
|
+
timeVariable?: string;
|
|
35
|
+
timeseriesIdentifiers?: string[];
|
|
36
|
+
/** Re-guess every setting after the core change, not just the impacted ones. */
|
|
37
|
+
fullReguess?: boolean;
|
|
38
|
+
}
|
|
12
39
|
export interface MlTaskCreateResult extends Record<string, unknown> {
|
|
13
40
|
analysisId?: string;
|
|
14
41
|
mlTaskId?: string;
|
|
@@ -91,6 +118,14 @@ export interface MlTaskDeployResult extends Record<string, unknown> {
|
|
|
91
118
|
savedModelId?: string;
|
|
92
119
|
trainRecipeName?: string;
|
|
93
120
|
}
|
|
121
|
+
/** Query of POST .../guess; repeated `timeseriesIdentifiers` as dataikuapi sends a list (pure; shared with `--plan`). */
|
|
122
|
+
export declare function mlTaskReguessQuery(opts: MlTaskReguessOptions): URLSearchParams;
|
|
123
|
+
/**
|
|
124
|
+
* Task-creation body shared by `create` (existing analysis) and
|
|
125
|
+
* `createForDataset` (new analysis), with dataikuapi's defaults (pure; shared
|
|
126
|
+
* with `--plan`).
|
|
127
|
+
*/
|
|
128
|
+
export declare function mlTaskCreateBody(opts: MlTaskCreateFields): Record<string, unknown>;
|
|
94
129
|
export declare class MlTasksResource extends BaseResource {
|
|
95
130
|
/** Create a prediction or clustering task in an existing visual analysis. */
|
|
96
131
|
create(opts: MlTaskCreateOptions): Promise<MlTaskCreateResult>;
|
|
@@ -122,5 +157,33 @@ export declare class MlTasksResource extends BaseResource {
|
|
|
122
157
|
deployToFlow(opts: MlTaskDeployOptions): Promise<MlTaskDeployResult>;
|
|
123
158
|
/** Delete an ML task and its trained models. */
|
|
124
159
|
delete(analysisId: string, mlTaskId: string, projectKey?: string): Promise<void>;
|
|
160
|
+
/** Every ML task of the project, with its visual analysis (GET /models/lab/). */
|
|
161
|
+
list(projectKey?: string): Promise<MlTaskListItem[]>;
|
|
162
|
+
/**
|
|
163
|
+
* Create a new visual analysis on `inputDataset` with one ML task
|
|
164
|
+
* (POST /models/lab/). Returns `{analysisId, mlTaskId}`; DSS guesses the
|
|
165
|
+
* settings asynchronously (watch `status().guessing`).
|
|
166
|
+
*/
|
|
167
|
+
createForDataset(opts: MlTaskCreateForDatasetOptions): Promise<MlTaskCreateResult>;
|
|
168
|
+
/**
|
|
169
|
+
* Re-guess the task settings: all of them, or (prediction tasks) after
|
|
170
|
+
* setting a new prediction type, target, time variable, or series
|
|
171
|
+
* identifiers — only the impacted settings unless `fullReguess`.
|
|
172
|
+
*/
|
|
173
|
+
reguess(analysisId: string, mlTaskId: string, opts?: MlTaskReguessOptions, projectKey?: string): Promise<void>;
|
|
174
|
+
/**
|
|
175
|
+
* Time series forecasting tasks: change the forecasting parameters
|
|
176
|
+
* (`forecastHorizon`, `validationHorizons`, `timestepParams`,
|
|
177
|
+
* `updateAlgorithmSettings`) and re-guess the impacted algorithm settings.
|
|
178
|
+
*/
|
|
179
|
+
reguessForecasting(analysisId: string, mlTaskId: string, params: Record<string, unknown>, projectKey?: string): Promise<void>;
|
|
180
|
+
/**
|
|
181
|
+
* One trained lab model as a {@link TrainedModel}: diagnostics
|
|
182
|
+
* (subpopulation analyses, partial dependencies, timeseries residuals and
|
|
183
|
+
* per-series results), documentation, scoring exports, user metadata.
|
|
184
|
+
*/
|
|
185
|
+
model(analysisId: string, mlTaskId: string, modelId: string, projectKey?: string): TrainedModel;
|
|
186
|
+
/** GET a generated lab model documentation (docx) by the `exportId` of its finished future. */
|
|
187
|
+
downloadModelDocumentation(exportId: string, projectKey?: string): Promise<Response>;
|
|
125
188
|
private taskPath;
|
|
126
189
|
}
|