dataiku-sdk 3.6.1 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +14 -6
  2. package/bin/bun-version.js +14 -0
  3. package/bin/dss.js +11 -15
  4. package/dist/build-metadata.json +1 -1
  5. package/dist/packages/types/src/index.d.ts +114 -5
  6. package/dist/packages/types/src/index.js +75 -2
  7. package/dist/src/cli/coerce.d.ts +2 -0
  8. package/dist/src/cli/coerce.js +20 -16
  9. package/dist/src/cli/command-syntax.json +3369 -144
  10. package/dist/src/cli/commands/analysis.js +19 -1
  11. package/dist/src/cli/commands/bundle.js +22 -0
  12. package/dist/src/cli/commands/connection.js +26 -2
  13. package/dist/src/cli/commands/data-quality.js +33 -1
  14. package/dist/src/cli/commands/dataset.js +76 -9
  15. package/dist/src/cli/commands/discussion.js +15 -0
  16. package/dist/src/cli/commands/doctor.js +1 -1
  17. package/dist/src/cli/commands/flow-zone.js +241 -46
  18. package/dist/src/cli/commands/flow.d.ts +5 -0
  19. package/dist/src/cli/commands/flow.js +57 -0
  20. package/dist/src/cli/commands/future.js +8 -0
  21. package/dist/src/cli/commands/index.js +6 -0
  22. package/dist/src/cli/commands/job.js +44 -6
  23. package/dist/src/cli/commands/metrics.js +21 -1
  24. package/dist/src/cli/commands/ml-task.d.ts +5 -0
  25. package/dist/src/cli/commands/ml-task.js +86 -16
  26. package/dist/src/cli/commands/mlflow.d.ts +11 -0
  27. package/dist/src/cli/commands/mlflow.js +121 -0
  28. package/dist/src/cli/commands/plugin.js +5 -11
  29. package/dist/src/cli/commands/project-deployer.js +70 -0
  30. package/dist/src/cli/commands/project-library.js +2 -5
  31. package/dist/src/cli/commands/project.js +8 -0
  32. package/dist/src/cli/commands/recipe.d.ts +13 -0
  33. package/dist/src/cli/commands/recipe.js +130 -7
  34. package/dist/src/cli/commands/saved-model.d.ts +1 -1
  35. package/dist/src/cli/commands/saved-model.js +9 -76
  36. package/dist/src/cli/commands/streaming-endpoint.js +30 -1
  37. package/dist/src/cli/commands/webapp.js +8 -0
  38. package/dist/src/cli/commands/wiki.js +78 -0
  39. package/dist/src/cli/commands/workspace.js +8 -0
  40. package/dist/src/cli/contract.d.ts +16 -15
  41. package/dist/src/cli/contract.js +150 -3407
  42. package/dist/src/cli/doctor.d.ts +13 -0
  43. package/dist/src/cli/doctor.js +43 -1
  44. package/dist/src/cli/flags.d.ts +14 -1
  45. package/dist/src/cli/flags.js +106 -8
  46. package/dist/src/cli/helpers/app-successor.js +1 -2
  47. package/dist/src/cli/helpers/dataset.d.ts +12 -0
  48. package/dist/src/cli/helpers/dataset.js +36 -0
  49. package/dist/src/cli/helpers/documentation.d.ts +21 -0
  50. package/dist/src/cli/helpers/documentation.js +53 -0
  51. package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
  52. package/dist/src/cli/helpers/flow-zone.js +144 -4
  53. package/dist/src/cli/helpers/trained-model.d.ts +26 -0
  54. package/dist/src/cli/helpers/trained-model.js +144 -0
  55. package/dist/src/cli/list-output.d.ts +7 -0
  56. package/dist/src/cli/list-output.js +77 -0
  57. package/dist/src/cli/list-shapes.d.ts +21 -0
  58. package/dist/src/cli/list-shapes.js +111 -0
  59. package/dist/src/cli/output-schemas.d.ts +0 -14
  60. package/dist/src/cli/output-schemas.js +4 -7
  61. package/dist/src/cli/output.d.ts +0 -1
  62. package/dist/src/cli/output.js +33 -19
  63. package/dist/src/cli/plans.d.ts +14 -0
  64. package/dist/src/cli/plans.js +3740 -0
  65. package/dist/src/cli/types.d.ts +2 -0
  66. package/dist/src/cli/usage.d.ts +7 -0
  67. package/dist/src/cli/usage.js +20 -1
  68. package/dist/src/cli/version.d.ts +2 -2
  69. package/dist/src/cli/version.js +2 -2
  70. package/dist/src/cli.js +137 -115
  71. package/dist/src/client.d.ts +52 -43
  72. package/dist/src/client.js +267 -96
  73. package/dist/src/errors.d.ts +22 -2
  74. package/dist/src/errors.js +49 -50
  75. package/dist/src/generated/action-output-schemas.json +345 -89
  76. package/dist/src/index.d.ts +5 -1
  77. package/dist/src/index.js +3 -0
  78. package/dist/src/resources/analyses.d.ts +8 -0
  79. package/dist/src/resources/analyses.js +13 -1
  80. package/dist/src/resources/bundles.d.ts +34 -0
  81. package/dist/src/resources/bundles.js +67 -0
  82. package/dist/src/resources/connections.d.ts +9 -0
  83. package/dist/src/resources/connections.js +15 -0
  84. package/dist/src/resources/data-quality.d.ts +20 -0
  85. package/dist/src/resources/data-quality.js +30 -0
  86. package/dist/src/resources/dataset-create.d.ts +38 -0
  87. package/dist/src/resources/dataset-create.js +135 -0
  88. package/dist/src/resources/datasets.d.ts +46 -15
  89. package/dist/src/resources/datasets.js +120 -112
  90. package/dist/src/resources/discussions.d.ts +5 -0
  91. package/dist/src/resources/discussions.js +7 -0
  92. package/dist/src/resources/documentation.d.ts +25 -0
  93. package/dist/src/resources/documentation.js +16 -0
  94. package/dist/src/resources/flow-zones.d.ts +29 -3
  95. package/dist/src/resources/flow-zones.js +53 -3
  96. package/dist/src/resources/flow.d.ts +48 -0
  97. package/dist/src/resources/flow.js +65 -0
  98. package/dist/src/resources/futures.d.ts +23 -0
  99. package/dist/src/resources/futures.js +32 -0
  100. package/dist/src/resources/jobs.d.ts +34 -0
  101. package/dist/src/resources/jobs.js +170 -3
  102. package/dist/src/resources/metrics.d.ts +10 -0
  103. package/dist/src/resources/metrics.js +10 -0
  104. package/dist/src/resources/ml-tasks.d.ts +65 -2
  105. package/dist/src/resources/ml-tasks.js +102 -28
  106. package/dist/src/resources/mlflow-extension.d.ts +73 -0
  107. package/dist/src/resources/mlflow-extension.js +105 -0
  108. package/dist/src/resources/projects.d.ts +15 -0
  109. package/dist/src/resources/projects.js +18 -3
  110. package/dist/src/resources/recipe-create.d.ts +79 -0
  111. package/dist/src/resources/recipe-create.js +454 -0
  112. package/dist/src/resources/recipes.d.ts +28 -29
  113. package/dist/src/resources/recipes.js +201 -267
  114. package/dist/src/resources/saved-models.d.ts +10 -6
  115. package/dist/src/resources/saved-models.js +16 -14
  116. package/dist/src/resources/scenarios.js +8 -14
  117. package/dist/src/resources/sql.js +52 -13
  118. package/dist/src/resources/streaming-endpoints.d.ts +12 -0
  119. package/dist/src/resources/streaming-endpoints.js +15 -0
  120. package/dist/src/resources/trained-model.d.ts +60 -0
  121. package/dist/src/resources/trained-model.js +98 -0
  122. package/dist/src/resources/webapps.d.ts +8 -0
  123. package/dist/src/resources/webapps.js +10 -0
  124. package/dist/src/resources/wiki.d.ts +52 -1
  125. package/dist/src/resources/wiki.js +96 -0
  126. package/dist/src/resources/workspaces.d.ts +2 -0
  127. package/dist/src/resources/workspaces.js +4 -0
  128. package/dist/src/utils/flow-analysis.js +4 -0
  129. package/dist/src/utils/response-file.js +20 -9
  130. package/dist/src/utils/secret-sanitize.d.ts +7 -0
  131. package/dist/src/utils/secret-sanitize.js +46 -0
  132. package/package.json +25 -26
  133. package/plugin.json +1 -1
  134. package/skills/dataiku-dss/SKILL.md +1 -1
  135. package/skills/dataiku-dss/references/authentication.md +3 -1
  136. package/skills/dataiku-dss/references/coding.md +1 -0
  137. package/skills/dataiku-dss/references/discovery.md +4 -6
  138. package/skills/dataiku-dss/references/flow-maps.md +2 -2
  139. package/skills/dataiku-dss/references/troubleshooting.md +3 -2
@@ -0,0 +1,65 @@
1
+ import { ClientValidationError, } from "../errors.js";
2
+ import { BaseResource, } from "./base.js";
3
+ import { startDocumentation, } from "./documentation.js";
4
+ import { dssTask, } from "./futures.js";
5
+ /**
6
+ * Flow-wide tools: schema propagation from a dataset through every downstream
7
+ * recipe, and flow documentation export. Zones and the flow graph live on
8
+ * `flowZones`; per-recipe schema updates on `recipes.updateSchema`.
9
+ */
10
+ export class FlowResource extends BaseResource {
11
+ flowPath(projectKey) {
12
+ return `/public/api/projects/${this.enc(projectKey)}/flow`;
13
+ }
14
+ /**
15
+ * Start propagating `datasetName`'s schema through the flow
16
+ * (POST /flow/tools/propagate-schema/, dataikuapi
17
+ * `new_schema_propagation(...).start()`). Returns the DSS future.
18
+ */
19
+ async propagateSchema(datasetName, opts = {}) {
20
+ const name = datasetName.trim();
21
+ if (name.length === 0) {
22
+ throw new ClientValidationError("propagateSchema requires a dataset name.", "validation_failed");
23
+ }
24
+ const pk = this.resolveProjectKey(opts.projectKey);
25
+ const raw = await this.client.post(`${this.flowPath(pk)}/tools/propagate-schema/`, schemaPropagationBody(pk, name, opts));
26
+ return dssTask(raw, "Schema propagation");
27
+ }
28
+ async propagateSchemaAndWait(datasetName, opts = {}) {
29
+ return this.client.futures.waitTask(await this.propagateSchema(datasetName, opts), opts);
30
+ }
31
+ /**
32
+ * Start generating the flow documentation (docx) from the DSS default
33
+ * template, an uploaded docx template, or a docx in a managed folder.
34
+ * The finished future's result carries the `exportId` to download.
35
+ */
36
+ async generateDocumentation(template = { kind: "default", }, projectKey) {
37
+ const base = `${this.flowPath(projectKey)}/documentation`;
38
+ return startDocumentation(this.client, {
39
+ default: `${base}/generate`,
40
+ file: `${base}/generate-with-template`,
41
+ folder: `${base}/generate-with-template-in-folder`,
42
+ }, template, "Flow documentation generation");
43
+ }
44
+ /** GET a generated flow documentation (docx) as a binary stream. */
45
+ async downloadDocumentation(exportId, projectKey) {
46
+ return this.client.stream(`${this.flowPath(projectKey)}/documentation/generated/${encodeURIComponent(exportId)}`);
47
+ }
48
+ }
49
+ /** Request body of POST /flow/tools/propagate-schema/ (pure; shared with `--plan`). */
50
+ export function schemaPropagationBody(projectKey, datasetName, opts) {
51
+ return {
52
+ options: {
53
+ recipeUpdateOptions: {
54
+ byType: opts.recipeUpdateOptions?.byType ?? {},
55
+ byName: opts.recipeUpdateOptions?.byName ?? {},
56
+ },
57
+ defaultPartitionValuesByDimension: opts.defaultPartitionValuesByDimension ?? {},
58
+ partitionsByComputable: opts.partitionsByComputable ?? {},
59
+ excludedRecipes: opts.excludedRecipes ?? [],
60
+ markAsOkRecipes: opts.markAsOkRecipes ?? [],
61
+ autoRebuild: opts.autoRebuild ?? true,
62
+ },
63
+ sources: [{ projectKey, id: datasetName, },],
64
+ };
65
+ }
@@ -4,7 +4,30 @@ export interface FutureWaitOptions {
4
4
  pollIntervalMs?: number;
5
5
  timeoutMs?: number;
6
6
  }
7
+ /**
8
+ * A DSS long task as started (dataikuapi `DSSFuture.from_resp`): `jobId`
9
+ * addresses a running task in `futures.wait`; a task DSS finished inline comes
10
+ * back without one, as its final state (`hasResult`, `result`).
11
+ */
12
+ export interface DssTask {
13
+ jobId?: string;
14
+ hasResult?: boolean;
15
+ result?: unknown;
16
+ [key: string]: unknown;
17
+ }
18
+ /** Validate a task-start response: an object whose `jobId`, when present, is a string. */
19
+ export declare function dssTask(raw: unknown, operation: string): DssTask;
7
20
  export declare class FuturesResource extends BaseResource {
21
+ /**
22
+ * Long tasks in progress (GET /futures/): the caller's own by default,
23
+ * every user's with `allUsers`, running scenarios too with `withScenarios`.
24
+ */
25
+ list(opts?: {
26
+ allUsers?: boolean;
27
+ withScenarios?: boolean;
28
+ }): Promise<FutureState[]>;
29
+ /** Wait for a started task; a task DSS already finished inline is returned as is. */
30
+ waitTask(task: DssTask, opts?: FutureWaitOptions): Promise<FutureWaitResult | DssTask>;
8
31
  get(futureId: string, opts?: {
9
32
  timeoutMs?: number;
10
33
  noRetry?: boolean;
@@ -1,6 +1,18 @@
1
+ import { unexpectedResponseError, } from "../errors.js";
1
2
  import { FutureStateSchema, FutureWaitResultSchema, } from "../schemas.js";
2
3
  import { computeNextPollDelayMs, DEFAULT_POLL_INTERVAL_MS, DEFAULT_TIMEOUT_MS, isRequestDeadlineError, } from "../utils/polling.js";
3
4
  import { BaseResource, } from "./base.js";
5
+ /** Validate a task-start response: an object whose `jobId`, when present, is a string. */
6
+ export function dssTask(raw, operation) {
7
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
8
+ throw unexpectedResponseError(`${operation} returned no task state.`);
9
+ }
10
+ const state = { ...raw, };
11
+ if (state.jobId !== undefined && typeof state.jobId !== "string") {
12
+ throw unexpectedResponseError(`${operation} returned a non-string jobId.`);
13
+ }
14
+ return state;
15
+ }
4
16
  function sleep(ms) {
5
17
  return new Promise((resolve) => setTimeout(resolve, ms));
6
18
  }
@@ -22,6 +34,26 @@ function waitState(state) {
22
34
  return "RUNNING";
23
35
  }
24
36
  export class FuturesResource extends BaseResource {
37
+ /**
38
+ * Long tasks in progress (GET /futures/): the caller's own by default,
39
+ * every user's with `allUsers`, running scenarios too with `withScenarios`.
40
+ */
41
+ async list(opts = {}) {
42
+ const query = new URLSearchParams({
43
+ allUsers: String(opts.allUsers === true),
44
+ withScenarios: String(opts.withScenarios === true),
45
+ });
46
+ return this.client.get(`/public/api/futures/?${query}`);
47
+ }
48
+ /** Wait for a started task; a task DSS already finished inline is returned as is. */
49
+ async waitTask(task, opts = {}) {
50
+ if (task.jobId === undefined)
51
+ return task;
52
+ return this.wait(task.jobId, {
53
+ pollIntervalMs: opts.pollIntervalMs,
54
+ timeoutMs: opts.timeoutMs,
55
+ });
56
+ }
25
57
  async get(futureId, opts = {}) {
26
58
  return this.state(futureId, {
27
59
  peek: false,
@@ -23,12 +23,30 @@ export interface JobLogSummary {
23
23
  }
24
24
  /** Why a requested job log could not be retrieved. */
25
25
  export type JobLogUnavailableReason = "not_found" | "error";
26
+ /**
27
+ * A requested build target that no flow object can ever produce: a managed
28
+ * dataset with no recipe writing it.
29
+ */
30
+ export interface JobUnbuildableTarget {
31
+ type: "DATASET";
32
+ id: string;
33
+ projectKey: string;
34
+ reason: "no_producing_recipe";
35
+ }
36
+ /** Why a job that ended DONE is reported as failed because it built nothing. */
37
+ export interface JobNothingToBuildFailure {
38
+ code: "nothing_to_build";
39
+ message: string;
40
+ targets: JobUnbuildableTarget[];
41
+ }
26
42
  /** Result of a completed job wait, including log-unavailable metadata. */
27
43
  export type JobWaitOutcome = JobWaitResult & {
28
44
  logSummary?: JobLogSummary;
29
45
  logUnavailable?: JobLogUnavailableReason;
30
46
  /** True when the job itself no longer exists on the server. */
31
47
  removed?: boolean;
48
+ /** Set (with `success: false`) when a DONE job ran no activity and a target has no producer. */
49
+ failure?: JobNothingToBuildFailure;
32
50
  };
33
51
  export interface JobBuildTarget {
34
52
  id: string;
@@ -120,6 +138,12 @@ export declare class JobsResource extends BaseResource {
120
138
  * confirmed gone. Explicit `includeLogs`/`summary` requests stay honest:
121
139
  * `log`/`logSummary` are only present when logs were actually retrieved.
122
140
  *
141
+ * A DONE job that ran no activity (`progress.total === 0`) is also what DSS
142
+ * reports for an already-up-to-date build, so it stays successful unless an
143
+ * output is a managed dataset that no recipe writes: then nothing can ever
144
+ * build it, and the outcome is `success: false` with
145
+ * `failure.code === "nothing_to_build"`.
146
+ *
123
147
  * On timeout, returns `{ success: false, ... }` rather than throwing.
124
148
  */
125
149
  wait(jobId: string, opts?: {
@@ -133,6 +157,16 @@ export declare class JobsResource extends BaseResource {
133
157
  timeoutMs?: number;
134
158
  projectKey?: string;
135
159
  }): Promise<JobWaitOutcome>;
160
+ /**
161
+ * Dataset outputs of a job that ran no activity which nothing can produce.
162
+ * DSS ends such a job DONE exactly like an already-up-to-date build, so the
163
+ * job status cannot tell them apart; the flow graph can: a target is
164
+ * unbuildable when it is a managed dataset (DSS owns its storage, so a
165
+ * recipe is the only way to fill it) that no recipe writes. Unmanaged
166
+ * datasets (uploads, external tables) are sources and legitimately build
167
+ * nothing. Lookups that fail leave the DONE outcome untouched.
168
+ */
169
+ private unbuildableTargets;
136
170
  /**
137
171
  * Probe whether a job still exists. Returns `true` when the server reports
138
172
  * it as not found, `false` when it is still visible, and `undefined` when
@@ -1,6 +1,7 @@
1
1
  import { ClientValidationError, DataikuError, } from "../errors.js";
2
2
  import { JobSummaryArraySchema, } from "../schemas.js";
3
3
  import { computeNextPollDelayMs, DEFAULT_POLL_INTERVAL_MS, DEFAULT_TIMEOUT_MS, isRequestDeadlineError, } from "../utils/polling.js";
4
+ import { asRecord, } from "../utils/records.js";
4
5
  import { BaseResource, } from "./base.js";
5
6
  const DEFAULT_MAX_LOG_LINES = 500;
6
7
  /**
@@ -73,6 +74,36 @@ function jobLogLines(log) {
73
74
  */
74
75
  const LEGACY_STDOUT_RECORD = /^\s*stdout:/i;
75
76
  const LEGACY_STDERR_RECORD = /^\s*stderr:/i;
77
+ /**
78
+ * A DSS backend log record starts with a timestamped header line
79
+ * (`[YYYY/MM/DD-HH:MM:SS.mmm]`); every following non-timestamped line is a
80
+ * continuation of that record (exception chain, JVM frames, ...).
81
+ */
82
+ const DSS_RECORD_HEADER = /^\[\d{4}\/\d{2}\/\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}\]/;
83
+ const ERROR_KEYWORDS = /\b(error|failed|failure|exception|traceback)\b/i;
84
+ /**
85
+ * Exception class names ending in the `Exception`/`Error` suffix (e.g.
86
+ * `java.lang.ClassCastException`, `ValueError`): no word boundary precedes the
87
+ * suffix inside a compound name, so the keyword regex alone misses them.
88
+ */
89
+ const ERROR_CLASS_NAME = /\b\w*(?:Exception|Error)\b/;
90
+ const PY_TRACEBACK_HEAD = /^\s*Traceback \(most recent call last\):$/;
91
+ const PY_FILE_FRAME = /^\s*File "(?:[^"\\]|\\.)*", line \d+/;
92
+ const JVM_STACK_FRAME = /^\s+at\s/;
93
+ const JVM_MORE_FRAMES = /^\s*\.\.\. \d+ more\b/;
94
+ /**
95
+ * Message payload of a DSS record line: timestamp header and the
96
+ * `[thread] [LEVEL] [logger] -` prefix stripped, so continuation logic can be
97
+ * shared between timestamped records and raw log lines.
98
+ */
99
+ function dssRecordMessage(line) {
100
+ return line
101
+ .replace(/^\[\d{4}\/\d{2}\/\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}\]\s?/, "")
102
+ .replace(/^\[[^\]]*\] \[[^\]]*\] \[[^\]]*\]\s+-\s?/, "");
103
+ }
104
+ function isErrorContent(line) {
105
+ return ERROR_KEYWORDS.test(line) || ERROR_CLASS_NAME.test(line) || PY_FILE_FRAME.test(line);
106
+ }
76
107
  function lineMatchesLogFilter(line, filter) {
77
108
  const normalized = line.toLowerCase();
78
109
  switch (filter) {
@@ -82,17 +113,87 @@ function lineMatchesLogFilter(line, filter) {
82
113
  case "stderr":
83
114
  return normalized.includes("[null-err-") || LEGACY_STDERR_RECORD.test(line);
84
115
  case "errors":
85
- return /\b(error|failed|failure|exception|traceback)\b/i.test(line);
116
+ return isErrorContent(line);
86
117
  case "user":
87
118
  return !/^\d{4}[-/]\d{2}[-/]\d{2}/.test(line)
88
119
  && !normalized.includes("backend-log")
89
120
  && !normalized.includes("debug");
90
121
  }
91
122
  }
123
+ /**
124
+ * Errors filter over DSS records. A record's exception spans lines: the
125
+ * timestamped header carries the level/message, untimestamped continuations
126
+ * carry the exception chain. A record whose header matches the error keywords
127
+ * keeps its cause-bearing continuation lines (exception headers like
128
+ * `pkg.FooException: msg`, `Caused by:` chains, Python `Traceback` essentials)
129
+ * and drops bare JVM stack frames. A Python traceback body is tracked as state,
130
+ * so `File "..."` frames and the user-code source line survive even though they
131
+ * carry no error keyword — including when DSS re-logs each child-stderr line as
132
+ * its own timestamped record.
133
+ */
134
+ function filterJobLogToErrors(lines) {
135
+ const kept = [];
136
+ let recordMatches = false;
137
+ let pyTraceback = false;
138
+ for (const line of lines) {
139
+ const isHeader = DSS_RECORD_HEADER.test(line);
140
+ const message = dssRecordMessage(line);
141
+ if (pyTraceback) {
142
+ if (JVM_STACK_FRAME.test(line) || JVM_MORE_FRAMES.test(line)) {
143
+ if (isHeader)
144
+ recordMatches = false;
145
+ continue;
146
+ }
147
+ if (/^\s+\S/.test(message)) {
148
+ kept.push(line); // `File "..."` frame or user-code source line
149
+ if (isHeader)
150
+ recordMatches = false;
151
+ continue;
152
+ }
153
+ if (ERROR_CLASS_NAME.test(message)) {
154
+ kept.push(line); // terminating exception line
155
+ pyTraceback = false;
156
+ if (isHeader)
157
+ recordMatches = false;
158
+ continue;
159
+ }
160
+ pyTraceback = false;
161
+ }
162
+ if (isHeader) {
163
+ recordMatches = isErrorContent(line) || PY_FILE_FRAME.test(message);
164
+ if (recordMatches && (PY_TRACEBACK_HEAD.test(message) || PY_FILE_FRAME.test(message))) {
165
+ pyTraceback = true;
166
+ }
167
+ if (recordMatches)
168
+ kept.push(line);
169
+ continue;
170
+ }
171
+ if (PY_TRACEBACK_HEAD.test(message)) {
172
+ kept.push(line);
173
+ pyTraceback = true;
174
+ continue;
175
+ }
176
+ if (recordMatches
177
+ && !JVM_STACK_FRAME.test(line)
178
+ && !JVM_MORE_FRAMES.test(line)
179
+ && isErrorContent(line))
180
+ kept.push(line);
181
+ }
182
+ return kept.join("\n");
183
+ }
184
+ /**
185
+ * Filter a job log. DSS logs (any timestamped record) use the record-aware
186
+ * errors pass; untimestamped (non-DSS) logs keep the per-line behaviour, since
187
+ * no record structure can be inferred.
188
+ */
92
189
  function filterJobLog(log, filter) {
93
190
  if (!filter)
94
191
  return log;
95
- return jobLogLines(log).filter((line) => lineMatchesLogFilter(line, filter)).join("\n");
192
+ const lines = jobLogLines(log);
193
+ if (filter !== "errors" || !lines.some((line) => DSS_RECORD_HEADER.test(line))) {
194
+ return lines.filter((line) => lineMatchesLogFilter(line, filter)).join("\n");
195
+ }
196
+ return filterJobLogToErrors(lines);
96
197
  }
97
198
  function limitJobLog(log, maxLines) {
98
199
  if (!log)
@@ -163,6 +264,10 @@ function summarizeJobLog(state, log, maxLines, elapsedMs) {
163
264
  ...(progress ? { progress, } : {}),
164
265
  };
165
266
  }
267
+ function nothingToBuildMessage(targets) {
268
+ const names = targets.map((target) => `${target.projectKey}.${target.id}`).join(", ");
269
+ return `The job ran no activity: ${names} ${targets.length === 1 ? "is a managed dataset" : "are managed datasets"} that no recipe produces, so nothing was built. Create a recipe that writes ${targets.length === 1 ? "it" : "them"} or build the recipe's output instead.`;
270
+ }
166
271
  export class JobsResource extends BaseResource {
167
272
  /** List jobs in a project. */
168
273
  async list(projectKey) {
@@ -275,6 +380,12 @@ export class JobsResource extends BaseResource {
275
380
  * confirmed gone. Explicit `includeLogs`/`summary` requests stay honest:
276
381
  * `log`/`logSummary` are only present when logs were actually retrieved.
277
382
  *
383
+ * A DONE job that ran no activity (`progress.total === 0`) is also what DSS
384
+ * reports for an already-up-to-date build, so it stays successful unless an
385
+ * output is a managed dataset that no recipe writes: then nothing can ever
386
+ * build it, and the outcome is `success: false` with
387
+ * `failure.code === "nothing_to_build"`.
388
+ *
278
389
  * On timeout, returns `{ success: false, ... }` rather than throwing.
279
390
  */
280
391
  async wait(jobId, opts) {
@@ -392,8 +503,18 @@ export class JobsResource extends BaseResource {
392
503
  }
393
504
  }
394
505
  }
506
+ const unbuildable = success && gs.total === 0
507
+ ? await this.unbuildableTargets(def.outputs, def.projectKey ?? opts?.projectKey)
508
+ : [];
509
+ const failure = unbuildable.length > 0
510
+ ? {
511
+ code: "nothing_to_build",
512
+ message: nothingToBuildMessage(unbuildable),
513
+ targets: unbuildable,
514
+ }
515
+ : undefined;
395
516
  return {
396
- success,
517
+ success: success && failure === undefined,
397
518
  jobId: def.id ?? jobId,
398
519
  state,
399
520
  type: def.type ?? "unknown",
@@ -405,6 +526,7 @@ export class JobsResource extends BaseResource {
405
526
  running: gs.running ?? 0,
406
527
  total: gs.total ?? null,
407
528
  },
529
+ ...(failure !== undefined ? { failure, } : {}),
408
530
  ...(log !== undefined ? { log, } : {}),
409
531
  ...(logSummary !== undefined ? { logSummary, } : {}),
410
532
  ...(logUnavailable !== undefined ? { logUnavailable, } : {}),
@@ -423,6 +545,51 @@ export class JobsResource extends BaseResource {
423
545
  await sleep(Math.min(nextDelayMs, timeout - elapsedMs));
424
546
  }
425
547
  }
548
+ /**
549
+ * Dataset outputs of a job that ran no activity which nothing can produce.
550
+ * DSS ends such a job DONE exactly like an already-up-to-date build, so the
551
+ * job status cannot tell them apart; the flow graph can: a target is
552
+ * unbuildable when it is a managed dataset (DSS owns its storage, so a
553
+ * recipe is the only way to fill it) that no recipe writes. Unmanaged
554
+ * datasets (uploads, external tables) are sources and legitimately build
555
+ * nothing. Lookups that fail leave the DONE outcome untouched.
556
+ */
557
+ async unbuildableTargets(outputs, fallbackProjectKey) {
558
+ if (!Array.isArray(outputs))
559
+ return [];
560
+ const writtenByProject = new Map();
561
+ const unbuildable = [];
562
+ for (const output of outputs) {
563
+ const record = asRecord(output);
564
+ if (record?.type !== "DATASET" || typeof record.targetDataset !== "string")
565
+ continue;
566
+ const id = record.targetDataset;
567
+ const projectKey = typeof record.targetDatasetProjectKey === "string"
568
+ ? record.targetDatasetProjectKey
569
+ : fallbackProjectKey;
570
+ if (projectKey === undefined)
571
+ continue;
572
+ try {
573
+ const details = await this.client.datasets.get(id, projectKey);
574
+ if (details.managed !== true)
575
+ continue;
576
+ let written = writtenByProject.get(projectKey);
577
+ if (written === undefined) {
578
+ const { graph, } = await this.client.projects.flowTopology(projectKey);
579
+ written = new Set(graph.edges.filter((edge) => edge.relation === "writes").map((edge) => edge.to));
580
+ writtenByProject.set(projectKey, written);
581
+ }
582
+ if (!written.has(id)) {
583
+ unbuildable.push({ type: "DATASET", id, projectKey, reason: "no_producing_recipe", });
584
+ }
585
+ }
586
+ catch (error) {
587
+ if (!(error instanceof DataikuError || error instanceof ClientValidationError))
588
+ throw error;
589
+ }
590
+ }
591
+ return unbuildable;
592
+ }
426
593
  /**
427
594
  * Probe whether a job still exists. Returns `true` when the server reports
428
595
  * it as not found, `false` when it is still visible, and `undefined` when
@@ -37,6 +37,16 @@ export declare class MetricsResource extends BaseResource {
37
37
  getDatasetMetrics(datasetName: string, projectKey?: string): Promise<MetricValues>;
38
38
  /** Compute the dataset metrics configured in DSS for the default partition. */
39
39
  computeDatasetMetrics(datasetName: string, projectKey?: string): Promise<MetricComputationReport>;
40
+ /**
41
+ * Run the dataset's checks (POST /actions/runChecks/{?partitions}). Without
42
+ * `checks` DSS runs the checks configured on the dataset; with `checks` it
43
+ * runs those definitions instead (body `{"checks": [...]}`).
44
+ */
45
+ runDatasetChecks(datasetName: string, opts?: {
46
+ projectKey?: string;
47
+ partitions?: string;
48
+ checks?: unknown[];
49
+ }): Promise<MetricComputationReport>;
40
50
  /** Get the last metric values for a managed folder. */
41
51
  getFolderMetrics(folderId: string, projectKey?: string): Promise<MetricValues>;
42
52
  /** Get the metric history for a dataset's global partition. */
@@ -16,6 +16,16 @@ export class MetricsResource extends BaseResource {
16
16
  const pk = this.enc(projectKey);
17
17
  return this.client.post(`${datasetPath(pk, datasetName)}/actions/computeMetrics?partition=`);
18
18
  }
19
+ /**
20
+ * Run the dataset's checks (POST /actions/runChecks/{?partitions}). Without
21
+ * `checks` DSS runs the checks configured on the dataset; with `checks` it
22
+ * runs those definitions instead (body `{"checks": [...]}`).
23
+ */
24
+ async runDatasetChecks(datasetName, opts = {}) {
25
+ const pk = this.enc(opts.projectKey);
26
+ const query = opts.partitions ? `?partitions=${encodeURIComponent(opts.partitions)}` : "";
27
+ return this.client.post(`${datasetPath(pk, datasetName)}/actions/runChecks/${query}`, opts.checks ? { checks: opts.checks, } : undefined);
28
+ }
19
29
  /** Get the last metric values for a managed folder. */
20
30
  async getFolderMetrics(folderId, projectKey) {
21
31
  const pk = this.enc(projectKey);
@@ -1,14 +1,41 @@
1
1
  import { BaseResource } from "./base.js";
2
+ import { TrainedModel } from "./trained-model.js";
2
3
  export type MlTaskType = "PREDICTION" | "CLUSTERING";
3
- export interface MlTaskCreateOptions {
4
- analysisId: string;
4
+ /** ML task definition; time series forecasting is PREDICTION with predictionType TIMESERIES_FORECAST. */
5
+ export interface MlTaskCreateFields {
5
6
  taskType: MlTaskType;
6
7
  targetVariable?: string;
7
8
  predictionType?: string;
9
+ /** Forecasting tasks: the parsed-date time column. */
10
+ timeVariable?: string;
11
+ /** Forecasting tasks: columns identifying each series of a multi-series dataset. */
12
+ timeseriesIdentifiers?: string[];
8
13
  backendType?: string;
9
14
  guessPolicy?: string;
10
15
  projectKey?: string;
11
16
  }
17
+ export interface MlTaskCreateOptions extends MlTaskCreateFields {
18
+ analysisId: string;
19
+ }
20
+ export interface MlTaskCreateForDatasetOptions extends MlTaskCreateFields {
21
+ inputDataset: string;
22
+ }
23
+ export interface MlTaskListItem extends Record<string, unknown> {
24
+ analysisId?: string;
25
+ mlTaskId?: string;
26
+ analysisName?: string;
27
+ mlTaskName?: string;
28
+ taskType?: string;
29
+ inputDataset?: string;
30
+ }
31
+ export interface MlTaskReguessOptions {
32
+ predictionType?: string;
33
+ targetVariable?: string;
34
+ timeVariable?: string;
35
+ timeseriesIdentifiers?: string[];
36
+ /** Re-guess every setting after the core change, not just the impacted ones. */
37
+ fullReguess?: boolean;
38
+ }
12
39
  export interface MlTaskCreateResult extends Record<string, unknown> {
13
40
  analysisId?: string;
14
41
  mlTaskId?: string;
@@ -91,6 +118,14 @@ export interface MlTaskDeployResult extends Record<string, unknown> {
91
118
  savedModelId?: string;
92
119
  trainRecipeName?: string;
93
120
  }
121
+ /** Query of POST .../guess; repeated `timeseriesIdentifiers` as dataikuapi sends a list (pure; shared with `--plan`). */
122
+ export declare function mlTaskReguessQuery(opts: MlTaskReguessOptions): URLSearchParams;
123
+ /**
124
+ * Task-creation body shared by `create` (existing analysis) and
125
+ * `createForDataset` (new analysis), with dataikuapi's defaults (pure; shared
126
+ * with `--plan`).
127
+ */
128
+ export declare function mlTaskCreateBody(opts: MlTaskCreateFields): Record<string, unknown>;
94
129
  export declare class MlTasksResource extends BaseResource {
95
130
  /** Create a prediction or clustering task in an existing visual analysis. */
96
131
  create(opts: MlTaskCreateOptions): Promise<MlTaskCreateResult>;
@@ -122,5 +157,33 @@ export declare class MlTasksResource extends BaseResource {
122
157
  deployToFlow(opts: MlTaskDeployOptions): Promise<MlTaskDeployResult>;
123
158
  /** Delete an ML task and its trained models. */
124
159
  delete(analysisId: string, mlTaskId: string, projectKey?: string): Promise<void>;
160
+ /** Every ML task of the project, with its visual analysis (GET /models/lab/). */
161
+ list(projectKey?: string): Promise<MlTaskListItem[]>;
162
+ /**
163
+ * Create a new visual analysis on `inputDataset` with one ML task
164
+ * (POST /models/lab/). Returns `{analysisId, mlTaskId}`; DSS guesses the
165
+ * settings asynchronously (watch `status().guessing`).
166
+ */
167
+ createForDataset(opts: MlTaskCreateForDatasetOptions): Promise<MlTaskCreateResult>;
168
+ /**
169
+ * Re-guess the task settings: all of them, or (prediction tasks) after
170
+ * setting a new prediction type, target, time variable, or series
171
+ * identifiers — only the impacted settings unless `fullReguess`.
172
+ */
173
+ reguess(analysisId: string, mlTaskId: string, opts?: MlTaskReguessOptions, projectKey?: string): Promise<void>;
174
+ /**
175
+ * Time series forecasting tasks: change the forecasting parameters
176
+ * (`forecastHorizon`, `validationHorizons`, `timestepParams`,
177
+ * `updateAlgorithmSettings`) and re-guess the impacted algorithm settings.
178
+ */
179
+ reguessForecasting(analysisId: string, mlTaskId: string, params: Record<string, unknown>, projectKey?: string): Promise<void>;
180
+ /**
181
+ * One trained lab model as a {@link TrainedModel}: diagnostics
182
+ * (subpopulation analyses, partial dependencies, timeseries residuals and
183
+ * per-series results), documentation, scoring exports, user metadata.
184
+ */
185
+ model(analysisId: string, mlTaskId: string, modelId: string, projectKey?: string): TrainedModel;
186
+ /** GET a generated lab model documentation (docx) by the `exportId` of its finished future. */
187
+ downloadModelDocumentation(exportId: string, projectKey?: string): Promise<Response>;
125
188
  private taskPath;
126
189
  }