dataiku-sdk 3.8.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +4 -0
- package/dist/packages/types/src/index.js +4 -0
- package/dist/src/cli/coerce.d.ts +6 -0
- package/dist/src/cli/coerce.js +9 -0
- package/dist/src/cli/command-syntax.json +36 -0
- package/dist/src/cli/commands/app.js +2 -2
- package/dist/src/cli/commands/code-env.js +5 -6
- package/dist/src/cli/commands/code.js +16 -2
- package/dist/src/cli/commands/dashboard.js +1 -2
- package/dist/src/cli/commands/dataset.js +2 -2
- package/dist/src/cli/commands/flow-zone.js +1 -1
- package/dist/src/cli/commands/job.js +101 -22
- package/dist/src/cli/commands/plugin.js +7 -7
- package/dist/src/cli/commands/project-library.js +2 -2
- package/dist/src/cli/commands/project.js +25 -7
- package/dist/src/cli/commands/recipe.js +11 -19
- package/dist/src/cli/commands/scenario.js +2 -4
- package/dist/src/cli/contract.js +29 -10
- package/dist/src/cli/env.js +2 -3
- package/dist/src/cli/helpers/app-permissions.d.ts +1 -1
- package/dist/src/cli/helpers/app-permissions.js +2 -2
- package/dist/src/cli/helpers/cleanup.d.ts +2 -0
- package/dist/src/cli/helpers/cleanup.js +122 -8
- package/dist/src/cli/helpers/flow-zone.d.ts +1 -1
- package/dist/src/cli/helpers/flow-zone.js +3 -4
- package/dist/src/cli/list-output.d.ts +5 -0
- package/dist/src/cli/list-output.js +11 -2
- package/dist/src/cli/list-shapes.d.ts +6 -0
- package/dist/src/cli/list-shapes.js +7 -1
- package/dist/src/cli/output-schemas.d.ts +1 -1
- package/dist/src/cli/output-schemas.js +28 -4
- package/dist/src/cli/output.js +7 -0
- package/dist/src/cli/project-map-output.d.ts +24 -0
- package/dist/src/cli/project-map-output.js +21 -0
- package/dist/src/cli/read-shapes.d.ts +14 -0
- package/dist/src/cli/read-shapes.js +114 -0
- package/dist/src/cli/result-shapes.d.ts +6 -0
- package/dist/src/cli/result-shapes.js +235 -0
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +4 -5
- package/dist/src/cli.js +77 -33
- package/dist/src/client.d.ts +14 -2
- package/dist/src/client.js +18 -10
- package/dist/src/generated/action-output-schemas.json +242 -35
- package/dist/src/index.d.ts +1 -1
- package/dist/src/resources/datasets.d.ts +11 -1
- package/dist/src/resources/datasets.js +91 -30
- package/dist/src/resources/jobs.d.ts +37 -6
- package/dist/src/resources/jobs.js +110 -44
- package/dist/src/resources/projects.js +2 -2
- package/dist/src/resources/recipes.d.ts +12 -1
- package/dist/src/resources/recipes.js +39 -12
- package/dist/src/resources/scenarios.d.ts +8 -1
- package/dist/src/resources/scenarios.js +51 -12
- package/dist/src/resources/wiki.js +3 -7
- package/dist/src/skill.js +1 -2
- package/dist/src/utils/project-archive.d.ts +25 -0
- package/dist/src/utils/project-archive.js +216 -5
- package/dist/src/utils/response-file.js +15 -8
- package/dist/src/utils/secret-sanitize.d.ts +11 -2
- package/dist/src/utils/secret-sanitize.js +23 -6
- package/package.json +13 -14
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
- package/skills/dataiku-dss/references/discovery.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +2 -2
package/README.md
CHANGED
|
@@ -16,7 +16,7 @@ Malformed user/group listings and scenario run histories fail explicitly instead
|
|
|
16
16
|
|
|
17
17
|
## Platform support
|
|
18
18
|
|
|
19
|
-
The published `dss` CLI requires Bun >= 1.4.
|
|
19
|
+
The published `dss` CLI requires Bun >= 1.4.3 and supports Linux, macOS, and Windows. CI runs Bun 1.4.3 on all three operating systems. The runtime dependency is pure JavaScript, so the same package also runs on every Bun-supported x64 and ARM64 system.
|
|
20
20
|
|
|
21
21
|
Run directly with Bun:
|
|
22
22
|
|
|
@@ -29,7 +29,7 @@ Or install the npm binary:
|
|
|
29
29
|
```text
|
|
30
30
|
npm install --global dataiku-sdk
|
|
31
31
|
```
|
|
32
|
-
The installed `dss` bin bootstraps Bun with automatic `.env` preloading disabled, so Bun >= 1.4.
|
|
32
|
+
The installed `dss` bin bootstraps Bun with automatic `.env` preloading disabled, so Bun >= 1.4.3 must be on `PATH`; the CLI's own `.env` handling stays authoritative. `bunx --bun` forces Bun's Node-compatibility mode, which provides the same no-preload guarantee across operating systems.
|
|
33
33
|
|
|
34
34
|
Bun is the only supported runtime and the package manager. Examples below assume an installed `dss` binary. From a checkout, use `bun --no-env-file src/cli.ts ...` or the packaged launcher `bun --no-env-file ./bin/dss.js ...`. From another working directory, pass the checkout's absolute `bin/dss.js` path to Bun.
|
|
35
35
|
`--no-env-file` disables Bun's automatic preloading only; the CLI still applies its documented `.env` handling unless `DATAIKU_DISABLE_ENV=1` is set.
|
|
@@ -181,7 +181,7 @@ then update `tests/live-coverage.ts` when the
|
|
|
181
181
|
registry changes; catalogue drift fails the offline gate. Plan/dry-run calls do not count
|
|
182
182
|
as executed live coverage.
|
|
183
183
|
|
|
184
|
-
Regular `bun run
|
|
184
|
+
Regular `bun run verify` and `bun run lint` include the live runner and fixtures.
|
|
185
185
|
Lint includes project-owned rules in `tools/oxlint/index.ts`: accumulator-copy, widen-then-assert, and chained-assertion errors, alongside the native accumulating-spread check.
|
|
186
186
|
For focused verification:
|
|
187
187
|
|
|
@@ -520,8 +520,10 @@ UI-click smoke test. The CLI does not guess private endpoints for those operatio
|
|
|
520
520
|
|
|
521
521
|
`project map` returns one agent-oriented JSON view of the Flow: normalized nodes and edges,
|
|
522
522
|
joined zone metadata, cycle-safe topological layers, weakly connected components, diagnostics, and
|
|
523
|
-
a stable `topologyFingerprint`.
|
|
524
|
-
`
|
|
523
|
+
a stable `topologyFingerprint`. Output is deduplicated: node `name` appears only when it differs from
|
|
524
|
+
`id`; each node carries `zoneId`/`componentId` (zone names live in `zones[]`, membership is derived
|
|
525
|
+
from those ids, so `zones[]`/`components[]` omit `nodeIds`); `warnings` appears only when non-empty and
|
|
526
|
+
`truncation` only when the map was cut. `zones` preserves zone color, position, and item count.
|
|
525
527
|
|
|
526
528
|
```bash
|
|
527
529
|
dss project map --project-key MYPROJ
|
package/dist/build-metadata.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"format":1,"buildRevision":"
|
|
1
|
+
{"format":1,"buildRevision":"ab176cb1de853bfbdc58627b38a131ce1fc1681d"}
|
|
@@ -284,6 +284,10 @@ export declare const JobWaitResultSchema: import("@sinclair/typebox").TObject<{
|
|
|
284
284
|
pollCount: import("@sinclair/typebox").TNumber;
|
|
285
285
|
success: import("@sinclair/typebox").TBoolean;
|
|
286
286
|
timedOut: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
287
|
+
/** Present on a wait timeout: the wait never cancels the job, so this is `false`. */
|
|
288
|
+
cancelled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
289
|
+
/** CLI only, on a wait timeout: the command that resumes waiting for the still-running job. */
|
|
290
|
+
resume: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
287
291
|
progress: import("@sinclair/typebox").TObject<{
|
|
288
292
|
done: import("@sinclair/typebox").TNumber;
|
|
289
293
|
failed: import("@sinclair/typebox").TNumber;
|
|
@@ -308,6 +308,10 @@ export const JobWaitResultSchema = Type.Object({
|
|
|
308
308
|
pollCount: Type.Number(),
|
|
309
309
|
success: Type.Boolean(),
|
|
310
310
|
timedOut: Type.Optional(Type.Boolean()),
|
|
311
|
+
/** Present on a wait timeout: the wait never cancels the job, so this is `false`. */
|
|
312
|
+
cancelled: Type.Optional(Type.Boolean()),
|
|
313
|
+
/** CLI only, on a wait timeout: the command that resumes waiting for the still-running job. */
|
|
314
|
+
resume: Type.Optional(Type.String()),
|
|
311
315
|
progress: Type.Object({
|
|
312
316
|
done: Type.Number(),
|
|
313
317
|
failed: Type.Number(),
|
package/dist/src/cli/coerce.d.ts
CHANGED
|
@@ -21,6 +21,12 @@ export declare function parseJsonObject(text: string, source: string): Record<st
|
|
|
21
21
|
export declare function json(v: string | boolean | undefined, source?: string): Record<string, unknown> | undefined;
|
|
22
22
|
export declare function readStdinText(): string;
|
|
23
23
|
export declare function readInputText(path: string, flag: string): string;
|
|
24
|
+
/**
|
|
25
|
+
* Async UTF-8 read of a user-supplied path (BOM kept, like `readFileSync`).
|
|
26
|
+
* Stays on node:fs: Bun 1.4.3 `Bun.file(path)` reads of a missing path ended the
|
|
27
|
+
* process on Windows CI instead of rejecting with ENOENT.
|
|
28
|
+
*/
|
|
29
|
+
export declare function readTextFile(path: string): Promise<string>;
|
|
24
30
|
export declare function seenJsonInputCredentials(): readonly string[];
|
|
25
31
|
export declare function jsonInput(flags: Record<string, string | boolean>): Record<string, unknown> | undefined;
|
|
26
32
|
export declare function unknownJsonInput(flags: Record<string, string | boolean>): unknown;
|
package/dist/src/cli/coerce.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
|
+
import { readFile, } from "node:fs/promises";
|
|
2
3
|
import { credentialValues, } from "../utils/secret-sanitize.js";
|
|
3
4
|
import { UsageError, } from "./usage.js";
|
|
4
5
|
function safeValuePreview(value) {
|
|
@@ -174,6 +175,14 @@ export function readInputText(path, flag) {
|
|
|
174
175
|
throw new UsageError(`Could not read ${flag} file: ${path}`, "validation_failed", "Verify that the path names an existing, readable file.", { flag, path, ...(cause ? { cause, } : {}), });
|
|
175
176
|
}
|
|
176
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* Async UTF-8 read of a user-supplied path (BOM kept, like `readFileSync`).
|
|
180
|
+
* Stays on node:fs: Bun 1.4.3 `Bun.file(path)` reads of a missing path ended the
|
|
181
|
+
* process on Windows CI instead of rejecting with ENOENT.
|
|
182
|
+
*/
|
|
183
|
+
export async function readTextFile(path) {
|
|
184
|
+
return readFile(path, "utf-8");
|
|
185
|
+
}
|
|
177
186
|
/**
|
|
178
187
|
* Credential values seen in this process's JSON inputs. DSS error text can
|
|
179
188
|
* echo the request payload, so error reports redact these.
|
|
@@ -4513,6 +4513,15 @@
|
|
|
4513
4513
|
{
|
|
4514
4514
|
"arg": "name"
|
|
4515
4515
|
},
|
|
4516
|
+
{
|
|
4517
|
+
"optional": [
|
|
4518
|
+
[
|
|
4519
|
+
{
|
|
4520
|
+
"flag": "full"
|
|
4521
|
+
}
|
|
4522
|
+
]
|
|
4523
|
+
]
|
|
4524
|
+
},
|
|
4516
4525
|
{
|
|
4517
4526
|
"optional": [
|
|
4518
4527
|
[
|
|
@@ -4531,6 +4540,15 @@
|
|
|
4531
4540
|
{
|
|
4532
4541
|
"arg": "name"
|
|
4533
4542
|
},
|
|
4543
|
+
{
|
|
4544
|
+
"optional": [
|
|
4545
|
+
[
|
|
4546
|
+
{
|
|
4547
|
+
"flag": "full"
|
|
4548
|
+
}
|
|
4549
|
+
]
|
|
4550
|
+
]
|
|
4551
|
+
},
|
|
4534
4552
|
{
|
|
4535
4553
|
"optional": [
|
|
4536
4554
|
[
|
|
@@ -7108,6 +7126,15 @@
|
|
|
7108
7126
|
{
|
|
7109
7127
|
"arg": "id"
|
|
7110
7128
|
},
|
|
7129
|
+
{
|
|
7130
|
+
"optional": [
|
|
7131
|
+
[
|
|
7132
|
+
{
|
|
7133
|
+
"flag": "full"
|
|
7134
|
+
}
|
|
7135
|
+
]
|
|
7136
|
+
]
|
|
7137
|
+
},
|
|
7111
7138
|
{
|
|
7112
7139
|
"optional": [
|
|
7113
7140
|
[
|
|
@@ -12890,6 +12917,15 @@
|
|
|
12890
12917
|
"dss",
|
|
12891
12918
|
"project",
|
|
12892
12919
|
"get",
|
|
12920
|
+
{
|
|
12921
|
+
"optional": [
|
|
12922
|
+
[
|
|
12923
|
+
{
|
|
12924
|
+
"flag": "full"
|
|
12925
|
+
}
|
|
12926
|
+
]
|
|
12927
|
+
]
|
|
12928
|
+
},
|
|
12893
12929
|
{
|
|
12894
12930
|
"optional": [
|
|
12895
12931
|
[
|
|
@@ -754,7 +754,7 @@ export const appCommands = withUsage("app", {
|
|
|
754
754
|
const usage = commandUsage("app", "permissions-diff");
|
|
755
755
|
requireNoArgs(_a, usage);
|
|
756
756
|
const file = requiredStringFlag(f, "file", usage);
|
|
757
|
-
const snapshot = readAppPermissionsSnapshot(file);
|
|
757
|
+
const snapshot = await readAppPermissionsSnapshot(file);
|
|
758
758
|
const projectKey = c.resolveProjectKey(f["project-key"]);
|
|
759
759
|
assertAppPermissionsSnapshotBinding(snapshot, projectKey, c.getBaseUrl());
|
|
760
760
|
const currentProjectIncarnationHash = await requireProjectIncarnationHash(c, projectKey);
|
|
@@ -779,7 +779,7 @@ export const appCommands = withUsage("app", {
|
|
|
779
779
|
const usage = commandUsage("app", "permissions-restore");
|
|
780
780
|
requireNoArgs(_a, usage);
|
|
781
781
|
const file = requiredStringFlag(f, "file", usage);
|
|
782
|
-
const snapshot = readAppPermissionsSnapshot(file);
|
|
782
|
+
const snapshot = await readAppPermissionsSnapshot(file);
|
|
783
783
|
const projectKey = c.resolveProjectKey(f["project-key"]);
|
|
784
784
|
assertAppPermissionsSnapshotBinding(snapshot, projectKey, c.getBaseUrl());
|
|
785
785
|
const currentProjectIncarnationHash = await requireProjectIncarnationHash(c, projectKey);
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { readFileSync, writeFileSync, } from "node:fs";
|
|
2
1
|
import { ClientValidationError, } from "../../errors.js";
|
|
3
2
|
import { SHA256_HEX_PATTERN, stableHash, } from "../../utils/stable-hash.js";
|
|
4
|
-
import { json, jsonInput, num, numFlag, parseBooleanOption, } from "../coerce.js";
|
|
3
|
+
import { json, jsonInput, num, numFlag, parseBooleanOption, readTextFile, } from "../coerce.js";
|
|
5
4
|
import { executionMode, } from "../flags.js";
|
|
6
5
|
import { enqueueCliWarning, isNotFoundError, readIfExists, skipResult, } from "../output.js";
|
|
7
6
|
import { commandUsage, withUsage, } from "../syntax.js";
|
|
@@ -49,9 +48,9 @@ function splitPackageSpec(raw) {
|
|
|
49
48
|
* (set-packages replaces specPackageList wholesale); only a call with no
|
|
50
49
|
* package source flag at all is a usage error.
|
|
51
50
|
*/
|
|
52
|
-
function codeEnvPackageList(flags) {
|
|
51
|
+
async function codeEnvPackageList(flags) {
|
|
53
52
|
const file = flags["file"];
|
|
54
|
-
const packages = typeof file === "string" ? splitPackageSpec(
|
|
53
|
+
const packages = typeof file === "string" ? splitPackageSpec(await readTextFile(file)) : [];
|
|
55
54
|
if (typeof flags["packages"] === "string") {
|
|
56
55
|
packages.push(...splitPackageSpec(flags["packages"]));
|
|
57
56
|
}
|
|
@@ -122,7 +121,7 @@ export const codeEnvCommands = withUsage("code-env", {
|
|
|
122
121
|
});
|
|
123
122
|
const output = f["output"];
|
|
124
123
|
if (typeof output === "string" && output.trim().length > 0) {
|
|
125
|
-
|
|
124
|
+
await Bun.write(output, result.log);
|
|
126
125
|
if (result.truncated || result.tailed) {
|
|
127
126
|
const limits = [
|
|
128
127
|
result.tailed ? `tail limited to ${maxLines ?? 500} lines` : undefined,
|
|
@@ -257,7 +256,7 @@ export const codeEnvCommands = withUsage("code-env", {
|
|
|
257
256
|
const usage = commandUsage("code-env", "set-packages");
|
|
258
257
|
requireArgs(a, 2, usage);
|
|
259
258
|
const envLang = requireEnvLang(a[0], usage);
|
|
260
|
-
const packages = codeEnvPackageList(f);
|
|
259
|
+
const packages = await codeEnvPackageList(f);
|
|
261
260
|
const installCorePackages = parseBooleanOption(f["install-core-packages"], "--install-core-packages");
|
|
262
261
|
const expectHash = validateExpectHash(f["expect-hash"]);
|
|
263
262
|
if (executionMode(f).dryRun) {
|
|
@@ -115,19 +115,33 @@ export const codeCommands = withUsage("code", {
|
|
|
115
115
|
maxLogBytes: run.maxLogBytes,
|
|
116
116
|
cleanup: run.cleanup,
|
|
117
117
|
...(run.timedOut === true ? { timedOut: true, timeoutMs: run.timeoutMs, } : {}),
|
|
118
|
+
...(run.outputTruncated === true ? { outputTruncated: true, } : {}),
|
|
119
|
+
...(run.outputOmittedBytes !== undefined
|
|
120
|
+
? { outputOmittedBytes: run.outputOmittedBytes, }
|
|
121
|
+
: {}),
|
|
118
122
|
};
|
|
119
123
|
if (f["full-log"] === true || run.output === undefined) {
|
|
120
124
|
result.log = run.log;
|
|
121
125
|
}
|
|
126
|
+
if (run.outputTruncated === true) {
|
|
127
|
+
enqueueCliWarning({
|
|
128
|
+
code: "code_run_output_truncated",
|
|
129
|
+
hint: run.outputOmittedBytes !== undefined
|
|
130
|
+
? `Script output exceeded the capture budget; the first ${run.outputOmittedBytes} bytes were dropped and \`output\` holds the last part. Raise --max-log-bytes (the budget is a quarter of it) or print less.`
|
|
131
|
+
: "The run log exceeded --max-log-bytes, so the start of the script output was cut and `output` holds the last part. Raise --max-log-bytes or print less.",
|
|
132
|
+
...(run.outputOmittedBytes !== undefined ? { omittedBytes: run.outputOmittedBytes, } : {}),
|
|
133
|
+
maxLogBytes: run.maxLogBytes,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
122
136
|
if (run.cleanup.status === "failed") {
|
|
123
137
|
reportCleanupFailure({ scenarioId: run.scenarioId, error: run.cleanup.error ?? "unknown", });
|
|
124
138
|
}
|
|
125
139
|
return result;
|
|
126
140
|
},
|
|
127
|
-
description: "Run Python through a throwaway custom-python scenario and return captured output plus outcome. --timeout caps waiting; --max-log-bytes caps
|
|
141
|
+
description: "Run Python through a throwaway custom-python scenario and return captured output plus outcome. --timeout caps waiting; --max-log-bytes caps the log tail read (script output keeps its last quarter; a code_run_output_truncated warning reports cuts); --full-log returns the capped raw log. Exits 4 unless outcome is SUCCESS.",
|
|
128
142
|
examples: [
|
|
129
143
|
"dss code run --file inspect.py",
|
|
130
|
-
"dss code run --file inspect.py --env
|
|
144
|
+
"dss code run --file inspect.py --env py311_pandas",
|
|
131
145
|
"cat snippet.py | dss code run --stdin",
|
|
132
146
|
"dss code run --file inspect.py --full-log",
|
|
133
147
|
],
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as fs from "node:fs";
|
|
2
1
|
import { ClientValidationError, } from "../../errors.js";
|
|
3
2
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
4
3
|
import { jsonInput, parseBooleanOption, } from "../coerce.js";
|
|
@@ -73,7 +72,7 @@ export const dashboardCommands = withUsage("dashboard", {
|
|
|
73
72
|
if (!isPdf) {
|
|
74
73
|
throw new ClientValidationError("DSS dashboard export did not return a PDF.", "validation_failed", "Confirm DSS graphics export is configured and retry the export.");
|
|
75
74
|
}
|
|
76
|
-
|
|
75
|
+
await Bun.write(outputPath, bytes);
|
|
77
76
|
return {
|
|
78
77
|
path: outputPath,
|
|
79
78
|
bytes: bytes.length,
|
|
@@ -143,7 +143,7 @@ export const datasetCommands = withUsage("dataset", {
|
|
|
143
143
|
requireArgs(a, 1, commandUsage("dataset", "get"));
|
|
144
144
|
return c.datasets.get(a[0], f["project-key"]);
|
|
145
145
|
},
|
|
146
|
-
description: "Get
|
|
146
|
+
description: "Get the settings for a dataset (compact by default: empty values and metrics/version metadata are dropped, structure kept for get-edit-update; --full returns the raw DSS object).",
|
|
147
147
|
examples: ["dss dataset get orders", "dss dataset get orders --project-key MYPROJ",],
|
|
148
148
|
},
|
|
149
149
|
schema: {
|
|
@@ -506,7 +506,7 @@ export const datasetCommands = withUsage("dataset", {
|
|
|
506
506
|
requireArgs(a, 1, commandUsage("dataset", "info"));
|
|
507
507
|
return c.datasets.info(a[0], f["project-key"]);
|
|
508
508
|
},
|
|
509
|
-
description: "Get
|
|
509
|
+
description: "Get dataset info: type, connection, format, schema columns, flow recipes, buildability, data quality, and last modification (compact by default; --full returns the raw DSS info object).",
|
|
510
510
|
examples: ["dss dataset info orders", "dss dataset info orders --project-key MYPROJ",],
|
|
511
511
|
},
|
|
512
512
|
"column-lineage": {
|
|
@@ -275,7 +275,7 @@ export const flowZoneCommands = withUsage("flow-zone", {
|
|
|
275
275
|
handler: async (c, _a, f) => {
|
|
276
276
|
const usage = commandUsage("flow-zone", "organize");
|
|
277
277
|
const pk = f["project-key"];
|
|
278
|
-
const parsed = readFlowZoneOrganizePlan(f, usage);
|
|
278
|
+
const parsed = await readFlowZoneOrganizePlan(f, usage);
|
|
279
279
|
const sync = f["sync"] === true;
|
|
280
280
|
const validateObjects = f["validate-objects"] === true;
|
|
281
281
|
const recreateOnMove = f["recreate-on-move"] === true;
|
|
@@ -1,12 +1,57 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { ClientValidationError, DataikuError, } from "../../errors.js";
|
|
1
|
+
import { resolve, } from "node:path";
|
|
2
|
+
import { ClientValidationError, DataikuError, dataikuErrorCode, } from "../../errors.js";
|
|
4
3
|
import { asRecord, } from "../../utils/records.js";
|
|
5
4
|
import { jobBuildTargetTypeFromFlags, jobLogFilterFromFlag, maxLogLinesFromFlags, num, } from "../coerce.js";
|
|
6
5
|
import { executionMode, } from "../flags.js";
|
|
6
|
+
import { compactValue, } from "../list-output.js";
|
|
7
|
+
import { COMPACT_LIST_FIELDS, } from "../list-shapes.js";
|
|
7
8
|
import { encodedProjectEndpoint, enqueueCliWarning, } from "../output.js";
|
|
8
9
|
import { commandUsage, withUsage, } from "../syntax.js";
|
|
9
10
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
11
|
+
/** Lines `job log` prints to stdout when no `--max-lines` is given. */
|
|
12
|
+
const DEFAULT_STDOUT_LOG_LINES = 100;
|
|
13
|
+
/**
|
|
14
|
+
* Tail a (filtered) job log for stdout. `0`/`-1` return everything; a
|
|
15
|
+
* truncated tail enqueues a `log_truncated` warning carrying the shown and
|
|
16
|
+
* total line counts.
|
|
17
|
+
*/
|
|
18
|
+
function tailJobLogForStdout(log, limit) {
|
|
19
|
+
if (!log || limit === 0 || limit === -1)
|
|
20
|
+
return log;
|
|
21
|
+
const lines = log.split(/\r?\n/);
|
|
22
|
+
const hasTrailingLineBreak = lines[lines.length - 1] === "";
|
|
23
|
+
if (hasTrailingLineBreak)
|
|
24
|
+
lines.pop();
|
|
25
|
+
const shown = Math.max(1, limit);
|
|
26
|
+
if (lines.length <= shown)
|
|
27
|
+
return log;
|
|
28
|
+
enqueueCliWarning({
|
|
29
|
+
code: "log_truncated",
|
|
30
|
+
shown,
|
|
31
|
+
total: lines.length,
|
|
32
|
+
hint: "Raise --max-lines (0 = all), add --errors-only, or use --output PATH.",
|
|
33
|
+
});
|
|
34
|
+
const tail = lines.slice(-shown).join("\n");
|
|
35
|
+
return hasTrailingLineBreak ? `${tail}\n` : tail;
|
|
36
|
+
}
|
|
37
|
+
const RESPONSE_TOO_LARGE_STATUS_TEXT = "Response Too Large";
|
|
38
|
+
/** POSIX-shell quoting for a resume command argument. */
|
|
39
|
+
function shellArg(value) {
|
|
40
|
+
return /^[\w.:@%+=,/-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
|
|
41
|
+
}
|
|
42
|
+
/** Command that resumes waiting for a job whose wait timed out. */
|
|
43
|
+
function jobWaitResumeCommand(jobId, projectKey) {
|
|
44
|
+
return `dss job wait ${shellArg(jobId)}${projectKey ? ` --project-key ${shellArg(projectKey)}` : ""}`;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A wait that timed out only stopped polling: the job is still running in DSS.
|
|
48
|
+
* Attach the command that resumes waiting; other results pass through.
|
|
49
|
+
*/
|
|
50
|
+
function withWaitResume(result, projectKey) {
|
|
51
|
+
if (result.timedOut !== true)
|
|
52
|
+
return result;
|
|
53
|
+
return { ...result, resume: jobWaitResumeCommand(result.jobId, projectKey), };
|
|
54
|
+
}
|
|
10
55
|
/**
|
|
11
56
|
* With an empty output schema DSS ends a dataset build DONE after writing no
|
|
12
57
|
* columns and no rows, without an error. Flag that after a successful build so
|
|
@@ -94,8 +139,9 @@ function jobSummaryType(job) {
|
|
|
94
139
|
?? "unknown";
|
|
95
140
|
}
|
|
96
141
|
function jobSummaryState(job) {
|
|
142
|
+
const listState = compactValue(job, COMPACT_LIST_FIELDS["job"]["state"]);
|
|
97
143
|
return stringPath(job, ["baseStatus", "state",])
|
|
98
|
-
??
|
|
144
|
+
?? (typeof listState === "string" ? listState : undefined)
|
|
99
145
|
?? "unknown";
|
|
100
146
|
}
|
|
101
147
|
function filteredJobList(jobs, flags) {
|
|
@@ -176,7 +222,19 @@ function jobDurationMs(details) {
|
|
|
176
222
|
}
|
|
177
223
|
async function jobInspectionSummary(client, jobId, flags) {
|
|
178
224
|
const projectKey = flags["project-key"];
|
|
179
|
-
|
|
225
|
+
// A job-detail body over the buffered response cap must not sink the whole
|
|
226
|
+
// summary: keep the log part and report the details failure structurally.
|
|
227
|
+
let details = {};
|
|
228
|
+
let detailsError;
|
|
229
|
+
try {
|
|
230
|
+
details = await client.jobs.get(jobId, projectKey);
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
if (!(error instanceof DataikuError && error.statusText === RESPONSE_TOO_LARGE_STATUS_TEXT)) {
|
|
234
|
+
throw error;
|
|
235
|
+
}
|
|
236
|
+
detailsError = { code: dataikuErrorCode(error.category), message: error.message, };
|
|
237
|
+
}
|
|
180
238
|
let log;
|
|
181
239
|
let logError;
|
|
182
240
|
try {
|
|
@@ -207,6 +265,7 @@ async function jobInspectionSummary(client, jobId, flags) {
|
|
|
207
265
|
jobId: jobSummaryId(details, jobId),
|
|
208
266
|
state: jobSummaryState(details),
|
|
209
267
|
type: jobSummaryType(details),
|
|
268
|
+
...(detailsError ? { detailsError, } : {}),
|
|
210
269
|
...(durationMs !== undefined ? { durationMs, } : {}),
|
|
211
270
|
...(outputs !== undefined ? { outputs, } : {}),
|
|
212
271
|
warnings: jobWarningSummary(details, log),
|
|
@@ -227,7 +286,7 @@ async function jobInspectionSummary(client, jobId, flags) {
|
|
|
227
286
|
export const jobCommands = withUsage("job", {
|
|
228
287
|
list: {
|
|
229
288
|
handler: async (c, _a, f) => filteredJobList(await c.jobs.list(f["project-key"]), f),
|
|
230
|
-
description: "List recent jobs, optionally filtered for automation.",
|
|
289
|
+
description: "List recent jobs, optionally filtered for automation. In-progress jobs, which DSS lists without a state, show RUNNING; `job get` has the exact phase.",
|
|
231
290
|
examples: ["dss job list --state DONE --latest", "dss job list --contains WLM225S --limit 10",],
|
|
232
291
|
},
|
|
233
292
|
get: {
|
|
@@ -235,7 +294,7 @@ export const jobCommands = withUsage("job", {
|
|
|
235
294
|
requireArgs(a, 1, commandUsage("job", "get"));
|
|
236
295
|
return c.jobs.get(a[0], f["project-key"]);
|
|
237
296
|
},
|
|
238
|
-
description: "Get job
|
|
297
|
+
description: "Get job state, outputs, progress, error, and per-activity status (compact by default; --full returns the raw DSS job object). The log tail is never included: use `dss job log ID --errors-only`.",
|
|
239
298
|
examples: ["dss job get JOB_ID",],
|
|
240
299
|
},
|
|
241
300
|
summary: {
|
|
@@ -252,23 +311,41 @@ export const jobCommands = withUsage("job", {
|
|
|
252
311
|
const logFilter = f["errors-only"] === true
|
|
253
312
|
? "errors"
|
|
254
313
|
: jobLogFilterFromFlag(f["log-filter"]);
|
|
255
|
-
const
|
|
314
|
+
const outputFile = f["output"]
|
|
315
|
+
?? f["output-file"];
|
|
316
|
+
const requestedLines = maxLogLinesFromFlags(f);
|
|
317
|
+
// Stdout is tailed here (not in the SDK) so the total can be reported.
|
|
318
|
+
const details = await c.jobs.logDetails(a[0], {
|
|
256
319
|
activity: f["activity"],
|
|
257
320
|
logId: f["log-id"],
|
|
258
321
|
logFilter,
|
|
259
|
-
maxLogLines:
|
|
322
|
+
maxLogLines: outputFile ? requestedLines : -1,
|
|
260
323
|
projectKey: f["project-key"],
|
|
261
324
|
});
|
|
262
|
-
const
|
|
263
|
-
|
|
325
|
+
const log = details.text;
|
|
326
|
+
if (details.truncated) {
|
|
327
|
+
enqueueCliWarning({
|
|
328
|
+
code: "log_truncated",
|
|
329
|
+
reason: "byte_cap",
|
|
330
|
+
earlyLinesDropped: true,
|
|
331
|
+
maxLogBytes: details.maxLogBytes,
|
|
332
|
+
hint: "Only the most recent part of the log was retrieved; early lines (and any earlier errors) were dropped. Use `dss job get` for job-level and per-activity errors.",
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
if (logFilter === "errors" && log.length === 0) {
|
|
336
|
+
enqueueCliWarning({
|
|
337
|
+
code: "no_error_lines",
|
|
338
|
+
scannedLines: details.scannedLines,
|
|
339
|
+
hint: `No error lines matched in ${details.scannedLines} scanned log lines. Run \`dss job get ${a[0]}\` for the job-level and per-activity errors, or \`dss job log ${a[0]}\` without --errors-only for the full log.`,
|
|
340
|
+
});
|
|
341
|
+
}
|
|
264
342
|
if (!outputFile)
|
|
265
|
-
return log;
|
|
343
|
+
return tailJobLogForStdout(log, requestedLines ?? DEFAULT_STDOUT_LOG_LINES);
|
|
266
344
|
const outputPath = resolve(outputFile);
|
|
267
|
-
await
|
|
268
|
-
await writeFile(outputPath, log.endsWith("\n") ? log : `${log}\n`, "utf-8");
|
|
345
|
+
await Bun.write(outputPath, log.endsWith("\n") ? log : `${log}\n`);
|
|
269
346
|
return outputPath;
|
|
270
347
|
},
|
|
271
|
-
description: "Get public API job log output; find why a job failed. Use --errors-only
|
|
348
|
+
description: "Get public API job log output (last 100 lines by default; --max-lines N, 0 = all); find why a job failed. Use --errors-only for error/traceback lines and --output PATH to write it to a file (stdout returns the path). --log-id is accepted for UI parity; API keys cannot read browser-only cat-activity-log files.",
|
|
272
349
|
examples: [
|
|
273
350
|
"dss job log JOB_ID",
|
|
274
351
|
"dss job log JOB_ID --activity main --max-log-lines 200",
|
|
@@ -321,7 +398,7 @@ export const jobCommands = withUsage("job", {
|
|
|
321
398
|
};
|
|
322
399
|
}
|
|
323
400
|
if (f["wait"] === true) {
|
|
324
|
-
const result = await c.jobs.buildAndWait(a[0], { ...options, projectKey: pk, });
|
|
401
|
+
const result = withWaitResume(await c.jobs.buildAndWait(a[0], { ...options, projectKey: pk, }), pk);
|
|
325
402
|
await warnIfBuiltDatasetHasNoColumns(c, a[0], options.targetType, result.success, pk);
|
|
326
403
|
return result;
|
|
327
404
|
}
|
|
@@ -361,7 +438,7 @@ export const jobCommands = withUsage("job", {
|
|
|
361
438
|
method: "POST",
|
|
362
439
|
};
|
|
363
440
|
}
|
|
364
|
-
const result = await c.jobs.buildAndWait(a[0], { ...options, projectKey: pk, });
|
|
441
|
+
const result = withWaitResume(await c.jobs.buildAndWait(a[0], { ...options, projectKey: pk, }), pk);
|
|
365
442
|
await warnIfBuiltDatasetHasNoColumns(c, a[0], options.targetType, result.success, pk);
|
|
366
443
|
return result;
|
|
367
444
|
},
|
|
@@ -374,17 +451,19 @@ export const jobCommands = withUsage("job", {
|
|
|
374
451
|
],
|
|
375
452
|
},
|
|
376
453
|
wait: {
|
|
377
|
-
handler: (c, a, f) => {
|
|
454
|
+
handler: async (c, a, f) => {
|
|
378
455
|
requireArgs(a, 1, commandUsage("job", "wait"));
|
|
379
|
-
|
|
456
|
+
const projectKey = f["project-key"];
|
|
457
|
+
const result = await c.jobs.wait(a[0], {
|
|
380
458
|
includeLogs: f["include-logs"] === true,
|
|
381
459
|
logFilter: jobLogFilterFromFlag(f["log-filter"]),
|
|
382
460
|
maxLogLines: maxLogLinesFromFlags(f),
|
|
383
461
|
pollIntervalMs: num(f["poll-interval"], "--poll-interval"),
|
|
384
462
|
timeoutMs: num(f["timeout"], "--timeout"),
|
|
385
463
|
summary: f["summary"] === true,
|
|
386
|
-
projectKey
|
|
464
|
+
projectKey,
|
|
387
465
|
});
|
|
466
|
+
return withWaitResume(result, projectKey);
|
|
388
467
|
},
|
|
389
468
|
description: "Wait for an existing job to complete.",
|
|
390
469
|
examples: [
|
|
@@ -404,7 +483,7 @@ export const jobCommands = withUsage("job", {
|
|
|
404
483
|
summary: f["summary"] !== false,
|
|
405
484
|
projectKey: f["project-key"],
|
|
406
485
|
};
|
|
407
|
-
const jobs = await Promise.all(a.map((jobId) => c.jobs.wait(jobId, options)));
|
|
486
|
+
const jobs = await Promise.all(a.map(async (jobId) => withWaitResume(await c.jobs.wait(jobId, options), options.projectKey)));
|
|
408
487
|
return aggregateJobWaitResults(jobs, String(f["until"] ?? "all-done"));
|
|
409
488
|
},
|
|
410
489
|
description: "Monitor one or more existing jobs and summarize progress counters from logs.",
|
|
@@ -422,7 +501,7 @@ export const jobCommands = withUsage("job", {
|
|
|
422
501
|
summary: true,
|
|
423
502
|
projectKey: f["project-key"],
|
|
424
503
|
};
|
|
425
|
-
const jobs = await Promise.all(a.map((jobId) => c.jobs.wait(jobId, options)));
|
|
504
|
+
const jobs = await Promise.all(a.map(async (jobId) => withWaitResume(await c.jobs.wait(jobId, options), options.projectKey)));
|
|
426
505
|
return aggregateJobWaitResults(jobs, String(f["until"] ?? "all-done"));
|
|
427
506
|
},
|
|
428
507
|
description: "Watch one or more existing jobs with progress extraction enabled.",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { readFile, } from "node:fs/promises";
|
|
2
2
|
import { validatedPluginGitUrl, validatePluginDestinationPath, validatePluginId, validatePluginName, validatePluginPath, } from "../../resources/plugins.js";
|
|
3
3
|
import { writeResponseToFile, } from "../../utils/response-file.js";
|
|
4
|
-
import { num, readStdinText, sha256Hex, } from "../coerce.js";
|
|
4
|
+
import { num, readStdinText, readTextFile, sha256Hex, } from "../coerce.js";
|
|
5
5
|
import { executionMode, } from "../flags.js";
|
|
6
6
|
import { planResult, } from "../output.js";
|
|
7
7
|
import { commandUsage, withUsage, } from "../syntax.js";
|
|
@@ -281,7 +281,7 @@ export const pluginCommands = withUsage("plugin", {
|
|
|
281
281
|
requireArgs(a, 1, SETTINGS_SET_USAGE);
|
|
282
282
|
const pluginId = validatePluginId(a[0]);
|
|
283
283
|
const mode = executionMode(f);
|
|
284
|
-
const settings = readSettingsInput(f);
|
|
284
|
+
const settings = await readSettingsInput(f);
|
|
285
285
|
if (mode.dryRun) {
|
|
286
286
|
return pluginPlan("settings-set", {
|
|
287
287
|
method: "POST",
|
|
@@ -330,7 +330,7 @@ export const pluginCommands = withUsage("plugin", {
|
|
|
330
330
|
},
|
|
331
331
|
description: "Create the code env of a plugin. Returns the DSS future (jobId); --wait polls it to completion.",
|
|
332
332
|
examples: [
|
|
333
|
-
"dss plugin code-env-create my-plugin --python-interpreter
|
|
333
|
+
"dss plugin code-env-create my-plugin --python-interpreter PYTHON311",
|
|
334
334
|
"dss plugin code-env-create my-plugin --conda --wait --timeout 300000",
|
|
335
335
|
],
|
|
336
336
|
},
|
|
@@ -629,7 +629,7 @@ export const pluginCommands = withUsage("plugin", {
|
|
|
629
629
|
});
|
|
630
630
|
}
|
|
631
631
|
const content = typeof f["file"] === "string"
|
|
632
|
-
?
|
|
632
|
+
? await readFile(f["file"])
|
|
633
633
|
: typeof f["content"] === "string"
|
|
634
634
|
? f["content"]
|
|
635
635
|
: await readStdinText();
|
|
@@ -738,10 +738,10 @@ function normalizeCreationMode(value) {
|
|
|
738
738
|
}
|
|
739
739
|
throw new UsageError("--creation-mode must be one of EMPTY, GIT_CLONE, GIT_EXPORT.", "invalid_enum");
|
|
740
740
|
}
|
|
741
|
-
function readSettingsInput(flags) {
|
|
741
|
+
async function readSettingsInput(flags) {
|
|
742
742
|
let text;
|
|
743
743
|
if (typeof flags["file"] === "string") {
|
|
744
|
-
text =
|
|
744
|
+
text = await readTextFile(flags["file"]);
|
|
745
745
|
}
|
|
746
746
|
else if (typeof flags["content"] === "string") {
|
|
747
747
|
text = flags["content"];
|
|
@@ -159,7 +159,7 @@ export const projectLibraryCommands = withUsage("project-library", {
|
|
|
159
159
|
const content = typeof f["content"] === "string"
|
|
160
160
|
? f["content"]
|
|
161
161
|
: typeof f["file"] === "string"
|
|
162
|
-
? fs.
|
|
162
|
+
? await fs.promises.readFile(f["file"])
|
|
163
163
|
: await readStdinText();
|
|
164
164
|
const result = await c.projectLibrary.addOrUpdateFile(a[0], content, pk, expectSha256 !== undefined ? { expectSha256, } : undefined);
|
|
165
165
|
return {
|
|
@@ -184,7 +184,7 @@ export const projectLibraryCommands = withUsage("project-library", {
|
|
|
184
184
|
const local = typeof f["content"] === "string"
|
|
185
185
|
? f["content"]
|
|
186
186
|
: typeof f["file"] === "string"
|
|
187
|
-
? fs.
|
|
187
|
+
? await fs.promises.readFile(f["file"])
|
|
188
188
|
: await readStdinText();
|
|
189
189
|
return c.projectLibrary.diffFile(a[0], local, pk, maxLines !== undefined ? { maxLines, } : {});
|
|
190
190
|
},
|