dataiku-sdk 3.9.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +4 -0
- package/dist/packages/types/src/index.js +4 -0
- package/dist/src/cli/coerce.d.ts +6 -0
- package/dist/src/cli/coerce.js +9 -0
- package/dist/src/cli/commands/app.js +2 -2
- package/dist/src/cli/commands/code-env.js +5 -6
- package/dist/src/cli/commands/code.js +16 -2
- package/dist/src/cli/commands/dashboard.js +1 -2
- package/dist/src/cli/commands/flow-zone.js +1 -1
- package/dist/src/cli/commands/job.js +67 -16
- package/dist/src/cli/commands/plugin.js +7 -7
- package/dist/src/cli/commands/project-library.js +2 -2
- package/dist/src/cli/commands/project.js +21 -4
- package/dist/src/cli/commands/recipe.js +11 -19
- package/dist/src/cli/commands/scenario.js +2 -4
- package/dist/src/cli/contract.js +27 -5
- package/dist/src/cli/env.js +2 -3
- package/dist/src/cli/helpers/app-permissions.d.ts +1 -1
- package/dist/src/cli/helpers/app-permissions.js +2 -2
- package/dist/src/cli/helpers/cleanup.d.ts +2 -0
- package/dist/src/cli/helpers/cleanup.js +122 -8
- package/dist/src/cli/helpers/flow-zone.d.ts +1 -1
- package/dist/src/cli/helpers/flow-zone.js +3 -4
- package/dist/src/cli/list-output.d.ts +5 -0
- package/dist/src/cli/list-output.js +11 -2
- package/dist/src/cli/list-shapes.d.ts +6 -0
- package/dist/src/cli/list-shapes.js +7 -1
- package/dist/src/cli/output-schemas.d.ts +1 -1
- package/dist/src/cli/output-schemas.js +1 -1
- package/dist/src/cli/output.js +7 -0
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +4 -5
- package/dist/src/cli.js +66 -17
- package/dist/src/client.d.ts +14 -2
- package/dist/src/client.js +18 -10
- package/dist/src/generated/action-output-schemas.json +18 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/resources/datasets.d.ts +11 -1
- package/dist/src/resources/datasets.js +91 -30
- package/dist/src/resources/jobs.d.ts +35 -5
- package/dist/src/resources/jobs.js +68 -41
- package/dist/src/resources/projects.js +2 -2
- package/dist/src/resources/recipes.d.ts +12 -1
- package/dist/src/resources/recipes.js +39 -12
- package/dist/src/resources/scenarios.d.ts +8 -1
- package/dist/src/resources/scenarios.js +51 -12
- package/dist/src/resources/wiki.js +3 -7
- package/dist/src/skill.js +1 -2
- package/dist/src/utils/project-archive.d.ts +25 -0
- package/dist/src/utils/project-archive.js +216 -5
- package/dist/src/utils/response-file.js +15 -8
- package/package.json +11 -12
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
package/dist/src/cli/contract.js
CHANGED
|
@@ -248,6 +248,13 @@ const ALLOWED_CLEANUP_ACTIONS = new Set([
|
|
|
248
248
|
"project-library delete",
|
|
249
249
|
"streaming-endpoint delete",
|
|
250
250
|
"workspace delete",
|
|
251
|
+
"saved-model delete",
|
|
252
|
+
"project-folder delete",
|
|
253
|
+
"data-collection delete",
|
|
254
|
+
"plugin delete",
|
|
255
|
+
"connection delete",
|
|
256
|
+
"user delete",
|
|
257
|
+
"group delete",
|
|
251
258
|
]);
|
|
252
259
|
export function isAllowedCleanupAction(resource, action) {
|
|
253
260
|
return ALLOWED_CLEANUP_ACTIONS.has(`${resource} ${action}`);
|
|
@@ -391,7 +398,7 @@ function structuredExamples(examples, examplePayload) {
|
|
|
391
398
|
* (`dss commands run`, `dss agent contract`) never loads the TypeBox graph.
|
|
392
399
|
* The content matches `typeBoxCommandOutputSchemas()` exactly; parity and
|
|
393
400
|
* freshness are enforced by tests/cli/discovery-output-schemas.test.ts and
|
|
394
|
-
* `bun run
|
|
401
|
+
* `bun run verify`.
|
|
395
402
|
*/
|
|
396
403
|
const actionOutputSchemas = actionOutputSchemasMetadata
|
|
397
404
|
.schemas;
|
|
@@ -764,6 +771,20 @@ const STRING_OUTPUT_ACTIONS = new Set([
|
|
|
764
771
|
"log",
|
|
765
772
|
"log-url",
|
|
766
773
|
]);
|
|
774
|
+
/**
|
|
775
|
+
* `resource.action` shapes that differ from the bare-action-name sets above:
|
|
776
|
+
* `contents`/`usages`/`project-timeline` are array-named elsewhere, and
|
|
777
|
+
* `download`/`diff` are string-named elsewhere, but these handlers return the
|
|
778
|
+
* opposite shape. Pinned by tests/cli/output-shapes.test.ts.
|
|
779
|
+
*/
|
|
780
|
+
const SCOPED_OUTPUT_SHAPES = {
|
|
781
|
+
"folder.contents": "array",
|
|
782
|
+
"plugin.usages": "object",
|
|
783
|
+
"plugin.download": "object",
|
|
784
|
+
"dataset.download": "object",
|
|
785
|
+
"project-library.get": "string",
|
|
786
|
+
"project-library.diff": "object",
|
|
787
|
+
};
|
|
767
788
|
function inferOutputShape(resource, action) {
|
|
768
789
|
if (resource === "project-git") {
|
|
769
790
|
if (action === "branches" || action === "tags" || action === "list-libraries")
|
|
@@ -777,14 +798,15 @@ function inferOutputShape(resource, action) {
|
|
|
777
798
|
|| resource === "version") {
|
|
778
799
|
return "object";
|
|
779
800
|
}
|
|
801
|
+
// Resource-scoped rules come first: the bare-name sets below classify by
|
|
802
|
+
// action alone and would otherwise misfile same-named actions elsewhere.
|
|
803
|
+
const scoped = SCOPED_OUTPUT_SHAPES[`${resource}.${action}`];
|
|
804
|
+
if (scoped !== undefined)
|
|
805
|
+
return scoped;
|
|
780
806
|
if (/^list(-|$)/.test(action))
|
|
781
807
|
return "array";
|
|
782
808
|
if (ARRAY_OUTPUT_ACTIONS.has(action))
|
|
783
809
|
return "array";
|
|
784
|
-
if (resource === "dataset" && action === "download")
|
|
785
|
-
return "object";
|
|
786
|
-
if (resource === "project-library" && action === "get")
|
|
787
|
-
return "string";
|
|
788
810
|
if (STRING_OUTPUT_ACTIONS.has(action))
|
|
789
811
|
return "string";
|
|
790
812
|
return "object";
|
package/dist/src/cli/env.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
|
-
import {
|
|
3
|
-
import { fileURLToPath, } from "node:url";
|
|
2
|
+
import { resolve, } from "node:path";
|
|
4
3
|
const projectEnvVars = new Set();
|
|
5
4
|
const installEnvVars = new Set();
|
|
6
5
|
/**
|
|
@@ -42,7 +41,7 @@ export function loadEnvFile() {
|
|
|
42
41
|
// CLI. First writer wins below, so cwd must be listed first.
|
|
43
42
|
const dirs = [
|
|
44
43
|
process.cwd(),
|
|
45
|
-
resolve(
|
|
44
|
+
resolve(import.meta.dir, "..", ".."),
|
|
46
45
|
];
|
|
47
46
|
for (const [index, dir,] of dirs.entries()) {
|
|
48
47
|
try {
|
|
@@ -43,7 +43,7 @@ export declare function buildAppPermissionsSnapshot(projectKey: string, dssUrl:
|
|
|
43
43
|
export declare function writeAppPermissionsSnapshot(filePath: string, snapshot: AppPermissionsSnapshot): void;
|
|
44
44
|
/** Validate envelope shape, version, and hash integrity. Never echoes permission payloads. */
|
|
45
45
|
export declare function parseAppPermissionsSnapshot(text: string, source: string): AppPermissionsSnapshot;
|
|
46
|
-
export declare function readAppPermissionsSnapshot(filePath: string): AppPermissionsSnapshot
|
|
46
|
+
export declare function readAppPermissionsSnapshot(filePath: string): Promise<AppPermissionsSnapshot>;
|
|
47
47
|
/**
|
|
48
48
|
* Refuse to use a snapshot outside the exact project *and* DSS server it was captured
|
|
49
49
|
* from. Access-control rules are server-scoped: the same project key, users, and groups
|
|
@@ -153,10 +153,10 @@ export function parseAppPermissionsSnapshot(text, source) {
|
|
|
153
153
|
permissions,
|
|
154
154
|
};
|
|
155
155
|
}
|
|
156
|
-
export function readAppPermissionsSnapshot(filePath) {
|
|
156
|
+
export async function readAppPermissionsSnapshot(filePath) {
|
|
157
157
|
let text;
|
|
158
158
|
try {
|
|
159
|
-
text = fs.
|
|
159
|
+
text = await fs.promises.readFile(filePath, "utf-8");
|
|
160
160
|
}
|
|
161
161
|
catch (error) {
|
|
162
162
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
import type { CleanupLedgerEntry } from "../../utils/cleanup-ledger.js";
|
|
2
2
|
export declare function projectArg(projectKey: string | undefined): string[];
|
|
3
|
+
/** Input field that identifies the created object, when the result does not. */
|
|
4
|
+
export declare function cleanupInputIdentifierField(resource: string, action: string): string | undefined;
|
|
3
5
|
export declare function cleanupLedgerEntry(resource: string, action: string, args: string[], flags: Record<string, string | boolean>, result: unknown, projectKey: string | undefined): CleanupLedgerEntry | undefined;
|
|
@@ -35,6 +35,30 @@ function createInputFromFlags(flags) {
|
|
|
35
35
|
return undefined;
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Creates whose DSS response carries no identifier: the cleanup ledger pins the
|
|
40
|
+
* created object from this field of the `--data`/`--data-file` input instead.
|
|
41
|
+
* `--stdin` cannot be replayed, so the CLI rejects it with `--record-cleanup`
|
|
42
|
+
* before mutating.
|
|
43
|
+
*/
|
|
44
|
+
const CLEANUP_INPUT_IDENTIFIER_FIELDS = {
|
|
45
|
+
"workspace.create": "workspaceKey",
|
|
46
|
+
"connection.create": "name",
|
|
47
|
+
"user.create": "login",
|
|
48
|
+
"group.create": "name",
|
|
49
|
+
};
|
|
50
|
+
/** Input field that identifies the created object, when the result does not. */
|
|
51
|
+
export function cleanupInputIdentifierField(resource, action) {
|
|
52
|
+
return CLEANUP_INPUT_IDENTIFIER_FIELDS[`${resource}.${action}`];
|
|
53
|
+
}
|
|
54
|
+
function inputIdentifier(resource, action, flags) {
|
|
55
|
+
const field = cleanupInputIdentifierField(resource, action);
|
|
56
|
+
if (!field)
|
|
57
|
+
return undefined;
|
|
58
|
+
const input = resultRecord(createInputFromFlags(flags));
|
|
59
|
+
const identifier = stringField(input, [field,]);
|
|
60
|
+
return identifier ? { identifier, input, } : undefined;
|
|
61
|
+
}
|
|
38
62
|
export function cleanupLedgerEntry(resource, action, args, flags, result, projectKey) {
|
|
39
63
|
const record = resultRecord(result);
|
|
40
64
|
// A notebook save is ledger-eligible only when it created the notebook:
|
|
@@ -77,6 +101,22 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
77
101
|
cleanup: { argv: ["dataset", "delete", name, "--if-exists", ...withProject,], },
|
|
78
102
|
};
|
|
79
103
|
}
|
|
104
|
+
case "dataset.create-managed": {
|
|
105
|
+
// The handler returns `created` as the dataset name and the project
|
|
106
|
+
// DSS created it in, which wins over the requested project key.
|
|
107
|
+
const name = stringField(record, ["created", "name",]) ?? flags["name"];
|
|
108
|
+
if (!name)
|
|
109
|
+
return undefined;
|
|
110
|
+
const landedProject = stringField(record, ["projectKey",]) ?? project;
|
|
111
|
+
return {
|
|
112
|
+
...base,
|
|
113
|
+
...(landedProject ? { projectKey: landedProject, } : {}),
|
|
114
|
+
name,
|
|
115
|
+
cleanup: {
|
|
116
|
+
argv: ["dataset", "delete", name, "--if-exists", ...projectArg(landedProject),],
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
80
120
|
case "recipe.create": {
|
|
81
121
|
const name = stringField(record, ["created", "recipeName", "name",])
|
|
82
122
|
?? flags["name"];
|
|
@@ -199,6 +239,78 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
199
239
|
cleanup: { argv: ["ml-task", "delete", analysisId, taskId, ...withProject,], },
|
|
200
240
|
};
|
|
201
241
|
}
|
|
242
|
+
case "ml-task.create-for-dataset": {
|
|
243
|
+
// This create opens a new visual analysis around the task, so cleanup
|
|
244
|
+
// removes the whole analysis rather than leaving an empty one behind.
|
|
245
|
+
const analysisId = stringField(record, ["analysisId",]);
|
|
246
|
+
if (!analysisId)
|
|
247
|
+
return undefined;
|
|
248
|
+
return {
|
|
249
|
+
...base,
|
|
250
|
+
id: analysisId,
|
|
251
|
+
cleanup: { argv: ["analysis", "delete", analysisId, "--if-exists", ...withProject,], },
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
case "saved-model.create-external": {
|
|
255
|
+
const id = stringField(record, ["id",]);
|
|
256
|
+
if (!id)
|
|
257
|
+
return undefined;
|
|
258
|
+
const landedProject = stringField(record, ["projectKey",]) ?? project;
|
|
259
|
+
return {
|
|
260
|
+
...base,
|
|
261
|
+
...(landedProject ? { projectKey: landedProject, } : {}),
|
|
262
|
+
id,
|
|
263
|
+
name: args[0],
|
|
264
|
+
cleanup: {
|
|
265
|
+
argv: ["saved-model", "delete", id, "--if-exists", ...projectArg(landedProject),],
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
case "project-folder.create-child": {
|
|
270
|
+
// `created` falls back to the requested name when DSS omits the id,
|
|
271
|
+
// and a name cannot address a folder delete: only `id` pins it.
|
|
272
|
+
const id = stringField(record, ["id",]);
|
|
273
|
+
if (!id)
|
|
274
|
+
return undefined;
|
|
275
|
+
return {
|
|
276
|
+
...base,
|
|
277
|
+
id,
|
|
278
|
+
name: flags["name"],
|
|
279
|
+
cleanup: { argv: ["project-folder", "delete", id, "--if-exists",], },
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
case "data-collection.create": {
|
|
283
|
+
const id = stringField(record, ["id",]);
|
|
284
|
+
if (!id)
|
|
285
|
+
return undefined;
|
|
286
|
+
return {
|
|
287
|
+
...base,
|
|
288
|
+
id,
|
|
289
|
+
cleanup: { argv: ["data-collection", "delete", id,], },
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
case "plugin.create-dev": {
|
|
293
|
+
const id = stringField(record, ["created",]) ?? args[0];
|
|
294
|
+
if (!id)
|
|
295
|
+
return undefined;
|
|
296
|
+
return {
|
|
297
|
+
...base,
|
|
298
|
+
id,
|
|
299
|
+
cleanup: { argv: ["plugin", "delete", id,], },
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
case "connection.create":
|
|
303
|
+
case "user.create":
|
|
304
|
+
case "group.create": {
|
|
305
|
+
const pinned = inputIdentifier(resource, action, flags);
|
|
306
|
+
if (!pinned)
|
|
307
|
+
return undefined;
|
|
308
|
+
return {
|
|
309
|
+
...base,
|
|
310
|
+
name: pinned.identifier,
|
|
311
|
+
cleanup: { argv: [resource, "delete", pinned.identifier, "--if-exists",], },
|
|
312
|
+
};
|
|
313
|
+
}
|
|
202
314
|
case "data-quality.create-rule": {
|
|
203
315
|
const ruleId = stringField(record, ["id", "created",]);
|
|
204
316
|
if (!ruleId)
|
|
@@ -222,8 +334,11 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
222
334
|
cleanup: { argv: ["code-env", "delete", lang, name, "--if-exists",], },
|
|
223
335
|
};
|
|
224
336
|
}
|
|
225
|
-
case "streaming-endpoint.create":
|
|
226
|
-
|
|
337
|
+
case "streaming-endpoint.create":
|
|
338
|
+
case "streaming-endpoint.create-managed": {
|
|
339
|
+
const id = stringField(record, ["created",]) ?? args[0];
|
|
340
|
+
if (!id)
|
|
341
|
+
return undefined;
|
|
227
342
|
return {
|
|
228
343
|
...base,
|
|
229
344
|
id,
|
|
@@ -240,15 +355,14 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
240
355
|
};
|
|
241
356
|
}
|
|
242
357
|
case "workspace.create": {
|
|
243
|
-
const
|
|
244
|
-
|
|
245
|
-
if (!workspaceKey)
|
|
358
|
+
const pinned = inputIdentifier(resource, action, flags);
|
|
359
|
+
if (!pinned)
|
|
246
360
|
return undefined;
|
|
247
361
|
return {
|
|
248
362
|
...base,
|
|
249
|
-
id:
|
|
250
|
-
name: stringField(input, ["displayName",]),
|
|
251
|
-
cleanup: { argv: ["workspace", "delete",
|
|
363
|
+
id: pinned.identifier,
|
|
364
|
+
name: stringField(pinned.input, ["displayName",]),
|
|
365
|
+
cleanup: { argv: ["workspace", "delete", pinned.identifier,], },
|
|
252
366
|
};
|
|
253
367
|
}
|
|
254
368
|
case "project-library.create-file":
|
|
@@ -56,7 +56,7 @@ export declare function dedupeFlowZonePlanItems(items: FlowZoneItemInput[]): Flo
|
|
|
56
56
|
export declare function flowZonePlanItemKeys(plan: FlowZoneOrganizePlan): Set<string>;
|
|
57
57
|
export declare function validateUniqueFlowZoneAssignments(plan: FlowZoneOrganizePlan): void;
|
|
58
58
|
export declare function parseFlowZoneOrganizePlan(input: Record<string, unknown>): FlowZoneOrganizePlan;
|
|
59
|
-
export declare function readFlowZoneOrganizePlan(flags: Record<string, string | boolean>, usage: string): FlowZoneOrganizePlan
|
|
59
|
+
export declare function readFlowZoneOrganizePlan(flags: Record<string, string | boolean>, usage: string): Promise<FlowZoneOrganizePlan>;
|
|
60
60
|
export declare function findFlowZoneForPlan(zones: FlowZone[], plan: FlowZoneOrganizeZonePlan): FlowZone | undefined;
|
|
61
61
|
export declare function ensureFlowZonePlanTarget(plan: FlowZoneOrganizeZonePlan, existing: FlowZone | undefined): void;
|
|
62
62
|
export declare function flowZoneExplicitItems(zone: FlowZone): FlowZoneItemInput[];
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { readFileSync, } from "node:fs";
|
|
2
1
|
import { ClientValidationError, } from "../../errors.js";
|
|
3
2
|
import { asRecord, } from "../../utils/records.js";
|
|
4
3
|
import { compareStrings, } from "../../utils/stable-hash.js";
|
|
5
|
-
import { finiteNumberField, jsonInput, optionalStringField, parseJsonObject, requiredStringArray, splitCsvFlag, } from "../coerce.js";
|
|
4
|
+
import { finiteNumberField, jsonInput, optionalStringField, parseJsonObject, readTextFile, requiredStringArray, splitCsvFlag, } from "../coerce.js";
|
|
6
5
|
import { commandUsage, } from "../syntax.js";
|
|
7
6
|
import { UsageError, } from "../usage.js";
|
|
8
7
|
export function flowZoneId(value) {
|
|
@@ -257,9 +256,9 @@ export function parseFlowZoneOrganizePlan(input) {
|
|
|
257
256
|
validateUniqueFlowZoneAssignments(plan);
|
|
258
257
|
return plan;
|
|
259
258
|
}
|
|
260
|
-
export function readFlowZoneOrganizePlan(flags, usage) {
|
|
259
|
+
export async function readFlowZoneOrganizePlan(flags, usage) {
|
|
261
260
|
const data = typeof flags["file"] === "string"
|
|
262
|
-
? parseJsonObject(
|
|
261
|
+
? parseJsonObject(await readTextFile(flags["file"]), flags["file"])
|
|
263
262
|
: jsonInput(flags);
|
|
264
263
|
if (!data) {
|
|
265
264
|
throw new UsageError(`--data, --data-file, --file, or --stdin is required. Usage: ${usage}`, "missing_required_flag");
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import { type CompactListField } from "./list-shapes.js";
|
|
2
|
+
type Item = Record<string, unknown>;
|
|
3
|
+
/** One compact field of a DSS list item, as `RESOURCE.list` prints it. */
|
|
4
|
+
export declare function compactValue(item: Item, spec: CompactListField): unknown;
|
|
1
5
|
/**
|
|
2
6
|
* Shape a `*.list` result for agents: `--contains` filters on identifier
|
|
3
7
|
* fields, `--limit` caps the count (a `list_truncated` warning reports the
|
|
@@ -5,3 +9,4 @@
|
|
|
5
9
|
* `--full` or `--fields` asks for the DSS objects.
|
|
6
10
|
*/
|
|
7
11
|
export declare function shapeListResult(resource: string, result: unknown, flags: Record<string, string | boolean>): unknown;
|
|
12
|
+
export {};
|
|
@@ -4,10 +4,19 @@ import { enqueueCliWarning, } from "./output.js";
|
|
|
4
4
|
function record(value) {
|
|
5
5
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
6
6
|
}
|
|
7
|
-
function
|
|
7
|
+
function valueAt(item, path) {
|
|
8
8
|
let value = item;
|
|
9
|
-
for (const key of
|
|
9
|
+
for (const key of path.split("."))
|
|
10
10
|
value = record(value)[key];
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
/** One compact field of a DSS list item, as `RESOURCE.list` prints it. */
|
|
14
|
+
export function compactValue(item, spec) {
|
|
15
|
+
const value = valueAt(item, spec.path);
|
|
16
|
+
if (value === undefined && spec.fallback
|
|
17
|
+
&& valueAt(item, spec.fallback.when) === spec.fallback.equals) {
|
|
18
|
+
return spec.fallback.value;
|
|
19
|
+
}
|
|
11
20
|
if (spec.roleRefs !== true)
|
|
12
21
|
return value;
|
|
13
22
|
// Dataset refs across every role of recipe inputs/outputs (`{main:{items:[{ref}]}}`).
|
|
@@ -9,6 +9,12 @@ export interface CompactListField {
|
|
|
9
9
|
path: string;
|
|
10
10
|
/** Flatten recipe roles (`{main:{items:[{ref}]}}`) to the dataset refs. */
|
|
11
11
|
roleRefs?: true;
|
|
12
|
+
/** Value when `path` is absent and the item's `when` path equals `equals`. */
|
|
13
|
+
fallback?: {
|
|
14
|
+
when: string;
|
|
15
|
+
equals: unknown;
|
|
16
|
+
value: unknown;
|
|
17
|
+
};
|
|
12
18
|
schema: Record<string, unknown>;
|
|
13
19
|
}
|
|
14
20
|
/**
|
|
@@ -51,7 +51,12 @@ export const COMPACT_LIST_FIELDS = {
|
|
|
51
51
|
id: field("def.id", STRING),
|
|
52
52
|
name: field("def.name", STRING),
|
|
53
53
|
type: field("def.type", STRING),
|
|
54
|
-
|
|
54
|
+
// DSS lists in-progress jobs with `stableState: false` and no `state`
|
|
55
|
+
// (COMPUTING_DEPS or RUNNING); `job get` reports the exact phase.
|
|
56
|
+
state: {
|
|
57
|
+
...field("state", STRING),
|
|
58
|
+
fallback: { when: "stableState", equals: false, value: "RUNNING", },
|
|
59
|
+
},
|
|
55
60
|
startTime: field("startTime", NUMBER),
|
|
56
61
|
endTime: field("endTime", NUMBER),
|
|
57
62
|
},
|
|
@@ -83,6 +88,7 @@ export const COMPACT_LIST_FIELDS = {
|
|
|
83
88
|
id: field("id", STRING),
|
|
84
89
|
version: field("version", STRING),
|
|
85
90
|
label: field("meta.label", STRING),
|
|
91
|
+
isDev: field("isDev", BOOLEAN),
|
|
86
92
|
},
|
|
87
93
|
macro: {
|
|
88
94
|
runnableType: field("runnableType", STRING),
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* the single source of truth for `src/generated/action-output-schemas.json`,
|
|
4
4
|
* which discovery reads as plain data so no TypeBox schema graph loads on the
|
|
5
5
|
* discovery path. Regenerate with
|
|
6
|
-
* `bun scripts/generate-action-output-schemas.mjs`; `bun run
|
|
6
|
+
* `bun scripts/generate-action-output-schemas.mjs`; `bun run verify` fails when
|
|
7
7
|
* the generated file is stale. SDK validation keeps using the TypeBox schema
|
|
8
8
|
* objects from `packages/types` directly and never depends on the generated file.
|
|
9
9
|
*/
|
|
@@ -72,7 +72,7 @@ let commandOutputSchemas;
|
|
|
72
72
|
* the single source of truth for `src/generated/action-output-schemas.json`,
|
|
73
73
|
* which discovery reads as plain data so no TypeBox schema graph loads on the
|
|
74
74
|
* discovery path. Regenerate with
|
|
75
|
-
* `bun scripts/generate-action-output-schemas.mjs`; `bun run
|
|
75
|
+
* `bun scripts/generate-action-output-schemas.mjs`; `bun run verify` fails when
|
|
76
76
|
* the generated file is stale. SDK validation keeps using the TypeBox schema
|
|
77
77
|
* objects from `packages/types` directly and never depends on the generated file.
|
|
78
78
|
*/
|
package/dist/src/cli/output.js
CHANGED
|
@@ -193,6 +193,13 @@ export function commandFailureMessage(result) {
|
|
|
193
193
|
if (typeof message === "string" && message.length > 0)
|
|
194
194
|
return message;
|
|
195
195
|
}
|
|
196
|
+
if (record.timedOut === true && typeof record.jobId === "string"
|
|
197
|
+
&& typeof record.progress === "object" && typeof record.state === "string") {
|
|
198
|
+
// A job wait result: the timeout only stopped polling, the DSS job is
|
|
199
|
+
// still running (and was not cancelled).
|
|
200
|
+
const resume = typeof record.resume === "string" ? ` Resume with: ${record.resume}` : "";
|
|
201
|
+
return `Timed out waiting for job ${record.jobId} after ${record.elapsedMs}ms (last known state: ${record.state}); the job is still running in DSS and was not cancelled.${resume}`;
|
|
202
|
+
}
|
|
196
203
|
const state = typeof record.state === "string"
|
|
197
204
|
? record.state
|
|
198
205
|
: typeof record.outcome === "string"
|
|
@@ -49,7 +49,7 @@ export declare function buildVersionPayload(input: {
|
|
|
49
49
|
export declare const PACKAGE_ROOT: string | undefined;
|
|
50
50
|
export declare const CLI_VERSION: string;
|
|
51
51
|
export declare const CLI_GIT_REVISION: string | undefined;
|
|
52
|
-
export declare const AGENT_CONTRACT_VERSION =
|
|
52
|
+
export declare const AGENT_CONTRACT_VERSION = 5;
|
|
53
53
|
export declare const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
|
|
54
|
-
export declare const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-
|
|
54
|
+
export declare const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v5.json";
|
|
55
55
|
export declare function cliVersionResult(): CliVersionPayload;
|
package/dist/src/cli/version.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { readdirSync, readFileSync, statSync, } from "node:fs";
|
|
2
2
|
import { dirname, resolve, } from "node:path";
|
|
3
|
-
import { fileURLToPath, } from "node:url";
|
|
4
3
|
export function findPackageRoot() {
|
|
5
|
-
let dir =
|
|
4
|
+
let dir = import.meta.dir;
|
|
6
5
|
for (let i = 0; i < 5; i++) {
|
|
7
6
|
try {
|
|
8
7
|
readFileSync(resolve(dir, "package.json"), "utf-8");
|
|
@@ -133,7 +132,7 @@ export function gitRevision(packageRoot) {
|
|
|
133
132
|
const full = gitFullRevision(packageRoot);
|
|
134
133
|
return full ? full.slice(0, 7) : undefined;
|
|
135
134
|
}
|
|
136
|
-
export function detectLoadSource(modulePath =
|
|
135
|
+
export function detectLoadSource(modulePath = import.meta.path) {
|
|
137
136
|
return modulePath.replaceAll("\\", "/").includes("/dist/") ? "dist" : "source";
|
|
138
137
|
}
|
|
139
138
|
export function buildMetadataRevision(packageRoot) {
|
|
@@ -204,9 +203,9 @@ export function buildVersionPayload(input) {
|
|
|
204
203
|
export const PACKAGE_ROOT = findPackageRoot();
|
|
205
204
|
export const CLI_VERSION = packageVersion(PACKAGE_ROOT);
|
|
206
205
|
export const CLI_GIT_REVISION = gitRevision(PACKAGE_ROOT);
|
|
207
|
-
export const AGENT_CONTRACT_VERSION =
|
|
206
|
+
export const AGENT_CONTRACT_VERSION = 5;
|
|
208
207
|
export const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
|
|
209
|
-
export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-
|
|
208
|
+
export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v5.json";
|
|
210
209
|
export function cliVersionResult() {
|
|
211
210
|
// Only code actually loaded from dist/ may report dist provenance; an
|
|
212
211
|
// inherited DSS_LOAD_SOURCE on a source run is ignored.
|
package/dist/src/cli.js
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import * as fs from "node:fs/promises";
|
|
3
2
|
import { homedir, } from "node:os";
|
|
4
3
|
import { resolve, } from "node:path";
|
|
5
|
-
import { num, seenJsonInputCredentials, unknownJsonInput, } from "./cli/coerce.js";
|
|
4
|
+
import { num, resultRecord, seenJsonInputCredentials, stringField, unknownJsonInput, } from "./cli/coerce.js";
|
|
6
5
|
import { commands, } from "./cli/commands/index.js";
|
|
7
6
|
import { ambientProjectKey, loadEnvFile, } from "./cli/env.js";
|
|
8
7
|
import { executionMode, FLAG_ALIASES, isFlagValueToken, parseArgs, rawPositionals, } from "./cli/flags.js";
|
|
9
|
-
import { cleanupLedgerEntry, } from "./cli/helpers/cleanup.js";
|
|
8
|
+
import { cleanupInputIdentifierField, cleanupLedgerEntry, } from "./cli/helpers/cleanup.js";
|
|
10
9
|
import { commandFailureExitCode, CommandResultFailure, enqueueCliWarning, flushCliWarnings, isFailedWaitResult, planResult, projectResultFields, setOutputFieldProjection, writeCommandResult, } from "./cli/output.js";
|
|
11
10
|
import { shapeCommandResult, } from "./cli/result-shapes.js";
|
|
12
11
|
import { currentCommandContext, resolveCredentials, resolveLoginCredentials, } from "./cli/runtime.js";
|
|
@@ -786,7 +785,7 @@ async function commandsRunResult(flags) {
|
|
|
786
785
|
: projectResultFields(registry, selectors, () => Object.keys(contract.commandActionSummary()).sort());
|
|
787
786
|
if (typeof outputPath === "string") {
|
|
788
787
|
const exported = selectedRegistry;
|
|
789
|
-
await
|
|
788
|
+
await Bun.write(outputPath, `${JSON.stringify(exported)}\n`);
|
|
790
789
|
return { path: outputPath, };
|
|
791
790
|
}
|
|
792
791
|
return selectedRegistry;
|
|
@@ -802,8 +801,9 @@ function assertCleanupLedgerSupported(resource, action, flags) {
|
|
|
802
801
|
if (!contract.supportsCleanupLedger(resource, action)) {
|
|
803
802
|
throw new UsageError(`--record-cleanup is not supported for ${resource} ${action}.`);
|
|
804
803
|
}
|
|
805
|
-
|
|
806
|
-
|
|
804
|
+
const pinnedField = cleanupInputIdentifierField(resource, action);
|
|
805
|
+
if (pinnedField !== undefined && flags["stdin"] === true) {
|
|
806
|
+
throw new UsageError(`${resource} ${action} cannot combine --record-cleanup with --stdin because the cleanup identifier cannot be replayed from a consumed stream.`, "conflicting_input_sources", `Use --data or --data-file so the ${pinnedField} remains available for the cleanup ledger.`);
|
|
807
807
|
}
|
|
808
808
|
}
|
|
809
809
|
/**
|
|
@@ -856,11 +856,36 @@ async function preflightCleanupLedgerForFlags(client, flags) {
|
|
|
856
856
|
}
|
|
857
857
|
}
|
|
858
858
|
/**
|
|
859
|
-
*
|
|
860
|
-
*
|
|
861
|
-
*
|
|
862
|
-
*
|
|
863
|
-
|
|
859
|
+
* What the command result already says about the created object, so a failed
|
|
860
|
+
* ledger write never hides it: the `created`/`saved` receipt, the object `id`
|
|
861
|
+
* and `name`, and the project key DSS landed it in. Fields stay distinct (a
|
|
862
|
+
* name is not an id) and absent ones are omitted rather than guessed.
|
|
863
|
+
*/
|
|
864
|
+
function createdObjectFromResult(result, fallbackProjectKey) {
|
|
865
|
+
const record = resultRecord(result);
|
|
866
|
+
const created = stringField(record, ["created", "saved", "usedProjectKey",]);
|
|
867
|
+
const id = stringField(record, ["id",]);
|
|
868
|
+
const name = stringField(record, ["name",]);
|
|
869
|
+
const projectKey = stringField(record, ["projectKey", "usedProjectKey", "targetProjectKey",])
|
|
870
|
+
?? fallbackProjectKey;
|
|
871
|
+
return {
|
|
872
|
+
...(created ? { created, } : {}),
|
|
873
|
+
...(id ? { id, } : {}),
|
|
874
|
+
...(name ? { name, } : {}),
|
|
875
|
+
...(projectKey ? { projectKey, } : {}),
|
|
876
|
+
};
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Append the cleanup entry for a completed handler result. Callers must call
|
|
880
|
+
* this before any failure exit code is raised: an indeterminate post-POST
|
|
881
|
+
* outcome stays addressable only when the ledger records it first. Eligibility
|
|
882
|
+
* (dry runs, skipped work, an explicit `cleanupEligible:false`) is decided by
|
|
883
|
+
* `cleanupLedgerEntry`, never here.
|
|
884
|
+
*
|
|
885
|
+
* The mutation has already happened when this runs, so every failure here is
|
|
886
|
+
* an `ambiguous_outcome` that says so (`mutationSucceeded: true`,
|
|
887
|
+
* `cleanupRecorded: false`) and carries the created identifier/project key:
|
|
888
|
+
* callers must never retry the create, only clean up or record it themselves.
|
|
864
889
|
*/
|
|
865
890
|
async function recordCleanupLedgerEntry(client, resource, action, args, flags, result, projectKey) {
|
|
866
891
|
const ledgerPath = flags["record-cleanup"];
|
|
@@ -868,15 +893,33 @@ async function recordCleanupLedgerEntry(client, resource, action, args, flags, r
|
|
|
868
893
|
return;
|
|
869
894
|
const entry = cleanupLedgerEntry(resource, action, args, flags, result, projectKey);
|
|
870
895
|
if (entry) {
|
|
871
|
-
|
|
896
|
+
try {
|
|
897
|
+
await appendCleanupLedgerEntry(ledgerPath, entry, client.getBaseUrl());
|
|
898
|
+
}
|
|
899
|
+
catch (error) {
|
|
900
|
+
throw new ClientValidationError(`Mutation completed but the cleanup entry for ${resource} ${action} could not be recorded: ${error instanceof Error ? error.message : String(error)}`, "ambiguous_outcome", "The mutation already succeeded: do not retry it. Remove the created object manually or append this cleanup entry to the ledger yourself.", {
|
|
901
|
+
resource,
|
|
902
|
+
action,
|
|
903
|
+
ledgerPath,
|
|
904
|
+
mutationSucceeded: true,
|
|
905
|
+
cleanupRecorded: false,
|
|
906
|
+
...createdObjectFromResult(result, projectKey),
|
|
907
|
+
cleanup: entry.cleanup,
|
|
908
|
+
});
|
|
909
|
+
}
|
|
872
910
|
return;
|
|
873
911
|
}
|
|
874
|
-
const record =
|
|
875
|
-
? result
|
|
876
|
-
: {};
|
|
912
|
+
const record = resultRecord(result);
|
|
877
913
|
if (record.skipped !== undefined || record.cleanupEligible === false)
|
|
878
914
|
return;
|
|
879
|
-
throw new ClientValidationError(`Mutation completed but no cleanup entry could be derived for ${resource} ${action}.`, "ambiguous_outcome", "Inspect the created object and remove it manually;
|
|
915
|
+
throw new ClientValidationError(`Mutation completed but no cleanup entry could be derived for ${resource} ${action}.`, "ambiguous_outcome", "The mutation already succeeded: do not retry it. Inspect the created object and remove it manually; the requested cleanup ledger does not contain it.", {
|
|
916
|
+
resource,
|
|
917
|
+
action,
|
|
918
|
+
ledgerPath,
|
|
919
|
+
mutationSucceeded: true,
|
|
920
|
+
cleanupRecorded: false,
|
|
921
|
+
...createdObjectFromResult(result, projectKey),
|
|
922
|
+
});
|
|
880
923
|
}
|
|
881
924
|
async function runBatch(flags) {
|
|
882
925
|
const payload = unknownJsonInput(flags);
|
|
@@ -1306,15 +1349,21 @@ function buildErrorReport(err, contextOverride) {
|
|
|
1306
1349
|
if (err instanceof ClientValidationError
|
|
1307
1350
|
&& (err.code === "ambiguous_outcome" || err.code === "assertion_failed"
|
|
1308
1351
|
|| err.code === "long_running_failure")) {
|
|
1352
|
+
// The DSS response behind an indeterminate outcome stays visible (status,
|
|
1353
|
+
// sanitized message), e.g. a 500 that names the conflicting project key.
|
|
1354
|
+
const cause = err.cause instanceof DataikuError ? err.cause : undefined;
|
|
1355
|
+
const diagnostics = cause ? dssDiagnosticDetails(cause) : {};
|
|
1356
|
+
const details = { ...diagnostics, ...err.details, };
|
|
1309
1357
|
return {
|
|
1310
1358
|
type: "error",
|
|
1311
1359
|
error: err.message,
|
|
1312
1360
|
code: err.code,
|
|
1313
1361
|
category: "dss",
|
|
1314
1362
|
exitCode,
|
|
1363
|
+
...(cause ? { status: cause.status, } : {}),
|
|
1315
1364
|
retryable: false,
|
|
1316
1365
|
...(err.hint ? { hint: err.hint, } : {}),
|
|
1317
|
-
...(
|
|
1366
|
+
...(Object.keys(details).length > 0 ? { details, } : {}),
|
|
1318
1367
|
...scope,
|
|
1319
1368
|
};
|
|
1320
1369
|
}
|
package/dist/src/client.d.ts
CHANGED
|
@@ -153,7 +153,13 @@ export declare class DataikuClient {
|
|
|
153
153
|
private readonly maxResponseBodyBytes;
|
|
154
154
|
private readonly retryMaxAttempts;
|
|
155
155
|
private readonly verbose;
|
|
156
|
-
|
|
156
|
+
/**
|
|
157
|
+
* Connection pool carrying this client's TLS settings (custom CA, disabled
|
|
158
|
+
* verification). A session never shares connections with plain `fetch` or
|
|
159
|
+
* another client, so a connection opened with relaxed TLS cannot serve a
|
|
160
|
+
* client that expects strict verification. Default-TLS clients use plain fetch.
|
|
161
|
+
*/
|
|
162
|
+
private readonly session;
|
|
157
163
|
private readonly onTrace;
|
|
158
164
|
private readonly onValidationWarning;
|
|
159
165
|
private projectsResource?;
|
|
@@ -294,7 +300,8 @@ export declare class DataikuClient {
|
|
|
294
300
|
* When the response exceeds the cap, earlier bytes are discarded, only
|
|
295
301
|
* the tail is kept and `truncated` is true — so callers that need the end
|
|
296
302
|
* of a large document (e.g. job logs) stay bounded without losing the most
|
|
297
|
-
* recent content. The
|
|
303
|
+
* recent content. The request timeout applies per received chunk (a stall
|
|
304
|
+
* deadline), not to the whole transfer.
|
|
298
305
|
*/
|
|
299
306
|
getTextTailLimited(path: string, maxBytes: number): Promise<{
|
|
300
307
|
text: string;
|
|
@@ -385,6 +392,11 @@ export declare class DataikuClient {
|
|
|
385
392
|
* Returns `truncated: true` when bytes had to be dropped. Bytes that fall
|
|
386
393
|
* in the middle of a UTF-8 character are dropped from both ends so the
|
|
387
394
|
* decoded text never contains a replacement character.
|
|
395
|
+
*
|
|
396
|
+
* The request timeout applies per chunk (a stall deadline), like the
|
|
397
|
+
* streaming path: a large body that keeps arriving is read to EOF however
|
|
398
|
+
* long the whole transfer takes, while a body that stops sending for longer
|
|
399
|
+
* than the timeout fails with the body-read timeout error.
|
|
388
400
|
*/
|
|
389
401
|
private readBodyTextTail;
|
|
390
402
|
private buildResponseTooLargeError;
|