dataiku-sdk 3.7.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +40 -4
- package/dist/packages/types/src/index.js +22 -1
- package/dist/src/cli/coerce.d.ts +1 -0
- package/dist/src/cli/coerce.js +1 -1
- package/dist/src/cli/command-syntax.json +156 -0
- package/dist/src/cli/commands/dataset.js +44 -8
- package/dist/src/cli/commands/flow-zone.js +241 -46
- package/dist/src/cli/commands/job.js +52 -9
- package/dist/src/cli/commands/project.js +4 -3
- package/dist/src/cli/commands/recipe.d.ts +4 -0
- package/dist/src/cli/commands/recipe.js +38 -1
- package/dist/src/cli/commands/wiki.js +56 -0
- package/dist/src/cli/contract.js +13 -8
- package/dist/src/cli/doctor.js +2 -0
- package/dist/src/cli/flags.d.ts +7 -0
- package/dist/src/cli/flags.js +28 -2
- package/dist/src/cli/helpers/dataset.d.ts +12 -0
- package/dist/src/cli/helpers/dataset.js +36 -0
- package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
- package/dist/src/cli/helpers/flow-zone.js +144 -4
- package/dist/src/cli/output-schemas.js +27 -3
- package/dist/src/cli/output.js +6 -0
- package/dist/src/cli/plans.js +58 -2
- package/dist/src/cli/project-map-output.d.ts +24 -0
- package/dist/src/cli/project-map-output.js +21 -0
- package/dist/src/cli/read-shapes.d.ts +14 -0
- package/dist/src/cli/read-shapes.js +114 -0
- package/dist/src/cli/result-shapes.d.ts +6 -0
- package/dist/src/cli/result-shapes.js +235 -0
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +2 -2
- package/dist/src/cli.js +23 -20
- package/dist/src/errors.d.ts +11 -1
- package/dist/src/errors.js +18 -3
- package/dist/src/generated/action-output-schemas.json +224 -35
- package/dist/src/resources/dataset-create.d.ts +19 -0
- package/dist/src/resources/dataset-create.js +69 -0
- package/dist/src/resources/datasets.d.ts +21 -0
- package/dist/src/resources/datasets.js +61 -1
- package/dist/src/resources/flow-zones.d.ts +29 -3
- package/dist/src/resources/flow-zones.js +53 -3
- package/dist/src/resources/jobs.d.ts +36 -1
- package/dist/src/resources/jobs.js +111 -5
- package/dist/src/resources/projects.d.ts +9 -0
- package/dist/src/resources/projects.js +10 -3
- package/dist/src/resources/recipe-create.d.ts +17 -3
- package/dist/src/resources/recipe-create.js +149 -33
- package/dist/src/resources/recipes.js +76 -23
- package/dist/src/resources/sql.js +52 -13
- package/dist/src/resources/wiki.d.ts +47 -1
- package/dist/src/resources/wiki.js +89 -0
- package/dist/src/utils/flow-analysis.js +4 -0
- package/dist/src/utils/secret-sanitize.d.ts +11 -2
- package/dist/src/utils/secret-sanitize.js +23 -6
- package/package.json +4 -4
- package/plugin.json +1 -1
- package/skills/dataiku-dss/references/discovery.md +2 -2
- package/skills/dataiku-dss/references/flow-maps.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +3 -3
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { findWikiAttachment, statWikiAttachmentFile, wikiAttachmentNotFound, wikiAttachments, } from "../../resources/wiki.js";
|
|
1
2
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
2
3
|
import { jsonInput, textInput, } from "../coerce.js";
|
|
3
4
|
import { executionMode, } from "../flags.js";
|
|
@@ -117,6 +118,61 @@ export const wikiCommands = withUsage("wiki", {
|
|
|
117
118
|
description: "Delete a wiki article.",
|
|
118
119
|
examples: ["dss wiki delete ARTICLE_ID --dry-run",],
|
|
119
120
|
},
|
|
121
|
+
attach: {
|
|
122
|
+
handler: async (c, a, f) => {
|
|
123
|
+
requireArgs(a, 2, commandUsage("wiki", "attach"));
|
|
124
|
+
const pk = f["project-key"];
|
|
125
|
+
const file = await statWikiAttachmentFile(a[1]);
|
|
126
|
+
if (executionMode(f).dryRun) {
|
|
127
|
+
const current = await c.wiki.get(a[0], pk);
|
|
128
|
+
return {
|
|
129
|
+
dryRun: true,
|
|
130
|
+
action: "attach",
|
|
131
|
+
resource: "wiki",
|
|
132
|
+
article: a[0],
|
|
133
|
+
file,
|
|
134
|
+
current: wikiAttachments(current),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
const result = await c.wiki.attach(a[0], a[1], pk);
|
|
138
|
+
return { attached: result.attachment.smartId, resource: "wiki", ...result, };
|
|
139
|
+
},
|
|
140
|
+
description: "Upload a local file and attach it to a wiki article; returns its smartId.",
|
|
141
|
+
examples: ["dss wiki attach ARTICLE_ID ./diagram.png --dry-run",],
|
|
142
|
+
},
|
|
143
|
+
detach: {
|
|
144
|
+
handler: async (c, a, f) => {
|
|
145
|
+
requireArgs(a, 2, commandUsage("wiki", "detach"));
|
|
146
|
+
const pk = f["project-key"];
|
|
147
|
+
const ifExists = f["if-exists"] === true;
|
|
148
|
+
const dryRun = executionMode(f).dryRun;
|
|
149
|
+
if (dryRun || ifExists) {
|
|
150
|
+
const current = ifExists
|
|
151
|
+
? await readIfExists(() => c.wiki.get(a[0], pk))
|
|
152
|
+
: await c.wiki.get(a[0], pk);
|
|
153
|
+
const attachment = current && findWikiAttachment(current, a[1]);
|
|
154
|
+
if (!current || !attachment) {
|
|
155
|
+
if (!ifExists && current)
|
|
156
|
+
throw wikiAttachmentNotFound(current, a[1]);
|
|
157
|
+
return skipResult("wiki", a[1], "missing", { article: a[0], });
|
|
158
|
+
}
|
|
159
|
+
if (dryRun) {
|
|
160
|
+
return {
|
|
161
|
+
dryRun: true,
|
|
162
|
+
action: "detach",
|
|
163
|
+
resource: "wiki",
|
|
164
|
+
article: a[0],
|
|
165
|
+
smartId: a[1],
|
|
166
|
+
current: attachment,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
const result = await c.wiki.detach(a[0], a[1], pk);
|
|
171
|
+
return { resource: "wiki", ...result, };
|
|
172
|
+
},
|
|
173
|
+
description: "Remove one attachment from a wiki article by smartId.",
|
|
174
|
+
examples: ["dss wiki detach ARTICLE_ID SMART_ID --if-exists",],
|
|
175
|
+
},
|
|
120
176
|
"update-settings": {
|
|
121
177
|
handler: async (c, _a, f) => {
|
|
122
178
|
const pk = f["project-key"];
|
package/dist/src/cli/contract.js
CHANGED
|
@@ -411,6 +411,8 @@ function payloadJsonSchema(payloadSchema) {
|
|
|
411
411
|
if (payloadSchema.contentType === "text/plain") {
|
|
412
412
|
return { type: "string", contentMediaType: "text/plain", };
|
|
413
413
|
}
|
|
414
|
+
if (payloadSchema.jsonSchema)
|
|
415
|
+
return payloadSchema.jsonSchema;
|
|
414
416
|
return payloadSchema.jsonShape === "array"
|
|
415
417
|
? { type: "array", items: true, }
|
|
416
418
|
: { type: "object", additionalProperties: true, };
|
|
@@ -722,7 +724,7 @@ function inferSideEffect(resource, action) {
|
|
|
722
724
|
}
|
|
723
725
|
if (READ_ACTIONS.has(action))
|
|
724
726
|
return "read";
|
|
725
|
-
if (/^(create|clone|restore|update|delete|set|save|upload|run|build|abort|move|refresh|clear|unload|install|login|logout|add|remove|publish|activate|deploy|import|export|preload|upgrade|start|stop|restart|duplicate|put|rename|reply|compute|organize)/
|
|
727
|
+
if (/^(create|clone|restore|update|delete|set|save|upload|run|build|abort|move|refresh|clear|unload|install|login|logout|add|remove|publish|activate|deploy|import|export|preload|upgrade|start|stop|restart|duplicate|put|rename|reply|compute|organize|attach|detach)/
|
|
726
728
|
.test(action)
|
|
727
729
|
// Compound actions whose mutating verb is a suffix (e.g. permissions-set,
|
|
728
730
|
// dataset-compute, saved-model external-metadata-put).
|
|
@@ -876,6 +878,8 @@ export function supportsCleanupLedger(resource, action) {
|
|
|
876
878
|
const EXPLICIT_DESTRUCTIVE_KEYS = {
|
|
877
879
|
"dataset.upload-file": true,
|
|
878
880
|
"sql.query": true,
|
|
881
|
+
// Detaching drops the attachment from the article; no public route restores it.
|
|
882
|
+
"wiki.detach": true,
|
|
879
883
|
// Plugin dev-Git and content deletions discard work with no undo route.
|
|
880
884
|
"plugin.delete": true,
|
|
881
885
|
"plugin.delete-git-remote": true,
|
|
@@ -961,8 +965,10 @@ function inferIdempotency(resource, sideEffect, action, syntax) {
|
|
|
961
965
|
if (action.startsWith("create") && syntaxHasFlag(syntax, "if-not-exists")) {
|
|
962
966
|
return "if-not-exists";
|
|
963
967
|
}
|
|
964
|
-
if (action.startsWith("delete")
|
|
968
|
+
if ((action.startsWith("delete") || `${resource}.${action}` === "wiki.detach")
|
|
969
|
+
&& syntaxHasFlag(syntax, "if-exists")) {
|
|
965
970
|
return "if-exists";
|
|
971
|
+
}
|
|
966
972
|
if (`${resource}.${action}` === "app.set-manifest-version")
|
|
967
973
|
return "none";
|
|
968
974
|
// `app delete-instance` converges without an `--if-exists` flag: an absent
|
|
@@ -1062,6 +1068,8 @@ export function buildRegistryEntry(resource, action, meta) {
|
|
|
1062
1068
|
const payloadSchema = meta.payloadSchema
|
|
1063
1069
|
?? EXPLICIT_REGISTRY_OVERRIDES[registryKey(resource, action)]?.payloadSchema
|
|
1064
1070
|
?? inferPayloadSchema(inputContract);
|
|
1071
|
+
// The precise JSON Schema is published once, as `schemas.input`.
|
|
1072
|
+
const { jsonSchema: _jsonSchema, ...payloadSummary } = payloadSchema ?? {};
|
|
1065
1073
|
const examplePayload = meta.examplePayload
|
|
1066
1074
|
?? EXPLICIT_REGISTRY_OVERRIDES[registryKey(resource, action)]?.examplePayload;
|
|
1067
1075
|
const inferredCleanupCommand = meta.cleanupCommand
|
|
@@ -1126,7 +1134,7 @@ export function buildRegistryEntry(resource, action, meta) {
|
|
|
1126
1134
|
optionalFlags: uniqueOptionalFlags,
|
|
1127
1135
|
...(requiredOneOf.length > 0 ? { requiredOneOf, } : {}),
|
|
1128
1136
|
...(inputGroups.length > 0 ? { requiredInputGroups: inputGroups, } : {}),
|
|
1129
|
-
...(payloadSchema ? { payloadSchema, } : {}),
|
|
1137
|
+
...(payloadSchema ? { payloadSchema: payloadSummary, } : {}),
|
|
1130
1138
|
get schemas() {
|
|
1131
1139
|
return schemas ??= buildCommandSchemas(resource, action, flagMetadata, uniqueRequiredFlags, requiredOneOf, payloadSchema, outputShape, meta.usage);
|
|
1132
1140
|
},
|
|
@@ -1392,11 +1400,10 @@ function commandRegistryJsonSchema() {
|
|
|
1392
1400
|
function errorEnvelopeJsonSchema() {
|
|
1393
1401
|
return {
|
|
1394
1402
|
type: "object",
|
|
1395
|
-
required: ["type", "
|
|
1403
|
+
required: ["type", "error", "code", "category", "exitCode",],
|
|
1396
1404
|
additionalProperties: true,
|
|
1397
1405
|
properties: {
|
|
1398
1406
|
type: { const: "error", },
|
|
1399
|
-
ok: { const: false, },
|
|
1400
1407
|
error: { type: "string", },
|
|
1401
1408
|
code: { type: "string", },
|
|
1402
1409
|
category: { enum: ["usage", "permission_or_environment", "dss", "internal",], },
|
|
@@ -1406,8 +1413,6 @@ function errorEnvelopeJsonSchema() {
|
|
|
1406
1413
|
retryable: { type: "boolean", },
|
|
1407
1414
|
requestId: { type: "string", },
|
|
1408
1415
|
details: { type: "object", additionalProperties: true, },
|
|
1409
|
-
resource: { type: "string", },
|
|
1410
|
-
action: { type: "string", },
|
|
1411
1416
|
projectKey: { type: "string", },
|
|
1412
1417
|
},
|
|
1413
1418
|
};
|
|
@@ -1518,7 +1523,7 @@ export function buildAgentContract() {
|
|
|
1518
1523
|
failure: "structured-error-object",
|
|
1519
1524
|
failureResultDetailLimitBytes: 65_536,
|
|
1520
1525
|
fieldProjection: "Missing --fields paths: null on stdout; field_projection_missing on stderr.",
|
|
1521
|
-
|
|
1526
|
+
compactResults: "*.list, job get, dataset get/info, project get: compact; --full/--fields use DSS objects; lists take --contains/--limit (list_truncated warning).",
|
|
1522
1527
|
richFailureResults: "doctor/batch/cleanup failures: own compact {ok:false,...} result on stdout and command exit code; no wrapper.",
|
|
1523
1528
|
},
|
|
1524
1529
|
stderr: {
|
package/dist/src/cli/doctor.js
CHANGED
|
@@ -11,6 +11,8 @@ export function errorDetails(error) {
|
|
|
11
11
|
retryable: error.retryable,
|
|
12
12
|
status: error.status,
|
|
13
13
|
statusText: error.statusText,
|
|
14
|
+
message: error.message,
|
|
15
|
+
retryHint: error.retryHint,
|
|
14
16
|
};
|
|
15
17
|
}
|
|
16
18
|
return { message: error instanceof Error ? error.message : String(error), };
|
package/dist/src/cli/flags.d.ts
CHANGED
|
@@ -18,6 +18,13 @@ export declare const REPEATABLE_VALUE_FLAGS: Set<string>;
|
|
|
18
18
|
export declare const KNOWN_LONG_FLAGS: Set<string>;
|
|
19
19
|
export declare function normalizeLongFlag(rawFlagName: string, argv: string[]): string;
|
|
20
20
|
export declare function isNegativeNumberToken(value: string): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Whether the token after a value flag is that flag's value rather than the
|
|
23
|
+
* next flag. `-` (stdin), negative numbers, and anything not flag-shaped
|
|
24
|
+
* qualify, so a SQL value beginning with a `-- comment` is a value; `--` alone
|
|
25
|
+
* and `--flag`/`--flag=x` are not.
|
|
26
|
+
*/
|
|
27
|
+
export declare function isFlagValueToken(token: string): boolean;
|
|
21
28
|
export declare function requireFlagValue(flagLabel: string, next: string | undefined): string;
|
|
22
29
|
export declare function setParsedFlagValue(flags: Record<string, string | boolean>, flagName: string, value: string): void;
|
|
23
30
|
export interface ParsedArgs {
|
package/dist/src/cli/flags.js
CHANGED
|
@@ -81,6 +81,7 @@ export const BOOLEAN_FLAGS = new Set([
|
|
|
81
81
|
"allow-same-path",
|
|
82
82
|
"sync",
|
|
83
83
|
"validate-objects",
|
|
84
|
+
"recreate-on-move",
|
|
84
85
|
"errors-only",
|
|
85
86
|
"keep",
|
|
86
87
|
"full-log",
|
|
@@ -223,6 +224,8 @@ export const VALUE_FLAGS = new Set([
|
|
|
223
224
|
"render",
|
|
224
225
|
"prediction-type",
|
|
225
226
|
"project-key",
|
|
227
|
+
"query",
|
|
228
|
+
"query-file",
|
|
226
229
|
"recipe",
|
|
227
230
|
"reference",
|
|
228
231
|
"repository",
|
|
@@ -353,9 +356,32 @@ export function normalizeLongFlag(rawFlagName, argv) {
|
|
|
353
356
|
export function isNegativeNumberToken(value) {
|
|
354
357
|
return value.startsWith("-") && Number.isFinite(Number(value));
|
|
355
358
|
}
|
|
359
|
+
/** `--name` or `--name=value`: a token that parses as a long flag. */
|
|
360
|
+
const LONG_FLAG_SHAPE = /^--[A-Za-z][A-Za-z0-9-]*(?:=|$)/;
|
|
361
|
+
/**
|
|
362
|
+
* Whether the token after a value flag is that flag's value rather than the
|
|
363
|
+
* next flag. `-` (stdin), negative numbers, and anything not flag-shaped
|
|
364
|
+
* qualify, so a SQL value beginning with a `-- comment` is a value; `--` alone
|
|
365
|
+
* and `--flag`/`--flag=x` are not.
|
|
366
|
+
*/
|
|
367
|
+
export function isFlagValueToken(token) {
|
|
368
|
+
if (token === "-" || isNegativeNumberToken(token))
|
|
369
|
+
return true;
|
|
370
|
+
if (!token.startsWith("-"))
|
|
371
|
+
return true;
|
|
372
|
+
return token.startsWith("--") && token !== "--" && !LONG_FLAG_SHAPE.test(token);
|
|
373
|
+
}
|
|
356
374
|
export function requireFlagValue(flagLabel, next) {
|
|
357
|
-
if (next === undefined ||
|
|
358
|
-
|
|
375
|
+
if (next === undefined || !isFlagValueToken(next)) {
|
|
376
|
+
const inlineForm = flagLabel.startsWith("--") ? `${flagLabel}=VALUE` : undefined;
|
|
377
|
+
const hint = next === undefined || !next.startsWith("-")
|
|
378
|
+
? undefined
|
|
379
|
+
: flagLabel === "--sql"
|
|
380
|
+
? 'A SQL value that starts with `--` (a comment) can be passed as --sql="-- ...", from a file with --sql-file PATH, or on stdin with --stdin.'
|
|
381
|
+
: inlineForm
|
|
382
|
+
? `If the value itself starts with \`-\`, pass it as ${inlineForm}.`
|
|
383
|
+
: undefined;
|
|
384
|
+
throw new UsageError(`Flag ${flagLabel} requires a value.`, "missing_required_flag", hint);
|
|
359
385
|
}
|
|
360
386
|
return next;
|
|
361
387
|
}
|
|
@@ -1,2 +1,14 @@
|
|
|
1
1
|
import type { DatasetDetails } from "../../schemas.js";
|
|
2
|
+
export interface DatasetSqlSourceFlags {
|
|
3
|
+
table?: string;
|
|
4
|
+
dbSchema?: string;
|
|
5
|
+
catalog?: string;
|
|
6
|
+
query?: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* The SQL source of `dataset create` (`--table [--schema S] [--catalog C]` or
|
|
10
|
+
* `--query SQL`/`--query-file PATH`), validated against the dataset type.
|
|
11
|
+
* Empty when no SQL flag is given.
|
|
12
|
+
*/
|
|
13
|
+
export declare function datasetSqlSourceFromFlags(flags: Record<string, string | boolean>, dsType: string): DatasetSqlSourceFlags;
|
|
2
14
|
export declare function datasetSourceSummary(details: DatasetDetails): Record<string, unknown>;
|
|
@@ -1,3 +1,39 @@
|
|
|
1
|
+
import { assertSqlDatasetSource, } from "../../resources/dataset-create.js";
|
|
2
|
+
import { readInputText, stripUtf8Bom, } from "../coerce.js";
|
|
3
|
+
import { UsageError, } from "../usage.js";
|
|
4
|
+
/**
|
|
5
|
+
* The SQL source of `dataset create` (`--table [--schema S] [--catalog C]` or
|
|
6
|
+
* `--query SQL`/`--query-file PATH`), validated against the dataset type.
|
|
7
|
+
* Empty when no SQL flag is given.
|
|
8
|
+
*/
|
|
9
|
+
export function datasetSqlSourceFromFlags(flags, dsType) {
|
|
10
|
+
const text = (flag) => {
|
|
11
|
+
const value = flags[flag];
|
|
12
|
+
if (value === undefined)
|
|
13
|
+
return undefined;
|
|
14
|
+
if (typeof value !== "string") {
|
|
15
|
+
throw new UsageError(`--${flag} requires a value.`, "missing_required_flag");
|
|
16
|
+
}
|
|
17
|
+
return value;
|
|
18
|
+
};
|
|
19
|
+
if (flags["query"] !== undefined && flags["query-file"] !== undefined) {
|
|
20
|
+
throw new UsageError("Conflicting input sources: --query, --query-file. Provide exactly one.", "conflicting_input_sources", undefined, { sources: ["query", "query-file",], });
|
|
21
|
+
}
|
|
22
|
+
const queryFile = text("query-file");
|
|
23
|
+
const inline = text("query");
|
|
24
|
+
const source = {
|
|
25
|
+
...(text("table") !== undefined ? { table: text("table"), } : {}),
|
|
26
|
+
...(text("schema") !== undefined ? { dbSchema: text("schema"), } : {}),
|
|
27
|
+
...(text("catalog") !== undefined ? { catalog: text("catalog"), } : {}),
|
|
28
|
+
...(queryFile !== undefined
|
|
29
|
+
? { query: stripUtf8Bom(readInputText(queryFile, "--query-file")), }
|
|
30
|
+
: inline !== undefined
|
|
31
|
+
? { query: inline, }
|
|
32
|
+
: {}),
|
|
33
|
+
};
|
|
34
|
+
assertSqlDatasetSource(dsType, source);
|
|
35
|
+
return source;
|
|
36
|
+
}
|
|
1
37
|
export function datasetSourceSummary(details) {
|
|
2
38
|
const params = details.params ?? {};
|
|
3
39
|
return {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { DataikuClient } from "../../client.js";
|
|
2
2
|
import type { FlowZoneItemInput } from "../../resources/flow-zones.js";
|
|
3
3
|
import type { FlowZone, FlowZoneObjectType, FlowZonePosition } from "../../schemas.js";
|
|
4
|
+
import type { NormalizedFlowMap } from "../../utils/flow-map.js";
|
|
4
5
|
export declare function flowZoneId(value: string): string;
|
|
5
6
|
export declare function flowZoneName(value: string | boolean | undefined): string;
|
|
6
7
|
export declare function flowZoneColor(value: string | boolean | undefined): string | undefined;
|
|
@@ -18,6 +19,8 @@ export interface FlowZoneOrganizeZonePlan {
|
|
|
18
19
|
color?: string;
|
|
19
20
|
position?: FlowZonePosition;
|
|
20
21
|
items: FlowZoneItemInput[];
|
|
22
|
+
/** Recipe/output unit members added by {@link applyFlowZoneUnits}; also present in `items`. */
|
|
23
|
+
impliedItems?: FlowZoneItemInput[];
|
|
21
24
|
}
|
|
22
25
|
export interface FlowZoneOrganizePlan {
|
|
23
26
|
topologyFingerprint?: string;
|
|
@@ -60,7 +63,25 @@ export declare function flowZoneExplicitItems(zone: FlowZone): FlowZoneItemInput
|
|
|
60
63
|
export declare function flowZonePlanFromZones(zones: FlowZone[], topologyFingerprint: string): FlowZoneOrganizePlan;
|
|
61
64
|
export declare function flowZoneMoveDelta(existing: FlowZone | undefined, plannedItems: FlowZoneItemInput[]): FlowZoneItemInput[];
|
|
62
65
|
export declare function flowZonePruneItems(existing: FlowZone | undefined, plannedItemKeys: Set<string>): FlowZoneItemInput[];
|
|
66
|
+
/**
|
|
67
|
+
* The position an existing zone must move to, if the plan changes it. DSS
|
|
68
|
+
* honors a zone position only on creation, so a move means recreating the zone.
|
|
69
|
+
*/
|
|
70
|
+
export declare function flowZoneRepositionTarget(plan: FlowZoneOrganizeZonePlan, existing: FlowZone): FlowZonePosition | undefined;
|
|
63
71
|
export declare function flowZoneOrganizeStep(plan: FlowZoneOrganizeZonePlan, existing: FlowZone | undefined, sync: boolean, plannedItemKeys: Set<string>): Record<string, unknown>;
|
|
72
|
+
/**
|
|
73
|
+
* Enforce the DSS recipe/output zone invariant on a plan. Rejects plans that
|
|
74
|
+
* place members of one unit in different zones (they would never converge),
|
|
75
|
+
* and adds the unlisted members of each planned unit to that zone as
|
|
76
|
+
* `impliedItems`, so moves, `--sync` pruning, and reruns all agree with what
|
|
77
|
+
* DSS actually does.
|
|
78
|
+
*/
|
|
79
|
+
export declare function applyFlowZoneUnits(plan: FlowZoneOrganizePlan, graph: NormalizedFlowMap, zones: FlowZone[], projectKey: string | undefined): FlowZoneOrganizePlan;
|
|
80
|
+
/**
|
|
81
|
+
* Default-zone objects an organization plan has to place: recipes (their
|
|
82
|
+
* outputs follow) and objects no recipe produces, such as uploaded sources.
|
|
83
|
+
*/
|
|
84
|
+
export declare function flowZoneUnassignedItems(graph: NormalizedFlowMap, zones: FlowZone[]): FlowZoneItemInput[];
|
|
64
85
|
export declare function flowZoneValidationBucket(index: FlowZoneOrganizeValidationIndex, objectType: FlowZoneObjectType): Set<string>;
|
|
65
86
|
export declare function flowZoneValidationIndex(client: DataikuClient, projectKey: string | undefined): Promise<FlowZoneOrganizeValidationIndex>;
|
|
66
87
|
export declare function validateFlowZoneOrganizeObjects(client: DataikuClient, plan: FlowZoneOrganizePlan, projectKey: string | undefined): Promise<FlowZoneOrganizeValidationResult>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
|
+
import { ClientValidationError, } from "../../errors.js";
|
|
2
3
|
import { asRecord, } from "../../utils/records.js";
|
|
3
4
|
import { compareStrings, } from "../../utils/stable-hash.js";
|
|
4
5
|
import { finiteNumberField, jsonInput, optionalStringField, parseJsonObject, requiredStringArray, splitCsvFlag, } from "../coerce.js";
|
|
@@ -319,6 +320,16 @@ export function flowZonePruneItems(existing, plannedItemKeys) {
|
|
|
319
320
|
return [];
|
|
320
321
|
return flowZoneExplicitItems(existing).filter((item) => !plannedItemKeys.has(flowZoneItemKey(item)));
|
|
321
322
|
}
|
|
323
|
+
/**
|
|
324
|
+
* The position an existing zone must move to, if the plan changes it. DSS
|
|
325
|
+
* honors a zone position only on creation, so a move means recreating the zone.
|
|
326
|
+
*/
|
|
327
|
+
export function flowZoneRepositionTarget(plan, existing) {
|
|
328
|
+
return plan.position !== undefined
|
|
329
|
+
&& !flowZoneSamePosition(flowZoneCurrentPosition(existing), plan.position)
|
|
330
|
+
? plan.position
|
|
331
|
+
: undefined;
|
|
332
|
+
}
|
|
322
333
|
export function flowZoneOrganizeStep(plan, existing, sync, plannedItemKeys) {
|
|
323
334
|
ensureFlowZonePlanTarget(plan, existing);
|
|
324
335
|
const update = {};
|
|
@@ -326,10 +337,8 @@ export function flowZoneOrganizeStep(plan, existing, sync, plannedItemKeys) {
|
|
|
326
337
|
update.name = plan.name;
|
|
327
338
|
if (existing && plan.color && plan.color !== existing.color)
|
|
328
339
|
update.color = plan.color;
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
update.position = plan.position;
|
|
332
|
-
}
|
|
340
|
+
const recreatePosition = existing ? flowZoneRepositionTarget(plan, existing) : undefined;
|
|
341
|
+
const recreate = recreatePosition ? { position: recreatePosition, } : undefined;
|
|
333
342
|
const pruneItems = sync ? flowZonePruneItems(existing, plannedItemKeys) : [];
|
|
334
343
|
const moveItems = flowZoneMoveDelta(existing, plan.items);
|
|
335
344
|
return {
|
|
@@ -341,10 +350,141 @@ export function flowZoneOrganizeStep(plan, existing, sync, plannedItemKeys) {
|
|
|
341
350
|
},
|
|
342
351
|
...(existing ? { existing: flowZoneSummary(existing), } : { create: true, }),
|
|
343
352
|
...(Object.keys(update).length > 0 ? { update, } : {}),
|
|
353
|
+
...(recreate ? { recreate, } : {}),
|
|
344
354
|
moveItems,
|
|
355
|
+
...(plan.impliedItems && plan.impliedItems.length > 0
|
|
356
|
+
? { impliedItems: plan.impliedItems, }
|
|
357
|
+
: {}),
|
|
345
358
|
...(pruneItems.length > 0 ? { pruneItems, } : {}),
|
|
346
359
|
};
|
|
347
360
|
}
|
|
361
|
+
function flowNodeObjectType(node, knownTypes) {
|
|
362
|
+
switch (node.kind) {
|
|
363
|
+
case "recipe":
|
|
364
|
+
return "RECIPE";
|
|
365
|
+
case "dataset":
|
|
366
|
+
return "DATASET";
|
|
367
|
+
case "folder":
|
|
368
|
+
return "MANAGED_FOLDER";
|
|
369
|
+
default:
|
|
370
|
+
return knownTypes.get(node.id);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* DSS keeps a recipe and its outputs in one zone: moving either drags the
|
|
375
|
+
* others. Each unit is keyed by its recipe id and lists the recipe plus every
|
|
376
|
+
* object the recipe writes.
|
|
377
|
+
*/
|
|
378
|
+
function flowZoneUnits(graph) {
|
|
379
|
+
const kindById = new Map(graph.nodes.map((node) => [node.id, node.kind,]));
|
|
380
|
+
const units = new Map();
|
|
381
|
+
for (const node of graph.nodes) {
|
|
382
|
+
if (node.kind === "recipe")
|
|
383
|
+
units.set(node.id, [node.id,]);
|
|
384
|
+
}
|
|
385
|
+
for (const edge of graph.edges) {
|
|
386
|
+
if (kindById.get(edge.from) !== "recipe" || kindById.get(edge.to) === "recipe")
|
|
387
|
+
continue;
|
|
388
|
+
units.get(edge.from)?.push(edge.to);
|
|
389
|
+
}
|
|
390
|
+
return units;
|
|
391
|
+
}
|
|
392
|
+
function explicitTypesById(zones) {
|
|
393
|
+
const types = new Map();
|
|
394
|
+
for (const zone of zones) {
|
|
395
|
+
for (const item of flowZoneItems(zone)) {
|
|
396
|
+
if (!item.projectKey)
|
|
397
|
+
types.set(item.objectId, item.objectType);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
return types;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Enforce the DSS recipe/output zone invariant on a plan. Rejects plans that
|
|
404
|
+
* place members of one unit in different zones (they would never converge),
|
|
405
|
+
* and adds the unlisted members of each planned unit to that zone as
|
|
406
|
+
* `impliedItems`, so moves, `--sync` pruning, and reruns all agree with what
|
|
407
|
+
* DSS actually does.
|
|
408
|
+
*/
|
|
409
|
+
export function applyFlowZoneUnits(plan, graph, zones, projectKey) {
|
|
410
|
+
const zoneIndexById = new Map();
|
|
411
|
+
plan.zones.forEach((zonePlan, index) => {
|
|
412
|
+
for (const item of zonePlan.items) {
|
|
413
|
+
// Shared objects from other projects never move with a local recipe.
|
|
414
|
+
if (!item.projectKey || item.projectKey === projectKey)
|
|
415
|
+
zoneIndexById.set(item.objectId, index);
|
|
416
|
+
}
|
|
417
|
+
});
|
|
418
|
+
const nodeById = new Map(graph.nodes.map((node) => [node.id, node,]));
|
|
419
|
+
const knownTypes = explicitTypesById(zones);
|
|
420
|
+
const conflicts = [];
|
|
421
|
+
const implied = plan.zones.map(() => []);
|
|
422
|
+
for (const [recipe, members,] of flowZoneUnits(graph)) {
|
|
423
|
+
const planned = members.filter((member) => zoneIndexById.has(member));
|
|
424
|
+
if (planned.length === 0)
|
|
425
|
+
continue;
|
|
426
|
+
const indexes = new Set(planned.map((member) => zoneIndexById.get(member)));
|
|
427
|
+
if (indexes.size > 1) {
|
|
428
|
+
const byZone = {};
|
|
429
|
+
for (const member of planned) {
|
|
430
|
+
const label = flowZonePlanLabel(plan.zones[zoneIndexById.get(member)]);
|
|
431
|
+
(byZone[label] ??= []).push(member);
|
|
432
|
+
}
|
|
433
|
+
conflicts.push({ recipe, zones: byZone, });
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
const [index,] = indexes;
|
|
437
|
+
for (const member of members) {
|
|
438
|
+
if (zoneIndexById.has(member))
|
|
439
|
+
continue;
|
|
440
|
+
const node = nodeById.get(member);
|
|
441
|
+
const objectType = node ? flowNodeObjectType(node, knownTypes) : undefined;
|
|
442
|
+
if (objectType)
|
|
443
|
+
implied[index].push({ objectType, objectId: member, });
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
if (conflicts.length > 0) {
|
|
447
|
+
throw new ClientValidationError(`Flow zone plan splits ${conflicts.length} recipe(s) from their outputs across zones; DSS keeps a recipe and its outputs in one zone.`, "validation_failed", "Assign each recipe to one zone (its outputs follow automatically), or list the recipe and all its outputs in the same zone.", { conflicts, });
|
|
448
|
+
}
|
|
449
|
+
return {
|
|
450
|
+
...plan,
|
|
451
|
+
zones: plan.zones.map((zonePlan, index) => implied[index].length === 0
|
|
452
|
+
? zonePlan
|
|
453
|
+
: {
|
|
454
|
+
...zonePlan,
|
|
455
|
+
items: [...zonePlan.items, ...implied[index],],
|
|
456
|
+
impliedItems: implied[index],
|
|
457
|
+
}),
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Default-zone objects an organization plan has to place: recipes (their
|
|
462
|
+
* outputs follow) and objects no recipe produces, such as uploaded sources.
|
|
463
|
+
*/
|
|
464
|
+
export function flowZoneUnassignedItems(graph, zones) {
|
|
465
|
+
const assigned = new Set();
|
|
466
|
+
for (const zone of zones) {
|
|
467
|
+
if (zone.id === "default")
|
|
468
|
+
continue;
|
|
469
|
+
for (const item of flowZoneExplicitItems(zone))
|
|
470
|
+
assigned.add(item.objectId);
|
|
471
|
+
}
|
|
472
|
+
const produced = new Set();
|
|
473
|
+
for (const members of flowZoneUnits(graph).values()) {
|
|
474
|
+
for (const member of members.slice(1))
|
|
475
|
+
produced.add(member);
|
|
476
|
+
}
|
|
477
|
+
const knownTypes = explicitTypesById(zones);
|
|
478
|
+
const items = [];
|
|
479
|
+
for (const node of graph.nodes) {
|
|
480
|
+
if (assigned.has(node.id) || produced.has(node.id))
|
|
481
|
+
continue;
|
|
482
|
+
const objectType = flowNodeObjectType(node, knownTypes);
|
|
483
|
+
if (objectType)
|
|
484
|
+
items.push({ objectType, objectId: node.id, });
|
|
485
|
+
}
|
|
486
|
+
return items.sort((a, b) => compareStrings(flowZoneItemKey(a), flowZoneItemKey(b)));
|
|
487
|
+
}
|
|
348
488
|
export function flowZoneValidationBucket(index, objectType) {
|
|
349
489
|
switch (objectType) {
|
|
350
490
|
case "DATASET":
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* Imported by the generator and the discovery parity test only.
|
|
14
14
|
*/
|
|
15
15
|
import { compactListOutputSchemas, } from "./list-shapes.js";
|
|
16
|
+
import { compactReadOutputSchemas, DATASET_GET_DROPPED_KEYS, PROJECT_GET_DROPPED_KEYS, } from "./read-shapes.js";
|
|
16
17
|
const PROJECT_LIBRARY_ITEM_OUTPUT_SCHEMA = {
|
|
17
18
|
type: "object",
|
|
18
19
|
additionalProperties: false,
|
|
@@ -50,6 +51,21 @@ const NOTEBOOK_SAVE_SQL_OUTPUT_SCHEMA = {
|
|
|
50
51
|
requested: { type: "string", },
|
|
51
52
|
},
|
|
52
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* Discovery schema of a compact settings object (`dataset get`, `project get`):
|
|
56
|
+
* the SDK TypeBox schema minus the keys the CLI drops, with only the
|
|
57
|
+
* identifiers that survive pruning required. SDK validation keeps the
|
|
58
|
+
* unmodified TypeBox schema.
|
|
59
|
+
*/
|
|
60
|
+
function settingsOutputSchema(schema, droppedKeys, extraProperties = {}) {
|
|
61
|
+
const properties = Object.fromEntries(Object.entries(schema["properties"]).filter(([key,]) => !droppedKeys.includes(key)));
|
|
62
|
+
const required = (schema["required"] ?? []).filter((key) => !droppedKeys.includes(key));
|
|
63
|
+
return {
|
|
64
|
+
...schema,
|
|
65
|
+
properties: { ...properties, ...extraProperties, },
|
|
66
|
+
required: [...required, ...Object.keys(extraProperties),],
|
|
67
|
+
};
|
|
68
|
+
}
|
|
53
69
|
let commandOutputSchemas;
|
|
54
70
|
/**
|
|
55
71
|
* Canonical TypeBox-derived output schemas, keyed by `resource.action`. This is
|
|
@@ -196,9 +212,15 @@ export function typeBoxCommandOutputSchemas() {
|
|
|
196
212
|
maxLines: { type: "integer", minimum: 0, },
|
|
197
213
|
},
|
|
198
214
|
},
|
|
199
|
-
"project.get": ProjectDetailsSchema,
|
|
215
|
+
"project.get": settingsOutputSchema(ProjectDetailsSchema, PROJECT_GET_DROPPED_KEYS, {
|
|
216
|
+
deniedPermissions: {
|
|
217
|
+
type: "array",
|
|
218
|
+
items: { type: "string", },
|
|
219
|
+
description: "Permission flags DSS reports as false; absent flags are granted.",
|
|
220
|
+
},
|
|
221
|
+
}),
|
|
200
222
|
"project.metadata": ProjectMetadataSchema,
|
|
201
|
-
"dataset.get": DatasetDetailsSchema,
|
|
223
|
+
"dataset.get": settingsOutputSchema(DatasetDetailsSchema, DATASET_GET_DROPPED_KEYS),
|
|
202
224
|
"dataset.schema": DatasetSchemaSchema,
|
|
203
225
|
"job.wait": JobWaitResultSchema,
|
|
204
226
|
"job.monitor": JobWaitResultSchema,
|
|
@@ -224,7 +246,9 @@ export function typeBoxCommandOutputSchemas() {
|
|
|
224
246
|
},
|
|
225
247
|
],
|
|
226
248
|
},
|
|
227
|
-
// `*.list`
|
|
249
|
+
// `*.list` and the four large reads default to compact shapes; `--full`
|
|
250
|
+
// returns the DSS objects.
|
|
228
251
|
...compactListOutputSchemas(),
|
|
252
|
+
...compactReadOutputSchemas(),
|
|
229
253
|
};
|
|
230
254
|
}
|
package/dist/src/cli/output.js
CHANGED
|
@@ -187,6 +187,12 @@ export class CommandResultFailure extends Error {
|
|
|
187
187
|
export function commandFailureMessage(result) {
|
|
188
188
|
if (isFailedWaitResult(result)) {
|
|
189
189
|
const record = result;
|
|
190
|
+
const failure = record.failure;
|
|
191
|
+
if (failure !== null && typeof failure === "object" && !Array.isArray(failure)) {
|
|
192
|
+
const message = failure.message;
|
|
193
|
+
if (typeof message === "string" && message.length > 0)
|
|
194
|
+
return message;
|
|
195
|
+
}
|
|
190
196
|
const state = typeof record.state === "string"
|
|
191
197
|
? record.state
|
|
192
198
|
: typeof record.outcome === "string"
|