dataiku-sdk 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -19
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +39 -4
- package/dist/packages/types/src/index.js +31 -2
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +66 -18
- package/dist/src/cli/commands/app.js +4 -3
- package/dist/src/cli/commands/bundle.js +51 -16
- package/dist/src/cli/commands/code-env.js +219 -57
- package/dist/src/cli/commands/code.d.ts +19 -0
- package/dist/src/cli/commands/code.js +87 -15
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +17 -11
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +42 -13
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +2 -1
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +3 -2
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +82 -50
- package/dist/src/cli/commands/project-git.js +8 -5
- package/dist/src/cli/commands/project-library.d.ts +21 -0
- package/dist/src/cli/commands/project-library.js +128 -56
- package/dist/src/cli/commands/project.js +19 -4
- package/dist/src/cli/commands/recipe.js +51 -25
- package/dist/src/cli/commands/saved-model.js +4 -3
- package/dist/src/cli/commands/scenario.js +9 -8
- package/dist/src/cli/commands/sql.d.ts +10 -0
- package/dist/src/cli/commands/sql.js +42 -19
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +68 -13
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +16 -1
- package/dist/src/cli/contract.js +967 -186
- package/dist/src/cli/env.d.ts +9 -0
- package/dist/src/cli/env.js +20 -2
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +29 -1
- package/dist/src/cli/helpers/cleanup.js +104 -3
- package/dist/src/cli/helpers/flow-zone.d.ts +3 -0
- package/dist/src/cli/helpers/flow-zone.js +33 -3
- package/dist/src/cli/helpers/recipe.d.ts +20 -0
- package/dist/src/cli/helpers/recipe.js +158 -3
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +33 -6
- package/dist/src/cli/runtime.d.ts +35 -1
- package/dist/src/cli/runtime.js +125 -18
- package/dist/src/cli/types.d.ts +7 -0
- package/dist/src/cli.js +226 -95
- package/dist/src/client.d.ts +63 -1
- package/dist/src/client.js +305 -53
- package/dist/src/errors.d.ts +8 -2
- package/dist/src/errors.js +42 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +2 -1
- package/dist/src/resources/api-services.d.ts +48 -6
- package/dist/src/resources/api-services.js +54 -8
- package/dist/src/resources/bundles.d.ts +51 -6
- package/dist/src/resources/bundles.js +39 -9
- package/dist/src/resources/code-envs.d.ts +25 -3
- package/dist/src/resources/code-envs.js +104 -2
- package/dist/src/resources/datasets.d.ts +20 -0
- package/dist/src/resources/datasets.js +232 -52
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/jobs.d.ts +35 -10
- package/dist/src/resources/jobs.js +92 -17
- package/dist/src/resources/notebooks.d.ts +77 -7
- package/dist/src/resources/notebooks.js +143 -19
- package/dist/src/resources/project-git.d.ts +12 -0
- package/dist/src/resources/project-git.js +39 -8
- package/dist/src/resources/project-library.d.ts +122 -7
- package/dist/src/resources/project-library.js +494 -25
- package/dist/src/resources/projects.d.ts +5 -4
- package/dist/src/resources/projects.js +21 -19
- package/dist/src/resources/scenarios.d.ts +30 -0
- package/dist/src/resources/scenarios.js +73 -27
- package/dist/src/resources/sql.d.ts +6 -0
- package/dist/src/resources/sql.js +23 -3
- package/dist/src/resources/webapps.d.ts +35 -4
- package/dist/src/resources/webapps.js +54 -5
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +18 -0
- package/dist/src/skill.js +93 -9
- package/dist/src/utils/flow-analysis.d.ts +53 -0
- package/dist/src/utils/flow-analysis.js +382 -0
- package/dist/src/utils/project-archive.d.ts +41 -2
- package/dist/src/utils/project-archive.js +159 -16
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/package.json +9 -7
- package/packages/types/dist/index.d.ts +39 -4
- package/packages/types/dist/index.js +31 -2
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +44 -197
- package/skills/dataiku-dss/references/app-releases.md +27 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
package/dist/src/cli/env.d.ts
CHANGED
|
@@ -1,2 +1,11 @@
|
|
|
1
|
+
export type EnvFileProvenance = "project" | "install";
|
|
2
|
+
/**
|
|
3
|
+
* Provenance of an environment variable injected by loadEnvFile:
|
|
4
|
+
* - "project": set from the invocation directory's `.env` (working tree, untrusted)
|
|
5
|
+
* - "install": set from the CLI install/root directory's `.env` (shipped defaults)
|
|
6
|
+
* Native environment variables (present before loadEnvFile runs) are not
|
|
7
|
+
* recorded here and resolve to their default source.
|
|
8
|
+
*/
|
|
9
|
+
export declare function envFileProvenance(name: string): EnvFileProvenance | undefined;
|
|
1
10
|
export declare function dataikuEnvironmentEnabled(): boolean;
|
|
2
11
|
export declare function loadEnvFile(): void;
|
package/dist/src/cli/env.js
CHANGED
|
@@ -1,6 +1,22 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
2
|
import { dirname, resolve, } from "node:path";
|
|
3
3
|
import { fileURLToPath, } from "node:url";
|
|
4
|
+
const projectEnvVars = new Set();
|
|
5
|
+
const installEnvVars = new Set();
|
|
6
|
+
/**
|
|
7
|
+
* Provenance of an environment variable injected by loadEnvFile:
|
|
8
|
+
* - "project": set from the invocation directory's `.env` (working tree, untrusted)
|
|
9
|
+
* - "install": set from the CLI install/root directory's `.env` (shipped defaults)
|
|
10
|
+
* Native environment variables (present before loadEnvFile runs) are not
|
|
11
|
+
* recorded here and resolve to their default source.
|
|
12
|
+
*/
|
|
13
|
+
export function envFileProvenance(name) {
|
|
14
|
+
if (projectEnvVars.has(name))
|
|
15
|
+
return "project";
|
|
16
|
+
if (installEnvVars.has(name))
|
|
17
|
+
return "install";
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
4
20
|
export function dataikuEnvironmentEnabled() {
|
|
5
21
|
return process.env.DATAIKU_DISABLE_ENV !== "1";
|
|
6
22
|
}
|
|
@@ -14,7 +30,7 @@ export function loadEnvFile() {
|
|
|
14
30
|
process.cwd(),
|
|
15
31
|
resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."),
|
|
16
32
|
];
|
|
17
|
-
for (const dir of dirs) {
|
|
33
|
+
for (const [index, dir,] of dirs.entries()) {
|
|
18
34
|
try {
|
|
19
35
|
const content = readFileSync(resolve(dir, ".env"), "utf-8");
|
|
20
36
|
for (const line of content.split("\n")) {
|
|
@@ -26,8 +42,10 @@ export function loadEnvFile() {
|
|
|
26
42
|
continue;
|
|
27
43
|
const key = trimmed.slice(0, eq).trim();
|
|
28
44
|
const val = trimmed.slice(eq + 1).trim().replace(/^['"]|['"]$/g, "");
|
|
29
|
-
if (!process.env[key])
|
|
45
|
+
if (!process.env[key]) {
|
|
30
46
|
process.env[key] = val;
|
|
47
|
+
(index === 0 ? projectEnvVars : installEnvVars).add(key);
|
|
48
|
+
}
|
|
31
49
|
}
|
|
32
50
|
}
|
|
33
51
|
catch {
|
package/dist/src/cli/flags.d.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
export declare function executionMode(flags: Record<string, string | boolean>): {
|
|
2
|
+
readonly plan: boolean;
|
|
3
|
+
readonly dryRun: boolean;
|
|
4
|
+
};
|
|
1
5
|
export declare const BOOLEAN_FLAGS: Set<string>;
|
|
2
6
|
export declare const SHORT_FLAGS: Record<string, string>;
|
|
3
7
|
/** Long-flag aliases: these are normalized to the canonical name in parseArgs. */
|
package/dist/src/cli/flags.js
CHANGED
|
@@ -1,4 +1,17 @@
|
|
|
1
|
+
import { parseBooleanOption, } from "./coerce.js";
|
|
1
2
|
import { unsupportedHelpFlag, UsageError, } from "./usage.js";
|
|
3
|
+
/** Planning wins dispatch; dryRun remains set for combined plan metadata. */
|
|
4
|
+
const EXECUTION_MODES = [
|
|
5
|
+
Object.freeze({ plan: false, dryRun: false, }),
|
|
6
|
+
Object.freeze({ plan: false, dryRun: true, }),
|
|
7
|
+
Object.freeze({ plan: true, dryRun: false, }),
|
|
8
|
+
Object.freeze({ plan: true, dryRun: true, }),
|
|
9
|
+
];
|
|
10
|
+
export function executionMode(flags) {
|
|
11
|
+
const plan = parseBooleanOption(flags["plan"], "--plan") ?? false;
|
|
12
|
+
const dryRun = parseBooleanOption(flags["dry-run"], "--dry-run") ?? false;
|
|
13
|
+
return EXECUTION_MODES[(plan ? 2 : 0) + (dryRun ? 1 : 0)];
|
|
14
|
+
}
|
|
2
15
|
export const BOOLEAN_FLAGS = new Set([
|
|
3
16
|
"verbose",
|
|
4
17
|
"version",
|
|
@@ -46,6 +59,8 @@ export const BOOLEAN_FLAGS = new Set([
|
|
|
46
59
|
"no-add-to-python-path",
|
|
47
60
|
"delete-directory",
|
|
48
61
|
"peek",
|
|
62
|
+
"raw-data",
|
|
63
|
+
"all",
|
|
49
64
|
]);
|
|
50
65
|
export const SHORT_FLAGS = {
|
|
51
66
|
v: "verbose",
|
|
@@ -96,6 +111,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
96
111
|
"env-version",
|
|
97
112
|
"expect-hash",
|
|
98
113
|
"expect-project-incarnation",
|
|
114
|
+
"expect-sha256",
|
|
99
115
|
"expected",
|
|
100
116
|
"expect-version",
|
|
101
117
|
"data-file",
|
|
@@ -105,6 +121,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
105
121
|
"file-name",
|
|
106
122
|
"file-type",
|
|
107
123
|
"env",
|
|
124
|
+
"evaluate-standards-checks",
|
|
108
125
|
"install-core-packages",
|
|
109
126
|
"fuzzy-on",
|
|
110
127
|
"fuzzy-distance",
|
|
@@ -121,6 +138,10 @@ export const VALUE_FLAGS = new Set([
|
|
|
121
138
|
"join-type",
|
|
122
139
|
"package",
|
|
123
140
|
"packages",
|
|
141
|
+
"published-service-id",
|
|
142
|
+
"published-project-key",
|
|
143
|
+
"release-notes",
|
|
144
|
+
"scenarios",
|
|
124
145
|
"local",
|
|
125
146
|
"login",
|
|
126
147
|
"manifest-version",
|
|
@@ -156,6 +177,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
156
177
|
"path",
|
|
157
178
|
"path-in-repository",
|
|
158
179
|
"preview",
|
|
180
|
+
"render",
|
|
159
181
|
"prediction-type",
|
|
160
182
|
"project-key",
|
|
161
183
|
"recipe",
|
|
@@ -264,7 +286,13 @@ export function parseArgs(argv) {
|
|
|
264
286
|
if (eqIdx !== -1) {
|
|
265
287
|
const raw = arg.slice(2, eqIdx);
|
|
266
288
|
const flagName = normalizeLongFlag(raw);
|
|
267
|
-
|
|
289
|
+
const value = arg.slice(eqIdx + 1);
|
|
290
|
+
if (BOOLEAN_FLAGS.has(flagName)) {
|
|
291
|
+
flags[flagName] = parseBooleanOption(value, `--${flagName}`);
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
setParsedFlagValue(flags, flagName, value);
|
|
295
|
+
}
|
|
268
296
|
}
|
|
269
297
|
else {
|
|
270
298
|
const rawFlagName = arg.slice(2);
|
|
@@ -1,12 +1,51 @@
|
|
|
1
|
+
import { readFileSync, } from "node:fs";
|
|
1
2
|
import { resultRecord, stringField, } from "../coerce.js";
|
|
2
3
|
export function projectArg(projectKey) {
|
|
3
4
|
return projectKey ? ["--project-key", projectKey,] : [];
|
|
4
5
|
}
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
/** Snapshot of the create input for actions whose identifiers live in the JSON
|
|
7
|
+
* payload rather than positional args (e.g. `workspace.create`). Only
|
|
8
|
+
* `--data`/`--data-file` are inspected: restating `--stdin` would re-read a
|
|
9
|
+
* stream the command already consumed, so a stdin-backed create cannot pin
|
|
10
|
+
* identifiers here and the caller must skip it.
|
|
11
|
+
*/
|
|
12
|
+
function createInputFromFlags(flags) {
|
|
13
|
+
const text = typeof flags["data"] === "string"
|
|
14
|
+
? flags["data"]
|
|
15
|
+
: typeof flags["data-file"] === "string"
|
|
16
|
+
? (() => {
|
|
17
|
+
try {
|
|
18
|
+
return readFileSync(flags["data-file"], "utf-8");
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
})()
|
|
24
|
+
: undefined;
|
|
25
|
+
if (text === undefined)
|
|
26
|
+
return undefined;
|
|
27
|
+
try {
|
|
28
|
+
const parsed = JSON.parse(text);
|
|
29
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
|
|
30
|
+
? parsed
|
|
31
|
+
: undefined;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// An unparsable input cannot pin the resource: no entry, no guess.
|
|
8
35
|
return undefined;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function cleanupLedgerEntry(resource, action, args, flags, result, projectKey) {
|
|
9
39
|
const record = resultRecord(result);
|
|
40
|
+
// A notebook save is ledger-eligible only when it created the notebook:
|
|
41
|
+
// an update overwrote something that already existed, so there is nothing
|
|
42
|
+
// for cleanup to remove, and a dry run never mutated anything.
|
|
43
|
+
const isCreatedNotebookSave = (action === "save-jupyter" || action === "save-sql")
|
|
44
|
+
&& record.created === true;
|
|
45
|
+
if (!isCreatedNotebookSave
|
|
46
|
+
&& !(action.startsWith("create") || action === "clone" || action === "duplicate"
|
|
47
|
+
|| action === "import" || action === "upload"))
|
|
48
|
+
return undefined;
|
|
10
49
|
if (record.skipped !== undefined)
|
|
11
50
|
return undefined;
|
|
12
51
|
// Only an explicit `cleanupEligible:false` suppresses the ledger: absent or
|
|
@@ -183,6 +222,44 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
183
222
|
cleanup: { argv: ["code-env", "delete", lang, name, "--if-exists",], },
|
|
184
223
|
};
|
|
185
224
|
}
|
|
225
|
+
case "streaming-endpoint.create": {
|
|
226
|
+
const id = args[0];
|
|
227
|
+
return {
|
|
228
|
+
...base,
|
|
229
|
+
id,
|
|
230
|
+
cleanup: { argv: ["streaming-endpoint", "delete", id, ...withProject,], },
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
case "meaning.create": {
|
|
234
|
+
const id = args[0];
|
|
235
|
+
return {
|
|
236
|
+
...base,
|
|
237
|
+
id,
|
|
238
|
+
name: args[1],
|
|
239
|
+
cleanup: { argv: ["meaning", "delete", id, "--if-exists",], },
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
case "workspace.create": {
|
|
243
|
+
const input = resultRecord(createInputFromFlags(flags));
|
|
244
|
+
const workspaceKey = stringField(input, ["workspaceKey",]);
|
|
245
|
+
if (!workspaceKey)
|
|
246
|
+
return undefined;
|
|
247
|
+
return {
|
|
248
|
+
...base,
|
|
249
|
+
id: workspaceKey,
|
|
250
|
+
name: stringField(input, ["displayName",]),
|
|
251
|
+
cleanup: { argv: ["workspace", "delete", workspaceKey,], },
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
case "project-library.create-file":
|
|
255
|
+
case "project-library.create-folder": {
|
|
256
|
+
const path = args[0];
|
|
257
|
+
return {
|
|
258
|
+
...base,
|
|
259
|
+
path,
|
|
260
|
+
cleanup: { argv: ["project-library", "delete", path, ...withProject,], },
|
|
261
|
+
};
|
|
262
|
+
}
|
|
186
263
|
case "folder.upload":
|
|
187
264
|
return {
|
|
188
265
|
...base,
|
|
@@ -190,6 +267,30 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
190
267
|
path: args[1],
|
|
191
268
|
cleanup: { argv: ["folder", "delete-file", args[0], args[1], ...withProject,], },
|
|
192
269
|
};
|
|
270
|
+
case "notebook.save-jupyter": {
|
|
271
|
+
const name = args[0];
|
|
272
|
+
if (!name)
|
|
273
|
+
return undefined;
|
|
274
|
+
return {
|
|
275
|
+
...base,
|
|
276
|
+
name,
|
|
277
|
+
cleanup: {
|
|
278
|
+
argv: ["notebook", "delete-jupyter", name, "--if-exists", ...withProject,],
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
case "notebook.save-sql": {
|
|
283
|
+
const id = args[0];
|
|
284
|
+
if (!id)
|
|
285
|
+
return undefined;
|
|
286
|
+
return {
|
|
287
|
+
...base,
|
|
288
|
+
id,
|
|
289
|
+
cleanup: {
|
|
290
|
+
argv: ["notebook", "delete-sql", id, "--if-exists", ...withProject,],
|
|
291
|
+
},
|
|
292
|
+
};
|
|
293
|
+
}
|
|
193
294
|
case "project.import": {
|
|
194
295
|
const usedProjectKey = stringField(record, ["usedProjectKey",]);
|
|
195
296
|
const recordedIncarnation = stringField(record, ["projectIncarnationHash",]);
|
|
@@ -20,6 +20,7 @@ export interface FlowZoneOrganizeZonePlan {
|
|
|
20
20
|
items: FlowZoneItemInput[];
|
|
21
21
|
}
|
|
22
22
|
export interface FlowZoneOrganizePlan {
|
|
23
|
+
topologyFingerprint?: string;
|
|
23
24
|
zones: FlowZoneOrganizeZonePlan[];
|
|
24
25
|
}
|
|
25
26
|
export interface FlowZoneOrganizeValidationIssue {
|
|
@@ -56,6 +57,8 @@ export declare function readFlowZoneOrganizePlan(flags: Record<string, string |
|
|
|
56
57
|
export declare function findFlowZoneForPlan(zones: FlowZone[], plan: FlowZoneOrganizeZonePlan): FlowZone | undefined;
|
|
57
58
|
export declare function ensureFlowZonePlanTarget(plan: FlowZoneOrganizeZonePlan, existing: FlowZone | undefined): void;
|
|
58
59
|
export declare function flowZoneExplicitItems(zone: FlowZone): FlowZoneItemInput[];
|
|
60
|
+
export declare function flowZonePlanFromZones(zones: FlowZone[], topologyFingerprint: string): FlowZoneOrganizePlan;
|
|
61
|
+
export declare function flowZoneMoveDelta(existing: FlowZone | undefined, plannedItems: FlowZoneItemInput[]): FlowZoneItemInput[];
|
|
59
62
|
export declare function flowZonePruneItems(existing: FlowZone | undefined, plannedItemKeys: Set<string>): FlowZoneItemInput[];
|
|
60
63
|
export declare function flowZoneOrganizeStep(plan: FlowZoneOrganizeZonePlan, existing: FlowZone | undefined, sync: boolean, plannedItemKeys: Set<string>): Record<string, unknown>;
|
|
61
64
|
export declare function flowZoneValidationBucket(index: FlowZoneOrganizeValidationIndex, objectType: FlowZoneObjectType): Set<string>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
|
+
import { compareStrings, } from "../../utils/stable-hash.js";
|
|
2
3
|
import { finiteNumberField, jsonInput, optionalStringField, parseJsonObject, plainRecord, requiredStringArray, splitCsvFlag, } from "../coerce.js";
|
|
3
4
|
import { UsageError, } from "../usage.js";
|
|
4
5
|
export function flowZoneId(value) {
|
|
@@ -201,10 +202,15 @@ export function validateUniqueFlowZoneAssignments(plan) {
|
|
|
201
202
|
}
|
|
202
203
|
export function parseFlowZoneOrganizePlan(input) {
|
|
203
204
|
const zones = input.zones;
|
|
204
|
-
if (!Array.isArray(zones)
|
|
205
|
-
throw new UsageError("Flow zone organize plan must include a
|
|
205
|
+
if (!Array.isArray(zones)) {
|
|
206
|
+
throw new UsageError("Flow zone organize plan must include a zones array.", "validation_failed");
|
|
207
|
+
}
|
|
208
|
+
const topologyFingerprint = optionalStringField(input, ["topologyFingerprint",]);
|
|
209
|
+
if (topologyFingerprint !== undefined && !/^[0-9a-f]{64}$/.test(topologyFingerprint)) {
|
|
210
|
+
throw new UsageError("topologyFingerprint must be a lowercase SHA-256 hash.", "validation_failed");
|
|
206
211
|
}
|
|
207
212
|
const plan = {
|
|
213
|
+
...(topologyFingerprint ? { topologyFingerprint, } : {}),
|
|
208
214
|
zones: zones.map((value, index) => {
|
|
209
215
|
const source = `zones[${index}]`;
|
|
210
216
|
const record = plainRecord(value);
|
|
@@ -284,6 +290,29 @@ export function flowZoneExplicitItems(zone) {
|
|
|
284
290
|
...(item.projectKey ? { projectKey: item.projectKey, } : {}),
|
|
285
291
|
}));
|
|
286
292
|
}
|
|
293
|
+
export function flowZonePlanFromZones(zones, topologyFingerprint) {
|
|
294
|
+
return {
|
|
295
|
+
topologyFingerprint,
|
|
296
|
+
zones: [...zones,]
|
|
297
|
+
.sort((a, b) => compareStrings(a.id, b.id))
|
|
298
|
+
.map((zone) => {
|
|
299
|
+
const position = flowZoneCurrentPosition(zone);
|
|
300
|
+
return {
|
|
301
|
+
id: zone.id,
|
|
302
|
+
name: zone.name,
|
|
303
|
+
...(zone.color ? { color: zone.color, } : {}),
|
|
304
|
+
...(position ? { position, } : {}),
|
|
305
|
+
items: flowZoneExplicitItems(zone).sort((a, b) => compareStrings(flowZoneItemKey(a), flowZoneItemKey(b))),
|
|
306
|
+
};
|
|
307
|
+
}),
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
export function flowZoneMoveDelta(existing, plannedItems) {
|
|
311
|
+
if (!existing)
|
|
312
|
+
return plannedItems;
|
|
313
|
+
const existingKeys = new Set(flowZoneExplicitItems(existing).map(flowZoneItemKey));
|
|
314
|
+
return plannedItems.filter((item) => !existingKeys.has(flowZoneItemKey(item)));
|
|
315
|
+
}
|
|
287
316
|
export function flowZonePruneItems(existing, plannedItemKeys) {
|
|
288
317
|
if (!existing)
|
|
289
318
|
return [];
|
|
@@ -301,6 +330,7 @@ export function flowZoneOrganizeStep(plan, existing, sync, plannedItemKeys) {
|
|
|
301
330
|
update.position = plan.position;
|
|
302
331
|
}
|
|
303
332
|
const pruneItems = sync ? flowZonePruneItems(existing, plannedItemKeys) : [];
|
|
333
|
+
const moveItems = flowZoneMoveDelta(existing, plan.items);
|
|
304
334
|
return {
|
|
305
335
|
target: {
|
|
306
336
|
...(plan.id ? { id: plan.id, } : {}),
|
|
@@ -310,7 +340,7 @@ export function flowZoneOrganizeStep(plan, existing, sync, plannedItemKeys) {
|
|
|
310
340
|
},
|
|
311
341
|
...(existing ? { existing: flowZoneSummary(existing), } : { create: true, }),
|
|
312
342
|
...(Object.keys(update).length > 0 ? { update, } : {}),
|
|
313
|
-
moveItems
|
|
343
|
+
moveItems,
|
|
314
344
|
...(pruneItems.length > 0 ? { pruneItems, } : {}),
|
|
315
345
|
};
|
|
316
346
|
}
|
|
@@ -1,4 +1,24 @@
|
|
|
1
1
|
export declare function recipeBackupPath(recipeName: string, backupDir: string): string;
|
|
2
|
+
/**
|
|
3
|
+
* Prepare the backup directory as an owner-only, symlink-free directory.
|
|
4
|
+
*
|
|
5
|
+
* Every untrusted component is checked with `lstat` and must be a real
|
|
6
|
+
* directory — `mkdir { recursive: true, }` would silently traverse a symlink
|
|
7
|
+
* planted along the path (e.g. `.dss-backups` pointing at an attacker-chosen
|
|
8
|
+
* home) and create with ambient permissions. Missing components are created
|
|
9
|
+
* one at a time with mode 0700, and the final directory is clamped to 0700
|
|
10
|
+
* through a no-follow descriptor so the verification cannot be raced by a swap.
|
|
11
|
+
*/
|
|
12
|
+
export declare function ensureRecipeBackupDir(backupDir: string): void;
|
|
13
|
+
/**
|
|
14
|
+
* Write a recipe backup file as a brand-new owner-only file.
|
|
15
|
+
*
|
|
16
|
+
* `O_EXCL` refuses any pre-existing destination (including a planted symlink)
|
|
17
|
+
* instead of clobbering it, `O_NOFOLLOW` blocks a symlink destination, and the
|
|
18
|
+
* file is created with mode 0600 — the payload must never sit on disk under
|
|
19
|
+
* ambient (umask-derived) permissions.
|
|
20
|
+
*/
|
|
21
|
+
export declare function writeRecipeBackup(backupPath: string, content: string): void;
|
|
2
22
|
export declare function recipeCodeEnv(recipe: Record<string, unknown>): unknown;
|
|
3
23
|
export declare function recipeGraph(recipe: Record<string, unknown>): Record<string, unknown>;
|
|
4
24
|
export declare function recipeBackupDocument(recipeName: string, projectKey: string | undefined, current: {
|
|
@@ -1,12 +1,167 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import { tmpdir, } from "node:os";
|
|
3
|
+
import { join, parse as parsePath, relative as relativePath, resolve as resolvePath, sep, } from "node:path";
|
|
3
4
|
import { sanitizeFileName, } from "../../utils/sanitize.js";
|
|
4
5
|
import { normalizeLineEndings, sha256Hex, stableHash, } from "../coerce.js";
|
|
5
6
|
import { UsageError, } from "../usage.js";
|
|
7
|
+
/** Backups may embed recipe payloads with secrets: keep them owner-only. */
|
|
8
|
+
const RECIPE_BACKUP_DIR_MODE = 0o700;
|
|
9
|
+
const RECIPE_BACKUP_FILE_MODE = 0o600;
|
|
10
|
+
/**
|
|
11
|
+
* Backup files are created exclusively — an existing destination is a conflict
|
|
12
|
+
* (a stale file, or a pre-planted symlink), never something to overwrite — and
|
|
13
|
+
* O_NOFOLLOW refuses a symlink at the destination itself.
|
|
14
|
+
*/
|
|
15
|
+
const RECIPE_BACKUP_FILE_OPEN_FLAGS = fs.constants.O_WRONLY | fs.constants.O_CREAT
|
|
16
|
+
| fs.constants.O_EXCL
|
|
17
|
+
| (typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0);
|
|
18
|
+
/** Verify the backup directory through a descriptor that rejects symlinking. */
|
|
19
|
+
const RECIPE_BACKUP_DIR_OPEN_FLAGS = fs.constants.O_RDONLY
|
|
20
|
+
| (typeof fs.constants.O_DIRECTORY === "number" ? fs.constants.O_DIRECTORY : 0)
|
|
21
|
+
| (typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0);
|
|
6
22
|
export function recipeBackupPath(recipeName, backupDir) {
|
|
7
23
|
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
8
24
|
return join(backupDir, `${sanitizeFileName(recipeName, "recipe")}-${stamp}.recipe-backup.json`);
|
|
9
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Split a backup directory into a trusted base and the untrusted tail that is
|
|
28
|
+
* verified component by component.
|
|
29
|
+
*
|
|
30
|
+
* Only two prefixes are trusted, and only after canonicalisation: the
|
|
31
|
+
* invocation directory (the CLI already reads its `.env`) and the OS temp
|
|
32
|
+
* directory. Canonicalising absorbs OS-level links the user never named —
|
|
33
|
+
* macOS `/var -> /private/var`, Windows 8.3 temp aliases — while every
|
|
34
|
+
* component the user's path introduces beneath that prefix is still walked
|
|
35
|
+
* with `lstat`, which is where a planted `.dss-backups` symlink lives. A path
|
|
36
|
+
* under neither prefix is walked in full from its volume root.
|
|
37
|
+
*/
|
|
38
|
+
function splitBackupDir(absolute) {
|
|
39
|
+
for (const trusted of [process.cwd(), tmpdir(),]) {
|
|
40
|
+
let canonical;
|
|
41
|
+
try {
|
|
42
|
+
canonical = fs.realpathSync(trusted);
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const literal = resolvePath(trusted);
|
|
48
|
+
const bases = literal === canonical ? [canonical,] : [literal, canonical,];
|
|
49
|
+
for (const candidateBase of bases) {
|
|
50
|
+
const rel = relativePath(candidateBase, absolute);
|
|
51
|
+
if (rel === "" || rel.startsWith("..") || parsePath(rel).root)
|
|
52
|
+
continue;
|
|
53
|
+
return { base: canonical, tail: rel.split(sep).filter((part) => part.length > 0), };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const { root, } = parsePath(absolute);
|
|
57
|
+
return {
|
|
58
|
+
base: root,
|
|
59
|
+
tail: relativePath(root, absolute).split(sep).filter((part) => part.length > 0),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Prepare the backup directory as an owner-only, symlink-free directory.
|
|
64
|
+
*
|
|
65
|
+
* Every untrusted component is checked with `lstat` and must be a real
|
|
66
|
+
* directory — `mkdir { recursive: true, }` would silently traverse a symlink
|
|
67
|
+
* planted along the path (e.g. `.dss-backups` pointing at an attacker-chosen
|
|
68
|
+
* home) and create with ambient permissions. Missing components are created
|
|
69
|
+
* one at a time with mode 0700, and the final directory is clamped to 0700
|
|
70
|
+
* through a no-follow descriptor so the verification cannot be raced by a swap.
|
|
71
|
+
*/
|
|
72
|
+
export function ensureRecipeBackupDir(backupDir) {
|
|
73
|
+
const absolute = resolvePath(backupDir);
|
|
74
|
+
if (parsePath(absolute).root === absolute) {
|
|
75
|
+
throw new UsageError(`Recipe backup directory "${backupDir}" is the filesystem root. Choose a dedicated backup directory.`, "usage_error");
|
|
76
|
+
}
|
|
77
|
+
const { base, tail, } = splitBackupDir(absolute);
|
|
78
|
+
let current = base;
|
|
79
|
+
for (const part of tail) {
|
|
80
|
+
current = join(current, part);
|
|
81
|
+
let stats;
|
|
82
|
+
try {
|
|
83
|
+
stats = fs.lstatSync(current);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
fs.mkdirSync(current, { mode: RECIPE_BACKUP_DIR_MODE, });
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (stats.isSymbolicLink()) {
|
|
90
|
+
throw new UsageError(`Recipe backup directory "${backupDir}" traverses the symlink "${current}". Refusing to write backups through a symlink.`, "usage_error");
|
|
91
|
+
}
|
|
92
|
+
if (!stats.isDirectory()) {
|
|
93
|
+
throw new UsageError(`Recipe backup path component "${current}" is not a directory.`, "usage_error");
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const target = current;
|
|
97
|
+
let dirFd;
|
|
98
|
+
try {
|
|
99
|
+
dirFd = fs.openSync(target, RECIPE_BACKUP_DIR_OPEN_FLAGS);
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
const code = error.code;
|
|
103
|
+
if (code === "ELOOP" || code === "ENOTDIR") {
|
|
104
|
+
throw new UsageError(`Recipe backup directory "${backupDir}" is not a real, non-symlinked directory.`, "usage_error");
|
|
105
|
+
}
|
|
106
|
+
throw error;
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
// Windows has no POSIX directory modes and fchmod on a directory
|
|
110
|
+
// descriptor fails with EPERM; the owner-only guarantee there comes from
|
|
111
|
+
// the user profile ACLs, so only the identity verification below applies.
|
|
112
|
+
if (process.platform !== "win32")
|
|
113
|
+
fs.fchmodSync(dirFd, RECIPE_BACKUP_DIR_MODE);
|
|
114
|
+
const verified = fs.fstatSync(dirFd);
|
|
115
|
+
if (!verified.isDirectory()) {
|
|
116
|
+
throw new UsageError(`Recipe backup path "${target}" is not a directory.`, "usage_error");
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
fs.closeSync(dirFd);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/** Drain a backup payload: one `writeSync` may cover only part of the buffer. */
|
|
124
|
+
function writeAllSync(fd, payload) {
|
|
125
|
+
let offset = 0;
|
|
126
|
+
while (offset < payload.length) {
|
|
127
|
+
const written = fs.writeSync(fd, payload, offset, payload.length - offset, offset);
|
|
128
|
+
if (written <= 0) {
|
|
129
|
+
throw new Error(`Stalled writing recipe backup at byte ${offset}.`);
|
|
130
|
+
}
|
|
131
|
+
offset += written;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Write a recipe backup file as a brand-new owner-only file.
|
|
136
|
+
*
|
|
137
|
+
* `O_EXCL` refuses any pre-existing destination (including a planted symlink)
|
|
138
|
+
* instead of clobbering it, `O_NOFOLLOW` blocks a symlink destination, and the
|
|
139
|
+
* file is created with mode 0600 — the payload must never sit on disk under
|
|
140
|
+
* ambient (umask-derived) permissions.
|
|
141
|
+
*/
|
|
142
|
+
export function writeRecipeBackup(backupPath, content) {
|
|
143
|
+
const payload = Buffer.from(content, "utf-8");
|
|
144
|
+
let fd;
|
|
145
|
+
try {
|
|
146
|
+
fd = fs.openSync(backupPath, RECIPE_BACKUP_FILE_OPEN_FLAGS, RECIPE_BACKUP_FILE_MODE);
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
const code = error.code;
|
|
150
|
+
if (code === "EEXIST" || code === "ELOOP") {
|
|
151
|
+
throw new UsageError(`Recipe backup destination already exists: ${backupPath}. Refusing to overwrite an existing path.`, "usage_error");
|
|
152
|
+
}
|
|
153
|
+
throw error;
|
|
154
|
+
}
|
|
155
|
+
try {
|
|
156
|
+
// O_EXCL guarantees a fresh inode; clamp the mode so the payload is never
|
|
157
|
+
// published under wider permissions even if creation flags change.
|
|
158
|
+
fs.fchmodSync(fd, RECIPE_BACKUP_FILE_MODE);
|
|
159
|
+
writeAllSync(fd, payload);
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
fs.closeSync(fd);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
10
165
|
export function recipeCodeEnv(recipe) {
|
|
11
166
|
const params = recipe.params;
|
|
12
167
|
if (!params || typeof params !== "object" || Array.isArray(params))
|
|
@@ -36,7 +191,7 @@ export function recipeBackupDocument(recipeName, projectKey, current) {
|
|
|
36
191
|
};
|
|
37
192
|
}
|
|
38
193
|
export function readRecipeBackup(backupPath) {
|
|
39
|
-
const raw = readFileSync(backupPath, "utf-8");
|
|
194
|
+
const raw = fs.readFileSync(backupPath, "utf-8");
|
|
40
195
|
try {
|
|
41
196
|
const parsed = JSON.parse(raw);
|
|
42
197
|
if (parsed && typeof parsed === "object" && parsed.resource === "recipe")
|
package/dist/src/cli/output.d.ts
CHANGED
|
@@ -38,6 +38,7 @@ export declare function skipResult(resource: string, id: string, reason: "exists
|
|
|
38
38
|
export declare function planResult(resource: string, action: string, options: {
|
|
39
39
|
asyncKind: string;
|
|
40
40
|
endpoint?: string;
|
|
41
|
+
exact?: boolean;
|
|
41
42
|
exitCodesOnFailure: Record<string, number>;
|
|
42
43
|
identifiers?: Record<string, unknown>;
|
|
43
44
|
idempotency: string;
|
|
@@ -45,6 +46,7 @@ export declare function planResult(resource: string, action: string, options: {
|
|
|
45
46
|
payload?: unknown;
|
|
46
47
|
localWrites?: unknown;
|
|
47
48
|
plannedAndDryRun?: boolean;
|
|
49
|
+
reason?: string;
|
|
48
50
|
requests?: unknown;
|
|
49
51
|
wait?: unknown;
|
|
50
52
|
}): Record<string, unknown>;
|
package/dist/src/cli/output.js
CHANGED
|
@@ -23,11 +23,16 @@ export function flushCliWarnings() {
|
|
|
23
23
|
function resolveFieldPath(source, field) {
|
|
24
24
|
let current = source;
|
|
25
25
|
for (const segment of field.split(".")) {
|
|
26
|
-
if (current === null || typeof current !== "object" || Array.isArray(current))
|
|
27
|
-
return null;
|
|
28
|
-
|
|
26
|
+
if (current === null || typeof current !== "object" || Array.isArray(current)) {
|
|
27
|
+
return { found: false, value: null, };
|
|
28
|
+
}
|
|
29
|
+
const record = current;
|
|
30
|
+
if (!Object.prototype.hasOwnProperty.call(record, segment)) {
|
|
31
|
+
return { found: false, value: null, };
|
|
32
|
+
}
|
|
33
|
+
current = record[segment];
|
|
29
34
|
}
|
|
30
|
-
return current ?? null;
|
|
35
|
+
return { found: true, value: current ?? null, };
|
|
31
36
|
}
|
|
32
37
|
export function pickResultFields(item, fields) {
|
|
33
38
|
if (!item || typeof item !== "object" || Array.isArray(item))
|
|
@@ -35,24 +40,44 @@ export function pickResultFields(item, fields) {
|
|
|
35
40
|
const source = item;
|
|
36
41
|
const picked = {};
|
|
37
42
|
for (const field of fields)
|
|
38
|
-
picked[field] = resolveFieldPath(source, field);
|
|
43
|
+
picked[field] = resolveFieldPath(source, field).value;
|
|
39
44
|
return picked;
|
|
40
45
|
}
|
|
46
|
+
function projectionRecords(result) {
|
|
47
|
+
const items = Array.isArray(result) ? result : [result,];
|
|
48
|
+
return items.filter((item) => item !== null && typeof item === "object" && !Array.isArray(item));
|
|
49
|
+
}
|
|
50
|
+
function warnUnknownProjectionFields(result, fields) {
|
|
51
|
+
const records = projectionRecords(result);
|
|
52
|
+
if (records.length === 0)
|
|
53
|
+
return;
|
|
54
|
+
const unknownFields = fields.filter((field) => !records.some((record) => resolveFieldPath(record, field).found));
|
|
55
|
+
if (unknownFields.length === 0)
|
|
56
|
+
return;
|
|
57
|
+
const availableFields = [...new Set(records.flatMap((record) => Object.keys(record))),].sort();
|
|
58
|
+
enqueueCliWarning({
|
|
59
|
+
code: "field_projection_missing",
|
|
60
|
+
fields: unknownFields,
|
|
61
|
+
availableFields,
|
|
62
|
+
hint: "Use fields from the command's schemas.output contract or an unprojected result.",
|
|
63
|
+
});
|
|
64
|
+
}
|
|
41
65
|
/**
|
|
42
66
|
* Project the top-level fields callers asked for via --fields. Arrays are mapped
|
|
43
67
|
* element-wise; scalars and string results pass through untouched. Requested keys
|
|
44
68
|
* that are absent become null so every row keeps a stable, predictable shape.
|
|
45
69
|
*/
|
|
46
70
|
export function projectResultFields(result, fields) {
|
|
71
|
+
warnUnknownProjectionFields(result, fields);
|
|
47
72
|
if (Array.isArray(result))
|
|
48
73
|
return result.map((item) => pickResultFields(item, fields));
|
|
49
74
|
return pickResultFields(result, fields);
|
|
50
75
|
}
|
|
51
76
|
export function writeCommandResult(result) {
|
|
52
|
-
flushCliWarnings();
|
|
53
77
|
const projected = outputFieldProjection
|
|
54
78
|
? projectResultFields(result, outputFieldProjection)
|
|
55
79
|
: result;
|
|
80
|
+
flushCliWarnings();
|
|
56
81
|
process.stdout.write(`${JSON.stringify(projected ?? { ok: true, })}\n`);
|
|
57
82
|
}
|
|
58
83
|
export function transientBodyWithTargetContext(body, target, elapsedMs) {
|
|
@@ -187,6 +212,8 @@ export function planResult(resource, action, options) {
|
|
|
187
212
|
action,
|
|
188
213
|
resource,
|
|
189
214
|
...(options.plannedAndDryRun ? { plannedAndDryRun: true, } : {}),
|
|
215
|
+
...(options.exact !== undefined ? { exact: options.exact, } : {}),
|
|
216
|
+
...(options.reason !== undefined ? { reason: options.reason, } : {}),
|
|
190
217
|
...options.identifiers,
|
|
191
218
|
...(options.method ? { method: options.method, } : {}),
|
|
192
219
|
...(options.endpoint ? { endpoint: options.endpoint, } : {}),
|