dataiku-sdk 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +107 -17
- package/README.md +144 -17
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +39 -4
- package/dist/packages/types/src/index.js +31 -2
- package/dist/src/cli/commands/api-service.js +62 -16
- package/dist/src/cli/commands/bundle.js +49 -15
- package/dist/src/cli/commands/code-env.js +210 -49
- package/dist/src/cli/commands/code.d.ts +19 -0
- package/dist/src/cli/commands/code.js +87 -15
- package/dist/src/cli/commands/dataset.js +8 -3
- package/dist/src/cli/commands/flow-zone.js +33 -5
- package/dist/src/cli/commands/notebook.js +72 -41
- package/dist/src/cli/commands/project-git.js +8 -5
- package/dist/src/cli/commands/project-library.d.ts +21 -0
- package/dist/src/cli/commands/project-library.js +121 -50
- package/dist/src/cli/commands/project.js +16 -2
- package/dist/src/cli/commands/recipe.js +37 -12
- package/dist/src/cli/commands/sql.d.ts +10 -0
- package/dist/src/cli/commands/sql.js +39 -17
- package/dist/src/cli/commands/webapp.js +67 -13
- package/dist/src/cli/contract.d.ts +15 -0
- package/dist/src/cli/contract.js +870 -107
- package/dist/src/cli/env.d.ts +9 -0
- package/dist/src/cli/env.js +20 -2
- package/dist/src/cli/flags.js +9 -0
- package/dist/src/cli/helpers/cleanup.js +104 -3
- package/dist/src/cli/helpers/flow-zone.d.ts +3 -0
- package/dist/src/cli/helpers/flow-zone.js +33 -3
- package/dist/src/cli/helpers/recipe.d.ts +20 -0
- package/dist/src/cli/helpers/recipe.js +158 -3
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +33 -6
- package/dist/src/cli/runtime.d.ts +35 -1
- package/dist/src/cli/runtime.js +125 -18
- package/dist/src/cli/types.d.ts +7 -0
- package/dist/src/cli.js +135 -28
- package/dist/src/client.d.ts +60 -0
- package/dist/src/client.js +237 -42
- package/dist/src/errors.d.ts +8 -2
- package/dist/src/errors.js +40 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +2 -1
- package/dist/src/resources/api-services.d.ts +48 -6
- package/dist/src/resources/api-services.js +54 -8
- package/dist/src/resources/bundles.d.ts +51 -6
- package/dist/src/resources/bundles.js +39 -9
- package/dist/src/resources/code-envs.d.ts +25 -3
- package/dist/src/resources/code-envs.js +104 -2
- package/dist/src/resources/datasets.d.ts +20 -0
- package/dist/src/resources/datasets.js +232 -52
- package/dist/src/resources/jobs.d.ts +35 -10
- package/dist/src/resources/jobs.js +92 -17
- package/dist/src/resources/notebooks.d.ts +77 -7
- package/dist/src/resources/notebooks.js +143 -19
- package/dist/src/resources/project-git.d.ts +12 -0
- package/dist/src/resources/project-git.js +39 -8
- package/dist/src/resources/project-library.d.ts +122 -7
- package/dist/src/resources/project-library.js +494 -25
- package/dist/src/resources/projects.d.ts +5 -2
- package/dist/src/resources/projects.js +18 -2
- package/dist/src/resources/scenarios.d.ts +30 -0
- package/dist/src/resources/scenarios.js +73 -27
- package/dist/src/resources/sql.d.ts +6 -0
- package/dist/src/resources/sql.js +22 -2
- package/dist/src/resources/webapps.d.ts +35 -4
- package/dist/src/resources/webapps.js +54 -5
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +9 -0
- package/dist/src/skill.js +37 -4
- package/dist/src/utils/flow-analysis.d.ts +53 -0
- package/dist/src/utils/flow-analysis.js +382 -0
- package/dist/src/utils/project-archive.d.ts +41 -2
- package/dist/src/utils/project-archive.js +159 -16
- package/package.json +5 -5
- package/packages/types/dist/index.d.ts +39 -4
- package/packages/types/dist/index.js +31 -2
- package/plugin.json +2 -2
- package/skills/dataiku-dss/SKILL.md +75 -48
package/dist/src/cli/runtime.js
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
import { loadCredentials, } from "../config.js";
|
|
2
|
-
import { dataikuEnvironmentEnabled, } from "./env.js";
|
|
2
|
+
import { dataikuEnvironmentEnabled, envFileProvenance, } from "./env.js";
|
|
3
|
+
import { UsageError, } from "./usage.js";
|
|
4
|
+
function envVarSource(name) {
|
|
5
|
+
return envFileProvenance(name) ?? "env";
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Reject an endpoint (URL/TLS) taken from the project `.env` when the API key
|
|
9
|
+
* would come from anywhere else: project-supplied endpoints must not harvest a
|
|
10
|
+
* user key and transmit it. A project `.env` that supplies the URL and the key
|
|
11
|
+
* together stays supported, as does `DATAIKU_DISABLE_ENV=1`, which stops the
|
|
12
|
+
* loader from recording any project provenance.
|
|
13
|
+
*/
|
|
14
|
+
export function assertCredentialBinding(sources) {
|
|
15
|
+
if (sources.apiKey === undefined || sources.apiKey === "project")
|
|
16
|
+
return;
|
|
17
|
+
const projectEndpoint = sources.url === "project"
|
|
18
|
+
|| sources.tlsRejectUnauthorized === "project"
|
|
19
|
+
|| sources.caCertPath === "project";
|
|
20
|
+
if (!projectEndpoint)
|
|
21
|
+
return;
|
|
22
|
+
throw new UsageError("Refusing to pair credentials: the DSS URL/TLS configuration comes from the project .env, but the API key comes from another source.", "conflicting_input_sources", "Supply both DATAIKU_URL and DATAIKU_API_KEY in the project .env, pass --url/--api-key explicitly, or set DATAIKU_DISABLE_ENV=1 to ignore the project .env.", {
|
|
23
|
+
url: sources.url,
|
|
24
|
+
apiKey: sources.apiKey,
|
|
25
|
+
...(sources.tlsRejectUnauthorized === "project"
|
|
26
|
+
? { tlsRejectUnauthorized: sources.tlsRejectUnauthorized, }
|
|
27
|
+
: {}),
|
|
28
|
+
...(sources.caCertPath === "project"
|
|
29
|
+
? { caCertPath: sources.caCertPath, }
|
|
30
|
+
: {}),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
3
33
|
function parseTlsRejectUnauthorizedEnv(value) {
|
|
4
34
|
if (value === undefined)
|
|
5
35
|
return undefined;
|
|
@@ -10,15 +40,38 @@ function parseTlsRejectUnauthorizedEnv(value) {
|
|
|
10
40
|
return true;
|
|
11
41
|
return undefined;
|
|
12
42
|
}
|
|
13
|
-
export function resolveTlsSettings(flags, saved) {
|
|
43
|
+
export function resolveTlsSettings(flags, saved, sources = {}) {
|
|
14
44
|
let tlsRejectUnauthorized = flags["insecure"] === true ? false : undefined;
|
|
15
45
|
let caCertPath = flags["ca-cert"];
|
|
16
|
-
|
|
17
|
-
|
|
46
|
+
if (flags["insecure"] === true)
|
|
47
|
+
sources.tlsRejectUnauthorized = "flag";
|
|
48
|
+
if (caCertPath !== undefined)
|
|
49
|
+
sources.caCertPath = "flag";
|
|
50
|
+
if (tlsRejectUnauthorized === undefined) {
|
|
51
|
+
const envValue = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
52
|
+
const parsed = parseTlsRejectUnauthorizedEnv(envValue);
|
|
53
|
+
if (envValue !== undefined && parsed !== undefined) {
|
|
54
|
+
tlsRejectUnauthorized = parsed;
|
|
55
|
+
sources.tlsRejectUnauthorized = envVarSource("NODE_TLS_REJECT_UNAUTHORIZED");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (caCertPath === undefined) {
|
|
59
|
+
const envCaCert = process.env.NODE_EXTRA_CA_CERTS;
|
|
60
|
+
if (envCaCert !== undefined) {
|
|
61
|
+
caCertPath = envCaCert;
|
|
62
|
+
sources.caCertPath = envVarSource("NODE_EXTRA_CA_CERTS");
|
|
63
|
+
}
|
|
64
|
+
}
|
|
18
65
|
if (tlsRejectUnauthorized === undefined) {
|
|
19
66
|
tlsRejectUnauthorized = saved?.tlsRejectUnauthorized;
|
|
67
|
+
if (tlsRejectUnauthorized !== undefined)
|
|
68
|
+
sources.tlsRejectUnauthorized = "saved";
|
|
69
|
+
}
|
|
70
|
+
if (caCertPath === undefined) {
|
|
71
|
+
caCertPath = saved?.caCertPath;
|
|
72
|
+
if (caCertPath !== undefined)
|
|
73
|
+
sources.caCertPath = "saved";
|
|
20
74
|
}
|
|
21
|
-
caCertPath ??= saved?.caCertPath;
|
|
22
75
|
return { tlsRejectUnauthorized, caCertPath, };
|
|
23
76
|
}
|
|
24
77
|
export const currentCommandContext = {};
|
|
@@ -32,28 +85,82 @@ export function resolveCredentials(flags) {
|
|
|
32
85
|
let url = hasUrlFlag ? flags["url"] : undefined;
|
|
33
86
|
let apiKey = hasApiKeyFlag ? flags["api-key"] : undefined;
|
|
34
87
|
let projectKey = hasProjectKeyFlag ? flags["project-key"] : undefined;
|
|
88
|
+
const sources = {};
|
|
89
|
+
if (hasUrlFlag)
|
|
90
|
+
sources.url = "flag";
|
|
91
|
+
if (hasApiKeyFlag)
|
|
92
|
+
sources.apiKey = "flag";
|
|
35
93
|
const saved = loadCredentials();
|
|
36
94
|
const useEnv = dataikuEnvironmentEnabled();
|
|
37
95
|
if (useEnv) {
|
|
38
|
-
if (!hasUrlFlag)
|
|
39
|
-
url
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if (!
|
|
43
|
-
|
|
96
|
+
if (!hasUrlFlag && process.env.DATAIKU_URL !== undefined) {
|
|
97
|
+
url = process.env.DATAIKU_URL;
|
|
98
|
+
sources.url = envVarSource("DATAIKU_URL");
|
|
99
|
+
}
|
|
100
|
+
if (!hasApiKeyFlag && process.env.DATAIKU_API_KEY !== undefined) {
|
|
101
|
+
apiKey = process.env.DATAIKU_API_KEY;
|
|
102
|
+
sources.apiKey = envVarSource("DATAIKU_API_KEY");
|
|
103
|
+
}
|
|
104
|
+
if (!hasProjectKeyFlag && process.env.DATAIKU_PROJECT_KEY !== undefined) {
|
|
105
|
+
projectKey = process.env.DATAIKU_PROJECT_KEY;
|
|
106
|
+
}
|
|
44
107
|
}
|
|
45
108
|
if (saved) {
|
|
46
|
-
if (!hasUrlFlag)
|
|
47
|
-
url
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (!
|
|
51
|
-
|
|
109
|
+
if (!hasUrlFlag && url === undefined) {
|
|
110
|
+
url = saved.url;
|
|
111
|
+
sources.url = "saved";
|
|
112
|
+
}
|
|
113
|
+
if (!hasApiKeyFlag && apiKey === undefined) {
|
|
114
|
+
apiKey = saved.apiKey;
|
|
115
|
+
sources.apiKey = "saved";
|
|
116
|
+
}
|
|
117
|
+
if (!hasProjectKeyFlag && projectKey === undefined) {
|
|
118
|
+
projectKey = saved.projectKey;
|
|
119
|
+
}
|
|
52
120
|
}
|
|
121
|
+
const tlsSettings = resolveTlsSettings(flags, saved ?? undefined, sources);
|
|
122
|
+
assertCredentialBinding(sources);
|
|
53
123
|
return {
|
|
54
124
|
url: url ?? "",
|
|
55
125
|
apiKey: apiKey ?? "",
|
|
56
126
|
projectKey,
|
|
57
|
-
...
|
|
127
|
+
...tlsSettings,
|
|
58
128
|
};
|
|
59
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* Resolve credentials for `auth login` from explicit flags and the process
|
|
132
|
+
* environment only — never from saved credentials — with provenance tracking
|
|
133
|
+
* and binding enforcement. The call throws before transmitting anything when
|
|
134
|
+
* the project `.env` supplies the URL/TLS configuration while the API key
|
|
135
|
+
* comes from another source.
|
|
136
|
+
*/
|
|
137
|
+
export function resolveLoginCredentials(flags) {
|
|
138
|
+
const hasUrlFlag = typeof flags["url"] === "string";
|
|
139
|
+
const hasApiKeyFlag = typeof flags["api-key"] === "string";
|
|
140
|
+
const hasProjectKeyFlag = typeof flags["project-key"] === "string";
|
|
141
|
+
let url = hasUrlFlag ? flags["url"] : undefined;
|
|
142
|
+
let apiKey = hasApiKeyFlag ? flags["api-key"] : undefined;
|
|
143
|
+
let projectKey = hasProjectKeyFlag ? flags["project-key"] : undefined;
|
|
144
|
+
const sources = {};
|
|
145
|
+
if (hasUrlFlag)
|
|
146
|
+
sources.url = "flag";
|
|
147
|
+
if (hasApiKeyFlag)
|
|
148
|
+
sources.apiKey = "flag";
|
|
149
|
+
const useEnv = dataikuEnvironmentEnabled();
|
|
150
|
+
if (useEnv) {
|
|
151
|
+
if (!hasUrlFlag && process.env.DATAIKU_URL !== undefined) {
|
|
152
|
+
url = process.env.DATAIKU_URL;
|
|
153
|
+
sources.url = envVarSource("DATAIKU_URL");
|
|
154
|
+
}
|
|
155
|
+
if (!hasApiKeyFlag && process.env.DATAIKU_API_KEY !== undefined) {
|
|
156
|
+
apiKey = process.env.DATAIKU_API_KEY;
|
|
157
|
+
sources.apiKey = envVarSource("DATAIKU_API_KEY");
|
|
158
|
+
}
|
|
159
|
+
if (!hasProjectKeyFlag && process.env.DATAIKU_PROJECT_KEY !== undefined) {
|
|
160
|
+
projectKey = process.env.DATAIKU_PROJECT_KEY;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
const tlsSettings = resolveTlsSettings(flags, undefined, sources);
|
|
164
|
+
assertCredentialBinding(sources);
|
|
165
|
+
return { url: url ?? "", apiKey: apiKey ?? "", projectKey, tlsSettings, };
|
|
166
|
+
}
|
package/dist/src/cli/types.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ export interface CommandPayloadSchema {
|
|
|
5
5
|
stdin?: boolean;
|
|
6
6
|
dataFlag?: boolean;
|
|
7
7
|
dataFileFlag?: boolean;
|
|
8
|
+
contentType?: "application/json" | "text/plain";
|
|
8
9
|
jsonShape?: "object" | "array";
|
|
9
10
|
}
|
|
10
11
|
export interface CommandFlagChoice {
|
|
@@ -21,6 +22,12 @@ export interface CommandRegistryOverride {
|
|
|
21
22
|
export interface CommandMeta extends CommandRegistryOverride {
|
|
22
23
|
handler: CommandHandler;
|
|
23
24
|
localHandler?: LocalCommandHandler;
|
|
25
|
+
/**
|
|
26
|
+
* Pure, DSS-free argument validation run before credentials are resolved,
|
|
27
|
+
* so a bad flag value is reported as its own usage error even when no
|
|
28
|
+
* credentials are configured, instead of being masked by a missing-URL error.
|
|
29
|
+
*/
|
|
30
|
+
validate?: (args: string[], flags: Record<string, string | boolean>) => void;
|
|
24
31
|
usage: string;
|
|
25
32
|
description?: string;
|
|
26
33
|
examples?: string[];
|
package/dist/src/cli.js
CHANGED
|
@@ -7,8 +7,8 @@ import { runDoctor, runFixtures, } from "./cli/doctor.js";
|
|
|
7
7
|
import { dataikuEnvironmentEnabled, loadEnvFile, } from "./cli/env.js";
|
|
8
8
|
import { FLAG_ALIASES, isNegativeNumberToken, parseArgs, SHORT_FLAGS, VALUE_FLAGS, } from "./cli/flags.js";
|
|
9
9
|
import { cleanupLedgerEntry, } from "./cli/helpers/cleanup.js";
|
|
10
|
-
import { commandFailureExitCode,
|
|
11
|
-
import { currentCommandContext, resolveCredentials,
|
|
10
|
+
import { commandFailureExitCode, CommandResultFailure, flushCliWarnings, isFailedWaitResult, planResult, projectResultFields, setOutputFieldProjection, writeCommandResult, } from "./cli/output.js";
|
|
11
|
+
import { currentCommandContext, resolveCredentials, resolveLoginCredentials, } from "./cli/runtime.js";
|
|
12
12
|
import { COMMANDS_RUN_HINT, missingActionError, noCommandError, requireArgs, unknownActionError, unknownResourceError, UsageError, } from "./cli/usage.js";
|
|
13
13
|
import { cliVersionResult, } from "./cli/version.js";
|
|
14
14
|
import { DataikuClient, } from "./client.js";
|
|
@@ -153,6 +153,11 @@ async function runCleanup(flags) {
|
|
|
153
153
|
try {
|
|
154
154
|
const parsed = parseArgs(entry.cleanup.argv);
|
|
155
155
|
const [resource, action, ...args] = parsed.positional;
|
|
156
|
+
Object.assign(currentCommandContext, {
|
|
157
|
+
resource,
|
|
158
|
+
action,
|
|
159
|
+
projectKey: parsed.flags["project-key"],
|
|
160
|
+
});
|
|
156
161
|
if (!resource || !action || !isAllowedCleanupAction(resource, action)
|
|
157
162
|
|| !commands[resource]?.[action]) {
|
|
158
163
|
throw new UsageError(`Invalid cleanup argv: ${entry.cleanup.argv.join(" ")}`);
|
|
@@ -173,13 +178,18 @@ async function runCleanup(flags) {
|
|
|
173
178
|
const failure = {
|
|
174
179
|
index,
|
|
175
180
|
cleanup: entry.cleanup,
|
|
176
|
-
error:
|
|
177
|
-
result,
|
|
181
|
+
error: buildErrorReport(new CommandResultFailure(result, 4)),
|
|
178
182
|
};
|
|
179
183
|
failures.push(failure);
|
|
180
184
|
if (flags["continue-on-error"] !== true) {
|
|
181
185
|
return {
|
|
182
|
-
result: {
|
|
186
|
+
result: {
|
|
187
|
+
applied: true,
|
|
188
|
+
steps,
|
|
189
|
+
results: applied,
|
|
190
|
+
failures,
|
|
191
|
+
...summarizeCleanupFailures(failures),
|
|
192
|
+
},
|
|
183
193
|
exitCode: 2,
|
|
184
194
|
};
|
|
185
195
|
}
|
|
@@ -191,19 +201,31 @@ async function runCleanup(flags) {
|
|
|
191
201
|
const failure = {
|
|
192
202
|
index,
|
|
193
203
|
cleanup: entry.cleanup,
|
|
194
|
-
error:
|
|
204
|
+
error: buildErrorReport(error),
|
|
195
205
|
};
|
|
196
206
|
failures.push(failure);
|
|
197
207
|
if (flags["continue-on-error"] !== true) {
|
|
198
208
|
return {
|
|
199
|
-
result: {
|
|
209
|
+
result: {
|
|
210
|
+
applied: true,
|
|
211
|
+
steps,
|
|
212
|
+
results: applied,
|
|
213
|
+
failures,
|
|
214
|
+
...summarizeCleanupFailures(failures),
|
|
215
|
+
},
|
|
200
216
|
exitCode: 2,
|
|
201
217
|
};
|
|
202
218
|
}
|
|
203
219
|
}
|
|
204
220
|
}
|
|
205
221
|
return {
|
|
206
|
-
result: {
|
|
222
|
+
result: {
|
|
223
|
+
applied: true,
|
|
224
|
+
steps,
|
|
225
|
+
results: applied,
|
|
226
|
+
failures,
|
|
227
|
+
...summarizeCleanupFailures(failures),
|
|
228
|
+
},
|
|
207
229
|
exitCode: failures.length > 0 ? 2 : 0,
|
|
208
230
|
};
|
|
209
231
|
}
|
|
@@ -656,23 +678,7 @@ function runInstallSkill(flags) {
|
|
|
656
678
|
}
|
|
657
679
|
const META_PLAN_EXIT_CODES = { usage: 1, error: 2, transient: 3, };
|
|
658
680
|
function authLoginPlan(flags) {
|
|
659
|
-
const tlsSettings =
|
|
660
|
-
const useEnv = dataikuEnvironmentEnabled();
|
|
661
|
-
const url = typeof flags["url"] === "string"
|
|
662
|
-
? flags["url"]
|
|
663
|
-
: useEnv
|
|
664
|
-
? process.env.DATAIKU_URL ?? ""
|
|
665
|
-
: "";
|
|
666
|
-
const apiKey = typeof flags["api-key"] === "string"
|
|
667
|
-
? flags["api-key"]
|
|
668
|
-
: useEnv
|
|
669
|
-
? process.env.DATAIKU_API_KEY ?? ""
|
|
670
|
-
: "";
|
|
671
|
-
const projectKey = typeof flags["project-key"] === "string"
|
|
672
|
-
? flags["project-key"]
|
|
673
|
-
: useEnv
|
|
674
|
-
? process.env.DATAIKU_PROJECT_KEY
|
|
675
|
-
: undefined;
|
|
681
|
+
const { url, apiKey, projectKey, tlsSettings, } = resolveLoginCredentials(flags);
|
|
676
682
|
if (!url || !apiKey) {
|
|
677
683
|
throw new UsageError("Missing --url and/or --api-key for auth login.", "missing_required_flag", "Pass --url and --api-key, or set DATAIKU_URL and DATAIKU_API_KEY.", { requiredFlags: ["url", "api-key",], env: ["DATAIKU_URL", "DATAIKU_API_KEY",], });
|
|
678
684
|
}
|
|
@@ -861,6 +867,9 @@ function assertCleanupLedgerSupported(resource, action, flags) {
|
|
|
861
867
|
if (!supportsCleanupLedger(resource, action)) {
|
|
862
868
|
throw new UsageError(`--record-cleanup is not supported for ${resource} ${action}.`);
|
|
863
869
|
}
|
|
870
|
+
if (resource === "workspace" && action === "create" && flags["stdin"] === true) {
|
|
871
|
+
throw new UsageError("workspace create cannot combine --record-cleanup with --stdin because the cleanup identifier cannot be replayed from a consumed stream.", "conflicting_input_sources", "Use --data or --data-file so the workspaceKey remains available for the cleanup ledger.");
|
|
872
|
+
}
|
|
864
873
|
}
|
|
865
874
|
/**
|
|
866
875
|
* Preflight a requested cleanup ledger path before any remote mutation so an
|
|
@@ -923,8 +932,16 @@ async function recordCleanupLedgerEntry(client, resource, action, args, flags, r
|
|
|
923
932
|
if (typeof ledgerPath !== "string" || flags["dry-run"] === true)
|
|
924
933
|
return;
|
|
925
934
|
const entry = cleanupLedgerEntry(resource, action, args, flags, result, projectKey);
|
|
926
|
-
if (entry)
|
|
935
|
+
if (entry) {
|
|
927
936
|
await appendCleanupLedgerEntry(ledgerPath, entry, client.getBaseUrl());
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
const record = result !== null && typeof result === "object" && !Array.isArray(result)
|
|
940
|
+
? result
|
|
941
|
+
: {};
|
|
942
|
+
if (record.skipped !== undefined || record.cleanupEligible === false)
|
|
943
|
+
return;
|
|
944
|
+
throw new ClientValidationError(`Mutation completed but no cleanup entry could be derived for ${resource} ${action}.`, "ambiguous_outcome", "Inspect the created object and remove it manually; do not assume the requested cleanup ledger contains it.", { resource, action, ledgerPath, });
|
|
928
945
|
}
|
|
929
946
|
async function runBatch(flags) {
|
|
930
947
|
const payload = unknownJsonInput(flags);
|
|
@@ -1102,16 +1119,46 @@ async function runBatch(flags) {
|
|
|
1102
1119
|
}
|
|
1103
1120
|
}
|
|
1104
1121
|
const ok = firstFailureExit === undefined;
|
|
1122
|
+
const failureSummary = summarizeFailureEnvelopes(results.flatMap((step) => step.error ? [{ index: step.index, error: step.error, },] : []));
|
|
1105
1123
|
return {
|
|
1106
1124
|
result: {
|
|
1107
1125
|
ok,
|
|
1108
1126
|
total: steps.length,
|
|
1109
1127
|
completed: results.filter((step) => step.ok !== null).length,
|
|
1128
|
+
...failureSummary,
|
|
1110
1129
|
steps: results,
|
|
1111
1130
|
},
|
|
1112
1131
|
exitCode: ok ? 0 : firstFailureExit ?? 2,
|
|
1113
1132
|
};
|
|
1114
1133
|
}
|
|
1134
|
+
function summarizeFailureEnvelopes(failures) {
|
|
1135
|
+
const failureCodes = failures.map(({ index, error, }) => ({
|
|
1136
|
+
index,
|
|
1137
|
+
code: error.code,
|
|
1138
|
+
category: error.category,
|
|
1139
|
+
exitCode: error.exitCode,
|
|
1140
|
+
retryable: error.retryable === true,
|
|
1141
|
+
}));
|
|
1142
|
+
return {
|
|
1143
|
+
failed: failures.length,
|
|
1144
|
+
retryable: failures.length > 0 && failureCodes.every((failure) => failure.retryable),
|
|
1145
|
+
hasRetryableFailures: failureCodes.some((failure) => failure.retryable),
|
|
1146
|
+
failureCodes,
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
function summarizeCleanupFailures(failures) {
|
|
1150
|
+
return summarizeFailureEnvelopes(failures.flatMap((failure) => {
|
|
1151
|
+
const index = failure.index;
|
|
1152
|
+
const error = failure.error;
|
|
1153
|
+
return typeof index === "number"
|
|
1154
|
+
&& error !== null
|
|
1155
|
+
&& typeof error === "object"
|
|
1156
|
+
&& !Array.isArray(error)
|
|
1157
|
+
&& error.type === "error"
|
|
1158
|
+
? [{ index, error: error, },]
|
|
1159
|
+
: [];
|
|
1160
|
+
}));
|
|
1161
|
+
}
|
|
1115
1162
|
function rawFlagValue(argv, flagName) {
|
|
1116
1163
|
const longFlag = `--${flagName}`;
|
|
1117
1164
|
for (let index = 0; index < argv.length; index++) {
|
|
@@ -1206,13 +1253,48 @@ function requestIdFromBody(body) {
|
|
|
1206
1253
|
return undefined;
|
|
1207
1254
|
}
|
|
1208
1255
|
const MISSING_PROJECT_KEY_ERROR_PREFIX = "projectKey is required";
|
|
1256
|
+
function isAmbiguousMutationFailure(err, context) {
|
|
1257
|
+
if (!(err instanceof DataikuError))
|
|
1258
|
+
return false;
|
|
1259
|
+
const method = err.retry?.method.toUpperCase();
|
|
1260
|
+
if (!method || method === "GET" || method === "HEAD")
|
|
1261
|
+
return false;
|
|
1262
|
+
if (err.status !== 0 && err.status < 500)
|
|
1263
|
+
return false;
|
|
1264
|
+
const entry = context.resource && context.action
|
|
1265
|
+
? cachedCommandRegistry()[context.resource]?.[context.action]
|
|
1266
|
+
: undefined;
|
|
1267
|
+
return entry?.sideEffect === "write" && entry.idempotency === "none";
|
|
1268
|
+
}
|
|
1269
|
+
const MAX_FAILURE_RESULT_BYTES = 64 * 1024;
|
|
1270
|
+
function boundedFailureResult(result) {
|
|
1271
|
+
let serialized;
|
|
1272
|
+
try {
|
|
1273
|
+
serialized = JSON.stringify(result);
|
|
1274
|
+
}
|
|
1275
|
+
catch {
|
|
1276
|
+
return { result: "[unserializable command result]", resultTruncated: true, };
|
|
1277
|
+
}
|
|
1278
|
+
if (serialized === undefined || Buffer.byteLength(serialized, "utf8") <= MAX_FAILURE_RESULT_BYTES) {
|
|
1279
|
+
return { result, };
|
|
1280
|
+
}
|
|
1281
|
+
return {
|
|
1282
|
+
resultPreview: Buffer.from(serialized, "utf8").subarray(0, MAX_FAILURE_RESULT_BYTES).toString("utf8"),
|
|
1283
|
+
resultBytes: Buffer.byteLength(serialized, "utf8"),
|
|
1284
|
+
resultTruncated: true,
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1209
1287
|
function errorExitCode(err) {
|
|
1210
1288
|
if (err instanceof CommandResultFailure)
|
|
1211
1289
|
return err.exitCode;
|
|
1290
|
+
if (err instanceof ClientValidationError && err.code === "assertion_failed")
|
|
1291
|
+
return 4;
|
|
1212
1292
|
if (err instanceof ClientValidationError && err.code === "ambiguous_outcome")
|
|
1213
1293
|
return 2;
|
|
1214
1294
|
if (err instanceof UsageError || err instanceof ClientValidationError)
|
|
1215
1295
|
return 1;
|
|
1296
|
+
if (isAmbiguousMutationFailure(err, rawCommandContext()))
|
|
1297
|
+
return 2;
|
|
1216
1298
|
if (err instanceof DataikuError)
|
|
1217
1299
|
return err.category === "transient" ? 3 : 2;
|
|
1218
1300
|
if (err instanceof Error && err.message.startsWith(MISSING_PROJECT_KEY_ERROR_PREFIX))
|
|
@@ -1222,7 +1304,8 @@ function errorExitCode(err) {
|
|
|
1222
1304
|
function buildErrorReport(err) {
|
|
1223
1305
|
const context = rawCommandContext();
|
|
1224
1306
|
const exitCode = errorExitCode(err);
|
|
1225
|
-
if (err instanceof ClientValidationError
|
|
1307
|
+
if (err instanceof ClientValidationError
|
|
1308
|
+
&& (err.code === "ambiguous_outcome" || err.code === "assertion_failed")) {
|
|
1226
1309
|
return {
|
|
1227
1310
|
type: "error",
|
|
1228
1311
|
ok: false,
|
|
@@ -1230,6 +1313,7 @@ function buildErrorReport(err) {
|
|
|
1230
1313
|
code: err.code,
|
|
1231
1314
|
category: "dss",
|
|
1232
1315
|
exitCode,
|
|
1316
|
+
retryable: false,
|
|
1233
1317
|
...(err.hint ? { hint: err.hint, } : {}),
|
|
1234
1318
|
...(err.details ? { details: err.details, } : {}),
|
|
1235
1319
|
...context,
|
|
@@ -1259,7 +1343,29 @@ function buildErrorReport(err) {
|
|
|
1259
1343
|
code: err.code,
|
|
1260
1344
|
category: "dss",
|
|
1261
1345
|
exitCode: err.exitCode,
|
|
1262
|
-
details:
|
|
1346
|
+
details: boundedFailureResult(err.result),
|
|
1347
|
+
...context,
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
if (isAmbiguousMutationFailure(err, context)) {
|
|
1351
|
+
const method = err.retry?.method.toUpperCase() ?? "mutation";
|
|
1352
|
+
return {
|
|
1353
|
+
type: "error",
|
|
1354
|
+
ok: false,
|
|
1355
|
+
error: `The ${method} request failed after dispatch; the mutation outcome is unknown.`,
|
|
1356
|
+
code: "ambiguous_outcome",
|
|
1357
|
+
category: "dss",
|
|
1358
|
+
exitCode: 2,
|
|
1359
|
+
hint: "Read the target state and verify whether the mutation took effect before retrying.",
|
|
1360
|
+
status: err.status,
|
|
1361
|
+
retryable: false,
|
|
1362
|
+
requestId: err.requestId ?? requestIdFromBody(err.body),
|
|
1363
|
+
details: {
|
|
1364
|
+
dssCategory: err.category,
|
|
1365
|
+
statusText: canonicalStatusText(err.status),
|
|
1366
|
+
idempotency: "none",
|
|
1367
|
+
...(err.retry ? { retry: err.retry, } : {}),
|
|
1368
|
+
},
|
|
1263
1369
|
...context,
|
|
1264
1370
|
};
|
|
1265
1371
|
}
|
|
@@ -1387,6 +1493,7 @@ async function main() {
|
|
|
1387
1493
|
writeCommandResult(result);
|
|
1388
1494
|
return;
|
|
1389
1495
|
}
|
|
1496
|
+
actionMeta.validate?.(args, flags);
|
|
1390
1497
|
const { url, apiKey, projectKey, tlsRejectUnauthorized, caCertPath, } = resolveCredentials(flags);
|
|
1391
1498
|
currentCommandContext.projectKey = projectKey;
|
|
1392
1499
|
if (!url) {
|
package/dist/src/client.d.ts
CHANGED
|
@@ -71,6 +71,12 @@ export interface DataikuClientConfig {
|
|
|
71
71
|
requestTimeoutMs?: number;
|
|
72
72
|
/** Max retry attempts for idempotent requests (default 4, capped at 10) */
|
|
73
73
|
retryMaxAttempts?: number;
|
|
74
|
+
/**
|
|
75
|
+
* Maximum bytes buffered from a single response body by the text/JSON
|
|
76
|
+
* consumer methods (default 50 MiB). Responses larger than this are
|
|
77
|
+
* rejected with a DataikuError after the stream is cancelled.
|
|
78
|
+
*/
|
|
79
|
+
maxResponseBodyBytes?: number;
|
|
74
80
|
/** Emit HTTP request/response trace events. Defaults to JSONL on stderr when verbose is true. */
|
|
75
81
|
verbose?: boolean;
|
|
76
82
|
onTrace?: (event: DataikuClientTraceEvent) => void;
|
|
@@ -91,6 +97,7 @@ export declare class DataikuClient {
|
|
|
91
97
|
private readonly apiKey;
|
|
92
98
|
private readonly defaultProjectKey;
|
|
93
99
|
private readonly requestTimeoutMs;
|
|
100
|
+
private readonly maxResponseBodyBytes;
|
|
94
101
|
private readonly retryMaxAttempts;
|
|
95
102
|
private readonly verbose;
|
|
96
103
|
private readonly tlsOptions;
|
|
@@ -176,6 +183,11 @@ export declare class DataikuClient {
|
|
|
176
183
|
*/
|
|
177
184
|
getBaseUrl(): string;
|
|
178
185
|
getRequestTimeoutMs(): number;
|
|
186
|
+
/**
|
|
187
|
+
* The configured maximum number of bytes buffered from a single response
|
|
188
|
+
* body by the text/JSON consumer methods (default 50 MiB).
|
|
189
|
+
*/
|
|
190
|
+
getMaxResponseBodyBytes(): number;
|
|
179
191
|
resolveProjectKey(paramValue?: string): string;
|
|
180
192
|
get<T = unknown>(path: string): Promise<T>;
|
|
181
193
|
/**
|
|
@@ -187,11 +199,34 @@ export declare class DataikuClient {
|
|
|
187
199
|
data: T;
|
|
188
200
|
meta: DataikuClientResponseMeta;
|
|
189
201
|
}>;
|
|
202
|
+
/**
|
|
203
|
+
* GET returning the entire text body. The body is buffered with a hard
|
|
204
|
+
* byte cap (`maxResponseBodyBytes`, default 50 MiB) and read under the
|
|
205
|
+
* request deadline: a response exceeding the cap throws a DataikuError
|
|
206
|
+
* instead of being silently truncated, and a stalled body times out.
|
|
207
|
+
*/
|
|
190
208
|
getText(path: string): Promise<string>;
|
|
209
|
+
/**
|
|
210
|
+
* GET returning at most the first `maxBytes` bytes of a text body.
|
|
211
|
+
* When the response exceeds the cap the stream is cancelled, only the
|
|
212
|
+
* collected prefix is returned and `truncated` is true. The read runs
|
|
213
|
+
* under the request deadline: a stalled body throws a DataikuError.
|
|
214
|
+
*/
|
|
191
215
|
getTextLimited(path: string, maxBytes: number): Promise<{
|
|
192
216
|
text: string;
|
|
193
217
|
truncated: boolean;
|
|
194
218
|
}>;
|
|
219
|
+
/**
|
|
220
|
+
* GET returning at most the last `maxBytes` bytes of a text body.
|
|
221
|
+
* When the response exceeds the cap, earlier bytes are discarded, only
|
|
222
|
+
* the tail is kept and `truncated` is true — so callers that need the end
|
|
223
|
+
* of a large document (e.g. job logs) stay bounded without losing the most
|
|
224
|
+
* recent content. The read runs under the request deadline.
|
|
225
|
+
*/
|
|
226
|
+
getTextTailLimited(path: string, maxBytes: number): Promise<{
|
|
227
|
+
text: string;
|
|
228
|
+
truncated: boolean;
|
|
229
|
+
}>;
|
|
195
230
|
/**
|
|
196
231
|
* POST with optional transient retry. Only opt in when repeating the request
|
|
197
232
|
* is safe: the server may have accepted an attempt whose response was lost.
|
|
@@ -199,6 +234,10 @@ export declare class DataikuClient {
|
|
|
199
234
|
post<T = unknown>(path: string, body?: unknown, options?: {
|
|
200
235
|
retryMaxAttempts?: number;
|
|
201
236
|
}): Promise<T>;
|
|
237
|
+
/**
|
|
238
|
+
* POST returning the text body, under the same bounded buffering rules as
|
|
239
|
+
* {@link getText} (byte cap `maxResponseBodyBytes`, request deadline).
|
|
240
|
+
*/
|
|
202
241
|
postText(path: string, body?: unknown): Promise<string>;
|
|
203
242
|
postStream(path: string, body?: unknown): Promise<Response>;
|
|
204
243
|
put<T = unknown>(path: string, body: unknown): Promise<T>;
|
|
@@ -229,5 +268,26 @@ export declare class DataikuClient {
|
|
|
229
268
|
warn(method: string, errors: string[]): void;
|
|
230
269
|
private requestIdFromHeaders;
|
|
231
270
|
private parseJsonResponse;
|
|
271
|
+
/**
|
|
272
|
+
* Read at most `maxBytes` of a 2xx response body as UTF-8 text.
|
|
273
|
+
*
|
|
274
|
+
* The body is consumed with the request deadline still active (measured
|
|
275
|
+
* from the first read): a stalled body throws a DataikuError instead of
|
|
276
|
+
* hanging. When the stream yields more than the cap, the reader cancels the
|
|
277
|
+
* body and returns the collected prefix with `truncated: true`; truncated
|
|
278
|
+
* text is trimmed to a UTF-8 character boundary so it never decodes beyond
|
|
279
|
+
* the byte cap.
|
|
280
|
+
*/
|
|
281
|
+
private readBoundedBodyText;
|
|
282
|
+
/**
|
|
283
|
+
* Read a 2xx response body keeping only the last `maxBytes` bytes.
|
|
284
|
+
* Earlier bytes are discarded as the stream advances, so memory stays
|
|
285
|
+
* bounded by the cap plus one chunk even for arbitrarily large bodies.
|
|
286
|
+
* Returns `truncated: true` when bytes had to be dropped. Bytes that fall
|
|
287
|
+
* in the middle of a UTF-8 character are dropped from both ends so the
|
|
288
|
+
* decoded text never contains a replacement character.
|
|
289
|
+
*/
|
|
290
|
+
private readBodyTextTail;
|
|
291
|
+
private buildResponseTooLargeError;
|
|
232
292
|
private fetchWithRetry;
|
|
233
293
|
}
|