dataiku-sdk 0.12.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -15
- package/bin/dss.js +21 -2
- package/dist/build-metadata.json +1 -0
- package/dist/packages/types/src/index.d.ts +128 -3
- package/dist/packages/types/src/index.js +106 -3
- package/dist/src/cli/coerce.d.ts +2 -1
- package/dist/src/cli/coerce.js +84 -26
- package/dist/src/cli/commands/app.js +6 -6
- package/dist/src/cli/commands/code.js +2 -2
- package/dist/src/cli/commands/data-quality.js +50 -8
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +197 -12
- package/dist/src/cli/commands/folder.js +5 -4
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/index.js +2 -0
- package/dist/src/cli/commands/job.js +26 -13
- package/dist/src/cli/commands/notebook.js +1 -1
- package/dist/src/cli/commands/project-git.d.ts +2 -0
- package/dist/src/cli/commands/project-git.js +563 -0
- package/dist/src/cli/commands/project.js +118 -14
- package/dist/src/cli/commands/recipe.js +14 -17
- package/dist/src/cli/commands/scenario.js +4 -4
- package/dist/src/cli/contract.d.ts +6 -4
- package/dist/src/cli/contract.js +595 -68
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +50 -19
- package/dist/src/cli/flags.js +23 -2
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/helpers/dataset.js +1 -1
- package/dist/src/cli/output.d.ts +5 -2
- package/dist/src/cli/output.js +13 -12
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +1 -1
- package/dist/src/cli/usage.js +2 -1
- package/dist/src/cli/version.d.ts +48 -6
- package/dist/src/cli/version.js +177 -11
- package/dist/src/cli.js +94 -39
- package/dist/src/client.d.ts +31 -1
- package/dist/src/client.js +39 -7
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/index.d.ts +6 -4
- package/dist/src/index.js +3 -1
- package/dist/src/resources/connections.d.ts +3 -1
- package/dist/src/resources/connections.js +82 -21
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +241 -30
- package/dist/src/resources/folders.js +7 -3
- package/dist/src/resources/project-git.d.ts +154 -0
- package/dist/src/resources/project-git.js +427 -0
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +127 -16
- package/dist/src/resources/recipes.js +4 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +624 -0
- package/package.json +6 -5
- package/packages/types/dist/index.d.ts +128 -3
- package/packages/types/dist/index.js +106 -3
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +26 -19
package/dist/src/cli/version.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFileSync, } from "node:fs";
|
|
1
|
+
import { readdirSync, readFileSync, statSync, } from "node:fs";
|
|
2
2
|
import { dirname, resolve, } from "node:path";
|
|
3
3
|
import { fileURLToPath, } from "node:url";
|
|
4
4
|
export function findPackageRoot() {
|
|
@@ -36,28 +36,194 @@ export function gitDirectory(packageRoot) {
|
|
|
36
36
|
}
|
|
37
37
|
return resolve(packageRoot, ".git");
|
|
38
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* A full lowercase hexadecimal revision is the only provenance claim trusted.
|
|
41
|
+
* Git is never invoked from runtime version reporting; revisions are resolved
|
|
42
|
+
* directly from the filesystem layouts Git documents.
|
|
43
|
+
*/
|
|
44
|
+
const FULL_REVISION_PATTERN = /^[0-9a-f]{40}$/;
|
|
45
|
+
export function isFullRevision(value) {
|
|
46
|
+
return typeof value === "string" && FULL_REVISION_PATTERN.test(value);
|
|
47
|
+
}
|
|
48
|
+
function readGitFile(base, relative) {
|
|
49
|
+
try {
|
|
50
|
+
return readFileSync(resolve(base, relative), "utf-8").trim();
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function isSafeRefName(ref) {
|
|
57
|
+
return (ref.length > 0
|
|
58
|
+
&& !ref.startsWith("/")
|
|
59
|
+
&& !ref.endsWith("/")
|
|
60
|
+
&& !ref.includes("\\")
|
|
61
|
+
&& !ref.split("/").includes(".."));
|
|
62
|
+
}
|
|
63
|
+
/** Resolve an entry from packed-refs; peeled `^` lines never match a ref. */
|
|
64
|
+
function packedRefFile(base, ref) {
|
|
65
|
+
let contents;
|
|
66
|
+
try {
|
|
67
|
+
contents = readFileSync(resolve(base, "packed-refs"), "utf-8");
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
for (const line of contents.split("\n")) {
|
|
73
|
+
const trimmed = line.trim();
|
|
74
|
+
if (trimmed.length === 0 || trimmed.startsWith("#") || trimmed.startsWith("^"))
|
|
75
|
+
continue;
|
|
76
|
+
const splitAt = trimmed.indexOf(" ");
|
|
77
|
+
if (splitAt <= 0)
|
|
78
|
+
continue;
|
|
79
|
+
const hash = trimmed.slice(0, splitAt);
|
|
80
|
+
if (isFullRevision(hash) && trimmed.slice(splitAt + 1).trim() === ref)
|
|
81
|
+
return hash;
|
|
82
|
+
}
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The shared git dir behind a linked worktree or submodule checkout: the
|
|
87
|
+
* `commondir` file inside the worktree's git dir, resolved relative to that
|
|
88
|
+
* dir. Normal checkouts (no `commondir` file) share their own git dir.
|
|
89
|
+
*/
|
|
90
|
+
function gitCommonDirectory(gitDir) {
|
|
91
|
+
try {
|
|
92
|
+
const commondir = readFileSync(resolve(gitDir, "commondir"), "utf-8").trim();
|
|
93
|
+
return commondir.length > 0 ? resolve(gitDir, commondir) : undefined;
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Full checkout HEAD revision (40-lowercase-hex) — the basis for staleness
|
|
101
|
+
* comparison. Resolution follows Git's own layouts: a detached revision, then
|
|
102
|
+
* loose refs in the worktree git dir, then the shared store reached through
|
|
103
|
+
* `commondir`, then packed-refs in each. Anything that is not a full
|
|
104
|
+
* lowercase hexadecimal revision is rejected.
|
|
105
|
+
*/
|
|
106
|
+
export function gitFullRevision(packageRoot) {
|
|
107
|
+
if (!packageRoot)
|
|
108
|
+
return undefined;
|
|
109
|
+
const gitDir = gitDirectory(packageRoot);
|
|
110
|
+
const head = readGitFile(gitDir, "HEAD");
|
|
111
|
+
if (!head)
|
|
112
|
+
return undefined;
|
|
113
|
+
if (isFullRevision(head))
|
|
114
|
+
return head;
|
|
115
|
+
if (!head.startsWith("ref:"))
|
|
116
|
+
return undefined;
|
|
117
|
+
const ref = head.slice("ref:".length).trim();
|
|
118
|
+
if (!isSafeRefName(ref))
|
|
119
|
+
return undefined;
|
|
120
|
+
for (const base of [gitDir, gitCommonDirectory(gitDir),]) {
|
|
121
|
+
if (!base)
|
|
122
|
+
continue;
|
|
123
|
+
const loose = readGitFile(base, ref);
|
|
124
|
+
if (loose && isFullRevision(loose))
|
|
125
|
+
return loose;
|
|
126
|
+
const packed = packedRefFile(base, ref);
|
|
127
|
+
if (packed)
|
|
128
|
+
return packed;
|
|
129
|
+
}
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
39
132
|
export function gitRevision(packageRoot) {
|
|
133
|
+
const full = gitFullRevision(packageRoot);
|
|
134
|
+
return full ? full.slice(0, 7) : undefined;
|
|
135
|
+
}
|
|
136
|
+
export function detectLoadSource(modulePath = fileURLToPath(import.meta.url)) {
|
|
137
|
+
return modulePath.replaceAll("\\", "/").includes("/dist/") ? "dist" : "source";
|
|
138
|
+
}
|
|
139
|
+
export function buildMetadataRevision(packageRoot) {
|
|
40
140
|
if (!packageRoot)
|
|
41
141
|
return undefined;
|
|
42
142
|
try {
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
const full = readFileSync(resolve(gitDir, ref), "utf-8").trim();
|
|
49
|
-
return full.slice(0, 7);
|
|
143
|
+
const metadata = JSON.parse(readFileSync(resolve(packageRoot, "dist", "build-metadata.json"), "utf-8"));
|
|
144
|
+
return typeof metadata.buildRevision === "string"
|
|
145
|
+
&& isFullRevision(metadata.buildRevision)
|
|
146
|
+
? metadata.buildRevision
|
|
147
|
+
: undefined;
|
|
50
148
|
}
|
|
51
149
|
catch {
|
|
52
150
|
return undefined;
|
|
53
151
|
}
|
|
54
152
|
}
|
|
153
|
+
/**
|
|
154
|
+
* Whether compiled CLI output predates any source input. The build metadata
|
|
155
|
+
* file is written after `tsc`, so its mtime is the build-completion marker.
|
|
156
|
+
* Installed packages omit `src/`; without both sides there is no drift claim.
|
|
157
|
+
*/
|
|
158
|
+
export function sourceTreeNewerThanBuild(packageRoot) {
|
|
159
|
+
if (!packageRoot)
|
|
160
|
+
return false;
|
|
161
|
+
try {
|
|
162
|
+
const buildMtimeMs = statSync(resolve(packageRoot, "dist", "build-metadata.json")).mtimeMs;
|
|
163
|
+
const pending = [resolve(packageRoot, "src"),];
|
|
164
|
+
while (pending.length > 0) {
|
|
165
|
+
const path = pending.pop();
|
|
166
|
+
if (!path)
|
|
167
|
+
break;
|
|
168
|
+
const stat = statSync(path);
|
|
169
|
+
if (stat.mtimeMs > buildMtimeMs)
|
|
170
|
+
return true;
|
|
171
|
+
if (!stat.isDirectory())
|
|
172
|
+
continue;
|
|
173
|
+
for (const entry of readdirSync(path, { withFileTypes: true, })) {
|
|
174
|
+
if (entry.isDirectory() || entry.isFile()) {
|
|
175
|
+
pending.push(resolve(path, entry.name));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
export function detectRuntime() {
|
|
186
|
+
return process.versions.bun ? "bun" : "node";
|
|
187
|
+
}
|
|
188
|
+
export function buildVersionPayload(input) {
|
|
189
|
+
const isDist = input.loadSource === "dist";
|
|
190
|
+
// Source execution never claims a build revision, even if the environment
|
|
191
|
+
// happens to carry one.
|
|
192
|
+
const buildRevision = isDist && input.buildRevision ? input.buildRevision : null;
|
|
193
|
+
const staleBuild = isDist
|
|
194
|
+
&& ((buildRevision !== null
|
|
195
|
+
&& input.checkoutFullRevision !== undefined
|
|
196
|
+
&& input.checkoutFullRevision !== buildRevision)
|
|
197
|
+
|| input.sourceNewerThanBuild === true);
|
|
198
|
+
return {
|
|
199
|
+
version: input.packageVersion,
|
|
200
|
+
gitRevision: input.checkoutRevision ?? null,
|
|
201
|
+
source: isDist ? "dist" : "source",
|
|
202
|
+
runtime: input.runtime,
|
|
203
|
+
buildRevision,
|
|
204
|
+
staleBuild,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
55
207
|
export const PACKAGE_ROOT = findPackageRoot();
|
|
56
208
|
export const CLI_VERSION = packageVersion(PACKAGE_ROOT);
|
|
57
209
|
export const CLI_GIT_REVISION = gitRevision(PACKAGE_ROOT);
|
|
58
|
-
export const AGENT_CONTRACT_VERSION =
|
|
210
|
+
export const AGENT_CONTRACT_VERSION = 2;
|
|
59
211
|
export const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
|
|
60
|
-
export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-
|
|
212
|
+
export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v2.json";
|
|
61
213
|
export function cliVersionResult() {
|
|
62
|
-
|
|
214
|
+
const loadSource = process.env["DSS_LOAD_SOURCE"] ?? detectLoadSource();
|
|
215
|
+
const envBuildRevision = process.env["DSS_BUILD_REVISION"];
|
|
216
|
+
return buildVersionPayload({
|
|
217
|
+
packageVersion: CLI_VERSION,
|
|
218
|
+
checkoutRevision: CLI_GIT_REVISION,
|
|
219
|
+
checkoutFullRevision: gitFullRevision(PACKAGE_ROOT),
|
|
220
|
+
loadSource,
|
|
221
|
+
buildRevision: isFullRevision(envBuildRevision)
|
|
222
|
+
? envBuildRevision
|
|
223
|
+
: (loadSource === "dist" ? buildMetadataRevision(PACKAGE_ROOT) : undefined),
|
|
224
|
+
sourceNewerThanBuild: loadSource === "dist"
|
|
225
|
+
? sourceTreeNewerThanBuild(PACKAGE_ROOT)
|
|
226
|
+
: false,
|
|
227
|
+
runtime: detectRuntime(),
|
|
228
|
+
});
|
|
63
229
|
}
|
package/dist/src/cli.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import * as fs from "node:fs/promises";
|
|
2
3
|
import { num, unknownJsonInput, } from "./cli/coerce.js";
|
|
3
4
|
import { commands, } from "./cli/commands/index.js";
|
|
4
|
-
import { AGENT_CONTRACT_USAGE, AUTH_ACTIONS, BATCH_EXAMPLE_PAYLOAD, BATCH_HINT, BATCH_PLAN_EXIT_CODES, BATCH_USAGE, buildAgentContract, buildCommandRegistry, buildMutationPlan, CLEANUP_USAGE, COMMANDS_USAGE, inferRequiresProject, isAllowedCleanupAction, supportsCleanupLedger, } from "./cli/contract.js";
|
|
5
|
+
import { AGENT_CONTRACT_USAGE, AUTH_ACTIONS, BATCH_EXAMPLE_PAYLOAD, BATCH_HINT, BATCH_PLAN_EXIT_CODES, BATCH_USAGE, buildAgentContract, buildCommandRegistry, buildMutationPlan, CLEANUP_USAGE, commandActionSummary, COMMANDS_USAGE, inferRequiresProject, isAllowedCleanupAction, supportsCleanupLedger, } from "./cli/contract.js";
|
|
5
6
|
import { runDoctor, runFixtures, } from "./cli/doctor.js";
|
|
6
7
|
import { dataikuEnvironmentEnabled, loadEnvFile, } from "./cli/env.js";
|
|
7
8
|
import { FLAG_ALIASES, isNegativeNumberToken, parseArgs, SHORT_FLAGS, VALUE_FLAGS, } from "./cli/flags.js";
|
|
8
9
|
import { cleanupLedgerEntry, } from "./cli/helpers/cleanup.js";
|
|
9
|
-
import { commandFailureExitCode, commandFailureMessage, CommandResultFailure, isFailedWaitResult, planResult, projectResultFields,
|
|
10
|
+
import { commandFailureExitCode, commandFailureMessage, CommandResultFailure, flushCliWarnings, isFailedWaitResult, planResult, projectResultFields, setOutputFieldProjection, writeCommandResult, } from "./cli/output.js";
|
|
10
11
|
import { currentCommandContext, resolveCredentials, resolveTlsSettings, } from "./cli/runtime.js";
|
|
11
12
|
import { COMMANDS_RUN_HINT, missingActionError, noCommandError, requireArgs, unknownActionError, unknownResourceError, UsageError, } from "./cli/usage.js";
|
|
12
13
|
import { cliVersionResult, } from "./cli/version.js";
|
|
@@ -31,19 +32,28 @@ import { canonicalDssUrl, } from "./utils/dss-url.js";
|
|
|
31
32
|
// ---------------------------------------------------------------------------
|
|
32
33
|
// Auth commands (run before client creation)
|
|
33
34
|
// ---------------------------------------------------------------------------
|
|
34
|
-
function
|
|
35
|
+
function findUnboundLifecycleCleanup(entries) {
|
|
35
36
|
for (const [index, entry,] of entries.entries()) {
|
|
36
37
|
const parsed = parseArgs(entry.cleanup.argv);
|
|
37
38
|
const [resource, action,] = parsed.positional;
|
|
38
|
-
if (resource
|
|
39
|
+
if (resource === "app" && action === "delete-instance") {
|
|
40
|
+
if (parsed.flags["unconfirmed-creation"] === true)
|
|
41
|
+
continue;
|
|
42
|
+
const expected = parsed.flags["expect-project-incarnation"];
|
|
43
|
+
if (typeof expected !== "string")
|
|
44
|
+
return { index, resource, action, reason: "missing", };
|
|
45
|
+
if (!/^[0-9a-f]{64}$/.test(expected))
|
|
46
|
+
return { index, resource, action, reason: "invalid", };
|
|
39
47
|
continue;
|
|
40
|
-
|
|
48
|
+
}
|
|
49
|
+
if (resource === "project" && action === "delete") {
|
|
50
|
+
const expected = parsed.flags["expect-project-incarnation"];
|
|
51
|
+
if (typeof expected !== "string")
|
|
52
|
+
return { index, resource, action, reason: "missing", };
|
|
53
|
+
if (!/^[0-9a-f]{64}$/.test(expected))
|
|
54
|
+
return { index, resource, action, reason: "invalid", };
|
|
41
55
|
continue;
|
|
42
|
-
|
|
43
|
-
if (typeof expected !== "string")
|
|
44
|
-
return { index, reason: "missing", };
|
|
45
|
-
if (!/^[0-9a-f]{64}$/.test(expected))
|
|
46
|
-
return { index, reason: "invalid", };
|
|
56
|
+
}
|
|
47
57
|
}
|
|
48
58
|
return undefined;
|
|
49
59
|
}
|
|
@@ -83,8 +93,8 @@ async function runCleanup(flags) {
|
|
|
83
93
|
if (!apiKey) {
|
|
84
94
|
throw new UsageError("Missing API key. Set DATAIKU_API_KEY or pass --api-key.", "missing_required_flag");
|
|
85
95
|
}
|
|
86
|
-
const requestTimeoutMs = num(flags["request-timeout"]);
|
|
87
|
-
const retryMaxAttempts = num(flags["retries"]);
|
|
96
|
+
const requestTimeoutMs = num(flags["request-timeout"], "--request-timeout");
|
|
97
|
+
const retryMaxAttempts = num(flags["retries"], "--retries");
|
|
88
98
|
const client = new DataikuClient({
|
|
89
99
|
url,
|
|
90
100
|
apiKey,
|
|
@@ -118,9 +128,10 @@ async function runCleanup(flags) {
|
|
|
118
128
|
};
|
|
119
129
|
}
|
|
120
130
|
// Lifecycle binding is as load-bearing as server binding. Validate the whole
|
|
121
|
-
// reversed plan before step one so a later legacy app
|
|
122
|
-
// earlier cleanup applied and then fail on project-key
|
|
123
|
-
|
|
131
|
+
// reversed plan before step one so a later legacy app or project entry
|
|
132
|
+
// cannot leave an earlier cleanup applied and then fail on project-key
|
|
133
|
+
// reuse protection.
|
|
134
|
+
const lifecycleViolation = findUnboundLifecycleCleanup(ordered);
|
|
124
135
|
if (lifecycleViolation) {
|
|
125
136
|
return {
|
|
126
137
|
result: {
|
|
@@ -128,8 +139,8 @@ async function runCleanup(flags) {
|
|
|
128
139
|
steps,
|
|
129
140
|
lifecycleError: {
|
|
130
141
|
entryIndex: lifecycleViolation.index,
|
|
131
|
-
resource:
|
|
132
|
-
action:
|
|
142
|
+
resource: lifecycleViolation.resource,
|
|
143
|
+
action: lifecycleViolation.action,
|
|
133
144
|
reason: lifecycleViolation.reason,
|
|
134
145
|
},
|
|
135
146
|
},
|
|
@@ -146,6 +157,11 @@ async function runCleanup(flags) {
|
|
|
146
157
|
|| !commands[resource]?.[action]) {
|
|
147
158
|
throw new UsageError(`Invalid cleanup argv: ${entry.cleanup.argv.join(" ")}`);
|
|
148
159
|
}
|
|
160
|
+
if (resource === "project"
|
|
161
|
+
&& action === "delete"
|
|
162
|
+
&& typeof parsed.flags["expect-project-incarnation"] !== "string") {
|
|
163
|
+
throw new UsageError("Project cleanup entry is not bound to a project incarnation. Refusing deletion after possible project-key reuse.", "validation_failed", "Capture a new cleanup entry from the current CLI. Legacy entries cannot safely delete projects.");
|
|
164
|
+
}
|
|
149
165
|
if (resource === "app"
|
|
150
166
|
&& action === "delete-instance"
|
|
151
167
|
&& parsed.flags["unconfirmed-creation"] !== true
|
|
@@ -252,11 +268,11 @@ function supportedCommandFlags(entry) {
|
|
|
252
268
|
return supported;
|
|
253
269
|
}
|
|
254
270
|
function scopedCommandDiscoveryHint(resource, action) {
|
|
255
|
-
return `Use \`dss commands run --fields ${resource}.${action}
|
|
271
|
+
return `Use \`dss commands run --fields ${resource}.${action}\` to list the flags this command supports.`;
|
|
256
272
|
}
|
|
257
273
|
/**
|
|
258
274
|
* Resolve the registry field selectors for `commands run --fields`. Omitting the flag
|
|
259
|
-
* selects the
|
|
275
|
+
* selects the action summary; `--output` exports the full registry, but an explicitly supplied value carrying no selector is a
|
|
260
276
|
* usage error: an agent that asked for a scoped subset must never be handed the full
|
|
261
277
|
* registry instead.
|
|
262
278
|
*/
|
|
@@ -419,6 +435,7 @@ function validateMetaCommandInputs(resource, action, flags) {
|
|
|
419
435
|
throw missingActionError("commands", ["run",], COMMANDS_USAGE);
|
|
420
436
|
if (action !== "run")
|
|
421
437
|
throw unknownActionError("commands", action, ["run",]);
|
|
438
|
+
validateSupportedCommandFlags("commands", "run", flags);
|
|
422
439
|
commandRegistrySelectors(flags);
|
|
423
440
|
return action;
|
|
424
441
|
}
|
|
@@ -516,9 +533,10 @@ function batchStepCommandContext(argv) {
|
|
|
516
533
|
}
|
|
517
534
|
return { resource: positionals[0], action: positionals[1], };
|
|
518
535
|
}
|
|
519
|
-
const SENSITIVE_ARGV_VALUE_FLAGS =
|
|
520
|
-
"api-key",
|
|
521
|
-
|
|
536
|
+
const SENSITIVE_ARGV_VALUE_FLAGS = {
|
|
537
|
+
"api-key": true,
|
|
538
|
+
repository: true,
|
|
539
|
+
};
|
|
522
540
|
function redactArgv(argv) {
|
|
523
541
|
const redacted = [];
|
|
524
542
|
for (let index = 0; index < argv.length; index++) {
|
|
@@ -531,7 +549,7 @@ function redactArgv(argv) {
|
|
|
531
549
|
const eqIdx = arg.indexOf("=");
|
|
532
550
|
const rawFlagName = eqIdx === -1 ? arg.slice(2) : arg.slice(2, eqIdx);
|
|
533
551
|
const flagName = FLAG_ALIASES[rawFlagName] ?? rawFlagName;
|
|
534
|
-
if (SENSITIVE_ARGV_VALUE_FLAGS
|
|
552
|
+
if (SENSITIVE_ARGV_VALUE_FLAGS[flagName] === true) {
|
|
535
553
|
if (eqIdx !== -1) {
|
|
536
554
|
redacted.push(`${arg.slice(0, eqIdx + 1)}***`);
|
|
537
555
|
}
|
|
@@ -763,6 +781,7 @@ async function runMetaCommand(resource, action, flags) {
|
|
|
763
781
|
currentCommandContext.action = action;
|
|
764
782
|
if (action !== "run")
|
|
765
783
|
throw unknownActionError("commands", action, ["run",]);
|
|
784
|
+
validateSupportedCommandFlags("commands", "run", flags);
|
|
766
785
|
const selectors = commandRegistrySelectors(flags);
|
|
767
786
|
const registry = buildCommandRegistry();
|
|
768
787
|
for (const selector of selectors) {
|
|
@@ -778,7 +797,20 @@ async function runMetaCommand(resource, action, flags) {
|
|
|
778
797
|
throw unknownActionError(selectedResource, selectedAction, Object.keys(resourceActions));
|
|
779
798
|
}
|
|
780
799
|
}
|
|
781
|
-
|
|
800
|
+
const outputPath = flags["output"];
|
|
801
|
+
const selectedRegistry = selectors.length === 0
|
|
802
|
+
? registry
|
|
803
|
+
: projectResultFields(registry, selectors);
|
|
804
|
+
if (typeof outputPath === "string") {
|
|
805
|
+
const exported = selectedRegistry;
|
|
806
|
+
await fs.writeFile(outputPath, `${JSON.stringify(exported)}\n`, "utf-8");
|
|
807
|
+
return { action, result: { path: outputPath, }, exitCode: 0, };
|
|
808
|
+
}
|
|
809
|
+
return {
|
|
810
|
+
action,
|
|
811
|
+
result: selectors.length === 0 ? commandActionSummary(registry) : selectedRegistry,
|
|
812
|
+
exitCode: 0,
|
|
813
|
+
};
|
|
782
814
|
}
|
|
783
815
|
if (resource === "version") {
|
|
784
816
|
if (action !== undefined && action !== "run") {
|
|
@@ -972,8 +1004,8 @@ async function runBatch(flags) {
|
|
|
972
1004
|
apiKey,
|
|
973
1005
|
projectKey,
|
|
974
1006
|
verbose: flags["verbose"] === true,
|
|
975
|
-
requestTimeoutMs: num(flags["request-timeout"]),
|
|
976
|
-
retryMaxAttempts: num(flags["retries"]),
|
|
1007
|
+
requestTimeoutMs: num(flags["request-timeout"], "--request-timeout"),
|
|
1008
|
+
retryMaxAttempts: num(flags["retries"], "--retries"),
|
|
977
1009
|
tlsRejectUnauthorized,
|
|
978
1010
|
caCertPath,
|
|
979
1011
|
});
|
|
@@ -1050,7 +1082,9 @@ async function runBatch(flags) {
|
|
|
1050
1082
|
const stepFields = typeof stepFieldsFlag === "string"
|
|
1051
1083
|
? stepFieldsFlag.split(",").map((field) => field.trim()).filter((field) => field.length > 0)
|
|
1052
1084
|
: [];
|
|
1053
|
-
const stepResult = stepFields.length > 0
|
|
1085
|
+
const stepResult = stepFields.length > 0 && !(resource === "commands" && action === "run")
|
|
1086
|
+
? projectResultFields(result, stepFields)
|
|
1087
|
+
: result;
|
|
1054
1088
|
results.push({
|
|
1055
1089
|
index,
|
|
1056
1090
|
args: redactArgv(argv),
|
|
@@ -1175,6 +1209,8 @@ const MISSING_PROJECT_KEY_ERROR_PREFIX = "projectKey is required";
|
|
|
1175
1209
|
function errorExitCode(err) {
|
|
1176
1210
|
if (err instanceof CommandResultFailure)
|
|
1177
1211
|
return err.exitCode;
|
|
1212
|
+
if (err instanceof ClientValidationError && err.code === "ambiguous_outcome")
|
|
1213
|
+
return 2;
|
|
1178
1214
|
if (err instanceof UsageError || err instanceof ClientValidationError)
|
|
1179
1215
|
return 1;
|
|
1180
1216
|
if (err instanceof DataikuError)
|
|
@@ -1186,6 +1222,19 @@ function errorExitCode(err) {
|
|
|
1186
1222
|
function buildErrorReport(err) {
|
|
1187
1223
|
const context = rawCommandContext();
|
|
1188
1224
|
const exitCode = errorExitCode(err);
|
|
1225
|
+
if (err instanceof ClientValidationError && err.code === "ambiguous_outcome") {
|
|
1226
|
+
return {
|
|
1227
|
+
type: "error",
|
|
1228
|
+
ok: false,
|
|
1229
|
+
error: err.message,
|
|
1230
|
+
code: err.code,
|
|
1231
|
+
category: "dss",
|
|
1232
|
+
exitCode,
|
|
1233
|
+
...(err.hint ? { hint: err.hint, } : {}),
|
|
1234
|
+
...(err.details ? { details: err.details, } : {}),
|
|
1235
|
+
...context,
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1189
1238
|
if (err instanceof UsageError || err instanceof ClientValidationError) {
|
|
1190
1239
|
return {
|
|
1191
1240
|
type: "error",
|
|
@@ -1268,7 +1317,8 @@ function buildErrorReport(err) {
|
|
|
1268
1317
|
};
|
|
1269
1318
|
}
|
|
1270
1319
|
function writeErrorReport(err) {
|
|
1271
|
-
|
|
1320
|
+
flushCliWarnings();
|
|
1321
|
+
process.stdout.write(`${JSON.stringify(buildErrorReport(err))}\n`);
|
|
1272
1322
|
}
|
|
1273
1323
|
// ---------------------------------------------------------------------------
|
|
1274
1324
|
// Main
|
|
@@ -1276,12 +1326,14 @@ function writeErrorReport(err) {
|
|
|
1276
1326
|
async function main() {
|
|
1277
1327
|
loadEnvFile();
|
|
1278
1328
|
const { positional, flags, } = parseArgs(process.argv.slice(2));
|
|
1279
|
-
setCompactJsonOutput(flags["json"] === true);
|
|
1280
1329
|
const fieldsFlag = flags["fields"];
|
|
1281
1330
|
if (typeof fieldsFlag === "string") {
|
|
1282
1331
|
const selected = fieldsFlag.split(",").map((field) => field.trim()).filter((field) => field.length > 0);
|
|
1283
|
-
if (selected.length > 0)
|
|
1284
|
-
|
|
1332
|
+
if (selected.length > 0) {
|
|
1333
|
+
const commandsRunProjection = positional[0] === "commands" && positional[1] === "run";
|
|
1334
|
+
if (!commandsRunProjection)
|
|
1335
|
+
setOutputFieldProjection(selected);
|
|
1336
|
+
}
|
|
1285
1337
|
}
|
|
1286
1338
|
if (flags["version"] === true) {
|
|
1287
1339
|
writeCommandResult(cliVersionResult());
|
|
@@ -1324,6 +1376,14 @@ async function main() {
|
|
|
1324
1376
|
writeCommandResult(plan);
|
|
1325
1377
|
return;
|
|
1326
1378
|
}
|
|
1379
|
+
if (actionMeta.localHandler) {
|
|
1380
|
+
const result = await actionMeta.localHandler(args, flags);
|
|
1381
|
+
const failureExitCode = commandFailureExitCode(result);
|
|
1382
|
+
if (failureExitCode !== undefined)
|
|
1383
|
+
throw new CommandResultFailure(result, failureExitCode);
|
|
1384
|
+
writeCommandResult(result);
|
|
1385
|
+
return;
|
|
1386
|
+
}
|
|
1327
1387
|
const { url, apiKey, projectKey, tlsRejectUnauthorized, caCertPath, } = resolveCredentials(flags);
|
|
1328
1388
|
currentCommandContext.projectKey = projectKey;
|
|
1329
1389
|
if (!url) {
|
|
@@ -1332,8 +1392,8 @@ async function main() {
|
|
|
1332
1392
|
if (!apiKey) {
|
|
1333
1393
|
throw new UsageError("Missing API key.", "missing_required_flag", "Set DATAIKU_API_KEY or pass --api-key.", { requiredFlags: ["api-key",], env: ["DATAIKU_API_KEY",], });
|
|
1334
1394
|
}
|
|
1335
|
-
const requestTimeoutMs = num(flags["request-timeout"]);
|
|
1336
|
-
const retryMaxAttempts = num(flags["retries"]);
|
|
1395
|
+
const requestTimeoutMs = num(flags["request-timeout"], "--request-timeout");
|
|
1396
|
+
const retryMaxAttempts = num(flags["retries"], "--retries");
|
|
1337
1397
|
const client = new DataikuClient({
|
|
1338
1398
|
url,
|
|
1339
1399
|
apiKey,
|
|
@@ -1351,12 +1411,7 @@ async function main() {
|
|
|
1351
1411
|
const failureExitCode = commandFailureExitCode(result);
|
|
1352
1412
|
if (failureExitCode !== undefined)
|
|
1353
1413
|
throw new CommandResultFailure(result, failureExitCode);
|
|
1354
|
-
|
|
1355
|
-
process.stdout.write(result);
|
|
1356
|
-
}
|
|
1357
|
-
else {
|
|
1358
|
-
writeCommandResult(result);
|
|
1359
|
-
}
|
|
1414
|
+
writeCommandResult(result);
|
|
1360
1415
|
}
|
|
1361
1416
|
main().catch((err) => {
|
|
1362
1417
|
writeErrorReport(err);
|
package/dist/src/client.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { MetricsResource } from "./resources/metrics.js";
|
|
|
21
21
|
import { MlTasksResource } from "./resources/ml-tasks.js";
|
|
22
22
|
import { ModelEvaluationStoresResource } from "./resources/model-evaluation-stores.js";
|
|
23
23
|
import { NotebooksResource } from "./resources/notebooks.js";
|
|
24
|
+
import { ProjectGitResource } from "./resources/project-git.js";
|
|
24
25
|
import { ProjectLibraryResource } from "./resources/project-library.js";
|
|
25
26
|
import { ProjectsResource } from "./resources/projects.js";
|
|
26
27
|
import { RecipesResource } from "./resources/recipes.js";
|
|
@@ -44,6 +45,21 @@ export interface DataikuClientTraceEvent {
|
|
|
44
45
|
elapsedMs?: number;
|
|
45
46
|
detail?: string;
|
|
46
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Selected response metadata captured alongside a public API response body.
|
|
50
|
+
* A fixed whitelist of non-sensitive headers: authorization data is never
|
|
51
|
+
* returned. Null means the header was absent.
|
|
52
|
+
*/
|
|
53
|
+
export interface DataikuClientResponseMeta {
|
|
54
|
+
/** `DSS-Version` — version of the DSS backend answering the request. */
|
|
55
|
+
dssVersion: string | null;
|
|
56
|
+
/** `DSS-API-Version` — version of the API server handling the request. */
|
|
57
|
+
dssApiVersion: string | null;
|
|
58
|
+
/** `Date` — origin server timestamp, when present. */
|
|
59
|
+
date: string | null;
|
|
60
|
+
/** Server-provided request id, when present. */
|
|
61
|
+
requestId: string | null;
|
|
62
|
+
}
|
|
47
63
|
export interface DataikuClientConfig {
|
|
48
64
|
/** DSS base URL (e.g. https://dss.example.com) */
|
|
49
65
|
url: string;
|
|
@@ -104,6 +120,7 @@ export declare class DataikuClient {
|
|
|
104
120
|
private bundlesResource?;
|
|
105
121
|
private projectDeployerResource?;
|
|
106
122
|
private projectLibraryResource?;
|
|
123
|
+
private projectGitResource?;
|
|
107
124
|
private streamingEndpointsResource?;
|
|
108
125
|
private continuousActivitiesResource?;
|
|
109
126
|
private statisticsResource?;
|
|
@@ -139,6 +156,7 @@ export declare class DataikuClient {
|
|
|
139
156
|
get bundles(): BundlesResource;
|
|
140
157
|
get projectDeployer(): ProjectDeployerResource;
|
|
141
158
|
get projectLibrary(): ProjectLibraryResource;
|
|
159
|
+
get projectGit(): ProjectGitResource;
|
|
142
160
|
get streamingEndpoints(): StreamingEndpointsResource;
|
|
143
161
|
get continuousActivities(): ContinuousActivitiesResource;
|
|
144
162
|
get statistics(): StatisticsResource;
|
|
@@ -160,6 +178,15 @@ export declare class DataikuClient {
|
|
|
160
178
|
getRequestTimeoutMs(): number;
|
|
161
179
|
resolveProjectKey(paramValue?: string): string;
|
|
162
180
|
get<T = unknown>(path: string): Promise<T>;
|
|
181
|
+
/**
|
|
182
|
+
* GET returning the parsed JSON body plus selected response metadata
|
|
183
|
+
* (documented DSS version headers, server date, request id). Authorization
|
|
184
|
+
* headers are never included.
|
|
185
|
+
*/
|
|
186
|
+
getWithMetadata<T = unknown>(path: string): Promise<{
|
|
187
|
+
data: T;
|
|
188
|
+
meta: DataikuClientResponseMeta;
|
|
189
|
+
}>;
|
|
163
190
|
getText(path: string): Promise<string>;
|
|
164
191
|
getTextLimited(path: string, maxBytes: number): Promise<{
|
|
165
192
|
text: string;
|
|
@@ -177,10 +204,13 @@ export declare class DataikuClient {
|
|
|
177
204
|
put<T = unknown>(path: string, body: unknown): Promise<T>;
|
|
178
205
|
del(path: string): Promise<void>;
|
|
179
206
|
putVoid(path: string, body: unknown): Promise<void>;
|
|
180
|
-
|
|
207
|
+
private uploadResponse;
|
|
208
|
+
upload(path: string, filePath: string, fileName?: string): Promise<void>;
|
|
209
|
+
uploadJson<T>(path: string, filePath: string, fileName?: string): Promise<T>;
|
|
181
210
|
stream(path: string): Promise<Response>;
|
|
182
211
|
private getHeaders;
|
|
183
212
|
private getAnyHeaders;
|
|
213
|
+
private responseMeta;
|
|
184
214
|
private logTrace;
|
|
185
215
|
/**
|
|
186
216
|
* Validate raw data against a TypeBox schema, throwing on structural mismatch.
|
package/dist/src/client.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { readFileSync, } from "node:fs";
|
|
1
|
+
import { openAsBlob, readFileSync, } from "node:fs";
|
|
2
|
+
import { basename, } from "node:path";
|
|
2
3
|
import { getCACertificates, } from "node:tls";
|
|
3
4
|
import { Value, } from "@sinclair/typebox/value";
|
|
4
5
|
import { safeParseSchema, } from "./schemas.js";
|
|
@@ -25,6 +26,7 @@ import { MetricsResource, } from "./resources/metrics.js";
|
|
|
25
26
|
import { MlTasksResource, } from "./resources/ml-tasks.js";
|
|
26
27
|
import { ModelEvaluationStoresResource, } from "./resources/model-evaluation-stores.js";
|
|
27
28
|
import { NotebooksResource, } from "./resources/notebooks.js";
|
|
29
|
+
import { ProjectGitResource, } from "./resources/project-git.js";
|
|
28
30
|
import { ProjectLibraryResource, } from "./resources/project-library.js";
|
|
29
31
|
import { ProjectsResource, } from "./resources/projects.js";
|
|
30
32
|
import { RecipesResource, } from "./resources/recipes.js";
|
|
@@ -189,6 +191,7 @@ export class DataikuClient {
|
|
|
189
191
|
bundlesResource;
|
|
190
192
|
projectDeployerResource;
|
|
191
193
|
projectLibraryResource;
|
|
194
|
+
projectGitResource;
|
|
192
195
|
streamingEndpointsResource;
|
|
193
196
|
continuousActivitiesResource;
|
|
194
197
|
statisticsResource;
|
|
@@ -272,6 +275,9 @@ export class DataikuClient {
|
|
|
272
275
|
get projectLibrary() {
|
|
273
276
|
return (this.projectLibraryResource ??= new ProjectLibraryResource(this));
|
|
274
277
|
}
|
|
278
|
+
get projectGit() {
|
|
279
|
+
return (this.projectGitResource ??= new ProjectGitResource(this));
|
|
280
|
+
}
|
|
275
281
|
get streamingEndpoints() {
|
|
276
282
|
return (this.streamingEndpointsResource ??= new StreamingEndpointsResource(this));
|
|
277
283
|
}
|
|
@@ -355,6 +361,19 @@ export class DataikuClient {
|
|
|
355
361
|
});
|
|
356
362
|
return this.parseJsonResponse(res);
|
|
357
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* GET returning the parsed JSON body plus selected response metadata
|
|
366
|
+
* (documented DSS version headers, server date, request id). Authorization
|
|
367
|
+
* headers are never included.
|
|
368
|
+
*/
|
|
369
|
+
async getWithMetadata(path) {
|
|
370
|
+
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
371
|
+
method: "GET",
|
|
372
|
+
headers: this.getHeaders(),
|
|
373
|
+
});
|
|
374
|
+
const data = await this.parseJsonResponse(res);
|
|
375
|
+
return { data, meta: this.responseMeta(res.headers), };
|
|
376
|
+
}
|
|
358
377
|
async getText(path) {
|
|
359
378
|
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
360
379
|
method: "GET",
|
|
@@ -459,19 +478,23 @@ export class DataikuClient {
|
|
|
459
478
|
body: JSON.stringify(body),
|
|
460
479
|
});
|
|
461
480
|
}
|
|
462
|
-
async
|
|
463
|
-
const { openAsBlob, } = await import("node:fs");
|
|
464
|
-
const { basename, } = await import("node:path");
|
|
481
|
+
async uploadResponse(path, filePath, fileName) {
|
|
465
482
|
const fileBlob = await openAsBlob(filePath);
|
|
466
|
-
const fileName = basename(filePath);
|
|
467
483
|
const formData = new FormData();
|
|
468
|
-
formData.append("file", fileBlob, fileName);
|
|
469
|
-
|
|
484
|
+
formData.append("file", fileBlob, fileName ?? basename(filePath));
|
|
485
|
+
return this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
470
486
|
method: "POST",
|
|
471
487
|
headers: { Authorization: `Bearer ${this.apiKey}`, },
|
|
472
488
|
body: formData,
|
|
473
489
|
});
|
|
474
490
|
}
|
|
491
|
+
async upload(path, filePath, fileName) {
|
|
492
|
+
await this.uploadResponse(path, filePath, fileName);
|
|
493
|
+
}
|
|
494
|
+
async uploadJson(path, filePath, fileName) {
|
|
495
|
+
const response = await this.uploadResponse(path, filePath, fileName);
|
|
496
|
+
return this.parseJsonResponse(response);
|
|
497
|
+
}
|
|
475
498
|
async stream(path) {
|
|
476
499
|
return this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
477
500
|
method: "GET",
|
|
@@ -492,6 +515,15 @@ export class DataikuClient {
|
|
|
492
515
|
Accept: "*/*",
|
|
493
516
|
};
|
|
494
517
|
}
|
|
518
|
+
responseMeta(headers) {
|
|
519
|
+
// Fixed non-sensitive whitelist. `Headers.get` is case-insensitive.
|
|
520
|
+
return {
|
|
521
|
+
dssVersion: headers.get("dss-version"),
|
|
522
|
+
dssApiVersion: headers.get("dss-api-version"),
|
|
523
|
+
date: headers.get("date"),
|
|
524
|
+
requestId: this.requestIdFromHeaders(headers) ?? null,
|
|
525
|
+
};
|
|
526
|
+
}
|
|
495
527
|
logTrace(event) {
|
|
496
528
|
if (this.verbose)
|
|
497
529
|
this.onTrace({ type: "trace", ...event, });
|