dataiku-sdk 3.6.0 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/bin/bun-version.js +14 -0
- package/bin/dss.js +23 -12
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +74 -1
- package/dist/packages/types/src/index.js +53 -1
- package/dist/src/cli/coerce.d.ts +1 -0
- package/dist/src/cli/coerce.js +19 -15
- package/dist/src/cli/command-syntax.json +3521 -416
- package/dist/src/cli/commands/analysis.js +19 -1
- package/dist/src/cli/commands/bundle.js +22 -0
- package/dist/src/cli/commands/connection.js +26 -2
- package/dist/src/cli/commands/data-quality.js +33 -1
- package/dist/src/cli/commands/dataset.js +35 -4
- package/dist/src/cli/commands/discussion.js +15 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/flow.d.ts +5 -0
- package/dist/src/cli/commands/flow.js +57 -0
- package/dist/src/cli/commands/future.js +8 -0
- package/dist/src/cli/commands/index.js +6 -0
- package/dist/src/cli/commands/job.js +29 -6
- package/dist/src/cli/commands/metrics.js +21 -1
- package/dist/src/cli/commands/ml-task.d.ts +5 -0
- package/dist/src/cli/commands/ml-task.js +86 -16
- package/dist/src/cli/commands/mlflow.d.ts +11 -0
- package/dist/src/cli/commands/mlflow.js +121 -0
- package/dist/src/cli/commands/plugin.js +5 -11
- package/dist/src/cli/commands/project-deployer.js +70 -0
- package/dist/src/cli/commands/project-library.js +2 -5
- package/dist/src/cli/commands/project.js +8 -0
- package/dist/src/cli/commands/recipe.d.ts +9 -0
- package/dist/src/cli/commands/recipe.js +93 -7
- package/dist/src/cli/commands/saved-model.d.ts +1 -1
- package/dist/src/cli/commands/saved-model.js +9 -76
- package/dist/src/cli/commands/streaming-endpoint.js +30 -1
- package/dist/src/cli/commands/webapp.js +8 -0
- package/dist/src/cli/commands/wiki.js +22 -0
- package/dist/src/cli/commands/workspace.js +8 -0
- package/dist/src/cli/contract.d.ts +16 -15
- package/dist/src/cli/contract.js +139 -3404
- package/dist/src/cli/doctor.d.ts +13 -0
- package/dist/src/cli/doctor.js +41 -1
- package/dist/src/cli/flags.d.ts +7 -1
- package/dist/src/cli/flags.js +78 -6
- package/dist/src/cli/helpers/app-successor.js +1 -2
- package/dist/src/cli/helpers/documentation.d.ts +21 -0
- package/dist/src/cli/helpers/documentation.js +53 -0
- package/dist/src/cli/helpers/trained-model.d.ts +26 -0
- package/dist/src/cli/helpers/trained-model.js +144 -0
- package/dist/src/cli/list-output.d.ts +7 -0
- package/dist/src/cli/list-output.js +77 -0
- package/dist/src/cli/list-shapes.d.ts +21 -0
- package/dist/src/cli/list-shapes.js +111 -0
- package/dist/src/cli/output-schemas.d.ts +0 -14
- package/dist/src/cli/output-schemas.js +4 -7
- package/dist/src/cli/output.d.ts +0 -1
- package/dist/src/cli/output.js +27 -19
- package/dist/src/cli/plans.d.ts +14 -0
- package/dist/src/cli/plans.js +3684 -0
- package/dist/src/cli/usage.d.ts +7 -0
- package/dist/src/cli/usage.js +20 -1
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +2 -2
- package/dist/src/cli.js +127 -113
- package/dist/src/client.d.ts +52 -43
- package/dist/src/client.js +267 -96
- package/dist/src/errors.d.ts +11 -1
- package/dist/src/errors.js +31 -47
- package/dist/src/generated/action-output-schemas.json +345 -89
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +3 -0
- package/dist/src/resources/analyses.d.ts +8 -0
- package/dist/src/resources/analyses.js +13 -1
- package/dist/src/resources/bundles.d.ts +34 -0
- package/dist/src/resources/bundles.js +67 -0
- package/dist/src/resources/connections.d.ts +9 -0
- package/dist/src/resources/connections.js +15 -0
- package/dist/src/resources/data-quality.d.ts +20 -0
- package/dist/src/resources/data-quality.js +30 -0
- package/dist/src/resources/dataset-create.d.ts +19 -0
- package/dist/src/resources/dataset-create.js +66 -0
- package/dist/src/resources/datasets.d.ts +25 -15
- package/dist/src/resources/datasets.js +59 -111
- package/dist/src/resources/discussions.d.ts +5 -0
- package/dist/src/resources/discussions.js +7 -0
- package/dist/src/resources/documentation.d.ts +25 -0
- package/dist/src/resources/documentation.js +16 -0
- package/dist/src/resources/flow.d.ts +48 -0
- package/dist/src/resources/flow.js +65 -0
- package/dist/src/resources/futures.d.ts +23 -0
- package/dist/src/resources/futures.js +32 -0
- package/dist/src/resources/jobs.js +102 -2
- package/dist/src/resources/metrics.d.ts +10 -0
- package/dist/src/resources/metrics.js +10 -0
- package/dist/src/resources/ml-tasks.d.ts +65 -2
- package/dist/src/resources/ml-tasks.js +102 -28
- package/dist/src/resources/mlflow-extension.d.ts +73 -0
- package/dist/src/resources/mlflow-extension.js +105 -0
- package/dist/src/resources/projects.d.ts +6 -0
- package/dist/src/resources/projects.js +8 -0
- package/dist/src/resources/recipe-create.d.ts +65 -0
- package/dist/src/resources/recipe-create.js +338 -0
- package/dist/src/resources/recipes.d.ts +28 -29
- package/dist/src/resources/recipes.js +126 -245
- package/dist/src/resources/saved-models.d.ts +10 -6
- package/dist/src/resources/saved-models.js +16 -14
- package/dist/src/resources/scenarios.js +8 -14
- package/dist/src/resources/streaming-endpoints.d.ts +12 -0
- package/dist/src/resources/streaming-endpoints.js +15 -0
- package/dist/src/resources/trained-model.d.ts +60 -0
- package/dist/src/resources/trained-model.js +98 -0
- package/dist/src/resources/webapps.d.ts +8 -0
- package/dist/src/resources/webapps.js +10 -0
- package/dist/src/resources/wiki.d.ts +5 -0
- package/dist/src/resources/wiki.js +7 -0
- package/dist/src/resources/workspaces.d.ts +2 -0
- package/dist/src/resources/workspaces.js +4 -0
- package/dist/src/utils/response-file.js +20 -9
- package/dist/src/utils/secret-sanitize.d.ts +7 -0
- package/dist/src/utils/secret-sanitize.js +46 -0
- package/package.json +25 -26
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
- package/skills/dataiku-dss/references/authentication.md +3 -1
- package/skills/dataiku-dss/references/coding.md +1 -0
- package/skills/dataiku-dss/references/discovery.md +4 -6
- package/skills/dataiku-dss/references/troubleshooting.md +3 -2
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ Agent-only TypeScript SDK and `dss` CLI for Dataiku DSS automation.
|
|
|
4
4
|
|
|
5
5
|
## API coverage
|
|
6
6
|
|
|
7
|
-
The SDK and CLI include scenario run management; project, recipe, and dataset metadata; managed datasets, lineage, and table imports; external saved models and MLflow
|
|
7
|
+
The SDK and CLI include scenario run management; project, recipe, and dataset metadata; managed datasets, lineage, and table imports; flow schema propagation and documentation; grouping and prepare recipes built from flags; Visual ML tasks, model diagnostics (subpopulation analyses, partial dependencies, time series residuals) and model documentation for lab models and saved-model versions; external saved models, MLflow imports, and the MLflow experiment-tracking extension; Project Deployer infrastructures, deployments, and published projects; LLM completions and embeddings; knowledge banks; project folders and data collections; users, groups, connections, macros, and plugin administration. Discover exact inputs with `dss commands run` and scoped command contracts rather than assuming flags from API field names. MLflow archive/folder imports and evaluations resolve the external container config explicitly: DSS 15 requires the `containerExecConfigName` parameter, so the SDK/CLI send `NONE` rather than omitting it (the plan surface matches the executed request).
|
|
8
8
|
|
|
9
9
|
The [DSS 15 coverage matrix](docs/API-COVERAGE-DSS-15.md) maps documented operations to SDK methods and CLI actions, records documentation discrepancies, and identifies remaining gaps. A [machine-readable matrix](docs/API-COVERAGE-DSS-15.json) is also available. Coverage is not full DSS parity or live compatibility certification: availability depends on DSS version, permissions, licensing, and installed capabilities. One known documentation discrepancy: `plugin git-branches` is served by GET on DSS 15 (the same route rejects POST with 405 `rawMethodPOSTnotsupported`, contrary to the public docs).
|
|
10
10
|
|
|
@@ -16,7 +16,7 @@ Malformed user/group listings and scenario run histories fail explicitly instead
|
|
|
16
16
|
|
|
17
17
|
## Platform support
|
|
18
18
|
|
|
19
|
-
The published `dss` CLI requires Bun >= 1.4.
|
|
19
|
+
The published `dss` CLI requires Bun >= 1.4.2 and supports Linux, macOS, and Windows. CI runs Bun 1.4.2 on all three operating systems. The runtime dependency is pure JavaScript, so the same package also runs on every Bun-supported x64 and ARM64 system.
|
|
20
20
|
|
|
21
21
|
Run directly with Bun:
|
|
22
22
|
|
|
@@ -29,7 +29,7 @@ Or install the npm binary:
|
|
|
29
29
|
```text
|
|
30
30
|
npm install --global dataiku-sdk
|
|
31
31
|
```
|
|
32
|
-
The installed `dss` bin bootstraps Bun with automatic `.env` preloading disabled, so Bun >= 1.4.
|
|
32
|
+
The installed `dss` bin bootstraps Bun with automatic `.env` preloading disabled, so Bun >= 1.4.2 must be on `PATH`; the CLI's own `.env` handling stays authoritative. `bunx --bun` forces Bun's Node-compatibility mode, which provides the same no-preload guarantee across operating systems.
|
|
33
33
|
|
|
34
34
|
Bun is the only supported runtime and the package manager. Examples below assume an installed `dss` binary. From a checkout, use `bun --no-env-file src/cli.ts ...` or the packaged launcher `bun --no-env-file ./bin/dss.js ...`. From another working directory, pass the checkout's absolute `bin/dss.js` path to Bun.
|
|
35
35
|
`--no-env-file` disables Bun's automatic preloading only; the CLI still applies its documented `.env` handling unless `DATAIKU_DISABLE_ENV=1` is set.
|
|
@@ -201,21 +201,29 @@ essentials: exactly one compact JSON value on stdout per command (void success i
|
|
|
201
201
|
failures as one structured JSON error object on stdout with a nonzero exit code, stderr reserved
|
|
202
202
|
for JSONL diagnostics only, and exit codes `0` success / `1` usage error / `2` DSS or internal
|
|
203
203
|
error / `3` transient DSS error / `4` failed long-running result or assertion. `--retries N`
|
|
204
|
-
controls idempotent GET retries only
|
|
204
|
+
controls idempotent GET retries only, and only transient failures (408/425/429, 5xx without a known
|
|
205
|
+
non-transient cause, connection resets, timeouts) are retried: a refused connection, unknown host,
|
|
206
|
+
bad URL, or untrusted certificate fails at once. `dss sql query --start-retries N` retries transient failures
|
|
205
207
|
while starting a query. For portable multi-step mutations, prefer `dss batch`; shell chaining and
|
|
206
208
|
pipeline exit semantics differ across platforms.
|
|
207
209
|
|
|
210
|
+
`*.list` commands print compact items (identifiers, kind, next-step fields) by default; `--full`
|
|
211
|
+
returns the DSS objects, `--contains TEXT` filters on identifiers, and `--limit N` caps the count
|
|
212
|
+
with a `list_truncated` warning.
|
|
213
|
+
|
|
208
214
|
Discover the command surface with scoped calls:
|
|
209
215
|
|
|
210
216
|
```text
|
|
211
217
|
dss commands run
|
|
218
|
+
dss commands run --contains "propagate schema"
|
|
212
219
|
dss commands run --fields dataset.create.usage,dataset.create.description,dataset.create.flags,dataset.create.examples
|
|
213
220
|
dss commands run --output commands.json
|
|
214
221
|
dss agent contract --fields protocol,agentContractVersion,cli,stdio,planning,compatibility
|
|
215
222
|
```
|
|
216
223
|
|
|
217
|
-
`dss commands run` prints the compact resource/action summary
|
|
218
|
-
|
|
224
|
+
`dss commands run` prints the compact resource/action summary, and `--contains TEXT` finds the top 5
|
|
225
|
+
actions for a concept; registry entries never dump to stdout — they travel only via `--output PATH`.
|
|
226
|
+
See the canonical skill for the full `--fields` projection rules and planning workflow.
|
|
219
227
|
|
|
220
228
|
## Agent skill installation
|
|
221
229
|
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a Bun version string is at least `minimum`, comparing only the
|
|
3
|
+
* release part (major.minor.patch). Canary builds print e.g.
|
|
4
|
+
* `1.4.3-canary.12+abc`; npm-style semver ranges (and Bun.semver) reject every
|
|
5
|
+
* prerelease, which would lock out anyone on a canary channel.
|
|
6
|
+
*/
|
|
7
|
+
export function versionAtLeast(actual, minimum,) {
|
|
8
|
+
const a = actual.split(/[-+]/,)[0].split(".",).map(Number,);
|
|
9
|
+
const m = minimum.split(".",).map(Number,);
|
|
10
|
+
for (let i = 0; i < m.length; i++) {
|
|
11
|
+
if ((a[i] ?? 0) !== m[i]) return (a[i] ?? 0) > m[i];
|
|
12
|
+
}
|
|
13
|
+
return true;
|
|
14
|
+
}
|
package/bin/dss.js
CHANGED
|
@@ -7,6 +7,7 @@ import { spawnSync, } from "node:child_process";
|
|
|
7
7
|
import { readFileSync, } from "node:fs";
|
|
8
8
|
import * as path from "node:path";
|
|
9
9
|
import { fileURLToPath, pathToFileURL, } from "node:url";
|
|
10
|
+
import { versionAtLeast, } from "./bun-version.js";
|
|
10
11
|
|
|
11
12
|
const runningUnderBun = typeof process.versions.bun === "string";
|
|
12
13
|
const bunNodeCompatibilityMode = runningUnderBun
|
|
@@ -26,21 +27,33 @@ const MINIMUM_BUN = JSON.parse(
|
|
|
26
27
|
).engines.bun.replace(/^>=/, "",);
|
|
27
28
|
|
|
28
29
|
if (!runningUnderBun) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
)
|
|
30
|
+
// Check before spawning: an old Bun may reject --no-env-file during argument
|
|
31
|
+
// parsing, before any launcher code could report a JSON error.
|
|
32
|
+
const probe = spawnSync("bun", ["--version",], { encoding: "utf-8", },);
|
|
33
|
+
const found = probe.error ? undefined : probe.stdout.trim();
|
|
34
|
+
const result = found !== undefined && versionAtLeast(found, MINIMUM_BUN,)
|
|
35
|
+
? spawnSync(
|
|
36
|
+
"bun",
|
|
37
|
+
["--no-env-file", fileURLToPath(import.meta.url,), ...process.argv.slice(2,),],
|
|
38
|
+
{ stdio: "inherit", env: process.env, },
|
|
39
|
+
)
|
|
40
|
+
: {
|
|
41
|
+
error: new Error(
|
|
42
|
+
found ? `Bun ${found} is older than the required ${MINIMUM_BUN}.` : "bun not found",
|
|
43
|
+
),
|
|
44
|
+
};
|
|
34
45
|
if (result.error) {
|
|
35
46
|
process.stdout.write(`${
|
|
36
47
|
JSON.stringify({
|
|
37
48
|
type: "error",
|
|
38
49
|
ok: false,
|
|
39
|
-
error:
|
|
50
|
+
error: `Unable to start the required Bun runtime: ${result.error.message}`,
|
|
40
51
|
code: "internal_error",
|
|
41
52
|
category: "internal",
|
|
42
53
|
exitCode: 2,
|
|
43
|
-
hint:
|
|
54
|
+
hint: found
|
|
55
|
+
? `Update Bun to ${MINIMUM_BUN} or newer: run \`bun upgrade\`.`
|
|
56
|
+
: `Install Bun ${MINIMUM_BUN} or newer (https://bun.sh) and ensure \`bun\` is on PATH.`,
|
|
44
57
|
},)
|
|
45
58
|
}\n`,);
|
|
46
59
|
process.exitCode = 2;
|
|
@@ -49,7 +62,7 @@ if (!runningUnderBun) {
|
|
|
49
62
|
} else {
|
|
50
63
|
process.exitCode = result.status ?? 1;
|
|
51
64
|
}
|
|
52
|
-
} else if (!
|
|
65
|
+
} else if (!versionAtLeast(Bun.version, MINIMUM_BUN,)) {
|
|
53
66
|
process.stdout.write(`${
|
|
54
67
|
JSON.stringify({
|
|
55
68
|
type: "error",
|
|
@@ -58,7 +71,7 @@ if (!runningUnderBun) {
|
|
|
58
71
|
code: "internal_error",
|
|
59
72
|
category: "internal",
|
|
60
73
|
exitCode: 2,
|
|
61
|
-
hint: `
|
|
74
|
+
hint: `Update Bun to ${MINIMUM_BUN} or newer: run \`bun upgrade\`.`,
|
|
62
75
|
},)
|
|
63
76
|
}\n`,);
|
|
64
77
|
process.exitCode = 2;
|
|
@@ -91,7 +104,6 @@ if (!runningUnderBun) {
|
|
|
91
104
|
// build revision; only a full lowercase hexadecimal revision is accepted, so an
|
|
92
105
|
// inherited or corrupt variable never stands in for packaged metadata.
|
|
93
106
|
if (usesDistCli) {
|
|
94
|
-
process.env.DSS_LOAD_SOURCE = "dist";
|
|
95
107
|
delete process.env.DSS_BUILD_REVISION;
|
|
96
108
|
try {
|
|
97
109
|
const metadata = await Bun.file(path.resolve(here, "../dist/build-metadata.json",),).json();
|
|
@@ -103,8 +115,7 @@ if (!runningUnderBun) {
|
|
|
103
115
|
// Dist without build metadata: provenance still reports dist source.
|
|
104
116
|
}
|
|
105
117
|
} else {
|
|
106
|
-
//
|
|
107
|
-
delete process.env.DSS_LOAD_SOURCE;
|
|
118
|
+
// Only a dist run may carry a build revision; never trust an inherited one.
|
|
108
119
|
delete process.env.DSS_BUILD_REVISION;
|
|
109
120
|
}
|
|
110
121
|
|
package/dist/build-metadata.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"format":1,"buildRevision":"
|
|
1
|
+
{"format":1,"buildRevision":"a980b6ec9bd11f68d319f7034152d7f9dd4ccd83"}
|
|
@@ -166,6 +166,24 @@ export declare const RecipeCreateOptionsSchema: import("@sinclair/typebox").TObj
|
|
|
166
166
|
fuzzyNormalize: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
167
167
|
projectKey: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
168
168
|
outputFolder: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
169
|
+
/** Grouping: key columns. */
|
|
170
|
+
groupBy: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
171
|
+
/** Grouping: `COL:FN[+FN]` specs (count, countDistinct, sum, avg, min, max, median, stddev, first, last, concat, ...). */
|
|
172
|
+
aggregate: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
173
|
+
/** Grouping: ordering column for first/last aggregates. */
|
|
174
|
+
orderBy: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
175
|
+
/** Prepare: `OLD=NEW` column renames. */
|
|
176
|
+
rename: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
177
|
+
/** Prepare: `COL=VALUE` fills for empty cells. */
|
|
178
|
+
fillEmpty: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
179
|
+
/** Prepare: one `NAME=GREL_EXPRESSION` computed column. */
|
|
180
|
+
formula: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
181
|
+
/** Prepare: keep only rows where this GREL expression is true. */
|
|
182
|
+
filter: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
183
|
+
/** Prepare: columns to remove. */
|
|
184
|
+
dropColumns: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
185
|
+
/** Prepare: the only columns to keep. */
|
|
186
|
+
keepColumns: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
169
187
|
}>;
|
|
170
188
|
export type RecipeCreateOptions = Static<typeof RecipeCreateOptionsSchema>;
|
|
171
189
|
export declare const RecipeCreateResultSchema: import("@sinclair/typebox").TObject<{
|
|
@@ -173,13 +191,68 @@ export declare const RecipeCreateResultSchema: import("@sinclair/typebox").TObje
|
|
|
173
191
|
type: import("@sinclair/typebox").TString;
|
|
174
192
|
createdDatasets: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
175
193
|
joinConfigured: import("@sinclair/typebox").TBoolean;
|
|
194
|
+
/** Grouping keys/aggregates or prepare steps from flags were written to the payload. */
|
|
195
|
+
payloadConfigured: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
176
196
|
outputProvisioningFallbackUsed: import("@sinclair/typebox").TBoolean;
|
|
177
197
|
outputFolder: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
178
198
|
temporaryOutputDataset: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
179
199
|
temporaryOutputDatasetDeleted: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
180
|
-
|
|
200
|
+
/** Output datasets whose schema DSS computed and the client applied after creation. */
|
|
201
|
+
outputSchemaUpdated: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
202
|
+
/** Why DSS could not compute the output schema; the recipe itself was created. */
|
|
203
|
+
outputSchemaUpdateError: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
181
204
|
}>;
|
|
182
205
|
export type RecipeCreateResult = Static<typeof RecipeCreateResultSchema>;
|
|
206
|
+
/** One output from GET /recipes/{name}/schema-update (dataikuapi compute_schema_updates). */
|
|
207
|
+
export declare const RecipeSchemaUpdateComputableSchema: import("@sinclair/typebox").TObject<{
|
|
208
|
+
type: import("@sinclair/typebox").TString;
|
|
209
|
+
id: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
210
|
+
datasetName: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
211
|
+
previousSchemaWasEmpty: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
212
|
+
incompatibilities: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
213
|
+
newSchema: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
|
|
214
|
+
columns: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TUnknown>>;
|
|
215
|
+
}>>;
|
|
216
|
+
}>;
|
|
217
|
+
export type RecipeSchemaUpdateComputable = Static<typeof RecipeSchemaUpdateComputableSchema>;
|
|
218
|
+
export declare const RecipeSchemaUpdatesSchema: import("@sinclair/typebox").TObject<{
|
|
219
|
+
totalIncompatibilities: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
220
|
+
computables: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
221
|
+
type: import("@sinclair/typebox").TString;
|
|
222
|
+
id: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
223
|
+
datasetName: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
224
|
+
previousSchemaWasEmpty: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
225
|
+
incompatibilities: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
226
|
+
newSchema: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
|
|
227
|
+
columns: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TUnknown>>;
|
|
228
|
+
}>>;
|
|
229
|
+
}>>;
|
|
230
|
+
recipeChanges: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnknown>>;
|
|
231
|
+
}>;
|
|
232
|
+
export type RecipeSchemaUpdates = Static<typeof RecipeSchemaUpdatesSchema>;
|
|
233
|
+
export declare const RecipeUpdateSchemaResultSchema: import("@sinclair/typebox").TObject<{
|
|
234
|
+
recipeName: import("@sinclair/typebox").TString;
|
|
235
|
+
/** Output ids whose schema was replaced by the DSS-computed schema. */
|
|
236
|
+
updated: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
237
|
+
/** Output ids whose schema differs from the computed one but was left as is (`derivedOutputs`, `dryRun`). */
|
|
238
|
+
pending: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
239
|
+
/** Output ids whose current schema already matches. */
|
|
240
|
+
unchanged: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
241
|
+
totalIncompatibilities: import("@sinclair/typebox").TNumber;
|
|
242
|
+
computables: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
243
|
+
type: import("@sinclair/typebox").TString;
|
|
244
|
+
id: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
245
|
+
datasetName: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
246
|
+
previousSchemaWasEmpty: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
247
|
+
incompatibilities: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
|
|
248
|
+
newSchema: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
|
|
249
|
+
columns: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TUnknown>>;
|
|
250
|
+
}>>;
|
|
251
|
+
}>>;
|
|
252
|
+
/** `datasetsNeedingAction` entries from the DSS apply responses. */
|
|
253
|
+
datasetsNeedingAction: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnknown>>;
|
|
254
|
+
}>;
|
|
255
|
+
export type RecipeUpdateSchemaResult = Static<typeof RecipeUpdateSchemaResultSchema>;
|
|
183
256
|
export declare const JobSummarySchema: import("@sinclair/typebox").TObject<{
|
|
184
257
|
def: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
|
|
185
258
|
id: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
@@ -211,17 +211,69 @@ export const RecipeCreateOptionsSchema = Type.Object({
|
|
|
211
211
|
fuzzyNormalize: Type.Optional(Type.Boolean()),
|
|
212
212
|
projectKey: Type.Optional(Type.String()),
|
|
213
213
|
outputFolder: Type.Optional(Type.String()),
|
|
214
|
+
/** Grouping: key columns. */
|
|
215
|
+
groupBy: Type.Optional(Type.Array(Type.String())),
|
|
216
|
+
/** Grouping: `COL:FN[+FN]` specs (count, countDistinct, sum, avg, min, max, median, stddev, first, last, concat, ...). */
|
|
217
|
+
aggregate: Type.Optional(Type.Array(Type.String())),
|
|
218
|
+
/** Grouping: ordering column for first/last aggregates. */
|
|
219
|
+
orderBy: Type.Optional(Type.String()),
|
|
220
|
+
/** Prepare: `OLD=NEW` column renames. */
|
|
221
|
+
rename: Type.Optional(Type.Array(Type.String())),
|
|
222
|
+
/** Prepare: `COL=VALUE` fills for empty cells. */
|
|
223
|
+
fillEmpty: Type.Optional(Type.Array(Type.String())),
|
|
224
|
+
/** Prepare: one `NAME=GREL_EXPRESSION` computed column. */
|
|
225
|
+
formula: Type.Optional(Type.String()),
|
|
226
|
+
/** Prepare: keep only rows where this GREL expression is true. */
|
|
227
|
+
filter: Type.Optional(Type.String()),
|
|
228
|
+
/** Prepare: columns to remove. */
|
|
229
|
+
dropColumns: Type.Optional(Type.Array(Type.String())),
|
|
230
|
+
/** Prepare: the only columns to keep. */
|
|
231
|
+
keepColumns: Type.Optional(Type.Array(Type.String())),
|
|
214
232
|
});
|
|
215
233
|
export const RecipeCreateResultSchema = Type.Object({
|
|
216
234
|
recipeName: Type.String(),
|
|
217
235
|
type: Type.String(),
|
|
218
236
|
createdDatasets: Type.Array(Type.String()),
|
|
219
237
|
joinConfigured: Type.Boolean(),
|
|
238
|
+
/** Grouping keys/aggregates or prepare steps from flags were written to the payload. */
|
|
239
|
+
payloadConfigured: Type.Optional(Type.Boolean()),
|
|
220
240
|
outputProvisioningFallbackUsed: Type.Boolean(),
|
|
221
241
|
outputFolder: Type.Optional(Type.String()),
|
|
222
242
|
temporaryOutputDataset: Type.Optional(Type.String()),
|
|
223
243
|
temporaryOutputDatasetDeleted: Type.Optional(Type.Boolean()),
|
|
224
|
-
|
|
244
|
+
/** Output datasets whose schema DSS computed and the client applied after creation. */
|
|
245
|
+
outputSchemaUpdated: Type.Optional(Type.Array(Type.String())),
|
|
246
|
+
/** Why DSS could not compute the output schema; the recipe itself was created. */
|
|
247
|
+
outputSchemaUpdateError: Type.Optional(Type.String()),
|
|
248
|
+
});
|
|
249
|
+
/** One output from GET /recipes/{name}/schema-update (dataikuapi compute_schema_updates). */
|
|
250
|
+
export const RecipeSchemaUpdateComputableSchema = Type.Object({
|
|
251
|
+
type: Type.String(),
|
|
252
|
+
id: Type.Optional(Type.String()),
|
|
253
|
+
datasetName: Type.Optional(Type.String()),
|
|
254
|
+
previousSchemaWasEmpty: Type.Optional(Type.Boolean()),
|
|
255
|
+
incompatibilities: Type.Optional(Type.Array(Type.String())),
|
|
256
|
+
newSchema: Type.Optional(Type.Object({
|
|
257
|
+
columns: Type.Array(Type.Record(Type.String(), Type.Unknown())),
|
|
258
|
+
}, { additionalProperties: true, })),
|
|
259
|
+
}, { additionalProperties: true, });
|
|
260
|
+
export const RecipeSchemaUpdatesSchema = Type.Object({
|
|
261
|
+
totalIncompatibilities: Type.Optional(Type.Number()),
|
|
262
|
+
computables: Type.Array(RecipeSchemaUpdateComputableSchema),
|
|
263
|
+
recipeChanges: Type.Optional(Type.Array(Type.Unknown())),
|
|
264
|
+
}, { additionalProperties: true, });
|
|
265
|
+
export const RecipeUpdateSchemaResultSchema = Type.Object({
|
|
266
|
+
recipeName: Type.String(),
|
|
267
|
+
/** Output ids whose schema was replaced by the DSS-computed schema. */
|
|
268
|
+
updated: Type.Array(Type.String()),
|
|
269
|
+
/** Output ids whose schema differs from the computed one but was left as is (`derivedOutputs`, `dryRun`). */
|
|
270
|
+
pending: Type.Array(Type.String()),
|
|
271
|
+
/** Output ids whose current schema already matches. */
|
|
272
|
+
unchanged: Type.Array(Type.String()),
|
|
273
|
+
totalIncompatibilities: Type.Number(),
|
|
274
|
+
computables: Type.Array(RecipeSchemaUpdateComputableSchema),
|
|
275
|
+
/** `datasetsNeedingAction` entries from the DSS apply responses. */
|
|
276
|
+
datasetsNeedingAction: Type.Optional(Type.Array(Type.Unknown())),
|
|
225
277
|
});
|
|
226
278
|
// ---------------------------------------------------------------------------
|
|
227
279
|
// Jobs
|
package/dist/src/cli/coerce.d.ts
CHANGED
|
@@ -20,6 +20,7 @@ export declare function expectJsonObject(value: unknown, source: string): Record
|
|
|
20
20
|
export declare function parseJsonObject(text: string, source: string): Record<string, unknown>;
|
|
21
21
|
export declare function json(v: string | boolean | undefined, source?: string): Record<string, unknown> | undefined;
|
|
22
22
|
export declare function readStdinText(): string;
|
|
23
|
+
export declare function seenJsonInputCredentials(): readonly string[];
|
|
23
24
|
export declare function jsonInput(flags: Record<string, string | boolean>): Record<string, unknown> | undefined;
|
|
24
25
|
export declare function unknownJsonInput(flags: Record<string, string | boolean>): unknown;
|
|
25
26
|
export declare function schemaColumnsInput(flags: Record<string, string | boolean>, usage: string): Array<{
|
package/dist/src/cli/coerce.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
|
+
import { credentialValues, } from "../utils/secret-sanitize.js";
|
|
2
3
|
import { UsageError, } from "./usage.js";
|
|
3
4
|
function safeValuePreview(value) {
|
|
4
5
|
return value.length <= 32 ? value : `${value.slice(0, 32)}…`;
|
|
@@ -173,31 +174,34 @@ function readInputText(path, flag) {
|
|
|
173
174
|
throw new UsageError(`Could not read ${flag} file: ${path}`, "validation_failed", "Verify that the path names an existing, readable file.", { flag, path, ...(cause ? { cause, } : {}), });
|
|
174
175
|
}
|
|
175
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* Credential values seen in this process's JSON inputs. DSS error text can
|
|
179
|
+
* echo the request payload, so error reports redact these.
|
|
180
|
+
*/
|
|
181
|
+
const jsonInputCredentials = [];
|
|
182
|
+
export function seenJsonInputCredentials() {
|
|
183
|
+
return jsonInputCredentials;
|
|
184
|
+
}
|
|
185
|
+
function parseJsonSource(text, source, kind) {
|
|
186
|
+
const value = kind === "object" ? parseJsonObject(text, source) : parseJsonValue(text, source);
|
|
187
|
+
jsonInputCredentials.push(...credentialValues(value));
|
|
188
|
+
return value;
|
|
189
|
+
}
|
|
176
190
|
function resolveJsonSourceText(flags, kind) {
|
|
177
191
|
const source = selectSingleSource(flags, JSON_SOURCE_FLAGS);
|
|
178
192
|
if (source === undefined)
|
|
179
193
|
return undefined;
|
|
180
|
-
if (source === "stdin")
|
|
181
|
-
|
|
182
|
-
return kind === "object"
|
|
183
|
-
? parseJsonObject(jsonText, "stdin")
|
|
184
|
-
: parseJsonValue(jsonText, "stdin");
|
|
185
|
-
}
|
|
194
|
+
if (source === "stdin")
|
|
195
|
+
return parseJsonSource(readStdinText(), "stdin", kind);
|
|
186
196
|
if (source === "data-file") {
|
|
187
197
|
const fileValue = flags["data-file"];
|
|
188
198
|
if (typeof fileValue === "string") {
|
|
189
|
-
|
|
190
|
-
return kind === "object"
|
|
191
|
-
? parseJsonObject(jsonText, fileValue)
|
|
192
|
-
: parseJsonValue(jsonText, fileValue);
|
|
199
|
+
return parseJsonSource(readInputText(fileValue, "--data-file"), fileValue, kind);
|
|
193
200
|
}
|
|
194
201
|
}
|
|
195
202
|
const dataValue = flags["data"];
|
|
196
|
-
if (typeof dataValue === "string")
|
|
197
|
-
return
|
|
198
|
-
? parseJsonObject(dataValue, "--data")
|
|
199
|
-
: parseJsonValue(dataValue, "--data");
|
|
200
|
-
}
|
|
203
|
+
if (typeof dataValue === "string")
|
|
204
|
+
return parseJsonSource(dataValue, "--data", kind);
|
|
201
205
|
throw new UsageError("JSON input source is present but has no value.", "invalid_flag_value");
|
|
202
206
|
}
|
|
203
207
|
export function jsonInput(flags) {
|