dataiku-sdk 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -1
- package/bin/dss.js +20 -1
- package/dist/build-metadata.json +1 -0
- package/dist/src/cli/commands/data-quality.js +42 -0
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +187 -6
- package/dist/src/cli/commands/job.js +15 -2
- package/dist/src/cli/commands/project.js +116 -12
- package/dist/src/cli/contract.d.ts +1 -0
- package/dist/src/cli/contract.js +182 -13
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +44 -13
- package/dist/src/cli/flags.js +2 -0
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/output.d.ts +3 -1
- package/dist/src/cli/output.js +11 -6
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/version.d.ts +46 -4
- package/dist/src/cli/version.js +175 -9
- package/dist/src/cli.js +51 -13
- package/dist/src/client.d.ts +28 -1
- package/dist/src/client.js +34 -7
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.js +1 -0
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +199 -28
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +127 -16
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +624 -0
- package/package.json +6 -5
- package/plugin.json +1 -1
package/README.md
CHANGED
|
@@ -20,6 +20,7 @@ npm install --global dataiku-sdk
|
|
|
20
20
|
|
|
21
21
|
Bun is the primary development runtime and package manager. Examples below assume an installed `dss` binary. From a checkout, use `bun --no-env-file src/cli.ts ...` or the cross-runtime `bun --no-env-file ./bin/dss.js ...` launcher. Node users can invoke the same launcher as `node ./bin/dss.js ...`; if `dist/` is absent, it delegates to Bun. From another working directory, pass the checkout's absolute `bin/dss.js` path to Bun or Node.
|
|
22
22
|
`--no-env-file` disables Bun's automatic preloading only; the CLI still applies its documented `.env` handling unless `DATAIKU_DISABLE_ENV=1` is set.
|
|
23
|
+
`dss version` reports whether the running code came from `source` or `dist`, the Node/Bun runtime, the packaged build revision when available, and `staleBuild` when that revision differs from the checkout. Release builds generate `dist/build-metadata.json`; source snapshots without Git metadata still build but report no build revision.
|
|
23
24
|
|
|
24
25
|
## CLI contract
|
|
25
26
|
|
|
@@ -119,7 +120,14 @@ The command saves credentials and returns `{ "saved": true, "path": "..." }`.
|
|
|
119
120
|
dss version
|
|
120
121
|
dss doctor --fast
|
|
121
122
|
dss project list
|
|
123
|
+
dss project inspect-archive ./project.zip
|
|
124
|
+
dss project import ./project.zip --target-project-key IMPORTED_PROJECT --record-cleanup cleanup.jsonl
|
|
122
125
|
dss dataset list --project-key MYPROJ
|
|
126
|
+
dss dataset files uploaded_input --project-key MYPROJ
|
|
127
|
+
dss dataset upload-file uploaded_input ./new.csv --file-name new.csv --project-key MYPROJ
|
|
128
|
+
dss dataset assert-count jmp_doe_ready --expected 12 --project-key MYPROJ
|
|
129
|
+
dss dataset assert-schema jmp_doe_ready --data-file expected-schema.json --project-key MYPROJ
|
|
130
|
+
dss data-quality assert-results jmp_doe_ready --project-key MYPROJ
|
|
123
131
|
dss recipe get-payload compute_orders --project-key MYPROJ
|
|
124
132
|
dss recipe get-payload compute_orders --output code.py --project-key MYPROJ
|
|
125
133
|
dss install-skill --dry-run
|
|
@@ -136,6 +144,28 @@ dss app permissions-restore --project-key MYAPP_INSTANCE --file permissions.json
|
|
|
136
144
|
dss cleanup --file cleanup.jsonl
|
|
137
145
|
dss cleanup --file cleanup.jsonl --apply
|
|
138
146
|
```
|
|
147
|
+
`project import` performs both public API stages: archive upload, then import processing. It
|
|
148
|
+
first validates ZIP integrity, safe unique member paths, the export manifest, and provable Flow
|
|
149
|
+
references locally. Verified success reports DSS's actual project key, explicit remapping, and a
|
|
150
|
+
`creationTag` incarnation hash; `--record-cleanup` records an incarnation-bound guarded delete.
|
|
151
|
+
It exits nonzero when DSS returns `success:false` even if both HTTP requests succeeded. A timeout,
|
|
152
|
+
5xx, malformed response, missing used key, or failed post-import identity read is
|
|
153
|
+
`ambiguous_outcome` and retains the temporary import ID for inspection before retry.
|
|
154
|
+
|
|
155
|
+
`dataset upload-file` only adds a filename that is not already present and verifies its byte
|
|
156
|
+
length afterward. DSS 14.7 exposes no public UploadedFiles delete or replacement endpoint;
|
|
157
|
+
the command refuses an existing filename before POST instead of triggering DSS's opaque 500.
|
|
158
|
+
For replacement, import a successor project archive containing the desired upload.
|
|
159
|
+
|
|
160
|
+
`dataset preview` returns `truncated` and `limit`; it probes one row past the requested cap instead
|
|
161
|
+
of presenting a capped sample as an exact count. `dataset assert-count`, `dataset assert-schema`,
|
|
162
|
+
and `data-quality assert-results` return `satisfied` and exit `4`/`assertion_failed` on a failed
|
|
163
|
+
postcondition, so they can be used as `batch` steps. Multi-job `job monitor` and `job watch`
|
|
164
|
+
aggregate child failures and also exit `4` when any watched job fails.
|
|
165
|
+
|
|
166
|
+
`doctor` reads the documented `DSS-Version` and `DSS-API-Version` response headers plus the HTTP
|
|
167
|
+
`Date` header from the public project-list call. Feature support remains behavioral: the CLI does
|
|
168
|
+
not call an undocumented feature endpoint or infer licensed capabilities from a version string.
|
|
139
169
|
|
|
140
170
|
### Application release safety
|
|
141
171
|
|
|
@@ -214,7 +244,9 @@ conditional DELETE. Cleanup rechecks type and `creationTag` immediately before d
|
|
|
214
244
|
detected key reuse, but cannot eliminate replacement in the remaining check-to-DELETE gap.
|
|
215
245
|
`cleanup --apply` validates the full ledger before issuing any request and rejects legacy entries
|
|
216
246
|
without server identity, mixed-server ledgers, entries for another DSS URL, and app cleanup entries
|
|
217
|
-
without a valid incarnation binding.
|
|
247
|
+
without a valid incarnation binding. Project-import cleanup uses the same rule: the ledger binds
|
|
248
|
+
the actual imported key and verified `creationTag` hash, and `project delete
|
|
249
|
+
--expect-project-incarnation` refuses a reused or unverifiable key before DELETE.
|
|
218
250
|
|
|
219
251
|
Saving `version`/`versionNotes` in the public app manifest is a metadata write, not a publish
|
|
220
252
|
transaction. If the manifest PUT or its verification read has an ambiguous transport/server
|
package/bin/dss.js
CHANGED
|
@@ -56,9 +56,28 @@ if (hasFlag(["--insecure", "--skip-tls-verify",],)) {
|
|
|
56
56
|
) {
|
|
57
57
|
env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
|
58
58
|
}
|
|
59
|
-
|
|
60
59
|
const runningOnBun = Boolean(process.versions.bun,);
|
|
61
60
|
const usesSourceCli = cliPath === sourceCliPath;
|
|
61
|
+
|
|
62
|
+
// Published dist carries dist/build-metadata.json with the revision it was
|
|
63
|
+
// built from. Forward it to the CLI so provenance can report dist source and
|
|
64
|
+
// its build revision; only a full lowercase hexadecimal revision is accepted,
|
|
65
|
+
// so an inherited or corrupt variable never stands in for packaged metadata.
|
|
66
|
+
const distMetadataPath = path.resolve(here, "../dist/build-metadata.json",);
|
|
67
|
+
if (!usesSourceCli) {
|
|
68
|
+
env.DSS_LOAD_SOURCE = "dist";
|
|
69
|
+
delete env.DSS_BUILD_REVISION;
|
|
70
|
+
try {
|
|
71
|
+
const metadata = JSON.parse(fs.readFileSync(distMetadataPath, "utf-8",),);
|
|
72
|
+
const revision = metadata.buildRevision;
|
|
73
|
+
if (typeof revision === "string" && /^[0-9a-f]{40}$/.test(revision,)) {
|
|
74
|
+
env.DSS_BUILD_REVISION = revision;
|
|
75
|
+
}
|
|
76
|
+
} catch {
|
|
77
|
+
// Dist without build metadata: provenance still reports dist source.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
62
81
|
const runtimeBin = runningOnBun ? process.execPath : usesSourceCli ? "bun" : process.execPath;
|
|
63
82
|
const runtimeArgs = runningOnBun || usesSourceCli ? ["--no-env-file",] : ["--use-system-ca",];
|
|
64
83
|
const result = spawnSync(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"format":1,"buildRevision":"70487e2275a70acce6862f126a89b1bc852ebb11"}
|
|
@@ -2,6 +2,18 @@ import { deepMerge, } from "../../utils/deep-merge.js";
|
|
|
2
2
|
import { num, parseBooleanOption, requiredJsonInput, } from "../coerce.js";
|
|
3
3
|
import { encodedProjectEndpoint, readIfExists, skipResult, } from "../output.js";
|
|
4
4
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
5
|
+
/**
|
|
6
|
+
* A data quality result counts as passing only when it actually carries a
|
|
7
|
+
* verdict and every verdict it carries reports OK. A result with no outcome
|
|
8
|
+
* and no status has proven nothing, and a result whose fields disagree
|
|
9
|
+
* (outcome ERROR alongside status OK) is a failure — never a pass.
|
|
10
|
+
*/
|
|
11
|
+
function resultReportsOk(result) {
|
|
12
|
+
const verdicts = [result.outcome, result.status,]
|
|
13
|
+
.filter((value) => typeof value === "string" && value.trim().length > 0)
|
|
14
|
+
.map((value) => value.trim().toUpperCase());
|
|
15
|
+
return verdicts.length > 0 && verdicts.every((value) => value === "OK");
|
|
16
|
+
}
|
|
5
17
|
export const dataQualityCommands = {
|
|
6
18
|
rules: {
|
|
7
19
|
handler: (c, a, f) => {
|
|
@@ -160,6 +172,36 @@ export const dataQualityCommands = {
|
|
|
160
172
|
description: "Get latest data quality rule results for a dataset.",
|
|
161
173
|
examples: ["dss data-quality last-results orders",],
|
|
162
174
|
},
|
|
175
|
+
"assert-results": {
|
|
176
|
+
handler: async (c, a, f) => {
|
|
177
|
+
requireArgs(a, 1, "dss data-quality assert-results <dataset>");
|
|
178
|
+
const ruleId = f["rule-id"];
|
|
179
|
+
const pk = f["project-key"];
|
|
180
|
+
const results = await c.dataQuality.lastResults(a[0], { ruleId, projectKey: pk, });
|
|
181
|
+
const selected = results.filter((result) => ruleId === undefined || result.id === ruleId || result.ruleId === ruleId);
|
|
182
|
+
const failed = selected
|
|
183
|
+
.filter((result) => !resultReportsOk(result))
|
|
184
|
+
.map((result) => ({
|
|
185
|
+
ruleId: result.id ?? result.ruleId ?? result.name ?? null,
|
|
186
|
+
outcome: result.outcome ?? null,
|
|
187
|
+
status: result.status ?? null,
|
|
188
|
+
}));
|
|
189
|
+
return {
|
|
190
|
+
satisfied: selected.length > 0 && failed.length === 0,
|
|
191
|
+
dataset: a[0],
|
|
192
|
+
...(ruleId !== undefined ? { ruleId, } : {}),
|
|
193
|
+
checked: selected.length,
|
|
194
|
+
failed,
|
|
195
|
+
...(selected.length === 0 ? { reason: "no_results", } : {}),
|
|
196
|
+
};
|
|
197
|
+
},
|
|
198
|
+
usage: "dss data-quality assert-results <dataset> [--rule-id ID] [--project-key KEY]",
|
|
199
|
+
description: "Assert the latest data quality results: at least one selected rule must exist and every selected result must report OK. Returns { satisfied, checked, failed } with failed rule ids/outcomes only; a failed assertion exits 4 with assertion_failed.",
|
|
200
|
+
examples: [
|
|
201
|
+
"dss data-quality assert-results orders",
|
|
202
|
+
"dss data-quality assert-results orders --rule-id RULE_ID",
|
|
203
|
+
],
|
|
204
|
+
},
|
|
163
205
|
history: {
|
|
164
206
|
handler: (c, a, f) => {
|
|
165
207
|
requireArgs(a, 1, "dss data-quality history <dataset>");
|
|
@@ -1,2 +1,25 @@
|
|
|
1
1
|
import type { CommandMeta } from "../types.js";
|
|
2
|
+
export interface DatasetSchemaDifference {
|
|
3
|
+
/** Path from the schema root in dot/bracket notation, e.g. `columns[2].type`. */
|
|
4
|
+
path: string;
|
|
5
|
+
kind: "added" | "removed" | "changed";
|
|
6
|
+
/** SHA-256 of the deterministic JSON rendering of the expected subtree. */
|
|
7
|
+
expectedHash?: string;
|
|
8
|
+
/** SHA-256 of the deterministic JSON rendering of the actual subtree. */
|
|
9
|
+
actualHash?: string;
|
|
10
|
+
}
|
|
11
|
+
export interface DatasetSchemaComparison {
|
|
12
|
+
equal: boolean;
|
|
13
|
+
expectedHash: string;
|
|
14
|
+
actualHash: string;
|
|
15
|
+
differences: DatasetSchemaDifference[];
|
|
16
|
+
totalDifferences: number;
|
|
17
|
+
differencesTruncated: boolean;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Compare two schema values deterministically — object key order never matters
|
|
21
|
+
* — and report each structural difference as a concise per-path entry carrying
|
|
22
|
+
* subtree hashes instead of subtree values.
|
|
23
|
+
*/
|
|
24
|
+
export declare function compareDatasetSchemas(expected: unknown, actual: unknown): DatasetSchemaComparison;
|
|
2
25
|
export declare const datasetCommands: Record<string, CommandMeta>;
|
|
@@ -1,10 +1,102 @@
|
|
|
1
1
|
import { buildDatasetCloneSettings, } from "../../resources/datasets.js";
|
|
2
2
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
3
|
-
import {
|
|
3
|
+
import { compareStrings, stableHash, } from "../../utils/stable-hash.js";
|
|
4
|
+
import { jsonInput, num, schemaColumnsInput, unknownJsonInput, } from "../coerce.js";
|
|
4
5
|
import { datasetSourceSummary, } from "../helpers/dataset.js";
|
|
5
6
|
import { moveCreatedItemsToZone, resolveFlowZoneIdFromFlags, } from "../helpers/flow-zone.js";
|
|
6
7
|
import { enqueueCliWarning, readIfExists, skipResult, } from "../output.js";
|
|
7
8
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
9
|
+
const MAX_SCHEMA_DIFFERENCES = 50;
|
|
10
|
+
function isPlainObject(value) {
|
|
11
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
12
|
+
}
|
|
13
|
+
function appendSchemaPath(parent, key) {
|
|
14
|
+
return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)
|
|
15
|
+
? (parent ? `${parent}.${key}` : key)
|
|
16
|
+
: `${parent}[${JSON.stringify(key)}]`;
|
|
17
|
+
}
|
|
18
|
+
function collectSchemaDifferences(expected, actual, path, acc) {
|
|
19
|
+
// Hash equality is the deterministic equivalence test: structurally equal
|
|
20
|
+
// subtrees (any key order) match exactly and are never descended into.
|
|
21
|
+
if (stableHash(expected) === stableHash(actual))
|
|
22
|
+
return;
|
|
23
|
+
if (isPlainObject(expected) && isPlainObject(actual)) {
|
|
24
|
+
const keys = [...new Set([...Object.keys(expected), ...Object.keys(actual),]),].sort(compareStrings);
|
|
25
|
+
for (const key of keys) {
|
|
26
|
+
const nextPath = appendSchemaPath(path, key);
|
|
27
|
+
const hasExpected = key in expected;
|
|
28
|
+
const hasActual = key in actual;
|
|
29
|
+
if (hasExpected && hasActual) {
|
|
30
|
+
collectSchemaDifferences(expected[key], actual[key], nextPath, acc);
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
acc.total += 1;
|
|
34
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
35
|
+
acc.items.push({
|
|
36
|
+
path: nextPath,
|
|
37
|
+
kind: hasActual ? "added" : "removed",
|
|
38
|
+
...(hasActual
|
|
39
|
+
? { actualHash: stableHash(actual[key]), }
|
|
40
|
+
: { expectedHash: stableHash(expected[key]), }),
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (Array.isArray(expected) && Array.isArray(actual)) {
|
|
47
|
+
const shared = Math.min(expected.length, actual.length);
|
|
48
|
+
for (let i = 0; i < shared; i++) {
|
|
49
|
+
collectSchemaDifferences(expected[i], actual[i], `${path}[${i}]`, acc);
|
|
50
|
+
}
|
|
51
|
+
for (let i = shared; i < expected.length; i++) {
|
|
52
|
+
acc.total += 1;
|
|
53
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
54
|
+
acc.items.push({
|
|
55
|
+
path: `${path}[${i}]`,
|
|
56
|
+
kind: "removed",
|
|
57
|
+
expectedHash: stableHash(expected[i]),
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
for (let i = shared; i < actual.length; i++) {
|
|
62
|
+
acc.total += 1;
|
|
63
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
64
|
+
acc.items.push({
|
|
65
|
+
path: `${path}[${i}]`,
|
|
66
|
+
kind: "added",
|
|
67
|
+
actualHash: stableHash(actual[i]),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
acc.total += 1;
|
|
74
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
75
|
+
acc.items.push({
|
|
76
|
+
path: path || "$",
|
|
77
|
+
kind: "changed",
|
|
78
|
+
expectedHash: stableHash(expected),
|
|
79
|
+
actualHash: stableHash(actual),
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Compare two schema values deterministically — object key order never matters
|
|
85
|
+
* — and report each structural difference as a concise per-path entry carrying
|
|
86
|
+
* subtree hashes instead of subtree values.
|
|
87
|
+
*/
|
|
88
|
+
export function compareDatasetSchemas(expected, actual) {
|
|
89
|
+
const acc = { items: [], total: 0, };
|
|
90
|
+
collectSchemaDifferences(expected, actual, "", acc);
|
|
91
|
+
return {
|
|
92
|
+
equal: acc.items.length === 0,
|
|
93
|
+
expectedHash: stableHash(expected),
|
|
94
|
+
actualHash: stableHash(actual),
|
|
95
|
+
differences: acc.items,
|
|
96
|
+
totalDifferences: acc.total,
|
|
97
|
+
differencesTruncated: acc.items.length < acc.total,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
8
100
|
export const datasetCommands = {
|
|
9
101
|
list: {
|
|
10
102
|
handler: (c, _a, f) => c.datasets.list(f["project-key"]),
|
|
@@ -69,6 +161,34 @@ export const datasetCommands = {
|
|
|
69
161
|
description: "Show backing connection, catalog/schema/table, path, and format for a dataset.",
|
|
70
162
|
examples: ["dss dataset source orders",],
|
|
71
163
|
},
|
|
164
|
+
files: {
|
|
165
|
+
handler: (c, a, f) => {
|
|
166
|
+
requireArgs(a, 1, "dss dataset files <name> [--project-key KEY]");
|
|
167
|
+
return c.datasets.listUploadedFiles(a[0], f["project-key"]);
|
|
168
|
+
},
|
|
169
|
+
usage: "dss dataset files <name> [--project-key KEY]",
|
|
170
|
+
description: "List files stored by an UploadedFiles dataset.",
|
|
171
|
+
examples: ["dss dataset files uploaded_input",],
|
|
172
|
+
},
|
|
173
|
+
"upload-file": {
|
|
174
|
+
handler: (c, a, f) => {
|
|
175
|
+
const usage = "dss dataset upload-file <name> <localPath> --file-name NAME [--project-key KEY]";
|
|
176
|
+
requireArgs(a, 2, usage);
|
|
177
|
+
const fileName = f["file-name"];
|
|
178
|
+
if (!fileName || fileName.trim() === "") {
|
|
179
|
+
throw new UsageError(`--file-name is required. Usage: ${usage}`);
|
|
180
|
+
}
|
|
181
|
+
return c.datasets.uploadDatasetFile(a[0], a[1], {
|
|
182
|
+
fileName,
|
|
183
|
+
projectKey: f["project-key"],
|
|
184
|
+
});
|
|
185
|
+
},
|
|
186
|
+
usage: "dss dataset upload-file <name> <localPath> --file-name NAME [--project-key KEY]",
|
|
187
|
+
description: "Add one new file to an UploadedFiles dataset and verify the stored byte length.",
|
|
188
|
+
examples: [
|
|
189
|
+
"dss dataset upload-file uploaded_input ./input.csv --file-name input.csv",
|
|
190
|
+
],
|
|
191
|
+
},
|
|
72
192
|
"refresh-schema": {
|
|
73
193
|
handler: async (c, a, f) => {
|
|
74
194
|
const usage = "dss dataset refresh-schema <name> [--data JSON | --data-file PATH | --stdin] [--dry-run] [--project-key KEY]";
|
|
@@ -105,18 +225,79 @@ export const datasetCommands = {
|
|
|
105
225
|
examples: ["dss dataset validate-build orders",],
|
|
106
226
|
},
|
|
107
227
|
preview: {
|
|
108
|
-
handler: (c, a, f) => {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
228
|
+
handler: async (c, a, f) => {
|
|
229
|
+
const previewUsage = "dss dataset preview <name> [--max-rows N] [--rows N] [--project-key KEY] [--timeout MS]";
|
|
230
|
+
requireArgs(a, 1, previewUsage);
|
|
231
|
+
const maxRows = num(f["max-rows"], "--max-rows");
|
|
232
|
+
if (maxRows !== undefined && (!Number.isSafeInteger(maxRows) || maxRows <= 0 || maxRows > 500)) {
|
|
233
|
+
throw new UsageError(`--max-rows must be a positive integer no greater than 500 (got "${String(f["max-rows"])}"). `
|
|
234
|
+
+ `Usage: ${previewUsage}`, "validation_failed");
|
|
235
|
+
}
|
|
236
|
+
const result = await c.datasets.preview(a[0], {
|
|
237
|
+
maxRows,
|
|
112
238
|
projectKey: f["project-key"],
|
|
113
239
|
timeoutMs: num(f["timeout"], "--timeout"),
|
|
114
240
|
});
|
|
241
|
+
if (result.truncated) {
|
|
242
|
+
enqueueCliWarning({
|
|
243
|
+
code: "dataset_preview_truncated",
|
|
244
|
+
message: `Preview of '${a[0]}' stopped at the ${result.limit}-row cap; the dataset has more rows. ${result.limit >= 500
|
|
245
|
+
? "The preview cap of 500 rows is the public maximum; use `dss dataset download` for more rows."
|
|
246
|
+
: "Re-run with --max-rows N (up to 500) for more rows."}`,
|
|
247
|
+
dataset: a[0],
|
|
248
|
+
rows: result.rowCount,
|
|
249
|
+
limit: result.limit,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
return result;
|
|
115
253
|
},
|
|
116
254
|
usage: "dss dataset preview <name> [--max-rows N] [--rows N] [--project-key KEY] [--timeout MS]",
|
|
117
|
-
description: "Preview dataset rows (--rows is an alias for --max-rows).",
|
|
255
|
+
description: "Preview dataset rows (--rows is an alias for --max-rows). Returns { columns, rows, rowCount, truncated, limit }; when the dataset has more rows than the cap, truncated is true and a dataset_preview_truncated warning is written to stderr.",
|
|
118
256
|
examples: ["dss dataset preview orders", "dss dataset preview orders --rows 5",],
|
|
119
257
|
},
|
|
258
|
+
"assert-count": {
|
|
259
|
+
handler: async (c, a, f) => {
|
|
260
|
+
const assertUsage = "dss dataset assert-count <dataset> --expected N [--project-key KEY]";
|
|
261
|
+
requireArgs(a, 1, assertUsage);
|
|
262
|
+
const expected = num(f["expected"], "--expected");
|
|
263
|
+
if (expected === undefined || !Number.isSafeInteger(expected) || expected < 0) {
|
|
264
|
+
throw new UsageError(`--expected must be a non-negative integer (got "${String(f["expected"])}"). `
|
|
265
|
+
+ `Usage: ${assertUsage}`, "validation_failed");
|
|
266
|
+
}
|
|
267
|
+
const result = await c.datasets.assertRowCount(a[0], expected, f["project-key"]);
|
|
268
|
+
return { ...result, dataset: a[0], };
|
|
269
|
+
},
|
|
270
|
+
usage: "dss dataset assert-count <dataset> --expected N [--project-key KEY]",
|
|
271
|
+
description: "Stream the dataset and assert it holds exactly --expected rows, probing at most N+1 rows. Returns { satisfied, expected, count, exact }; a mismatch exits 4 with assertion_failed.",
|
|
272
|
+
examples: ["dss dataset assert-count orders --expected 1000",],
|
|
273
|
+
},
|
|
274
|
+
"assert-schema": {
|
|
275
|
+
handler: async (c, a, f) => {
|
|
276
|
+
const assertUsage = "dss dataset assert-schema <dataset> (--data JSON | --data-file PATH | --stdin) [--project-key KEY]";
|
|
277
|
+
requireArgs(a, 1, assertUsage);
|
|
278
|
+
const expected = unknownJsonInput(f);
|
|
279
|
+
if (expected === undefined) {
|
|
280
|
+
throw new UsageError(`--data, --data-file, or --stdin is required. Usage: ${assertUsage}`);
|
|
281
|
+
}
|
|
282
|
+
const actual = await c.datasets.getSchemaObject(a[0], f["project-key"]);
|
|
283
|
+
const comparison = compareDatasetSchemas(expected, actual);
|
|
284
|
+
return {
|
|
285
|
+
satisfied: comparison.equal,
|
|
286
|
+
dataset: a[0],
|
|
287
|
+
expectedHash: comparison.expectedHash,
|
|
288
|
+
actualHash: comparison.actualHash,
|
|
289
|
+
totalDifferences: comparison.totalDifferences,
|
|
290
|
+
...(comparison.differencesTruncated ? { differencesTruncated: true, } : {}),
|
|
291
|
+
differences: comparison.differences,
|
|
292
|
+
};
|
|
293
|
+
},
|
|
294
|
+
usage: "dss dataset assert-schema <dataset> (--data JSON | --data-file PATH | --stdin) [--project-key KEY]",
|
|
295
|
+
description: "Assert the dataset's full schema equals the expected schema object; reports concise per-path differences with sha256 subtree hashes. A mismatch exits 4 with assertion_failed.",
|
|
296
|
+
examples: [
|
|
297
|
+
`dss dataset assert-schema orders --data-file expected-schema.json`,
|
|
298
|
+
`dss dataset assert-schema orders --data '{"columns":[]}'`,
|
|
299
|
+
],
|
|
300
|
+
},
|
|
120
301
|
metadata: {
|
|
121
302
|
handler: (c, a, f) => {
|
|
122
303
|
requireArgs(a, 1, "dss dataset metadata <name>");
|
|
@@ -33,6 +33,19 @@ function firstNumberPath(value, paths) {
|
|
|
33
33
|
}
|
|
34
34
|
return undefined;
|
|
35
35
|
}
|
|
36
|
+
function aggregateJobWaitResults(jobs, until) {
|
|
37
|
+
if (jobs.length === 1)
|
|
38
|
+
return jobs[0];
|
|
39
|
+
const failed = jobs.find((job) => !job.success);
|
|
40
|
+
return {
|
|
41
|
+
elapsedMs: jobs.reduce((maximum, job) => Math.max(maximum, job.elapsedMs), 0),
|
|
42
|
+
jobs,
|
|
43
|
+
pollCount: jobs.reduce((total, job) => total + job.pollCount, 0),
|
|
44
|
+
state: failed?.state ?? "DONE",
|
|
45
|
+
success: failed === undefined,
|
|
46
|
+
until,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
36
49
|
function jobSummaryId(job, fallback) {
|
|
37
50
|
return stringPath(job, ["baseStatus", "def", "id",])
|
|
38
51
|
?? stringPath(job, ["def", "id",])
|
|
@@ -357,7 +370,7 @@ export const jobCommands = {
|
|
|
357
370
|
projectKey: f["project-key"],
|
|
358
371
|
};
|
|
359
372
|
const jobs = await Promise.all(a.map((jobId) => c.jobs.wait(jobId, options)));
|
|
360
|
-
return
|
|
373
|
+
return aggregateJobWaitResults(jobs, String(f["until"] ?? "all-done"));
|
|
361
374
|
},
|
|
362
375
|
usage: "dss job monitor <id...> [--summary] [--include-logs] [--log-filter stdout|stderr|user|errors] [--max-log-lines N] [--timeout MS] [--poll-interval MS] [--until all-done] [--project-key KEY]",
|
|
363
376
|
description: "Monitor one or more existing jobs and summarize progress counters from logs.",
|
|
@@ -376,7 +389,7 @@ export const jobCommands = {
|
|
|
376
389
|
projectKey: f["project-key"],
|
|
377
390
|
};
|
|
378
391
|
const jobs = await Promise.all(a.map((jobId) => c.jobs.wait(jobId, options)));
|
|
379
|
-
return
|
|
392
|
+
return aggregateJobWaitResults(jobs, String(f["until"] ?? "all-done"));
|
|
380
393
|
},
|
|
381
394
|
usage: "dss job watch <id...> [--include-logs] [--log-filter stdout|stderr|user|errors] [--max-log-lines N] [--timeout MS] [--poll-interval MS] [--until all-done] [--project-key KEY]",
|
|
382
395
|
description: "Watch one or more existing jobs with progress extraction enabled.",
|
|
@@ -1,6 +1,16 @@
|
|
|
1
|
+
import { ClientValidationError, DataikuError, } from "../../errors.js";
|
|
1
2
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
2
|
-
import {
|
|
3
|
+
import { inspectProjectArchive, } from "../../utils/project-archive.js";
|
|
4
|
+
import { projectIncarnationHash, } from "../../utils/project-incarnation.js";
|
|
5
|
+
import { jsonInput, num, requiredJsonInput, requiredStringFlag, } from "../coerce.js";
|
|
6
|
+
import { CommandResultFailure, } from "../output.js";
|
|
3
7
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
8
|
+
const INSPECT_ARCHIVE_USAGE = "dss project inspect-archive <file>";
|
|
9
|
+
/** Local, DSS-free archive inspection shared by the local handler and the command handler. */
|
|
10
|
+
function inspectArchiveCommand(args) {
|
|
11
|
+
requireArgs(args, 1, INSPECT_ARCHIVE_USAGE);
|
|
12
|
+
return inspectProjectArchive(args[0]);
|
|
13
|
+
}
|
|
4
14
|
export const projectCommands = {
|
|
5
15
|
list: {
|
|
6
16
|
handler: (c) => c.projects.list(),
|
|
@@ -56,12 +66,75 @@ export const projectCommands = {
|
|
|
56
66
|
},
|
|
57
67
|
delete: {
|
|
58
68
|
handler: async (c, a, f) => {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
69
|
+
const usage = "dss project delete <projectKey> [--drop-data] [--if-exists] [--dry-run] [--expect-project-incarnation HASH]";
|
|
70
|
+
requireArgs(a, 1, usage);
|
|
71
|
+
const projectKey = a[0];
|
|
72
|
+
const expectedIncarnation = f["expect-project-incarnation"] === undefined
|
|
73
|
+
? undefined
|
|
74
|
+
: requiredStringFlag(f, "expect-project-incarnation", usage);
|
|
75
|
+
if (expectedIncarnation !== undefined
|
|
76
|
+
&& !/^[0-9a-f]{64}$/.test(expectedIncarnation)) {
|
|
77
|
+
throw new UsageError("--expect-project-incarnation must be a 64-character lowercase SHA-256 hash.", "validation_failed");
|
|
78
|
+
}
|
|
79
|
+
const guarded = f["if-exists"] === true
|
|
80
|
+
|| f["dry-run"] === true
|
|
81
|
+
|| expectedIncarnation !== undefined;
|
|
82
|
+
if (!guarded) {
|
|
83
|
+
// Purely user-requested delete without any guard flag: keep the
|
|
84
|
+
// historical direct DELETE, with no preflight probe.
|
|
85
|
+
await c.projects.deleteProject(projectKey, f["drop-data"] === true);
|
|
86
|
+
return { deleted: projectKey, };
|
|
87
|
+
}
|
|
88
|
+
// Guarded path: existence and identity must be verified before any
|
|
89
|
+
// DELETE. A missing creationTag or an incarnation mismatch refuses
|
|
90
|
+
// before the DELETE under possible project-key reuse.
|
|
91
|
+
let currentIncarnation;
|
|
92
|
+
try {
|
|
93
|
+
const details = await c.projects.get(projectKey);
|
|
94
|
+
if (expectedIncarnation !== undefined) {
|
|
95
|
+
currentIncarnation = projectIncarnationHash(projectKey, details);
|
|
96
|
+
if (currentIncarnation === undefined) {
|
|
97
|
+
throw new ClientValidationError(`DSS did not return creationTag identity for project ${projectKey}. Refusing to delete without project-incarnation verification.`, "validation_failed", "Project-key reuse cannot be excluded. Refusing the DELETE; re-read the project and retry with a hash captured from that incarnation.", { projectKey, });
|
|
98
|
+
}
|
|
99
|
+
if (currentIncarnation !== expectedIncarnation) {
|
|
100
|
+
throw new ClientValidationError(`Project ${projectKey} is not the same project incarnation recorded for this operation. Refusing to delete after project-key reuse.`, "validation_failed", "Re-read the current project and retry only with an artifact captured from that incarnation.", {
|
|
101
|
+
projectKey,
|
|
102
|
+
expectedProjectIncarnationHash: expectedIncarnation,
|
|
103
|
+
currentProjectIncarnationHash: currentIncarnation,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error instanceof DataikuError && error.status === 404) {
|
|
110
|
+
// Already absent: converge only when the caller asked for it.
|
|
111
|
+
if (f["if-exists"] !== true)
|
|
112
|
+
throw error;
|
|
113
|
+
return { deleted: false, alreadyAbsent: true, projectKey, };
|
|
114
|
+
}
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
if (f["dry-run"] === true) {
|
|
118
|
+
return {
|
|
119
|
+
deleted: false,
|
|
120
|
+
dryRun: true,
|
|
121
|
+
projectKey,
|
|
122
|
+
...(currentIncarnation !== undefined
|
|
123
|
+
? { projectIncarnationHash: currentIncarnation, }
|
|
124
|
+
: {}),
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
await c.projects.deleteProject(projectKey, f["drop-data"] === true);
|
|
128
|
+
return {
|
|
129
|
+
deleted: true,
|
|
130
|
+
projectKey,
|
|
131
|
+
...(currentIncarnation !== undefined
|
|
132
|
+
? { projectIncarnationHash: currentIncarnation, }
|
|
133
|
+
: {}),
|
|
134
|
+
};
|
|
62
135
|
},
|
|
63
|
-
usage: "dss project delete <projectKey> [--drop-data]",
|
|
64
|
-
description: "Delete a project (destructive). --drop-data also clears managed datasets.",
|
|
136
|
+
usage: "dss project delete <projectKey> [--drop-data] [--if-exists] [--dry-run] [--expect-project-incarnation HASH]",
|
|
137
|
+
description: "Delete a project (destructive). --drop-data also clears managed datasets; --if-exists treats an absent project as already deleted; --dry-run verifies without deleting; --expect-project-incarnation HASH refuses deletion unless the project still matches the recorded incarnation.",
|
|
65
138
|
examples: ["dss project delete MY_PROJ",],
|
|
66
139
|
},
|
|
67
140
|
duplicate: {
|
|
@@ -87,13 +160,44 @@ export const projectCommands = {
|
|
|
87
160
|
examples: ["dss project export MY_PROJ --output ./my_proj.zip",],
|
|
88
161
|
},
|
|
89
162
|
import: {
|
|
90
|
-
handler: async (c, a) => {
|
|
91
|
-
|
|
92
|
-
|
|
163
|
+
handler: async (c, a, f) => {
|
|
164
|
+
const usage = "dss project import <filePath> [--target-project-key KEY] [--data JSON|--data-file PATH|--stdin]";
|
|
165
|
+
requireArgs(a, 1, usage);
|
|
166
|
+
const settings = jsonInput(f) ?? {};
|
|
167
|
+
const rawTarget = f["target-project-key"];
|
|
168
|
+
const targetProjectKey = rawTarget?.trim();
|
|
169
|
+
if (rawTarget !== undefined && targetProjectKey === "") {
|
|
170
|
+
throw new UsageError(`--target-project-key must not be empty. Usage: ${usage}`);
|
|
171
|
+
}
|
|
172
|
+
const settingsTarget = settings.targetProjectKey;
|
|
173
|
+
if (settingsTarget !== undefined
|
|
174
|
+
&& (typeof settingsTarget !== "string" || settingsTarget.trim() === "")) {
|
|
175
|
+
throw new UsageError(`targetProjectKey in import settings must be a non-empty string. Usage: ${usage}`);
|
|
176
|
+
}
|
|
177
|
+
if (targetProjectKey !== undefined
|
|
178
|
+
&& settingsTarget !== undefined
|
|
179
|
+
&& targetProjectKey !== settingsTarget.trim()) {
|
|
180
|
+
throw new UsageError(`--target-project-key conflicts with targetProjectKey in import settings. Usage: ${usage}`);
|
|
181
|
+
}
|
|
182
|
+
const result = await c.projects.importProjectFromArchive(a[0], targetProjectKey === undefined ? settings : { ...settings, targetProjectKey, });
|
|
183
|
+
if (result.success !== true) {
|
|
184
|
+
throw new CommandResultFailure({ ...result, resource: "project", action: "import", stage: "process", }, 2);
|
|
185
|
+
}
|
|
186
|
+
return result;
|
|
93
187
|
},
|
|
94
|
-
usage: "dss project import <filePath>",
|
|
95
|
-
description: "Upload a project archive
|
|
96
|
-
examples: [
|
|
188
|
+
usage: "dss project import <filePath> [--target-project-key KEY] [--data JSON|--data-file PATH|--stdin]",
|
|
189
|
+
description: "Upload and process a project archive, optionally overriding its project key. The archive is validated locally before upload; verified success reports the used project key, explicit request/actual remapping, and the landed incarnation hash.",
|
|
190
|
+
examples: [
|
|
191
|
+
"dss project import ./my_proj.zip",
|
|
192
|
+
"dss project import ./my_proj.zip --target-project-key MY_PROJ",
|
|
193
|
+
],
|
|
194
|
+
},
|
|
195
|
+
"inspect-archive": {
|
|
196
|
+
handler: async (_c, a) => inspectArchiveCommand(a),
|
|
197
|
+
localHandler: inspectArchiveCommand,
|
|
198
|
+
usage: INSPECT_ARCHIVE_USAGE,
|
|
199
|
+
description: "Read-only local inspection of a project export archive: member names, sizes, manifest validity, archive issues, and the source project key. Never contacts DSS.",
|
|
200
|
+
examples: ["dss project inspect-archive ./my_proj.zip",],
|
|
97
201
|
},
|
|
98
202
|
"permissions-get": {
|
|
99
203
|
handler: (c, _a, f) => c.projects.getPermissions(f["project-key"]),
|
|
@@ -101,6 +101,7 @@ export declare function commandPlanShape(resource: string, action: string, args:
|
|
|
101
101
|
payload?: unknown;
|
|
102
102
|
localWrites?: unknown;
|
|
103
103
|
wait?: unknown;
|
|
104
|
+
requests?: unknown;
|
|
104
105
|
};
|
|
105
106
|
export declare function buildMutationPlan(resource: string, action: string, meta: CommandMeta, args: string[], flags: Record<string, string | boolean>): Record<string, unknown>;
|
|
106
107
|
/**
|