dataiku-sdk 0.12.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -15
- package/bin/dss.js +21 -2
- package/dist/build-metadata.json +1 -0
- package/dist/packages/types/src/index.d.ts +128 -3
- package/dist/packages/types/src/index.js +106 -3
- package/dist/src/cli/coerce.d.ts +2 -1
- package/dist/src/cli/coerce.js +84 -26
- package/dist/src/cli/commands/app.js +6 -6
- package/dist/src/cli/commands/code.js +2 -2
- package/dist/src/cli/commands/data-quality.js +50 -8
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +197 -12
- package/dist/src/cli/commands/folder.js +5 -4
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/index.js +2 -0
- package/dist/src/cli/commands/job.js +26 -13
- package/dist/src/cli/commands/notebook.js +1 -1
- package/dist/src/cli/commands/project-git.d.ts +2 -0
- package/dist/src/cli/commands/project-git.js +563 -0
- package/dist/src/cli/commands/project.js +118 -14
- package/dist/src/cli/commands/recipe.js +14 -17
- package/dist/src/cli/commands/scenario.js +4 -4
- package/dist/src/cli/contract.d.ts +6 -4
- package/dist/src/cli/contract.js +595 -68
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +50 -19
- package/dist/src/cli/flags.js +23 -2
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/helpers/dataset.js +1 -1
- package/dist/src/cli/output.d.ts +5 -2
- package/dist/src/cli/output.js +13 -12
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +1 -1
- package/dist/src/cli/usage.js +2 -1
- package/dist/src/cli/version.d.ts +48 -6
- package/dist/src/cli/version.js +177 -11
- package/dist/src/cli.js +94 -39
- package/dist/src/client.d.ts +31 -1
- package/dist/src/client.js +39 -7
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/index.d.ts +6 -4
- package/dist/src/index.js +3 -1
- package/dist/src/resources/connections.d.ts +3 -1
- package/dist/src/resources/connections.js +82 -21
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +241 -30
- package/dist/src/resources/folders.js +7 -3
- package/dist/src/resources/project-git.d.ts +154 -0
- package/dist/src/resources/project-git.js +427 -0
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +127 -16
- package/dist/src/resources/recipes.js +4 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +624 -0
- package/package.json +6 -5
- package/packages/types/dist/index.d.ts +128 -3
- package/packages/types/dist/index.js +106 -3
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +26 -19
package/dist/src/cli/coerce.js
CHANGED
|
@@ -1,10 +1,31 @@
|
|
|
1
1
|
import { readFileSync, } from "node:fs";
|
|
2
2
|
import { UsageError, } from "./usage.js";
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
function safeValuePreview(value) {
|
|
4
|
+
return value.length <= 32 ? value : `${value.slice(0, 32)}…`;
|
|
5
|
+
}
|
|
6
|
+
export function num(v, flagName) {
|
|
7
|
+
if (v === undefined || v === false)
|
|
5
8
|
return undefined;
|
|
9
|
+
if (typeof v !== "string") {
|
|
10
|
+
throw new UsageError(`${flagName ?? "Numeric flag"} requires a value.`, "invalid_flag_value", undefined, flagName ? { flag: flagName, } : {});
|
|
11
|
+
}
|
|
12
|
+
if (v.trim().length === 0) {
|
|
13
|
+
throw new UsageError(`Invalid numeric value for ${flagName ?? "flag"}.`, "invalid_flag_value", undefined, flagName ? { flag: flagName, } : {});
|
|
14
|
+
}
|
|
6
15
|
const n = Number(v);
|
|
7
|
-
|
|
16
|
+
if (!Number.isFinite(n)) {
|
|
17
|
+
throw new UsageError(`Invalid numeric value for ${flagName ?? "flag"}: ${safeValuePreview(v)}.`, "invalid_flag_value", undefined, {
|
|
18
|
+
...(flagName ? { flag: flagName, } : {}),
|
|
19
|
+
value: safeValuePreview(v),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
return n;
|
|
23
|
+
}
|
|
24
|
+
export function numFlag(flags, flagNames) {
|
|
25
|
+
const flagName = flagNames.find((name) => flags[name] !== undefined && flags[name] !== false);
|
|
26
|
+
if (flagName === undefined)
|
|
27
|
+
return undefined;
|
|
28
|
+
return num(flags[flagName], `--${flagName}`);
|
|
8
29
|
}
|
|
9
30
|
export function jobBuildTargetType(v) {
|
|
10
31
|
if (v === undefined)
|
|
@@ -21,7 +42,7 @@ export function jobBuildTargetTypeFromFlags(flags) {
|
|
|
21
42
|
return jobBuildTargetType(flags["target-type"] ?? flags["type"]);
|
|
22
43
|
}
|
|
23
44
|
export function maxLogLinesFromFlags(flags) {
|
|
24
|
-
return
|
|
45
|
+
return numFlag(flags, ["max-log-lines", "max-lines",]);
|
|
25
46
|
}
|
|
26
47
|
export function jobLogFilterFromFlag(v) {
|
|
27
48
|
if (v === undefined)
|
|
@@ -124,32 +145,62 @@ export function parseJsonObject(text, source) {
|
|
|
124
145
|
return expectJsonObject(parseJsonValue(text, source), source);
|
|
125
146
|
}
|
|
126
147
|
export function json(v, source = "JSON flag") {
|
|
127
|
-
if (
|
|
148
|
+
if (v === undefined || v === false)
|
|
128
149
|
return undefined;
|
|
150
|
+
if (typeof v !== "string" || v.trim().length === 0) {
|
|
151
|
+
throw new UsageError(`${source} requires a JSON value.`, "invalid_flag_value");
|
|
152
|
+
}
|
|
129
153
|
return parseJsonObject(v, source);
|
|
130
154
|
}
|
|
131
155
|
export function readStdinText() {
|
|
132
156
|
return readFileSync(0, "utf-8");
|
|
133
157
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
158
|
+
const JSON_SOURCE_FLAGS = ["stdin", "data-file", "data",];
|
|
159
|
+
const TEXT_SOURCE_FLAGS = ["content", "file",];
|
|
160
|
+
function selectSingleSource(flags, sourceFlags) {
|
|
161
|
+
const provided = sourceFlags.filter((name) => {
|
|
162
|
+
const value = flags[name];
|
|
163
|
+
if (value === undefined || value === false)
|
|
164
|
+
return false;
|
|
165
|
+
return name === "stdin" ? parseBooleanOption(value, "--stdin") === true : true;
|
|
166
|
+
});
|
|
167
|
+
if (provided.length > 1) {
|
|
168
|
+
throw new UsageError(`Conflicting input sources: ${provided.map((name) => `--${name}`).join(" or ")}. Provide exactly one.`, "conflicting_input_sources", undefined, { sources: provided, });
|
|
139
169
|
}
|
|
140
|
-
|
|
141
|
-
return parseJsonObject(flags["data"], "--data");
|
|
142
|
-
return undefined;
|
|
170
|
+
return provided[0];
|
|
143
171
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
172
|
+
function resolveJsonSourceText(flags, kind) {
|
|
173
|
+
const source = selectSingleSource(flags, JSON_SOURCE_FLAGS);
|
|
174
|
+
if (source === undefined)
|
|
175
|
+
return undefined;
|
|
176
|
+
if (source === "stdin") {
|
|
177
|
+
const jsonText = readStdinText();
|
|
178
|
+
return kind === "object"
|
|
179
|
+
? parseJsonObject(jsonText, "stdin")
|
|
180
|
+
: parseJsonValue(jsonText, "stdin");
|
|
149
181
|
}
|
|
150
|
-
if (
|
|
151
|
-
|
|
152
|
-
|
|
182
|
+
if (source === "data-file") {
|
|
183
|
+
const fileValue = flags["data-file"];
|
|
184
|
+
if (typeof fileValue === "string") {
|
|
185
|
+
const jsonText = readFileSync(fileValue, "utf-8");
|
|
186
|
+
return kind === "object"
|
|
187
|
+
? parseJsonObject(jsonText, fileValue)
|
|
188
|
+
: parseJsonValue(jsonText, fileValue);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const dataValue = flags["data"];
|
|
192
|
+
if (typeof dataValue === "string") {
|
|
193
|
+
return kind === "object"
|
|
194
|
+
? parseJsonObject(dataValue, "--data")
|
|
195
|
+
: parseJsonValue(dataValue, "--data");
|
|
196
|
+
}
|
|
197
|
+
throw new UsageError("JSON input source is present but has no value.", "invalid_flag_value");
|
|
198
|
+
}
|
|
199
|
+
export function jsonInput(flags) {
|
|
200
|
+
return resolveJsonSourceText(flags, "object");
|
|
201
|
+
}
|
|
202
|
+
export function unknownJsonInput(flags) {
|
|
203
|
+
return resolveJsonSourceText(flags, "any");
|
|
153
204
|
}
|
|
154
205
|
export function schemaColumnsInput(flags, usage) {
|
|
155
206
|
const input = unknownJsonInput(flags);
|
|
@@ -184,11 +235,18 @@ export function schemaColumnsInput(flags, usage) {
|
|
|
184
235
|
});
|
|
185
236
|
}
|
|
186
237
|
export function textInput(flags) {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
238
|
+
const source = selectSingleSource(flags, TEXT_SOURCE_FLAGS);
|
|
239
|
+
if (source === undefined)
|
|
240
|
+
return undefined;
|
|
241
|
+
if (source === "file") {
|
|
242
|
+
const fileValue = flags["file"];
|
|
243
|
+
if (typeof fileValue === "string")
|
|
244
|
+
return readFileSync(fileValue, "utf-8");
|
|
245
|
+
}
|
|
246
|
+
const contentValue = flags["content"];
|
|
247
|
+
if (typeof contentValue === "string")
|
|
248
|
+
return contentValue;
|
|
249
|
+
throw new UsageError("Text input source is present but has no value.", "invalid_flag_value");
|
|
192
250
|
}
|
|
193
251
|
export function requiredJsonInput(flags, message) {
|
|
194
252
|
const data = jsonInput(flags);
|
|
@@ -264,8 +264,8 @@ export const appCommands = {
|
|
|
264
264
|
let waited;
|
|
265
265
|
try {
|
|
266
266
|
waited = await c.futures.wait(jobId, {
|
|
267
|
-
pollIntervalMs: num(f["poll-interval"]),
|
|
268
|
-
timeoutMs: num(f["timeout"]),
|
|
267
|
+
pollIntervalMs: num(f["poll-interval"], "--poll-interval"),
|
|
268
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
269
269
|
});
|
|
270
270
|
}
|
|
271
271
|
catch (error) {
|
|
@@ -353,8 +353,8 @@ export const appCommands = {
|
|
|
353
353
|
...(f["name"] !== undefined ? { name: f["name"], } : {}),
|
|
354
354
|
copyPermissions: parseBooleanOption(f["copy-permissions"], "--copy-permissions") ?? false,
|
|
355
355
|
dryRun: parseBooleanOption(f["dry-run"], "--dry-run") ?? false,
|
|
356
|
-
timeoutMs: num(f["timeout"]),
|
|
357
|
-
pollIntervalMs: num(f["poll-interval"]),
|
|
356
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
357
|
+
pollIntervalMs: num(f["poll-interval"], "--poll-interval"),
|
|
358
358
|
}, CREATE_SUCCESSOR_USAGE);
|
|
359
359
|
},
|
|
360
360
|
usage: CREATE_SUCCESSOR_USAGE,
|
|
@@ -550,8 +550,8 @@ export const appCommands = {
|
|
|
550
550
|
let waited;
|
|
551
551
|
try {
|
|
552
552
|
waited = await c.futures.wait(futureId, {
|
|
553
|
-
pollIntervalMs: num(f["poll-interval"]),
|
|
554
|
-
timeoutMs: num(f["timeout"]),
|
|
553
|
+
pollIntervalMs: num(f["poll-interval"], "--poll-interval"),
|
|
554
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
555
555
|
});
|
|
556
556
|
}
|
|
557
557
|
catch (error) {
|
|
@@ -32,9 +32,9 @@ export const codeCommands = {
|
|
|
32
32
|
const run = await c.scenarios.runScript(script, {
|
|
33
33
|
envName: f["env"],
|
|
34
34
|
projectKey: f["project-key"],
|
|
35
|
-
timeoutMs: num(f["timeout"]),
|
|
35
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
36
36
|
keepScenario: f["keep"] === true,
|
|
37
|
-
maxLogBytes: num(f["max-log-bytes"]),
|
|
37
|
+
maxLogBytes: num(f["max-log-bytes"], "--max-log-bytes"),
|
|
38
38
|
});
|
|
39
39
|
const result = {
|
|
40
40
|
outcome: run.outcome,
|
|
@@ -2,6 +2,18 @@ import { deepMerge, } from "../../utils/deep-merge.js";
|
|
|
2
2
|
import { num, parseBooleanOption, requiredJsonInput, } from "../coerce.js";
|
|
3
3
|
import { encodedProjectEndpoint, readIfExists, skipResult, } from "../output.js";
|
|
4
4
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
5
|
+
/**
|
|
6
|
+
* A data quality result counts as passing only when it actually carries a
|
|
7
|
+
* verdict and every verdict it carries reports OK. A result with no outcome
|
|
8
|
+
* and no status has proven nothing, and a result whose fields disagree
|
|
9
|
+
* (outcome ERROR alongside status OK) is a failure — never a pass.
|
|
10
|
+
*/
|
|
11
|
+
function resultReportsOk(result) {
|
|
12
|
+
const verdicts = [result.outcome, result.status,]
|
|
13
|
+
.filter((value) => typeof value === "string" && value.trim().length > 0)
|
|
14
|
+
.map((value) => value.trim().toUpperCase());
|
|
15
|
+
return verdicts.length > 0 && verdicts.every((value) => value === "OK");
|
|
16
|
+
}
|
|
5
17
|
export const dataQualityCommands = {
|
|
6
18
|
rules: {
|
|
7
19
|
handler: (c, a, f) => {
|
|
@@ -160,14 +172,44 @@ export const dataQualityCommands = {
|
|
|
160
172
|
description: "Get latest data quality rule results for a dataset.",
|
|
161
173
|
examples: ["dss data-quality last-results orders",],
|
|
162
174
|
},
|
|
175
|
+
"assert-results": {
|
|
176
|
+
handler: async (c, a, f) => {
|
|
177
|
+
requireArgs(a, 1, "dss data-quality assert-results <dataset>");
|
|
178
|
+
const ruleId = f["rule-id"];
|
|
179
|
+
const pk = f["project-key"];
|
|
180
|
+
const results = await c.dataQuality.lastResults(a[0], { ruleId, projectKey: pk, });
|
|
181
|
+
const selected = results.filter((result) => ruleId === undefined || result.id === ruleId || result.ruleId === ruleId);
|
|
182
|
+
const failed = selected
|
|
183
|
+
.filter((result) => !resultReportsOk(result))
|
|
184
|
+
.map((result) => ({
|
|
185
|
+
ruleId: result.id ?? result.ruleId ?? result.name ?? null,
|
|
186
|
+
outcome: result.outcome ?? null,
|
|
187
|
+
status: result.status ?? null,
|
|
188
|
+
}));
|
|
189
|
+
return {
|
|
190
|
+
satisfied: selected.length > 0 && failed.length === 0,
|
|
191
|
+
dataset: a[0],
|
|
192
|
+
...(ruleId !== undefined ? { ruleId, } : {}),
|
|
193
|
+
checked: selected.length,
|
|
194
|
+
failed,
|
|
195
|
+
...(selected.length === 0 ? { reason: "no_results", } : {}),
|
|
196
|
+
};
|
|
197
|
+
},
|
|
198
|
+
usage: "dss data-quality assert-results <dataset> [--rule-id ID] [--project-key KEY]",
|
|
199
|
+
description: "Assert the latest data quality results: at least one selected rule must exist and every selected result must report OK. Returns { satisfied, checked, failed } with failed rule ids/outcomes only; a failed assertion exits 4 with assertion_failed.",
|
|
200
|
+
examples: [
|
|
201
|
+
"dss data-quality assert-results orders",
|
|
202
|
+
"dss data-quality assert-results orders --rule-id RULE_ID",
|
|
203
|
+
],
|
|
204
|
+
},
|
|
163
205
|
history: {
|
|
164
206
|
handler: (c, a, f) => {
|
|
165
207
|
requireArgs(a, 1, "dss data-quality history <dataset>");
|
|
166
208
|
return c.dataQuality.history(a[0], {
|
|
167
|
-
minTimestamp: num(f["min-timestamp"]),
|
|
168
|
-
maxTimestamp: num(f["max-timestamp"]),
|
|
169
|
-
resultsPerPage: num(f["results-per-page"]),
|
|
170
|
-
page: num(f["page"]),
|
|
209
|
+
minTimestamp: num(f["min-timestamp"], "--min-timestamp"),
|
|
210
|
+
maxTimestamp: num(f["max-timestamp"], "--max-timestamp"),
|
|
211
|
+
resultsPerPage: num(f["results-per-page"], "--results-per-page"),
|
|
212
|
+
page: num(f["page"], "--page"),
|
|
171
213
|
ruleId: f["rule-id"],
|
|
172
214
|
projectKey: f["project-key"],
|
|
173
215
|
});
|
|
@@ -187,8 +229,8 @@ export const dataQualityCommands = {
|
|
|
187
229
|
},
|
|
188
230
|
"project-timeline": {
|
|
189
231
|
handler: (c, _a, f) => c.dataQuality.projectTimeline({
|
|
190
|
-
minTimestamp: num(f["min-timestamp"]),
|
|
191
|
-
maxTimestamp: num(f["max-timestamp"]),
|
|
232
|
+
minTimestamp: num(f["min-timestamp"], "--min-timestamp"),
|
|
233
|
+
maxTimestamp: num(f["max-timestamp"], "--max-timestamp"),
|
|
192
234
|
projectKey: f["project-key"],
|
|
193
235
|
}),
|
|
194
236
|
usage: "dss data-quality project-timeline [--min-timestamp MS] [--max-timestamp MS] [--project-key KEY]",
|
|
@@ -201,10 +243,10 @@ export const dataQualityCommands = {
|
|
|
201
243
|
const pk = f["project-key"];
|
|
202
244
|
const options = {
|
|
203
245
|
partition: f["partition"],
|
|
204
|
-
pollIntervalMs: num(f["poll-interval"]),
|
|
246
|
+
pollIntervalMs: num(f["poll-interval"], "--poll-interval"),
|
|
205
247
|
ruleId: f["rule-id"],
|
|
206
248
|
projectKey: pk,
|
|
207
|
-
timeoutMs: num(f["timeout"]),
|
|
249
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
208
250
|
};
|
|
209
251
|
if (f["dry-run"] === true) {
|
|
210
252
|
const params = new URLSearchParams();
|
|
@@ -1,2 +1,25 @@
|
|
|
1
1
|
import type { CommandMeta } from "../types.js";
|
|
2
|
+
export interface DatasetSchemaDifference {
|
|
3
|
+
/** Path from the schema root in dot/bracket notation, e.g. `columns[2].type`. */
|
|
4
|
+
path: string;
|
|
5
|
+
kind: "added" | "removed" | "changed";
|
|
6
|
+
/** SHA-256 of the deterministic JSON rendering of the expected subtree. */
|
|
7
|
+
expectedHash?: string;
|
|
8
|
+
/** SHA-256 of the deterministic JSON rendering of the actual subtree. */
|
|
9
|
+
actualHash?: string;
|
|
10
|
+
}
|
|
11
|
+
export interface DatasetSchemaComparison {
|
|
12
|
+
equal: boolean;
|
|
13
|
+
expectedHash: string;
|
|
14
|
+
actualHash: string;
|
|
15
|
+
differences: DatasetSchemaDifference[];
|
|
16
|
+
totalDifferences: number;
|
|
17
|
+
differencesTruncated: boolean;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Compare two schema values deterministically — object key order never matters
|
|
21
|
+
* — and report each structural difference as a concise per-path entry carrying
|
|
22
|
+
* subtree hashes instead of subtree values.
|
|
23
|
+
*/
|
|
24
|
+
export declare function compareDatasetSchemas(expected: unknown, actual: unknown): DatasetSchemaComparison;
|
|
2
25
|
export declare const datasetCommands: Record<string, CommandMeta>;
|
|
@@ -1,10 +1,102 @@
|
|
|
1
1
|
import { buildDatasetCloneSettings, } from "../../resources/datasets.js";
|
|
2
2
|
import { deepMerge, } from "../../utils/deep-merge.js";
|
|
3
|
-
import {
|
|
3
|
+
import { compareStrings, stableHash, } from "../../utils/stable-hash.js";
|
|
4
|
+
import { jsonInput, num, schemaColumnsInput, unknownJsonInput, } from "../coerce.js";
|
|
4
5
|
import { datasetSourceSummary, } from "../helpers/dataset.js";
|
|
5
6
|
import { moveCreatedItemsToZone, resolveFlowZoneIdFromFlags, } from "../helpers/flow-zone.js";
|
|
6
7
|
import { enqueueCliWarning, readIfExists, skipResult, } from "../output.js";
|
|
7
8
|
import { requireArgs, UsageError, } from "../usage.js";
|
|
9
|
+
const MAX_SCHEMA_DIFFERENCES = 50;
|
|
10
|
+
function isPlainObject(value) {
|
|
11
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
12
|
+
}
|
|
13
|
+
function appendSchemaPath(parent, key) {
|
|
14
|
+
return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)
|
|
15
|
+
? (parent ? `${parent}.${key}` : key)
|
|
16
|
+
: `${parent}[${JSON.stringify(key)}]`;
|
|
17
|
+
}
|
|
18
|
+
function collectSchemaDifferences(expected, actual, path, acc) {
|
|
19
|
+
// Hash equality is the deterministic equivalence test: structurally equal
|
|
20
|
+
// subtrees (any key order) match exactly and are never descended into.
|
|
21
|
+
if (stableHash(expected) === stableHash(actual))
|
|
22
|
+
return;
|
|
23
|
+
if (isPlainObject(expected) && isPlainObject(actual)) {
|
|
24
|
+
const keys = [...new Set([...Object.keys(expected), ...Object.keys(actual),]),].sort(compareStrings);
|
|
25
|
+
for (const key of keys) {
|
|
26
|
+
const nextPath = appendSchemaPath(path, key);
|
|
27
|
+
const hasExpected = key in expected;
|
|
28
|
+
const hasActual = key in actual;
|
|
29
|
+
if (hasExpected && hasActual) {
|
|
30
|
+
collectSchemaDifferences(expected[key], actual[key], nextPath, acc);
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
acc.total += 1;
|
|
34
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
35
|
+
acc.items.push({
|
|
36
|
+
path: nextPath,
|
|
37
|
+
kind: hasActual ? "added" : "removed",
|
|
38
|
+
...(hasActual
|
|
39
|
+
? { actualHash: stableHash(actual[key]), }
|
|
40
|
+
: { expectedHash: stableHash(expected[key]), }),
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (Array.isArray(expected) && Array.isArray(actual)) {
|
|
47
|
+
const shared = Math.min(expected.length, actual.length);
|
|
48
|
+
for (let i = 0; i < shared; i++) {
|
|
49
|
+
collectSchemaDifferences(expected[i], actual[i], `${path}[${i}]`, acc);
|
|
50
|
+
}
|
|
51
|
+
for (let i = shared; i < expected.length; i++) {
|
|
52
|
+
acc.total += 1;
|
|
53
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
54
|
+
acc.items.push({
|
|
55
|
+
path: `${path}[${i}]`,
|
|
56
|
+
kind: "removed",
|
|
57
|
+
expectedHash: stableHash(expected[i]),
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
for (let i = shared; i < actual.length; i++) {
|
|
62
|
+
acc.total += 1;
|
|
63
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
64
|
+
acc.items.push({
|
|
65
|
+
path: `${path}[${i}]`,
|
|
66
|
+
kind: "added",
|
|
67
|
+
actualHash: stableHash(actual[i]),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
acc.total += 1;
|
|
74
|
+
if (acc.items.length < MAX_SCHEMA_DIFFERENCES) {
|
|
75
|
+
acc.items.push({
|
|
76
|
+
path: path || "$",
|
|
77
|
+
kind: "changed",
|
|
78
|
+
expectedHash: stableHash(expected),
|
|
79
|
+
actualHash: stableHash(actual),
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Compare two schema values deterministically — object key order never matters
|
|
85
|
+
* — and report each structural difference as a concise per-path entry carrying
|
|
86
|
+
* subtree hashes instead of subtree values.
|
|
87
|
+
*/
|
|
88
|
+
export function compareDatasetSchemas(expected, actual) {
|
|
89
|
+
const acc = { items: [], total: 0, };
|
|
90
|
+
collectSchemaDifferences(expected, actual, "", acc);
|
|
91
|
+
return {
|
|
92
|
+
equal: acc.items.length === 0,
|
|
93
|
+
expectedHash: stableHash(expected),
|
|
94
|
+
actualHash: stableHash(actual),
|
|
95
|
+
differences: acc.items,
|
|
96
|
+
totalDifferences: acc.total,
|
|
97
|
+
differencesTruncated: acc.items.length < acc.total,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
8
100
|
export const datasetCommands = {
|
|
9
101
|
list: {
|
|
10
102
|
handler: (c, _a, f) => c.datasets.list(f["project-key"]),
|
|
@@ -69,6 +161,34 @@ export const datasetCommands = {
|
|
|
69
161
|
description: "Show backing connection, catalog/schema/table, path, and format for a dataset.",
|
|
70
162
|
examples: ["dss dataset source orders",],
|
|
71
163
|
},
|
|
164
|
+
files: {
|
|
165
|
+
handler: (c, a, f) => {
|
|
166
|
+
requireArgs(a, 1, "dss dataset files <name> [--project-key KEY]");
|
|
167
|
+
return c.datasets.listUploadedFiles(a[0], f["project-key"]);
|
|
168
|
+
},
|
|
169
|
+
usage: "dss dataset files <name> [--project-key KEY]",
|
|
170
|
+
description: "List files stored by an UploadedFiles dataset.",
|
|
171
|
+
examples: ["dss dataset files uploaded_input",],
|
|
172
|
+
},
|
|
173
|
+
"upload-file": {
|
|
174
|
+
handler: (c, a, f) => {
|
|
175
|
+
const usage = "dss dataset upload-file <name> <localPath> --file-name NAME [--project-key KEY]";
|
|
176
|
+
requireArgs(a, 2, usage);
|
|
177
|
+
const fileName = f["file-name"];
|
|
178
|
+
if (!fileName || fileName.trim() === "") {
|
|
179
|
+
throw new UsageError(`--file-name is required. Usage: ${usage}`);
|
|
180
|
+
}
|
|
181
|
+
return c.datasets.uploadDatasetFile(a[0], a[1], {
|
|
182
|
+
fileName,
|
|
183
|
+
projectKey: f["project-key"],
|
|
184
|
+
});
|
|
185
|
+
},
|
|
186
|
+
usage: "dss dataset upload-file <name> <localPath> --file-name NAME [--project-key KEY]",
|
|
187
|
+
description: "Add one new file to an UploadedFiles dataset and verify the stored byte length.",
|
|
188
|
+
examples: [
|
|
189
|
+
"dss dataset upload-file uploaded_input ./input.csv --file-name input.csv",
|
|
190
|
+
],
|
|
191
|
+
},
|
|
72
192
|
"refresh-schema": {
|
|
73
193
|
handler: async (c, a, f) => {
|
|
74
194
|
const usage = "dss dataset refresh-schema <name> [--data JSON | --data-file PATH | --stdin] [--dry-run] [--project-key KEY]";
|
|
@@ -105,18 +225,79 @@ export const datasetCommands = {
|
|
|
105
225
|
examples: ["dss dataset validate-build orders",],
|
|
106
226
|
},
|
|
107
227
|
preview: {
|
|
108
|
-
handler: (c, a, f) => {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
228
|
+
handler: async (c, a, f) => {
|
|
229
|
+
const previewUsage = "dss dataset preview <name> [--max-rows N] [--rows N] [--project-key KEY] [--timeout MS]";
|
|
230
|
+
requireArgs(a, 1, previewUsage);
|
|
231
|
+
const maxRows = num(f["max-rows"], "--max-rows");
|
|
232
|
+
if (maxRows !== undefined && (!Number.isSafeInteger(maxRows) || maxRows <= 0 || maxRows > 500)) {
|
|
233
|
+
throw new UsageError(`--max-rows must be a positive integer no greater than 500 (got "${String(f["max-rows"])}"). `
|
|
234
|
+
+ `Usage: ${previewUsage}`, "validation_failed");
|
|
235
|
+
}
|
|
236
|
+
const result = await c.datasets.preview(a[0], {
|
|
237
|
+
maxRows,
|
|
112
238
|
projectKey: f["project-key"],
|
|
113
|
-
timeoutMs: num(f["timeout"]),
|
|
239
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
114
240
|
});
|
|
241
|
+
if (result.truncated) {
|
|
242
|
+
enqueueCliWarning({
|
|
243
|
+
code: "dataset_preview_truncated",
|
|
244
|
+
message: `Preview of '${a[0]}' stopped at the ${result.limit}-row cap; the dataset has more rows. ${result.limit >= 500
|
|
245
|
+
? "The preview cap of 500 rows is the public maximum; use `dss dataset download` for more rows."
|
|
246
|
+
: "Re-run with --max-rows N (up to 500) for more rows."}`,
|
|
247
|
+
dataset: a[0],
|
|
248
|
+
rows: result.rowCount,
|
|
249
|
+
limit: result.limit,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
return result;
|
|
115
253
|
},
|
|
116
254
|
usage: "dss dataset preview <name> [--max-rows N] [--rows N] [--project-key KEY] [--timeout MS]",
|
|
117
|
-
description: "Preview dataset rows (--rows is an alias for --max-rows).",
|
|
255
|
+
description: "Preview dataset rows (--rows is an alias for --max-rows). Returns { columns, rows, rowCount, truncated, limit }; when the dataset has more rows than the cap, truncated is true and a dataset_preview_truncated warning is written to stderr.",
|
|
118
256
|
examples: ["dss dataset preview orders", "dss dataset preview orders --rows 5",],
|
|
119
257
|
},
|
|
258
|
+
"assert-count": {
|
|
259
|
+
handler: async (c, a, f) => {
|
|
260
|
+
const assertUsage = "dss dataset assert-count <dataset> --expected N [--project-key KEY]";
|
|
261
|
+
requireArgs(a, 1, assertUsage);
|
|
262
|
+
const expected = num(f["expected"], "--expected");
|
|
263
|
+
if (expected === undefined || !Number.isSafeInteger(expected) || expected < 0) {
|
|
264
|
+
throw new UsageError(`--expected must be a non-negative integer (got "${String(f["expected"])}"). `
|
|
265
|
+
+ `Usage: ${assertUsage}`, "validation_failed");
|
|
266
|
+
}
|
|
267
|
+
const result = await c.datasets.assertRowCount(a[0], expected, f["project-key"]);
|
|
268
|
+
return { ...result, dataset: a[0], };
|
|
269
|
+
},
|
|
270
|
+
usage: "dss dataset assert-count <dataset> --expected N [--project-key KEY]",
|
|
271
|
+
description: "Stream the dataset and assert it holds exactly --expected rows, probing at most N+1 rows. Returns { satisfied, expected, count, exact }; a mismatch exits 4 with assertion_failed.",
|
|
272
|
+
examples: ["dss dataset assert-count orders --expected 1000",],
|
|
273
|
+
},
|
|
274
|
+
"assert-schema": {
|
|
275
|
+
handler: async (c, a, f) => {
|
|
276
|
+
const assertUsage = "dss dataset assert-schema <dataset> (--data JSON | --data-file PATH | --stdin) [--project-key KEY]";
|
|
277
|
+
requireArgs(a, 1, assertUsage);
|
|
278
|
+
const expected = unknownJsonInput(f);
|
|
279
|
+
if (expected === undefined) {
|
|
280
|
+
throw new UsageError(`--data, --data-file, or --stdin is required. Usage: ${assertUsage}`);
|
|
281
|
+
}
|
|
282
|
+
const actual = await c.datasets.getSchemaObject(a[0], f["project-key"]);
|
|
283
|
+
const comparison = compareDatasetSchemas(expected, actual);
|
|
284
|
+
return {
|
|
285
|
+
satisfied: comparison.equal,
|
|
286
|
+
dataset: a[0],
|
|
287
|
+
expectedHash: comparison.expectedHash,
|
|
288
|
+
actualHash: comparison.actualHash,
|
|
289
|
+
totalDifferences: comparison.totalDifferences,
|
|
290
|
+
...(comparison.differencesTruncated ? { differencesTruncated: true, } : {}),
|
|
291
|
+
differences: comparison.differences,
|
|
292
|
+
};
|
|
293
|
+
},
|
|
294
|
+
usage: "dss dataset assert-schema <dataset> (--data JSON | --data-file PATH | --stdin) [--project-key KEY]",
|
|
295
|
+
description: "Assert the dataset's full schema equals the expected schema object; reports concise per-path differences with sha256 subtree hashes. A mismatch exits 4 with assertion_failed.",
|
|
296
|
+
examples: [
|
|
297
|
+
`dss dataset assert-schema orders --data-file expected-schema.json`,
|
|
298
|
+
`dss dataset assert-schema orders --data '{"columns":[]}'`,
|
|
299
|
+
],
|
|
300
|
+
},
|
|
120
301
|
metadata: {
|
|
121
302
|
handler: (c, a, f) => {
|
|
122
303
|
requireArgs(a, 1, "dss dataset metadata <name>");
|
|
@@ -132,7 +313,7 @@ export const datasetCommands = {
|
|
|
132
313
|
const result = await c.datasets.download(a[0], {
|
|
133
314
|
outputPath: f["output"],
|
|
134
315
|
projectKey: f["project-key"],
|
|
135
|
-
limit: num(f["limit"]),
|
|
316
|
+
limit: num(f["limit"], "--limit"),
|
|
136
317
|
});
|
|
137
318
|
if (result.truncated) {
|
|
138
319
|
enqueueCliWarning({
|
|
@@ -166,8 +347,11 @@ export const datasetCommands = {
|
|
|
166
347
|
const name = f["name"];
|
|
167
348
|
const connection = f["connection"];
|
|
168
349
|
const dsType = f["type"];
|
|
169
|
-
if (!name || !
|
|
170
|
-
throw new UsageError("--name
|
|
350
|
+
if (!name || !dsType) {
|
|
351
|
+
throw new UsageError("--name and --type are required. Usage: dss dataset create --name NAME --type TYPE [--connection CONN]");
|
|
352
|
+
}
|
|
353
|
+
if (!connection && dsType.toLowerCase() !== "uploadedfiles") {
|
|
354
|
+
throw new UsageError("--connection is required unless --type is UploadedFiles.");
|
|
171
355
|
}
|
|
172
356
|
const payload = {
|
|
173
357
|
datasetName: name,
|
|
@@ -198,9 +382,10 @@ export const datasetCommands = {
|
|
|
198
382
|
const moved = await moveCreatedItemsToZone(c, f, [{ objectId: name, objectType: "DATASET", },], pk);
|
|
199
383
|
return { created: name, resource: "dataset", ...moved, };
|
|
200
384
|
},
|
|
201
|
-
usage: "dss dataset create --name NAME --
|
|
202
|
-
description: "Create a new dataset.",
|
|
385
|
+
usage: "dss dataset create --name NAME --type TYPE [--connection CONN] [--zone ZONE|--zone-id ID] [--if-not-exists] [--dry-run] [--project-key KEY]",
|
|
386
|
+
description: "Create a new dataset. UploadedFiles uses the server's default upload connection when omitted.",
|
|
203
387
|
examples: [
|
|
388
|
+
"dss dataset create --name uploads --type UploadedFiles",
|
|
204
389
|
"dss dataset create --name orders --connection filesystem --type Filesystem",
|
|
205
390
|
"dss dataset create --name orders --connection filesystem --type Filesystem --zone Experiments --dry-run",
|
|
206
391
|
],
|
|
@@ -18,8 +18,8 @@ export const folderCommands = {
|
|
|
18
18
|
const type = f["type"];
|
|
19
19
|
const connection = f["connection"];
|
|
20
20
|
const pk = f["project-key"];
|
|
21
|
-
if (!name
|
|
22
|
-
throw new UsageError("--name
|
|
21
|
+
if (!name) {
|
|
22
|
+
throw new UsageError("--name is required. Usage: dss folder create --name NAME [--type TYPE] [--connection CONN] [--path PATH]");
|
|
23
23
|
}
|
|
24
24
|
const payload = {
|
|
25
25
|
name,
|
|
@@ -48,9 +48,10 @@ export const folderCommands = {
|
|
|
48
48
|
const created = await c.folders.create(payload);
|
|
49
49
|
return { created: created.id ?? name, resource: "folder", ...created, };
|
|
50
50
|
},
|
|
51
|
-
usage: "dss folder create --name NAME --type TYPE --connection CONN [--path PATH] [--if-not-exists] [--dry-run] [--project-key KEY]",
|
|
52
|
-
description: "Create a managed folder.",
|
|
51
|
+
usage: "dss folder create --name NAME [--type TYPE] [--connection CONN] [--path PATH] [--if-not-exists] [--dry-run] [--project-key KEY]",
|
|
52
|
+
description: "Create a managed folder; when omitted, the connection is selected from writable managed-folder storage and DSS infers the folder type.",
|
|
53
53
|
examples: [
|
|
54
|
+
"dss folder create --name exports",
|
|
54
55
|
"dss folder create --name exports --type S3 --connection s3_connection",
|
|
55
56
|
"dss folder create --name exports --type S3 --connection s3_connection --path /dataiku/MYPROJ/exports --dry-run",
|
|
56
57
|
],
|
|
@@ -24,8 +24,8 @@ export const futureCommands = {
|
|
|
24
24
|
handler: (c, a, f) => {
|
|
25
25
|
requireArgs(a, 1, "dss future wait <id>");
|
|
26
26
|
return c.futures.wait(a[0], {
|
|
27
|
-
pollIntervalMs: num(f["poll-interval"]),
|
|
28
|
-
timeoutMs: num(f["timeout"]),
|
|
27
|
+
pollIntervalMs: num(f["poll-interval"], "--poll-interval"),
|
|
28
|
+
timeoutMs: num(f["timeout"], "--timeout"),
|
|
29
29
|
});
|
|
30
30
|
},
|
|
31
31
|
usage: "dss future wait <id> [--timeout MS] [--poll-interval MS]",
|
|
@@ -24,6 +24,7 @@ import { mlTaskCommands, } from "./ml-task.js";
|
|
|
24
24
|
import { modelEvaluationStoreCommands, } from "./model-evaluation-store.js";
|
|
25
25
|
import { notebookCommands, } from "./notebook.js";
|
|
26
26
|
import { projectDeployerCommands, } from "./project-deployer.js";
|
|
27
|
+
import { projectGitCommands, } from "./project-git.js";
|
|
27
28
|
import { projectLibraryCommands, } from "./project-library.js";
|
|
28
29
|
import { projectCommands, } from "./project.js";
|
|
29
30
|
import { recipeCommands, } from "./recipe.js";
|
|
@@ -49,6 +50,7 @@ export const commands = {
|
|
|
49
50
|
"api-deployer": apiDeployerCommands,
|
|
50
51
|
bundle: bundleCommands,
|
|
51
52
|
"project-deployer": projectDeployerCommands,
|
|
53
|
+
"project-git": projectGitCommands,
|
|
52
54
|
"project-library": projectLibraryCommands,
|
|
53
55
|
"streaming-endpoint": streamingEndpointCommands,
|
|
54
56
|
"continuous-activity": continuousActivityCommands,
|