dataiku-sdk 3.6.1 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/bin/bun-version.js +14 -0
- package/bin/dss.js +11 -15
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +114 -5
- package/dist/packages/types/src/index.js +75 -2
- package/dist/src/cli/coerce.d.ts +2 -0
- package/dist/src/cli/coerce.js +20 -16
- package/dist/src/cli/command-syntax.json +3369 -144
- package/dist/src/cli/commands/analysis.js +19 -1
- package/dist/src/cli/commands/bundle.js +22 -0
- package/dist/src/cli/commands/connection.js +26 -2
- package/dist/src/cli/commands/data-quality.js +33 -1
- package/dist/src/cli/commands/dataset.js +76 -9
- package/dist/src/cli/commands/discussion.js +15 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/flow-zone.js +241 -46
- package/dist/src/cli/commands/flow.d.ts +5 -0
- package/dist/src/cli/commands/flow.js +57 -0
- package/dist/src/cli/commands/future.js +8 -0
- package/dist/src/cli/commands/index.js +6 -0
- package/dist/src/cli/commands/job.js +44 -6
- package/dist/src/cli/commands/metrics.js +21 -1
- package/dist/src/cli/commands/ml-task.d.ts +5 -0
- package/dist/src/cli/commands/ml-task.js +86 -16
- package/dist/src/cli/commands/mlflow.d.ts +11 -0
- package/dist/src/cli/commands/mlflow.js +121 -0
- package/dist/src/cli/commands/plugin.js +5 -11
- package/dist/src/cli/commands/project-deployer.js +70 -0
- package/dist/src/cli/commands/project-library.js +2 -5
- package/dist/src/cli/commands/project.js +8 -0
- package/dist/src/cli/commands/recipe.d.ts +13 -0
- package/dist/src/cli/commands/recipe.js +130 -7
- package/dist/src/cli/commands/saved-model.d.ts +1 -1
- package/dist/src/cli/commands/saved-model.js +9 -76
- package/dist/src/cli/commands/streaming-endpoint.js +30 -1
- package/dist/src/cli/commands/webapp.js +8 -0
- package/dist/src/cli/commands/wiki.js +78 -0
- package/dist/src/cli/commands/workspace.js +8 -0
- package/dist/src/cli/contract.d.ts +16 -15
- package/dist/src/cli/contract.js +150 -3407
- package/dist/src/cli/doctor.d.ts +13 -0
- package/dist/src/cli/doctor.js +43 -1
- package/dist/src/cli/flags.d.ts +14 -1
- package/dist/src/cli/flags.js +106 -8
- package/dist/src/cli/helpers/app-successor.js +1 -2
- package/dist/src/cli/helpers/dataset.d.ts +12 -0
- package/dist/src/cli/helpers/dataset.js +36 -0
- package/dist/src/cli/helpers/documentation.d.ts +21 -0
- package/dist/src/cli/helpers/documentation.js +53 -0
- package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
- package/dist/src/cli/helpers/flow-zone.js +144 -4
- package/dist/src/cli/helpers/trained-model.d.ts +26 -0
- package/dist/src/cli/helpers/trained-model.js +144 -0
- package/dist/src/cli/list-output.d.ts +7 -0
- package/dist/src/cli/list-output.js +77 -0
- package/dist/src/cli/list-shapes.d.ts +21 -0
- package/dist/src/cli/list-shapes.js +111 -0
- package/dist/src/cli/output-schemas.d.ts +0 -14
- package/dist/src/cli/output-schemas.js +4 -7
- package/dist/src/cli/output.d.ts +0 -1
- package/dist/src/cli/output.js +33 -19
- package/dist/src/cli/plans.d.ts +14 -0
- package/dist/src/cli/plans.js +3740 -0
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +7 -0
- package/dist/src/cli/usage.js +20 -1
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +2 -2
- package/dist/src/cli.js +137 -115
- package/dist/src/client.d.ts +52 -43
- package/dist/src/client.js +267 -96
- package/dist/src/errors.d.ts +22 -2
- package/dist/src/errors.js +49 -50
- package/dist/src/generated/action-output-schemas.json +345 -89
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +3 -0
- package/dist/src/resources/analyses.d.ts +8 -0
- package/dist/src/resources/analyses.js +13 -1
- package/dist/src/resources/bundles.d.ts +34 -0
- package/dist/src/resources/bundles.js +67 -0
- package/dist/src/resources/connections.d.ts +9 -0
- package/dist/src/resources/connections.js +15 -0
- package/dist/src/resources/data-quality.d.ts +20 -0
- package/dist/src/resources/data-quality.js +30 -0
- package/dist/src/resources/dataset-create.d.ts +38 -0
- package/dist/src/resources/dataset-create.js +135 -0
- package/dist/src/resources/datasets.d.ts +46 -15
- package/dist/src/resources/datasets.js +120 -112
- package/dist/src/resources/discussions.d.ts +5 -0
- package/dist/src/resources/discussions.js +7 -0
- package/dist/src/resources/documentation.d.ts +25 -0
- package/dist/src/resources/documentation.js +16 -0
- package/dist/src/resources/flow-zones.d.ts +29 -3
- package/dist/src/resources/flow-zones.js +53 -3
- package/dist/src/resources/flow.d.ts +48 -0
- package/dist/src/resources/flow.js +65 -0
- package/dist/src/resources/futures.d.ts +23 -0
- package/dist/src/resources/futures.js +32 -0
- package/dist/src/resources/jobs.d.ts +34 -0
- package/dist/src/resources/jobs.js +170 -3
- package/dist/src/resources/metrics.d.ts +10 -0
- package/dist/src/resources/metrics.js +10 -0
- package/dist/src/resources/ml-tasks.d.ts +65 -2
- package/dist/src/resources/ml-tasks.js +102 -28
- package/dist/src/resources/mlflow-extension.d.ts +73 -0
- package/dist/src/resources/mlflow-extension.js +105 -0
- package/dist/src/resources/projects.d.ts +15 -0
- package/dist/src/resources/projects.js +18 -3
- package/dist/src/resources/recipe-create.d.ts +79 -0
- package/dist/src/resources/recipe-create.js +454 -0
- package/dist/src/resources/recipes.d.ts +28 -29
- package/dist/src/resources/recipes.js +201 -267
- package/dist/src/resources/saved-models.d.ts +10 -6
- package/dist/src/resources/saved-models.js +16 -14
- package/dist/src/resources/scenarios.js +8 -14
- package/dist/src/resources/sql.js +52 -13
- package/dist/src/resources/streaming-endpoints.d.ts +12 -0
- package/dist/src/resources/streaming-endpoints.js +15 -0
- package/dist/src/resources/trained-model.d.ts +60 -0
- package/dist/src/resources/trained-model.js +98 -0
- package/dist/src/resources/webapps.d.ts +8 -0
- package/dist/src/resources/webapps.js +10 -0
- package/dist/src/resources/wiki.d.ts +52 -1
- package/dist/src/resources/wiki.js +96 -0
- package/dist/src/resources/workspaces.d.ts +2 -0
- package/dist/src/resources/workspaces.js +4 -0
- package/dist/src/utils/flow-analysis.js +4 -0
- package/dist/src/utils/response-file.js +20 -9
- package/dist/src/utils/secret-sanitize.d.ts +7 -0
- package/dist/src/utils/secret-sanitize.js +46 -0
- package/package.json +25 -26
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
- package/skills/dataiku-dss/references/authentication.md +3 -1
- package/skills/dataiku-dss/references/coding.md +1 -0
- package/skills/dataiku-dss/references/discovery.md +4 -6
- package/skills/dataiku-dss/references/flow-maps.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +3 -2
|
@@ -1,23 +1,15 @@
|
|
|
1
1
|
import { writeFile, } from "node:fs/promises";
|
|
2
2
|
import { resolve, } from "node:path";
|
|
3
3
|
import { ClientValidationError, DataikuError, nonJsonResponseBody, unexpectedResponseError, } from "../errors.js";
|
|
4
|
-
import { ProjectMetadataSchema, RecipeSummaryArraySchema, } from "../schemas.js";
|
|
4
|
+
import { ProjectMetadataSchema, RecipeSchemaUpdatesSchema, RecipeSummaryArraySchema, } from "../schemas.js";
|
|
5
5
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
6
6
|
import { asRecord, } from "../utils/records.js";
|
|
7
7
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
8
8
|
import { BaseResource, } from "./base.js";
|
|
9
|
+
import { applyGroupingPayload, asString, buildRecipeCreateRequest, recipeInputItems, recipeOutputSchemaIsComputable, schemaUpdateComputableId, schemaUpdateIsPending, } from "./recipe-create.js";
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// Helpers: type narrowing
|
|
11
12
|
// ---------------------------------------------------------------------------
|
|
12
|
-
function asString(value) {
|
|
13
|
-
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
14
|
-
}
|
|
15
|
-
function asStringArray(value) {
|
|
16
|
-
if (!Array.isArray(value))
|
|
17
|
-
return undefined;
|
|
18
|
-
const out = value.filter((v) => typeof v === "string" && v.length > 0);
|
|
19
|
-
return out.length > 0 ? out : undefined;
|
|
20
|
-
}
|
|
21
13
|
function parseRecipePayload(payload) {
|
|
22
14
|
if (!payload)
|
|
23
15
|
return {};
|
|
@@ -30,48 +22,6 @@ function parseRecipePayload(payload) {
|
|
|
30
22
|
}
|
|
31
23
|
return asRecord(parsed) ?? {};
|
|
32
24
|
}
|
|
33
|
-
function parseJoinKeyPairs(values, optionName = "joinOn") {
|
|
34
|
-
const pairs = [];
|
|
35
|
-
for (const raw of values) {
|
|
36
|
-
const token = raw.trim();
|
|
37
|
-
if (!token)
|
|
38
|
-
continue;
|
|
39
|
-
const eq = token.indexOf("=");
|
|
40
|
-
if (eq === -1) {
|
|
41
|
-
pairs.push({ left: token, right: token, });
|
|
42
|
-
continue;
|
|
43
|
-
}
|
|
44
|
-
const left = token.slice(0, eq).trim();
|
|
45
|
-
const right = token.slice(eq + 1).trim();
|
|
46
|
-
if (!left || !right) {
|
|
47
|
-
throw new ClientValidationError(`${optionName} values must use COL or LEFT=RIGHT.`, "validation_failed");
|
|
48
|
-
}
|
|
49
|
-
pairs.push({ left, right, });
|
|
50
|
-
}
|
|
51
|
-
return pairs;
|
|
52
|
-
}
|
|
53
|
-
function normalizeFuzzyDistance(value) {
|
|
54
|
-
const normalized = typeof value === "string" && value.trim().length > 0
|
|
55
|
-
? value.trim().toUpperCase()
|
|
56
|
-
: "DAMERAU_LEVENSHTEIN";
|
|
57
|
-
if (normalized === "DAMERAU_LEVENSHTEIN")
|
|
58
|
-
return "LEVENSHTEIN";
|
|
59
|
-
if (normalized === "HAMMING" || normalized === "JACCARD" || normalized === "COSINE"
|
|
60
|
-
|| normalized === "EUCLIDEAN") {
|
|
61
|
-
return normalized;
|
|
62
|
-
}
|
|
63
|
-
throw new ClientValidationError("fuzzyDistance must be one of DAMERAU_LEVENSHTEIN, HAMMING, JACCARD, COSINE, or EUCLIDEAN.", "validation_failed");
|
|
64
|
-
}
|
|
65
|
-
function normalizeJoinType(value) {
|
|
66
|
-
const normalized = typeof value === "string" && value.trim().length > 0
|
|
67
|
-
? value.trim().toUpperCase()
|
|
68
|
-
: "LEFT";
|
|
69
|
-
if (normalized === "LEFT" || normalized === "INNER" || normalized === "RIGHT"
|
|
70
|
-
|| normalized === "FULL") {
|
|
71
|
-
return normalized;
|
|
72
|
-
}
|
|
73
|
-
throw new ClientValidationError("joinType must be one of LEFT, INNER, RIGHT, or FULL.", "validation_failed");
|
|
74
|
-
}
|
|
75
25
|
function recipeVirtualInputs(payload, inputCount) {
|
|
76
26
|
const current = Array.isArray(payload.virtualInputs) ? payload.virtualInputs : [];
|
|
77
27
|
const virtualInputs = [];
|
|
@@ -136,27 +86,6 @@ function recipeOutputItems(recipe) {
|
|
|
136
86
|
}
|
|
137
87
|
return result;
|
|
138
88
|
}
|
|
139
|
-
function recipeInputItems(recipe) {
|
|
140
|
-
const inputs = asRecord(recipe.inputs);
|
|
141
|
-
if (!inputs)
|
|
142
|
-
return [];
|
|
143
|
-
const result = [];
|
|
144
|
-
const seen = new Set();
|
|
145
|
-
for (const [role, roleValue,] of Object.entries(inputs)) {
|
|
146
|
-
const items = asRecord(roleValue)?.items;
|
|
147
|
-
if (!Array.isArray(items))
|
|
148
|
-
continue;
|
|
149
|
-
for (const itemValue of items) {
|
|
150
|
-
const item = asRecord(itemValue);
|
|
151
|
-
const ref = asString(item?.ref);
|
|
152
|
-
if (!ref || seen.has(ref))
|
|
153
|
-
continue;
|
|
154
|
-
seen.add(ref);
|
|
155
|
-
result.push({ ref, role, });
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
return result;
|
|
159
|
-
}
|
|
160
89
|
function rewriteRefs(value, rewrites) {
|
|
161
90
|
if (Object.keys(rewrites).length === 0)
|
|
162
91
|
return value;
|
|
@@ -295,95 +224,17 @@ function shouldRetryRecipeCreateWithOutputProvisioning(error) {
|
|
|
295
224
|
// ---------------------------------------------------------------------------
|
|
296
225
|
// Resource
|
|
297
226
|
// ---------------------------------------------------------------------------
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
const
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
const requestedOutputDataset = asString(opts.outputDataset);
|
|
310
|
-
let inputs = asRecord(opts.inputs);
|
|
311
|
-
let outputs = asRecord(opts.outputs);
|
|
312
|
-
if (!inputs) {
|
|
313
|
-
inputs = {
|
|
314
|
-
main: {
|
|
315
|
-
items: inputDatasets?.map((ref) => ({ ref, deps: [], })) ?? [],
|
|
316
|
-
},
|
|
317
|
-
};
|
|
318
|
-
}
|
|
319
|
-
// Auto-generate name if not provided
|
|
320
|
-
const outputNameForDefaultRecipe = requestedOutputDataset ?? outputFolder;
|
|
321
|
-
const name = opts.name ?? (type && outputNameForDefaultRecipe
|
|
322
|
-
? `${type}_${outputNameForDefaultRecipe}`
|
|
323
|
-
: undefined);
|
|
324
|
-
const temporaryOutputDataset = outputFolder && !requestedOutputDataset && name
|
|
325
|
-
? `${name}_folder_output_marker`
|
|
326
|
-
: undefined;
|
|
327
|
-
const outputDataset = requestedOutputDataset ?? temporaryOutputDataset;
|
|
328
|
-
if (!outputs && outputDataset) {
|
|
329
|
-
outputs = {
|
|
330
|
-
main: {
|
|
331
|
-
items: [{ ref: outputDataset, appendMode: false, },],
|
|
332
|
-
},
|
|
333
|
-
};
|
|
334
|
-
}
|
|
335
|
-
if (!type || !name || !outputs) {
|
|
336
|
-
throw new ClientValidationError("type and outputDataset/outputFolder or (name + outputs) are required for create.", "validation_failed");
|
|
337
|
-
}
|
|
338
|
-
const joinCols = typeof opts.joinOn === "string" ? [opts.joinOn,] : asStringArray(opts.joinOn);
|
|
339
|
-
const joinKeys = type === "join" && joinCols?.length
|
|
340
|
-
? parseJoinKeyPairs(joinCols)
|
|
341
|
-
: undefined;
|
|
342
|
-
const fuzzyOn = opts.fuzzyOn ?? (type === "fuzzyjoin" ? opts.joinOn : undefined);
|
|
343
|
-
const fuzzyCols = typeof fuzzyOn === "string" ? [fuzzyOn,] : asStringArray(fuzzyOn);
|
|
344
|
-
const fuzzyKeys = type === "fuzzyjoin" && fuzzyCols?.length
|
|
345
|
-
? parseJoinKeyPairs(fuzzyCols, "fuzzyOn")
|
|
346
|
-
: undefined;
|
|
347
|
-
const normalizedJoinType = joinKeys?.length || fuzzyKeys?.length
|
|
348
|
-
? normalizeJoinType(rawJoinType)
|
|
349
|
-
: undefined;
|
|
350
|
-
const fuzzyDistance = fuzzyKeys?.length ? normalizeFuzzyDistance(opts.fuzzyDistance) : undefined;
|
|
351
|
-
const fuzzyThreshold = opts.fuzzyThreshold ?? 1;
|
|
352
|
-
if (!Number.isFinite(fuzzyThreshold)) {
|
|
353
|
-
throw new ClientValidationError("fuzzyThreshold must be a finite number.", "validation_failed");
|
|
354
|
-
}
|
|
355
|
-
const recipePrototype = {
|
|
356
|
-
type,
|
|
357
|
-
name,
|
|
358
|
-
projectKey: pk,
|
|
359
|
-
...(type === "fuzzyjoin" ? {} : { inputs, }),
|
|
360
|
-
outputs,
|
|
361
|
-
};
|
|
362
|
-
const creationSettings = {};
|
|
363
|
-
if (payload !== undefined) {
|
|
364
|
-
creationSettings.script = payload;
|
|
365
|
-
}
|
|
366
|
-
if (type === "fuzzyjoin") {
|
|
367
|
-
creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
|
|
368
|
-
}
|
|
369
|
-
return {
|
|
370
|
-
creationSettings,
|
|
371
|
-
fuzzyDistance,
|
|
372
|
-
fuzzyKeys,
|
|
373
|
-
fuzzyThreshold,
|
|
374
|
-
inputDatasets,
|
|
375
|
-
inputs,
|
|
376
|
-
joinKeys,
|
|
377
|
-
name,
|
|
378
|
-
normalizedJoinType,
|
|
379
|
-
outputFolder,
|
|
380
|
-
outputs,
|
|
381
|
-
payload,
|
|
382
|
-
rawConnection,
|
|
383
|
-
recipePrototype,
|
|
384
|
-
temporaryOutputDataset,
|
|
385
|
-
type,
|
|
386
|
-
};
|
|
227
|
+
// DSS renames some recipe types server-side (e.g. prediction_scoring becomes
|
|
228
|
+
// "score_<inputDataset>") and documents the creation response as the final
|
|
229
|
+
// unique name: Body {"name": "recipe1"} ("Returns the final unique name of the
|
|
230
|
+
// recipe"). The response name is the contract — falling back to the requested
|
|
231
|
+
// name would mask a missing/invalid receipt, so a body without it is an error.
|
|
232
|
+
function requireRecipeName(created) {
|
|
233
|
+
const name = asString(created?.["name"]);
|
|
234
|
+
if (!name) {
|
|
235
|
+
throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
236
|
+
}
|
|
237
|
+
return name;
|
|
387
238
|
}
|
|
388
239
|
export class RecipesResource extends BaseResource {
|
|
389
240
|
/** List all recipes in a project. */
|
|
@@ -636,13 +487,18 @@ export class RecipesResource extends BaseResource {
|
|
|
636
487
|
async create(opts) {
|
|
637
488
|
const pk = this.resolveProjectKey(opts.projectKey);
|
|
638
489
|
const enc = encodeURIComponent(pk);
|
|
639
|
-
const { creationSettings, fuzzyDistance, fuzzyKeys, fuzzyThreshold, inputDatasets, inputs, joinKeys,
|
|
490
|
+
const { creationSettings, fuzzyDistance, fuzzyKeys, fuzzyThreshold, grouping, inputDatasets, inputs, joinKeys, normalizedJoinType, outputFolder, outputs, prepareSteps: prepareStepList, rawConnection, recipePrototype, temporaryOutputDataset, type, } = buildRecipeCreateRequest(opts, pk);
|
|
640
491
|
const createRecipe = () => this.client.post(`/public/api/projects/${enc}/recipes/`, {
|
|
641
492
|
recipePrototype,
|
|
642
493
|
creationSettings,
|
|
643
494
|
});
|
|
644
495
|
const createdDatasets = [];
|
|
645
496
|
let usedOutputProvisioningFallback = false;
|
|
497
|
+
// Missing outputs are created through POST /datasets/managed, like the DSS UI
|
|
498
|
+
// and dataikuapi's new_managed_dataset: DSS picks the dataset type, storage
|
|
499
|
+
// path, and format from the connection (an S3 connection yields an S3
|
|
500
|
+
// dataset under the connection's naming rule), so the client never guesses
|
|
501
|
+
// type-specific params.
|
|
646
502
|
const provisionOutputDatasets = async () => {
|
|
647
503
|
const existingDs = await this.client.get(`/public/api/projects/${enc}/datasets/`);
|
|
648
504
|
let outputConnection = asString(rawConnection);
|
|
@@ -655,126 +511,91 @@ export class RecipesResource extends BaseResource {
|
|
|
655
511
|
if (!outputConnection)
|
|
656
512
|
return;
|
|
657
513
|
const existingNames = new Set([...existingDs.map((d) => d.name), ...createdDatasets,]);
|
|
658
|
-
const connectionSample = existingDs.find((d) => d.params?.connection === outputConnection && d.type);
|
|
659
|
-
const inferredOutputType = connectionSample?.type ?? "Filesystem";
|
|
660
514
|
const outputRoles = outputs;
|
|
661
515
|
for (const role of Object.values(outputRoles)) {
|
|
662
516
|
for (const item of role.items ?? []) {
|
|
663
517
|
if (!item.ref || existingNames.has(item.ref))
|
|
664
518
|
continue;
|
|
665
|
-
|
|
666
|
-
? {
|
|
667
|
-
projectKey: pk,
|
|
668
|
-
name: item.ref,
|
|
669
|
-
type: inferredOutputType,
|
|
670
|
-
params: {
|
|
671
|
-
connection: outputConnection,
|
|
672
|
-
path: `/dataiku/${pk}/${item.ref}`,
|
|
673
|
-
metastoreTableName: item.ref,
|
|
674
|
-
},
|
|
675
|
-
formatType: "csv",
|
|
676
|
-
formatParams: {
|
|
677
|
-
style: "excel",
|
|
678
|
-
charset: "utf8",
|
|
679
|
-
separator: "\t",
|
|
680
|
-
quoteChar: '"',
|
|
681
|
-
escapeChar: "\\",
|
|
682
|
-
dateSerializationFormat: "ISO",
|
|
683
|
-
arrayMapFormat: "json",
|
|
684
|
-
parseHeaderRow: true,
|
|
685
|
-
compress: "gz",
|
|
686
|
-
},
|
|
687
|
-
managed: true,
|
|
688
|
-
}
|
|
689
|
-
: {
|
|
690
|
-
projectKey: pk,
|
|
691
|
-
name: item.ref,
|
|
692
|
-
type: inferredOutputType,
|
|
693
|
-
params: {
|
|
694
|
-
connection: outputConnection,
|
|
695
|
-
mode: "table",
|
|
696
|
-
table: item.ref,
|
|
697
|
-
...(connectionSample?.params?.schema
|
|
698
|
-
? { schema: connectionSample.params.schema, }
|
|
699
|
-
: {}),
|
|
700
|
-
...(connectionSample?.params?.catalog
|
|
701
|
-
? { catalog: connectionSample.params.catalog, }
|
|
702
|
-
: {}),
|
|
703
|
-
},
|
|
704
|
-
managed: connectionSample?.managed ?? false,
|
|
705
|
-
};
|
|
706
|
-
await this.client.post(`/public/api/projects/${enc}/datasets/`, datasetBody);
|
|
519
|
+
await this.client.datasets.createManaged({ name: item.ref, connection: outputConnection, }, pk);
|
|
707
520
|
existingNames.add(item.ref);
|
|
708
|
-
|
|
709
|
-
createdDatasets.push(item.ref);
|
|
521
|
+
createdDatasets.push(item.ref);
|
|
710
522
|
}
|
|
711
523
|
}
|
|
712
524
|
};
|
|
713
525
|
let finalRecipeName;
|
|
714
|
-
//
|
|
715
|
-
//
|
|
716
|
-
//
|
|
717
|
-
//
|
|
718
|
-
//
|
|
719
|
-
//
|
|
720
|
-
|
|
721
|
-
if (
|
|
722
|
-
await provisionOutputDatasets();
|
|
723
|
-
const created = await createRecipe();
|
|
724
|
-
const receivedName = asString(created?.["name"]);
|
|
725
|
-
if (!receivedName) {
|
|
726
|
-
throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
727
|
-
}
|
|
728
|
-
finalRecipeName = receivedName;
|
|
729
|
-
}
|
|
730
|
-
catch (error) {
|
|
731
|
-
if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
|
|
526
|
+
// Datasets this call provisioned must not outlive a failed create: the
|
|
527
|
+
// failure is rethrown unchanged, with the rollback outcome in `details`.
|
|
528
|
+
// A failure can be ambiguous (timeout, 5xx after DSS committed, a body
|
|
529
|
+
// without the name), so a dataset some recipe already writes is kept:
|
|
530
|
+
// deleting it would break that recipe. DSS may rename the recipe, so the
|
|
531
|
+
// check reads the flow graph rather than the requested recipe name.
|
|
532
|
+
const rollbackProvisionedDatasets = async (error) => {
|
|
533
|
+
if (createdDatasets.length === 0)
|
|
732
534
|
throw error;
|
|
535
|
+
const deleted = [];
|
|
536
|
+
const failed = [];
|
|
537
|
+
const kept = [];
|
|
538
|
+
let producers;
|
|
539
|
+
try {
|
|
540
|
+
const { graph, } = await this.client.projects.flowTopology(pk);
|
|
541
|
+
producers = new Map(graph.edges.filter((edge) => edge.relation === "writes").map((edge) => [edge.to, edge.from,]));
|
|
733
542
|
}
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
if (!retryName) {
|
|
739
|
-
throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
|
|
543
|
+
catch (graphError) {
|
|
544
|
+
const reason = `could not read the flow graph to check for a recipe writing it (${graphError instanceof Error ? graphError.message : String(graphError)})`;
|
|
545
|
+
for (const name of createdDatasets)
|
|
546
|
+
kept.push({ name, reason, });
|
|
740
547
|
}
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
if (type.toLowerCase() === "sync" && !outputFolder) {
|
|
749
|
-
const inputRefs = recipeInputItems({ inputs, }).map((item) => item.ref);
|
|
750
|
-
const outputDatasetRefs = recipeOutputItems({ outputs, })
|
|
751
|
-
.filter((item) => item.type !== "MANAGED_FOLDER")
|
|
752
|
-
.map((item) => item.ref);
|
|
753
|
-
if (inputRefs.length === 1 && outputDatasetRefs.length > 0) {
|
|
754
|
-
const inputSchema = await this.client.datasets
|
|
755
|
-
.schema(inputRefs[0], pk)
|
|
756
|
-
.catch(() => undefined);
|
|
757
|
-
const inputColumns = inputSchema?.columns;
|
|
758
|
-
if (inputColumns && inputColumns.length > 0) {
|
|
759
|
-
const propagated = [];
|
|
760
|
-
for (const ref of outputDatasetRefs) {
|
|
761
|
-
const current = await this.client.datasets
|
|
762
|
-
.schema(ref, pk)
|
|
763
|
-
.catch(() => undefined);
|
|
764
|
-
if (current && current.columns.length === 0) {
|
|
765
|
-
await this.client.datasets.updateSchema(ref, inputColumns, pk);
|
|
766
|
-
propagated.push(ref);
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
if (propagated.length > 0)
|
|
770
|
-
syncOutputSchemaPropagated = propagated;
|
|
548
|
+
for (const name of createdDatasets) {
|
|
549
|
+
if (!producers)
|
|
550
|
+
break;
|
|
551
|
+
const producer = producers.get(name);
|
|
552
|
+
if (producer !== undefined) {
|
|
553
|
+
kept.push({ name, reason: `written by recipe ${producer}`, });
|
|
554
|
+
continue;
|
|
771
555
|
}
|
|
556
|
+
try {
|
|
557
|
+
await this.client.del(`/public/api/projects/${enc}/datasets/${encodeURIComponent(name)}`);
|
|
558
|
+
deleted.push(name);
|
|
559
|
+
}
|
|
560
|
+
catch (deleteError) {
|
|
561
|
+
failed.push({
|
|
562
|
+
name,
|
|
563
|
+
error: deleteError instanceof Error ? deleteError.message : String(deleteError),
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
if (error instanceof DataikuError || error instanceof ClientValidationError) {
|
|
568
|
+
error.details = {
|
|
569
|
+
...error.details,
|
|
570
|
+
createdDatasetsDeleted: deleted,
|
|
571
|
+
createdDatasetsNotDeleted: failed,
|
|
572
|
+
...(kept.length > 0 ? { createdDatasetsKept: kept, } : {}),
|
|
573
|
+
};
|
|
574
|
+
}
|
|
575
|
+
throw error;
|
|
576
|
+
};
|
|
577
|
+
try {
|
|
578
|
+
try {
|
|
579
|
+
if (rawConnection)
|
|
580
|
+
await provisionOutputDatasets();
|
|
581
|
+
finalRecipeName = requireRecipeName(await createRecipe());
|
|
582
|
+
}
|
|
583
|
+
catch (error) {
|
|
584
|
+
if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
|
|
585
|
+
throw error;
|
|
586
|
+
}
|
|
587
|
+
usedOutputProvisioningFallback = true;
|
|
588
|
+
await provisionOutputDatasets();
|
|
589
|
+
finalRecipeName = requireRecipeName(await createRecipe());
|
|
772
590
|
}
|
|
773
591
|
}
|
|
592
|
+
catch (error) {
|
|
593
|
+
await rollbackProvisionedDatasets(error);
|
|
594
|
+
}
|
|
774
595
|
// For join recipes: configure join conditions after creation
|
|
775
596
|
let joinConfigured = false;
|
|
776
597
|
if (type === "join" && joinKeys?.length) {
|
|
777
|
-
const rnEnc = encodeURIComponent(
|
|
598
|
+
const rnEnc = encodeURIComponent(finalRecipeName);
|
|
778
599
|
const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
|
|
779
600
|
const joinPayload = parseRecipePayload(full.payload);
|
|
780
601
|
const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
|
|
@@ -807,7 +628,7 @@ export class RecipesResource extends BaseResource {
|
|
|
807
628
|
joinConfigured = true;
|
|
808
629
|
}
|
|
809
630
|
if (type === "fuzzyjoin" && fuzzyKeys?.length) {
|
|
810
|
-
const rnEnc = encodeURIComponent(
|
|
631
|
+
const rnEnc = encodeURIComponent(finalRecipeName);
|
|
811
632
|
const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
|
|
812
633
|
const fuzzyPayload = parseRecipePayload(full.payload);
|
|
813
634
|
const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
|
|
@@ -851,9 +672,27 @@ export class RecipesResource extends BaseResource {
|
|
|
851
672
|
});
|
|
852
673
|
joinConfigured = true;
|
|
853
674
|
}
|
|
675
|
+
// Grouping keys/aggregates and prepare steps edit the payload DSS created.
|
|
676
|
+
let payloadConfigured = false;
|
|
677
|
+
if (grouping || prepareStepList.length > 0) {
|
|
678
|
+
const rnEnc = encodeURIComponent(finalRecipeName);
|
|
679
|
+
const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
|
|
680
|
+
const created = parseRecipePayload(full.payload);
|
|
681
|
+
const configured = grouping
|
|
682
|
+
? applyGroupingPayload(created, grouping)
|
|
683
|
+
: {
|
|
684
|
+
...created,
|
|
685
|
+
steps: [...(Array.isArray(created.steps) ? created.steps : []), ...prepareStepList,],
|
|
686
|
+
};
|
|
687
|
+
await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, {
|
|
688
|
+
...full,
|
|
689
|
+
payload: JSON.stringify(configured),
|
|
690
|
+
});
|
|
691
|
+
payloadConfigured = true;
|
|
692
|
+
}
|
|
854
693
|
let temporaryOutputDatasetDeleted;
|
|
855
694
|
if (outputFolder) {
|
|
856
|
-
await this.update(
|
|
695
|
+
await this.update(finalRecipeName, {
|
|
857
696
|
recipe: {
|
|
858
697
|
outputs: {
|
|
859
698
|
main: {
|
|
@@ -875,18 +714,113 @@ export class RecipesResource extends BaseResource {
|
|
|
875
714
|
}
|
|
876
715
|
}
|
|
877
716
|
}
|
|
717
|
+
// DSS computes visual-recipe output schemas only when asked (the UI asks on
|
|
718
|
+
// save; dataikuapi exposes compute_schema_updates). Without this, a join or
|
|
719
|
+
// grouping output keeps an empty schema and its build "succeeds" with no
|
|
720
|
+
// columns. Only outputs this call created, or whose schema is still empty,
|
|
721
|
+
// are written: an existing schema is never replaced implicitly.
|
|
722
|
+
let outputSchemaUpdated;
|
|
723
|
+
let outputSchemaUpdateError;
|
|
724
|
+
if (!outputFolder && recipeOutputSchemaIsComputable(type)) {
|
|
725
|
+
try {
|
|
726
|
+
const created = new Set(createdDatasets);
|
|
727
|
+
const { updated, } = await this.applySchemaUpdates(finalRecipeName, await this.computeSchemaUpdates(finalRecipeName, pk), pk, (computable) => computable.previousSchemaWasEmpty === true
|
|
728
|
+
|| created.has(schemaUpdateComputableId(computable)));
|
|
729
|
+
if (updated.length > 0)
|
|
730
|
+
outputSchemaUpdated = updated;
|
|
731
|
+
}
|
|
732
|
+
catch (error) {
|
|
733
|
+
if (!(error instanceof DataikuError))
|
|
734
|
+
throw error;
|
|
735
|
+
outputSchemaUpdateError = error.message.split("\n")[0];
|
|
736
|
+
}
|
|
737
|
+
}
|
|
878
738
|
return {
|
|
739
|
+
...(opts.name && opts.name !== finalRecipeName ? { requestedName: opts.name, } : {}),
|
|
879
740
|
recipeName: finalRecipeName,
|
|
880
741
|
type,
|
|
881
742
|
createdDatasets,
|
|
882
743
|
joinConfigured,
|
|
744
|
+
...(payloadConfigured ? { payloadConfigured, } : {}),
|
|
883
745
|
outputProvisioningFallbackUsed: usedOutputProvisioningFallback,
|
|
884
|
-
...(
|
|
746
|
+
...(outputSchemaUpdated ? { outputSchemaUpdated, } : {}),
|
|
747
|
+
...(outputSchemaUpdateError ? { outputSchemaUpdateError, } : {}),
|
|
885
748
|
...(outputFolder ? { outputFolder, } : {}),
|
|
886
749
|
...(temporaryOutputDataset ? { temporaryOutputDataset, } : {}),
|
|
887
750
|
...(temporaryOutputDatasetDeleted !== undefined ? { temporaryOutputDatasetDeleted, } : {}),
|
|
888
751
|
};
|
|
889
752
|
}
|
|
753
|
+
/**
|
|
754
|
+
* Compute the output schemas DSS derives from the recipe's current settings
|
|
755
|
+
* (GET /recipes/{name}/schema-update, dataikuapi `compute_schema_updates`).
|
|
756
|
+
* DSS rejects code recipes, whose code sets the schema at run time.
|
|
757
|
+
*/
|
|
758
|
+
async computeSchemaUpdates(recipeName, projectKey) {
|
|
759
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/recipes/${encodeURIComponent(recipeName)}/schema-update`);
|
|
760
|
+
return this.client.safeParse(RecipeSchemaUpdatesSchema, raw, "recipes.computeSchemaUpdates");
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* Write each pending computed schema accepted by `select` through
|
|
764
|
+
* actions/updateOutputSchema. Pending outputs `select` rejects are reported,
|
|
765
|
+
* not written.
|
|
766
|
+
*/
|
|
767
|
+
async applySchemaUpdates(recipeName, updates, projectKey, select) {
|
|
768
|
+
const endpoint = `/public/api/projects/${this.enc(projectKey)}/recipes/${encodeURIComponent(recipeName)}/actions/updateOutputSchema`;
|
|
769
|
+
const updated = [];
|
|
770
|
+
const pending = [];
|
|
771
|
+
const unchanged = [];
|
|
772
|
+
const datasetsNeedingAction = [];
|
|
773
|
+
for (const computable of updates.computables) {
|
|
774
|
+
const computableId = schemaUpdateComputableId(computable);
|
|
775
|
+
if (!schemaUpdateIsPending(computable)) {
|
|
776
|
+
unchanged.push(computableId);
|
|
777
|
+
continue;
|
|
778
|
+
}
|
|
779
|
+
if (!select(computable)) {
|
|
780
|
+
pending.push(computableId);
|
|
781
|
+
continue;
|
|
782
|
+
}
|
|
783
|
+
const applied = await this.client.post(endpoint, { computableType: computable.type, computableId, newSchema: computable.newSchema, });
|
|
784
|
+
updated.push(computableId);
|
|
785
|
+
if (applied?.datasetsNeedingAction?.length) {
|
|
786
|
+
datasetsNeedingAction.push(...applied.datasetsNeedingAction);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
return { updated, pending, unchanged, datasetsNeedingAction, };
|
|
790
|
+
}
|
|
791
|
+
/**
|
|
792
|
+
* Replace each output schema that differs from the DSS-computed one
|
|
793
|
+
* (dataikuapi `compute_schema_updates().apply()`, the UI's "Update schema").
|
|
794
|
+
* With `derivedOutputs`, only outputs whose schema is empty or listed there
|
|
795
|
+
* are written: callers pass the outputs whose schema matched the computed
|
|
796
|
+
* one before an edit, so DSS-derived schemas follow the edit while
|
|
797
|
+
* hand-edited ones are left alone. `dryRun` writes nothing. Skipped
|
|
798
|
+
* differing outputs are listed under `pending`. Code recipes are rejected
|
|
799
|
+
* before the schema computation: DSS cannot compute their output schema.
|
|
800
|
+
*/
|
|
801
|
+
async updateSchema(recipeName, opts) {
|
|
802
|
+
const { recipe, } = await this.get(recipeName, { projectKey: opts?.projectKey, });
|
|
803
|
+
const recipeType = typeof recipe.type === "string" ? recipe.type : "";
|
|
804
|
+
if (!recipeOutputSchemaIsComputable(recipeType)) {
|
|
805
|
+
throw new ClientValidationError(`DSS cannot compute output schemas for ${recipeType} recipes: the code sets the schema when it runs. Build the recipe, or set columns with dss dataset refresh-schema.`, "validation_failed");
|
|
806
|
+
}
|
|
807
|
+
const updates = await this.computeSchemaUpdates(recipeName, opts?.projectKey);
|
|
808
|
+
const derived = opts?.derivedOutputs ? new Set(opts.derivedOutputs) : undefined;
|
|
809
|
+
const applied = await this.applySchemaUpdates(recipeName, updates, opts?.projectKey, (computable) => opts?.dryRun !== true
|
|
810
|
+
&& (derived === undefined || computable.previousSchemaWasEmpty === true
|
|
811
|
+
|| derived.has(schemaUpdateComputableId(computable))));
|
|
812
|
+
return {
|
|
813
|
+
recipeName,
|
|
814
|
+
updated: applied.updated,
|
|
815
|
+
pending: applied.pending,
|
|
816
|
+
unchanged: applied.unchanged,
|
|
817
|
+
totalIncompatibilities: updates.totalIncompatibilities ?? 0,
|
|
818
|
+
computables: updates.computables,
|
|
819
|
+
...(applied.datasetsNeedingAction.length > 0
|
|
820
|
+
? { datasetsNeedingAction: applied.datasetsNeedingAction, }
|
|
821
|
+
: {}),
|
|
822
|
+
};
|
|
823
|
+
}
|
|
890
824
|
/**
|
|
891
825
|
* Update a recipe by merging the patch into the current definition.
|
|
892
826
|
* The `recipe` sub-object is deep-merged to preserve nested fields.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { BaseResource } from "./base.js";
|
|
2
|
+
import { type ScoringJarOptions, TrainedModel } from "./trained-model.js";
|
|
2
3
|
export interface SavedModelListItem extends Record<string, unknown> {
|
|
3
4
|
id?: string;
|
|
4
5
|
name?: string;
|
|
@@ -64,11 +65,6 @@ export interface ExternalModelVersionEvaluateOptions extends Record<string, unkn
|
|
|
64
65
|
useOptimalThreshold?: boolean;
|
|
65
66
|
skipExpensiveReports?: boolean;
|
|
66
67
|
}
|
|
67
|
-
/** Query options for the scoring JAR export (GET .../scoring-jar). */
|
|
68
|
-
export interface SavedModelScoringJarOptions extends Record<string, unknown> {
|
|
69
|
-
fullClassName?: string;
|
|
70
|
-
includeLibs?: boolean;
|
|
71
|
-
}
|
|
72
68
|
export declare class SavedModelsResource extends BaseResource {
|
|
73
69
|
/** List saved models in a project. */
|
|
74
70
|
list(projectKey?: string): Promise<SavedModelListItem[]>;
|
|
@@ -159,7 +155,7 @@ export declare class SavedModelsResource extends BaseResource {
|
|
|
159
155
|
* responsible for consuming the response body (e.g. writeResponseToFile);
|
|
160
156
|
* the stream is read under the client body deadline.
|
|
161
157
|
*/
|
|
162
|
-
downloadScoringJar(options:
|
|
158
|
+
downloadScoringJar(options: ScoringJarOptions | undefined, savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
|
|
163
159
|
/** GET the scoring PMML of a version as a binary/XML stream. */
|
|
164
160
|
downloadScoringPmml(savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
|
|
165
161
|
/**
|
|
@@ -167,6 +163,14 @@ export declare class SavedModelsResource extends BaseResource {
|
|
|
167
163
|
* "userMeta" field of a previously-retrieved version-details object.
|
|
168
164
|
*/
|
|
169
165
|
setUserMeta(savedModelId: string, versionId: string, userMeta: Record<string, unknown>, projectKey?: string): Promise<void>;
|
|
166
|
+
/**
|
|
167
|
+
* One saved-model version as a {@link TrainedModel}: diagnostics
|
|
168
|
+
* (subpopulation analyses, partial dependencies, timeseries residuals and
|
|
169
|
+
* per-series results), documentation, scoring exports, user metadata.
|
|
170
|
+
*/
|
|
171
|
+
version(savedModelId: string, versionId: string, projectKey?: string): TrainedModel;
|
|
172
|
+
/** GET a generated saved-model version documentation (docx) by the `exportId` of its finished future. */
|
|
173
|
+
downloadDocumentation(exportId: string, projectKey?: string): Promise<Response>;
|
|
170
174
|
/** Delete a saved model and all of its versions. */
|
|
171
175
|
delete(savedModelId: string, projectKey?: string): Promise<void>;
|
|
172
176
|
private mlflowImportQuery;
|