dataiku-sdk 3.6.1 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +14 -6
  2. package/bin/bun-version.js +14 -0
  3. package/bin/dss.js +11 -15
  4. package/dist/build-metadata.json +1 -1
  5. package/dist/packages/types/src/index.d.ts +114 -5
  6. package/dist/packages/types/src/index.js +75 -2
  7. package/dist/src/cli/coerce.d.ts +2 -0
  8. package/dist/src/cli/coerce.js +20 -16
  9. package/dist/src/cli/command-syntax.json +3369 -144
  10. package/dist/src/cli/commands/analysis.js +19 -1
  11. package/dist/src/cli/commands/bundle.js +22 -0
  12. package/dist/src/cli/commands/connection.js +26 -2
  13. package/dist/src/cli/commands/data-quality.js +33 -1
  14. package/dist/src/cli/commands/dataset.js +76 -9
  15. package/dist/src/cli/commands/discussion.js +15 -0
  16. package/dist/src/cli/commands/doctor.js +1 -1
  17. package/dist/src/cli/commands/flow-zone.js +241 -46
  18. package/dist/src/cli/commands/flow.d.ts +5 -0
  19. package/dist/src/cli/commands/flow.js +57 -0
  20. package/dist/src/cli/commands/future.js +8 -0
  21. package/dist/src/cli/commands/index.js +6 -0
  22. package/dist/src/cli/commands/job.js +44 -6
  23. package/dist/src/cli/commands/metrics.js +21 -1
  24. package/dist/src/cli/commands/ml-task.d.ts +5 -0
  25. package/dist/src/cli/commands/ml-task.js +86 -16
  26. package/dist/src/cli/commands/mlflow.d.ts +11 -0
  27. package/dist/src/cli/commands/mlflow.js +121 -0
  28. package/dist/src/cli/commands/plugin.js +5 -11
  29. package/dist/src/cli/commands/project-deployer.js +70 -0
  30. package/dist/src/cli/commands/project-library.js +2 -5
  31. package/dist/src/cli/commands/project.js +8 -0
  32. package/dist/src/cli/commands/recipe.d.ts +13 -0
  33. package/dist/src/cli/commands/recipe.js +130 -7
  34. package/dist/src/cli/commands/saved-model.d.ts +1 -1
  35. package/dist/src/cli/commands/saved-model.js +9 -76
  36. package/dist/src/cli/commands/streaming-endpoint.js +30 -1
  37. package/dist/src/cli/commands/webapp.js +8 -0
  38. package/dist/src/cli/commands/wiki.js +78 -0
  39. package/dist/src/cli/commands/workspace.js +8 -0
  40. package/dist/src/cli/contract.d.ts +16 -15
  41. package/dist/src/cli/contract.js +150 -3407
  42. package/dist/src/cli/doctor.d.ts +13 -0
  43. package/dist/src/cli/doctor.js +43 -1
  44. package/dist/src/cli/flags.d.ts +14 -1
  45. package/dist/src/cli/flags.js +106 -8
  46. package/dist/src/cli/helpers/app-successor.js +1 -2
  47. package/dist/src/cli/helpers/dataset.d.ts +12 -0
  48. package/dist/src/cli/helpers/dataset.js +36 -0
  49. package/dist/src/cli/helpers/documentation.d.ts +21 -0
  50. package/dist/src/cli/helpers/documentation.js +53 -0
  51. package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
  52. package/dist/src/cli/helpers/flow-zone.js +144 -4
  53. package/dist/src/cli/helpers/trained-model.d.ts +26 -0
  54. package/dist/src/cli/helpers/trained-model.js +144 -0
  55. package/dist/src/cli/list-output.d.ts +7 -0
  56. package/dist/src/cli/list-output.js +77 -0
  57. package/dist/src/cli/list-shapes.d.ts +21 -0
  58. package/dist/src/cli/list-shapes.js +111 -0
  59. package/dist/src/cli/output-schemas.d.ts +0 -14
  60. package/dist/src/cli/output-schemas.js +4 -7
  61. package/dist/src/cli/output.d.ts +0 -1
  62. package/dist/src/cli/output.js +33 -19
  63. package/dist/src/cli/plans.d.ts +14 -0
  64. package/dist/src/cli/plans.js +3740 -0
  65. package/dist/src/cli/types.d.ts +2 -0
  66. package/dist/src/cli/usage.d.ts +7 -0
  67. package/dist/src/cli/usage.js +20 -1
  68. package/dist/src/cli/version.d.ts +2 -2
  69. package/dist/src/cli/version.js +2 -2
  70. package/dist/src/cli.js +137 -115
  71. package/dist/src/client.d.ts +52 -43
  72. package/dist/src/client.js +267 -96
  73. package/dist/src/errors.d.ts +22 -2
  74. package/dist/src/errors.js +49 -50
  75. package/dist/src/generated/action-output-schemas.json +345 -89
  76. package/dist/src/index.d.ts +5 -1
  77. package/dist/src/index.js +3 -0
  78. package/dist/src/resources/analyses.d.ts +8 -0
  79. package/dist/src/resources/analyses.js +13 -1
  80. package/dist/src/resources/bundles.d.ts +34 -0
  81. package/dist/src/resources/bundles.js +67 -0
  82. package/dist/src/resources/connections.d.ts +9 -0
  83. package/dist/src/resources/connections.js +15 -0
  84. package/dist/src/resources/data-quality.d.ts +20 -0
  85. package/dist/src/resources/data-quality.js +30 -0
  86. package/dist/src/resources/dataset-create.d.ts +38 -0
  87. package/dist/src/resources/dataset-create.js +135 -0
  88. package/dist/src/resources/datasets.d.ts +46 -15
  89. package/dist/src/resources/datasets.js +120 -112
  90. package/dist/src/resources/discussions.d.ts +5 -0
  91. package/dist/src/resources/discussions.js +7 -0
  92. package/dist/src/resources/documentation.d.ts +25 -0
  93. package/dist/src/resources/documentation.js +16 -0
  94. package/dist/src/resources/flow-zones.d.ts +29 -3
  95. package/dist/src/resources/flow-zones.js +53 -3
  96. package/dist/src/resources/flow.d.ts +48 -0
  97. package/dist/src/resources/flow.js +65 -0
  98. package/dist/src/resources/futures.d.ts +23 -0
  99. package/dist/src/resources/futures.js +32 -0
  100. package/dist/src/resources/jobs.d.ts +34 -0
  101. package/dist/src/resources/jobs.js +170 -3
  102. package/dist/src/resources/metrics.d.ts +10 -0
  103. package/dist/src/resources/metrics.js +10 -0
  104. package/dist/src/resources/ml-tasks.d.ts +65 -2
  105. package/dist/src/resources/ml-tasks.js +102 -28
  106. package/dist/src/resources/mlflow-extension.d.ts +73 -0
  107. package/dist/src/resources/mlflow-extension.js +105 -0
  108. package/dist/src/resources/projects.d.ts +15 -0
  109. package/dist/src/resources/projects.js +18 -3
  110. package/dist/src/resources/recipe-create.d.ts +79 -0
  111. package/dist/src/resources/recipe-create.js +454 -0
  112. package/dist/src/resources/recipes.d.ts +28 -29
  113. package/dist/src/resources/recipes.js +201 -267
  114. package/dist/src/resources/saved-models.d.ts +10 -6
  115. package/dist/src/resources/saved-models.js +16 -14
  116. package/dist/src/resources/scenarios.js +8 -14
  117. package/dist/src/resources/sql.js +52 -13
  118. package/dist/src/resources/streaming-endpoints.d.ts +12 -0
  119. package/dist/src/resources/streaming-endpoints.js +15 -0
  120. package/dist/src/resources/trained-model.d.ts +60 -0
  121. package/dist/src/resources/trained-model.js +98 -0
  122. package/dist/src/resources/webapps.d.ts +8 -0
  123. package/dist/src/resources/webapps.js +10 -0
  124. package/dist/src/resources/wiki.d.ts +52 -1
  125. package/dist/src/resources/wiki.js +96 -0
  126. package/dist/src/resources/workspaces.d.ts +2 -0
  127. package/dist/src/resources/workspaces.js +4 -0
  128. package/dist/src/utils/flow-analysis.js +4 -0
  129. package/dist/src/utils/response-file.js +20 -9
  130. package/dist/src/utils/secret-sanitize.d.ts +7 -0
  131. package/dist/src/utils/secret-sanitize.js +46 -0
  132. package/package.json +25 -26
  133. package/plugin.json +1 -1
  134. package/skills/dataiku-dss/SKILL.md +1 -1
  135. package/skills/dataiku-dss/references/authentication.md +3 -1
  136. package/skills/dataiku-dss/references/coding.md +1 -0
  137. package/skills/dataiku-dss/references/discovery.md +4 -6
  138. package/skills/dataiku-dss/references/flow-maps.md +2 -2
  139. package/skills/dataiku-dss/references/troubleshooting.md +3 -2
@@ -1,23 +1,15 @@
1
1
  import { writeFile, } from "node:fs/promises";
2
2
  import { resolve, } from "node:path";
3
3
  import { ClientValidationError, DataikuError, nonJsonResponseBody, unexpectedResponseError, } from "../errors.js";
4
- import { ProjectMetadataSchema, RecipeSummaryArraySchema, } from "../schemas.js";
4
+ import { ProjectMetadataSchema, RecipeSchemaUpdatesSchema, RecipeSummaryArraySchema, } from "../schemas.js";
5
5
  import { deepMerge, } from "../utils/deep-merge.js";
6
6
  import { asRecord, } from "../utils/records.js";
7
7
  import { sanitizeFileName, } from "../utils/sanitize.js";
8
8
  import { BaseResource, } from "./base.js";
9
+ import { applyGroupingPayload, asString, buildRecipeCreateRequest, recipeInputItems, recipeOutputSchemaIsComputable, schemaUpdateComputableId, schemaUpdateIsPending, } from "./recipe-create.js";
9
10
  // ---------------------------------------------------------------------------
10
11
  // Helpers: type narrowing
11
12
  // ---------------------------------------------------------------------------
12
- function asString(value) {
13
- return typeof value === "string" && value.length > 0 ? value : undefined;
14
- }
15
- function asStringArray(value) {
16
- if (!Array.isArray(value))
17
- return undefined;
18
- const out = value.filter((v) => typeof v === "string" && v.length > 0);
19
- return out.length > 0 ? out : undefined;
20
- }
21
13
  function parseRecipePayload(payload) {
22
14
  if (!payload)
23
15
  return {};
@@ -30,48 +22,6 @@ function parseRecipePayload(payload) {
30
22
  }
31
23
  return asRecord(parsed) ?? {};
32
24
  }
33
- function parseJoinKeyPairs(values, optionName = "joinOn") {
34
- const pairs = [];
35
- for (const raw of values) {
36
- const token = raw.trim();
37
- if (!token)
38
- continue;
39
- const eq = token.indexOf("=");
40
- if (eq === -1) {
41
- pairs.push({ left: token, right: token, });
42
- continue;
43
- }
44
- const left = token.slice(0, eq).trim();
45
- const right = token.slice(eq + 1).trim();
46
- if (!left || !right) {
47
- throw new ClientValidationError(`${optionName} values must use COL or LEFT=RIGHT.`, "validation_failed");
48
- }
49
- pairs.push({ left, right, });
50
- }
51
- return pairs;
52
- }
53
- function normalizeFuzzyDistance(value) {
54
- const normalized = typeof value === "string" && value.trim().length > 0
55
- ? value.trim().toUpperCase()
56
- : "DAMERAU_LEVENSHTEIN";
57
- if (normalized === "DAMERAU_LEVENSHTEIN")
58
- return "LEVENSHTEIN";
59
- if (normalized === "HAMMING" || normalized === "JACCARD" || normalized === "COSINE"
60
- || normalized === "EUCLIDEAN") {
61
- return normalized;
62
- }
63
- throw new ClientValidationError("fuzzyDistance must be one of DAMERAU_LEVENSHTEIN, HAMMING, JACCARD, COSINE, or EUCLIDEAN.", "validation_failed");
64
- }
65
- function normalizeJoinType(value) {
66
- const normalized = typeof value === "string" && value.trim().length > 0
67
- ? value.trim().toUpperCase()
68
- : "LEFT";
69
- if (normalized === "LEFT" || normalized === "INNER" || normalized === "RIGHT"
70
- || normalized === "FULL") {
71
- return normalized;
72
- }
73
- throw new ClientValidationError("joinType must be one of LEFT, INNER, RIGHT, or FULL.", "validation_failed");
74
- }
75
25
  function recipeVirtualInputs(payload, inputCount) {
76
26
  const current = Array.isArray(payload.virtualInputs) ? payload.virtualInputs : [];
77
27
  const virtualInputs = [];
@@ -136,27 +86,6 @@ function recipeOutputItems(recipe) {
136
86
  }
137
87
  return result;
138
88
  }
139
- function recipeInputItems(recipe) {
140
- const inputs = asRecord(recipe.inputs);
141
- if (!inputs)
142
- return [];
143
- const result = [];
144
- const seen = new Set();
145
- for (const [role, roleValue,] of Object.entries(inputs)) {
146
- const items = asRecord(roleValue)?.items;
147
- if (!Array.isArray(items))
148
- continue;
149
- for (const itemValue of items) {
150
- const item = asRecord(itemValue);
151
- const ref = asString(item?.ref);
152
- if (!ref || seen.has(ref))
153
- continue;
154
- seen.add(ref);
155
- result.push({ ref, role, });
156
- }
157
- }
158
- return result;
159
- }
160
89
  function rewriteRefs(value, rewrites) {
161
90
  if (Object.keys(rewrites).length === 0)
162
91
  return value;
@@ -295,95 +224,17 @@ function shouldRetryRecipeCreateWithOutputProvisioning(error) {
295
224
  // ---------------------------------------------------------------------------
296
225
  // Resource
297
226
  // ---------------------------------------------------------------------------
298
- /**
299
- * Pure request construction for POST /recipes/ (no DSS calls). Shared with
300
- * `recipe create --plan` so the plan equals the request the command sends.
301
- */
302
- export function buildRecipeCreateRequest(opts, pk) {
303
- const { type, payload, outputConnection: rawConnection, joinType: rawJoinType, } = opts;
304
- const outputFolder = asString(opts.outputFolder);
305
- // Build inputs/outputs from simple form (inputDatasets + outputDataset) or
306
- // advanced form (inputs + outputs); both may coexist — simple form wins when
307
- // the advanced form is absent.
308
- const inputDatasets = asStringArray(opts.inputDatasets);
309
- const requestedOutputDataset = asString(opts.outputDataset);
310
- let inputs = asRecord(opts.inputs);
311
- let outputs = asRecord(opts.outputs);
312
- if (!inputs) {
313
- inputs = {
314
- main: {
315
- items: inputDatasets?.map((ref) => ({ ref, deps: [], })) ?? [],
316
- },
317
- };
318
- }
319
- // Auto-generate name if not provided
320
- const outputNameForDefaultRecipe = requestedOutputDataset ?? outputFolder;
321
- const name = opts.name ?? (type && outputNameForDefaultRecipe
322
- ? `${type}_${outputNameForDefaultRecipe}`
323
- : undefined);
324
- const temporaryOutputDataset = outputFolder && !requestedOutputDataset && name
325
- ? `${name}_folder_output_marker`
326
- : undefined;
327
- const outputDataset = requestedOutputDataset ?? temporaryOutputDataset;
328
- if (!outputs && outputDataset) {
329
- outputs = {
330
- main: {
331
- items: [{ ref: outputDataset, appendMode: false, },],
332
- },
333
- };
334
- }
335
- if (!type || !name || !outputs) {
336
- throw new ClientValidationError("type and outputDataset/outputFolder or (name + outputs) are required for create.", "validation_failed");
337
- }
338
- const joinCols = typeof opts.joinOn === "string" ? [opts.joinOn,] : asStringArray(opts.joinOn);
339
- const joinKeys = type === "join" && joinCols?.length
340
- ? parseJoinKeyPairs(joinCols)
341
- : undefined;
342
- const fuzzyOn = opts.fuzzyOn ?? (type === "fuzzyjoin" ? opts.joinOn : undefined);
343
- const fuzzyCols = typeof fuzzyOn === "string" ? [fuzzyOn,] : asStringArray(fuzzyOn);
344
- const fuzzyKeys = type === "fuzzyjoin" && fuzzyCols?.length
345
- ? parseJoinKeyPairs(fuzzyCols, "fuzzyOn")
346
- : undefined;
347
- const normalizedJoinType = joinKeys?.length || fuzzyKeys?.length
348
- ? normalizeJoinType(rawJoinType)
349
- : undefined;
350
- const fuzzyDistance = fuzzyKeys?.length ? normalizeFuzzyDistance(opts.fuzzyDistance) : undefined;
351
- const fuzzyThreshold = opts.fuzzyThreshold ?? 1;
352
- if (!Number.isFinite(fuzzyThreshold)) {
353
- throw new ClientValidationError("fuzzyThreshold must be a finite number.", "validation_failed");
354
- }
355
- const recipePrototype = {
356
- type,
357
- name,
358
- projectKey: pk,
359
- ...(type === "fuzzyjoin" ? {} : { inputs, }),
360
- outputs,
361
- };
362
- const creationSettings = {};
363
- if (payload !== undefined) {
364
- creationSettings.script = payload;
365
- }
366
- if (type === "fuzzyjoin") {
367
- creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
368
- }
369
- return {
370
- creationSettings,
371
- fuzzyDistance,
372
- fuzzyKeys,
373
- fuzzyThreshold,
374
- inputDatasets,
375
- inputs,
376
- joinKeys,
377
- name,
378
- normalizedJoinType,
379
- outputFolder,
380
- outputs,
381
- payload,
382
- rawConnection,
383
- recipePrototype,
384
- temporaryOutputDataset,
385
- type,
386
- };
227
+ // DSS renames some recipe types server-side (e.g. prediction_scoring becomes
228
+ // "score_<inputDataset>") and documents the creation response as the final
229
+ // unique name: Body {"name": "recipe1"} ("Returns the final unique name of the
230
+ // recipe"). The response name is the contract — falling back to the requested
231
+ // name would mask a missing/invalid receipt, so a body without it is an error.
232
+ function requireRecipeName(created) {
233
+ const name = asString(created?.["name"]);
234
+ if (!name) {
235
+ throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
236
+ }
237
+ return name;
387
238
  }
388
239
  export class RecipesResource extends BaseResource {
389
240
  /** List all recipes in a project. */
@@ -636,13 +487,18 @@ export class RecipesResource extends BaseResource {
636
487
  async create(opts) {
637
488
  const pk = this.resolveProjectKey(opts.projectKey);
638
489
  const enc = encodeURIComponent(pk);
639
- const { creationSettings, fuzzyDistance, fuzzyKeys, fuzzyThreshold, inputDatasets, inputs, joinKeys, name, normalizedJoinType, outputFolder, outputs, rawConnection, recipePrototype, temporaryOutputDataset, type, } = buildRecipeCreateRequest(opts, pk);
490
+ const { creationSettings, fuzzyDistance, fuzzyKeys, fuzzyThreshold, grouping, inputDatasets, inputs, joinKeys, normalizedJoinType, outputFolder, outputs, prepareSteps: prepareStepList, rawConnection, recipePrototype, temporaryOutputDataset, type, } = buildRecipeCreateRequest(opts, pk);
640
491
  const createRecipe = () => this.client.post(`/public/api/projects/${enc}/recipes/`, {
641
492
  recipePrototype,
642
493
  creationSettings,
643
494
  });
644
495
  const createdDatasets = [];
645
496
  let usedOutputProvisioningFallback = false;
497
+ // Missing outputs are created through POST /datasets/managed, like the DSS UI
498
+ // and dataikuapi's new_managed_dataset: DSS picks the dataset type, storage
499
+ // path, and format from the connection (an S3 connection yields an S3
500
+ // dataset under the connection's naming rule), so the client never guesses
501
+ // type-specific params.
646
502
  const provisionOutputDatasets = async () => {
647
503
  const existingDs = await this.client.get(`/public/api/projects/${enc}/datasets/`);
648
504
  let outputConnection = asString(rawConnection);
@@ -655,126 +511,91 @@ export class RecipesResource extends BaseResource {
655
511
  if (!outputConnection)
656
512
  return;
657
513
  const existingNames = new Set([...existingDs.map((d) => d.name), ...createdDatasets,]);
658
- const connectionSample = existingDs.find((d) => d.params?.connection === outputConnection && d.type);
659
- const inferredOutputType = connectionSample?.type ?? "Filesystem";
660
514
  const outputRoles = outputs;
661
515
  for (const role of Object.values(outputRoles)) {
662
516
  for (const item of role.items ?? []) {
663
517
  if (!item.ref || existingNames.has(item.ref))
664
518
  continue;
665
- const datasetBody = inferredOutputType === "Filesystem"
666
- ? {
667
- projectKey: pk,
668
- name: item.ref,
669
- type: inferredOutputType,
670
- params: {
671
- connection: outputConnection,
672
- path: `/dataiku/${pk}/${item.ref}`,
673
- metastoreTableName: item.ref,
674
- },
675
- formatType: "csv",
676
- formatParams: {
677
- style: "excel",
678
- charset: "utf8",
679
- separator: "\t",
680
- quoteChar: '"',
681
- escapeChar: "\\",
682
- dateSerializationFormat: "ISO",
683
- arrayMapFormat: "json",
684
- parseHeaderRow: true,
685
- compress: "gz",
686
- },
687
- managed: true,
688
- }
689
- : {
690
- projectKey: pk,
691
- name: item.ref,
692
- type: inferredOutputType,
693
- params: {
694
- connection: outputConnection,
695
- mode: "table",
696
- table: item.ref,
697
- ...(connectionSample?.params?.schema
698
- ? { schema: connectionSample.params.schema, }
699
- : {}),
700
- ...(connectionSample?.params?.catalog
701
- ? { catalog: connectionSample.params.catalog, }
702
- : {}),
703
- },
704
- managed: connectionSample?.managed ?? false,
705
- };
706
- await this.client.post(`/public/api/projects/${enc}/datasets/`, datasetBody);
519
+ await this.client.datasets.createManaged({ name: item.ref, connection: outputConnection, }, pk);
707
520
  existingNames.add(item.ref);
708
- if (!createdDatasets.includes(item.ref))
709
- createdDatasets.push(item.ref);
521
+ createdDatasets.push(item.ref);
710
522
  }
711
523
  }
712
524
  };
713
525
  let finalRecipeName;
714
- // DSS renames some recipe types server-side (e.g. prediction_scoring
715
- // becomes "score_<inputDataset>") and documents the creation response as
716
- // the final unique name: Body {"name": "recipe1"} ("Returns the final
717
- // unique name of the recipe"). The response name is the contract —
718
- // falling back to the requested name would mask a missing/invalid
719
- // receipt, so a body without it is an error.
720
- try {
721
- if (rawConnection)
722
- await provisionOutputDatasets();
723
- const created = await createRecipe();
724
- const receivedName = asString(created?.["name"]);
725
- if (!receivedName) {
726
- throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
727
- }
728
- finalRecipeName = receivedName;
729
- }
730
- catch (error) {
731
- if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
526
+ // Datasets this call provisioned must not outlive a failed create: the
527
+ // failure is rethrown unchanged, with the rollback outcome in `details`.
528
+ // A failure can be ambiguous (timeout, 5xx after DSS committed, a body
529
+ // without the name), so a dataset some recipe already writes is kept:
530
+ // deleting it would break that recipe. DSS may rename the recipe, so the
531
+ // check reads the flow graph rather than the requested recipe name.
532
+ const rollbackProvisionedDatasets = async (error) => {
533
+ if (createdDatasets.length === 0)
732
534
  throw error;
535
+ const deleted = [];
536
+ const failed = [];
537
+ const kept = [];
538
+ let producers;
539
+ try {
540
+ const { graph, } = await this.client.projects.flowTopology(pk);
541
+ producers = new Map(graph.edges.filter((edge) => edge.relation === "writes").map((edge) => [edge.to, edge.from,]));
733
542
  }
734
- usedOutputProvisioningFallback = true;
735
- await provisionOutputDatasets();
736
- const retryCreated = await createRecipe();
737
- const retryName = asString(retryCreated?.["name"]);
738
- if (!retryName) {
739
- throw unexpectedResponseError('DSS create-recipe response did not include the final recipe name (documented body: {"name": ...}).');
543
+ catch (graphError) {
544
+ const reason = `could not read the flow graph to check for a recipe writing it (${graphError instanceof Error ? graphError.message : String(graphError)})`;
545
+ for (const name of createdDatasets)
546
+ kept.push({ name, reason, });
740
547
  }
741
- finalRecipeName = retryName;
742
- }
743
- // For sync recipes (a 1:1 copy) DSS does not propagate schema on build, so a
744
- // schemaless dataset output silently builds to an empty dataset. Copy the
745
- // single input's schema onto any schemaless dataset output so the build
746
- // materializes rows (mirrors the DSS UI, which sets the sync output schema).
747
- let syncOutputSchemaPropagated;
748
- if (type.toLowerCase() === "sync" && !outputFolder) {
749
- const inputRefs = recipeInputItems({ inputs, }).map((item) => item.ref);
750
- const outputDatasetRefs = recipeOutputItems({ outputs, })
751
- .filter((item) => item.type !== "MANAGED_FOLDER")
752
- .map((item) => item.ref);
753
- if (inputRefs.length === 1 && outputDatasetRefs.length > 0) {
754
- const inputSchema = await this.client.datasets
755
- .schema(inputRefs[0], pk)
756
- .catch(() => undefined);
757
- const inputColumns = inputSchema?.columns;
758
- if (inputColumns && inputColumns.length > 0) {
759
- const propagated = [];
760
- for (const ref of outputDatasetRefs) {
761
- const current = await this.client.datasets
762
- .schema(ref, pk)
763
- .catch(() => undefined);
764
- if (current && current.columns.length === 0) {
765
- await this.client.datasets.updateSchema(ref, inputColumns, pk);
766
- propagated.push(ref);
767
- }
768
- }
769
- if (propagated.length > 0)
770
- syncOutputSchemaPropagated = propagated;
548
+ for (const name of createdDatasets) {
549
+ if (!producers)
550
+ break;
551
+ const producer = producers.get(name);
552
+ if (producer !== undefined) {
553
+ kept.push({ name, reason: `written by recipe ${producer}`, });
554
+ continue;
771
555
  }
556
+ try {
557
+ await this.client.del(`/public/api/projects/${enc}/datasets/${encodeURIComponent(name)}`);
558
+ deleted.push(name);
559
+ }
560
+ catch (deleteError) {
561
+ failed.push({
562
+ name,
563
+ error: deleteError instanceof Error ? deleteError.message : String(deleteError),
564
+ });
565
+ }
566
+ }
567
+ if (error instanceof DataikuError || error instanceof ClientValidationError) {
568
+ error.details = {
569
+ ...error.details,
570
+ createdDatasetsDeleted: deleted,
571
+ createdDatasetsNotDeleted: failed,
572
+ ...(kept.length > 0 ? { createdDatasetsKept: kept, } : {}),
573
+ };
574
+ }
575
+ throw error;
576
+ };
577
+ try {
578
+ try {
579
+ if (rawConnection)
580
+ await provisionOutputDatasets();
581
+ finalRecipeName = requireRecipeName(await createRecipe());
582
+ }
583
+ catch (error) {
584
+ if (!shouldRetryRecipeCreateWithOutputProvisioning(error)) {
585
+ throw error;
586
+ }
587
+ usedOutputProvisioningFallback = true;
588
+ await provisionOutputDatasets();
589
+ finalRecipeName = requireRecipeName(await createRecipe());
772
590
  }
773
591
  }
592
+ catch (error) {
593
+ await rollbackProvisionedDatasets(error);
594
+ }
774
595
  // For join recipes: configure join conditions after creation
775
596
  let joinConfigured = false;
776
597
  if (type === "join" && joinKeys?.length) {
777
- const rnEnc = encodeURIComponent(name);
598
+ const rnEnc = encodeURIComponent(finalRecipeName);
778
599
  const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
779
600
  const joinPayload = parseRecipePayload(full.payload);
780
601
  const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
@@ -807,7 +628,7 @@ export class RecipesResource extends BaseResource {
807
628
  joinConfigured = true;
808
629
  }
809
630
  if (type === "fuzzyjoin" && fuzzyKeys?.length) {
810
- const rnEnc = encodeURIComponent(name);
631
+ const rnEnc = encodeURIComponent(finalRecipeName);
811
632
  const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
812
633
  const fuzzyPayload = parseRecipePayload(full.payload);
813
634
  const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
@@ -851,9 +672,27 @@ export class RecipesResource extends BaseResource {
851
672
  });
852
673
  joinConfigured = true;
853
674
  }
675
+ // Grouping keys/aggregates and prepare steps edit the payload DSS created.
676
+ let payloadConfigured = false;
677
+ if (grouping || prepareStepList.length > 0) {
678
+ const rnEnc = encodeURIComponent(finalRecipeName);
679
+ const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
680
+ const created = parseRecipePayload(full.payload);
681
+ const configured = grouping
682
+ ? applyGroupingPayload(created, grouping)
683
+ : {
684
+ ...created,
685
+ steps: [...(Array.isArray(created.steps) ? created.steps : []), ...prepareStepList,],
686
+ };
687
+ await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, {
688
+ ...full,
689
+ payload: JSON.stringify(configured),
690
+ });
691
+ payloadConfigured = true;
692
+ }
854
693
  let temporaryOutputDatasetDeleted;
855
694
  if (outputFolder) {
856
- await this.update(name, {
695
+ await this.update(finalRecipeName, {
857
696
  recipe: {
858
697
  outputs: {
859
698
  main: {
@@ -875,18 +714,113 @@ export class RecipesResource extends BaseResource {
875
714
  }
876
715
  }
877
716
  }
717
+ // DSS computes visual-recipe output schemas only when asked (the UI asks on
718
+ // save; dataikuapi exposes compute_schema_updates). Without this, a join or
719
+ // grouping output keeps an empty schema and its build "succeeds" with no
720
+ // columns. Only outputs this call created, or whose schema is still empty,
721
+ // are written: an existing schema is never replaced implicitly.
722
+ let outputSchemaUpdated;
723
+ let outputSchemaUpdateError;
724
+ if (!outputFolder && recipeOutputSchemaIsComputable(type)) {
725
+ try {
726
+ const created = new Set(createdDatasets);
727
+ const { updated, } = await this.applySchemaUpdates(finalRecipeName, await this.computeSchemaUpdates(finalRecipeName, pk), pk, (computable) => computable.previousSchemaWasEmpty === true
728
+ || created.has(schemaUpdateComputableId(computable)));
729
+ if (updated.length > 0)
730
+ outputSchemaUpdated = updated;
731
+ }
732
+ catch (error) {
733
+ if (!(error instanceof DataikuError))
734
+ throw error;
735
+ outputSchemaUpdateError = error.message.split("\n")[0];
736
+ }
737
+ }
878
738
  return {
739
+ ...(opts.name && opts.name !== finalRecipeName ? { requestedName: opts.name, } : {}),
879
740
  recipeName: finalRecipeName,
880
741
  type,
881
742
  createdDatasets,
882
743
  joinConfigured,
744
+ ...(payloadConfigured ? { payloadConfigured, } : {}),
883
745
  outputProvisioningFallbackUsed: usedOutputProvisioningFallback,
884
- ...(syncOutputSchemaPropagated ? { syncOutputSchemaPropagated, } : {}),
746
+ ...(outputSchemaUpdated ? { outputSchemaUpdated, } : {}),
747
+ ...(outputSchemaUpdateError ? { outputSchemaUpdateError, } : {}),
885
748
  ...(outputFolder ? { outputFolder, } : {}),
886
749
  ...(temporaryOutputDataset ? { temporaryOutputDataset, } : {}),
887
750
  ...(temporaryOutputDatasetDeleted !== undefined ? { temporaryOutputDatasetDeleted, } : {}),
888
751
  };
889
752
  }
753
+ /**
754
+ * Compute the output schemas DSS derives from the recipe's current settings
755
+ * (GET /recipes/{name}/schema-update, dataikuapi `compute_schema_updates`).
756
+ * DSS rejects code recipes, whose code sets the schema at run time.
757
+ */
758
+ async computeSchemaUpdates(recipeName, projectKey) {
759
+ const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/recipes/${encodeURIComponent(recipeName)}/schema-update`);
760
+ return this.client.safeParse(RecipeSchemaUpdatesSchema, raw, "recipes.computeSchemaUpdates");
761
+ }
762
+ /**
763
+ * Write each pending computed schema accepted by `select` through
764
+ * actions/updateOutputSchema. Pending outputs `select` rejects are reported,
765
+ * not written.
766
+ */
767
+ async applySchemaUpdates(recipeName, updates, projectKey, select) {
768
+ const endpoint = `/public/api/projects/${this.enc(projectKey)}/recipes/${encodeURIComponent(recipeName)}/actions/updateOutputSchema`;
769
+ const updated = [];
770
+ const pending = [];
771
+ const unchanged = [];
772
+ const datasetsNeedingAction = [];
773
+ for (const computable of updates.computables) {
774
+ const computableId = schemaUpdateComputableId(computable);
775
+ if (!schemaUpdateIsPending(computable)) {
776
+ unchanged.push(computableId);
777
+ continue;
778
+ }
779
+ if (!select(computable)) {
780
+ pending.push(computableId);
781
+ continue;
782
+ }
783
+ const applied = await this.client.post(endpoint, { computableType: computable.type, computableId, newSchema: computable.newSchema, });
784
+ updated.push(computableId);
785
+ if (applied?.datasetsNeedingAction?.length) {
786
+ datasetsNeedingAction.push(...applied.datasetsNeedingAction);
787
+ }
788
+ }
789
+ return { updated, pending, unchanged, datasetsNeedingAction, };
790
+ }
791
+ /**
792
+ * Replace each output schema that differs from the DSS-computed one
793
+ * (dataikuapi `compute_schema_updates().apply()`, the UI's "Update schema").
794
+ * With `derivedOutputs`, only outputs whose schema is empty or listed there
795
+ * are written: callers pass the outputs whose schema matched the computed
796
+ * one before an edit, so DSS-derived schemas follow the edit while
797
+ * hand-edited ones are left alone. `dryRun` writes nothing. Skipped
798
+ * differing outputs are listed under `pending`. Code recipes are rejected
799
+ * before the schema computation: DSS cannot compute their output schema.
800
+ */
801
+ async updateSchema(recipeName, opts) {
802
+ const { recipe, } = await this.get(recipeName, { projectKey: opts?.projectKey, });
803
+ const recipeType = typeof recipe.type === "string" ? recipe.type : "";
804
+ if (!recipeOutputSchemaIsComputable(recipeType)) {
805
+ throw new ClientValidationError(`DSS cannot compute output schemas for ${recipeType} recipes: the code sets the schema when it runs. Build the recipe, or set columns with dss dataset refresh-schema.`, "validation_failed");
806
+ }
807
+ const updates = await this.computeSchemaUpdates(recipeName, opts?.projectKey);
808
+ const derived = opts?.derivedOutputs ? new Set(opts.derivedOutputs) : undefined;
809
+ const applied = await this.applySchemaUpdates(recipeName, updates, opts?.projectKey, (computable) => opts?.dryRun !== true
810
+ && (derived === undefined || computable.previousSchemaWasEmpty === true
811
+ || derived.has(schemaUpdateComputableId(computable))));
812
+ return {
813
+ recipeName,
814
+ updated: applied.updated,
815
+ pending: applied.pending,
816
+ unchanged: applied.unchanged,
817
+ totalIncompatibilities: updates.totalIncompatibilities ?? 0,
818
+ computables: updates.computables,
819
+ ...(applied.datasetsNeedingAction.length > 0
820
+ ? { datasetsNeedingAction: applied.datasetsNeedingAction, }
821
+ : {}),
822
+ };
823
+ }
890
824
  /**
891
825
  * Update a recipe by merging the patch into the current definition.
892
826
  * The `recipe` sub-object is deep-merged to preserve nested fields.
@@ -1,4 +1,5 @@
1
1
  import { BaseResource } from "./base.js";
2
+ import { type ScoringJarOptions, TrainedModel } from "./trained-model.js";
2
3
  export interface SavedModelListItem extends Record<string, unknown> {
3
4
  id?: string;
4
5
  name?: string;
@@ -64,11 +65,6 @@ export interface ExternalModelVersionEvaluateOptions extends Record<string, unkn
64
65
  useOptimalThreshold?: boolean;
65
66
  skipExpensiveReports?: boolean;
66
67
  }
67
- /** Query options for the scoring JAR export (GET .../scoring-jar). */
68
- export interface SavedModelScoringJarOptions extends Record<string, unknown> {
69
- fullClassName?: string;
70
- includeLibs?: boolean;
71
- }
72
68
  export declare class SavedModelsResource extends BaseResource {
73
69
  /** List saved models in a project. */
74
70
  list(projectKey?: string): Promise<SavedModelListItem[]>;
@@ -159,7 +155,7 @@ export declare class SavedModelsResource extends BaseResource {
159
155
  * responsible for consuming the response body (e.g. writeResponseToFile);
160
156
  * the stream is read under the client body deadline.
161
157
  */
162
- downloadScoringJar(options: SavedModelScoringJarOptions | undefined, savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
158
+ downloadScoringJar(options: ScoringJarOptions | undefined, savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
163
159
  /** GET the scoring PMML of a version as a binary/XML stream. */
164
160
  downloadScoringPmml(savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
165
161
  /**
@@ -167,6 +163,14 @@ export declare class SavedModelsResource extends BaseResource {
167
163
  * "userMeta" field of a previously-retrieved version-details object.
168
164
  */
169
165
  setUserMeta(savedModelId: string, versionId: string, userMeta: Record<string, unknown>, projectKey?: string): Promise<void>;
166
+ /**
167
+ * One saved-model version as a {@link TrainedModel}: diagnostics
168
+ * (subpopulation analyses, partial dependencies, timeseries residuals and
169
+ * per-series results), documentation, scoring exports, user metadata.
170
+ */
171
+ version(savedModelId: string, versionId: string, projectKey?: string): TrainedModel;
172
+ /** GET a generated saved-model version documentation (docx) by the `exportId` of its finished future. */
173
+ downloadDocumentation(exportId: string, projectKey?: string): Promise<Response>;
170
174
  /** Delete a saved model and all of its versions. */
171
175
  delete(savedModelId: string, projectKey?: string): Promise<void>;
172
176
  private mlflowImportQuery;