dataiku-sdk 3.6.1 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/README.md +14 -6
  2. package/bin/bun-version.js +14 -0
  3. package/bin/dss.js +11 -15
  4. package/dist/build-metadata.json +1 -1
  5. package/dist/packages/types/src/index.d.ts +74 -1
  6. package/dist/packages/types/src/index.js +53 -1
  7. package/dist/src/cli/coerce.d.ts +1 -0
  8. package/dist/src/cli/coerce.js +19 -15
  9. package/dist/src/cli/command-syntax.json +3521 -416
  10. package/dist/src/cli/commands/analysis.js +19 -1
  11. package/dist/src/cli/commands/bundle.js +22 -0
  12. package/dist/src/cli/commands/connection.js +26 -2
  13. package/dist/src/cli/commands/data-quality.js +33 -1
  14. package/dist/src/cli/commands/dataset.js +35 -4
  15. package/dist/src/cli/commands/discussion.js +15 -0
  16. package/dist/src/cli/commands/doctor.js +1 -1
  17. package/dist/src/cli/commands/flow.d.ts +5 -0
  18. package/dist/src/cli/commands/flow.js +57 -0
  19. package/dist/src/cli/commands/future.js +8 -0
  20. package/dist/src/cli/commands/index.js +6 -0
  21. package/dist/src/cli/commands/job.js +29 -6
  22. package/dist/src/cli/commands/metrics.js +21 -1
  23. package/dist/src/cli/commands/ml-task.d.ts +5 -0
  24. package/dist/src/cli/commands/ml-task.js +86 -16
  25. package/dist/src/cli/commands/mlflow.d.ts +11 -0
  26. package/dist/src/cli/commands/mlflow.js +121 -0
  27. package/dist/src/cli/commands/plugin.js +5 -11
  28. package/dist/src/cli/commands/project-deployer.js +70 -0
  29. package/dist/src/cli/commands/project-library.js +2 -5
  30. package/dist/src/cli/commands/project.js +8 -0
  31. package/dist/src/cli/commands/recipe.d.ts +9 -0
  32. package/dist/src/cli/commands/recipe.js +93 -7
  33. package/dist/src/cli/commands/saved-model.d.ts +1 -1
  34. package/dist/src/cli/commands/saved-model.js +9 -76
  35. package/dist/src/cli/commands/streaming-endpoint.js +30 -1
  36. package/dist/src/cli/commands/webapp.js +8 -0
  37. package/dist/src/cli/commands/wiki.js +22 -0
  38. package/dist/src/cli/commands/workspace.js +8 -0
  39. package/dist/src/cli/contract.d.ts +16 -15
  40. package/dist/src/cli/contract.js +139 -3404
  41. package/dist/src/cli/doctor.d.ts +13 -0
  42. package/dist/src/cli/doctor.js +41 -1
  43. package/dist/src/cli/flags.d.ts +7 -1
  44. package/dist/src/cli/flags.js +78 -6
  45. package/dist/src/cli/helpers/app-successor.js +1 -2
  46. package/dist/src/cli/helpers/documentation.d.ts +21 -0
  47. package/dist/src/cli/helpers/documentation.js +53 -0
  48. package/dist/src/cli/helpers/trained-model.d.ts +26 -0
  49. package/dist/src/cli/helpers/trained-model.js +144 -0
  50. package/dist/src/cli/list-output.d.ts +7 -0
  51. package/dist/src/cli/list-output.js +77 -0
  52. package/dist/src/cli/list-shapes.d.ts +21 -0
  53. package/dist/src/cli/list-shapes.js +111 -0
  54. package/dist/src/cli/output-schemas.d.ts +0 -14
  55. package/dist/src/cli/output-schemas.js +4 -7
  56. package/dist/src/cli/output.d.ts +0 -1
  57. package/dist/src/cli/output.js +27 -19
  58. package/dist/src/cli/plans.d.ts +14 -0
  59. package/dist/src/cli/plans.js +3684 -0
  60. package/dist/src/cli/usage.d.ts +7 -0
  61. package/dist/src/cli/usage.js +20 -1
  62. package/dist/src/cli/version.d.ts +2 -2
  63. package/dist/src/cli/version.js +2 -2
  64. package/dist/src/cli.js +127 -113
  65. package/dist/src/client.d.ts +52 -43
  66. package/dist/src/client.js +267 -96
  67. package/dist/src/errors.d.ts +11 -1
  68. package/dist/src/errors.js +31 -47
  69. package/dist/src/generated/action-output-schemas.json +345 -89
  70. package/dist/src/index.d.ts +5 -1
  71. package/dist/src/index.js +3 -0
  72. package/dist/src/resources/analyses.d.ts +8 -0
  73. package/dist/src/resources/analyses.js +13 -1
  74. package/dist/src/resources/bundles.d.ts +34 -0
  75. package/dist/src/resources/bundles.js +67 -0
  76. package/dist/src/resources/connections.d.ts +9 -0
  77. package/dist/src/resources/connections.js +15 -0
  78. package/dist/src/resources/data-quality.d.ts +20 -0
  79. package/dist/src/resources/data-quality.js +30 -0
  80. package/dist/src/resources/dataset-create.d.ts +19 -0
  81. package/dist/src/resources/dataset-create.js +66 -0
  82. package/dist/src/resources/datasets.d.ts +25 -15
  83. package/dist/src/resources/datasets.js +59 -111
  84. package/dist/src/resources/discussions.d.ts +5 -0
  85. package/dist/src/resources/discussions.js +7 -0
  86. package/dist/src/resources/documentation.d.ts +25 -0
  87. package/dist/src/resources/documentation.js +16 -0
  88. package/dist/src/resources/flow.d.ts +48 -0
  89. package/dist/src/resources/flow.js +65 -0
  90. package/dist/src/resources/futures.d.ts +23 -0
  91. package/dist/src/resources/futures.js +32 -0
  92. package/dist/src/resources/jobs.js +102 -2
  93. package/dist/src/resources/metrics.d.ts +10 -0
  94. package/dist/src/resources/metrics.js +10 -0
  95. package/dist/src/resources/ml-tasks.d.ts +65 -2
  96. package/dist/src/resources/ml-tasks.js +102 -28
  97. package/dist/src/resources/mlflow-extension.d.ts +73 -0
  98. package/dist/src/resources/mlflow-extension.js +105 -0
  99. package/dist/src/resources/projects.d.ts +6 -0
  100. package/dist/src/resources/projects.js +8 -0
  101. package/dist/src/resources/recipe-create.d.ts +65 -0
  102. package/dist/src/resources/recipe-create.js +338 -0
  103. package/dist/src/resources/recipes.d.ts +28 -29
  104. package/dist/src/resources/recipes.js +126 -245
  105. package/dist/src/resources/saved-models.d.ts +10 -6
  106. package/dist/src/resources/saved-models.js +16 -14
  107. package/dist/src/resources/scenarios.js +8 -14
  108. package/dist/src/resources/streaming-endpoints.d.ts +12 -0
  109. package/dist/src/resources/streaming-endpoints.js +15 -0
  110. package/dist/src/resources/trained-model.d.ts +60 -0
  111. package/dist/src/resources/trained-model.js +98 -0
  112. package/dist/src/resources/webapps.d.ts +8 -0
  113. package/dist/src/resources/webapps.js +10 -0
  114. package/dist/src/resources/wiki.d.ts +5 -0
  115. package/dist/src/resources/wiki.js +7 -0
  116. package/dist/src/resources/workspaces.d.ts +2 -0
  117. package/dist/src/resources/workspaces.js +4 -0
  118. package/dist/src/utils/response-file.js +20 -9
  119. package/dist/src/utils/secret-sanitize.d.ts +7 -0
  120. package/dist/src/utils/secret-sanitize.js +46 -0
  121. package/package.json +25 -26
  122. package/plugin.json +1 -1
  123. package/skills/dataiku-dss/SKILL.md +1 -1
  124. package/skills/dataiku-dss/references/authentication.md +3 -1
  125. package/skills/dataiku-dss/references/coding.md +1 -0
  126. package/skills/dataiku-dss/references/discovery.md +4 -6
  127. package/skills/dataiku-dss/references/troubleshooting.md +3 -2
@@ -1,23 +1,15 @@
1
1
  import { writeFile, } from "node:fs/promises";
2
2
  import { resolve, } from "node:path";
3
3
  import { ClientValidationError, DataikuError, nonJsonResponseBody, unexpectedResponseError, } from "../errors.js";
4
- import { ProjectMetadataSchema, RecipeSummaryArraySchema, } from "../schemas.js";
4
+ import { ProjectMetadataSchema, RecipeSchemaUpdatesSchema, RecipeSummaryArraySchema, } from "../schemas.js";
5
5
  import { deepMerge, } from "../utils/deep-merge.js";
6
6
  import { asRecord, } from "../utils/records.js";
7
7
  import { sanitizeFileName, } from "../utils/sanitize.js";
8
8
  import { BaseResource, } from "./base.js";
9
+ import { applyGroupingPayload, asString, buildRecipeCreateRequest, recipeInputItems, recipeOutputSchemaIsComputable, schemaUpdateComputableId, schemaUpdateIsPending, } from "./recipe-create.js";
9
10
  // ---------------------------------------------------------------------------
10
11
  // Helpers: type narrowing
11
12
  // ---------------------------------------------------------------------------
12
- function asString(value) {
13
- return typeof value === "string" && value.length > 0 ? value : undefined;
14
- }
15
- function asStringArray(value) {
16
- if (!Array.isArray(value))
17
- return undefined;
18
- const out = value.filter((v) => typeof v === "string" && v.length > 0);
19
- return out.length > 0 ? out : undefined;
20
- }
21
13
  function parseRecipePayload(payload) {
22
14
  if (!payload)
23
15
  return {};
@@ -30,48 +22,6 @@ function parseRecipePayload(payload) {
30
22
  }
31
23
  return asRecord(parsed) ?? {};
32
24
  }
33
- function parseJoinKeyPairs(values, optionName = "joinOn") {
34
- const pairs = [];
35
- for (const raw of values) {
36
- const token = raw.trim();
37
- if (!token)
38
- continue;
39
- const eq = token.indexOf("=");
40
- if (eq === -1) {
41
- pairs.push({ left: token, right: token, });
42
- continue;
43
- }
44
- const left = token.slice(0, eq).trim();
45
- const right = token.slice(eq + 1).trim();
46
- if (!left || !right) {
47
- throw new ClientValidationError(`${optionName} values must use COL or LEFT=RIGHT.`, "validation_failed");
48
- }
49
- pairs.push({ left, right, });
50
- }
51
- return pairs;
52
- }
53
- function normalizeFuzzyDistance(value) {
54
- const normalized = typeof value === "string" && value.trim().length > 0
55
- ? value.trim().toUpperCase()
56
- : "DAMERAU_LEVENSHTEIN";
57
- if (normalized === "DAMERAU_LEVENSHTEIN")
58
- return "LEVENSHTEIN";
59
- if (normalized === "HAMMING" || normalized === "JACCARD" || normalized === "COSINE"
60
- || normalized === "EUCLIDEAN") {
61
- return normalized;
62
- }
63
- throw new ClientValidationError("fuzzyDistance must be one of DAMERAU_LEVENSHTEIN, HAMMING, JACCARD, COSINE, or EUCLIDEAN.", "validation_failed");
64
- }
65
- function normalizeJoinType(value) {
66
- const normalized = typeof value === "string" && value.trim().length > 0
67
- ? value.trim().toUpperCase()
68
- : "LEFT";
69
- if (normalized === "LEFT" || normalized === "INNER" || normalized === "RIGHT"
70
- || normalized === "FULL") {
71
- return normalized;
72
- }
73
- throw new ClientValidationError("joinType must be one of LEFT, INNER, RIGHT, or FULL.", "validation_failed");
74
- }
75
25
  function recipeVirtualInputs(payload, inputCount) {
76
26
  const current = Array.isArray(payload.virtualInputs) ? payload.virtualInputs : [];
77
27
  const virtualInputs = [];
@@ -136,27 +86,6 @@ function recipeOutputItems(recipe) {
136
86
  }
137
87
  return result;
138
88
  }
139
- function recipeInputItems(recipe) {
140
- const inputs = asRecord(recipe.inputs);
141
- if (!inputs)
142
- return [];
143
- const result = [];
144
- const seen = new Set();
145
- for (const [role, roleValue,] of Object.entries(inputs)) {
146
- const items = asRecord(roleValue)?.items;
147
- if (!Array.isArray(items))
148
- continue;
149
- for (const itemValue of items) {
150
- const item = asRecord(itemValue);
151
- const ref = asString(item?.ref);
152
- if (!ref || seen.has(ref))
153
- continue;
154
- seen.add(ref);
155
- result.push({ ref, role, });
156
- }
157
- }
158
- return result;
159
- }
160
89
  function rewriteRefs(value, rewrites) {
161
90
  if (Object.keys(rewrites).length === 0)
162
91
  return value;
@@ -295,96 +224,6 @@ function shouldRetryRecipeCreateWithOutputProvisioning(error) {
295
224
  // ---------------------------------------------------------------------------
296
225
  // Resource
297
226
  // ---------------------------------------------------------------------------
298
- /**
299
- * Pure request construction for POST /recipes/ (no DSS calls). Shared with
300
- * `recipe create --plan` so the plan equals the request the command sends.
301
- */
302
- export function buildRecipeCreateRequest(opts, pk) {
303
- const { type, payload, outputConnection: rawConnection, joinType: rawJoinType, } = opts;
304
- const outputFolder = asString(opts.outputFolder);
305
- // Build inputs/outputs from simple form (inputDatasets + outputDataset) or
306
- // advanced form (inputs + outputs); both may coexist — simple form wins when
307
- // the advanced form is absent.
308
- const inputDatasets = asStringArray(opts.inputDatasets);
309
- const requestedOutputDataset = asString(opts.outputDataset);
310
- let inputs = asRecord(opts.inputs);
311
- let outputs = asRecord(opts.outputs);
312
- if (!inputs) {
313
- inputs = {
314
- main: {
315
- items: inputDatasets?.map((ref) => ({ ref, deps: [], })) ?? [],
316
- },
317
- };
318
- }
319
- // Auto-generate name if not provided
320
- const outputNameForDefaultRecipe = requestedOutputDataset ?? outputFolder;
321
- const name = opts.name ?? (type && outputNameForDefaultRecipe
322
- ? `${type}_${outputNameForDefaultRecipe}`
323
- : undefined);
324
- const temporaryOutputDataset = outputFolder && !requestedOutputDataset && name
325
- ? `${name}_folder_output_marker`
326
- : undefined;
327
- const outputDataset = requestedOutputDataset ?? temporaryOutputDataset;
328
- if (!outputs && outputDataset) {
329
- outputs = {
330
- main: {
331
- items: [{ ref: outputDataset, appendMode: false, },],
332
- },
333
- };
334
- }
335
- if (!type || !name || !outputs) {
336
- throw new ClientValidationError("type and outputDataset/outputFolder or (name + outputs) are required for create.", "validation_failed");
337
- }
338
- const joinCols = typeof opts.joinOn === "string" ? [opts.joinOn,] : asStringArray(opts.joinOn);
339
- const joinKeys = type === "join" && joinCols?.length
340
- ? parseJoinKeyPairs(joinCols)
341
- : undefined;
342
- const fuzzyOn = opts.fuzzyOn ?? (type === "fuzzyjoin" ? opts.joinOn : undefined);
343
- const fuzzyCols = typeof fuzzyOn === "string" ? [fuzzyOn,] : asStringArray(fuzzyOn);
344
- const fuzzyKeys = type === "fuzzyjoin" && fuzzyCols?.length
345
- ? parseJoinKeyPairs(fuzzyCols, "fuzzyOn")
346
- : undefined;
347
- const normalizedJoinType = joinKeys?.length || fuzzyKeys?.length
348
- ? normalizeJoinType(rawJoinType)
349
- : undefined;
350
- const fuzzyDistance = fuzzyKeys?.length ? normalizeFuzzyDistance(opts.fuzzyDistance) : undefined;
351
- const fuzzyThreshold = opts.fuzzyThreshold ?? 1;
352
- if (!Number.isFinite(fuzzyThreshold)) {
353
- throw new ClientValidationError("fuzzyThreshold must be a finite number.", "validation_failed");
354
- }
355
- const recipePrototype = {
356
- type,
357
- name,
358
- projectKey: pk,
359
- ...(type === "fuzzyjoin" ? {} : { inputs, }),
360
- outputs,
361
- };
362
- const creationSettings = {};
363
- if (payload !== undefined) {
364
- creationSettings.script = payload;
365
- }
366
- if (type === "fuzzyjoin") {
367
- creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
368
- }
369
- return {
370
- creationSettings,
371
- fuzzyDistance,
372
- fuzzyKeys,
373
- fuzzyThreshold,
374
- inputDatasets,
375
- inputs,
376
- joinKeys,
377
- name,
378
- normalizedJoinType,
379
- outputFolder,
380
- outputs,
381
- payload,
382
- rawConnection,
383
- recipePrototype,
384
- temporaryOutputDataset,
385
- type,
386
- };
387
- }
388
227
  export class RecipesResource extends BaseResource {
389
228
  /** List all recipes in a project. */
390
229
  async list(projectKey) {
@@ -636,13 +475,18 @@ export class RecipesResource extends BaseResource {
636
475
  async create(opts) {
637
476
  const pk = this.resolveProjectKey(opts.projectKey);
638
477
  const enc = encodeURIComponent(pk);
639
- const { creationSettings, fuzzyDistance, fuzzyKeys, fuzzyThreshold, inputDatasets, inputs, joinKeys, name, normalizedJoinType, outputFolder, outputs, rawConnection, recipePrototype, temporaryOutputDataset, type, } = buildRecipeCreateRequest(opts, pk);
478
+ const { creationSettings, fuzzyDistance, fuzzyKeys, fuzzyThreshold, grouping, inputDatasets, inputs, joinKeys, normalizedJoinType, outputFolder, outputs, prepareSteps: prepareStepList, rawConnection, recipePrototype, temporaryOutputDataset, type, } = buildRecipeCreateRequest(opts, pk);
640
479
  const createRecipe = () => this.client.post(`/public/api/projects/${enc}/recipes/`, {
641
480
  recipePrototype,
642
481
  creationSettings,
643
482
  });
644
483
  const createdDatasets = [];
645
484
  let usedOutputProvisioningFallback = false;
485
+ // Missing outputs are created through POST /datasets/managed, like the DSS UI
486
+ // and dataikuapi's new_managed_dataset: DSS picks the dataset type, storage
487
+ // path, and format from the connection (an S3 connection yields an S3
488
+ // dataset under the connection's naming rule), so the client never guesses
489
+ // type-specific params.
646
490
  const provisionOutputDatasets = async () => {
647
491
  const existingDs = await this.client.get(`/public/api/projects/${enc}/datasets/`);
648
492
  let outputConnection = asString(rawConnection);
@@ -655,58 +499,14 @@ export class RecipesResource extends BaseResource {
655
499
  if (!outputConnection)
656
500
  return;
657
501
  const existingNames = new Set([...existingDs.map((d) => d.name), ...createdDatasets,]);
658
- const connectionSample = existingDs.find((d) => d.params?.connection === outputConnection && d.type);
659
- const inferredOutputType = connectionSample?.type ?? "Filesystem";
660
502
  const outputRoles = outputs;
661
503
  for (const role of Object.values(outputRoles)) {
662
504
  for (const item of role.items ?? []) {
663
505
  if (!item.ref || existingNames.has(item.ref))
664
506
  continue;
665
- const datasetBody = inferredOutputType === "Filesystem"
666
- ? {
667
- projectKey: pk,
668
- name: item.ref,
669
- type: inferredOutputType,
670
- params: {
671
- connection: outputConnection,
672
- path: `/dataiku/${pk}/${item.ref}`,
673
- metastoreTableName: item.ref,
674
- },
675
- formatType: "csv",
676
- formatParams: {
677
- style: "excel",
678
- charset: "utf8",
679
- separator: "\t",
680
- quoteChar: '"',
681
- escapeChar: "\\",
682
- dateSerializationFormat: "ISO",
683
- arrayMapFormat: "json",
684
- parseHeaderRow: true,
685
- compress: "gz",
686
- },
687
- managed: true,
688
- }
689
- : {
690
- projectKey: pk,
691
- name: item.ref,
692
- type: inferredOutputType,
693
- params: {
694
- connection: outputConnection,
695
- mode: "table",
696
- table: item.ref,
697
- ...(connectionSample?.params?.schema
698
- ? { schema: connectionSample.params.schema, }
699
- : {}),
700
- ...(connectionSample?.params?.catalog
701
- ? { catalog: connectionSample.params.catalog, }
702
- : {}),
703
- },
704
- managed: connectionSample?.managed ?? false,
705
- };
706
- await this.client.post(`/public/api/projects/${enc}/datasets/`, datasetBody);
507
+ await this.client.datasets.createManaged({ name: item.ref, connection: outputConnection, }, pk);
707
508
  existingNames.add(item.ref);
708
- if (!createdDatasets.includes(item.ref))
709
- createdDatasets.push(item.ref);
509
+ createdDatasets.push(item.ref);
710
510
  }
711
511
  }
712
512
  };
@@ -740,41 +540,10 @@ export class RecipesResource extends BaseResource {
740
540
  }
741
541
  finalRecipeName = retryName;
742
542
  }
743
- // For sync recipes (a 1:1 copy) DSS does not propagate schema on build, so a
744
- // schemaless dataset output silently builds to an empty dataset. Copy the
745
- // single input's schema onto any schemaless dataset output so the build
746
- // materializes rows (mirrors the DSS UI, which sets the sync output schema).
747
- let syncOutputSchemaPropagated;
748
- if (type.toLowerCase() === "sync" && !outputFolder) {
749
- const inputRefs = recipeInputItems({ inputs, }).map((item) => item.ref);
750
- const outputDatasetRefs = recipeOutputItems({ outputs, })
751
- .filter((item) => item.type !== "MANAGED_FOLDER")
752
- .map((item) => item.ref);
753
- if (inputRefs.length === 1 && outputDatasetRefs.length > 0) {
754
- const inputSchema = await this.client.datasets
755
- .schema(inputRefs[0], pk)
756
- .catch(() => undefined);
757
- const inputColumns = inputSchema?.columns;
758
- if (inputColumns && inputColumns.length > 0) {
759
- const propagated = [];
760
- for (const ref of outputDatasetRefs) {
761
- const current = await this.client.datasets
762
- .schema(ref, pk)
763
- .catch(() => undefined);
764
- if (current && current.columns.length === 0) {
765
- await this.client.datasets.updateSchema(ref, inputColumns, pk);
766
- propagated.push(ref);
767
- }
768
- }
769
- if (propagated.length > 0)
770
- syncOutputSchemaPropagated = propagated;
771
- }
772
- }
773
- }
774
543
  // For join recipes: configure join conditions after creation
775
544
  let joinConfigured = false;
776
545
  if (type === "join" && joinKeys?.length) {
777
- const rnEnc = encodeURIComponent(name);
546
+ const rnEnc = encodeURIComponent(finalRecipeName);
778
547
  const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
779
548
  const joinPayload = parseRecipePayload(full.payload);
780
549
  const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
@@ -807,7 +576,7 @@ export class RecipesResource extends BaseResource {
807
576
  joinConfigured = true;
808
577
  }
809
578
  if (type === "fuzzyjoin" && fuzzyKeys?.length) {
810
- const rnEnc = encodeURIComponent(name);
579
+ const rnEnc = encodeURIComponent(finalRecipeName);
811
580
  const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
812
581
  const fuzzyPayload = parseRecipePayload(full.payload);
813
582
  const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
@@ -851,9 +620,27 @@ export class RecipesResource extends BaseResource {
851
620
  });
852
621
  joinConfigured = true;
853
622
  }
623
+ // Grouping keys/aggregates and prepare steps edit the payload DSS created.
624
+ let payloadConfigured = false;
625
+ if (grouping || prepareStepList.length > 0) {
626
+ const rnEnc = encodeURIComponent(finalRecipeName);
627
+ const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
628
+ const created = parseRecipePayload(full.payload);
629
+ const configured = grouping
630
+ ? applyGroupingPayload(created, grouping)
631
+ : {
632
+ ...created,
633
+ steps: [...(Array.isArray(created.steps) ? created.steps : []), ...prepareStepList,],
634
+ };
635
+ await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, {
636
+ ...full,
637
+ payload: JSON.stringify(configured),
638
+ });
639
+ payloadConfigured = true;
640
+ }
854
641
  let temporaryOutputDatasetDeleted;
855
642
  if (outputFolder) {
856
- await this.update(name, {
643
+ await this.update(finalRecipeName, {
857
644
  recipe: {
858
645
  outputs: {
859
646
  main: {
@@ -875,18 +662,112 @@ export class RecipesResource extends BaseResource {
875
662
  }
876
663
  }
877
664
  }
665
+ // DSS computes visual-recipe output schemas only when asked (the UI asks on
666
+ // save; dataikuapi exposes compute_schema_updates). Without this, a join or
667
+ // grouping output keeps an empty schema and its build "succeeds" with no
668
+ // columns. Only outputs this call created, or whose schema is still empty,
669
+ // are written: an existing schema is never replaced implicitly.
670
+ let outputSchemaUpdated;
671
+ let outputSchemaUpdateError;
672
+ if (!outputFolder && recipeOutputSchemaIsComputable(type)) {
673
+ try {
674
+ const created = new Set(createdDatasets);
675
+ const { updated, } = await this.applySchemaUpdates(finalRecipeName, await this.computeSchemaUpdates(finalRecipeName, pk), pk, (computable) => computable.previousSchemaWasEmpty === true
676
+ || created.has(schemaUpdateComputableId(computable)));
677
+ if (updated.length > 0)
678
+ outputSchemaUpdated = updated;
679
+ }
680
+ catch (error) {
681
+ if (!(error instanceof DataikuError))
682
+ throw error;
683
+ outputSchemaUpdateError = error.message.split("\n")[0];
684
+ }
685
+ }
878
686
  return {
879
687
  recipeName: finalRecipeName,
880
688
  type,
881
689
  createdDatasets,
882
690
  joinConfigured,
691
+ ...(payloadConfigured ? { payloadConfigured, } : {}),
883
692
  outputProvisioningFallbackUsed: usedOutputProvisioningFallback,
884
- ...(syncOutputSchemaPropagated ? { syncOutputSchemaPropagated, } : {}),
693
+ ...(outputSchemaUpdated ? { outputSchemaUpdated, } : {}),
694
+ ...(outputSchemaUpdateError ? { outputSchemaUpdateError, } : {}),
885
695
  ...(outputFolder ? { outputFolder, } : {}),
886
696
  ...(temporaryOutputDataset ? { temporaryOutputDataset, } : {}),
887
697
  ...(temporaryOutputDatasetDeleted !== undefined ? { temporaryOutputDatasetDeleted, } : {}),
888
698
  };
889
699
  }
700
+ /**
701
+ * Compute the output schemas DSS derives from the recipe's current settings
702
+ * (GET /recipes/{name}/schema-update, dataikuapi `compute_schema_updates`).
703
+ * DSS rejects code recipes, whose code sets the schema at run time.
704
+ */
705
+ async computeSchemaUpdates(recipeName, projectKey) {
706
+ const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/recipes/${encodeURIComponent(recipeName)}/schema-update`);
707
+ return this.client.safeParse(RecipeSchemaUpdatesSchema, raw, "recipes.computeSchemaUpdates");
708
+ }
709
+ /**
710
+ * Write each pending computed schema accepted by `select` through
711
+ * actions/updateOutputSchema. Pending outputs `select` rejects are reported,
712
+ * not written.
713
+ */
714
+ async applySchemaUpdates(recipeName, updates, projectKey, select) {
715
+ const endpoint = `/public/api/projects/${this.enc(projectKey)}/recipes/${encodeURIComponent(recipeName)}/actions/updateOutputSchema`;
716
+ const updated = [];
717
+ const pending = [];
718
+ const unchanged = [];
719
+ const datasetsNeedingAction = [];
720
+ for (const computable of updates.computables) {
721
+ const computableId = schemaUpdateComputableId(computable);
722
+ if (!schemaUpdateIsPending(computable)) {
723
+ unchanged.push(computableId);
724
+ continue;
725
+ }
726
+ if (!select(computable)) {
727
+ pending.push(computableId);
728
+ continue;
729
+ }
730
+ const applied = await this.client.post(endpoint, { computableType: computable.type, computableId, newSchema: computable.newSchema, });
731
+ updated.push(computableId);
732
+ if (applied?.datasetsNeedingAction?.length) {
733
+ datasetsNeedingAction.push(...applied.datasetsNeedingAction);
734
+ }
735
+ }
736
+ return { updated, pending, unchanged, datasetsNeedingAction, };
737
+ }
738
+ /**
739
+ * Replace each output schema that differs from the DSS-computed one
740
+ * (dataikuapi `compute_schema_updates().apply()`, the UI's "Update schema").
741
+ * With `derivedOutputs`, only outputs whose schema is empty or listed there
742
+ * are written: callers pass the outputs whose schema matched the computed
743
+ * one before an edit, so DSS-derived schemas follow the edit while
744
+ * hand-edited ones are left alone. `dryRun` writes nothing. Skipped
745
+ * differing outputs are listed under `pending`. Code recipes are rejected
746
+ * before the schema computation: DSS cannot compute their output schema.
747
+ */
748
+ async updateSchema(recipeName, opts) {
749
+ const { recipe, } = await this.get(recipeName, { projectKey: opts?.projectKey, });
750
+ const recipeType = typeof recipe.type === "string" ? recipe.type : "";
751
+ if (!recipeOutputSchemaIsComputable(recipeType)) {
752
+ throw new ClientValidationError(`DSS cannot compute output schemas for ${recipeType} recipes: the code sets the schema when it runs. Build the recipe, or set columns with dss dataset refresh-schema.`, "validation_failed");
753
+ }
754
+ const updates = await this.computeSchemaUpdates(recipeName, opts?.projectKey);
755
+ const derived = opts?.derivedOutputs ? new Set(opts.derivedOutputs) : undefined;
756
+ const applied = await this.applySchemaUpdates(recipeName, updates, opts?.projectKey, (computable) => opts?.dryRun !== true
757
+ && (derived === undefined || computable.previousSchemaWasEmpty === true
758
+ || derived.has(schemaUpdateComputableId(computable))));
759
+ return {
760
+ recipeName,
761
+ updated: applied.updated,
762
+ pending: applied.pending,
763
+ unchanged: applied.unchanged,
764
+ totalIncompatibilities: updates.totalIncompatibilities ?? 0,
765
+ computables: updates.computables,
766
+ ...(applied.datasetsNeedingAction.length > 0
767
+ ? { datasetsNeedingAction: applied.datasetsNeedingAction, }
768
+ : {}),
769
+ };
770
+ }
890
771
  /**
891
772
  * Update a recipe by merging the patch into the current definition.
892
773
  * The `recipe` sub-object is deep-merged to preserve nested fields.
@@ -1,4 +1,5 @@
1
1
  import { BaseResource } from "./base.js";
2
+ import { type ScoringJarOptions, TrainedModel } from "./trained-model.js";
2
3
  export interface SavedModelListItem extends Record<string, unknown> {
3
4
  id?: string;
4
5
  name?: string;
@@ -64,11 +65,6 @@ export interface ExternalModelVersionEvaluateOptions extends Record<string, unkn
64
65
  useOptimalThreshold?: boolean;
65
66
  skipExpensiveReports?: boolean;
66
67
  }
67
- /** Query options for the scoring JAR export (GET .../scoring-jar). */
68
- export interface SavedModelScoringJarOptions extends Record<string, unknown> {
69
- fullClassName?: string;
70
- includeLibs?: boolean;
71
- }
72
68
  export declare class SavedModelsResource extends BaseResource {
73
69
  /** List saved models in a project. */
74
70
  list(projectKey?: string): Promise<SavedModelListItem[]>;
@@ -159,7 +155,7 @@ export declare class SavedModelsResource extends BaseResource {
159
155
  * responsible for consuming the response body (e.g. writeResponseToFile);
160
156
  * the stream is read under the client body deadline.
161
157
  */
162
- downloadScoringJar(options: SavedModelScoringJarOptions | undefined, savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
158
+ downloadScoringJar(options: ScoringJarOptions | undefined, savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
163
159
  /** GET the scoring PMML of a version as a binary/XML stream. */
164
160
  downloadScoringPmml(savedModelId: string, versionId: string, projectKey?: string): Promise<Response>;
165
161
  /**
@@ -167,6 +163,14 @@ export declare class SavedModelsResource extends BaseResource {
167
163
  * "userMeta" field of a previously-retrieved version-details object.
168
164
  */
169
165
  setUserMeta(savedModelId: string, versionId: string, userMeta: Record<string, unknown>, projectKey?: string): Promise<void>;
166
+ /**
167
+ * One saved-model version as a {@link TrainedModel}: diagnostics
168
+ * (subpopulation analyses, partial dependencies, timeseries residuals and
169
+ * per-series results), documentation, scoring exports, user metadata.
170
+ */
171
+ version(savedModelId: string, versionId: string, projectKey?: string): TrainedModel;
172
+ /** GET a generated saved-model version documentation (docx) by the `exportId` of its finished future. */
173
+ downloadDocumentation(exportId: string, projectKey?: string): Promise<Response>;
170
174
  /** Delete a saved model and all of its versions. */
171
175
  delete(savedModelId: string, projectKey?: string): Promise<void>;
172
176
  private mlflowImportQuery;
@@ -1,6 +1,7 @@
1
1
  import { basename, } from "node:path";
2
2
  import { ClientValidationError, } from "../errors.js";
3
3
  import { BaseResource, requireNonEmpty, requireNonEmptyArray, requireObject, } from "./base.js";
4
+ import { TrainedModel, } from "./trained-model.js";
4
5
  function boolQuery(value) {
5
6
  return value === undefined ? "true" : String(value);
6
7
  }
@@ -187,30 +188,31 @@ export class SavedModelsResource extends BaseResource {
187
188
  * the stream is read under the client body deadline.
188
189
  */
189
190
  async downloadScoringJar(options, savedModelId, versionId, projectKey) {
190
- const version = encodeURIComponent(requireNonEmpty(versionId, "versionId"));
191
- const query = new URLSearchParams();
192
- if (options?.fullClassName !== undefined) {
193
- query.set("fullClassName", options.fullClassName);
194
- }
195
- if (options?.includeLibs !== undefined) {
196
- query.set("includeLibs", String(options.includeLibs));
197
- }
198
- const suffix = [...query.keys(),].length > 0 ? `?${query.toString()}` : "";
199
- return this.client.stream(`${this.savedModelPath(savedModelId, projectKey)}/versions/${version}/scoring-jar${suffix}`);
191
+ return this.version(savedModelId, versionId, projectKey).downloadScoringJar(options);
200
192
  }
201
193
  /** GET the scoring PMML of a version as a binary/XML stream. */
202
194
  async downloadScoringPmml(savedModelId, versionId, projectKey) {
203
- const version = encodeURIComponent(requireNonEmpty(versionId, "versionId"));
204
- return this.client.stream(`${this.savedModelPath(savedModelId, projectKey)}/versions/${version}/scoring-pmml`);
195
+ return this.version(savedModelId, versionId, projectKey).downloadScoringPmml();
205
196
  }
206
197
  /**
207
198
  * Update the user metadata of a model version. The body must only be the
208
199
  * "userMeta" field of a previously-retrieved version-details object.
209
200
  */
210
201
  async setUserMeta(savedModelId, versionId, userMeta, projectKey) {
202
+ await this.version(savedModelId, versionId, projectKey).setUserMeta(userMeta);
203
+ }
204
+ /**
205
+ * One saved-model version as a {@link TrainedModel}: diagnostics
206
+ * (subpopulation analyses, partial dependencies, timeseries residuals and
207
+ * per-series results), documentation, scoring exports, user metadata.
208
+ */
209
+ version(savedModelId, versionId, projectKey) {
211
210
  const version = encodeURIComponent(requireNonEmpty(versionId, "versionId"));
212
- requireObject(userMeta, "userMeta");
213
- await this.client.putVoid(`${this.savedModelPath(savedModelId, projectKey)}/versions/${version}/user-meta`, userMeta);
211
+ return new TrainedModel(this.client, `${this.savedModelPath(savedModelId, projectKey)}/versions/${version}`);
212
+ }
213
+ /** GET a generated saved-model version documentation (docx) by the `exportId` of its finished future. */
214
+ async downloadDocumentation(exportId, projectKey) {
215
+ return this.client.stream(`/public/api/projects/${this.enc(projectKey)}/savedmodels/documentations/${encodeURIComponent(requireNonEmpty(exportId, "exportId"))}`);
214
216
  }
215
217
  /** Delete a saved model and all of its versions. */
216
218
  async delete(savedModelId, projectKey) {
@@ -1,4 +1,3 @@
1
- import { randomUUID, } from "node:crypto";
2
1
  import { ClientValidationError, DataikuError, } from "../errors.js";
3
2
  import { ScenarioDetailsSchema, ScenarioStatusSchema, ScenarioSummaryArraySchema, } from "../schemas.js";
4
3
  import { deepMerge, } from "../utils/deep-merge.js";
@@ -56,20 +55,15 @@ function scenarioRunSummaryFromRaw(entry) {
56
55
  if (!isRecord(entry))
57
56
  return { runId: "unknown", };
58
57
  const runId = typeof entry.runId === "string" ? entry.runId : "unknown";
59
- const optionalRecord = (value) => isRecord(value) ? value : undefined;
58
+ const start = optionalNumber(entry.start);
59
+ const end = optionalNumber(entry.end);
60
60
  return {
61
61
  runId,
62
- ...(optionalNumber(entry.start) !== undefined ? { start: optionalNumber(entry.start), } : {}),
63
- ...(optionalNumber(entry.end) !== undefined ? { end: optionalNumber(entry.end), } : {}),
64
- ...(optionalRecord(entry.scenario) !== undefined
65
- ? { scenario: optionalRecord(entry.scenario), }
66
- : {}),
67
- ...(optionalRecord(entry.variables) !== undefined
68
- ? { variables: optionalRecord(entry.variables), }
69
- : {}),
70
- ...(optionalRecord(entry.result) !== undefined
71
- ? { result: optionalRecord(entry.result), }
72
- : {}),
62
+ ...(start !== undefined ? { start, } : {}),
63
+ ...(end !== undefined ? { end, } : {}),
64
+ ...(isRecord(entry.scenario) ? { scenario: entry.scenario, } : {}),
65
+ ...(isRecord(entry.variables) ? { variables: entry.variables, } : {}),
66
+ ...(isRecord(entry.result) ? { result: entry.result, } : {}),
73
67
  };
74
68
  }
75
69
  /** Map a light-endpoint response to the documented ScenarioWithStatus shape. */
@@ -422,7 +416,7 @@ export class ScenariosResource extends BaseResource {
422
416
  async runScript(script, opts) {
423
417
  const pk = this.resolveProjectKey(opts?.projectKey);
424
418
  const pkEnc = this.enc(opts?.projectKey);
425
- const scenarioId = `dss_cli_code_run_${Date.now()}_${randomUUID().replace(/-/g, "")}`;
419
+ const scenarioId = `dss_cli_code_run_${Date.now()}_${crypto.randomUUID().replace(/-/g, "")}`;
426
420
  const base = `/public/api/projects/${pkEnc}/scenarios/${encodeURIComponent(scenarioId)}`;
427
421
  const envSelection = opts?.envName
428
422
  ? { envMode: "EXPLICIT_ENV", envName: opts.envName, }
@@ -38,6 +38,18 @@ export declare class StreamingEndpointsResource extends BaseResource {
38
38
  getSettings(id: string, projectKey?: string): Promise<StreamingEndpointSettings>;
39
39
  /** Create a streaming endpoint with type-specific params. */
40
40
  create(id: string, type: string, body: StreamingEndpointCreateParams, projectKey?: string): Promise<StreamingEndpointSettings>;
41
+ /**
42
+ * Create a managed streaming endpoint (POST /streamingendpoints/managed):
43
+ * DSS picks type and params from the connection and format option.
44
+ */
45
+ createManaged(id: string, creationSettings: {
46
+ connectionId: string;
47
+ formatOptionId?: string;
48
+ }, projectKey?: string): Promise<void>;
49
+ /** Get a streaming endpoint's schema (`{columns: [{name, type}]}`). */
50
+ getSchema(id: string, projectKey?: string): Promise<Record<string, unknown>>;
51
+ /** Replace a streaming endpoint's schema with an object obtained from `getSchema`, edited. */
52
+ setSchema(id: string, schema: Record<string, unknown>, projectKey?: string): Promise<void>;
41
53
  /** Update one streaming endpoint's raw settings. */
42
54
  updateSettings(id: string, body: Record<string, unknown>, projectKey?: string): Promise<void>;
43
55
  /** Delete a streaming endpoint from the flow without deleting underlying streaming data. */
@@ -23,6 +23,21 @@ export class StreamingEndpointsResource extends BaseResource {
23
23
  params: body,
24
24
  });
25
25
  }
26
+ /**
27
+ * Create a managed streaming endpoint (POST /streamingendpoints/managed):
28
+ * DSS picks type and params from the connection and format option.
29
+ */
30
+ async createManaged(id, creationSettings, projectKey) {
31
+ await this.client.post(`/public/api/projects/${this.enc(projectKey)}/streamingendpoints/managed`, { id, creationSettings, });
32
+ }
33
+ /** Get a streaming endpoint's schema (`{columns: [{name, type}]}`). */
34
+ async getSchema(id, projectKey) {
35
+ return this.client.get(`/public/api/projects/${this.enc(projectKey)}/streamingendpoints/${encodeURIComponent(id)}/schema`);
36
+ }
37
+ /** Replace a streaming endpoint's schema with an object obtained from `getSchema`, edited. */
38
+ async setSchema(id, schema, projectKey) {
39
+ await this.client.putVoid(`/public/api/projects/${this.enc(projectKey)}/streamingendpoints/${encodeURIComponent(id)}/schema`, schema);
40
+ }
26
41
  /** Update one streaming endpoint's raw settings. */
27
42
  async updateSettings(id, body, projectKey) {
28
43
  const endpointId = encodeURIComponent(id);