dataiku-sdk 3.6.1 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +14 -6
  2. package/bin/bun-version.js +14 -0
  3. package/bin/dss.js +11 -15
  4. package/dist/build-metadata.json +1 -1
  5. package/dist/packages/types/src/index.d.ts +114 -5
  6. package/dist/packages/types/src/index.js +75 -2
  7. package/dist/src/cli/coerce.d.ts +2 -0
  8. package/dist/src/cli/coerce.js +20 -16
  9. package/dist/src/cli/command-syntax.json +3369 -144
  10. package/dist/src/cli/commands/analysis.js +19 -1
  11. package/dist/src/cli/commands/bundle.js +22 -0
  12. package/dist/src/cli/commands/connection.js +26 -2
  13. package/dist/src/cli/commands/data-quality.js +33 -1
  14. package/dist/src/cli/commands/dataset.js +76 -9
  15. package/dist/src/cli/commands/discussion.js +15 -0
  16. package/dist/src/cli/commands/doctor.js +1 -1
  17. package/dist/src/cli/commands/flow-zone.js +241 -46
  18. package/dist/src/cli/commands/flow.d.ts +5 -0
  19. package/dist/src/cli/commands/flow.js +57 -0
  20. package/dist/src/cli/commands/future.js +8 -0
  21. package/dist/src/cli/commands/index.js +6 -0
  22. package/dist/src/cli/commands/job.js +44 -6
  23. package/dist/src/cli/commands/metrics.js +21 -1
  24. package/dist/src/cli/commands/ml-task.d.ts +5 -0
  25. package/dist/src/cli/commands/ml-task.js +86 -16
  26. package/dist/src/cli/commands/mlflow.d.ts +11 -0
  27. package/dist/src/cli/commands/mlflow.js +121 -0
  28. package/dist/src/cli/commands/plugin.js +5 -11
  29. package/dist/src/cli/commands/project-deployer.js +70 -0
  30. package/dist/src/cli/commands/project-library.js +2 -5
  31. package/dist/src/cli/commands/project.js +8 -0
  32. package/dist/src/cli/commands/recipe.d.ts +13 -0
  33. package/dist/src/cli/commands/recipe.js +130 -7
  34. package/dist/src/cli/commands/saved-model.d.ts +1 -1
  35. package/dist/src/cli/commands/saved-model.js +9 -76
  36. package/dist/src/cli/commands/streaming-endpoint.js +30 -1
  37. package/dist/src/cli/commands/webapp.js +8 -0
  38. package/dist/src/cli/commands/wiki.js +78 -0
  39. package/dist/src/cli/commands/workspace.js +8 -0
  40. package/dist/src/cli/contract.d.ts +16 -15
  41. package/dist/src/cli/contract.js +150 -3407
  42. package/dist/src/cli/doctor.d.ts +13 -0
  43. package/dist/src/cli/doctor.js +43 -1
  44. package/dist/src/cli/flags.d.ts +14 -1
  45. package/dist/src/cli/flags.js +106 -8
  46. package/dist/src/cli/helpers/app-successor.js +1 -2
  47. package/dist/src/cli/helpers/dataset.d.ts +12 -0
  48. package/dist/src/cli/helpers/dataset.js +36 -0
  49. package/dist/src/cli/helpers/documentation.d.ts +21 -0
  50. package/dist/src/cli/helpers/documentation.js +53 -0
  51. package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
  52. package/dist/src/cli/helpers/flow-zone.js +144 -4
  53. package/dist/src/cli/helpers/trained-model.d.ts +26 -0
  54. package/dist/src/cli/helpers/trained-model.js +144 -0
  55. package/dist/src/cli/list-output.d.ts +7 -0
  56. package/dist/src/cli/list-output.js +77 -0
  57. package/dist/src/cli/list-shapes.d.ts +21 -0
  58. package/dist/src/cli/list-shapes.js +111 -0
  59. package/dist/src/cli/output-schemas.d.ts +0 -14
  60. package/dist/src/cli/output-schemas.js +4 -7
  61. package/dist/src/cli/output.d.ts +0 -1
  62. package/dist/src/cli/output.js +33 -19
  63. package/dist/src/cli/plans.d.ts +14 -0
  64. package/dist/src/cli/plans.js +3740 -0
  65. package/dist/src/cli/types.d.ts +2 -0
  66. package/dist/src/cli/usage.d.ts +7 -0
  67. package/dist/src/cli/usage.js +20 -1
  68. package/dist/src/cli/version.d.ts +2 -2
  69. package/dist/src/cli/version.js +2 -2
  70. package/dist/src/cli.js +137 -115
  71. package/dist/src/client.d.ts +52 -43
  72. package/dist/src/client.js +267 -96
  73. package/dist/src/errors.d.ts +22 -2
  74. package/dist/src/errors.js +49 -50
  75. package/dist/src/generated/action-output-schemas.json +345 -89
  76. package/dist/src/index.d.ts +5 -1
  77. package/dist/src/index.js +3 -0
  78. package/dist/src/resources/analyses.d.ts +8 -0
  79. package/dist/src/resources/analyses.js +13 -1
  80. package/dist/src/resources/bundles.d.ts +34 -0
  81. package/dist/src/resources/bundles.js +67 -0
  82. package/dist/src/resources/connections.d.ts +9 -0
  83. package/dist/src/resources/connections.js +15 -0
  84. package/dist/src/resources/data-quality.d.ts +20 -0
  85. package/dist/src/resources/data-quality.js +30 -0
  86. package/dist/src/resources/dataset-create.d.ts +38 -0
  87. package/dist/src/resources/dataset-create.js +135 -0
  88. package/dist/src/resources/datasets.d.ts +46 -15
  89. package/dist/src/resources/datasets.js +120 -112
  90. package/dist/src/resources/discussions.d.ts +5 -0
  91. package/dist/src/resources/discussions.js +7 -0
  92. package/dist/src/resources/documentation.d.ts +25 -0
  93. package/dist/src/resources/documentation.js +16 -0
  94. package/dist/src/resources/flow-zones.d.ts +29 -3
  95. package/dist/src/resources/flow-zones.js +53 -3
  96. package/dist/src/resources/flow.d.ts +48 -0
  97. package/dist/src/resources/flow.js +65 -0
  98. package/dist/src/resources/futures.d.ts +23 -0
  99. package/dist/src/resources/futures.js +32 -0
  100. package/dist/src/resources/jobs.d.ts +34 -0
  101. package/dist/src/resources/jobs.js +170 -3
  102. package/dist/src/resources/metrics.d.ts +10 -0
  103. package/dist/src/resources/metrics.js +10 -0
  104. package/dist/src/resources/ml-tasks.d.ts +65 -2
  105. package/dist/src/resources/ml-tasks.js +102 -28
  106. package/dist/src/resources/mlflow-extension.d.ts +73 -0
  107. package/dist/src/resources/mlflow-extension.js +105 -0
  108. package/dist/src/resources/projects.d.ts +15 -0
  109. package/dist/src/resources/projects.js +18 -3
  110. package/dist/src/resources/recipe-create.d.ts +79 -0
  111. package/dist/src/resources/recipe-create.js +454 -0
  112. package/dist/src/resources/recipes.d.ts +28 -29
  113. package/dist/src/resources/recipes.js +201 -267
  114. package/dist/src/resources/saved-models.d.ts +10 -6
  115. package/dist/src/resources/saved-models.js +16 -14
  116. package/dist/src/resources/scenarios.js +8 -14
  117. package/dist/src/resources/sql.js +52 -13
  118. package/dist/src/resources/streaming-endpoints.d.ts +12 -0
  119. package/dist/src/resources/streaming-endpoints.js +15 -0
  120. package/dist/src/resources/trained-model.d.ts +60 -0
  121. package/dist/src/resources/trained-model.js +98 -0
  122. package/dist/src/resources/webapps.d.ts +8 -0
  123. package/dist/src/resources/webapps.js +10 -0
  124. package/dist/src/resources/wiki.d.ts +52 -1
  125. package/dist/src/resources/wiki.js +96 -0
  126. package/dist/src/resources/workspaces.d.ts +2 -0
  127. package/dist/src/resources/workspaces.js +4 -0
  128. package/dist/src/utils/flow-analysis.js +4 -0
  129. package/dist/src/utils/response-file.js +20 -9
  130. package/dist/src/utils/secret-sanitize.d.ts +7 -0
  131. package/dist/src/utils/secret-sanitize.js +46 -0
  132. package/package.json +25 -26
  133. package/plugin.json +1 -1
  134. package/skills/dataiku-dss/SKILL.md +1 -1
  135. package/skills/dataiku-dss/references/authentication.md +3 -1
  136. package/skills/dataiku-dss/references/coding.md +1 -0
  137. package/skills/dataiku-dss/references/discovery.md +4 -6
  138. package/skills/dataiku-dss/references/flow-maps.md +2 -2
  139. package/skills/dataiku-dss/references/troubleshooting.md +3 -2
@@ -1,15 +1,14 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as nodePath from "node:path";
3
- import { Readable, Transform, } from "node:stream";
4
- import { pipeline, } from "node:stream/promises";
5
3
  import { StringDecoder, } from "node:string_decoder";
6
- import { createGzip, } from "node:zlib";
7
4
  import { ClientValidationError, DataikuError, } from "../errors.js";
8
5
  import { DatasetDetailsSchema, DatasetSchemaSchema, DatasetSummaryArraySchema, } from "../schemas.js";
9
6
  import { deepMerge, } from "../utils/deep-merge.js";
7
+ import { isRecord, } from "../utils/records.js";
10
8
  import { sanitizeFileName, } from "../utils/sanitize.js";
11
9
  import { BaseResource, } from "./base.js";
12
10
  import { resolveAdminManagedStorageConnection, } from "./connections.js";
11
+ import { buildDatasetCreateBody, } from "./dataset-create.js";
13
12
  // ---------------------------------------------------------------------------
14
13
  // Helpers: TSV → CSV streaming conversion
15
14
  // ---------------------------------------------------------------------------
@@ -404,13 +403,14 @@ const CSV_BATCH_BYTES = 64 * 1024;
404
403
  function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
405
404
  const state = createTsvStreamState();
406
405
  const decoder = new StringDecoder("utf-8");
406
+ const encoder = new TextEncoder();
407
407
  const maxRows = Math.max(1, maxDataRows);
408
408
  let headerSeen = false;
409
409
  let done = false;
410
410
  // Bounded CSV output batching: emitted rows accumulate here and are joined
411
- // into a single push once the batch reaches CSV_BATCH_BYTES, so memory stays
412
- // bounded by the batch budget plus one escaped row (the join briefly holds a
413
- // second copy of the batch) while per-row push/callback overhead disappears.
411
+ // into a single enqueue once the batch reaches CSV_BATCH_BYTES, so memory
412
+ // stays bounded by the batch budget plus one escaped row (the join briefly
413
+ // holds a second copy of the batch) while per-row chunk overhead disappears.
414
414
  let batchLines = [];
415
415
  let batchBytes = 0;
416
416
  const flushBatch = (target) => {
@@ -419,7 +419,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
419
419
  const csv = batchLines.join("");
420
420
  batchLines = [];
421
421
  batchBytes = 0;
422
- target.push(csv);
422
+ target.enqueue(encoder.encode(csv));
423
423
  };
424
424
  const queueLine = (target, line) => {
425
425
  batchLines.push(line);
@@ -450,48 +450,27 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
450
450
  push(`${rowToCsv(row, spreadsheetSafe)}\n`);
451
451
  stats.rows += 1;
452
452
  };
453
- return new Transform({
454
- transform(chunk, _encoding, callback) {
453
+ // A byte-limit overflow throws out of transform/flush: the stream errors,
454
+ // which cancels the response and makes download() remove the partial file.
455
+ return new TransformStream({
456
+ transform(chunk, controller) {
457
+ consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
455
458
  if (done) {
456
- callback();
457
- return;
458
- }
459
- try {
460
- consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(this, line)));
461
- }
462
- catch (error) {
463
- // Byte-limit overflow: destroy the transform so the pipeline
464
- // cancels the response and the partial file is removed.
465
- callback(error instanceof Error ? error : new ClientValidationError(String(error), "internal_error"));
466
- return;
459
+ // Truncation reached: emit the batch queued so far, end the output,
460
+ // and cancel the rest of the response.
461
+ flushBatch(controller);
462
+ controller.terminate();
467
463
  }
468
- if (done) {
469
- // Truncation reached: emit the batch queued so far before EOF.
470
- flushBatch(this);
471
- this.push(null);
472
- }
473
- callback();
474
464
  },
475
- flush(callback) {
476
- if (done) {
477
- callback();
478
- return;
479
- }
480
- try {
481
- const tail = decoder.end();
482
- if (tail !== "") {
483
- consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(this, line)));
484
- }
485
- flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(this, line)));
486
- }
487
- catch (error) {
488
- callback(error instanceof Error ? error : new ClientValidationError(String(error), "internal_error"));
489
- return;
465
+ flush(controller) {
466
+ const tail = decoder.end();
467
+ if (tail !== "") {
468
+ consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(controller, line)));
490
469
  }
470
+ flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(controller, line)));
491
471
  // Emit the trailing partial batch; without this the last rows of a
492
472
  // download (below the batch budget) would be silently dropped.
493
- flushBatch(this);
494
- callback();
473
+ flushBatch(controller);
495
474
  },
496
475
  });
497
476
  }
@@ -518,60 +497,6 @@ function isMissingUploadedFilesTargetConnection(error) {
518
497
  return (detail.includes("without a target connection")
519
498
  || detail.includes("target connection is required"));
520
499
  }
521
- /** Wire body for POST /datasets/; shared with `dataset create --plan` so the plan matches the request. */
522
- export function buildDatasetCreateBody(opts) {
523
- if (opts.dsType.toLowerCase() === "uploadedfiles") {
524
- return {
525
- projectKey: opts.projectKey,
526
- name: opts.datasetName,
527
- type: opts.dsType,
528
- params: opts.connection ? { uploadConnection: opts.connection, } : {},
529
- };
530
- }
531
- if (!opts.connection) {
532
- throw new ClientValidationError("connection is required unless dsType is UploadedFiles.");
533
- }
534
- if (opts.table) {
535
- const params = {
536
- connection: opts.connection,
537
- mode: "table",
538
- table: opts.table,
539
- };
540
- if (opts.dbSchema)
541
- params.schema = opts.dbSchema;
542
- if (opts.catalog)
543
- params.catalog = opts.catalog;
544
- return {
545
- projectKey: opts.projectKey,
546
- name: opts.datasetName,
547
- type: opts.dsType,
548
- params,
549
- managed: opts.managed ?? false,
550
- };
551
- }
552
- return {
553
- projectKey: opts.projectKey,
554
- name: opts.datasetName,
555
- type: opts.dsType,
556
- params: {
557
- connection: opts.connection,
558
- path: `/dataiku/${opts.projectKey}/${opts.datasetName}`,
559
- },
560
- formatType: opts.formatType ?? "csv",
561
- formatParams: opts.formatParams ?? {
562
- style: "excel",
563
- charset: "utf8",
564
- separator: "\t",
565
- quoteChar: '"',
566
- escapeChar: "\\",
567
- dateSerializationFormat: "ISO",
568
- arrayMapFormat: "json",
569
- parseHeaderRow: true,
570
- compress: "gz",
571
- },
572
- managed: opts.managed ?? true,
573
- };
574
- }
575
500
  const DATASET_CLONE_PARAM_KEYS = [
576
501
  "connection",
577
502
  "uploadConnection",
@@ -595,6 +520,28 @@ function cloneDatasetParams(params) {
595
520
  }
596
521
  return cloned;
597
522
  }
523
+ /** Explain an empty detection result from the first error DSS put in the response body. */
524
+ function describeSqlDetectionFailure(raw) {
525
+ if (!isRecord(raw))
526
+ return "DSS returned no schema detection result.";
527
+ for (const [key, value,] of Object.entries(raw)) {
528
+ if (!key.endsWith("Error"))
529
+ continue;
530
+ if (typeof value === "string" && value !== "")
531
+ return value;
532
+ if (isRecord(value)) {
533
+ const message = value["detailedMessage"] ?? value["message"];
534
+ if (typeof message === "string" && message !== "")
535
+ return message;
536
+ }
537
+ }
538
+ const failedChecks = Object.entries(raw)
539
+ .filter(([key, value,]) => key.endsWith("OK") && value === false)
540
+ .map(([key,]) => key);
541
+ return failedChecks.length > 0
542
+ ? `DSS detected no columns (failed checks: ${failedChecks.join(", ")}).`
543
+ : "DSS detected no columns.";
544
+ }
598
545
  function parseUploadedFiles(raw, datasetName) {
599
546
  if (!Array.isArray(raw)) {
600
547
  throw new ClientValidationError(`DSS returned an invalid uploaded-files list for dataset ${datasetName}.`, "validation_failed", "Expected an array from the UploadedFiles dataset file endpoint.");
@@ -668,6 +615,40 @@ export class DatasetsResource extends BaseResource {
668
615
  const dsEnc = encodeURIComponent(datasetName);
669
616
  await this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`, { columns, });
670
617
  }
618
+ /**
619
+ * Detect the schema of a SQL dataset (table or query mode) with DSS and
620
+ * store it: `POST /actions/testAndDetectSettings/externalSQL` (what
621
+ * dataikuapi's `autodetect_settings` calls), then `PUT /schema` with the
622
+ * suggested schema. DSS reports connection, table, and query problems in the
623
+ * response body (HTTP 200), so a failed detection is returned with
624
+ * `detected: false` and the reason instead of thrown.
625
+ */
626
+ async detectSqlSchema(datasetName, projectKey) {
627
+ const base = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}`;
628
+ try {
629
+ const raw = await this.client.post(`${base}/actions/testAndDetectSettings/externalSQL`);
630
+ const detection = isRecord(raw) ? raw["schemaDetection"] : undefined;
631
+ const newSchema = isRecord(detection) ? detection["newSchema"] : undefined;
632
+ const rawColumns = isRecord(newSchema) && Array.isArray(newSchema["columns"])
633
+ ? newSchema["columns"]
634
+ : [];
635
+ const columns = rawColumns.flatMap((column) => isRecord(column) && typeof column["name"] === "string"
636
+ && typeof column["type"] === "string"
637
+ ? [{ name: column["name"], type: column["type"], },]
638
+ : []);
639
+ if (!isRecord(newSchema) || columns.length === 0) {
640
+ return { detected: false, columns: [], error: describeSqlDetectionFailure(raw), };
641
+ }
642
+ await this.client.put(`${base}/schema`, newSchema);
643
+ return { detected: true, columns, };
644
+ }
645
+ catch (error) {
646
+ if (error instanceof DataikuError) {
647
+ return { detected: false, columns: [], error: error.message.split("\n")[0] ?? "", };
648
+ }
649
+ throw error;
650
+ }
651
+ }
671
652
  /**
672
653
  * Get the raw dataset schema object exactly as the DSS schema endpoint
673
654
  * returns it, without schema validation — used for deterministic comparison.
@@ -676,6 +657,23 @@ export class DatasetsResource extends BaseResource {
676
657
  const dsEnc = encodeURIComponent(datasetName);
677
658
  return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`);
678
659
  }
660
+ /**
661
+ * TSV data stream of at most `limit` rows. A column or partition selection
662
+ * uses the documented POST variant of the data endpoint (body `columns`,
663
+ * `partitions`, `format`, `sampling`); otherwise the GET endpoint.
664
+ */
665
+ async dataStream(datasetName, limit, projectKey, selection) {
666
+ const path = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/data/`;
667
+ if (!selection?.columns?.length && !selection?.partitions) {
668
+ return this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}`);
669
+ }
670
+ return this.client.postStream(path, {
671
+ format: "tsv-excel-header",
672
+ ...(selection.columns?.length ? { columns: selection.columns, } : {}),
673
+ ...(selection.partitions ? { partitions: selection.partitions, } : {}),
674
+ sampling: { samplingMethod: "HEAD_SEQUENTIAL", maxRecords: limit, },
675
+ });
676
+ }
679
677
  /**
680
678
  * Preview dataset rows as structured data: column names plus row arrays,
681
679
  * mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
@@ -691,10 +689,9 @@ export class DatasetsResource extends BaseResource {
691
689
  async preview(datasetName, opts) {
692
690
  const maxRows = Math.max(1, Math.min(opts?.maxRows ?? 50, 500));
693
691
  const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
694
- const dsEnc = encodeURIComponent(datasetName);
695
692
  // Probe one row past the cap so callers learn whether the dataset holds
696
693
  // more rows than requested without materializing the full dataset.
697
- const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${maxRows + 1}`);
694
+ const res = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
698
695
  const onHeader = opts?.validateColumns
699
696
  ? (headerRow) => {
700
697
  const warnings = validateStreamColumns(headerRow, opts.validateColumns);
@@ -761,6 +758,20 @@ export class DatasetsResource extends BaseResource {
761
758
  const dsEnc = encodeURIComponent(datasetName);
762
759
  return this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/metadata`, metadata);
763
760
  }
761
+ /**
762
+ * Create or update the Hive table of an HDFS dataset so it matches the
763
+ * dataset schema (POST /actions/synchronizeHiveMetastore).
764
+ */
765
+ async synchronizeHiveMetastore(datasetName, projectKey) {
766
+ await this.client.post(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/actions/synchronizeHiveMetastore`);
767
+ }
768
+ /**
769
+ * Update an HDFS dataset (path, schema, partitioning, format) from its
770
+ * Hive table (POST /actions/updateFromHive).
771
+ */
772
+ async updateFromHive(datasetName, projectKey) {
773
+ await this.client.post(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/actions/updateFromHive`);
774
+ }
764
775
  /**
765
776
  * Create a managed dataset through the dedicated
766
777
  * `POST /projects/{projectKey}/datasets/managed` endpoint. DSS handles the
@@ -862,8 +873,7 @@ export class DatasetsResource extends BaseResource {
862
873
  */
863
874
  async download(datasetName, opts) {
864
875
  const limit = Math.max(1, opts?.limit ?? 100_000);
865
- const dsEnc = encodeURIComponent(datasetName);
866
- const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${limit + 1}`);
876
+ const res = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
867
877
  const safeDatasetName = sanitizeFileName(datasetName, "dataset");
868
878
  const filePath = opts?.outputPath?.endsWith(".gz") || opts?.outputPath?.endsWith(".csv")
869
879
  ? nodePath.resolve(opts.outputPath)
@@ -878,18 +888,13 @@ export class DatasetsResource extends BaseResource {
878
888
  : undefined;
879
889
  const shouldGzip = filePath.endsWith(".gz");
880
890
  const stats = { rows: 0, truncated: false, };
881
- const nodeStream = Readable.from(res.body, { objectMode: false, });
882
- const csvTransform = tsvToCsvTransform(limit, stats, onHeader, opts?.rawData !== true);
891
+ let csv = res.body.pipeThrough(tsvToCsvTransform(limit, stats, onHeader, opts?.rawData !== true));
892
+ if (shouldGzip)
893
+ csv = csv.pipeThrough(new CompressionStream("gzip"));
883
894
  fs.mkdirSync(nodePath.dirname(filePath), { recursive: true, });
884
- const fileOut = fs.createWriteStream(filePath);
885
895
  try {
886
- if (shouldGzip) {
887
- const gzip = createGzip();
888
- await pipeline(nodeStream, csvTransform, gzip, fileOut);
889
- }
890
- else {
891
- await pipeline(nodeStream, csvTransform, fileOut);
892
- }
896
+ // Native streaming write with backpressure; no Node stream layer.
897
+ await Bun.write(filePath, new Response(csv));
893
898
  }
894
899
  catch (error) {
895
900
  // A failed export (byte-limit overflow, network error) must never
@@ -911,7 +916,7 @@ export class DatasetsResource extends BaseResource {
911
916
  const enc = encodeURIComponent(pk);
912
917
  const explicitType = opts.dsType;
913
918
  let dsType = explicitType
914
- ?? (opts.table ? DEFAULT_DATABASE_DATASET_TYPE : DEFAULT_FILESYSTEM_DATASET_TYPE);
919
+ ?? (opts.table || opts.query ? DEFAULT_DATABASE_DATASET_TYPE : DEFAULT_FILESYSTEM_DATASET_TYPE);
915
920
  let body = buildDatasetCreateBody({
916
921
  projectKey: pk,
917
922
  datasetName: opts.datasetName,
@@ -920,6 +925,7 @@ export class DatasetsResource extends BaseResource {
920
925
  table: opts.table,
921
926
  dbSchema: opts.dbSchema,
922
927
  catalog: opts.catalog,
928
+ query: opts.query,
923
929
  formatType: opts.formatType,
924
930
  formatParams: opts.formatParams,
925
931
  managed: opts.managed,
@@ -951,6 +957,7 @@ export class DatasetsResource extends BaseResource {
951
957
  table: opts.table,
952
958
  dbSchema: opts.dbSchema,
953
959
  catalog: opts.catalog,
960
+ query: opts.query,
954
961
  formatType: opts.formatType,
955
962
  formatParams: opts.formatParams,
956
963
  managed: opts.managed,
@@ -975,6 +982,7 @@ export class DatasetsResource extends BaseResource {
975
982
  table: opts.table,
976
983
  dbSchema: opts.dbSchema,
977
984
  catalog: opts.catalog,
985
+ query: opts.query,
978
986
  formatType: opts.formatType,
979
987
  formatParams: opts.formatParams,
980
988
  managed: opts.managed,
@@ -21,6 +21,11 @@ export declare class DiscussionsResource extends BaseResource {
21
21
  create(objectType: string, objectId: string, topic: string, reply: string, projectKey?: string): Promise<Discussion>;
22
22
  /** Add a reply to an existing discussion. */
23
23
  reply(objectType: string, objectId: string, discussionId: string, text: string, projectKey?: string): Promise<Discussion>;
24
+ /**
25
+ * Update a discussion (topic, or the object it is attached to within the
26
+ * project): PUT the full discussion object obtained from `get`, edited.
27
+ */
28
+ update(objectType: string, objectId: string, discussionId: string, discussion: Record<string, unknown>, projectKey?: string): Promise<Discussion>;
24
29
  private objectDiscussionsPath;
25
30
  private discussionPath;
26
31
  }
@@ -16,6 +16,13 @@ export class DiscussionsResource extends BaseResource {
16
16
  async reply(objectType, objectId, discussionId, text, projectKey) {
17
17
  return this.client.post(`${this.discussionPath(objectType, objectId, discussionId, projectKey)}/replies/`, { reply: text, });
18
18
  }
19
+ /**
20
+ * Update a discussion (topic, or the object it is attached to within the
21
+ * project): PUT the full discussion object obtained from `get`, edited.
22
+ */
23
+ async update(objectType, objectId, discussionId, discussion, projectKey) {
24
+ return this.client.put(this.discussionPath(objectType, objectId, discussionId, projectKey), discussion);
25
+ }
19
26
  objectDiscussionsPath(objectType, objectId, projectKey) {
20
27
  const pk = this.enc(projectKey);
21
28
  const encodedType = encodeURIComponent(objectType);
@@ -0,0 +1,25 @@
1
+ import type { DataikuClient } from "../client.js";
2
+ import { type DssTask } from "./futures.js";
3
+ /** Documentation template: the DSS default, an uploaded docx, or a docx in a managed folder. */
4
+ export type DocumentationTemplate = {
5
+ kind: "default";
6
+ } | {
7
+ kind: "file";
8
+ filePath: string;
9
+ } | {
10
+ kind: "folder";
11
+ folderId: string;
12
+ path: string;
13
+ };
14
+ /** The three generation endpoints of one documented object (flow, lab model, saved-model version). */
15
+ export interface DocumentationRoutes {
16
+ default: string;
17
+ file: string;
18
+ folder: string;
19
+ }
20
+ /**
21
+ * Start a DSS documentation export (docx): POST the default route, upload the
22
+ * template as multipart `file`, or POST the folder route with `folderId` and
23
+ * `path`. The finished task's result carries the `exportId` to download.
24
+ */
25
+ export declare function startDocumentation(client: DataikuClient, routes: DocumentationRoutes, template: DocumentationTemplate, operation: string): Promise<DssTask>;
@@ -0,0 +1,16 @@
1
+ import { dssTask, } from "./futures.js";
2
+ /**
3
+ * Start a DSS documentation export (docx): POST the default route, upload the
4
+ * template as multipart `file`, or POST the folder route with `folderId` and
5
+ * `path`. The finished task's result carries the `exportId` to download.
6
+ */
7
+ export async function startDocumentation(client, routes, template, operation) {
8
+ if (template.kind === "file") {
9
+ return dssTask(await client.uploadJson(routes.file, template.filePath), operation);
10
+ }
11
+ if (template.kind === "folder") {
12
+ const query = new URLSearchParams({ folderId: template.folderId, path: template.path, });
13
+ return dssTask(await client.post(`${routes.folder}?${query}`), operation);
14
+ }
15
+ return dssTask(await client.post(routes.default), operation);
16
+ }
@@ -1,11 +1,21 @@
1
1
  import type { DataikuGetOptions } from "../client.js";
2
- import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZoneUpdateOptions } from "../schemas.js";
2
+ import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZonePosition, FlowZoneUpdateOptions } from "../schemas.js";
3
3
  import { BaseResource } from "./base.js";
4
4
  export type FlowZoneItemInput = {
5
5
  objectId: string;
6
6
  objectType: FlowZoneObjectType;
7
7
  projectKey?: string;
8
8
  };
9
+ export type FlowZoneRecreateOptions = {
10
+ position: FlowZonePosition;
11
+ name?: string;
12
+ color?: string;
13
+ projectKey?: string;
14
+ };
15
+ export type FlowZoneRecreateResult = {
16
+ zone: FlowZone;
17
+ replacedZoneId: string;
18
+ };
9
19
  export declare class FlowZonesResource extends BaseResource {
10
20
  /**
11
21
  * List all flow zones in a project. Pass `timeoutMs` to bound the whole
@@ -14,10 +24,26 @@ export declare class FlowZonesResource extends BaseResource {
14
24
  list(projectKey?: string, options?: DataikuGetOptions): Promise<FlowZone[]>;
15
25
  /** Get one flow zone by id. */
16
26
  get(zoneId: string, projectKey?: string): Promise<FlowZone>;
17
- /** Create a flow zone. */
27
+ /**
28
+ * Create a flow zone. DSS stores `position` only while the project's manual
29
+ * zone positioning is off; with it on, DSS auto-places the zone.
30
+ */
18
31
  create(opts: FlowZoneCreateOptions): Promise<FlowZone>;
19
- /** Update flow zone settings such as name, color, and manual position. */
32
+ /**
33
+ * Update flow zone name and color. DSS's public API ignores `position` on
34
+ * this endpoint; use {@link recreate} to move an existing zone.
35
+ */
20
36
  update(zoneId: string, opts: FlowZoneUpdateOptions): Promise<FlowZone>;
37
+ /**
38
+ * Replace a zone with a new one at `position`: DSS honors a zone position
39
+ * only on creation, and only while the project's manual zone positioning is
40
+ * off (otherwise it auto-places the new zone). The replacement receives the
41
+ * old zone's name, color, metadata (tags, custom fields, checklists, ...),
42
+ * items, and shared items; the old zone is deleted once it is empty. The zone
43
+ * id changes, so anything referencing the old id (for example scenario zone
44
+ * builds) must be updated.
45
+ */
46
+ recreate(zoneId: string, opts: FlowZoneRecreateOptions): Promise<FlowZoneRecreateResult>;
21
47
  /** Delete a flow zone. DSS moves its items back to the default zone. */
22
48
  delete(zoneId: string, projectKey?: string): Promise<void>;
23
49
  /** Move items into a flow zone. */
@@ -23,7 +23,10 @@ export class FlowZonesResource extends BaseResource {
23
23
  const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`);
24
24
  return this.client.safeParse(FlowZoneSchema, raw, "flowZones.get");
25
25
  }
26
- /** Create a flow zone. */
26
+ /**
27
+ * Create a flow zone. DSS stores `position` only while the project's manual
28
+ * zone positioning is off; with it on, DSS auto-places the zone.
29
+ */
27
30
  async create(opts) {
28
31
  const raw = await this.client.post(`/public/api/projects/${this.enc(opts.projectKey)}/flow/zones`, {
29
32
  name: opts.name,
@@ -32,17 +35,64 @@ export class FlowZonesResource extends BaseResource {
32
35
  });
33
36
  return this.client.safeParse(FlowZoneSchema, raw, "flowZones.create");
34
37
  }
35
- /** Update flow zone settings such as name, color, and manual position. */
38
+ /**
39
+ * Update flow zone name and color. DSS's public API ignores `position` on
40
+ * this endpoint; use {@link recreate} to move an existing zone.
41
+ */
36
42
  async update(zoneId, opts) {
37
43
  const current = await this.get(zoneId, opts.projectKey);
38
44
  const merged = deepMerge(current, {
39
45
  ...(opts.name !== undefined ? { name: opts.name, } : {}),
40
46
  ...(opts.color !== undefined ? { color: opts.color, } : {}),
41
- ...(opts.position !== undefined ? { position: opts.position, } : {}),
42
47
  });
43
48
  await this.client.putVoid(`/public/api/projects/${this.enc(opts.projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`, merged);
44
49
  return this.get(zoneId, opts.projectKey);
45
50
  }
51
+ /**
52
+ * Replace a zone with a new one at `position`: DSS honors a zone position
53
+ * only on creation, and only while the project's manual zone positioning is
54
+ * off (otherwise it auto-places the new zone). The replacement receives the
55
+ * old zone's name, color, metadata (tags, custom fields, checklists, ...),
56
+ * items, and shared items; the old zone is deleted once it is empty. The zone
57
+ * id changes, so anything referencing the old id (for example scenario zone
58
+ * builds) must be updated.
59
+ */
60
+ async recreate(zoneId, opts) {
61
+ const base = `/public/api/projects/${this.enc(opts.projectKey)}/flow/zones`;
62
+ const old = await this.get(zoneId, opts.projectKey);
63
+ const created = await this.create({
64
+ name: opts.name ?? old.name,
65
+ color: opts.color ?? old.color,
66
+ position: opts.position,
67
+ projectKey: opts.projectKey,
68
+ });
69
+ const completed = ["create",];
70
+ try {
71
+ const { id: _id, items, shared, position: _position, ...metadata } = old;
72
+ await this.client.putVoid(`${base}/${encodeURIComponent(created.id)}`, {
73
+ ...created,
74
+ ...metadata,
75
+ id: created.id,
76
+ name: created.name,
77
+ color: created.color,
78
+ position: created.position,
79
+ });
80
+ completed.push("metadata");
81
+ if (items && items.length > 0) {
82
+ await this.moveItems(created.id, items, opts.projectKey);
83
+ }
84
+ completed.push("items");
85
+ for (const item of shared ?? []) {
86
+ await this.client.post(`${base}/${encodeURIComponent(created.id)}/shared`, normalizeZoneItem(item));
87
+ }
88
+ completed.push("shared");
89
+ await this.delete(old.id, opts.projectKey);
90
+ }
91
+ catch (error) {
92
+ throw new ClientValidationError(`Recreating flow zone ${old.id} at a new position stopped after ${completed.at(-1)}; both zones may now exist.`, "ambiguous_outcome", "Inspect both zones with dss flow-zone list, then move remaining items and delete the zone you no longer need.", { replacedZoneId: old.id, zoneId: created.id, completed, }, { cause: error, });
93
+ }
94
+ return { zone: await this.get(created.id, opts.projectKey), replacedZoneId: old.id, };
95
+ }
46
96
  /** Delete a flow zone. DSS moves its items back to the default zone. */
47
97
  async delete(zoneId, projectKey) {
48
98
  await this.client.del(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`);
@@ -0,0 +1,48 @@
1
+ import type { FutureWaitResult } from "../schemas.js";
2
+ import { BaseResource } from "./base.js";
3
+ import { type DocumentationTemplate } from "./documentation.js";
4
+ import { type DssTask, type FutureWaitOptions } from "./futures.js";
5
+ /** Schema propagation options (dataikuapi `DSSSchemaPropagationRunBuilder` settings). */
6
+ export interface SchemaPropagationOptions {
7
+ projectKey?: string;
8
+ /** Rebuild datasets when propagation needs their data (DSS default: true). */
9
+ autoRebuild?: boolean;
10
+ /** Recipes where propagation stops (dataikuapi `stop_at`). */
11
+ excludedRecipes?: string[];
12
+ /** Recipes always considered OK (dataikuapi `mark_recipe_as_ok`). */
13
+ markAsOkRecipes?: string[];
14
+ /** Partitioned flows: default partition value per dimension when rebuilding. */
15
+ defaultPartitionValuesByDimension?: Record<string, string>;
16
+ /** Partitioned flows: partition per computable, keyed `PROJECTKEY.id`. */
17
+ partitionsByComputable?: Record<string, string>;
18
+ /** Per-recipe-type and per-recipe update options, as dataikuapi sends them. */
19
+ recipeUpdateOptions?: {
20
+ byType?: Record<string, unknown>;
21
+ byName?: Record<string, unknown>;
22
+ };
23
+ }
24
+ /**
25
+ * Flow-wide tools: schema propagation from a dataset through every downstream
26
+ * recipe, and flow documentation export. Zones and the flow graph live on
27
+ * `flowZones`; per-recipe schema updates on `recipes.updateSchema`.
28
+ */
29
+ export declare class FlowResource extends BaseResource {
30
+ private flowPath;
31
+ /**
32
+ * Start propagating `datasetName`'s schema through the flow
33
+ * (POST /flow/tools/propagate-schema/, dataikuapi
34
+ * `new_schema_propagation(...).start()`). Returns the DSS future.
35
+ */
36
+ propagateSchema(datasetName: string, opts?: SchemaPropagationOptions): Promise<DssTask>;
37
+ propagateSchemaAndWait(datasetName: string, opts?: SchemaPropagationOptions & FutureWaitOptions): Promise<FutureWaitResult | DssTask>;
38
+ /**
39
+ * Start generating the flow documentation (docx) from the DSS default
40
+ * template, an uploaded docx template, or a docx in a managed folder.
41
+ * The finished future's result carries the `exportId` to download.
42
+ */
43
+ generateDocumentation(template?: DocumentationTemplate, projectKey?: string): Promise<DssTask>;
44
+ /** GET a generated flow documentation (docx) as a binary stream. */
45
+ downloadDocumentation(exportId: string, projectKey?: string): Promise<Response>;
46
+ }
47
+ /** Request body of POST /flow/tools/propagate-schema/ (pure; shared with `--plan`). */
48
+ export declare function schemaPropagationBody(projectKey: string, datasetName: string, opts: SchemaPropagationOptions): Record<string, unknown>;