dataiku-sdk 3.2.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +151 -46
  2. package/dist/build-metadata.json +1 -1
  3. package/dist/packages/types/src/index.d.ts +102 -0
  4. package/dist/packages/types/src/index.js +76 -0
  5. package/dist/src/cli/commands/app.js +57 -15
  6. package/dist/src/cli/commands/bundle.js +1 -1
  7. package/dist/src/cli/commands/code-env.js +13 -7
  8. package/dist/src/cli/commands/code.js +1 -1
  9. package/dist/src/cli/commands/connection.d.ts +9 -0
  10. package/dist/src/cli/commands/connection.js +267 -1
  11. package/dist/src/cli/commands/data-collection.d.ts +2 -0
  12. package/dist/src/cli/commands/data-collection.js +164 -0
  13. package/dist/src/cli/commands/dataset.js +162 -33
  14. package/dist/src/cli/commands/future.js +2 -2
  15. package/dist/src/cli/commands/group.d.ts +2 -0
  16. package/dist/src/cli/commands/group.js +109 -0
  17. package/dist/src/cli/commands/index.js +151 -78
  18. package/dist/src/cli/commands/job.js +4 -2
  19. package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
  20. package/dist/src/cli/commands/knowledge-bank.js +80 -0
  21. package/dist/src/cli/commands/llm.d.ts +2 -0
  22. package/dist/src/cli/commands/llm.js +93 -0
  23. package/dist/src/cli/commands/macro.d.ts +2 -0
  24. package/dist/src/cli/commands/macro.js +221 -0
  25. package/dist/src/cli/commands/ml-task.js +18 -6
  26. package/dist/src/cli/commands/notebook.js +6 -2
  27. package/dist/src/cli/commands/plugin.d.ts +2 -0
  28. package/dist/src/cli/commands/plugin.js +799 -0
  29. package/dist/src/cli/commands/project-folder.d.ts +2 -0
  30. package/dist/src/cli/commands/project-folder.js +197 -0
  31. package/dist/src/cli/commands/project-git.js +15 -55
  32. package/dist/src/cli/commands/project.js +52 -6
  33. package/dist/src/cli/commands/recipe.js +36 -1
  34. package/dist/src/cli/commands/saved-model.d.ts +5 -0
  35. package/dist/src/cli/commands/saved-model.js +388 -1
  36. package/dist/src/cli/commands/scenario.js +139 -2
  37. package/dist/src/cli/commands/user.d.ts +8 -0
  38. package/dist/src/cli/commands/user.js +296 -0
  39. package/dist/src/cli/contract.d.ts +1 -2
  40. package/dist/src/cli/contract.js +1098 -297
  41. package/dist/src/cli/flags.js +30 -1
  42. package/dist/src/cli/helpers/app-successor.d.ts +19 -4
  43. package/dist/src/cli/helpers/app-successor.js +90 -30
  44. package/dist/src/cli/helpers/cleanup.js +3 -1
  45. package/dist/src/cli/helpers/flow-zone.js +1 -2
  46. package/dist/src/cli/output-schemas.d.ts +24 -0
  47. package/dist/src/cli/output-schemas.js +233 -0
  48. package/dist/src/cli/output.d.ts +1 -1
  49. package/dist/src/cli/output.js +23 -10
  50. package/dist/src/cli/usage.d.ts +1 -0
  51. package/dist/src/cli/usage.js +47 -1
  52. package/dist/src/cli.js +147 -41
  53. package/dist/src/client.d.ts +92 -5
  54. package/dist/src/client.js +202 -31
  55. package/dist/src/errors.d.ts +3 -0
  56. package/dist/src/errors.js +8 -4
  57. package/dist/src/generated/action-output-schemas.json +1735 -0
  58. package/dist/src/index.d.ts +16 -7
  59. package/dist/src/index.js +11 -2
  60. package/dist/src/resources/analyses.js +1 -8
  61. package/dist/src/resources/applications.d.ts +8 -6
  62. package/dist/src/resources/applications.js +21 -21
  63. package/dist/src/resources/base.d.ts +4 -0
  64. package/dist/src/resources/base.js +15 -0
  65. package/dist/src/resources/bundles.d.ts +9 -4
  66. package/dist/src/resources/bundles.js +10 -5
  67. package/dist/src/resources/connections.d.ts +78 -1
  68. package/dist/src/resources/connections.js +132 -1
  69. package/dist/src/resources/data-collections.d.ts +118 -0
  70. package/dist/src/resources/data-collections.js +49 -0
  71. package/dist/src/resources/datasets.d.ts +69 -0
  72. package/dist/src/resources/datasets.js +136 -5
  73. package/dist/src/resources/flow-zones.d.ts +6 -2
  74. package/dist/src/resources/flow-zones.js +6 -3
  75. package/dist/src/resources/futures.d.ts +6 -1
  76. package/dist/src/resources/futures.js +68 -22
  77. package/dist/src/resources/groups.d.ts +47 -0
  78. package/dist/src/resources/groups.js +56 -0
  79. package/dist/src/resources/jobs.d.ts +8 -13
  80. package/dist/src/resources/jobs.js +71 -34
  81. package/dist/src/resources/knowledge-banks.d.ts +58 -0
  82. package/dist/src/resources/knowledge-banks.js +61 -0
  83. package/dist/src/resources/llms.d.ts +92 -0
  84. package/dist/src/resources/llms.js +62 -0
  85. package/dist/src/resources/macros.d.ts +121 -0
  86. package/dist/src/resources/macros.js +231 -0
  87. package/dist/src/resources/ml-tasks.d.ts +37 -2
  88. package/dist/src/resources/ml-tasks.js +109 -20
  89. package/dist/src/resources/model-evaluation-stores.js +1 -8
  90. package/dist/src/resources/notebooks.d.ts +43 -7
  91. package/dist/src/resources/notebooks.js +53 -13
  92. package/dist/src/resources/plugins.d.ts +263 -0
  93. package/dist/src/resources/plugins.js +455 -0
  94. package/dist/src/resources/project-folders.d.ts +71 -0
  95. package/dist/src/resources/project-folders.js +57 -0
  96. package/dist/src/resources/project-git.d.ts +12 -12
  97. package/dist/src/resources/project-git.js +84 -82
  98. package/dist/src/resources/project-library.d.ts +0 -1
  99. package/dist/src/resources/project-library.js +1 -17
  100. package/dist/src/resources/projects.d.ts +23 -3
  101. package/dist/src/resources/projects.js +60 -34
  102. package/dist/src/resources/recipes.d.ts +15 -1
  103. package/dist/src/resources/recipes.js +73 -6
  104. package/dist/src/resources/saved-models.d.ts +134 -1
  105. package/dist/src/resources/saved-models.js +214 -6
  106. package/dist/src/resources/scenarios.d.ts +105 -0
  107. package/dist/src/resources/scenarios.js +232 -106
  108. package/dist/src/resources/sql.js +5 -1
  109. package/dist/src/resources/users.d.ts +129 -0
  110. package/dist/src/resources/users.js +128 -0
  111. package/dist/src/resources/webapps.d.ts +15 -2
  112. package/dist/src/resources/webapps.js +18 -4
  113. package/dist/src/schemas.d.ts +2 -2
  114. package/dist/src/schemas.js +1 -1
  115. package/dist/src/utils/deep-merge.d.ts +1 -1
  116. package/dist/src/utils/git-reference.d.ts +12 -0
  117. package/dist/src/utils/git-reference.js +38 -0
  118. package/dist/src/utils/polling.d.ts +20 -0
  119. package/dist/src/utils/polling.js +27 -0
  120. package/dist/src/utils/response-file.js +1 -1
  121. package/dist/src/utils/secret-sanitize.d.ts +54 -0
  122. package/dist/src/utils/secret-sanitize.js +101 -0
  123. package/package.json +9 -8
  124. package/packages/types/dist/index.d.ts +102 -0
  125. package/packages/types/dist/index.js +76 -0
  126. package/plugin.json +1 -1
  127. package/skills/dataiku-dss/SKILL.md +11 -7
  128. package/skills/dataiku-dss/references/app-releases.md +8 -8
  129. package/skills/dataiku-dss/references/authentication.md +6 -6
  130. package/skills/dataiku-dss/references/discovery.md +4 -4
  131. package/skills/dataiku-dss/references/mutations.md +1 -1
  132. package/skills/dataiku-dss/references/troubleshooting.md +2 -2
@@ -0,0 +1,118 @@
1
+ import { BaseResource } from "./base.js";
2
+ /**
3
+ * Data collections are instance-level curated groupings of DSS objects
4
+ * (datasets, saved models, agents, fine-tuned models, agent tools, semantic
5
+ * models) shared across projects.
6
+ *
7
+ * @see https://doc.dataiku.com/dss/api/15/rest/ (Data Collections section)
8
+ */
9
+ export type DataCollectionObjectReferenceType = "DATASET" | "SAVED_MODEL" | "FINE_TUNED_MODEL" | "AGENT" | "AGENT_TOOL" | "SEMANTIC_MODEL" | (string & {});
10
+ export interface DataCollectionPermission extends Record<string, unknown> {
11
+ /** Name of the user this permission applies to (exclusive with group). */
12
+ user?: string;
13
+ /** Name of the group this permission applies to (exclusive with user). */
14
+ group?: string;
15
+ /** true if the permission grants administrator access. */
16
+ admin?: boolean;
17
+ /** true if the permission grants contributor access. */
18
+ write?: boolean;
19
+ /** true if the permission grants member access. */
20
+ read?: boolean;
21
+ }
22
+ export interface DataCollectionSummary extends Record<string, unknown> {
23
+ /** Name of the collection. */
24
+ displayName?: string;
25
+ /** Color of the collection (#RRGBB syntax). */
26
+ color?: string | null;
27
+ /** The description of the collection (markdown). */
28
+ description?: string | null;
29
+ /** List of tags of the collection. */
30
+ tags?: string[] | null;
31
+ /** Id of the collection. */
32
+ id?: string;
33
+ /** Number of objects in the collection. */
34
+ itemCount?: number;
35
+ /** Number of datasets in the collection. */
36
+ datasetsCount?: number;
37
+ /** Number of saved models in the collection, excluding agents and fine tuned models. */
38
+ savedModelsCount?: number;
39
+ /** Number of fine tuned models in the collection. */
40
+ fineTunedModelsCount?: number;
41
+ /** Number of agents in the collection. */
42
+ agentsCount?: number;
43
+ /** Number of agent tools in the collection. */
44
+ agentToolsCount?: number;
45
+ /** Number of semantic models in the collection. */
46
+ semanticModelsCount?: number;
47
+ /** Timestamp of the last collection modification. */
48
+ lastModifiedOn?: number;
49
+ }
50
+ export interface DataCollectionSettings extends Record<string, unknown> {
51
+ /** Id of the collection. */
52
+ id?: string;
53
+ /** Name of the collection. */
54
+ displayName: string;
55
+ /** Color of the collection (#RRGGBB syntax). */
56
+ color?: string | null;
57
+ /** The description of the collection (markdown). */
58
+ description?: string | null;
59
+ /** List of tags of the collection. */
60
+ tags?: string[] | null;
61
+ /** List of permissions of the collection (READER, CONTRIBUTOR or ADMIN). */
62
+ permissions?: DataCollectionPermission[] | null;
63
+ }
64
+ export interface DataCollectionCreateRequest extends Omit<DataCollectionSettings, "id"> {
65
+ /**
66
+ * Id of the Collection (a random id is generated if not provided).
67
+ * Must match the [A-Za-z0-9]{8} server-side format when provided.
68
+ */
69
+ id?: string;
70
+ }
71
+ /**
72
+ * An object inside a collection: a reference to a DSS object hosted by a
73
+ * project. The documented shape covers datasets; `type` accepts the other
74
+ * collection member kinds the instance may report.
75
+ */
76
+ export interface DataCollectionObject extends Record<string, unknown> {
77
+ /** The type of this object (e.g. "DATASET"). */
78
+ type: DataCollectionObjectReferenceType;
79
+ /** The key of the project hosting the object. */
80
+ projectKey: string;
81
+ /** The id (name) of the object in its project. */
82
+ id: string;
83
+ }
84
+ export declare class DataCollectionsResource extends BaseResource {
85
+ /** List the collections on which the API key has READ privilege. */
86
+ list(): Promise<DataCollectionSummary[]>;
87
+ /**
88
+ * Create a collection. The creating user is added as an administrator.
89
+ * Requires the mayCreateDataCollections global permission. DSS returns
90
+ * `{ msg, id }`; the created id is surfaced to the caller.
91
+ */
92
+ create(body: DataCollectionCreateRequest): Promise<{
93
+ msg?: string;
94
+ id?: string;
95
+ }>;
96
+ /**
97
+ * Get the settings of a collection. Requires READ on the collection; the
98
+ * permissions array is omitted unless the caller is an ADMIN.
99
+ */
100
+ get(dataCollectionId: string): Promise<DataCollectionSettings>;
101
+ /**
102
+ * Update the settings of a collection. Only set a settings object obtained
103
+ * through {@link get}. Requires ADMIN on the collection.
104
+ */
105
+ update(dataCollectionId: string, settings: DataCollectionSettings): Promise<DataCollectionSettings>;
106
+ /** Permanently delete a collection. Requires ADMIN on the collection. */
107
+ delete(dataCollectionId: string): Promise<void>;
108
+ /** List the objects in a collection. Requires READ on the collection. */
109
+ listObjects(dataCollectionId: string): Promise<DataCollectionObject[]>;
110
+ /**
111
+ * Add an object to a collection. Requires WRITE (Contributor) on the
112
+ * collection, the mayPublishToDataCollections global group permission, and
113
+ * publishToDataCollections on the project hosting the object (if relevant).
114
+ */
115
+ addObject(dataCollectionId: string, object: DataCollectionObject): Promise<void>;
116
+ /** Remove a dataset from a collection. Requires WRITE on the collection. */
117
+ removeDataset(dataCollectionId: string, projectKey: string, datasetName: string): Promise<void>;
118
+ }
@@ -0,0 +1,49 @@
1
+ import { BaseResource, } from "./base.js";
2
+ export class DataCollectionsResource extends BaseResource {
3
+ /** List the collections on which the API key has READ privilege. */
4
+ async list() {
5
+ return this.client.get("/public/api/data-collections/");
6
+ }
7
+ /**
8
+ * Create a collection. The creating user is added as an administrator.
9
+ * Requires the mayCreateDataCollections global permission. DSS returns
10
+ * `{ msg, id }`; the created id is surfaced to the caller.
11
+ */
12
+ async create(body) {
13
+ return this.client.post("/public/api/data-collections/", body);
14
+ }
15
+ /**
16
+ * Get the settings of a collection. Requires READ on the collection; the
17
+ * permissions array is omitted unless the caller is an ADMIN.
18
+ */
19
+ async get(dataCollectionId) {
20
+ return this.client.get(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}`);
21
+ }
22
+ /**
23
+ * Update the settings of a collection. Only set a settings object obtained
24
+ * through {@link get}. Requires ADMIN on the collection.
25
+ */
26
+ async update(dataCollectionId, settings) {
27
+ return this.client.put(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}`, settings);
28
+ }
29
+ /** Permanently delete a collection. Requires ADMIN on the collection. */
30
+ async delete(dataCollectionId) {
31
+ await this.client.del(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}`);
32
+ }
33
+ /** List the objects in a collection. Requires READ on the collection. */
34
+ async listObjects(dataCollectionId) {
35
+ return this.client.get(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}/objects`);
36
+ }
37
+ /**
38
+ * Add an object to a collection. Requires WRITE (Contributor) on the
39
+ * collection, the mayPublishToDataCollections global group permission, and
40
+ * publishToDataCollections on the project hosting the object (if relevant).
41
+ */
42
+ async addObject(dataCollectionId, object) {
43
+ await this.client.post(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}/objects`, object);
44
+ }
45
+ /** Remove a dataset from a collection. Requires WRITE on the collection. */
46
+ async removeDataset(dataCollectionId, projectKey, datasetName) {
47
+ await this.client.del(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}/objects/dataset/${encodeURIComponent(projectKey)}/${encodeURIComponent(datasetName)}`);
48
+ }
49
+ }
@@ -17,6 +17,37 @@ export interface DatasetSchemaColumnInput {
17
17
  type: string;
18
18
  comment?: string;
19
19
  }
20
+ /** Options for creating a managed dataset through the dedicated endpoint. */
21
+ export interface DatasetManagedCreateOptions {
22
+ /** Name of the new dataset (unique in the project). */
23
+ name: string;
24
+ /** Connection where the dataset is stored. */
25
+ connection: string;
26
+ /** When the connection accepts several dataset types, the type to use. */
27
+ typeOptionId?: string;
28
+ /** Optional identifier of a file format option (e.g. CSV_EXCEL_GZIP, PARQUET_HIVE). */
29
+ formatOptionId?: string;
30
+ /**
31
+ * Partitioning option id. Use {@link DatasetManagedCreateOptions.copyPartitioningFrom}
32
+ * to build one from an existing dataset or folder reference.
33
+ */
34
+ partitioningOptionId?: string;
35
+ /**
36
+ * Copy partitioning from an existing dataset or folder instead of passing a
37
+ * raw `partitioningOptionId`. `type` defaults to `DATASET` and may be
38
+ * `FOLDER`.
39
+ */
40
+ copyPartitioningFrom?: {
41
+ ref: string;
42
+ type?: "DATASET" | "FOLDER";
43
+ };
44
+ }
45
+ /** Result of creating a managed dataset (the endpoint answers 204 with no body). */
46
+ export interface DatasetManagedCreateResult {
47
+ datasetName: string;
48
+ projectKey: string;
49
+ creationSettings: Record<string, unknown>;
50
+ }
20
51
  export interface DatasetCloneOptions {
21
52
  projectKey?: string;
22
53
  path?: string;
@@ -120,6 +151,44 @@ export declare class DatasetsResource extends BaseResource {
120
151
  }>;
121
152
  /** Get dataset metadata (tags, custom fields, checklists). */
122
153
  metadata(datasetName: string, projectKey?: string): Promise<Record<string, unknown>>;
154
+ /**
155
+ * Replace dataset metadata with the given object (faithful PUT per the DSS
156
+ * API contract: the caller GETs the metadata, edits it, and PUTs the full
157
+ * object back — no partial merge is applied client-side). `label`,
158
+ * `description`, `tags`, and `custom` are the documented writable fields;
159
+ * any additional server-provided metadata fields present in the object are
160
+ * preserved verbatim.
161
+ */
162
+ updateMetadata(datasetName: string, metadata: Record<string, unknown>, projectKey?: string): Promise<Record<string, unknown>>;
163
+ /**
164
+ * Create a managed dataset through the dedicated
165
+ * `POST /projects/{projectKey}/datasets/managed` endpoint. DSS handles the
166
+ * format/storage details: you select the name, the connection, and optional
167
+ * type/format/partitioning options. The endpoint answers `204` with no
168
+ * body; the returned settings echo what was sent.
169
+ */
170
+ createManaged(opts: DatasetManagedCreateOptions, projectKey?: string): Promise<DatasetManagedCreateResult>;
171
+ /**
172
+ * Get the full info object for a dataset (type, parameters, last build
173
+ * information, schema, etc.). The server returns a complex opaque object;
174
+ * it is surfaced without validation.
175
+ */
176
+ info(datasetName: string, projectKey?: string): Promise<Record<string, unknown>>;
177
+ /**
178
+ * Get the full column lineage (auto-computed and manual) of a column in
179
+ * this dataset. Relations with datasets from both local and foreign
180
+ * projects are included.
181
+ *
182
+ * Route per the official Python client:
183
+ * `GET /projects/{projectKey}/datasets/{datasetName}/column-lineage`
184
+ * with query `columnName` (required) and `maxDatasetCount` (optional,
185
+ * positive integer — the docs page renders a merged project-scoped path,
186
+ * but DSS's own client and route registry use this per-dataset form).
187
+ */
188
+ getColumnLineage(datasetName: string, column: string, opts?: {
189
+ maxDatasetCount?: number;
190
+ projectKey?: string;
191
+ }): Promise<unknown>;
123
192
  /**
124
193
  * Download dataset rows as a gzipped (or .csv) file, capped at `limit` rows
125
194
  * (default 100k). Returns the written path, the number of data rows written,
@@ -394,12 +394,39 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
394
394
  reader.releaseLock();
395
395
  }
396
396
  }
397
+ /**
398
+ * Encoded CSV bytes accumulated before one stream push. Rows are joined into
399
+ * batch-sized chunks so large downloads do not pay per-row stream overhead; a
400
+ * single escaped row may exceed the budget (rows are never split), and the
401
+ * batch is always flushed at EOF so trailing rows are never dropped.
402
+ */
403
+ const CSV_BATCH_BYTES = 64 * 1024;
397
404
  function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
398
405
  const state = createTsvStreamState();
399
406
  const decoder = new StringDecoder("utf-8");
400
407
  const maxRows = Math.max(1, maxDataRows);
401
408
  let headerSeen = false;
402
409
  let done = false;
410
+ // Bounded CSV output batching: emitted rows accumulate here and are joined
411
+ // into a single push once the batch reaches CSV_BATCH_BYTES, so memory stays
412
+ // bounded by the batch budget plus one escaped row (the join briefly holds a
413
+ // second copy of the batch) while per-row push/callback overhead disappears.
414
+ let batchLines = [];
415
+ let batchBytes = 0;
416
+ const flushBatch = (target) => {
417
+ if (batchLines.length === 0)
418
+ return;
419
+ const csv = batchLines.join("");
420
+ batchLines = [];
421
+ batchBytes = 0;
422
+ target.push(csv);
423
+ };
424
+ const queueLine = (target, line) => {
425
+ batchLines.push(line);
426
+ batchBytes += Buffer.byteLength(line);
427
+ if (batchBytes >= CSV_BATCH_BYTES)
428
+ flushBatch(target);
429
+ };
403
430
  let columnCount = 0;
404
431
  const handleRow = (row, push) => {
405
432
  if (done)
@@ -430,7 +457,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
430
457
  return;
431
458
  }
432
459
  try {
433
- consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => this.push(line)));
460
+ consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(this, line)));
434
461
  }
435
462
  catch (error) {
436
463
  // Byte-limit overflow: destroy the transform so the pipeline
@@ -438,8 +465,11 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
438
465
  callback(error instanceof Error ? error : new Error(String(error)));
439
466
  return;
440
467
  }
441
- if (done)
468
+ if (done) {
469
+ // Truncation reached: emit the batch queued so far before EOF.
470
+ flushBatch(this);
442
471
  this.push(null);
472
+ }
443
473
  callback();
444
474
  },
445
475
  flush(callback) {
@@ -450,14 +480,17 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
450
480
  try {
451
481
  const tail = decoder.end();
452
482
  if (tail !== "") {
453
- consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => this.push(line)));
483
+ consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(this, line)));
454
484
  }
455
- flushTsvStream(state, (row) => handleRow(row, (line) => this.push(line)));
485
+ flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(this, line)));
456
486
  }
457
487
  catch (error) {
458
488
  callback(error instanceof Error ? error : new Error(String(error)));
459
489
  return;
460
490
  }
491
+ // Emit the trailing partial batch; without this the last rows of a
492
+ // download (below the batch budget) would be silently dropped.
493
+ flushBatch(this);
461
494
  callback();
462
495
  },
463
496
  });
@@ -712,6 +745,104 @@ export class DatasetsResource extends BaseResource {
712
745
  const dsEnc = encodeURIComponent(datasetName);
713
746
  return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/metadata`);
714
747
  }
748
+ /**
749
+ * Replace dataset metadata with the given object (faithful PUT per the DSS
750
+ * API contract: the caller GETs the metadata, edits it, and PUTs the full
751
+ * object back — no partial merge is applied client-side). `label`,
752
+ * `description`, `tags`, and `custom` are the documented writable fields;
753
+ * any additional server-provided metadata fields present in the object are
754
+ * preserved verbatim.
755
+ */
756
+ async updateMetadata(datasetName, metadata, projectKey) {
757
+ if (metadata === null || Array.isArray(metadata)) {
758
+ throw new ClientValidationError("metadata must be a JSON object.");
759
+ }
760
+ const dsEnc = encodeURIComponent(datasetName);
761
+ return this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/metadata`, metadata);
762
+ }
763
+ /**
764
+ * Create a managed dataset through the dedicated
765
+ * `POST /projects/{projectKey}/datasets/managed` endpoint. DSS handles the
766
+ * format/storage details: you select the name, the connection, and optional
767
+ * type/format/partitioning options. The endpoint answers `204` with no
768
+ * body; the returned settings echo what was sent.
769
+ */
770
+ async createManaged(opts, projectKey) {
771
+ const name = typeof opts.name === "string" ? opts.name.trim() : "";
772
+ if (name === "") {
773
+ throw new ClientValidationError("name must be a non-empty dataset name.");
774
+ }
775
+ const connection = typeof opts.connection === "string" ? opts.connection.trim() : "";
776
+ if (connection === "") {
777
+ throw new ClientValidationError("connection must be a non-empty connection name.");
778
+ }
779
+ if (opts.partitioningOptionId !== undefined && opts.copyPartitioningFrom !== undefined) {
780
+ throw new ClientValidationError("Pass either partitioningOptionId or copyPartitioningFrom, not both.");
781
+ }
782
+ if (opts.copyPartitioningFrom !== undefined
783
+ && (typeof opts.copyPartitioningFrom.ref !== "string"
784
+ || opts.copyPartitioningFrom.ref.trim() === "")) {
785
+ throw new ClientValidationError("copyPartitioningFrom.ref must be a non-empty dataset or folder name.");
786
+ }
787
+ const pk = this.resolveProjectKey(projectKey);
788
+ const creationSettings = { connectionId: connection, };
789
+ if (opts.typeOptionId !== undefined)
790
+ creationSettings.typeOptionId = opts.typeOptionId;
791
+ if (opts.formatOptionId !== undefined || opts.partitioningOptionId !== undefined
792
+ || opts.copyPartitioningFrom !== undefined) {
793
+ const specificSettings = {};
794
+ if (opts.formatOptionId !== undefined) {
795
+ specificSettings.formatOptionId = opts.formatOptionId;
796
+ }
797
+ if (opts.partitioningOptionId !== undefined) {
798
+ specificSettings.partitioningOptionId = opts.partitioningOptionId;
799
+ }
800
+ else if (opts.copyPartitioningFrom !== undefined) {
801
+ const code = opts.copyPartitioningFrom.type === "FOLDER" ? "folder" : "dataset";
802
+ specificSettings.partitioningOptionId = `copy:${code}:${opts.copyPartitioningFrom.ref.trim()}`;
803
+ }
804
+ creationSettings.specificSettings = specificSettings;
805
+ }
806
+ await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/datasets/managed`, { name, creationSettings, });
807
+ return { datasetName: name, projectKey: pk, creationSettings, };
808
+ }
809
+ /**
810
+ * Get the full info object for a dataset (type, parameters, last build
811
+ * information, schema, etc.). The server returns a complex opaque object;
812
+ * it is surfaced without validation.
813
+ */
814
+ async info(datasetName, projectKey) {
815
+ const dsEnc = encodeURIComponent(datasetName);
816
+ return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/info`);
817
+ }
818
+ /**
819
+ * Get the full column lineage (auto-computed and manual) of a column in
820
+ * this dataset. Relations with datasets from both local and foreign
821
+ * projects are included.
822
+ *
823
+ * Route per the official Python client:
824
+ * `GET /projects/{projectKey}/datasets/{datasetName}/column-lineage`
825
+ * with query `columnName` (required) and `maxDatasetCount` (optional,
826
+ * positive integer — the docs page renders a merged project-scoped path,
827
+ * but DSS's own client and route registry use this per-dataset form).
828
+ */
829
+ async getColumnLineage(datasetName, column, opts = {}) {
830
+ const trimmed = typeof column === "string" ? column.trim() : "";
831
+ if (trimmed === "") {
832
+ throw new ClientValidationError("column must be a non-empty column name.");
833
+ }
834
+ const maxDatasetCount = opts.maxDatasetCount;
835
+ if (maxDatasetCount !== undefined
836
+ && (!Number.isSafeInteger(maxDatasetCount) || maxDatasetCount <= 0)) {
837
+ throw new ClientValidationError("maxDatasetCount must be a positive integer.");
838
+ }
839
+ const dsEnc = encodeURIComponent(datasetName);
840
+ const params = [`columnName=${encodeURIComponent(trimmed)}`,];
841
+ if (maxDatasetCount !== undefined) {
842
+ params.push(`maxDatasetCount=${encodeURIComponent(String(maxDatasetCount))}`);
843
+ }
844
+ return this.client.get(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/column-lineage?${params.join("&")}`);
845
+ }
715
846
  /**
716
847
  * Download dataset rows as a gzipped (or .csv) file, capped at `limit` rows
717
848
  * (default 100k). Returns the written path, the number of data rows written,
@@ -746,7 +877,7 @@ export class DatasetsResource extends BaseResource {
746
877
  : undefined;
747
878
  const shouldGzip = filePath.endsWith(".gz");
748
879
  const stats = { rows: 0, truncated: false, };
749
- const nodeStream = Readable.fromWeb(res.body);
880
+ const nodeStream = Readable.from(res.body, { objectMode: false, });
750
881
  const csvTransform = tsvToCsvTransform(limit, stats, onHeader, opts?.rawData !== true);
751
882
  fs.mkdirSync(nodePath.dirname(filePath), { recursive: true, });
752
883
  const fileOut = fs.createWriteStream(filePath);
@@ -1,3 +1,4 @@
1
+ import type { DataikuGetOptions } from "../client.js";
1
2
  import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZoneUpdateOptions } from "../schemas.js";
2
3
  import { BaseResource } from "./base.js";
3
4
  export type FlowZoneItemInput = {
@@ -6,8 +7,11 @@ export type FlowZoneItemInput = {
6
7
  projectKey?: string;
7
8
  };
8
9
  export declare class FlowZonesResource extends BaseResource {
9
- /** List all flow zones in a project. */
10
- list(projectKey?: string): Promise<FlowZone[]>;
10
+ /**
11
+ * List all flow zones in a project. Pass `timeoutMs` to bound the whole
12
+ * request (attempts, backoff, and body read) by a total budget.
13
+ */
14
+ list(projectKey?: string, options?: DataikuGetOptions): Promise<FlowZone[]>;
11
15
  /** Get one flow zone by id. */
12
16
  get(zoneId: string, projectKey?: string): Promise<FlowZone>;
13
17
  /** Create a flow zone. */
@@ -9,9 +9,12 @@ function normalizeZoneItem(item) {
9
9
  };
10
10
  }
11
11
  export class FlowZonesResource extends BaseResource {
12
- /** List all flow zones in a project. */
13
- async list(projectKey) {
14
- const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones`);
12
+ /**
13
+ * List all flow zones in a project. Pass `timeoutMs` to bound the whole
14
+ * request (attempts, backoff, and body read) by a total budget.
15
+ */
16
+ async list(projectKey, options) {
17
+ const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones`, options);
15
18
  return this.client.safeParse(FlowZoneArraySchema, raw, "flowZones.list");
16
19
  }
17
20
  /** Get one flow zone by id. */
@@ -5,10 +5,15 @@ export interface FutureWaitOptions {
5
5
  timeoutMs?: number;
6
6
  }
7
7
  export declare class FuturesResource extends BaseResource {
8
- get(futureId: string): Promise<FutureState>;
8
+ get(futureId: string, opts?: {
9
+ timeoutMs?: number;
10
+ noRetry?: boolean;
11
+ }): Promise<FutureState>;
9
12
  peek(futureId: string): Promise<FutureState>;
10
13
  state(futureId: string, opts?: {
11
14
  peek?: boolean;
15
+ timeoutMs?: number;
16
+ noRetry?: boolean;
12
17
  }): Promise<FutureState>;
13
18
  abort(futureId: string): Promise<void>;
14
19
  wait(futureId: string, opts?: FutureWaitOptions): Promise<FutureWaitResult>;
@@ -1,4 +1,5 @@
1
1
  import { FutureStateSchema, FutureWaitResultSchema, } from "../schemas.js";
2
+ import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
2
3
  import { BaseResource, } from "./base.js";
3
4
  const DEFAULT_POLL_INTERVAL_MS = 2_000;
4
5
  const DEFAULT_TIMEOUT_MS = 120_000;
@@ -23,8 +24,12 @@ function waitState(state) {
23
24
  return "RUNNING";
24
25
  }
25
26
  export class FuturesResource extends BaseResource {
26
- async get(futureId) {
27
- return this.state(futureId, { peek: false, });
27
+ async get(futureId, opts = {}) {
28
+ return this.state(futureId, {
29
+ peek: false,
30
+ timeoutMs: opts.timeoutMs,
31
+ noRetry: opts.noRetry,
32
+ });
28
33
  }
29
34
  async peek(futureId) {
30
35
  return this.state(futureId, { peek: true, });
@@ -32,23 +37,71 @@ export class FuturesResource extends BaseResource {
32
37
  async state(futureId, opts = {}) {
33
38
  const params = new URLSearchParams();
34
39
  params.set("peek", String(opts.peek === true));
35
- const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}`);
40
+ const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}`, { timeoutMs: opts.timeoutMs, noRetry: opts.noRetry, });
36
41
  return this.client.safeParse(FutureStateSchema, raw, "futures.state");
37
42
  }
38
43
  async abort(futureId) {
39
44
  await this.client.del(`/public/api/futures/${encodeURIComponent(futureId)}`);
40
45
  }
41
46
  async wait(futureId, opts = {}) {
42
- const baseIntervalMs = Math.max(1, opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
47
+ // Adaptive backoff is the default; an explicit pollIntervalMs is an
48
+ // exact contract the caller chose, so it is never adapted.
49
+ const explicitIntervalMs = opts.pollIntervalMs;
50
+ const baseIntervalMs = Math.max(1, explicitIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
51
+ const adaptiveEnabled = explicitIntervalMs === undefined;
43
52
  // A caller's budget is never rounded up to a whole poll interval: a 5ms
44
53
  // wait must answer in about 5ms. One state observation always happens
45
54
  // regardless, because the poll precedes the deadline check.
46
55
  const timeoutMs = Math.max(0, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
47
56
  const startedAt = Date.now();
48
57
  let pollCount = 0;
58
+ let lastState;
49
59
  while (true) {
60
+ const elapsedBeforeMs = Date.now() - startedAt;
61
+ // The first observation always happens, even when the budget is
62
+ // already spent. A later poll is never started after the deadline:
63
+ // the loop reports the structured timeout from the last observed
64
+ // state instead of issuing a request the budget cannot cover.
65
+ if (lastState !== undefined && elapsedBeforeMs >= timeoutMs) {
66
+ return this.client.safeParse(FutureWaitResultSchema, {
67
+ futureId,
68
+ jobId: lastState.jobId,
69
+ state: waitState(lastState),
70
+ elapsedMs: elapsedBeforeMs,
71
+ pollCount,
72
+ success: false,
73
+ timedOut: true,
74
+ hasResult: lastState.hasResult === true,
75
+ alive: lastState.alive,
76
+ aborted: lastState.aborted,
77
+ unknown: lastState.unknown,
78
+ }, "futures.wait");
79
+ }
50
80
  pollCount += 1;
51
- const state = await this.get(futureId);
81
+ // Requests issued while budget remains are bounded by the remaining
82
+ // time; a spent budget still issues exactly one transport attempt
83
+ // (no retries, client requestTimeoutMs cap) so the first observation
84
+ // reaches the server instead of failing before any attempt.
85
+ const remainingMs = timeoutMs - elapsedBeforeMs;
86
+ let state;
87
+ try {
88
+ state = await this.get(futureId, remainingMs > 0 ? { timeoutMs: remainingMs, } : { noRetry: true, });
89
+ }
90
+ catch (error) {
91
+ // The poll budget ran out mid-request: report the structured
92
+ // timeout instead of letting the transport deadline error escape.
93
+ if (!isRequestDeadlineError(error, startedAt + timeoutMs))
94
+ throw error;
95
+ return this.client.safeParse(FutureWaitResultSchema, {
96
+ futureId,
97
+ state: "RUNNING",
98
+ elapsedMs: Date.now() - startedAt,
99
+ pollCount,
100
+ success: false,
101
+ timedOut: true,
102
+ }, "futures.wait");
103
+ }
104
+ lastState = state;
52
105
  const elapsedMs = Date.now() - startedAt;
53
106
  const status = waitState(state);
54
107
  if (isFinished(state)) {
@@ -67,25 +120,18 @@ export class FuturesResource extends BaseResource {
67
120
  };
68
121
  return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
69
122
  }
70
- if (elapsedMs >= timeoutMs) {
71
- const result = {
72
- futureId,
73
- jobId: state.jobId,
74
- state: status,
75
- elapsedMs,
76
- pollCount,
77
- success: false,
78
- timedOut: true,
79
- hasResult: state.hasResult === true,
80
- alive: state.alive,
81
- aborted: state.aborted,
82
- unknown: state.unknown,
83
- };
84
- return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
85
- }
123
+ // Deadline reached: the guard at the top of the loop reports the
124
+ // timeout from this observation without issuing another request.
125
+ if (elapsedMs >= timeoutMs)
126
+ continue;
86
127
  // Only the remaining budget is slept: a longer sleep would report an
87
128
  // elapsed time the caller never authorized.
88
- await sleep(Math.min(baseIntervalMs, timeoutMs - elapsedMs));
129
+ const nextDelayMs = computeNextPollDelayMs({
130
+ pollCount,
131
+ baseIntervalMs,
132
+ adaptiveEnabled,
133
+ });
134
+ await sleep(Math.min(nextDelayMs, timeoutMs - elapsedMs));
89
135
  }
90
136
  }
91
137
  }