dataiku-sdk 3.8.0 → 4.0.0

package/dist/src/generated/action-output-schemas.json

@@ -363,6 +363,107 @@
                 }
             }
         },
+        "dataset.info": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+                "name"
+            ],
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "type": {
+                    "type": "string"
+                },
+                "managed": {
+                    "type": "boolean"
+                },
+                "connection": {
+                    "type": "string"
+                },
+                "formatType": {
+                    "type": "string"
+                },
+                "partitioned": {
+                    "type": "boolean"
+                },
+                "schema": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "required": [
+                            "name",
+                            "type"
+                        ],
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "type": {
+                                "type": "string"
+                            },
+                            "meaning": {
+                                "type": "string"
+                            },
+                            "comment": {
+                                "type": "string"
+                            }
+                        }
+                    }
+                },
+                "recipes": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "buildable": {
+                    "type": "boolean"
+                },
+                "upstreamBuildable": {
+                    "type": "boolean"
+                },
+                "downstreamBuildable": {
+                    "type": "boolean"
+                },
+                "tags": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "dataQuality": {
+                    "type": "object",
+                    "additionalProperties": true
+                },
+                "lastBuild": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                        "jobId": {
+                            "type": "string"
+                        },
+                        "success": {
+                            "type": "boolean"
+                        },
+                        "startTime": {
+                            "type": "number"
+                        },
+                        "endTime": {
+                            "type": "number"
+                        }
+                    }
+                },
+                "lastModifiedOn": {
+                    "type": "number"
+                },
+                "lastModifiedBy": {
+                    "type": "string"
+                }
+            }
+        },
         "dataset.list": {
             "type": "array",
             "items": {
@@ -825,6 +926,121 @@
                 }
             }
         },
+        "job.get": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+                "id",
+                "state"
+            ],
+            "properties": {
+                "id": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "type": {
+                    "type": "string"
+                },
+                "state": {
+                    "type": "string"
+                },
+                "initiator": {
+                    "type": "string"
+                },
+                "triggeredFrom": {
+                    "type": "string"
+                },
+                "startTime": {
+                    "type": "number"
+                },
+                "endTime": {
+                    "type": "number"
+                },
+                "outputs": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                            "type": {
+                                "type": "string"
+                            },
+                            "id": {
+                                "type": "string"
+                            },
+                            "projectKey": {
+                                "type": "string"
+                            },
+                            "partition": {
+                                "type": "string"
+                            }
+                        }
+                    }
+                },
+                "progress": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "number"
+                    }
+                },
+                "error": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                        "type": {
+                            "type": "string"
+                        },
+                        "message": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "activities": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                            "id": {
+                                "type": "string"
+                            },
+                            "recipe": {
+                                "type": "string"
+                            },
+                            "recipeType": {
+                                "type": "string"
+                            },
+                            "state": {
+                                "type": "string"
+                            },
+                            "totalTime": {
+                                "type": "number"
+                            },
+                            "warnings": {
+                                "type": "number"
+                            },
+                            "error": {
+                                "type": "object",
+                                "additionalProperties": false,
+                                "properties": {
+                                    "type": {
+                                        "type": "string"
+                                    },
+                                    "message": {
+                                        "type": "string"
+                                    }
+                                }
+                            },
+                            "message": {
+                                "type": "string"
+                            }
+                        }
+                    }
+                }
+            }
+        },
         "job.list": {
             "type": "array",
             "items": {
@@ -903,6 +1119,12 @@
                 "timedOut": {
                     "type": "boolean"
                 },
+                "cancelled": {
+                    "type": "boolean"
+                },
+                "resume": {
+                    "type": "string"
+                },
                 "progress": {
                     "type": "object",
                     "required": [
@@ -971,6 +1193,12 @@
                 "timedOut": {
                     "type": "boolean"
                 },
+                "cancelled": {
+                    "type": "boolean"
+                },
+                "resume": {
+                    "type": "string"
+                },
                 "progress": {
                     "type": "object",
                     "required": [
@@ -1361,6 +1589,12 @@
                             "string",
                             "null"
                         ]
+                    },
+                    "isDev": {
+                        "type": [
+                            "boolean",
+                            "null"
+                        ]
                     }
                 }
             }
@@ -1519,7 +1753,8 @@
             "type": "object",
             "required": [
                 "projectKey",
-                "name"
+                "name",
+                "deniedPermissions"
             ],
             "properties": {
                 "projectKey": {
@@ -1537,25 +1772,6 @@
                 "projectAppType": {
                     "type": "string"
                 },
-                "creationTag": {
-                    "additionalProperties": true,
-                    "type": "object",
-                    "properties": {
-                        "versionNumber": {
-                            "type": "number"
-                        },
-                        "lastModifiedOn": {
-                            "anyOf": [
-                                {
-                                    "type": "number"
-                                },
-                                {
-                                    "type": "string"
-                                }
-                            ]
-                        }
-                    }
-                },
                 "ownerLogin": {
                     "type": "string"
                 },
@@ -1565,21 +1781,12 @@
                         "type": "string"
                     }
                 },
-                "versionTag": {
-                    "additionalProperties": true,
-                    "type": "object",
-                    "required": [
-                        "versionNumber",
-                        "lastModifiedOn"
-                    ],
-                    "properties": {
-                        "versionNumber": {
-                            "type": "number"
-                        },
-                        "lastModifiedOn": {
-                            "type": "number"
-                        }
-                    }
+                "deniedPermissions": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    },
+                    "description": "Permission flags DSS reports as false; absent flags are granted."
                 }
             }
         },

package/dist/src/index.d.ts

@@ -22,7 +22,7 @@ export { FoldersResource, } from "./resources/folders.js";
 export { type DssTask, FuturesResource, type FutureWaitOptions, } from "./resources/futures.js";
 export { type DssGroup, type GroupCreateRequest, GroupsResource, } from "./resources/groups.js";
 export { InsightsResource, } from "./resources/insights.js";
-export { type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogFilter, type JobLogProgress, type JobLogSummary, type JobLogUnavailableReason, JobsResource, type JobWaitOutcome, parseJobLogProgress, } from "./resources/jobs.js";
+export { type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogDetails, type JobLogFilter, type JobLogProgress, type JobLogSummary, type JobLogUnavailableReason, JobsResource, type JobWaitOutcome, parseJobLogProgress, } from "./resources/jobs.js";
 export * from "./resources/knowledge-banks.js";
 export * from "./resources/llms.js";
 export { type MacroResultOptions, type MacroRunFailure, type MacroRunHandle, type MacroRunOptions, type MacroRunWaitedResult, MacrosResource, type MacroWaitOptions, } from "./resources/macros.js";

package/dist/src/resources/datasets.d.ts

@@ -135,9 +135,19 @@ export declare class DatasetsResource extends BaseResource {
     /**
      * TSV data stream of at most `limit` rows. A column or partition selection
      * uses the documented POST variant of the data endpoint (body `columns`,
-     * `partitions`, `format`, `sampling`); otherwise the GET endpoint.
+     * `partitions`, `format`, `sampling`); otherwise the GET endpoint. Both carry
+     * a client-generated `readSessionId` for {@link verifyStreamComplete}.
      */
     private dataStream;
+    /**
+     * A data stream that ended normally may still be incomplete: DSS sends 200
+     * headers before reading the data, so a server-side read failure only cuts
+     * the body short. As dataikuapi does, ask DSS whether the read session
+     * finished; it answers non-2xx (DataikuError) when it did not. Only called
+     * after the body was read to its end — a stream the client cancelled at
+     * its row cap is not complete by design.
+     */
+    private verifyStreamComplete;
     /**
      * Preview dataset rows as structured data: column names plus row arrays,
      * mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV

package/dist/src/resources/datasets.js

@@ -1,6 +1,5 @@
 import * as fs from "node:fs";
 import * as nodePath from "node:path";
-import { StringDecoder, } from "node:string_decoder";
 import { ClientValidationError, DataikuError, } from "../errors.js";
 import { DatasetDetailsSchema, DatasetSchemaSchema, DatasetSummaryArraySchema, } from "../schemas.js";
 import { deepMerge, } from "../utils/deep-merge.js";
@@ -28,8 +27,44 @@ function formatLimitBytes(bytes) {
         return `${bytes / (1024 * 1024)} MiB`;
     return `${bytes} B`;
 }
-function tsvLimitError(what, limit) {
-    return new DataikuError(0, "Dataset TSV Limit", `Dataset TSV ${what} exceeded ${formatLimitBytes(limit)}; streaming was aborted to guard against unbounded memory use.`);
+const TSV_ERROR_COLUMN_NAME_MAX = 40;
+/**
+ * Describe the column whose cell is being read when a field/row budget trips:
+ * the header name once the header row has been parsed, otherwise the 0-based
+ * column index. Error summaries are bounded (~200 chars), so the name is
+ * shortened to keep the `--columns` hint inside the visible message.
+ */
+function describeTsvColumn(state) {
+    const index = state.currentRow.length;
+    const name = state.header?.[index];
+    if (name === undefined)
+        return `column index ${index}`;
+    const shown = name.length > TSV_ERROR_COLUMN_NAME_MAX
+        ? `${name.slice(0, TSV_ERROR_COLUMN_NAME_MAX)}...`
+        : name;
+    return `column "${shown}"`;
+}
+/**
+ * A client-side size budget, not a DSS or transport failure: retrying the same
+ * read cannot succeed, so this is a non-retryable input error whose fix is a
+ * narrower selection (--columns).
+ */
+function tsvLimitError(what, limit, state) {
+    const exceeded = `Dataset TSV ${what} exceeded ${formatLimitBytes(limit)}`;
+    const cellLimit = formatLimitBytes(state.limits.maxFieldBytes);
+    let message;
+    if (what === "field") {
+        message = `${exceeded} in ${describeTsvColumn(state)} (a single cell is limited to ${cellLimit}); exclude it with --columns.`;
+    }
+    else if (what === "row") {
+        message = `${exceeded} (a single cell is limited to ${cellLimit}); reached in ${describeTsvColumn(state)}. Exclude large columns with --columns.`;
+    }
+    else {
+        message = `${exceeded}; streaming was aborted to guard against unbounded memory use.`;
+    }
+    return new ClientValidationError(message, "validation_failed", what === "response"
+        ? "Narrow the read with --columns, --partitions, or a lower row limit."
+        : "Select fewer columns with --columns so the oversized cell is not read.");
 }
 /** Characters that evaluate a spreadsheet cell as a formula. */
 const FORMULA_SIGILS = "=+-@";
@@ -75,6 +110,7 @@ function createTsvStreamState(limits = {
     return {
         currentField: "",
         currentRow: [],
+        header: undefined,
         inQuotes: false,
         pendingQuoteInQuotes: false,
         fieldBytes: 0,
@@ -91,10 +127,10 @@ function appendFieldText(state, text) {
     state.fieldBytes += bytes;
     state.rowBytes += bytes;
     if (state.fieldBytes > state.limits.maxFieldBytes) {
-        throw tsvLimitError("field", state.limits.maxFieldBytes);
+        throw tsvLimitError("field", state.limits.maxFieldBytes, state);
     }
     if (state.rowBytes > state.limits.maxRowBytes) {
-        throw tsvLimitError("row", state.limits.maxRowBytes);
+        throw tsvLimitError("row", state.limits.maxRowBytes, state);
     }
     state.currentField += text;
 }
@@ -105,11 +141,20 @@ function endField(state) {
     state.fieldBytes = 0;
     state.rowBytes += 1; // count the delimiter byte
 }
+/** Hand a completed row to `onRow`, remembering the first non-blank row as the header. */
+function emitRow(state, onRow) {
+    const row = state.currentRow;
+    state.currentRow = [];
+    state.rowBytes = 0;
+    if (state.header === undefined && !isBlankRow(row))
+        state.header = row;
+    onRow(row);
+}
 function consumeTsvChunk(text, state, onRow) {
     // Response budget: bound total decoded bytes regardless of row structure.
     state.responseBytes += Buffer.byteLength(text, "utf-8");
     if (state.responseBytes > state.limits.maxResponseBytes) {
-        throw tsvLimitError("response", state.limits.maxResponseBytes);
+        throw tsvLimitError("response", state.limits.maxResponseBytes, state);
     }
     let i = 0;
     if (state.pendingQuoteInQuotes) {
@@ -193,10 +238,7 @@ function consumeTsvChunk(text, state, onRow) {
         if (ch === "\n") {
             endField(state);
             i = special + 1;
-            const row = state.currentRow;
-            state.currentRow = [];
-            state.rowBytes = 0;
-            onRow(row);
+            emitRow(state, onRow);
             continue;
         }
         // "\r" — carriage return: skipped, as in the original parser.
@@ -213,10 +255,7 @@ function flushTsvStream(state, onRow) {
     state.currentRow.push(state.currentField);
     state.currentField = "";
     state.fieldBytes = 0;
-    const row = state.currentRow;
-    state.currentRow = [];
-    state.rowBytes = 0;
-    onRow(row);
+    emitRow(state, onRow);
 }
 function rowToCsv(row, spreadsheetSafe) {
     return row.map((field) => csvEscape(field, spreadsheetSafe)).join(",");
@@ -264,7 +303,7 @@ async function readChunkWithTimeout(reader, remainingMs, timeoutMs, action) {
 }
 async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
     const state = createTsvStreamState();
-    const decoder = new StringDecoder("utf-8");
+    const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
     let columns;
     const rows = [];
     let done = false;
@@ -304,10 +343,10 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
             const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "preview");
             if (result.done)
                 break;
-            consumeTsvChunk(decoder.write(result.value), state, handleRow);
+            consumeTsvChunk(decoder.decode(result.value, { stream: true, }), state, handleRow);
         }
         if (!done) {
-            const tail = decoder.end();
+            const tail = decoder.decode();
             if (tail !== "")
                 consumeTsvChunk(tail, state, handleRow);
             flushTsvStream(state, handleRow);
@@ -332,7 +371,7 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
  */
 async function collectTsvRowCount(body, stopAfter, timeoutMs) {
     const state = createTsvStreamState();
-    const decoder = new StringDecoder("utf-8");
+    const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
     let headerSeen = false;
     let count = 0;
     let bounded = false;
@@ -373,10 +412,10 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
             const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "row count assertion");
             if (result.done)
                 break;
-            consumeTsvChunk(decoder.write(result.value), state, handleRow);
+            consumeTsvChunk(decoder.decode(result.value, { stream: true, }), state, handleRow);
         }
         if (!bounded) {
-            const tail = decoder.end();
+            const tail = decoder.decode();
             if (tail !== "")
                 consumeTsvChunk(tail, state, handleRow);
             flushTsvStream(state, handleRow);
@@ -402,7 +441,7 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
 const CSV_BATCH_BYTES = 64 * 1024;
 function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
     const state = createTsvStreamState();
-    const decoder = new StringDecoder("utf-8");
+    const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
     const encoder = new TextEncoder();
     const maxRows = Math.max(1, maxDataRows);
     let headerSeen = false;
@@ -454,7 +493,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
     // which cancels the response and makes download() remove the partial file.
     return new TransformStream({
         transform(chunk, controller) {
-            consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
+            consumeTsvChunk(decoder.decode(chunk, { stream: true, }), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
             if (done) {
                 // Truncation reached: emit the batch queued so far, end the output,
                 // and cancel the rest of the response.
@@ -463,7 +502,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
             }
         },
         flush(controller) {
-            const tail = decoder.end();
+            const tail = decoder.decode();
             if (tail !== "") {
                 consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(controller, line)));
             }
@@ -660,19 +699,35 @@ export class DatasetsResource extends BaseResource {
     /**
      * TSV data stream of at most `limit` rows. A column or partition selection
      * uses the documented POST variant of the data endpoint (body `columns`,
-     * `partitions`, `format`, `sampling`); otherwise the GET endpoint.
+     * `partitions`, `format`, `sampling`); otherwise the GET endpoint. Both carry
+     * a client-generated `readSessionId` for {@link verifyStreamComplete}.
      */
     async dataStream(datasetName, limit, projectKey, selection) {
+        const readSessionId = crypto.randomUUID();
         const path = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/data/`;
         if (!selection?.columns?.length && !selection?.partitions) {
-            return this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}`);
+            const res = await this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}&readSessionId=${readSessionId}`);
+            return { res, readSessionId, };
         }
-        return this.client.postStream(path, {
+        const res = await this.client.postStream(path, {
             format: "tsv-excel-header",
             ...(selection.columns?.length ? { columns: selection.columns, } : {}),
             ...(selection.partitions ? { partitions: selection.partitions, } : {}),
             sampling: { samplingMethod: "HEAD_SEQUENTIAL", maxRecords: limit, },
+            readSessionId,
         });
+        return { res, readSessionId, };
+    }
+    /**
+     * A data stream that ended normally may still be incomplete: DSS sends 200
+     * headers before reading the data, so a server-side read failure only cuts
+     * the body short. As dataikuapi does, ask DSS whether the read session
+     * finished; it answers non-2xx (DataikuError) when it did not. Only called
+     * after the body was read to its end — a stream the client cancelled at
+     * its row cap is not complete by design.
+     */
+    async verifyStreamComplete(datasetName, projectKey, readSessionId) {
+        await this.client.getTextLimited(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/finish-streaming/?readSessionId=${readSessionId}`, 64 * 1024);
     }
     /**
      * Preview dataset rows as structured data: column names plus row arrays,
@@ -691,7 +746,7 @@ export class DatasetsResource extends BaseResource {
         const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
         // Probe one row past the cap so callers learn whether the dataset holds
         // more rows than requested without materializing the full dataset.
-        const res = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
+        const { res, readSessionId, } = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
         const onHeader = opts?.validateColumns
             ? (headerRow) => {
                 const warnings = validateStreamColumns(headerRow, opts.validateColumns);
@@ -704,6 +759,8 @@ export class DatasetsResource extends BaseResource {
             return { columns: [], rows: [], rowCount: 0, truncated: false, limit: maxRows, };
         }
         const { columns, rows, truncated, } = await collectPreviewRows(res.body, maxRows, timeoutMs, onHeader);
+        if (!truncated)
+            await this.verifyStreamComplete(datasetName, opts?.projectKey, readSessionId);
         return {
             columns: columns.map((name) => ({ name, })),
             rows,
@@ -730,12 +787,13 @@ export class DatasetsResource extends BaseResource {
         const target = expected;
         const probe = target + 1;
         const timeoutMs = Math.max(1, this.client.getRequestTimeoutMs());
-        const dsEnc = encodeURIComponent(datasetName);
-        const res = await this.client.stream(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${probe}`);
+        const { res, readSessionId, } = await this.dataStream(datasetName, probe, projectKey, undefined);
         if (!res.body) {
             return { expected: target, count: 0, exact: true, satisfied: target === 0, };
         }
         const { count, bounded, } = await collectTsvRowCount(res.body, probe, timeoutMs);
+        if (!bounded)
+            await this.verifyStreamComplete(datasetName, projectKey, readSessionId);
         return { expected: target, count, exact: !bounded, satisfied: !bounded && count === target, };
     }
     /** Get dataset metadata (tags, custom fields, checklists). */
@@ -873,7 +931,7 @@ export class DatasetsResource extends BaseResource {
      */
     async download(datasetName, opts) {
         const limit = Math.max(1, opts?.limit ?? 100_000);
-        const res = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
+        const { res, readSessionId, } = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
         const safeDatasetName = sanitizeFileName(datasetName, "dataset");
         const filePath = opts?.outputPath?.endsWith(".gz") || opts?.outputPath?.endsWith(".csv")
             ? nodePath.resolve(opts.outputPath)
@@ -895,6 +953,9 @@ export class DatasetsResource extends BaseResource {
         try {
             // Native streaming write with backpressure; no Node stream layer.
             await Bun.write(filePath, new Response(csv));
+            if (!stats.truncated) {
+                await this.verifyStreamComplete(datasetName, opts?.projectKey, readSessionId);
+            }
         }
         catch (error) {
             // A failed export (byte-limit overflow, network error) must never