dataiku-sdk 3.9.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +4 -3
  2. package/dist/build-metadata.json +1 -1
  3. package/dist/packages/types/src/index.d.ts +6 -0
  4. package/dist/packages/types/src/index.js +6 -0
  5. package/dist/src/cli/coerce.d.ts +6 -0
  6. package/dist/src/cli/coerce.js +9 -0
  7. package/dist/src/cli/command-syntax.json +216 -0
  8. package/dist/src/cli/commands/app.js +2 -2
  9. package/dist/src/cli/commands/code-env.js +10 -9
  10. package/dist/src/cli/commands/code.js +16 -2
  11. package/dist/src/cli/commands/dashboard.js +1 -2
  12. package/dist/src/cli/commands/dataset.js +6 -4
  13. package/dist/src/cli/commands/flow-zone.js +1 -1
  14. package/dist/src/cli/commands/job.js +70 -16
  15. package/dist/src/cli/commands/plugin.js +7 -7
  16. package/dist/src/cli/commands/project-library.js +2 -2
  17. package/dist/src/cli/commands/project.js +21 -4
  18. package/dist/src/cli/commands/recipe.js +34 -21
  19. package/dist/src/cli/commands/scenario.js +78 -12
  20. package/dist/src/cli/contract.js +30 -7
  21. package/dist/src/cli/env.d.ts +6 -0
  22. package/dist/src/cli/env.js +10 -3
  23. package/dist/src/cli/helpers/app-permissions.d.ts +1 -1
  24. package/dist/src/cli/helpers/app-permissions.js +2 -2
  25. package/dist/src/cli/helpers/cleanup.d.ts +2 -0
  26. package/dist/src/cli/helpers/cleanup.js +122 -8
  27. package/dist/src/cli/helpers/flow-zone.d.ts +1 -1
  28. package/dist/src/cli/helpers/flow-zone.js +3 -4
  29. package/dist/src/cli/list-output.d.ts +5 -0
  30. package/dist/src/cli/list-output.js +11 -2
  31. package/dist/src/cli/list-shapes.d.ts +6 -0
  32. package/dist/src/cli/list-shapes.js +7 -1
  33. package/dist/src/cli/output-schemas.d.ts +1 -1
  34. package/dist/src/cli/output-schemas.js +1 -1
  35. package/dist/src/cli/output.d.ts +2 -0
  36. package/dist/src/cli/output.js +31 -4
  37. package/dist/src/cli/read-shapes.js +90 -0
  38. package/dist/src/cli/result-shapes.d.ts +19 -1
  39. package/dist/src/cli/result-shapes.js +227 -4
  40. package/dist/src/cli/selector-suggestions.d.ts +14 -0
  41. package/dist/src/cli/selector-suggestions.js +56 -0
  42. package/dist/src/cli/usage.d.ts +1 -1
  43. package/dist/src/cli/usage.js +29 -4
  44. package/dist/src/cli/version.d.ts +2 -2
  45. package/dist/src/cli/version.js +4 -5
  46. package/dist/src/cli.js +128 -30
  47. package/dist/src/client.d.ts +15 -2
  48. package/dist/src/client.js +36 -11
  49. package/dist/src/errors.d.ts +2 -0
  50. package/dist/src/errors.js +12 -2
  51. package/dist/src/generated/action-output-schemas.json +232 -0
  52. package/dist/src/index.d.ts +1 -1
  53. package/dist/src/resources/datasets.d.ts +11 -1
  54. package/dist/src/resources/datasets.js +91 -30
  55. package/dist/src/resources/jobs.d.ts +35 -5
  56. package/dist/src/resources/jobs.js +68 -41
  57. package/dist/src/resources/projects.js +2 -2
  58. package/dist/src/resources/recipe-create.d.ts +4 -1
  59. package/dist/src/resources/recipe-create.js +11 -2
  60. package/dist/src/resources/recipes.d.ts +12 -1
  61. package/dist/src/resources/recipes.js +39 -12
  62. package/dist/src/resources/scenarios.d.ts +8 -1
  63. package/dist/src/resources/scenarios.js +51 -12
  64. package/dist/src/resources/sql.js +3 -3
  65. package/dist/src/resources/wiki.js +3 -7
  66. package/dist/src/skill.js +1 -2
  67. package/dist/src/utils/project-archive.d.ts +25 -0
  68. package/dist/src/utils/project-archive.js +216 -5
  69. package/dist/src/utils/response-file.js +15 -8
  70. package/package.json +12 -12
  71. package/plugin.json +1 -1
  72. package/skills/dataiku-dss/SKILL.md +15 -17
  73. package/skills/dataiku-dss/references/discovery.md +2 -2
  74. package/skills/dataiku-dss/references/troubleshooting.md +1 -0
@@ -1119,6 +1119,18 @@
1119
1119
  "timedOut": {
1120
1120
  "type": "boolean"
1121
1121
  },
1122
+ "cancelled": {
1123
+ "type": "boolean"
1124
+ },
1125
+ "resume": {
1126
+ "type": "string"
1127
+ },
1128
+ "resumeArgv": {
1129
+ "type": "array",
1130
+ "items": {
1131
+ "type": "string"
1132
+ }
1133
+ },
1122
1134
  "progress": {
1123
1135
  "type": "object",
1124
1136
  "required": [
@@ -1187,6 +1199,18 @@
1187
1199
  "timedOut": {
1188
1200
  "type": "boolean"
1189
1201
  },
1202
+ "cancelled": {
1203
+ "type": "boolean"
1204
+ },
1205
+ "resume": {
1206
+ "type": "string"
1207
+ },
1208
+ "resumeArgv": {
1209
+ "type": "array",
1210
+ "items": {
1211
+ "type": "string"
1212
+ }
1213
+ },
1190
1214
  "progress": {
1191
1215
  "type": "object",
1192
1216
  "required": [
@@ -1577,6 +1601,12 @@
1577
1601
  "string",
1578
1602
  "null"
1579
1603
  ]
1604
+ },
1605
+ "isDev": {
1606
+ "type": [
1607
+ "boolean",
1608
+ "null"
1609
+ ]
1580
1610
  }
1581
1611
  }
1582
1612
  }
@@ -1958,6 +1988,208 @@
1958
1988
  }
1959
1989
  }
1960
1990
  },
1991
+ "scenario.get-run": {
1992
+ "type": "object",
1993
+ "additionalProperties": false,
1994
+ "required": [
1995
+ "runId",
1996
+ "steps"
1997
+ ],
1998
+ "properties": {
1999
+ "runId": {
2000
+ "type": "string"
2001
+ },
2002
+ "scenarioId": {
2003
+ "type": "string"
2004
+ },
2005
+ "projectKey": {
2006
+ "type": "string"
2007
+ },
2008
+ "outcome": {
2009
+ "type": "string"
2010
+ },
2011
+ "start": {
2012
+ "type": "number"
2013
+ },
2014
+ "end": {
2015
+ "type": "number"
2016
+ },
2017
+ "durationMs": {
2018
+ "type": "number"
2019
+ },
2020
+ "trigger": {
2021
+ "type": "object",
2022
+ "additionalProperties": false,
2023
+ "properties": {
2024
+ "type": {
2025
+ "type": "string"
2026
+ },
2027
+ "name": {
2028
+ "type": "string"
2029
+ }
2030
+ }
2031
+ },
2032
+ "steps": {
2033
+ "type": "array",
2034
+ "items": {
2035
+ "type": "object",
2036
+ "additionalProperties": false,
2037
+ "properties": {
2038
+ "id": {
2039
+ "type": "string"
2040
+ },
2041
+ "name": {
2042
+ "type": "string"
2043
+ },
2044
+ "type": {
2045
+ "type": "string"
2046
+ },
2047
+ "outcome": {
2048
+ "type": "string"
2049
+ },
2050
+ "message": {
2051
+ "type": "string"
2052
+ },
2053
+ "datasets": {
2054
+ "type": "array",
2055
+ "items": {
2056
+ "type": "object",
2057
+ "additionalProperties": false,
2058
+ "properties": {
2059
+ "dataset": {
2060
+ "type": "string"
2061
+ },
2062
+ "projectKey": {
2063
+ "type": "string"
2064
+ },
2065
+ "partition": {
2066
+ "type": "string"
2067
+ },
2068
+ "outcome": {
2069
+ "type": "string"
2070
+ }
2071
+ }
2072
+ }
2073
+ },
2074
+ "jobs": {
2075
+ "type": "array",
2076
+ "items": {
2077
+ "type": "object",
2078
+ "additionalProperties": false,
2079
+ "properties": {
2080
+ "jobId": {
2081
+ "type": "string"
2082
+ },
2083
+ "outcome": {
2084
+ "type": "string"
2085
+ },
2086
+ "message": {
2087
+ "type": "string"
2088
+ }
2089
+ }
2090
+ }
2091
+ }
2092
+ }
2093
+ }
2094
+ },
2095
+ "failure": {
2096
+ "type": "object",
2097
+ "additionalProperties": false,
2098
+ "properties": {
2099
+ "stepId": {
2100
+ "type": "string"
2101
+ },
2102
+ "jobId": {
2103
+ "type": "string"
2104
+ },
2105
+ "dataset": {
2106
+ "type": "string"
2107
+ },
2108
+ "recipe": {
2109
+ "type": "string"
2110
+ },
2111
+ "inputs": {
2112
+ "type": "array",
2113
+ "items": {
2114
+ "type": "string"
2115
+ }
2116
+ },
2117
+ "message": {
2118
+ "type": "string"
2119
+ }
2120
+ }
2121
+ },
2122
+ "next": {
2123
+ "type": "array",
2124
+ "description": "Next commands to run, each as an argv array.",
2125
+ "items": {
2126
+ "type": "array",
2127
+ "items": {
2128
+ "type": "string"
2129
+ }
2130
+ }
2131
+ },
2132
+ "hint": {
2133
+ "type": "string",
2134
+ "description": "Edit-and-rerun commands when the failed recipe is known."
2135
+ }
2136
+ }
2137
+ },
2138
+ "scenario.last-runs": {
2139
+ "type": "array",
2140
+ "items": {
2141
+ "type": "object",
2142
+ "additionalProperties": false,
2143
+ "required": [
2144
+ "runId"
2145
+ ],
2146
+ "properties": {
2147
+ "runId": {
2148
+ "type": "string"
2149
+ },
2150
+ "outcome": {
2151
+ "type": "string"
2152
+ },
2153
+ "start": {
2154
+ "type": "number"
2155
+ },
2156
+ "end": {
2157
+ "type": "number"
2158
+ },
2159
+ "trigger": {
2160
+ "type": "object",
2161
+ "additionalProperties": false,
2162
+ "properties": {
2163
+ "type": {
2164
+ "type": "string"
2165
+ },
2166
+ "name": {
2167
+ "type": "string"
2168
+ }
2169
+ }
2170
+ },
2171
+ "failure": {
2172
+ "type": "object",
2173
+ "additionalProperties": false,
2174
+ "properties": {
2175
+ "message": {
2176
+ "type": "string"
2177
+ }
2178
+ }
2179
+ },
2180
+ "next": {
2181
+ "type": "array",
2182
+ "description": "Next commands to run, each as an argv array.",
2183
+ "items": {
2184
+ "type": "array",
2185
+ "items": {
2186
+ "type": "string"
2187
+ }
2188
+ }
2189
+ }
2190
+ }
2191
+ }
2192
+ },
1961
2193
  "scenario.list": {
1962
2194
  "type": "array",
1963
2195
  "items": {
@@ -22,7 +22,7 @@ export { FoldersResource, } from "./resources/folders.js";
22
22
  export { type DssTask, FuturesResource, type FutureWaitOptions, } from "./resources/futures.js";
23
23
  export { type DssGroup, type GroupCreateRequest, GroupsResource, } from "./resources/groups.js";
24
24
  export { InsightsResource, } from "./resources/insights.js";
25
- export { type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogFilter, type JobLogProgress, type JobLogSummary, type JobLogUnavailableReason, JobsResource, type JobWaitOutcome, parseJobLogProgress, } from "./resources/jobs.js";
25
+ export { type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogDetails, type JobLogFilter, type JobLogProgress, type JobLogSummary, type JobLogUnavailableReason, JobsResource, type JobWaitOutcome, parseJobLogProgress, } from "./resources/jobs.js";
26
26
  export * from "./resources/knowledge-banks.js";
27
27
  export * from "./resources/llms.js";
28
28
  export { type MacroResultOptions, type MacroRunFailure, type MacroRunHandle, type MacroRunOptions, type MacroRunWaitedResult, MacrosResource, type MacroWaitOptions, } from "./resources/macros.js";
@@ -135,9 +135,19 @@ export declare class DatasetsResource extends BaseResource {
135
135
  /**
136
136
  * TSV data stream of at most `limit` rows. A column or partition selection
137
137
  * uses the documented POST variant of the data endpoint (body `columns`,
138
- * `partitions`, `format`, `sampling`); otherwise the GET endpoint.
138
+ * `partitions`, `format`, `sampling`); otherwise the GET endpoint. Both carry
139
+ * a client-generated `readSessionId` for {@link verifyStreamComplete}.
139
140
  */
140
141
  private dataStream;
142
+ /**
143
+ * A data stream that ended normally may still be incomplete: DSS sends 200
144
+ * headers before reading the data, so a server-side read failure only cuts
145
+ * the body short. As dataikuapi does, ask DSS whether the read session
146
+ * finished; it answers non-2xx (DataikuError) when it did not. Only called
147
+ * after the body was read to its end — a stream the client cancelled at
148
+ * its row cap is not complete by design.
149
+ */
150
+ private verifyStreamComplete;
141
151
  /**
142
152
  * Preview dataset rows as structured data: column names plus row arrays,
143
153
  * mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
@@ -1,6 +1,5 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as nodePath from "node:path";
3
- import { StringDecoder, } from "node:string_decoder";
4
3
  import { ClientValidationError, DataikuError, } from "../errors.js";
5
4
  import { DatasetDetailsSchema, DatasetSchemaSchema, DatasetSummaryArraySchema, } from "../schemas.js";
6
5
  import { deepMerge, } from "../utils/deep-merge.js";
@@ -28,8 +27,44 @@ function formatLimitBytes(bytes) {
28
27
  return `${bytes / (1024 * 1024)} MiB`;
29
28
  return `${bytes} B`;
30
29
  }
31
- function tsvLimitError(what, limit) {
32
- return new DataikuError(0, "Dataset TSV Limit", `Dataset TSV ${what} exceeded ${formatLimitBytes(limit)}; streaming was aborted to guard against unbounded memory use.`);
30
+ const TSV_ERROR_COLUMN_NAME_MAX = 40;
31
+ /**
32
+ * Describe the column whose cell is being read when a field/row budget trips:
33
+ * the header name once the header row has been parsed, otherwise the 0-based
34
+ * column index. Error summaries are bounded (~200 chars), so the name is
35
+ * shortened to keep the `--columns` hint inside the visible message.
36
+ */
37
+ function describeTsvColumn(state) {
38
+ const index = state.currentRow.length;
39
+ const name = state.header?.[index];
40
+ if (name === undefined)
41
+ return `column index ${index}`;
42
+ const shown = name.length > TSV_ERROR_COLUMN_NAME_MAX
43
+ ? `${name.slice(0, TSV_ERROR_COLUMN_NAME_MAX)}...`
44
+ : name;
45
+ return `column "${shown}"`;
46
+ }
47
+ /**
48
+ * A client-side size budget, not a DSS or transport failure: retrying the same
49
+ * read cannot succeed, so this is a non-retryable input error whose fix is a
50
+ * narrower selection (--columns).
51
+ */
52
+ function tsvLimitError(what, limit, state) {
53
+ const exceeded = `Dataset TSV ${what} exceeded ${formatLimitBytes(limit)}`;
54
+ const cellLimit = formatLimitBytes(state.limits.maxFieldBytes);
55
+ let message;
56
+ if (what === "field") {
57
+ message = `${exceeded} in ${describeTsvColumn(state)} (a single cell is limited to ${cellLimit}); exclude it with --columns.`;
58
+ }
59
+ else if (what === "row") {
60
+ message = `${exceeded} (a single cell is limited to ${cellLimit}); reached in ${describeTsvColumn(state)}. Exclude large columns with --columns.`;
61
+ }
62
+ else {
63
+ message = `${exceeded}; streaming was aborted to guard against unbounded memory use.`;
64
+ }
65
+ return new ClientValidationError(message, "validation_failed", what === "response"
66
+ ? "Narrow the read with --columns, --partitions, or a lower row limit."
67
+ : "Select fewer columns with --columns so the oversized cell is not read.");
33
68
  }
34
69
  /** Characters that evaluate a spreadsheet cell as a formula. */
35
70
  const FORMULA_SIGILS = "=+-@";
@@ -75,6 +110,7 @@ function createTsvStreamState(limits = {
75
110
  return {
76
111
  currentField: "",
77
112
  currentRow: [],
113
+ header: undefined,
78
114
  inQuotes: false,
79
115
  pendingQuoteInQuotes: false,
80
116
  fieldBytes: 0,
@@ -91,10 +127,10 @@ function appendFieldText(state, text) {
91
127
  state.fieldBytes += bytes;
92
128
  state.rowBytes += bytes;
93
129
  if (state.fieldBytes > state.limits.maxFieldBytes) {
94
- throw tsvLimitError("field", state.limits.maxFieldBytes);
130
+ throw tsvLimitError("field", state.limits.maxFieldBytes, state);
95
131
  }
96
132
  if (state.rowBytes > state.limits.maxRowBytes) {
97
- throw tsvLimitError("row", state.limits.maxRowBytes);
133
+ throw tsvLimitError("row", state.limits.maxRowBytes, state);
98
134
  }
99
135
  state.currentField += text;
100
136
  }
@@ -105,11 +141,20 @@ function endField(state) {
105
141
  state.fieldBytes = 0;
106
142
  state.rowBytes += 1; // count the delimiter byte
107
143
  }
144
+ /** Hand a completed row to `onRow`, remembering the first non-blank row as the header. */
145
+ function emitRow(state, onRow) {
146
+ const row = state.currentRow;
147
+ state.currentRow = [];
148
+ state.rowBytes = 0;
149
+ if (state.header === undefined && !isBlankRow(row))
150
+ state.header = row;
151
+ onRow(row);
152
+ }
108
153
  function consumeTsvChunk(text, state, onRow) {
109
154
  // Response budget: bound total decoded bytes regardless of row structure.
110
155
  state.responseBytes += Buffer.byteLength(text, "utf-8");
111
156
  if (state.responseBytes > state.limits.maxResponseBytes) {
112
- throw tsvLimitError("response", state.limits.maxResponseBytes);
157
+ throw tsvLimitError("response", state.limits.maxResponseBytes, state);
113
158
  }
114
159
  let i = 0;
115
160
  if (state.pendingQuoteInQuotes) {
@@ -193,10 +238,7 @@ function consumeTsvChunk(text, state, onRow) {
193
238
  if (ch === "\n") {
194
239
  endField(state);
195
240
  i = special + 1;
196
- const row = state.currentRow;
197
- state.currentRow = [];
198
- state.rowBytes = 0;
199
- onRow(row);
241
+ emitRow(state, onRow);
200
242
  continue;
201
243
  }
202
244
  // "\r" — carriage return: skipped, as in the original parser.
@@ -213,10 +255,7 @@ function flushTsvStream(state, onRow) {
213
255
  state.currentRow.push(state.currentField);
214
256
  state.currentField = "";
215
257
  state.fieldBytes = 0;
216
- const row = state.currentRow;
217
- state.currentRow = [];
218
- state.rowBytes = 0;
219
- onRow(row);
258
+ emitRow(state, onRow);
220
259
  }
221
260
  function rowToCsv(row, spreadsheetSafe) {
222
261
  return row.map((field) => csvEscape(field, spreadsheetSafe)).join(",");
@@ -264,7 +303,7 @@ async function readChunkWithTimeout(reader, remainingMs, timeoutMs, action) {
264
303
  }
265
304
  async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
266
305
  const state = createTsvStreamState();
267
- const decoder = new StringDecoder("utf-8");
306
+ const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
268
307
  let columns;
269
308
  const rows = [];
270
309
  let done = false;
@@ -304,10 +343,10 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
304
343
  const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "preview");
305
344
  if (result.done)
306
345
  break;
307
- consumeTsvChunk(decoder.write(result.value), state, handleRow);
346
+ consumeTsvChunk(decoder.decode(result.value, { stream: true, }), state, handleRow);
308
347
  }
309
348
  if (!done) {
310
- const tail = decoder.end();
349
+ const tail = decoder.decode();
311
350
  if (tail !== "")
312
351
  consumeTsvChunk(tail, state, handleRow);
313
352
  flushTsvStream(state, handleRow);
@@ -332,7 +371,7 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
332
371
  */
333
372
  async function collectTsvRowCount(body, stopAfter, timeoutMs) {
334
373
  const state = createTsvStreamState();
335
- const decoder = new StringDecoder("utf-8");
374
+ const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
336
375
  let headerSeen = false;
337
376
  let count = 0;
338
377
  let bounded = false;
@@ -373,10 +412,10 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
373
412
  const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "row count assertion");
374
413
  if (result.done)
375
414
  break;
376
- consumeTsvChunk(decoder.write(result.value), state, handleRow);
415
+ consumeTsvChunk(decoder.decode(result.value, { stream: true, }), state, handleRow);
377
416
  }
378
417
  if (!bounded) {
379
- const tail = decoder.end();
418
+ const tail = decoder.decode();
380
419
  if (tail !== "")
381
420
  consumeTsvChunk(tail, state, handleRow);
382
421
  flushTsvStream(state, handleRow);
@@ -402,7 +441,7 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
402
441
  const CSV_BATCH_BYTES = 64 * 1024;
403
442
  function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
404
443
  const state = createTsvStreamState();
405
- const decoder = new StringDecoder("utf-8");
444
+ const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
406
445
  const encoder = new TextEncoder();
407
446
  const maxRows = Math.max(1, maxDataRows);
408
447
  let headerSeen = false;
@@ -454,7 +493,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
454
493
  // which cancels the response and makes download() remove the partial file.
455
494
  return new TransformStream({
456
495
  transform(chunk, controller) {
457
- consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
496
+ consumeTsvChunk(decoder.decode(chunk, { stream: true, }), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
458
497
  if (done) {
459
498
  // Truncation reached: emit the batch queued so far, end the output,
460
499
  // and cancel the rest of the response.
@@ -463,7 +502,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
463
502
  }
464
503
  },
465
504
  flush(controller) {
466
- const tail = decoder.end();
505
+ const tail = decoder.decode();
467
506
  if (tail !== "") {
468
507
  consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(controller, line)));
469
508
  }
@@ -660,19 +699,35 @@ export class DatasetsResource extends BaseResource {
660
699
  /**
661
700
  * TSV data stream of at most `limit` rows. A column or partition selection
662
701
  * uses the documented POST variant of the data endpoint (body `columns`,
663
- * `partitions`, `format`, `sampling`); otherwise the GET endpoint.
702
+ * `partitions`, `format`, `sampling`); otherwise the GET endpoint. Both carry
703
+ * a client-generated `readSessionId` for {@link verifyStreamComplete}.
664
704
  */
665
705
  async dataStream(datasetName, limit, projectKey, selection) {
706
+ const readSessionId = crypto.randomUUID();
666
707
  const path = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/data/`;
667
708
  if (!selection?.columns?.length && !selection?.partitions) {
668
- return this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}`);
709
+ const res = await this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}&readSessionId=${readSessionId}`);
710
+ return { res, readSessionId, };
669
711
  }
670
- return this.client.postStream(path, {
712
+ const res = await this.client.postStream(path, {
671
713
  format: "tsv-excel-header",
672
714
  ...(selection.columns?.length ? { columns: selection.columns, } : {}),
673
715
  ...(selection.partitions ? { partitions: selection.partitions, } : {}),
674
716
  sampling: { samplingMethod: "HEAD_SEQUENTIAL", maxRecords: limit, },
717
+ readSessionId,
675
718
  });
719
+ return { res, readSessionId, };
720
+ }
721
+ /**
722
+ * A data stream that ended normally may still be incomplete: DSS sends 200
723
+ * headers before reading the data, so a server-side read failure only cuts
724
+ * the body short. As dataikuapi does, ask DSS whether the read session
725
+ * finished; it answers non-2xx (DataikuError) when it did not. Only called
726
+ * after the body was read to its end — a stream the client cancelled at
727
+ * its row cap is not complete by design.
728
+ */
729
+ async verifyStreamComplete(datasetName, projectKey, readSessionId) {
730
+ await this.client.getTextLimited(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/finish-streaming/?readSessionId=${readSessionId}`, 64 * 1024);
676
731
  }
677
732
  /**
678
733
  * Preview dataset rows as structured data: column names plus row arrays,
@@ -691,7 +746,7 @@ export class DatasetsResource extends BaseResource {
691
746
  const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
692
747
  // Probe one row past the cap so callers learn whether the dataset holds
693
748
  // more rows than requested without materializing the full dataset.
694
- const res = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
749
+ const { res, readSessionId, } = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
695
750
  const onHeader = opts?.validateColumns
696
751
  ? (headerRow) => {
697
752
  const warnings = validateStreamColumns(headerRow, opts.validateColumns);
@@ -704,6 +759,8 @@ export class DatasetsResource extends BaseResource {
704
759
  return { columns: [], rows: [], rowCount: 0, truncated: false, limit: maxRows, };
705
760
  }
706
761
  const { columns, rows, truncated, } = await collectPreviewRows(res.body, maxRows, timeoutMs, onHeader);
762
+ if (!truncated)
763
+ await this.verifyStreamComplete(datasetName, opts?.projectKey, readSessionId);
707
764
  return {
708
765
  columns: columns.map((name) => ({ name, })),
709
766
  rows,
@@ -730,12 +787,13 @@ export class DatasetsResource extends BaseResource {
730
787
  const target = expected;
731
788
  const probe = target + 1;
732
789
  const timeoutMs = Math.max(1, this.client.getRequestTimeoutMs());
733
- const dsEnc = encodeURIComponent(datasetName);
734
- const res = await this.client.stream(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${probe}`);
790
+ const { res, readSessionId, } = await this.dataStream(datasetName, probe, projectKey, undefined);
735
791
  if (!res.body) {
736
792
  return { expected: target, count: 0, exact: true, satisfied: target === 0, };
737
793
  }
738
794
  const { count, bounded, } = await collectTsvRowCount(res.body, probe, timeoutMs);
795
+ if (!bounded)
796
+ await this.verifyStreamComplete(datasetName, projectKey, readSessionId);
739
797
  return { expected: target, count, exact: !bounded, satisfied: !bounded && count === target, };
740
798
  }
741
799
  /** Get dataset metadata (tags, custom fields, checklists). */
@@ -873,7 +931,7 @@ export class DatasetsResource extends BaseResource {
873
931
  */
874
932
  async download(datasetName, opts) {
875
933
  const limit = Math.max(1, opts?.limit ?? 100_000);
876
- const res = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
934
+ const { res, readSessionId, } = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
877
935
  const safeDatasetName = sanitizeFileName(datasetName, "dataset");
878
936
  const filePath = opts?.outputPath?.endsWith(".gz") || opts?.outputPath?.endsWith(".csv")
879
937
  ? nodePath.resolve(opts.outputPath)
@@ -895,6 +953,9 @@ export class DatasetsResource extends BaseResource {
895
953
  try {
896
954
  // Native streaming write with backpressure; no Node stream layer.
897
955
  await Bun.write(filePath, new Response(csv));
956
+ if (!stats.truncated) {
957
+ await this.verifyStreamComplete(datasetName, opts?.projectKey, readSessionId);
958
+ }
898
959
  }
899
960
  catch (error) {
900
961
  // A failed export (byte-limit overflow, network error) must never
@@ -9,6 +9,20 @@ import { BaseResource } from "./base.js";
9
9
  */
10
10
  export type JobBuildTargetType = "DATASET" | "MANAGED_FOLDER" | "MODEL_EVALUATION_STORE";
11
11
  export type JobLogFilter = "stdout" | "stderr" | "user" | "errors";
12
+ /** A retrieved job log plus what the byte-bounded download dropped or scanned. */
13
+ export interface JobLogDetails {
14
+ /** Redacted, filtered and line-limited log text (what {@link JobsResource.log} returns). */
15
+ text: string;
16
+ /**
17
+ * True when the log exceeded the byte cap and only its most recent bytes
18
+ * were retained: earlier lines were never scanned or filtered.
19
+ */
20
+ truncated: boolean;
21
+ /** Lines in the retained log before filtering and the line limit. */
22
+ scannedLines: number;
23
+ /** Byte cap applied to the download (the most recent this-many bytes are kept). */
24
+ maxLogBytes: number;
25
+ }
12
26
  export interface JobLogProgress {
13
27
  lastProgressLine?: string;
14
28
  doneLine?: string;
@@ -83,11 +97,14 @@ export declare class JobsResource extends BaseResource {
83
97
  * Use `0` or `-1` to return the log without line truncation. Credential-like
84
98
  * JSON pairs (`"jobTicketSecret":"..."`) are redacted.
85
99
  *
86
- * The download is byte-bounded and deadline-covered: at most `maxLogBytes`
87
- * (default 10 MiB) of the *most recent* log output is retained, so a
88
- * collaborator-influenced oversized log cannot exhaust client memory while
89
- * the line limit previously only applied after a full unbounded download.
90
- * Logs up to the cap are returned in full.
100
+ * The download is byte-bounded: at most `maxLogBytes` (default 10 MiB) of
101
+ * the *most recent* log output is retained, so a collaborator-influenced
102
+ * oversized log cannot exhaust client memory while the line limit
103
+ * previously only applied after a full unbounded download. The request
104
+ * timeout is a stall deadline (per received chunk), so a large log that
105
+ * keeps arriving is not cut off by a total-transfer budget. Logs up to the
106
+ * cap are returned in full. Use {@link logDetails} to learn whether bytes
107
+ * were dropped.
91
108
  */
92
109
  log(jobId: string, opts?: {
93
110
  activity?: string;
@@ -97,6 +114,19 @@ export declare class JobsResource extends BaseResource {
97
114
  maxLogBytes?: number;
98
115
  projectKey?: string;
99
116
  }): Promise<string>;
117
+ /**
118
+ * Same retrieval as {@link log}, additionally reporting whether the
119
+ * byte-bounded download dropped early log bytes (`truncated`) and how many
120
+ * lines were scanned before filtering (`scannedLines`).
121
+ */
122
+ logDetails(jobId: string, opts?: {
123
+ activity?: string;
124
+ logId?: string;
125
+ logFilter?: JobLogFilter;
126
+ maxLogLines?: number;
127
+ maxLogBytes?: number;
128
+ projectKey?: string;
129
+ }): Promise<JobLogDetails>;
100
130
  logFromUrl(logUrl: string, opts?: {
101
131
  maxLogLines?: number;
102
132
  maxLogBytes?: number;