dataiku-sdk 3.9.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +6 -0
- package/dist/packages/types/src/index.js +6 -0
- package/dist/src/cli/coerce.d.ts +6 -0
- package/dist/src/cli/coerce.js +9 -0
- package/dist/src/cli/command-syntax.json +216 -0
- package/dist/src/cli/commands/app.js +2 -2
- package/dist/src/cli/commands/code-env.js +10 -9
- package/dist/src/cli/commands/code.js +16 -2
- package/dist/src/cli/commands/dashboard.js +1 -2
- package/dist/src/cli/commands/dataset.js +6 -4
- package/dist/src/cli/commands/flow-zone.js +1 -1
- package/dist/src/cli/commands/job.js +70 -16
- package/dist/src/cli/commands/plugin.js +7 -7
- package/dist/src/cli/commands/project-library.js +2 -2
- package/dist/src/cli/commands/project.js +21 -4
- package/dist/src/cli/commands/recipe.js +34 -21
- package/dist/src/cli/commands/scenario.js +78 -12
- package/dist/src/cli/contract.js +30 -7
- package/dist/src/cli/env.d.ts +6 -0
- package/dist/src/cli/env.js +10 -3
- package/dist/src/cli/helpers/app-permissions.d.ts +1 -1
- package/dist/src/cli/helpers/app-permissions.js +2 -2
- package/dist/src/cli/helpers/cleanup.d.ts +2 -0
- package/dist/src/cli/helpers/cleanup.js +122 -8
- package/dist/src/cli/helpers/flow-zone.d.ts +1 -1
- package/dist/src/cli/helpers/flow-zone.js +3 -4
- package/dist/src/cli/list-output.d.ts +5 -0
- package/dist/src/cli/list-output.js +11 -2
- package/dist/src/cli/list-shapes.d.ts +6 -0
- package/dist/src/cli/list-shapes.js +7 -1
- package/dist/src/cli/output-schemas.d.ts +1 -1
- package/dist/src/cli/output-schemas.js +1 -1
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +31 -4
- package/dist/src/cli/read-shapes.js +90 -0
- package/dist/src/cli/result-shapes.d.ts +19 -1
- package/dist/src/cli/result-shapes.js +227 -4
- package/dist/src/cli/selector-suggestions.d.ts +14 -0
- package/dist/src/cli/selector-suggestions.js +56 -0
- package/dist/src/cli/usage.d.ts +1 -1
- package/dist/src/cli/usage.js +29 -4
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +4 -5
- package/dist/src/cli.js +128 -30
- package/dist/src/client.d.ts +15 -2
- package/dist/src/client.js +36 -11
- package/dist/src/errors.d.ts +2 -0
- package/dist/src/errors.js +12 -2
- package/dist/src/generated/action-output-schemas.json +232 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/resources/datasets.d.ts +11 -1
- package/dist/src/resources/datasets.js +91 -30
- package/dist/src/resources/jobs.d.ts +35 -5
- package/dist/src/resources/jobs.js +68 -41
- package/dist/src/resources/projects.js +2 -2
- package/dist/src/resources/recipe-create.d.ts +4 -1
- package/dist/src/resources/recipe-create.js +11 -2
- package/dist/src/resources/recipes.d.ts +12 -1
- package/dist/src/resources/recipes.js +39 -12
- package/dist/src/resources/scenarios.d.ts +8 -1
- package/dist/src/resources/scenarios.js +51 -12
- package/dist/src/resources/sql.js +3 -3
- package/dist/src/resources/wiki.js +3 -7
- package/dist/src/skill.js +1 -2
- package/dist/src/utils/project-archive.d.ts +25 -0
- package/dist/src/utils/project-archive.js +216 -5
- package/dist/src/utils/response-file.js +15 -8
- package/package.json +12 -12
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +15 -17
- package/skills/dataiku-dss/references/discovery.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +1 -0
|
@@ -1119,6 +1119,18 @@
|
|
|
1119
1119
|
"timedOut": {
|
|
1120
1120
|
"type": "boolean"
|
|
1121
1121
|
},
|
|
1122
|
+
"cancelled": {
|
|
1123
|
+
"type": "boolean"
|
|
1124
|
+
},
|
|
1125
|
+
"resume": {
|
|
1126
|
+
"type": "string"
|
|
1127
|
+
},
|
|
1128
|
+
"resumeArgv": {
|
|
1129
|
+
"type": "array",
|
|
1130
|
+
"items": {
|
|
1131
|
+
"type": "string"
|
|
1132
|
+
}
|
|
1133
|
+
},
|
|
1122
1134
|
"progress": {
|
|
1123
1135
|
"type": "object",
|
|
1124
1136
|
"required": [
|
|
@@ -1187,6 +1199,18 @@
|
|
|
1187
1199
|
"timedOut": {
|
|
1188
1200
|
"type": "boolean"
|
|
1189
1201
|
},
|
|
1202
|
+
"cancelled": {
|
|
1203
|
+
"type": "boolean"
|
|
1204
|
+
},
|
|
1205
|
+
"resume": {
|
|
1206
|
+
"type": "string"
|
|
1207
|
+
},
|
|
1208
|
+
"resumeArgv": {
|
|
1209
|
+
"type": "array",
|
|
1210
|
+
"items": {
|
|
1211
|
+
"type": "string"
|
|
1212
|
+
}
|
|
1213
|
+
},
|
|
1190
1214
|
"progress": {
|
|
1191
1215
|
"type": "object",
|
|
1192
1216
|
"required": [
|
|
@@ -1577,6 +1601,12 @@
|
|
|
1577
1601
|
"string",
|
|
1578
1602
|
"null"
|
|
1579
1603
|
]
|
|
1604
|
+
},
|
|
1605
|
+
"isDev": {
|
|
1606
|
+
"type": [
|
|
1607
|
+
"boolean",
|
|
1608
|
+
"null"
|
|
1609
|
+
]
|
|
1580
1610
|
}
|
|
1581
1611
|
}
|
|
1582
1612
|
}
|
|
@@ -1958,6 +1988,208 @@
|
|
|
1958
1988
|
}
|
|
1959
1989
|
}
|
|
1960
1990
|
},
|
|
1991
|
+
"scenario.get-run": {
|
|
1992
|
+
"type": "object",
|
|
1993
|
+
"additionalProperties": false,
|
|
1994
|
+
"required": [
|
|
1995
|
+
"runId",
|
|
1996
|
+
"steps"
|
|
1997
|
+
],
|
|
1998
|
+
"properties": {
|
|
1999
|
+
"runId": {
|
|
2000
|
+
"type": "string"
|
|
2001
|
+
},
|
|
2002
|
+
"scenarioId": {
|
|
2003
|
+
"type": "string"
|
|
2004
|
+
},
|
|
2005
|
+
"projectKey": {
|
|
2006
|
+
"type": "string"
|
|
2007
|
+
},
|
|
2008
|
+
"outcome": {
|
|
2009
|
+
"type": "string"
|
|
2010
|
+
},
|
|
2011
|
+
"start": {
|
|
2012
|
+
"type": "number"
|
|
2013
|
+
},
|
|
2014
|
+
"end": {
|
|
2015
|
+
"type": "number"
|
|
2016
|
+
},
|
|
2017
|
+
"durationMs": {
|
|
2018
|
+
"type": "number"
|
|
2019
|
+
},
|
|
2020
|
+
"trigger": {
|
|
2021
|
+
"type": "object",
|
|
2022
|
+
"additionalProperties": false,
|
|
2023
|
+
"properties": {
|
|
2024
|
+
"type": {
|
|
2025
|
+
"type": "string"
|
|
2026
|
+
},
|
|
2027
|
+
"name": {
|
|
2028
|
+
"type": "string"
|
|
2029
|
+
}
|
|
2030
|
+
}
|
|
2031
|
+
},
|
|
2032
|
+
"steps": {
|
|
2033
|
+
"type": "array",
|
|
2034
|
+
"items": {
|
|
2035
|
+
"type": "object",
|
|
2036
|
+
"additionalProperties": false,
|
|
2037
|
+
"properties": {
|
|
2038
|
+
"id": {
|
|
2039
|
+
"type": "string"
|
|
2040
|
+
},
|
|
2041
|
+
"name": {
|
|
2042
|
+
"type": "string"
|
|
2043
|
+
},
|
|
2044
|
+
"type": {
|
|
2045
|
+
"type": "string"
|
|
2046
|
+
},
|
|
2047
|
+
"outcome": {
|
|
2048
|
+
"type": "string"
|
|
2049
|
+
},
|
|
2050
|
+
"message": {
|
|
2051
|
+
"type": "string"
|
|
2052
|
+
},
|
|
2053
|
+
"datasets": {
|
|
2054
|
+
"type": "array",
|
|
2055
|
+
"items": {
|
|
2056
|
+
"type": "object",
|
|
2057
|
+
"additionalProperties": false,
|
|
2058
|
+
"properties": {
|
|
2059
|
+
"dataset": {
|
|
2060
|
+
"type": "string"
|
|
2061
|
+
},
|
|
2062
|
+
"projectKey": {
|
|
2063
|
+
"type": "string"
|
|
2064
|
+
},
|
|
2065
|
+
"partition": {
|
|
2066
|
+
"type": "string"
|
|
2067
|
+
},
|
|
2068
|
+
"outcome": {
|
|
2069
|
+
"type": "string"
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
},
|
|
2074
|
+
"jobs": {
|
|
2075
|
+
"type": "array",
|
|
2076
|
+
"items": {
|
|
2077
|
+
"type": "object",
|
|
2078
|
+
"additionalProperties": false,
|
|
2079
|
+
"properties": {
|
|
2080
|
+
"jobId": {
|
|
2081
|
+
"type": "string"
|
|
2082
|
+
},
|
|
2083
|
+
"outcome": {
|
|
2084
|
+
"type": "string"
|
|
2085
|
+
},
|
|
2086
|
+
"message": {
|
|
2087
|
+
"type": "string"
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
},
|
|
2095
|
+
"failure": {
|
|
2096
|
+
"type": "object",
|
|
2097
|
+
"additionalProperties": false,
|
|
2098
|
+
"properties": {
|
|
2099
|
+
"stepId": {
|
|
2100
|
+
"type": "string"
|
|
2101
|
+
},
|
|
2102
|
+
"jobId": {
|
|
2103
|
+
"type": "string"
|
|
2104
|
+
},
|
|
2105
|
+
"dataset": {
|
|
2106
|
+
"type": "string"
|
|
2107
|
+
},
|
|
2108
|
+
"recipe": {
|
|
2109
|
+
"type": "string"
|
|
2110
|
+
},
|
|
2111
|
+
"inputs": {
|
|
2112
|
+
"type": "array",
|
|
2113
|
+
"items": {
|
|
2114
|
+
"type": "string"
|
|
2115
|
+
}
|
|
2116
|
+
},
|
|
2117
|
+
"message": {
|
|
2118
|
+
"type": "string"
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
},
|
|
2122
|
+
"next": {
|
|
2123
|
+
"type": "array",
|
|
2124
|
+
"description": "Next commands to run, each as an argv array.",
|
|
2125
|
+
"items": {
|
|
2126
|
+
"type": "array",
|
|
2127
|
+
"items": {
|
|
2128
|
+
"type": "string"
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
},
|
|
2132
|
+
"hint": {
|
|
2133
|
+
"type": "string",
|
|
2134
|
+
"description": "Edit-and-rerun commands when the failed recipe is known."
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
},
|
|
2138
|
+
"scenario.last-runs": {
|
|
2139
|
+
"type": "array",
|
|
2140
|
+
"items": {
|
|
2141
|
+
"type": "object",
|
|
2142
|
+
"additionalProperties": false,
|
|
2143
|
+
"required": [
|
|
2144
|
+
"runId"
|
|
2145
|
+
],
|
|
2146
|
+
"properties": {
|
|
2147
|
+
"runId": {
|
|
2148
|
+
"type": "string"
|
|
2149
|
+
},
|
|
2150
|
+
"outcome": {
|
|
2151
|
+
"type": "string"
|
|
2152
|
+
},
|
|
2153
|
+
"start": {
|
|
2154
|
+
"type": "number"
|
|
2155
|
+
},
|
|
2156
|
+
"end": {
|
|
2157
|
+
"type": "number"
|
|
2158
|
+
},
|
|
2159
|
+
"trigger": {
|
|
2160
|
+
"type": "object",
|
|
2161
|
+
"additionalProperties": false,
|
|
2162
|
+
"properties": {
|
|
2163
|
+
"type": {
|
|
2164
|
+
"type": "string"
|
|
2165
|
+
},
|
|
2166
|
+
"name": {
|
|
2167
|
+
"type": "string"
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2170
|
+
},
|
|
2171
|
+
"failure": {
|
|
2172
|
+
"type": "object",
|
|
2173
|
+
"additionalProperties": false,
|
|
2174
|
+
"properties": {
|
|
2175
|
+
"message": {
|
|
2176
|
+
"type": "string"
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
},
|
|
2180
|
+
"next": {
|
|
2181
|
+
"type": "array",
|
|
2182
|
+
"description": "Next commands to run, each as an argv array.",
|
|
2183
|
+
"items": {
|
|
2184
|
+
"type": "array",
|
|
2185
|
+
"items": {
|
|
2186
|
+
"type": "string"
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
},
|
|
1961
2193
|
"scenario.list": {
|
|
1962
2194
|
"type": "array",
|
|
1963
2195
|
"items": {
|
package/dist/src/index.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ export { FoldersResource, } from "./resources/folders.js";
|
|
|
22
22
|
export { type DssTask, FuturesResource, type FutureWaitOptions, } from "./resources/futures.js";
|
|
23
23
|
export { type DssGroup, type GroupCreateRequest, GroupsResource, } from "./resources/groups.js";
|
|
24
24
|
export { InsightsResource, } from "./resources/insights.js";
|
|
25
|
-
export { type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogFilter, type JobLogProgress, type JobLogSummary, type JobLogUnavailableReason, JobsResource, type JobWaitOutcome, parseJobLogProgress, } from "./resources/jobs.js";
|
|
25
|
+
export { type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogDetails, type JobLogFilter, type JobLogProgress, type JobLogSummary, type JobLogUnavailableReason, JobsResource, type JobWaitOutcome, parseJobLogProgress, } from "./resources/jobs.js";
|
|
26
26
|
export * from "./resources/knowledge-banks.js";
|
|
27
27
|
export * from "./resources/llms.js";
|
|
28
28
|
export { type MacroResultOptions, type MacroRunFailure, type MacroRunHandle, type MacroRunOptions, type MacroRunWaitedResult, MacrosResource, type MacroWaitOptions, } from "./resources/macros.js";
|
|
@@ -135,9 +135,19 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
135
135
|
/**
|
|
136
136
|
* TSV data stream of at most `limit` rows. A column or partition selection
|
|
137
137
|
* uses the documented POST variant of the data endpoint (body `columns`,
|
|
138
|
-
* `partitions`, `format`, `sampling`); otherwise the GET endpoint.
|
|
138
|
+
* `partitions`, `format`, `sampling`); otherwise the GET endpoint. Both carry
|
|
139
|
+
* a client-generated `readSessionId` for {@link verifyStreamComplete}.
|
|
139
140
|
*/
|
|
140
141
|
private dataStream;
|
|
142
|
+
/**
|
|
143
|
+
* A data stream that ended normally may still be incomplete: DSS sends 200
|
|
144
|
+
* headers before reading the data, so a server-side read failure only cuts
|
|
145
|
+
* the body short. As dataikuapi does, ask DSS whether the read session
|
|
146
|
+
* finished; it answers non-2xx (DataikuError) when it did not. Only called
|
|
147
|
+
* after the body was read to its end — a stream the client cancelled at
|
|
148
|
+
* its row cap is not complete by design.
|
|
149
|
+
*/
|
|
150
|
+
private verifyStreamComplete;
|
|
141
151
|
/**
|
|
142
152
|
* Preview dataset rows as structured data: column names plus row arrays,
|
|
143
153
|
* mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as nodePath from "node:path";
|
|
3
|
-
import { StringDecoder, } from "node:string_decoder";
|
|
4
3
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
5
4
|
import { DatasetDetailsSchema, DatasetSchemaSchema, DatasetSummaryArraySchema, } from "../schemas.js";
|
|
6
5
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
@@ -28,8 +27,44 @@ function formatLimitBytes(bytes) {
|
|
|
28
27
|
return `${bytes / (1024 * 1024)} MiB`;
|
|
29
28
|
return `${bytes} B`;
|
|
30
29
|
}
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
const TSV_ERROR_COLUMN_NAME_MAX = 40;
|
|
31
|
+
/**
|
|
32
|
+
* Describe the column whose cell is being read when a field/row budget trips:
|
|
33
|
+
* the header name once the header row has been parsed, otherwise the 0-based
|
|
34
|
+
* column index. Error summaries are bounded (~200 chars), so the name is
|
|
35
|
+
* shortened to keep the `--columns` hint inside the visible message.
|
|
36
|
+
*/
|
|
37
|
+
function describeTsvColumn(state) {
|
|
38
|
+
const index = state.currentRow.length;
|
|
39
|
+
const name = state.header?.[index];
|
|
40
|
+
if (name === undefined)
|
|
41
|
+
return `column index ${index}`;
|
|
42
|
+
const shown = name.length > TSV_ERROR_COLUMN_NAME_MAX
|
|
43
|
+
? `${name.slice(0, TSV_ERROR_COLUMN_NAME_MAX)}...`
|
|
44
|
+
: name;
|
|
45
|
+
return `column "${shown}"`;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A client-side size budget, not a DSS or transport failure: retrying the same
|
|
49
|
+
* read cannot succeed, so this is a non-retryable input error whose fix is a
|
|
50
|
+
* narrower selection (--columns).
|
|
51
|
+
*/
|
|
52
|
+
function tsvLimitError(what, limit, state) {
|
|
53
|
+
const exceeded = `Dataset TSV ${what} exceeded ${formatLimitBytes(limit)}`;
|
|
54
|
+
const cellLimit = formatLimitBytes(state.limits.maxFieldBytes);
|
|
55
|
+
let message;
|
|
56
|
+
if (what === "field") {
|
|
57
|
+
message = `${exceeded} in ${describeTsvColumn(state)} (a single cell is limited to ${cellLimit}); exclude it with --columns.`;
|
|
58
|
+
}
|
|
59
|
+
else if (what === "row") {
|
|
60
|
+
message = `${exceeded} (a single cell is limited to ${cellLimit}); reached in ${describeTsvColumn(state)}. Exclude large columns with --columns.`;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
message = `${exceeded}; streaming was aborted to guard against unbounded memory use.`;
|
|
64
|
+
}
|
|
65
|
+
return new ClientValidationError(message, "validation_failed", what === "response"
|
|
66
|
+
? "Narrow the read with --columns, --partitions, or a lower row limit."
|
|
67
|
+
: "Select fewer columns with --columns so the oversized cell is not read.");
|
|
33
68
|
}
|
|
34
69
|
/** Characters that evaluate a spreadsheet cell as a formula. */
|
|
35
70
|
const FORMULA_SIGILS = "=+-@";
|
|
@@ -75,6 +110,7 @@ function createTsvStreamState(limits = {
|
|
|
75
110
|
return {
|
|
76
111
|
currentField: "",
|
|
77
112
|
currentRow: [],
|
|
113
|
+
header: undefined,
|
|
78
114
|
inQuotes: false,
|
|
79
115
|
pendingQuoteInQuotes: false,
|
|
80
116
|
fieldBytes: 0,
|
|
@@ -91,10 +127,10 @@ function appendFieldText(state, text) {
|
|
|
91
127
|
state.fieldBytes += bytes;
|
|
92
128
|
state.rowBytes += bytes;
|
|
93
129
|
if (state.fieldBytes > state.limits.maxFieldBytes) {
|
|
94
|
-
throw tsvLimitError("field", state.limits.maxFieldBytes);
|
|
130
|
+
throw tsvLimitError("field", state.limits.maxFieldBytes, state);
|
|
95
131
|
}
|
|
96
132
|
if (state.rowBytes > state.limits.maxRowBytes) {
|
|
97
|
-
throw tsvLimitError("row", state.limits.maxRowBytes);
|
|
133
|
+
throw tsvLimitError("row", state.limits.maxRowBytes, state);
|
|
98
134
|
}
|
|
99
135
|
state.currentField += text;
|
|
100
136
|
}
|
|
@@ -105,11 +141,20 @@ function endField(state) {
|
|
|
105
141
|
state.fieldBytes = 0;
|
|
106
142
|
state.rowBytes += 1; // count the delimiter byte
|
|
107
143
|
}
|
|
144
|
+
/** Hand a completed row to `onRow`, remembering the first non-blank row as the header. */
|
|
145
|
+
function emitRow(state, onRow) {
|
|
146
|
+
const row = state.currentRow;
|
|
147
|
+
state.currentRow = [];
|
|
148
|
+
state.rowBytes = 0;
|
|
149
|
+
if (state.header === undefined && !isBlankRow(row))
|
|
150
|
+
state.header = row;
|
|
151
|
+
onRow(row);
|
|
152
|
+
}
|
|
108
153
|
function consumeTsvChunk(text, state, onRow) {
|
|
109
154
|
// Response budget: bound total decoded bytes regardless of row structure.
|
|
110
155
|
state.responseBytes += Buffer.byteLength(text, "utf-8");
|
|
111
156
|
if (state.responseBytes > state.limits.maxResponseBytes) {
|
|
112
|
-
throw tsvLimitError("response", state.limits.maxResponseBytes);
|
|
157
|
+
throw tsvLimitError("response", state.limits.maxResponseBytes, state);
|
|
113
158
|
}
|
|
114
159
|
let i = 0;
|
|
115
160
|
if (state.pendingQuoteInQuotes) {
|
|
@@ -193,10 +238,7 @@ function consumeTsvChunk(text, state, onRow) {
|
|
|
193
238
|
if (ch === "\n") {
|
|
194
239
|
endField(state);
|
|
195
240
|
i = special + 1;
|
|
196
|
-
|
|
197
|
-
state.currentRow = [];
|
|
198
|
-
state.rowBytes = 0;
|
|
199
|
-
onRow(row);
|
|
241
|
+
emitRow(state, onRow);
|
|
200
242
|
continue;
|
|
201
243
|
}
|
|
202
244
|
// "\r" — carriage return: skipped, as in the original parser.
|
|
@@ -213,10 +255,7 @@ function flushTsvStream(state, onRow) {
|
|
|
213
255
|
state.currentRow.push(state.currentField);
|
|
214
256
|
state.currentField = "";
|
|
215
257
|
state.fieldBytes = 0;
|
|
216
|
-
|
|
217
|
-
state.currentRow = [];
|
|
218
|
-
state.rowBytes = 0;
|
|
219
|
-
onRow(row);
|
|
258
|
+
emitRow(state, onRow);
|
|
220
259
|
}
|
|
221
260
|
function rowToCsv(row, spreadsheetSafe) {
|
|
222
261
|
return row.map((field) => csvEscape(field, spreadsheetSafe)).join(",");
|
|
@@ -264,7 +303,7 @@ async function readChunkWithTimeout(reader, remainingMs, timeoutMs, action) {
|
|
|
264
303
|
}
|
|
265
304
|
async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
266
305
|
const state = createTsvStreamState();
|
|
267
|
-
const decoder = new
|
|
306
|
+
const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
|
|
268
307
|
let columns;
|
|
269
308
|
const rows = [];
|
|
270
309
|
let done = false;
|
|
@@ -304,10 +343,10 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
|
304
343
|
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "preview");
|
|
305
344
|
if (result.done)
|
|
306
345
|
break;
|
|
307
|
-
consumeTsvChunk(decoder.
|
|
346
|
+
consumeTsvChunk(decoder.decode(result.value, { stream: true, }), state, handleRow);
|
|
308
347
|
}
|
|
309
348
|
if (!done) {
|
|
310
|
-
const tail = decoder.
|
|
349
|
+
const tail = decoder.decode();
|
|
311
350
|
if (tail !== "")
|
|
312
351
|
consumeTsvChunk(tail, state, handleRow);
|
|
313
352
|
flushTsvStream(state, handleRow);
|
|
@@ -332,7 +371,7 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
|
332
371
|
*/
|
|
333
372
|
async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
334
373
|
const state = createTsvStreamState();
|
|
335
|
-
const decoder = new
|
|
374
|
+
const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
|
|
336
375
|
let headerSeen = false;
|
|
337
376
|
let count = 0;
|
|
338
377
|
let bounded = false;
|
|
@@ -373,10 +412,10 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
|
373
412
|
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "row count assertion");
|
|
374
413
|
if (result.done)
|
|
375
414
|
break;
|
|
376
|
-
consumeTsvChunk(decoder.
|
|
415
|
+
consumeTsvChunk(decoder.decode(result.value, { stream: true, }), state, handleRow);
|
|
377
416
|
}
|
|
378
417
|
if (!bounded) {
|
|
379
|
-
const tail = decoder.
|
|
418
|
+
const tail = decoder.decode();
|
|
380
419
|
if (tail !== "")
|
|
381
420
|
consumeTsvChunk(tail, state, handleRow);
|
|
382
421
|
flushTsvStream(state, handleRow);
|
|
@@ -402,7 +441,7 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
|
402
441
|
const CSV_BATCH_BYTES = 64 * 1024;
|
|
403
442
|
function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
|
|
404
443
|
const state = createTsvStreamState();
|
|
405
|
-
const decoder = new
|
|
444
|
+
const decoder = new TextDecoder("utf-8", { ignoreBOM: true, });
|
|
406
445
|
const encoder = new TextEncoder();
|
|
407
446
|
const maxRows = Math.max(1, maxDataRows);
|
|
408
447
|
let headerSeen = false;
|
|
@@ -454,7 +493,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
454
493
|
// which cancels the response and makes download() remove the partial file.
|
|
455
494
|
return new TransformStream({
|
|
456
495
|
transform(chunk, controller) {
|
|
457
|
-
consumeTsvChunk(decoder.
|
|
496
|
+
consumeTsvChunk(decoder.decode(chunk, { stream: true, }), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
|
|
458
497
|
if (done) {
|
|
459
498
|
// Truncation reached: emit the batch queued so far, end the output,
|
|
460
499
|
// and cancel the rest of the response.
|
|
@@ -463,7 +502,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
463
502
|
}
|
|
464
503
|
},
|
|
465
504
|
flush(controller) {
|
|
466
|
-
const tail = decoder.
|
|
505
|
+
const tail = decoder.decode();
|
|
467
506
|
if (tail !== "") {
|
|
468
507
|
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(controller, line)));
|
|
469
508
|
}
|
|
@@ -660,19 +699,35 @@ export class DatasetsResource extends BaseResource {
|
|
|
660
699
|
/**
|
|
661
700
|
* TSV data stream of at most `limit` rows. A column or partition selection
|
|
662
701
|
* uses the documented POST variant of the data endpoint (body `columns`,
|
|
663
|
-
* `partitions`, `format`, `sampling`); otherwise the GET endpoint.
|
|
702
|
+
* `partitions`, `format`, `sampling`); otherwise the GET endpoint. Both carry
|
|
703
|
+
* a client-generated `readSessionId` for {@link verifyStreamComplete}.
|
|
664
704
|
*/
|
|
665
705
|
async dataStream(datasetName, limit, projectKey, selection) {
|
|
706
|
+
const readSessionId = crypto.randomUUID();
|
|
666
707
|
const path = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/data/`;
|
|
667
708
|
if (!selection?.columns?.length && !selection?.partitions) {
|
|
668
|
-
|
|
709
|
+
const res = await this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}&readSessionId=${readSessionId}`);
|
|
710
|
+
return { res, readSessionId, };
|
|
669
711
|
}
|
|
670
|
-
|
|
712
|
+
const res = await this.client.postStream(path, {
|
|
671
713
|
format: "tsv-excel-header",
|
|
672
714
|
...(selection.columns?.length ? { columns: selection.columns, } : {}),
|
|
673
715
|
...(selection.partitions ? { partitions: selection.partitions, } : {}),
|
|
674
716
|
sampling: { samplingMethod: "HEAD_SEQUENTIAL", maxRecords: limit, },
|
|
717
|
+
readSessionId,
|
|
675
718
|
});
|
|
719
|
+
return { res, readSessionId, };
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* A data stream that ended normally may still be incomplete: DSS sends 200
|
|
723
|
+
* headers before reading the data, so a server-side read failure only cuts
|
|
724
|
+
* the body short. As dataikuapi does, ask DSS whether the read session
|
|
725
|
+
* finished; it answers non-2xx (DataikuError) when it did not. Only called
|
|
726
|
+
* after the body was read to its end — a stream the client cancelled at
|
|
727
|
+
* its row cap is not complete by design.
|
|
728
|
+
*/
|
|
729
|
+
async verifyStreamComplete(datasetName, projectKey, readSessionId) {
|
|
730
|
+
await this.client.getTextLimited(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/finish-streaming/?readSessionId=${readSessionId}`, 64 * 1024);
|
|
676
731
|
}
|
|
677
732
|
/**
|
|
678
733
|
* Preview dataset rows as structured data: column names plus row arrays,
|
|
@@ -691,7 +746,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
691
746
|
const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
|
|
692
747
|
// Probe one row past the cap so callers learn whether the dataset holds
|
|
693
748
|
// more rows than requested without materializing the full dataset.
|
|
694
|
-
const res = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
|
|
749
|
+
const { res, readSessionId, } = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
|
|
695
750
|
const onHeader = opts?.validateColumns
|
|
696
751
|
? (headerRow) => {
|
|
697
752
|
const warnings = validateStreamColumns(headerRow, opts.validateColumns);
|
|
@@ -704,6 +759,8 @@ export class DatasetsResource extends BaseResource {
|
|
|
704
759
|
return { columns: [], rows: [], rowCount: 0, truncated: false, limit: maxRows, };
|
|
705
760
|
}
|
|
706
761
|
const { columns, rows, truncated, } = await collectPreviewRows(res.body, maxRows, timeoutMs, onHeader);
|
|
762
|
+
if (!truncated)
|
|
763
|
+
await this.verifyStreamComplete(datasetName, opts?.projectKey, readSessionId);
|
|
707
764
|
return {
|
|
708
765
|
columns: columns.map((name) => ({ name, })),
|
|
709
766
|
rows,
|
|
@@ -730,12 +787,13 @@ export class DatasetsResource extends BaseResource {
|
|
|
730
787
|
const target = expected;
|
|
731
788
|
const probe = target + 1;
|
|
732
789
|
const timeoutMs = Math.max(1, this.client.getRequestTimeoutMs());
|
|
733
|
-
const
|
|
734
|
-
const res = await this.client.stream(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${probe}`);
|
|
790
|
+
const { res, readSessionId, } = await this.dataStream(datasetName, probe, projectKey, undefined);
|
|
735
791
|
if (!res.body) {
|
|
736
792
|
return { expected: target, count: 0, exact: true, satisfied: target === 0, };
|
|
737
793
|
}
|
|
738
794
|
const { count, bounded, } = await collectTsvRowCount(res.body, probe, timeoutMs);
|
|
795
|
+
if (!bounded)
|
|
796
|
+
await this.verifyStreamComplete(datasetName, projectKey, readSessionId);
|
|
739
797
|
return { expected: target, count, exact: !bounded, satisfied: !bounded && count === target, };
|
|
740
798
|
}
|
|
741
799
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
@@ -873,7 +931,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
873
931
|
*/
|
|
874
932
|
async download(datasetName, opts) {
|
|
875
933
|
const limit = Math.max(1, opts?.limit ?? 100_000);
|
|
876
|
-
const res = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
|
|
934
|
+
const { res, readSessionId, } = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
|
|
877
935
|
const safeDatasetName = sanitizeFileName(datasetName, "dataset");
|
|
878
936
|
const filePath = opts?.outputPath?.endsWith(".gz") || opts?.outputPath?.endsWith(".csv")
|
|
879
937
|
? nodePath.resolve(opts.outputPath)
|
|
@@ -895,6 +953,9 @@ export class DatasetsResource extends BaseResource {
|
|
|
895
953
|
try {
|
|
896
954
|
// Native streaming write with backpressure; no Node stream layer.
|
|
897
955
|
await Bun.write(filePath, new Response(csv));
|
|
956
|
+
if (!stats.truncated) {
|
|
957
|
+
await this.verifyStreamComplete(datasetName, opts?.projectKey, readSessionId);
|
|
958
|
+
}
|
|
898
959
|
}
|
|
899
960
|
catch (error) {
|
|
900
961
|
// A failed export (byte-limit overflow, network error) must never
|
|
@@ -9,6 +9,20 @@ import { BaseResource } from "./base.js";
|
|
|
9
9
|
*/
|
|
10
10
|
export type JobBuildTargetType = "DATASET" | "MANAGED_FOLDER" | "MODEL_EVALUATION_STORE";
|
|
11
11
|
export type JobLogFilter = "stdout" | "stderr" | "user" | "errors";
|
|
12
|
+
/** A retrieved job log plus what the byte-bounded download dropped or scanned. */
|
|
13
|
+
export interface JobLogDetails {
|
|
14
|
+
/** Redacted, filtered and line-limited log text (what {@link JobsResource.log} returns). */
|
|
15
|
+
text: string;
|
|
16
|
+
/**
|
|
17
|
+
* True when the log exceeded the byte cap and only its most recent bytes
|
|
18
|
+
* were retained: earlier lines were never scanned or filtered.
|
|
19
|
+
*/
|
|
20
|
+
truncated: boolean;
|
|
21
|
+
/** Lines in the retained log before filtering and the line limit. */
|
|
22
|
+
scannedLines: number;
|
|
23
|
+
/** Byte cap applied to the download (the most recent this-many bytes are kept). */
|
|
24
|
+
maxLogBytes: number;
|
|
25
|
+
}
|
|
12
26
|
export interface JobLogProgress {
|
|
13
27
|
lastProgressLine?: string;
|
|
14
28
|
doneLine?: string;
|
|
@@ -83,11 +97,14 @@ export declare class JobsResource extends BaseResource {
|
|
|
83
97
|
* Use `0` or `-1` to return the log without line truncation. Credential-like
|
|
84
98
|
* JSON pairs (`"jobTicketSecret":"..."`) are redacted.
|
|
85
99
|
*
|
|
86
|
-
* The download is byte-bounded
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
100
|
+
* The download is byte-bounded: at most `maxLogBytes` (default 10 MiB) of
|
|
101
|
+
* the *most recent* log output is retained, so a collaborator-influenced
|
|
102
|
+
* oversized log cannot exhaust client memory while the line limit
|
|
103
|
+
* previously only applied after a full unbounded download. The request
|
|
104
|
+
* timeout is a stall deadline (per received chunk), so a large log that
|
|
105
|
+
* keeps arriving is not cut off by a total-transfer budget. Logs up to the
|
|
106
|
+
* cap are returned in full. Use {@link logDetails} to learn whether bytes
|
|
107
|
+
* were dropped.
|
|
91
108
|
*/
|
|
92
109
|
log(jobId: string, opts?: {
|
|
93
110
|
activity?: string;
|
|
@@ -97,6 +114,19 @@ export declare class JobsResource extends BaseResource {
|
|
|
97
114
|
maxLogBytes?: number;
|
|
98
115
|
projectKey?: string;
|
|
99
116
|
}): Promise<string>;
|
|
117
|
+
/**
|
|
118
|
+
* Same retrieval as {@link log}, additionally reporting whether the
|
|
119
|
+
* byte-bounded download dropped early log bytes (`truncated`) and how many
|
|
120
|
+
* lines were scanned before filtering (`scannedLines`).
|
|
121
|
+
*/
|
|
122
|
+
logDetails(jobId: string, opts?: {
|
|
123
|
+
activity?: string;
|
|
124
|
+
logId?: string;
|
|
125
|
+
logFilter?: JobLogFilter;
|
|
126
|
+
maxLogLines?: number;
|
|
127
|
+
maxLogBytes?: number;
|
|
128
|
+
projectKey?: string;
|
|
129
|
+
}): Promise<JobLogDetails>;
|
|
100
130
|
logFromUrl(logUrl: string, opts?: {
|
|
101
131
|
maxLogLines?: number;
|
|
102
132
|
maxLogBytes?: number;
|