dataiku-sdk 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -19
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +39 -4
- package/dist/packages/types/src/index.js +31 -2
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +66 -18
- package/dist/src/cli/commands/app.js +4 -3
- package/dist/src/cli/commands/bundle.js +51 -16
- package/dist/src/cli/commands/code-env.js +219 -57
- package/dist/src/cli/commands/code.d.ts +19 -0
- package/dist/src/cli/commands/code.js +87 -15
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +17 -11
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +42 -13
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +2 -1
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +3 -2
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +82 -50
- package/dist/src/cli/commands/project-git.js +8 -5
- package/dist/src/cli/commands/project-library.d.ts +21 -0
- package/dist/src/cli/commands/project-library.js +128 -56
- package/dist/src/cli/commands/project.js +19 -4
- package/dist/src/cli/commands/recipe.js +51 -25
- package/dist/src/cli/commands/saved-model.js +4 -3
- package/dist/src/cli/commands/scenario.js +9 -8
- package/dist/src/cli/commands/sql.d.ts +10 -0
- package/dist/src/cli/commands/sql.js +42 -19
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +68 -13
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +16 -1
- package/dist/src/cli/contract.js +967 -186
- package/dist/src/cli/env.d.ts +9 -0
- package/dist/src/cli/env.js +20 -2
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +29 -1
- package/dist/src/cli/helpers/cleanup.js +104 -3
- package/dist/src/cli/helpers/flow-zone.d.ts +3 -0
- package/dist/src/cli/helpers/flow-zone.js +33 -3
- package/dist/src/cli/helpers/recipe.d.ts +20 -0
- package/dist/src/cli/helpers/recipe.js +158 -3
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +33 -6
- package/dist/src/cli/runtime.d.ts +35 -1
- package/dist/src/cli/runtime.js +125 -18
- package/dist/src/cli/types.d.ts +7 -0
- package/dist/src/cli.js +226 -95
- package/dist/src/client.d.ts +63 -1
- package/dist/src/client.js +305 -53
- package/dist/src/errors.d.ts +8 -2
- package/dist/src/errors.js +42 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +2 -1
- package/dist/src/resources/api-services.d.ts +48 -6
- package/dist/src/resources/api-services.js +54 -8
- package/dist/src/resources/bundles.d.ts +51 -6
- package/dist/src/resources/bundles.js +39 -9
- package/dist/src/resources/code-envs.d.ts +25 -3
- package/dist/src/resources/code-envs.js +104 -2
- package/dist/src/resources/datasets.d.ts +20 -0
- package/dist/src/resources/datasets.js +232 -52
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/jobs.d.ts +35 -10
- package/dist/src/resources/jobs.js +92 -17
- package/dist/src/resources/notebooks.d.ts +77 -7
- package/dist/src/resources/notebooks.js +143 -19
- package/dist/src/resources/project-git.d.ts +12 -0
- package/dist/src/resources/project-git.js +39 -8
- package/dist/src/resources/project-library.d.ts +122 -7
- package/dist/src/resources/project-library.js +494 -25
- package/dist/src/resources/projects.d.ts +5 -4
- package/dist/src/resources/projects.js +21 -19
- package/dist/src/resources/scenarios.d.ts +30 -0
- package/dist/src/resources/scenarios.js +73 -27
- package/dist/src/resources/sql.d.ts +6 -0
- package/dist/src/resources/sql.js +23 -3
- package/dist/src/resources/webapps.d.ts +35 -4
- package/dist/src/resources/webapps.js +54 -5
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +18 -0
- package/dist/src/skill.js +93 -9
- package/dist/src/utils/flow-analysis.d.ts +53 -0
- package/dist/src/utils/flow-analysis.js +382 -0
- package/dist/src/utils/project-archive.d.ts +41 -2
- package/dist/src/utils/project-archive.js +159 -16
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/package.json +9 -7
- package/packages/types/dist/index.d.ts +39 -4
- package/packages/types/dist/index.js +31 -2
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +44 -197
- package/skills/dataiku-dss/references/app-releases.md +27 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
package/dist/src/client.d.ts
CHANGED
|
@@ -67,10 +67,16 @@ export interface DataikuClientConfig {
|
|
|
67
67
|
apiKey: string;
|
|
68
68
|
/** Default project key — used when a resource method omits projectKey */
|
|
69
69
|
projectKey?: string;
|
|
70
|
-
/**
|
|
70
|
+
/** Request/header timeout and buffered-body budget; idle timeout per raw stream read (default 30 000 ms). */
|
|
71
71
|
requestTimeoutMs?: number;
|
|
72
72
|
/** Max retry attempts for idempotent requests (default 4, capped at 10) */
|
|
73
73
|
retryMaxAttempts?: number;
|
|
74
|
+
/**
|
|
75
|
+
* Maximum bytes buffered from a single response body by the text/JSON
|
|
76
|
+
* consumer methods (default 50 MiB). Responses larger than this are
|
|
77
|
+
* rejected with a DataikuError after the stream is cancelled.
|
|
78
|
+
*/
|
|
79
|
+
maxResponseBodyBytes?: number;
|
|
74
80
|
/** Emit HTTP request/response trace events. Defaults to JSONL on stderr when verbose is true. */
|
|
75
81
|
verbose?: boolean;
|
|
76
82
|
onTrace?: (event: DataikuClientTraceEvent) => void;
|
|
@@ -91,6 +97,7 @@ export declare class DataikuClient {
|
|
|
91
97
|
private readonly apiKey;
|
|
92
98
|
private readonly defaultProjectKey;
|
|
93
99
|
private readonly requestTimeoutMs;
|
|
100
|
+
private readonly maxResponseBodyBytes;
|
|
94
101
|
private readonly retryMaxAttempts;
|
|
95
102
|
private readonly verbose;
|
|
96
103
|
private readonly tlsOptions;
|
|
@@ -176,6 +183,11 @@ export declare class DataikuClient {
|
|
|
176
183
|
*/
|
|
177
184
|
getBaseUrl(): string;
|
|
178
185
|
getRequestTimeoutMs(): number;
|
|
186
|
+
/**
|
|
187
|
+
* The configured maximum number of bytes buffered from a single response
|
|
188
|
+
* body by the text/JSON consumer methods (default 50 MiB).
|
|
189
|
+
*/
|
|
190
|
+
getMaxResponseBodyBytes(): number;
|
|
179
191
|
resolveProjectKey(paramValue?: string): string;
|
|
180
192
|
get<T = unknown>(path: string): Promise<T>;
|
|
181
193
|
/**
|
|
@@ -187,11 +199,34 @@ export declare class DataikuClient {
|
|
|
187
199
|
data: T;
|
|
188
200
|
meta: DataikuClientResponseMeta;
|
|
189
201
|
}>;
|
|
202
|
+
/**
|
|
203
|
+
* GET returning the entire text body. The body is buffered with a hard
|
|
204
|
+
* byte cap (`maxResponseBodyBytes`, default 50 MiB) and read under the
|
|
205
|
+
* request deadline: a response exceeding the cap throws a DataikuError
|
|
206
|
+
* instead of being silently truncated, and a stalled body times out.
|
|
207
|
+
*/
|
|
190
208
|
getText(path: string): Promise<string>;
|
|
209
|
+
/**
|
|
210
|
+
* GET returning at most the first `maxBytes` bytes of a text body.
|
|
211
|
+
* When the response exceeds the cap the stream is cancelled, only the
|
|
212
|
+
* collected prefix is returned and `truncated` is true. The read runs
|
|
213
|
+
* under the request deadline: a stalled body throws a DataikuError.
|
|
214
|
+
*/
|
|
191
215
|
getTextLimited(path: string, maxBytes: number): Promise<{
|
|
192
216
|
text: string;
|
|
193
217
|
truncated: boolean;
|
|
194
218
|
}>;
|
|
219
|
+
/**
|
|
220
|
+
* GET returning at most the last `maxBytes` bytes of a text body.
|
|
221
|
+
* When the response exceeds the cap, earlier bytes are discarded, only
|
|
222
|
+
* the tail is kept and `truncated` is true — so callers that need the end
|
|
223
|
+
* of a large document (e.g. job logs) stay bounded without losing the most
|
|
224
|
+
* recent content. The read runs under the request deadline.
|
|
225
|
+
*/
|
|
226
|
+
getTextTailLimited(path: string, maxBytes: number): Promise<{
|
|
227
|
+
text: string;
|
|
228
|
+
truncated: boolean;
|
|
229
|
+
}>;
|
|
195
230
|
/**
|
|
196
231
|
* POST with optional transient retry. Only opt in when repeating the request
|
|
197
232
|
* is safe: the server may have accepted an attempt whose response was lost.
|
|
@@ -199,6 +234,10 @@ export declare class DataikuClient {
|
|
|
199
234
|
post<T = unknown>(path: string, body?: unknown, options?: {
|
|
200
235
|
retryMaxAttempts?: number;
|
|
201
236
|
}): Promise<T>;
|
|
237
|
+
/**
|
|
238
|
+
* POST returning the text body, under the same bounded buffering rules as
|
|
239
|
+
* {@link getText} (byte cap `maxResponseBodyBytes`, request deadline).
|
|
240
|
+
*/
|
|
202
241
|
postText(path: string, body?: unknown): Promise<string>;
|
|
203
242
|
postStream(path: string, body?: unknown): Promise<Response>;
|
|
204
243
|
put<T = unknown>(path: string, body: unknown): Promise<T>;
|
|
@@ -208,6 +247,8 @@ export declare class DataikuClient {
|
|
|
208
247
|
upload(path: string, filePath: string, fileName?: string): Promise<void>;
|
|
209
248
|
uploadJson<T>(path: string, filePath: string, fileName?: string): Promise<T>;
|
|
210
249
|
stream(path: string): Promise<Response>;
|
|
250
|
+
/** Bound each pending read, not total transfer time; preserve large downloads and backpressure. */
|
|
251
|
+
private withBodyDeadline;
|
|
211
252
|
private getHeaders;
|
|
212
253
|
private getAnyHeaders;
|
|
213
254
|
private responseMeta;
|
|
@@ -229,5 +270,26 @@ export declare class DataikuClient {
|
|
|
229
270
|
warn(method: string, errors: string[]): void;
|
|
230
271
|
private requestIdFromHeaders;
|
|
231
272
|
private parseJsonResponse;
|
|
273
|
+
/**
|
|
274
|
+
* Read at most `maxBytes` of a 2xx response body as UTF-8 text.
|
|
275
|
+
*
|
|
276
|
+
* The body is consumed with the request deadline still active (measured
|
|
277
|
+
* from the first read): a stalled body throws a DataikuError instead of
|
|
278
|
+
* hanging. When the stream yields more than the cap, the reader cancels the
|
|
279
|
+
* body and returns the collected prefix with `truncated: true`; truncated
|
|
280
|
+
* text is trimmed to a UTF-8 character boundary so it never decodes beyond
|
|
281
|
+
* the byte cap.
|
|
282
|
+
*/
|
|
283
|
+
private readBoundedBodyText;
|
|
284
|
+
/**
|
|
285
|
+
* Read a 2xx response body keeping only the last `maxBytes` bytes.
|
|
286
|
+
* Earlier bytes are discarded as the stream advances, so memory stays
|
|
287
|
+
* bounded by the cap plus one chunk even for arbitrarily large bodies.
|
|
288
|
+
* Returns `truncated: true` when bytes had to be dropped. Bytes that fall
|
|
289
|
+
* in the middle of a UTF-8 character are dropped from both ends so the
|
|
290
|
+
* decoded text never contains a replacement character.
|
|
291
|
+
*/
|
|
292
|
+
private readBodyTextTail;
|
|
293
|
+
private buildResponseTooLargeError;
|
|
232
294
|
private fetchWithRetry;
|
|
233
295
|
}
|
package/dist/src/client.js
CHANGED
|
@@ -3,7 +3,7 @@ import { basename, } from "node:path";
|
|
|
3
3
|
import { getCACertificates, } from "node:tls";
|
|
4
4
|
import { Value, } from "@sinclair/typebox/value";
|
|
5
5
|
import { safeParseSchema, } from "./schemas.js";
|
|
6
|
-
import { classifyDataikuError, ClientValidationError, DataikuError, } from "./errors.js";
|
|
6
|
+
import { classifyDataikuError, ClientValidationError, DataikuError, NON_JSON_RESPONSE_MARKER, } from "./errors.js";
|
|
7
7
|
import { AnalysesResource, } from "./resources/analyses.js";
|
|
8
8
|
import { ApiDeployerResource, } from "./resources/api-deployer.js";
|
|
9
9
|
import { ApiServicesResource, } from "./resources/api-services.js";
|
|
@@ -47,6 +47,14 @@ const MAX_RETRY_ATTEMPTS_CAP = 10;
|
|
|
47
47
|
const BASE_DELAY_MS = 2_000;
|
|
48
48
|
const MAX_BACKOFF_DELAY_MS = 30_000;
|
|
49
49
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
50
|
+
/**
|
|
51
|
+
* Default cap on bytes buffered from a single response body by the text/JSON
|
|
52
|
+
* consumers (getText, postText, parseJsonResponse). Bodies larger than this
|
|
53
|
+
* are rejected with a DataikuError after the stream is cancelled; raw
|
|
54
|
+
* streaming methods (stream, postStream) are unaffected.
|
|
55
|
+
*/
|
|
56
|
+
const DEFAULT_MAX_BUFFERED_BODY_BYTES = 50 * 1024 * 1024;
|
|
57
|
+
const RESPONSE_TOO_LARGE_STATUS_TEXT = "Response Too Large";
|
|
50
58
|
/* ------------------------------------------------------------------ */
|
|
51
59
|
/* Helpers */
|
|
52
60
|
/* ------------------------------------------------------------------ */
|
|
@@ -68,15 +76,6 @@ function isTransientError(status, body) {
|
|
|
68
76
|
function shouldRetryMethod(method) {
|
|
69
77
|
return method.toUpperCase() === "GET";
|
|
70
78
|
}
|
|
71
|
-
function concatBytes(chunks, total) {
|
|
72
|
-
const out = new Uint8Array(total);
|
|
73
|
-
let offset = 0;
|
|
74
|
-
for (const chunk of chunks) {
|
|
75
|
-
out.set(chunk, offset);
|
|
76
|
-
offset += chunk.byteLength;
|
|
77
|
-
}
|
|
78
|
-
return out;
|
|
79
|
-
}
|
|
80
79
|
/**
|
|
81
80
|
* Drop a trailing incomplete UTF-8 multibyte sequence so the bytes end on a
|
|
82
81
|
* character boundary. Guarantees the decoded string re-encodes to no more bytes
|
|
@@ -105,6 +104,43 @@ function trimToUtf8Boundary(bytes) {
|
|
|
105
104
|
return bytes;
|
|
106
105
|
return continuation + 1 < expected ? bytes.subarray(0, i) : bytes;
|
|
107
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Drop leading UTF-8 continuation bytes. A UTF-8 character never *starts*
|
|
109
|
+
* with a continuation byte, so any leading continuation bytes are the remains
|
|
110
|
+
* of a character split by a byte cut at the start of the retained range.
|
|
111
|
+
* Dropping them keeps the decoded text free of replacement characters.
|
|
112
|
+
*/
|
|
113
|
+
function trimFromUtf8BoundaryStart(bytes) {
|
|
114
|
+
let i = 0;
|
|
115
|
+
while (i < bytes.length && (bytes[i] & 0xC0) === 0x80) {
|
|
116
|
+
i++;
|
|
117
|
+
}
|
|
118
|
+
return i > 0 ? bytes.subarray(i) : bytes;
|
|
119
|
+
}
|
|
120
|
+
function buildBodyReadTimeoutError(timeoutMs) {
|
|
121
|
+
return new DataikuError(0, "Request Timeout", `Request timed out after ${timeoutMs}ms while reading the response body.`);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Read the next chunk of a response body under a hard deadline.
|
|
125
|
+
* When `remainingMs` elapses, the stream is cancelled and the caller's
|
|
126
|
+
* promise rejects with a DataikuError, so a stalled body cannot hang the
|
|
127
|
+
* consumers after headers already arrived.
|
|
128
|
+
*/
|
|
129
|
+
async function readChunkWithDeadline(reader, remainingMs, timeoutMs) {
|
|
130
|
+
const { promise, resolve, reject, } = Promise.withResolvers();
|
|
131
|
+
const timer = setTimeout(() => {
|
|
132
|
+
void reader.cancel(buildBodyReadTimeoutError(timeoutMs)).catch(() => { });
|
|
133
|
+
reject(buildBodyReadTimeoutError(timeoutMs));
|
|
134
|
+
}, remainingMs);
|
|
135
|
+
reader.read().then((result) => {
|
|
136
|
+
clearTimeout(timer);
|
|
137
|
+
resolve(result);
|
|
138
|
+
}, (error) => {
|
|
139
|
+
clearTimeout(timer);
|
|
140
|
+
reject(error);
|
|
141
|
+
});
|
|
142
|
+
return promise;
|
|
143
|
+
}
|
|
108
144
|
function buildRetryMetadata(method, enabled, maxAttempts, attempts, delaysMs, timedOut) {
|
|
109
145
|
return {
|
|
110
146
|
method,
|
|
@@ -161,6 +197,7 @@ export class DataikuClient {
|
|
|
161
197
|
apiKey;
|
|
162
198
|
defaultProjectKey;
|
|
163
199
|
requestTimeoutMs;
|
|
200
|
+
maxResponseBodyBytes;
|
|
164
201
|
retryMaxAttempts;
|
|
165
202
|
verbose;
|
|
166
203
|
tlsOptions;
|
|
@@ -326,6 +363,7 @@ export class DataikuClient {
|
|
|
326
363
|
this.apiKey = apiKey;
|
|
327
364
|
this.defaultProjectKey = config?.projectKey?.trim() || undefined;
|
|
328
365
|
this.requestTimeoutMs = config?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
366
|
+
this.maxResponseBodyBytes = config?.maxResponseBodyBytes ?? DEFAULT_MAX_BUFFERED_BODY_BYTES;
|
|
329
367
|
const rawMax = config?.retryMaxAttempts ?? DEFAULT_RETRY_MAX_ATTEMPTS;
|
|
330
368
|
this.retryMaxAttempts = Math.min(Math.max(1, rawMax), MAX_RETRY_ATTEMPTS_CAP);
|
|
331
369
|
this.verbose = config?.verbose === true;
|
|
@@ -344,6 +382,13 @@ export class DataikuClient {
|
|
|
344
382
|
getRequestTimeoutMs() {
|
|
345
383
|
return this.requestTimeoutMs;
|
|
346
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* The configured maximum number of bytes buffered from a single response
|
|
387
|
+
* body by the text/JSON consumer methods (default 50 MiB).
|
|
388
|
+
*/
|
|
389
|
+
getMaxResponseBodyBytes() {
|
|
390
|
+
return this.maxResponseBodyBytes;
|
|
391
|
+
}
|
|
347
392
|
/* ---- public: project key resolution ---- */
|
|
348
393
|
resolveProjectKey(paramValue) {
|
|
349
394
|
const pk = paramValue?.trim();
|
|
@@ -374,56 +419,50 @@ export class DataikuClient {
|
|
|
374
419
|
const data = await this.parseJsonResponse(res);
|
|
375
420
|
return { data, meta: this.responseMeta(res.headers), };
|
|
376
421
|
}
|
|
422
|
+
/**
|
|
423
|
+
* GET returning the entire text body. The body is buffered with a hard
|
|
424
|
+
* byte cap (`maxResponseBodyBytes`, default 50 MiB) and read under the
|
|
425
|
+
* request deadline: a response exceeding the cap throws a DataikuError
|
|
426
|
+
* instead of being silently truncated, and a stalled body times out.
|
|
427
|
+
*/
|
|
377
428
|
async getText(path) {
|
|
378
429
|
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
379
430
|
method: "GET",
|
|
380
431
|
headers: this.getAnyHeaders(),
|
|
381
432
|
});
|
|
382
|
-
|
|
433
|
+
const { text, truncated, } = await this.readBoundedBodyText(res, this.maxResponseBodyBytes);
|
|
434
|
+
if (truncated)
|
|
435
|
+
throw this.buildResponseTooLargeError(res, this.maxResponseBodyBytes);
|
|
436
|
+
return text;
|
|
383
437
|
}
|
|
438
|
+
/**
|
|
439
|
+
* GET returning at most the first `maxBytes` bytes of a text body.
|
|
440
|
+
* When the response exceeds the cap the stream is cancelled, only the
|
|
441
|
+
* collected prefix is returned and `truncated` is true. The read runs
|
|
442
|
+
* under the request deadline: a stalled body throws a DataikuError.
|
|
443
|
+
*/
|
|
384
444
|
async getTextLimited(path, maxBytes) {
|
|
385
445
|
const limit = Math.max(0, Math.floor(maxBytes));
|
|
386
446
|
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
387
447
|
method: "GET",
|
|
388
448
|
headers: this.getAnyHeaders(),
|
|
389
449
|
});
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
break;
|
|
407
|
-
}
|
|
408
|
-
chunks.push(value);
|
|
409
|
-
bytesRead += value.byteLength;
|
|
410
|
-
}
|
|
411
|
-
if (!truncated && bytesRead >= limit) {
|
|
412
|
-
// Buffer filled exactly on a chunk boundary; peek whether more data remains.
|
|
413
|
-
const { done, } = await reader.read();
|
|
414
|
-
if (!done)
|
|
415
|
-
truncated = true;
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
finally {
|
|
419
|
-
await reader.cancel();
|
|
420
|
-
}
|
|
421
|
-
const collected = concatBytes(chunks, bytesRead);
|
|
422
|
-
// On truncation, drop any trailing partial UTF-8 character so the decoded
|
|
423
|
-
// text never re-encodes beyond the byte cap (a stream-flush would otherwise
|
|
424
|
-
// emit a 3-byte replacement char for a split multibyte sequence).
|
|
425
|
-
const usable = truncated ? trimToUtf8Boundary(collected) : collected;
|
|
426
|
-
return { text: new TextDecoder().decode(usable), truncated, };
|
|
450
|
+
return this.readBoundedBodyText(res, limit);
|
|
451
|
+
}
|
|
452
|
+
/**
|
|
453
|
+
* GET returning at most the last `maxBytes` bytes of a text body.
|
|
454
|
+
* When the response exceeds the cap, earlier bytes are discarded, only
|
|
455
|
+
* the tail is kept and `truncated` is true — so callers that need the end
|
|
456
|
+
* of a large document (e.g. job logs) stay bounded without losing the most
|
|
457
|
+
* recent content. The read runs under the request deadline.
|
|
458
|
+
*/
|
|
459
|
+
async getTextTailLimited(path, maxBytes) {
|
|
460
|
+
const limit = Math.max(0, Math.floor(maxBytes));
|
|
461
|
+
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
462
|
+
method: "GET",
|
|
463
|
+
headers: this.getAnyHeaders(),
|
|
464
|
+
});
|
|
465
|
+
return this.readBodyTextTail(res, limit);
|
|
427
466
|
}
|
|
428
467
|
/**
|
|
429
468
|
* POST with optional transient retry. Only opt in when repeating the request
|
|
@@ -442,20 +481,28 @@ export class DataikuClient {
|
|
|
442
481
|
}, retryMaxAttempts);
|
|
443
482
|
return this.parseJsonResponse(res);
|
|
444
483
|
}
|
|
484
|
+
/**
|
|
485
|
+
* POST returning the text body, under the same bounded buffering rules as
|
|
486
|
+
* {@link getText} (byte cap `maxResponseBodyBytes`, request deadline).
|
|
487
|
+
*/
|
|
445
488
|
async postText(path, body) {
|
|
446
489
|
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
447
490
|
method: "POST",
|
|
448
491
|
headers: this.getHeaders(),
|
|
449
492
|
body: body !== undefined ? JSON.stringify(body) : undefined,
|
|
450
493
|
});
|
|
451
|
-
|
|
494
|
+
const { text, truncated, } = await this.readBoundedBodyText(res, this.maxResponseBodyBytes);
|
|
495
|
+
if (truncated)
|
|
496
|
+
throw this.buildResponseTooLargeError(res, this.maxResponseBodyBytes);
|
|
497
|
+
return text;
|
|
452
498
|
}
|
|
453
499
|
async postStream(path, body) {
|
|
454
|
-
|
|
500
|
+
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
455
501
|
method: "POST",
|
|
456
502
|
headers: { ...this.getAnyHeaders(), "Content-Type": "application/json", },
|
|
457
503
|
body: body !== undefined ? JSON.stringify(body) : undefined,
|
|
458
504
|
});
|
|
505
|
+
return this.withBodyDeadline(res);
|
|
459
506
|
}
|
|
460
507
|
async put(path, body) {
|
|
461
508
|
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
@@ -496,10 +543,51 @@ export class DataikuClient {
|
|
|
496
543
|
return this.parseJsonResponse(response);
|
|
497
544
|
}
|
|
498
545
|
async stream(path) {
|
|
499
|
-
|
|
546
|
+
const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
|
|
500
547
|
method: "GET",
|
|
501
548
|
headers: this.getAnyHeaders(),
|
|
502
549
|
});
|
|
550
|
+
return this.withBodyDeadline(res);
|
|
551
|
+
}
|
|
552
|
+
/** Bound each pending read, not total transfer time; preserve large downloads and backpressure. */
|
|
553
|
+
withBodyDeadline(res) {
|
|
554
|
+
if (!res.body)
|
|
555
|
+
return res;
|
|
556
|
+
const reader = res.body.getReader();
|
|
557
|
+
const timeoutMs = this.requestTimeoutMs;
|
|
558
|
+
const body = new ReadableStream({
|
|
559
|
+
async pull(controller) {
|
|
560
|
+
try {
|
|
561
|
+
const { done, value, } = await readChunkWithDeadline(reader, timeoutMs, timeoutMs);
|
|
562
|
+
if (done) {
|
|
563
|
+
controller.close();
|
|
564
|
+
reader.releaseLock();
|
|
565
|
+
}
|
|
566
|
+
else {
|
|
567
|
+
controller.enqueue(value);
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
catch (error) {
|
|
571
|
+
controller.error(error);
|
|
572
|
+
void reader.cancel(error).catch(() => { });
|
|
573
|
+
}
|
|
574
|
+
},
|
|
575
|
+
cancel(reason) {
|
|
576
|
+
void reader.cancel(reason).catch(() => { });
|
|
577
|
+
},
|
|
578
|
+
});
|
|
579
|
+
const response = new Response(body, {
|
|
580
|
+
status: res.status,
|
|
581
|
+
statusText: res.statusText,
|
|
582
|
+
headers: res.headers,
|
|
583
|
+
});
|
|
584
|
+
// ResponseInit has no fields for these fetch response properties.
|
|
585
|
+
Object.defineProperties(response, {
|
|
586
|
+
url: { value: res.url, },
|
|
587
|
+
redirected: { value: res.redirected, },
|
|
588
|
+
type: { value: res.type, },
|
|
589
|
+
});
|
|
590
|
+
return response;
|
|
503
591
|
}
|
|
504
592
|
/* ---- private: headers ---- */
|
|
505
593
|
getHeaders() {
|
|
@@ -571,7 +659,9 @@ export class DataikuClient {
|
|
|
571
659
|
return undefined;
|
|
572
660
|
}
|
|
573
661
|
async parseJsonResponse(res) {
|
|
574
|
-
const text = await res.
|
|
662
|
+
const { text, truncated, } = await this.readBoundedBodyText(res, this.maxResponseBodyBytes);
|
|
663
|
+
if (truncated)
|
|
664
|
+
throw this.buildResponseTooLargeError(res, this.maxResponseBodyBytes);
|
|
575
665
|
// SAFETY: Empty 2xx responses from DSS are surfaced to callers as undefined
|
|
576
666
|
// cast to T. This keeps existing call sites stable, but callers that rely on
|
|
577
667
|
// an object shape must guard explicitly before dereferencing the result.
|
|
@@ -582,8 +672,170 @@ export class DataikuClient {
|
|
|
582
672
|
}
|
|
583
673
|
catch {
|
|
584
674
|
const summary = text.length > 300 ? `${text.slice(0, 300)}…` : text;
|
|
585
|
-
throw new DataikuError(res.status, res.statusText || "Invalid JSON response",
|
|
675
|
+
throw new DataikuError(res.status, res.statusText || "Invalid JSON response", `${NON_JSON_RESPONSE_MARKER}: ${summary}`, undefined, this.requestIdFromHeaders(res.headers));
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
/* ---- private: bounded response body reading ---- */
|
|
679
|
+
/**
|
|
680
|
+
* Read at most `maxBytes` of a 2xx response body as UTF-8 text.
|
|
681
|
+
*
|
|
682
|
+
* The body is consumed with the request deadline still active (measured
|
|
683
|
+
* from the first read): a stalled body throws a DataikuError instead of
|
|
684
|
+
* hanging. When the stream yields more than the cap, the reader cancels the
|
|
685
|
+
* body and returns the collected prefix with `truncated: true`; truncated
|
|
686
|
+
* text is trimmed to a UTF-8 character boundary so it never decodes beyond
|
|
687
|
+
* the byte cap.
|
|
688
|
+
*/
|
|
689
|
+
async readBoundedBodyText(res, maxBytes) {
|
|
690
|
+
const limit = Math.max(0, Math.floor(maxBytes));
|
|
691
|
+
if (!res.body)
|
|
692
|
+
return { text: "", truncated: false, };
|
|
693
|
+
const reader = res.body.getReader();
|
|
694
|
+
const decoder = new TextDecoder();
|
|
695
|
+
const parts = [];
|
|
696
|
+
// Keep the final UTF-8 sequence undecoded until truncation is known.
|
|
697
|
+
// Four bytes also cover malformed sequences handled by trimToUtf8Boundary.
|
|
698
|
+
const tail = new Uint8Array(4);
|
|
699
|
+
let tailLength = 0;
|
|
700
|
+
const append = (chunk) => {
|
|
701
|
+
if (chunk.byteLength >= tail.length) {
|
|
702
|
+
if (tailLength > 0) {
|
|
703
|
+
parts.push(decoder.decode(tail.subarray(0, tailLength), { stream: true, }));
|
|
704
|
+
}
|
|
705
|
+
const end = chunk.byteLength - tail.length;
|
|
706
|
+
if (end > 0)
|
|
707
|
+
parts.push(decoder.decode(chunk.subarray(0, end), { stream: true, }));
|
|
708
|
+
tail.set(chunk.subarray(end));
|
|
709
|
+
tailLength = tail.length;
|
|
710
|
+
}
|
|
711
|
+
else {
|
|
712
|
+
const excess = Math.max(0, tailLength + chunk.byteLength - tail.length);
|
|
713
|
+
if (excess > 0) {
|
|
714
|
+
parts.push(decoder.decode(tail.subarray(0, excess), { stream: true, }));
|
|
715
|
+
tail.copyWithin(0, excess, tailLength);
|
|
716
|
+
tailLength -= excess;
|
|
717
|
+
}
|
|
718
|
+
tail.set(chunk, tailLength);
|
|
719
|
+
tailLength += chunk.byteLength;
|
|
720
|
+
}
|
|
721
|
+
};
|
|
722
|
+
let bytesRead = 0;
|
|
723
|
+
let truncated = false;
|
|
724
|
+
const startedAt = Date.now();
|
|
725
|
+
try {
|
|
726
|
+
while (bytesRead < limit) {
|
|
727
|
+
const remainingMs = this.requestTimeoutMs - (Date.now() - startedAt);
|
|
728
|
+
if (remainingMs <= 0)
|
|
729
|
+
throw buildBodyReadTimeoutError(this.requestTimeoutMs);
|
|
730
|
+
const { done, value, } = await readChunkWithDeadline(reader, remainingMs, this.requestTimeoutMs);
|
|
731
|
+
if (done)
|
|
732
|
+
break;
|
|
733
|
+
const room = limit - bytesRead;
|
|
734
|
+
if (value.byteLength > room) {
|
|
735
|
+
append(value.subarray(0, room));
|
|
736
|
+
bytesRead += room;
|
|
737
|
+
truncated = true;
|
|
738
|
+
break;
|
|
739
|
+
}
|
|
740
|
+
append(value);
|
|
741
|
+
bytesRead += value.byteLength;
|
|
742
|
+
}
|
|
743
|
+
if (!truncated && bytesRead >= limit) {
|
|
744
|
+
// Buffer filled exactly on a chunk boundary; peek whether more data remains.
|
|
745
|
+
const remainingMs = this.requestTimeoutMs - (Date.now() - startedAt);
|
|
746
|
+
if (remainingMs <= 0)
|
|
747
|
+
throw buildBodyReadTimeoutError(this.requestTimeoutMs);
|
|
748
|
+
const { done, } = await readChunkWithDeadline(reader, remainingMs, this.requestTimeoutMs);
|
|
749
|
+
if (!done)
|
|
750
|
+
truncated = true;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
finally {
|
|
754
|
+
void reader.cancel().catch(() => { });
|
|
755
|
+
}
|
|
756
|
+
const remaining = tail.subarray(0, tailLength);
|
|
757
|
+
parts.push(decoder.decode(truncated ? trimToUtf8Boundary(remaining) : remaining));
|
|
758
|
+
return { text: parts.join(""), truncated, };
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* Read a 2xx response body keeping only the last `maxBytes` bytes.
|
|
762
|
+
* Earlier bytes are discarded as the stream advances, so memory stays
|
|
763
|
+
* bounded by the cap plus one chunk even for arbitrarily large bodies.
|
|
764
|
+
* Returns `truncated: true` when bytes had to be dropped. Bytes that fall
|
|
765
|
+
* in the middle of a UTF-8 character are dropped from both ends so the
|
|
766
|
+
* decoded text never contains a replacement character.
|
|
767
|
+
*/
|
|
768
|
+
async readBodyTextTail(res, maxBytes) {
|
|
769
|
+
const limit = Math.max(0, Math.floor(maxBytes));
|
|
770
|
+
if (!res.body)
|
|
771
|
+
return { text: "", truncated: false, };
|
|
772
|
+
const reader = res.body.getReader();
|
|
773
|
+
const ring = new Uint8Array(limit);
|
|
774
|
+
let writeOffset = 0;
|
|
775
|
+
let retainedBytes = 0;
|
|
776
|
+
let truncated = false;
|
|
777
|
+
const startedAt = Date.now();
|
|
778
|
+
try {
|
|
779
|
+
while (true) {
|
|
780
|
+
const remainingMs = this.requestTimeoutMs - (Date.now() - startedAt);
|
|
781
|
+
if (remainingMs <= 0)
|
|
782
|
+
throw buildBodyReadTimeoutError(this.requestTimeoutMs);
|
|
783
|
+
const { done, value, } = await readChunkWithDeadline(reader, remainingMs, this.requestTimeoutMs);
|
|
784
|
+
if (done)
|
|
785
|
+
break;
|
|
786
|
+
if (limit === 0) {
|
|
787
|
+
truncated = value.byteLength > 0;
|
|
788
|
+
if (truncated)
|
|
789
|
+
break;
|
|
790
|
+
continue;
|
|
791
|
+
}
|
|
792
|
+
if (value.byteLength >= limit) {
|
|
793
|
+
ring.set(value.subarray(value.byteLength - limit));
|
|
794
|
+
truncated = truncated || retainedBytes > 0 || value.byteLength > limit;
|
|
795
|
+
retainedBytes = limit;
|
|
796
|
+
writeOffset = 0;
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
const firstLength = Math.min(value.byteLength, limit - writeOffset);
|
|
800
|
+
ring.set(value.subarray(0, firstLength), writeOffset);
|
|
801
|
+
const remainingLength = value.byteLength - firstLength;
|
|
802
|
+
if (remainingLength > 0) {
|
|
803
|
+
ring.set(value.subarray(firstLength));
|
|
804
|
+
}
|
|
805
|
+
writeOffset = (writeOffset + value.byteLength) % limit;
|
|
806
|
+
truncated = truncated || retainedBytes + value.byteLength > limit;
|
|
807
|
+
retainedBytes = Math.min(limit, retainedBytes + value.byteLength);
|
|
808
|
+
}
|
|
586
809
|
}
|
|
810
|
+
finally {
|
|
811
|
+
void reader.cancel().catch(() => { });
|
|
812
|
+
}
|
|
813
|
+
let collected;
|
|
814
|
+
if (retainedBytes === 0) {
|
|
815
|
+
collected = new Uint8Array(0);
|
|
816
|
+
}
|
|
817
|
+
else if (retainedBytes < limit) {
|
|
818
|
+
collected = ring.slice(0, retainedBytes);
|
|
819
|
+
}
|
|
820
|
+
else if (writeOffset === 0) {
|
|
821
|
+
collected = ring;
|
|
822
|
+
}
|
|
823
|
+
else {
|
|
824
|
+
collected = new Uint8Array(retainedBytes);
|
|
825
|
+
const suffixLength = limit - writeOffset;
|
|
826
|
+
collected.set(ring.subarray(writeOffset));
|
|
827
|
+
collected.set(ring.subarray(0, writeOffset), suffixLength);
|
|
828
|
+
}
|
|
829
|
+
// A byte cut inside a multi-byte sequence can leave continuation bytes at
|
|
830
|
+
// the start (and possibly the end) of the tail; drop both so the decoded
|
|
831
|
+
// text never starts or ends with a U+FFFD replacement character.
|
|
832
|
+
const usable = truncated
|
|
833
|
+
? trimToUtf8Boundary(trimFromUtf8BoundaryStart(collected))
|
|
834
|
+
: collected;
|
|
835
|
+
return { text: new TextDecoder().decode(usable), truncated, };
|
|
836
|
+
}
|
|
837
|
+
buildResponseTooLargeError(res, limit) {
|
|
838
|
+
return new DataikuError(res.status, RESPONSE_TOO_LARGE_STATUS_TEXT, `Response body exceeded the ${limit}-byte limit; the body was cancelled after ${limit} bytes.`, undefined, this.requestIdFromHeaders(res.headers));
|
|
587
839
|
}
|
|
588
840
|
/* ---- private: retry loop ---- */
|
|
589
841
|
async fetchWithRetry(url, init, retryMaxAttempts) {
|
package/dist/src/errors.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export type DataikuErrorCategory = "not_found" | "forbidden" | "validation" | "transient" | "unknown";
|
|
2
|
-
export type StableErrorCode = "usage_error" | "unknown_flag" | "missing_required_arg" | "missing_required_flag" | "invalid_flag_value" | "conflicting_input_sources" | "invalid_enum" | "not_found" | "permission_denied" | "target_absence_unverifiable" | "validation_failed" | "transient" | "long_running_failure" | "command_result_failure" | "assertion_failed" | "ambiguous_outcome" | "internal_error";
|
|
1
|
+
export type DataikuErrorCategory = "not_found" | "forbidden" | "validation" | "transient" | "unexpected_response" | "unknown";
|
|
2
|
+
export type StableErrorCode = "usage_error" | "unknown_flag" | "missing_required_arg" | "missing_required_flag" | "invalid_flag_value" | "conflicting_input_sources" | "invalid_enum" | "not_found" | "permission_denied" | "target_absence_unverifiable" | "validation_failed" | "transient" | "long_running_failure" | "command_result_failure" | "assertion_failed" | "ambiguous_outcome" | "unexpected_response" | "internal_error";
|
|
3
3
|
export declare function dataikuErrorCode(category: DataikuErrorCategory): StableErrorCode;
|
|
4
4
|
export interface DataikuErrorTaxonomy {
|
|
5
5
|
category: DataikuErrorCategory;
|
|
@@ -26,6 +26,12 @@ export declare class ClientValidationError extends Error {
|
|
|
26
26
|
readonly details?: Record<string, unknown>;
|
|
27
27
|
constructor(message: string, code?: StableErrorCode, hint?: string, details?: Record<string, unknown>);
|
|
28
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Message prefix the client stamps on a 2xx body that is not JSON. The
|
|
31
|
+
* classifier recognizes exactly this prefix, so the parse-failure path and the
|
|
32
|
+
* taxonomy stay in sync as one contract.
|
|
33
|
+
*/
|
|
34
|
+
export declare const NON_JSON_RESPONSE_MARKER = "Expected JSON response body but got non-JSON content";
|
|
29
35
|
export declare function classifyDataikuError(status: number, body: string): DataikuErrorTaxonomy;
|
|
30
36
|
export declare class DataikuError extends Error {
|
|
31
37
|
status: number;
|