dataiku-sdk 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/build-metadata.json +1 -1
- package/dist/src/cli/commands/code.js +1 -1
- package/dist/src/cli/commands/dataset.js +1 -1
- package/dist/src/cli/commands/job.js +4 -2
- package/dist/src/cli/commands/macro.js +2 -2
- package/dist/src/cli/commands/project-git.js +1 -1
- package/dist/src/cli/commands/scenario.js +1 -1
- package/dist/src/cli/contract.js +23 -208
- package/dist/src/cli/output-schemas.d.ts +24 -0
- package/dist/src/cli/output-schemas.js +233 -0
- package/dist/src/client.d.ts +10 -1
- package/dist/src/client.js +20 -5
- package/dist/src/errors.js +6 -4
- package/dist/src/generated/action-output-schemas.json +1735 -0
- package/dist/src/resources/datasets.js +37 -4
- package/dist/src/resources/flow-zones.d.ts +6 -2
- package/dist/src/resources/flow-zones.js +6 -3
- package/dist/src/resources/futures.d.ts +2 -0
- package/dist/src/resources/futures.js +38 -24
- package/dist/src/resources/jobs.js +37 -25
- package/dist/src/resources/ml-tasks.js +28 -19
- package/dist/src/resources/plugins.d.ts +0 -2
- package/dist/src/resources/plugins.js +0 -3
- package/dist/src/resources/project-git.d.ts +6 -12
- package/dist/src/resources/project-git.js +23 -48
- package/dist/src/resources/projects.d.ts +4 -2
- package/dist/src/resources/projects.js +33 -33
- package/dist/src/utils/git-reference.d.ts +12 -0
- package/dist/src/utils/git-reference.js +38 -0
- package/package.json +5 -5
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +4 -2
- package/skills/dataiku-dss/references/app-releases.md +2 -3
- package/skills/dataiku-dss/references/authentication.md +6 -6
- package/skills/dataiku-dss/references/mutations.md +1 -1
- package/skills/dataiku-dss/references/troubleshooting.md +2 -2
|
@@ -394,12 +394,39 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
|
394
394
|
reader.releaseLock();
|
|
395
395
|
}
|
|
396
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* Encoded CSV bytes accumulated before one stream push. Rows are joined into
|
|
399
|
+
* batch-sized chunks so large downloads do not pay per-row stream overhead; a
|
|
400
|
+
* single escaped row may exceed the budget (rows are never split), and the
|
|
401
|
+
* batch is always flushed at EOF so trailing rows are never dropped.
|
|
402
|
+
*/
|
|
403
|
+
const CSV_BATCH_BYTES = 64 * 1024;
|
|
397
404
|
function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
|
|
398
405
|
const state = createTsvStreamState();
|
|
399
406
|
const decoder = new StringDecoder("utf-8");
|
|
400
407
|
const maxRows = Math.max(1, maxDataRows);
|
|
401
408
|
let headerSeen = false;
|
|
402
409
|
let done = false;
|
|
410
|
+
// Bounded CSV output batching: emitted rows accumulate here and are joined
|
|
411
|
+
// into a single push once the batch reaches CSV_BATCH_BYTES, so memory stays
|
|
412
|
+
// bounded by the batch budget plus one escaped row (the join briefly holds a
|
|
413
|
+
// second copy of the batch) while per-row push/callback overhead disappears.
|
|
414
|
+
let batchLines = [];
|
|
415
|
+
let batchBytes = 0;
|
|
416
|
+
const flushBatch = (target) => {
|
|
417
|
+
if (batchLines.length === 0)
|
|
418
|
+
return;
|
|
419
|
+
const csv = batchLines.join("");
|
|
420
|
+
batchLines = [];
|
|
421
|
+
batchBytes = 0;
|
|
422
|
+
target.push(csv);
|
|
423
|
+
};
|
|
424
|
+
const queueLine = (target, line) => {
|
|
425
|
+
batchLines.push(line);
|
|
426
|
+
batchBytes += Buffer.byteLength(line);
|
|
427
|
+
if (batchBytes >= CSV_BATCH_BYTES)
|
|
428
|
+
flushBatch(target);
|
|
429
|
+
};
|
|
403
430
|
let columnCount = 0;
|
|
404
431
|
const handleRow = (row, push) => {
|
|
405
432
|
if (done)
|
|
@@ -430,7 +457,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
430
457
|
return;
|
|
431
458
|
}
|
|
432
459
|
try {
|
|
433
|
-
consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => this
|
|
460
|
+
consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
434
461
|
}
|
|
435
462
|
catch (error) {
|
|
436
463
|
// Byte-limit overflow: destroy the transform so the pipeline
|
|
@@ -438,8 +465,11 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
438
465
|
callback(error instanceof Error ? error : new Error(String(error)));
|
|
439
466
|
return;
|
|
440
467
|
}
|
|
441
|
-
if (done)
|
|
468
|
+
if (done) {
|
|
469
|
+
// Truncation reached: emit the batch queued so far before EOF.
|
|
470
|
+
flushBatch(this);
|
|
442
471
|
this.push(null);
|
|
472
|
+
}
|
|
443
473
|
callback();
|
|
444
474
|
},
|
|
445
475
|
flush(callback) {
|
|
@@ -450,14 +480,17 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
450
480
|
try {
|
|
451
481
|
const tail = decoder.end();
|
|
452
482
|
if (tail !== "") {
|
|
453
|
-
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => this
|
|
483
|
+
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
454
484
|
}
|
|
455
|
-
flushTsvStream(state, (row) => handleRow(row, (line) => this
|
|
485
|
+
flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
456
486
|
}
|
|
457
487
|
catch (error) {
|
|
458
488
|
callback(error instanceof Error ? error : new Error(String(error)));
|
|
459
489
|
return;
|
|
460
490
|
}
|
|
491
|
+
// Emit the trailing partial batch; without this the last rows of a
|
|
492
|
+
// download (below the batch budget) would be silently dropped.
|
|
493
|
+
flushBatch(this);
|
|
461
494
|
callback();
|
|
462
495
|
},
|
|
463
496
|
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { DataikuGetOptions } from "../client.js";
|
|
1
2
|
import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZoneUpdateOptions } from "../schemas.js";
|
|
2
3
|
import { BaseResource } from "./base.js";
|
|
3
4
|
export type FlowZoneItemInput = {
|
|
@@ -6,8 +7,11 @@ export type FlowZoneItemInput = {
|
|
|
6
7
|
projectKey?: string;
|
|
7
8
|
};
|
|
8
9
|
export declare class FlowZonesResource extends BaseResource {
|
|
9
|
-
/**
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* List all flow zones in a project. Pass `timeoutMs` to bound the whole
|
|
12
|
+
* request (attempts, backoff, and body read) by a total budget.
|
|
13
|
+
*/
|
|
14
|
+
list(projectKey?: string, options?: DataikuGetOptions): Promise<FlowZone[]>;
|
|
11
15
|
/** Get one flow zone by id. */
|
|
12
16
|
get(zoneId: string, projectKey?: string): Promise<FlowZone>;
|
|
13
17
|
/** Create a flow zone. */
|
|
@@ -9,9 +9,12 @@ function normalizeZoneItem(item) {
|
|
|
9
9
|
};
|
|
10
10
|
}
|
|
11
11
|
export class FlowZonesResource extends BaseResource {
|
|
12
|
-
/**
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
/**
|
|
13
|
+
* List all flow zones in a project. Pass `timeoutMs` to bound the whole
|
|
14
|
+
* request (attempts, backoff, and body read) by a total budget.
|
|
15
|
+
*/
|
|
16
|
+
async list(projectKey, options) {
|
|
17
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones`, options);
|
|
15
18
|
return this.client.safeParse(FlowZoneArraySchema, raw, "flowZones.list");
|
|
16
19
|
}
|
|
17
20
|
/** Get one flow zone by id. */
|
|
@@ -7,11 +7,13 @@ export interface FutureWaitOptions {
|
|
|
7
7
|
export declare class FuturesResource extends BaseResource {
|
|
8
8
|
get(futureId: string, opts?: {
|
|
9
9
|
timeoutMs?: number;
|
|
10
|
+
noRetry?: boolean;
|
|
10
11
|
}): Promise<FutureState>;
|
|
11
12
|
peek(futureId: string): Promise<FutureState>;
|
|
12
13
|
state(futureId: string, opts?: {
|
|
13
14
|
peek?: boolean;
|
|
14
15
|
timeoutMs?: number;
|
|
16
|
+
noRetry?: boolean;
|
|
15
17
|
}): Promise<FutureState>;
|
|
16
18
|
abort(futureId: string): Promise<void>;
|
|
17
19
|
wait(futureId: string, opts?: FutureWaitOptions): Promise<FutureWaitResult>;
|
|
@@ -25,7 +25,11 @@ function waitState(state) {
|
|
|
25
25
|
}
|
|
26
26
|
export class FuturesResource extends BaseResource {
|
|
27
27
|
async get(futureId, opts = {}) {
|
|
28
|
-
return this.state(futureId, {
|
|
28
|
+
return this.state(futureId, {
|
|
29
|
+
peek: false,
|
|
30
|
+
timeoutMs: opts.timeoutMs,
|
|
31
|
+
noRetry: opts.noRetry,
|
|
32
|
+
});
|
|
29
33
|
}
|
|
30
34
|
async peek(futureId) {
|
|
31
35
|
return this.state(futureId, { peek: true, });
|
|
@@ -33,7 +37,7 @@ export class FuturesResource extends BaseResource {
|
|
|
33
37
|
async state(futureId, opts = {}) {
|
|
34
38
|
const params = new URLSearchParams();
|
|
35
39
|
params.set("peek", String(opts.peek === true));
|
|
36
|
-
const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}`, { timeoutMs: opts.timeoutMs, });
|
|
40
|
+
const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}`, { timeoutMs: opts.timeoutMs, noRetry: opts.noRetry, });
|
|
37
41
|
return this.client.safeParse(FutureStateSchema, raw, "futures.state");
|
|
38
42
|
}
|
|
39
43
|
async abort(futureId) {
|
|
@@ -51,16 +55,37 @@ export class FuturesResource extends BaseResource {
|
|
|
51
55
|
const timeoutMs = Math.max(0, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
52
56
|
const startedAt = Date.now();
|
|
53
57
|
let pollCount = 0;
|
|
58
|
+
let lastState;
|
|
54
59
|
while (true) {
|
|
60
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
61
|
+
// The first observation always happens, even when the budget is
|
|
62
|
+
// already spent. A later poll is never started after the deadline:
|
|
63
|
+
// the loop reports the structured timeout from the last observed
|
|
64
|
+
// state instead of issuing a request the budget cannot cover.
|
|
65
|
+
if (lastState !== undefined && elapsedBeforeMs >= timeoutMs) {
|
|
66
|
+
return this.client.safeParse(FutureWaitResultSchema, {
|
|
67
|
+
futureId,
|
|
68
|
+
jobId: lastState.jobId,
|
|
69
|
+
state: waitState(lastState),
|
|
70
|
+
elapsedMs: elapsedBeforeMs,
|
|
71
|
+
pollCount,
|
|
72
|
+
success: false,
|
|
73
|
+
timedOut: true,
|
|
74
|
+
hasResult: lastState.hasResult === true,
|
|
75
|
+
alive: lastState.alive,
|
|
76
|
+
aborted: lastState.aborted,
|
|
77
|
+
unknown: lastState.unknown,
|
|
78
|
+
}, "futures.wait");
|
|
79
|
+
}
|
|
55
80
|
pollCount += 1;
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
const remainingMs = timeoutMs -
|
|
81
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
82
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
83
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
84
|
+
// reaches the server instead of failing before any attempt.
|
|
85
|
+
const remainingMs = timeoutMs - elapsedBeforeMs;
|
|
61
86
|
let state;
|
|
62
87
|
try {
|
|
63
|
-
state = await this.get(futureId, remainingMs > 0 ? { timeoutMs: remainingMs, } :
|
|
88
|
+
state = await this.get(futureId, remainingMs > 0 ? { timeoutMs: remainingMs, } : { noRetry: true, });
|
|
64
89
|
}
|
|
65
90
|
catch (error) {
|
|
66
91
|
// The poll budget ran out mid-request: report the structured
|
|
@@ -76,6 +101,7 @@ export class FuturesResource extends BaseResource {
|
|
|
76
101
|
timedOut: true,
|
|
77
102
|
}, "futures.wait");
|
|
78
103
|
}
|
|
104
|
+
lastState = state;
|
|
79
105
|
const elapsedMs = Date.now() - startedAt;
|
|
80
106
|
const status = waitState(state);
|
|
81
107
|
if (isFinished(state)) {
|
|
@@ -94,22 +120,10 @@ export class FuturesResource extends BaseResource {
|
|
|
94
120
|
};
|
|
95
121
|
return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
|
|
96
122
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
state: status,
|
|
102
|
-
elapsedMs,
|
|
103
|
-
pollCount,
|
|
104
|
-
success: false,
|
|
105
|
-
timedOut: true,
|
|
106
|
-
hasResult: state.hasResult === true,
|
|
107
|
-
alive: state.alive,
|
|
108
|
-
aborted: state.aborted,
|
|
109
|
-
unknown: state.unknown,
|
|
110
|
-
};
|
|
111
|
-
return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
|
|
112
|
-
}
|
|
123
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
124
|
+
// timeout from this observation without issuing another request.
|
|
125
|
+
if (elapsedMs >= timeoutMs)
|
|
126
|
+
continue;
|
|
113
127
|
// Only the remaining budget is slept: a longer sleep would report an
|
|
114
128
|
// elapsed time the caller never authorized.
|
|
115
129
|
const nextDelayMs = computeNextPollDelayMs({
|
|
@@ -289,19 +289,44 @@ export class JobsResource extends BaseResource {
|
|
|
289
289
|
const timeout = Math.max(baseIntervalMs, opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
290
290
|
const startedAt = Date.now();
|
|
291
291
|
let pollCount = 0;
|
|
292
|
+
let lastJ;
|
|
292
293
|
while (true) {
|
|
294
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
295
|
+
// The first observation always happens, even when the budget is
|
|
296
|
+
// already spent. A later poll is never started after the deadline:
|
|
297
|
+
// the loop reports the structured timeout from the last observed
|
|
298
|
+
// state instead of issuing a request the budget cannot cover.
|
|
299
|
+
if (lastJ !== undefined && elapsedBeforeMs >= timeout) {
|
|
300
|
+
const bs = lastJ.baseStatus ?? {};
|
|
301
|
+
const def = bs.def ?? {};
|
|
302
|
+
const gs = lastJ.globalState ?? {};
|
|
303
|
+
return {
|
|
304
|
+
success: false,
|
|
305
|
+
jobId,
|
|
306
|
+
state: bs.state ?? "unknown",
|
|
307
|
+
type: def.type ?? "unknown",
|
|
308
|
+
elapsedMs: elapsedBeforeMs,
|
|
309
|
+
pollCount,
|
|
310
|
+
timedOut: true,
|
|
311
|
+
progress: {
|
|
312
|
+
done: gs.done ?? 0,
|
|
313
|
+
failed: gs.failed ?? 0,
|
|
314
|
+
running: gs.running ?? 0,
|
|
315
|
+
total: gs.total ?? null,
|
|
316
|
+
},
|
|
317
|
+
};
|
|
318
|
+
}
|
|
293
319
|
pollCount += 1;
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
const
|
|
299
|
-
const remainingMs = timeout - elapsedBefore;
|
|
320
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
321
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
322
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
323
|
+
// reaches the server instead of failing before any attempt.
|
|
324
|
+
const remainingMs = timeout - elapsedBeforeMs;
|
|
300
325
|
let j;
|
|
301
326
|
try {
|
|
302
327
|
j = await this.client.get(`/public/api/projects/${projectEnc}/jobs/${jobEnc}/`, remainingMs > 0
|
|
303
328
|
? { timeoutMs: remainingMs, }
|
|
304
|
-
:
|
|
329
|
+
: { noRetry: true, });
|
|
305
330
|
}
|
|
306
331
|
catch (error) {
|
|
307
332
|
// The poll budget ran out mid-request: report the documented
|
|
@@ -325,6 +350,7 @@ export class JobsResource extends BaseResource {
|
|
|
325
350
|
},
|
|
326
351
|
};
|
|
327
352
|
}
|
|
353
|
+
lastJ = j;
|
|
328
354
|
const bs = j.baseStatus ?? {};
|
|
329
355
|
const def = bs.def ?? {};
|
|
330
356
|
const gs = j.globalState ?? {};
|
|
@@ -387,24 +413,10 @@ export class JobsResource extends BaseResource {
|
|
|
387
413
|
...(removed !== undefined ? { removed, } : {}),
|
|
388
414
|
};
|
|
389
415
|
}
|
|
390
|
-
// Timeout —
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
jobId,
|
|
395
|
-
state,
|
|
396
|
-
type: def.type ?? "unknown",
|
|
397
|
-
elapsedMs,
|
|
398
|
-
pollCount,
|
|
399
|
-
timedOut: true,
|
|
400
|
-
progress: {
|
|
401
|
-
done: gs.done ?? 0,
|
|
402
|
-
failed: gs.failed ?? 0,
|
|
403
|
-
running: gs.running ?? 0,
|
|
404
|
-
total: gs.total ?? null,
|
|
405
|
-
},
|
|
406
|
-
};
|
|
407
|
-
}
|
|
416
|
+
// Timeout — the guard at the top of the loop returns the failure
|
|
417
|
+
// result from this observation without issuing another request.
|
|
418
|
+
if (elapsedMs >= timeout)
|
|
419
|
+
continue;
|
|
408
420
|
const nextDelayMs = computeNextPollDelayMs({
|
|
409
421
|
pollCount,
|
|
410
422
|
baseIntervalMs,
|
|
@@ -117,20 +117,35 @@ export class MlTasksResource extends BaseResource {
|
|
|
117
117
|
const adaptiveEnabled = explicitIntervalMs === undefined;
|
|
118
118
|
const startedAt = Date.now();
|
|
119
119
|
let pollCount = 0;
|
|
120
|
-
|
|
121
|
-
// check, so `timeoutMs: 0` still reports the freshly-started state once.
|
|
120
|
+
let lastStatus;
|
|
122
121
|
while (true) {
|
|
122
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
123
|
+
// The first observation always happens, even when the budget is
|
|
124
|
+
// already spent. A later poll is never started after the deadline:
|
|
125
|
+
// the loop reports the structured timeout from the last observed
|
|
126
|
+
// status instead of issuing a request the budget cannot cover.
|
|
127
|
+
if (lastStatus !== undefined && elapsedBeforeMs >= deadlineMs) {
|
|
128
|
+
return {
|
|
129
|
+
sessionId: session.sessionId,
|
|
130
|
+
trainedModelIds: modelIdsFromStatus(lastStatus),
|
|
131
|
+
timedOut: true,
|
|
132
|
+
success: false,
|
|
133
|
+
state: "RUNNING",
|
|
134
|
+
elapsedMs: elapsedBeforeMs,
|
|
135
|
+
pollCount,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
123
138
|
pollCount += 1;
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
const remainingMs = deadlineMs -
|
|
139
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
140
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
141
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
142
|
+
// reaches the server instead of failing before any attempt.
|
|
143
|
+
const remainingMs = deadlineMs - elapsedBeforeMs;
|
|
129
144
|
let status;
|
|
130
145
|
try {
|
|
131
146
|
status = await this.client.get(`${this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey)}/status`, remainingMs > 0
|
|
132
147
|
? { timeoutMs: remainingMs, }
|
|
133
|
-
:
|
|
148
|
+
: { noRetry: true, });
|
|
134
149
|
}
|
|
135
150
|
catch (error) {
|
|
136
151
|
// The poll budget ran out: report the structured timeout instead
|
|
@@ -147,19 +162,13 @@ export class MlTasksResource extends BaseResource {
|
|
|
147
162
|
pollCount,
|
|
148
163
|
};
|
|
149
164
|
}
|
|
165
|
+
lastStatus = status;
|
|
150
166
|
const elapsedMs = Date.now() - startedAt;
|
|
151
167
|
if (status.training !== false) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
timedOut: true,
|
|
157
|
-
success: false,
|
|
158
|
-
state: "RUNNING",
|
|
159
|
-
elapsedMs,
|
|
160
|
-
pollCount,
|
|
161
|
-
};
|
|
162
|
-
}
|
|
168
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
169
|
+
// timeout from this status without issuing another request.
|
|
170
|
+
if (elapsedMs >= deadlineMs)
|
|
171
|
+
continue;
|
|
163
172
|
const nextDelayMs = computeNextPollDelayMs({
|
|
164
173
|
pollCount,
|
|
165
174
|
baseIntervalMs,
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { BaseResource } from "./base.js";
|
|
2
|
-
import { validateGitReferencePath } from "./project-git.js";
|
|
3
2
|
/** Summary entry of `/plugins/` as documented (id/version/isDev/meta). */
|
|
4
3
|
export interface PluginSummary {
|
|
5
4
|
id: string;
|
|
@@ -262,4 +261,3 @@ export declare function validatedPluginGitUrl(repository: string, field?: string
|
|
|
262
261
|
* CLI to decide which actions need a plugin id positional.
|
|
263
262
|
*/
|
|
264
263
|
export declare function isPluginIdCandidate(value: unknown): boolean;
|
|
265
|
-
export { validateGitReferencePath, };
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
2
2
|
import { BaseResource, requireNonEmpty, } from "./base.js";
|
|
3
|
-
import { validateGitReferencePath, } from "./project-git.js";
|
|
4
3
|
/* ------------------------------------------------------------------ */
|
|
5
4
|
/* Path helpers */
|
|
6
5
|
/* ------------------------------------------------------------------ */
|
|
@@ -454,5 +453,3 @@ export function validatedPluginGitUrl(repository, field = "Git repository URL")
|
|
|
454
453
|
export function isPluginIdCandidate(value) {
|
|
455
454
|
return typeof value === "string" && PLUGIN_ID_PATTERN.test(value);
|
|
456
455
|
}
|
|
457
|
-
// Re-exported for CLI consumers so a single import site covers path rules.
|
|
458
|
-
export { validateGitReferencePath, };
|
|
@@ -68,23 +68,17 @@ export interface ProjectGitFutureStateOptions {
|
|
|
68
68
|
* loops that must not hang on a stalled endpoint.
|
|
69
69
|
*/
|
|
70
70
|
timeoutMs?: number;
|
|
71
|
+
/**
|
|
72
|
+
* Issue exactly one transport attempt (no retries). Independent of
|
|
73
|
+
* `timeoutMs`: when both are supplied the total budget still applies. Used
|
|
74
|
+
* by the wait loop for the one observation whose wait budget is spent.
|
|
75
|
+
*/
|
|
76
|
+
noRetry?: boolean;
|
|
71
77
|
}
|
|
72
78
|
export interface ProjectGitFutureWaitOptions {
|
|
73
79
|
pollIntervalMs?: number;
|
|
74
80
|
timeoutMs?: number;
|
|
75
81
|
}
|
|
76
|
-
/**
|
|
77
|
-
* Validate and canonicalize a Git library reference path. This is the single
|
|
78
|
-
* validator used by both SDK dispatch and CLI mutation planning.
|
|
79
|
-
*
|
|
80
|
-
* Strip only the permitted outer '/' separators, then require nonempty
|
|
81
|
-
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
82
|
-
* path therefore cannot change which API resource a dispatcher's URL
|
|
83
|
-
* normalization would otherwise select.
|
|
84
|
-
*/
|
|
85
|
-
export declare function validateGitReferencePath(path: string): string;
|
|
86
|
-
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
87
|
-
export declare function encodeGitReferencePath(path: string): string;
|
|
88
82
|
export declare class ProjectGitResource extends BaseResource {
|
|
89
83
|
/** Working-copy status: current branch, remotes, tracking counts, changes. */
|
|
90
84
|
status(projectKey: string): Promise<ProjectGitStatus>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ClientValidationError, } from "../errors.js";
|
|
2
2
|
import { ProjectGitActionResultSchema, ProjectGitDiffResultSchema, ProjectGitFutureResponseSchema, ProjectGitFutureStateSchema, ProjectGitLibrariesSchema, ProjectGitLogResultSchema, ProjectGitRemoteSchema, ProjectGitStatusSchema, ProjectGitTagsSchema, } from "../schemas.js";
|
|
3
|
+
import { encodeGitReferencePath, validateGitReferencePath, } from "../utils/git-reference.js";
|
|
3
4
|
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
4
5
|
import { sanitizeErrorSecrets, } from "../utils/secret-sanitize.js";
|
|
5
6
|
import { BaseResource, } from "./base.js";
|
|
@@ -19,43 +20,6 @@ function sleep(ms) {
|
|
|
19
20
|
setTimeout(resolve, ms);
|
|
20
21
|
return promise;
|
|
21
22
|
}
|
|
22
|
-
function gitReferencePathError(path, message) {
|
|
23
|
-
return new ClientValidationError(message, "validation_failed", "Use a Git library reference path relative to the project library root: '/' -separated segment names, with no '.' or '..' segments.", { path, });
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Validate and canonicalize a Git library reference path. This is the single
|
|
27
|
-
* validator used by both SDK dispatch and CLI mutation planning.
|
|
28
|
-
*
|
|
29
|
-
* Strip only the permitted outer '/' separators, then require nonempty
|
|
30
|
-
* ordinary segments: no empty segments and no '.' or '..' segments. A hostile
|
|
31
|
-
* path therefore cannot change which API resource a dispatcher's URL
|
|
32
|
-
* normalization would otherwise select.
|
|
33
|
-
*/
|
|
34
|
-
export function validateGitReferencePath(path) {
|
|
35
|
-
if (typeof path !== "string" || path.trim() === "") {
|
|
36
|
-
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
37
|
-
}
|
|
38
|
-
const normalized = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
39
|
-
if (normalized === "") {
|
|
40
|
-
throw gitReferencePathError(path, "A Git library reference path is required.");
|
|
41
|
-
}
|
|
42
|
-
for (const segment of normalized.split("/")) {
|
|
43
|
-
if (segment === "") {
|
|
44
|
-
throw gitReferencePathError(path, "A Git library reference path must not contain empty segments (consecutive or repeated '/').");
|
|
45
|
-
}
|
|
46
|
-
if (segment === "." || segment === "..") {
|
|
47
|
-
throw gitReferencePathError(path, "A Git library reference path must not contain '.' or '..' segments.");
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
return normalized;
|
|
51
|
-
}
|
|
52
|
-
/** Validate a Git library reference path and URL-encode it per segment. */
|
|
53
|
-
export function encodeGitReferencePath(path) {
|
|
54
|
-
return validateGitReferencePath(path)
|
|
55
|
-
.split("/")
|
|
56
|
-
.map((segment) => encodeURIComponent(segment))
|
|
57
|
-
.join("/");
|
|
58
|
-
}
|
|
59
23
|
/**
|
|
60
24
|
* Reject `https://user:token@host/repo` before it reaches DSS: an embedded
|
|
61
25
|
* secret would be stored in the project's Git configuration and echoed back by
|
|
@@ -364,7 +328,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
364
328
|
* what polling loops want; the default consumes it like the Python client.
|
|
365
329
|
*/
|
|
366
330
|
async getFutureState(jobId, options = {}) {
|
|
367
|
-
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`, undefined, [], { timeoutMs: options.timeoutMs, });
|
|
331
|
+
const raw = await this.request("GET", `/futures/${encodeURIComponent(jobId)}${this.query({ peek: options.peek === true, })}`, undefined, [], { timeoutMs: options.timeoutMs, noRetry: options.noRetry, });
|
|
368
332
|
return this.client.safeParse(ProjectGitFutureStateSchema, raw ?? {}, "projectGit.getFutureState");
|
|
369
333
|
}
|
|
370
334
|
/** Abort a running future. */
|
|
@@ -389,14 +353,23 @@ export class ProjectGitResource extends BaseResource {
|
|
|
389
353
|
// interval; an explicit pollIntervalMs is an exact contract.
|
|
390
354
|
const adaptiveEnabled = options.pollIntervalMs === undefined;
|
|
391
355
|
let pollCount = 0;
|
|
356
|
+
let lastState;
|
|
392
357
|
const startedAt = Date.now();
|
|
393
358
|
while (true) {
|
|
359
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
360
|
+
// The first observation always happens, even when the budget is
|
|
361
|
+
// already spent. A later poll is never started after the deadline:
|
|
362
|
+
// the loop reports the documented timeout from the last observed
|
|
363
|
+
// state instead of issuing a request the budget cannot cover.
|
|
364
|
+
if (lastState !== undefined && elapsedBeforeMs >= timeoutMs) {
|
|
365
|
+
throw new Error(`Timed out after ${String(elapsedBeforeMs)}ms waiting for Dataiku future ${jobId}`);
|
|
366
|
+
}
|
|
394
367
|
// Unlike peek state, the ordinary state includes the completed result.
|
|
395
|
-
//
|
|
396
|
-
//
|
|
397
|
-
//
|
|
398
|
-
//
|
|
399
|
-
const remainingMs = timeoutMs -
|
|
368
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
369
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
370
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
371
|
+
// reaches the server instead of failing before any attempt.
|
|
372
|
+
const remainingMs = timeoutMs - elapsedBeforeMs;
|
|
400
373
|
pollCount += 1;
|
|
401
374
|
let state;
|
|
402
375
|
try {
|
|
@@ -405,7 +378,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
405
378
|
peek: false,
|
|
406
379
|
timeoutMs: remainingMs,
|
|
407
380
|
}
|
|
408
|
-
: { peek: false, });
|
|
381
|
+
: { peek: false, noRetry: true, });
|
|
409
382
|
}
|
|
410
383
|
catch (error) {
|
|
411
384
|
// The poll budget ran out mid-request: report the documented
|
|
@@ -415,6 +388,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
415
388
|
throw error;
|
|
416
389
|
throw new Error(`Timed out after ${String(Date.now() - startedAt)}ms waiting for Dataiku future ${jobId}`, { cause: error, });
|
|
417
390
|
}
|
|
391
|
+
lastState = state;
|
|
418
392
|
const failure = describeFutureFailure(state.error);
|
|
419
393
|
if (failure !== undefined) {
|
|
420
394
|
throw new Error(`Dataiku future ${jobId} failed: ${failure}`);
|
|
@@ -431,9 +405,10 @@ export class ProjectGitResource extends BaseResource {
|
|
|
431
405
|
throw new Error(`Dataiku future ${jobId} ended without producing a result`);
|
|
432
406
|
}
|
|
433
407
|
const elapsedMs = Date.now() - startedAt;
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
408
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
409
|
+
// timeout from this observation without issuing another request.
|
|
410
|
+
if (elapsedMs >= timeoutMs)
|
|
411
|
+
continue;
|
|
437
412
|
const nextDelayMs = computeNextPollDelayMs({
|
|
438
413
|
pollCount,
|
|
439
414
|
baseIntervalMs: pollIntervalMs,
|
|
@@ -466,7 +441,7 @@ export class ProjectGitResource extends BaseResource {
|
|
|
466
441
|
}
|
|
467
442
|
async request(method, path, body, secrets = [], opts = {}) {
|
|
468
443
|
try {
|
|
469
|
-
const res = await this.client.requestGit(method, `${GIT_API_BASE}${path}`, body, opts.timeoutMs === undefined ? undefined : Date.now() + opts.timeoutMs);
|
|
444
|
+
const res = await this.client.requestGit(method, `${GIT_API_BASE}${path}`, body, opts.timeoutMs === undefined ? undefined : Date.now() + opts.timeoutMs, opts.noRetry === true);
|
|
470
445
|
const text = await res.text();
|
|
471
446
|
if (text.trim() === "")
|
|
472
447
|
return undefined;
|
|
@@ -150,8 +150,10 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
150
150
|
* Build a normalized, optionally truncated flow map for a project.
|
|
151
151
|
*
|
|
152
152
|
* Fetches the flow graph and supplementary metadata (datasets, recipes,
|
|
153
|
-
* managed folders) in parallel.
|
|
154
|
-
*
|
|
153
|
+
* managed folders) in parallel. Each metadata fetch runs under a total
|
|
154
|
+
* `timeoutMs` budget: when the budget expires the in-flight request is
|
|
155
|
+
* aborted and the map is returned without that metadata, reported as a
|
|
156
|
+
* warning.
|
|
155
157
|
*/
|
|
156
158
|
map(opts?: FlowMapOptions & {
|
|
157
159
|
projectKey?: string;
|