dataiku-sdk 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -19
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +39 -4
- package/dist/packages/types/src/index.js +31 -2
- package/dist/src/auth.js +1 -1
- package/dist/src/cli/coerce.js +11 -2
- package/dist/src/cli/commands/analysis.js +4 -3
- package/dist/src/cli/commands/api-service.js +66 -18
- package/dist/src/cli/commands/app.js +4 -3
- package/dist/src/cli/commands/bundle.js +51 -16
- package/dist/src/cli/commands/code-env.js +219 -57
- package/dist/src/cli/commands/code.d.ts +19 -0
- package/dist/src/cli/commands/code.js +87 -15
- package/dist/src/cli/commands/dashboard.js +7 -6
- package/dist/src/cli/commands/data-quality.js +9 -8
- package/dist/src/cli/commands/dataset.js +17 -11
- package/dist/src/cli/commands/doctor.js +4 -2
- package/dist/src/cli/commands/flow-zone.js +42 -13
- package/dist/src/cli/commands/folder.js +9 -8
- package/dist/src/cli/commands/future.js +2 -1
- package/dist/src/cli/commands/insight.js +8 -7
- package/dist/src/cli/commands/job.js +4 -3
- package/dist/src/cli/commands/meaning.js +5 -4
- package/dist/src/cli/commands/ml-task.js +3 -2
- package/dist/src/cli/commands/model-evaluation-store.js +4 -3
- package/dist/src/cli/commands/notebook.js +82 -50
- package/dist/src/cli/commands/project-git.js +8 -5
- package/dist/src/cli/commands/project-library.d.ts +21 -0
- package/dist/src/cli/commands/project-library.js +128 -56
- package/dist/src/cli/commands/project.js +19 -4
- package/dist/src/cli/commands/recipe.js +51 -25
- package/dist/src/cli/commands/saved-model.js +4 -3
- package/dist/src/cli/commands/scenario.js +9 -8
- package/dist/src/cli/commands/sql.d.ts +10 -0
- package/dist/src/cli/commands/sql.js +42 -19
- package/dist/src/cli/commands/statistics.js +7 -6
- package/dist/src/cli/commands/variable.js +2 -1
- package/dist/src/cli/commands/webapp.js +68 -13
- package/dist/src/cli/commands/wiki.js +7 -6
- package/dist/src/cli/contract.d.ts +16 -1
- package/dist/src/cli/contract.js +967 -186
- package/dist/src/cli/env.d.ts +9 -0
- package/dist/src/cli/env.js +20 -2
- package/dist/src/cli/flags.d.ts +4 -0
- package/dist/src/cli/flags.js +29 -1
- package/dist/src/cli/helpers/cleanup.js +104 -3
- package/dist/src/cli/helpers/flow-zone.d.ts +3 -0
- package/dist/src/cli/helpers/flow-zone.js +33 -3
- package/dist/src/cli/helpers/recipe.d.ts +20 -0
- package/dist/src/cli/helpers/recipe.js +158 -3
- package/dist/src/cli/output.d.ts +2 -0
- package/dist/src/cli/output.js +33 -6
- package/dist/src/cli/runtime.d.ts +35 -1
- package/dist/src/cli/runtime.js +125 -18
- package/dist/src/cli/types.d.ts +7 -0
- package/dist/src/cli.js +226 -95
- package/dist/src/client.d.ts +63 -1
- package/dist/src/client.js +305 -53
- package/dist/src/errors.d.ts +8 -2
- package/dist/src/errors.js +42 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +2 -1
- package/dist/src/resources/api-services.d.ts +48 -6
- package/dist/src/resources/api-services.js +54 -8
- package/dist/src/resources/bundles.d.ts +51 -6
- package/dist/src/resources/bundles.js +39 -9
- package/dist/src/resources/code-envs.d.ts +25 -3
- package/dist/src/resources/code-envs.js +104 -2
- package/dist/src/resources/datasets.d.ts +20 -0
- package/dist/src/resources/datasets.js +232 -52
- package/dist/src/resources/folders.js +2 -1
- package/dist/src/resources/jobs.d.ts +35 -10
- package/dist/src/resources/jobs.js +92 -17
- package/dist/src/resources/notebooks.d.ts +77 -7
- package/dist/src/resources/notebooks.js +143 -19
- package/dist/src/resources/project-git.d.ts +12 -0
- package/dist/src/resources/project-git.js +39 -8
- package/dist/src/resources/project-library.d.ts +122 -7
- package/dist/src/resources/project-library.js +494 -25
- package/dist/src/resources/projects.d.ts +5 -4
- package/dist/src/resources/projects.js +21 -19
- package/dist/src/resources/scenarios.d.ts +30 -0
- package/dist/src/resources/scenarios.js +73 -27
- package/dist/src/resources/sql.d.ts +6 -0
- package/dist/src/resources/sql.js +23 -3
- package/dist/src/resources/webapps.d.ts +35 -4
- package/dist/src/resources/webapps.js +54 -5
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/skill.d.ts +18 -0
- package/dist/src/skill.js +93 -9
- package/dist/src/utils/flow-analysis.d.ts +53 -0
- package/dist/src/utils/flow-analysis.js +382 -0
- package/dist/src/utils/project-archive.d.ts +41 -2
- package/dist/src/utils/project-archive.js +159 -16
- package/dist/src/utils/response-file.d.ts +2 -0
- package/dist/src/utils/response-file.js +34 -0
- package/package.json +9 -7
- package/packages/types/dist/index.d.ts +39 -4
- package/packages/types/dist/index.js +31 -2
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +44 -197
- package/skills/dataiku-dss/references/app-releases.md +27 -0
- package/skills/dataiku-dss/references/authentication.md +39 -0
- package/skills/dataiku-dss/references/coding.md +8 -0
- package/skills/dataiku-dss/references/discovery.md +30 -0
- package/skills/dataiku-dss/references/flow-maps.md +12 -0
- package/skills/dataiku-dss/references/mutations.md +33 -0
- package/skills/dataiku-dss/references/troubleshooting.md +29 -0
|
@@ -34,6 +34,24 @@ const NON_DATASET_REFERENCE_TYPES = {
|
|
|
34
34
|
const MAX_JSON_MEMBER_BYTES = 16 * 1024 * 1024;
|
|
35
35
|
/** Upper bound for residue text scanning of individual members. */
|
|
36
36
|
const MAX_RESIDUE_SCAN_BYTES = 1024 * 1024;
|
|
37
|
+
export const PROJECT_ARCHIVE_DEFAULT_LIMITS = Object
|
|
38
|
+
.freeze({
|
|
39
|
+
maxMembers: 100_000,
|
|
40
|
+
maxCompressedBytesPerMember: 4 * 1024 * 1024 * 1024,
|
|
41
|
+
maxExpandedBytesPerMember: 4 * 1024 * 1024 * 1024,
|
|
42
|
+
maxCompressionRatio: 1_000,
|
|
43
|
+
maxTotalExpandedBytes: 16 * 1024 * 1024 * 1024,
|
|
44
|
+
});
|
|
45
|
+
/** Merge per-call overrides onto the exported default inspection limits. */
|
|
46
|
+
function effectiveLimits(overrides) {
|
|
47
|
+
const merged = { ...PROJECT_ARCHIVE_DEFAULT_LIMITS, };
|
|
48
|
+
for (const key of Object.keys(overrides)) {
|
|
49
|
+
const value = overrides[key];
|
|
50
|
+
if (value !== undefined)
|
|
51
|
+
merged[key] = value;
|
|
52
|
+
}
|
|
53
|
+
return merged;
|
|
54
|
+
}
|
|
37
55
|
function plainRecord(value) {
|
|
38
56
|
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
39
57
|
? value
|
|
@@ -107,12 +125,19 @@ function isUnsafeMemberPath(name) {
|
|
|
107
125
|
* own verification is bypassed for data-descriptor records (bit 3 set), so the
|
|
108
126
|
* manual `Bun.hash.crc32` computation covers every record uniformly.
|
|
109
127
|
*
|
|
110
|
-
* When `accumulate` is set, buffered content never exceeds `
|
|
111
|
-
* buffer is dropped as soon as the cap is reached while the member is
|
|
112
|
-
* streamed in full, so CRC/size verification stays complete and the size
|
|
128
|
+
* When `accumulate` is set, buffered content never exceeds `bufferMaxBytes`:
|
|
129
|
+
* the buffer is dropped as soon as the cap is reached while the member is
|
|
130
|
+
* still streamed in full, so CRC/size verification stays complete and the size
|
|
113
131
|
* mismatch (if the declared size lied) is still reported.
|
|
132
|
+
*
|
|
133
|
+
* The safety budgets bound decompression itself, not just buffering: once the
|
|
134
|
+
* per-member expanded-byte budget (`expandedBytes`) or the aggregate
|
|
135
|
+
* expanded-byte budget (`totalExpandedBytes`, tracked across members in
|
|
136
|
+
* `tally`) is crossed, the member stream is destroyed immediately and no
|
|
137
|
+
* further decompression or hashing happens. The caller must not compare the
|
|
138
|
+
* partial CRC/size result, which is why the result carries `aborted`.
|
|
114
139
|
*/
|
|
115
|
-
function streamMember(zip, entry, accumulate,
|
|
140
|
+
function streamMember(zip, entry, accumulate, bufferMaxBytes, limits) {
|
|
116
141
|
const { promise, resolve, reject, } = Promise.withResolvers();
|
|
117
142
|
zip.stream(entry, (err, stm) => {
|
|
118
143
|
if (err) {
|
|
@@ -125,14 +150,29 @@ function streamMember(zip, entry, accumulate, maxBytes = Number.POSITIVE_INFINIT
|
|
|
125
150
|
let bytes = 0;
|
|
126
151
|
let crc = 0;
|
|
127
152
|
let truncated = false;
|
|
153
|
+
let aborted = false;
|
|
128
154
|
const content = accumulate ? [] : undefined;
|
|
155
|
+
const abort = (reason) => {
|
|
156
|
+
aborted = reason;
|
|
157
|
+
stream.destroy();
|
|
158
|
+
resolve({ bytes, crc, content: undefined, truncated, aborted, });
|
|
159
|
+
};
|
|
129
160
|
stream.on("data", (chunk) => {
|
|
161
|
+
if (aborted)
|
|
162
|
+
return;
|
|
130
163
|
const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
131
164
|
bytes += buffer.length;
|
|
165
|
+
limits.tally.expandedBytes += buffer.length;
|
|
166
|
+
if (limits.tally.expandedBytes > limits.totalExpandedBytes) {
|
|
167
|
+
return abort("aggregate");
|
|
168
|
+
}
|
|
169
|
+
if (bytes > limits.expandedBytes) {
|
|
170
|
+
return abort("member");
|
|
171
|
+
}
|
|
132
172
|
crc = Bun.hash.crc32(buffer, crc) >>> 0;
|
|
133
173
|
if (!content || truncated)
|
|
134
174
|
return;
|
|
135
|
-
if (bytes >
|
|
175
|
+
if (bytes > bufferMaxBytes) {
|
|
136
176
|
truncated = true;
|
|
137
177
|
content.length = 0;
|
|
138
178
|
return;
|
|
@@ -140,14 +180,21 @@ function streamMember(zip, entry, accumulate, maxBytes = Number.POSITIVE_INFINIT
|
|
|
140
180
|
content.push(buffer);
|
|
141
181
|
});
|
|
142
182
|
stream.on("end", () => {
|
|
183
|
+
if (aborted)
|
|
184
|
+
return;
|
|
143
185
|
resolve({
|
|
144
186
|
bytes,
|
|
145
187
|
crc,
|
|
146
188
|
content: truncated ? undefined : content,
|
|
147
189
|
truncated,
|
|
190
|
+
aborted: false,
|
|
148
191
|
});
|
|
149
192
|
});
|
|
150
193
|
stream.on("error", (streamErr) => {
|
|
194
|
+
// A stream destroyed by a budget breach may surface a late error;
|
|
195
|
+
// the result was already resolved and the partial data is unusable.
|
|
196
|
+
if (aborted)
|
|
197
|
+
return;
|
|
151
198
|
reject(streamErr);
|
|
152
199
|
});
|
|
153
200
|
});
|
|
@@ -302,6 +349,17 @@ function residueIssue(member, counts) {
|
|
|
302
349
|
member,
|
|
303
350
|
};
|
|
304
351
|
}
|
|
352
|
+
/** Deterministic issue ordering: severity, then code, member, message. */
|
|
353
|
+
function sortIssues(issues) {
|
|
354
|
+
issues.sort((a, b) => {
|
|
355
|
+
const severityOrder = a.severity === b.severity ? 0 : a.severity === "error" ? -1 : 1;
|
|
356
|
+
return (severityOrder
|
|
357
|
+
|| a.code.localeCompare(b.code)
|
|
358
|
+
|| (a.member ?? "").localeCompare(b.member ?? "")
|
|
359
|
+
|| a.message.localeCompare(b.message));
|
|
360
|
+
});
|
|
361
|
+
return issues;
|
|
362
|
+
}
|
|
305
363
|
/**
|
|
306
364
|
* Inspect a DSS project export ZIP archive without extracting anything to
|
|
307
365
|
* disk: enumerate every central-directory record, validate member paths
|
|
@@ -311,8 +369,18 @@ function residueIssue(member, counts) {
|
|
|
311
369
|
* the generic dangling-reference/orphan-member graph checks. Identity,
|
|
312
370
|
* history and machine-specific residue is reported only as warnings carrying
|
|
313
371
|
* member paths and pattern counts, never matched values.
|
|
372
|
+
*
|
|
373
|
+
* Safety budgets (see {@link ProjectArchiveInspectionLimits}) bound every
|
|
374
|
+
* amount of work: archives with more members than `maxMembers` are reported
|
|
375
|
+
* without opening any member stream, members whose central-directory record
|
|
376
|
+
* declares a compressed size or compression ratio above budget are skipped
|
|
377
|
+
* without being read, and per-member/aggregate expanded-byte overruns
|
|
378
|
+
* destroy the member stream immediately. Pass a partial limits object as
|
|
379
|
+
* `overrides` to customize the budgets for a single inspection; importers
|
|
380
|
+
* use the exported defaults.
|
|
314
381
|
*/
|
|
315
|
-
export async function inspectProjectArchive(filePath) {
|
|
382
|
+
export async function inspectProjectArchive(filePath, overrides = {}) {
|
|
383
|
+
const limits = effectiveLimits(overrides);
|
|
316
384
|
let sizeBytes = 0;
|
|
317
385
|
let statError;
|
|
318
386
|
try {
|
|
@@ -373,6 +441,23 @@ export async function inspectProjectArchive(filePath) {
|
|
|
373
441
|
};
|
|
374
442
|
}
|
|
375
443
|
try {
|
|
444
|
+
// The member-count budget bounds inspection work per archive. When it
|
|
445
|
+
// is exceeded no member stream is opened: the archive reports a single
|
|
446
|
+
// error and cannot be imported.
|
|
447
|
+
if (records.length > limits.maxMembers) {
|
|
448
|
+
issues.push({
|
|
449
|
+
severity: "error",
|
|
450
|
+
code: "member_count_exceeded",
|
|
451
|
+
message: `archive contains ${records.length} members, above the inspection limit of ${limits.maxMembers}`,
|
|
452
|
+
});
|
|
453
|
+
return {
|
|
454
|
+
filePath,
|
|
455
|
+
sizeBytes,
|
|
456
|
+
memberCount: records.length,
|
|
457
|
+
valid: false,
|
|
458
|
+
issues,
|
|
459
|
+
};
|
|
460
|
+
}
|
|
376
461
|
const memberCounts = new Map();
|
|
377
462
|
for (const entry of records) {
|
|
378
463
|
memberCounts.set(entry.name, (memberCounts.get(entry.name) ?? 0) + 1);
|
|
@@ -398,6 +483,8 @@ export async function inspectProjectArchive(filePath) {
|
|
|
398
483
|
}
|
|
399
484
|
const manifestJson = new Map();
|
|
400
485
|
const recipes = new Map();
|
|
486
|
+
const tally = { expandedBytes: 0, };
|
|
487
|
+
let aggregateAborted = false;
|
|
401
488
|
for (const entry of records) {
|
|
402
489
|
if (!entry.isFile)
|
|
403
490
|
continue;
|
|
@@ -426,16 +513,44 @@ export async function inspectProjectArchive(filePath) {
|
|
|
426
513
|
|| name.startsWith(`${BUNDLED_DATA_ROOT}/`)
|
|
427
514
|
|| name.startsWith(`${UPLOADS_ROOT}/`))
|
|
428
515
|
&& entry.size <= MAX_RESIDUE_SCAN_BYTES;
|
|
516
|
+
// Budgets computed from the central-directory record abort before
|
|
517
|
+
// any decompression work: a member whose declared compressed size
|
|
518
|
+
// or expanded/compressed ratio is above budget is never streamed.
|
|
519
|
+
if (entry.compressedSize > limits.maxCompressedBytesPerMember) {
|
|
520
|
+
issues.push({
|
|
521
|
+
severity: "error",
|
|
522
|
+
code: "member_compressed_size_exceeded",
|
|
523
|
+
message: `member ${name}: declared compressed size exceeds the inspection limit and was not read`,
|
|
524
|
+
member: name,
|
|
525
|
+
});
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
if (entry.compressedSize > 0
|
|
529
|
+
&& entry.size / entry.compressedSize > limits.maxCompressionRatio) {
|
|
530
|
+
issues.push({
|
|
531
|
+
severity: "error",
|
|
532
|
+
code: "member_compression_ratio_exceeded",
|
|
533
|
+
message: `member ${name}: declared compression ratio exceeds the inspection limit and was not read`,
|
|
534
|
+
member: name,
|
|
535
|
+
});
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
429
538
|
let streamed;
|
|
430
539
|
try {
|
|
431
540
|
// The buffering cap applies to actual streamed bytes, not the
|
|
432
541
|
// declared size: a member can declare a small size and still
|
|
433
542
|
// inflate arbitrarily, which must not be buffered unbounded.
|
|
543
|
+
// The expanded-byte budgets bound decompression itself and
|
|
544
|
+
// destroy the member stream the moment one is crossed.
|
|
434
545
|
streamed = await streamMember(zip, entry, parseCandidate || scanCandidate, parseCandidate
|
|
435
546
|
? MAX_JSON_MEMBER_BYTES
|
|
436
547
|
: scanCandidate
|
|
437
548
|
? MAX_RESIDUE_SCAN_BYTES
|
|
438
|
-
: Number.POSITIVE_INFINITY
|
|
549
|
+
: Number.POSITIVE_INFINITY, {
|
|
550
|
+
expandedBytes: limits.maxExpandedBytesPerMember,
|
|
551
|
+
totalExpandedBytes: limits.maxTotalExpandedBytes,
|
|
552
|
+
tally,
|
|
553
|
+
});
|
|
439
554
|
}
|
|
440
555
|
catch (streamErr) {
|
|
441
556
|
streamed = undefined;
|
|
@@ -467,6 +582,26 @@ export async function inspectProjectArchive(filePath) {
|
|
|
467
582
|
}
|
|
468
583
|
if (!streamed)
|
|
469
584
|
continue;
|
|
585
|
+
if (streamed.aborted) {
|
|
586
|
+
issues.push(streamed.aborted === "aggregate"
|
|
587
|
+
? {
|
|
588
|
+
severity: "error",
|
|
589
|
+
code: "aggregate_inflation_limit_exceeded",
|
|
590
|
+
message: `member ${name}: aggregate expanded bytes exceed the inspection budget; remaining members were not inspected`,
|
|
591
|
+
member: name,
|
|
592
|
+
}
|
|
593
|
+
: {
|
|
594
|
+
severity: "error",
|
|
595
|
+
code: "member_inflation_limit_exceeded",
|
|
596
|
+
message: `member ${name}: data exceeds the per-member expanded-byte inspection budget and was not fully inspected`,
|
|
597
|
+
member: name,
|
|
598
|
+
});
|
|
599
|
+
if (streamed.aborted === "aggregate") {
|
|
600
|
+
aggregateAborted = true;
|
|
601
|
+
break;
|
|
602
|
+
}
|
|
603
|
+
continue;
|
|
604
|
+
}
|
|
470
605
|
if (streamed.bytes !== entry.size) {
|
|
471
606
|
issues.push({
|
|
472
607
|
severity: "error",
|
|
@@ -524,6 +659,20 @@ export async function inspectProjectArchive(filePath) {
|
|
|
524
659
|
}
|
|
525
660
|
}
|
|
526
661
|
}
|
|
662
|
+
// Once the aggregate expanded-byte budget is crossed, remaining
|
|
663
|
+
// members were not inspected: report the budget issue and stop, so no
|
|
664
|
+
// manifest/graph conclusion is drawn from the partial inspection.
|
|
665
|
+
if (aggregateAborted) {
|
|
666
|
+
sortIssues(issues);
|
|
667
|
+
return {
|
|
668
|
+
filePath,
|
|
669
|
+
sizeBytes,
|
|
670
|
+
memberCount: records.length,
|
|
671
|
+
sourceProjectKey: undefined,
|
|
672
|
+
valid: false,
|
|
673
|
+
issues,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
527
676
|
let sourceProjectKey;
|
|
528
677
|
let manifestDatasetNames;
|
|
529
678
|
const manifestMember = manifestJson.get(MANIFEST_MEMBER);
|
|
@@ -580,13 +729,7 @@ export async function inspectProjectArchive(filePath) {
|
|
|
580
729
|
const recipeNames = recipeDefinitionNames(records);
|
|
581
730
|
checkRecipeReferences(recipes, datasetNames, nonDatasetNames, issues);
|
|
582
731
|
checkOrphanMembers(records, recipeNames, anchored, issues);
|
|
583
|
-
issues
|
|
584
|
-
const severityOrder = a.severity === b.severity ? 0 : a.severity === "error" ? -1 : 1;
|
|
585
|
-
return (severityOrder
|
|
586
|
-
|| a.code.localeCompare(b.code)
|
|
587
|
-
|| (a.member ?? "").localeCompare(b.member ?? "")
|
|
588
|
-
|| a.message.localeCompare(b.message));
|
|
589
|
-
});
|
|
732
|
+
sortIssues(issues);
|
|
590
733
|
return {
|
|
591
734
|
filePath,
|
|
592
735
|
sizeBytes,
|
|
@@ -613,8 +756,8 @@ export async function inspectProjectArchive(filePath) {
|
|
|
613
756
|
* `validation_failed` code when the inspection contains any error-severity
|
|
614
757
|
* issue, so the archive cannot be imported.
|
|
615
758
|
*/
|
|
616
|
-
export async function assertImportableProjectArchive(filePath) {
|
|
617
|
-
const inspection = await inspectProjectArchive(filePath);
|
|
759
|
+
export async function assertImportableProjectArchive(filePath, overrides = {}) {
|
|
760
|
+
const inspection = await inspectProjectArchive(filePath, overrides);
|
|
618
761
|
const errorCount = inspection.issues.filter((issue) => issue.severity === "error").length;
|
|
619
762
|
if (errorCount > 0) {
|
|
620
763
|
throw new ClientValidationError(`project archive is not importable: ${errorCount} validation error(s) found`, "validation_failed", undefined, { filePath: inspection.filePath, issues: inspection.issues, });
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { randomUUID, } from "node:crypto";
|
|
2
|
+
import { createWriteStream, } from "node:fs";
|
|
3
|
+
import { chmod, rename, rm, stat, } from "node:fs/promises";
|
|
4
|
+
import { basename, dirname, join, } from "node:path";
|
|
5
|
+
import { Readable, } from "node:stream";
|
|
6
|
+
import { pipeline, } from "node:stream/promises";
|
|
7
|
+
/** Stream with backpressure; publish only a complete download, preserving existing output on failure. */
|
|
8
|
+
export async function writeResponseToFile(path, response, permissions = "private") {
|
|
9
|
+
let mode;
|
|
10
|
+
if (permissions === "preserve") {
|
|
11
|
+
try {
|
|
12
|
+
mode = (await stat(path)).mode & 0o777;
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (error.code !== "ENOENT")
|
|
16
|
+
throw error;
|
|
17
|
+
mode = 0o666 & ~process.umask();
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
const input = Readable.fromWeb(response.body);
|
|
21
|
+
const temporaryPath = join(dirname(path), `.${basename(path)}.tmp-${randomUUID()}`);
|
|
22
|
+
const output = createWriteStream(temporaryPath, { flags: "wx", mode: 0o600, });
|
|
23
|
+
try {
|
|
24
|
+
await pipeline(input, output);
|
|
25
|
+
// Keep partial contents private; restore publication permissions only after completion.
|
|
26
|
+
if (mode !== undefined)
|
|
27
|
+
await chmod(temporaryPath, mode);
|
|
28
|
+
await rename(temporaryPath, path);
|
|
29
|
+
return output.bytesWritten;
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
await rm(temporaryPath, { force: true, });
|
|
33
|
+
}
|
|
34
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dataiku-sdk",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.2.0",
|
|
4
4
|
"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"workspaces": [
|
|
@@ -25,13 +25,15 @@
|
|
|
25
25
|
"build": "tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
|
|
26
26
|
"prepack": "bun scripts/sync-plugin-version.mjs --check && tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
|
|
27
27
|
"prepublishOnly": "bun scripts/sync-plugin-version.mjs --check && tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
|
|
28
|
-
"check": "tsc --noEmit && bun scripts/sync-plugin-version.mjs --check",
|
|
28
|
+
"check": "tsc --noEmit && bun run check:live && bun scripts/sync-plugin-version.mjs --check",
|
|
29
29
|
"version": "bun scripts/sync-plugin-version.mjs && git add plugin.json",
|
|
30
|
-
"lint": "oxlint src/",
|
|
31
|
-
"lint:fix": "oxlint --fix src/",
|
|
30
|
+
"lint": "oxlint src/ scripts/live-suite.ts tests/live-*.ts",
|
|
31
|
+
"lint:fix": "oxlint --fix src/ scripts/live-suite.ts tests/live-*.ts",
|
|
32
32
|
"format": "dprint fmt",
|
|
33
33
|
"format:check": "dprint check",
|
|
34
34
|
"test": "bun test",
|
|
35
|
+
"test:live": "bun --no-env-file scripts/live-suite.ts",
|
|
36
|
+
"check:live": "tsc --noEmit -p tsconfig.live.json",
|
|
35
37
|
"test:tokens": "bun test tests/cli/agent-token-budget.test.ts",
|
|
36
38
|
"test:platform": "bun scripts/platform-smoke.mjs",
|
|
37
39
|
"test:integration": "cross-env RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-playground.test.ts",
|
|
@@ -42,11 +44,11 @@
|
|
|
42
44
|
"devDependencies": {
|
|
43
45
|
"@types/bun": "^1.4.0",
|
|
44
46
|
"cross-env": "^10.1.0",
|
|
45
|
-
"dprint": "^0.
|
|
47
|
+
"dprint": "^0.57.0",
|
|
46
48
|
"js-tiktoken": "1.0.21",
|
|
47
|
-
"oxlint": "^1.
|
|
49
|
+
"oxlint": "^1.81.0",
|
|
48
50
|
"typescript": "^7.0.2",
|
|
49
|
-
"typescript-language-server": "
|
|
51
|
+
"typescript-language-server": "6.0.0",
|
|
50
52
|
"typescript-language-service": "npm:typescript@5.9.3"
|
|
51
53
|
},
|
|
52
54
|
"dependencies": {
|
|
@@ -430,7 +430,6 @@ export type ConnectionSummary = Static<typeof ConnectionSummarySchema>;
|
|
|
430
430
|
export declare const CodeEnvSummarySchema: import("@sinclair/typebox").TObject<{
|
|
431
431
|
envName: import("@sinclair/typebox").TString;
|
|
432
432
|
envLang: import("@sinclair/typebox").TString;
|
|
433
|
-
pythonInterpreter: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
434
433
|
deploymentMode: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
435
434
|
}>;
|
|
436
435
|
export type CodeEnvSummary = Static<typeof CodeEnvSummarySchema>;
|
|
@@ -438,10 +437,42 @@ export declare const CodeEnvDetailsSchema: import("@sinclair/typebox").TObject<{
|
|
|
438
437
|
envName: import("@sinclair/typebox").TString;
|
|
439
438
|
envLang: import("@sinclair/typebox").TString;
|
|
440
439
|
pythonInterpreter: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
440
|
+
deploymentMode: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
441
441
|
requestedPackages: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
442
442
|
installedPackages: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
443
|
+
definitionHash: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
443
444
|
}>;
|
|
444
445
|
export type CodeEnvDetails = Static<typeof CodeEnvDetailsSchema>;
|
|
446
|
+
export declare const CodeEnvLogSummarySchema: import("@sinclair/typebox").TObject<{
|
|
447
|
+
name: import("@sinclair/typebox").TString;
|
|
448
|
+
}>;
|
|
449
|
+
export type CodeEnvLogSummary = Static<typeof CodeEnvLogSummarySchema>;
|
|
450
|
+
export declare const CodeEnvLogSummaryArraySchema: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
451
|
+
name: import("@sinclair/typebox").TString;
|
|
452
|
+
}>>;
|
|
453
|
+
export type CodeEnvLogSummaryArray = Static<typeof CodeEnvLogSummaryArraySchema>;
|
|
454
|
+
export declare const CodeEnvLogResultSchema: import("@sinclair/typebox").TObject<{
|
|
455
|
+
log: import("@sinclair/typebox").TString;
|
|
456
|
+
bytes: import("@sinclair/typebox").TNumber;
|
|
457
|
+
truncated: import("@sinclair/typebox").TBoolean;
|
|
458
|
+
tailed: import("@sinclair/typebox").TBoolean;
|
|
459
|
+
}>;
|
|
460
|
+
export type CodeEnvLogResult = Static<typeof CodeEnvLogResultSchema>;
|
|
461
|
+
export declare const CodeEnvGetLogOptionsSchema: import("@sinclair/typebox").TObject<{
|
|
462
|
+
maxBytes: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
463
|
+
maxLines: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
464
|
+
}>;
|
|
465
|
+
export type CodeEnvGetLogOptions = Static<typeof CodeEnvGetLogOptionsSchema>;
|
|
466
|
+
export declare const CodeEnvUpdateImagesOptionsSchema: import("@sinclair/typebox").TObject<{
|
|
467
|
+
envVersion: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
468
|
+
wait: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
469
|
+
}>;
|
|
470
|
+
export type CodeEnvUpdateImagesOptions = Static<typeof CodeEnvUpdateImagesOptionsSchema>;
|
|
471
|
+
export declare const CodeEnvVersionForProjectSchema: import("@sinclair/typebox").TObject<{
|
|
472
|
+
version: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
473
|
+
bundleId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
474
|
+
}>;
|
|
475
|
+
export type CodeEnvVersionForProject = Static<typeof CodeEnvVersionForProjectSchema>;
|
|
445
476
|
export declare const CodeEnvActionResultSchema: import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TUnknown>;
|
|
446
477
|
export type CodeEnvActionResult = Static<typeof CodeEnvActionResultSchema>;
|
|
447
478
|
export declare const CodeEnvCreateOptionsSchema: import("@sinclair/typebox").TObject<{
|
|
@@ -457,6 +488,7 @@ export type CodeEnvPackageList = Static<typeof CodeEnvPackageListSchema>;
|
|
|
457
488
|
export declare const CodeEnvSetPackagesOptionsSchema: import("@sinclair/typebox").TObject<{
|
|
458
489
|
packages: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TString, import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>]>;
|
|
459
490
|
installCorePackages: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
491
|
+
expectHash: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
460
492
|
}>;
|
|
461
493
|
export type CodeEnvSetPackagesOptions = Static<typeof CodeEnvSetPackagesOptionsSchema>;
|
|
462
494
|
export declare const CodeEnvUpdatePackagesOptionsSchema: import("@sinclair/typebox").TObject<{
|
|
@@ -583,7 +615,7 @@ export declare const DataQualityComputeResultSchema: import("@sinclair/typebox")
|
|
|
583
615
|
export type DataQualityComputeResult = Static<typeof DataQualityComputeResultSchema>;
|
|
584
616
|
export declare const JupyterCellSchema: import("@sinclair/typebox").TObject<{
|
|
585
617
|
cell_type: import("@sinclair/typebox").TString;
|
|
586
|
-
source: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
618
|
+
source: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TString, import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>]>;
|
|
587
619
|
metadata: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TUnknown>>;
|
|
588
620
|
outputs: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnknown>>;
|
|
589
621
|
execution_count: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TNull, import("@sinclair/typebox").TNumber]>>;
|
|
@@ -606,7 +638,7 @@ export declare const JupyterNotebookContentSchema: import("@sinclair/typebox").T
|
|
|
606
638
|
nbformat_minor: import("@sinclair/typebox").TNumber;
|
|
607
639
|
cells: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
608
640
|
cell_type: import("@sinclair/typebox").TString;
|
|
609
|
-
source: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>;
|
|
641
|
+
source: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TString, import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>]>;
|
|
610
642
|
metadata: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TUnknown>>;
|
|
611
643
|
outputs: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnknown>>;
|
|
612
644
|
execution_count: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TNull, import("@sinclair/typebox").TNumber]>>;
|
|
@@ -646,6 +678,9 @@ export declare const SqlNotebookContentSchema: import("@sinclair/typebox").TObje
|
|
|
646
678
|
}>>;
|
|
647
679
|
}>;
|
|
648
680
|
export type SqlNotebookContent = Static<typeof SqlNotebookContentSchema>;
|
|
681
|
+
/** Execution history of a SQL notebook: cell ID to the runs recorded for it. */
|
|
682
|
+
export declare const SqlNotebookHistorySchema: import("@sinclair/typebox").TRecord<import("@sinclair/typebox").TString, import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnknown>>;
|
|
683
|
+
export type SqlNotebookHistory = Static<typeof SqlNotebookHistorySchema>;
|
|
649
684
|
export declare const SqlQuerySchemaSchema: import("@sinclair/typebox").TObject<{
|
|
650
685
|
name: import("@sinclair/typebox").TString;
|
|
651
686
|
type: import("@sinclair/typebox").TString;
|
|
@@ -677,6 +712,7 @@ export declare const FlowMapOptionsSchema: import("@sinclair/typebox").TObject<{
|
|
|
677
712
|
maxNodes: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
678
713
|
maxEdges: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
679
714
|
includeRaw: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
715
|
+
render: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"ascii">, import("@sinclair/typebox").TLiteral<"mermaid">]>>;
|
|
680
716
|
}>;
|
|
681
717
|
export type FlowMapOptions = Static<typeof FlowMapOptionsSchema>;
|
|
682
718
|
/**
|
|
@@ -853,7 +889,6 @@ export declare const FolderItemArraySchema: import("@sinclair/typebox").TArray<i
|
|
|
853
889
|
export declare const CodeEnvSummaryArraySchema: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
854
890
|
envName: import("@sinclair/typebox").TString;
|
|
855
891
|
envLang: import("@sinclair/typebox").TString;
|
|
856
|
-
pythonInterpreter: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
857
892
|
deploymentMode: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
858
893
|
}>>;
|
|
859
894
|
export declare const WikiArticleDataArraySchema: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
@@ -444,16 +444,39 @@ export const ConnectionSummarySchema = Type.Object({
|
|
|
444
444
|
export const CodeEnvSummarySchema = Type.Object({
|
|
445
445
|
envName: Type.String(),
|
|
446
446
|
envLang: Type.String(),
|
|
447
|
-
pythonInterpreter: Type.Optional(Type.String()),
|
|
448
447
|
deploymentMode: Type.Optional(Type.String()),
|
|
449
448
|
}, { additionalProperties: true, });
|
|
450
449
|
export const CodeEnvDetailsSchema = Type.Object({
|
|
451
450
|
envName: Type.String(),
|
|
452
451
|
envLang: Type.String(),
|
|
453
452
|
pythonInterpreter: Type.Optional(Type.String()),
|
|
453
|
+
deploymentMode: Type.Optional(Type.String()),
|
|
454
454
|
requestedPackages: Type.Array(Type.String()),
|
|
455
455
|
installedPackages: Type.Array(Type.String()),
|
|
456
|
+
definitionHash: Type.Optional(Type.String()),
|
|
456
457
|
});
|
|
458
|
+
export const CodeEnvLogSummarySchema = Type.Object({
|
|
459
|
+
name: Type.String(),
|
|
460
|
+
}, { additionalProperties: true, });
|
|
461
|
+
export const CodeEnvLogSummaryArraySchema = Type.Array(CodeEnvLogSummarySchema);
|
|
462
|
+
export const CodeEnvLogResultSchema = Type.Object({
|
|
463
|
+
log: Type.String(),
|
|
464
|
+
bytes: Type.Number(),
|
|
465
|
+
truncated: Type.Boolean(),
|
|
466
|
+
tailed: Type.Boolean(),
|
|
467
|
+
}, { additionalProperties: false, });
|
|
468
|
+
export const CodeEnvGetLogOptionsSchema = Type.Object({
|
|
469
|
+
maxBytes: Type.Optional(Type.Number()),
|
|
470
|
+
maxLines: Type.Optional(Type.Number()),
|
|
471
|
+
}, { additionalProperties: false, });
|
|
472
|
+
export const CodeEnvUpdateImagesOptionsSchema = Type.Object({
|
|
473
|
+
envVersion: Type.Optional(Type.String()),
|
|
474
|
+
wait: Type.Optional(Type.Boolean()),
|
|
475
|
+
}, { additionalProperties: false, });
|
|
476
|
+
export const CodeEnvVersionForProjectSchema = Type.Object({
|
|
477
|
+
version: Type.Optional(Type.String()),
|
|
478
|
+
bundleId: Type.Optional(Type.String()),
|
|
479
|
+
}, { additionalProperties: true, });
|
|
457
480
|
export const CodeEnvActionResultSchema = Type.Record(Type.String(), Type.Unknown());
|
|
458
481
|
export const CodeEnvCreateOptionsSchema = Type.Object({
|
|
459
482
|
envLang: Type.String(),
|
|
@@ -469,6 +492,7 @@ export const CodeEnvPackageListSchema = Type.Union([
|
|
|
469
492
|
export const CodeEnvSetPackagesOptionsSchema = Type.Object({
|
|
470
493
|
packages: CodeEnvPackageListSchema,
|
|
471
494
|
installCorePackages: Type.Optional(Type.Boolean()),
|
|
495
|
+
expectHash: Type.Optional(Type.String()),
|
|
472
496
|
}, { additionalProperties: false, });
|
|
473
497
|
export const CodeEnvUpdatePackagesOptionsSchema = Type.Object({
|
|
474
498
|
forceRebuildEnv: Type.Optional(Type.Boolean()),
|
|
@@ -569,7 +593,9 @@ export const DataQualityComputeResultSchema = Type.Object({
|
|
|
569
593
|
// ---------------------------------------------------------------------------
|
|
570
594
|
export const JupyterCellSchema = Type.Object({
|
|
571
595
|
cell_type: Type.String(),
|
|
572
|
-
source
|
|
596
|
+
// nbformat permits a cell's source to be either a single string or an
|
|
597
|
+
// array of strings; DSS returns whichever form the notebook carries.
|
|
598
|
+
source: Type.Union([Type.String(), Type.Array(Type.String()),]),
|
|
573
599
|
metadata: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
|
|
574
600
|
outputs: Type.Optional(Type.Array(Type.Unknown())),
|
|
575
601
|
execution_count: Type.Optional(Type.Union([Type.Null(), Type.Number(),])),
|
|
@@ -617,6 +643,8 @@ export const SqlNotebookContentSchema = Type.Object({
|
|
|
617
643
|
connection: Type.String(),
|
|
618
644
|
cells: Type.Array(SqlNotebookCellSchema),
|
|
619
645
|
}, { additionalProperties: true, });
|
|
646
|
+
/** Execution history of a SQL notebook: cell ID to the runs recorded for it. */
|
|
647
|
+
export const SqlNotebookHistorySchema = Type.Record(Type.String(), Type.Array(Type.Unknown()));
|
|
620
648
|
// ---------------------------------------------------------------------------
|
|
621
649
|
// SQL Queries
|
|
622
650
|
// ---------------------------------------------------------------------------
|
|
@@ -642,6 +670,7 @@ export const FlowMapOptionsSchema = Type.Object({
|
|
|
642
670
|
maxNodes: Type.Optional(Type.Number()),
|
|
643
671
|
maxEdges: Type.Optional(Type.Number()),
|
|
644
672
|
includeRaw: Type.Optional(Type.Boolean()),
|
|
673
|
+
render: Type.Optional(Type.Union([Type.Literal("ascii"), Type.Literal("mermaid"),])),
|
|
645
674
|
});
|
|
646
675
|
// ---------------------------------------------------------------------------
|
|
647
676
|
// Project Git
|
package/plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "dataiku-sdk",
|
|
4
|
-
"version": "3.
|
|
4
|
+
"version": "3.2.0",
|
|
5
5
|
"description": "Dataiku DSS SDK and agent-oriented CLI for programmatic access to DSS REST APIs.",
|
|
6
6
|
"homepage": "https://github.com/clssck/dataiku-sdk",
|
|
7
7
|
"repository": "https://github.com/clssck/dataiku-sdk",
|