dataiku-sdk 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -10
- package/bin/dss.js +67 -78
- package/dist/build-metadata.json +1 -0
- package/dist/src/cli/commands/api-service.js +5 -4
- package/dist/src/cli/commands/app.js +45 -12
- package/dist/src/cli/commands/bundle.js +5 -4
- package/dist/src/cli/commands/data-quality.js +42 -0
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +187 -6
- package/dist/src/cli/commands/job.js +15 -2
- package/dist/src/cli/commands/project-library.js +4 -4
- package/dist/src/cli/commands/project.js +116 -12
- package/dist/src/cli/contract.d.ts +1 -0
- package/dist/src/cli/contract.js +194 -15
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +44 -13
- package/dist/src/cli/flags.js +2 -0
- package/dist/src/cli/helpers/app-successor.d.ts +15 -3
- package/dist/src/cli/helpers/app-successor.js +155 -15
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/output.d.ts +3 -1
- package/dist/src/cli/output.js +11 -6
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/version.d.ts +47 -4
- package/dist/src/cli/version.js +176 -9
- package/dist/src/cli.js +55 -14
- package/dist/src/client.d.ts +28 -1
- package/dist/src/client.js +34 -7
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.js +1 -0
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +199 -28
- package/dist/src/resources/folders.js +6 -8
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +122 -20
- package/dist/src/skill.js +1 -18
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +623 -0
- package/dist/src/utils/stable-hash.js +1 -2
- package/package.json +10 -10
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +27 -35
|
@@ -134,46 +134,52 @@ export function validateStreamColumns(headerRow, expectedColumns) {
|
|
|
134
134
|
}
|
|
135
135
|
return warnings;
|
|
136
136
|
}
|
|
137
|
-
function
|
|
138
|
-
return new DataikuError(0, "Request Timeout", `Dataset
|
|
137
|
+
function buildStreamTimeoutError(timeoutMs, action) {
|
|
138
|
+
return new DataikuError(0, "Request Timeout", `Dataset ${action} timed out after ${timeoutMs}ms while waiting for rows.`);
|
|
139
139
|
}
|
|
140
|
-
async function readChunkWithTimeout(reader, remainingMs, timeoutMs) {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
});
|
|
140
|
+
async function readChunkWithTimeout(reader, remainingMs, timeoutMs, action) {
|
|
141
|
+
const { promise, resolve, reject, } = Promise.withResolvers();
|
|
142
|
+
const timer = setTimeout(() => {
|
|
143
|
+
void reader.cancel(buildStreamTimeoutError(timeoutMs, action)).catch(() => { });
|
|
144
|
+
reject(buildStreamTimeoutError(timeoutMs, action));
|
|
145
|
+
}, remainingMs);
|
|
146
|
+
reader.read().then((result) => {
|
|
147
|
+
clearTimeout(timer);
|
|
148
|
+
resolve(result);
|
|
149
|
+
}, (error) => {
|
|
150
|
+
clearTimeout(timer);
|
|
151
|
+
reject(error);
|
|
153
152
|
});
|
|
153
|
+
return promise;
|
|
154
154
|
}
|
|
155
155
|
async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
156
156
|
const state = createTsvStreamState();
|
|
157
157
|
let columns;
|
|
158
158
|
const rows = [];
|
|
159
159
|
let done = false;
|
|
160
|
+
let truncated = false;
|
|
160
161
|
const startedAt = Date.now();
|
|
161
162
|
const reader = body.getReader();
|
|
163
|
+
let columnCount = 0;
|
|
162
164
|
const handleRow = (row) => {
|
|
163
|
-
if (done
|
|
165
|
+
if (done)
|
|
164
166
|
return;
|
|
165
167
|
if (columns === undefined) {
|
|
168
|
+
if (isBlankRow(row))
|
|
169
|
+
return;
|
|
166
170
|
columns = row;
|
|
171
|
+
columnCount = row.length;
|
|
167
172
|
onHeader?.(row);
|
|
168
173
|
return;
|
|
169
174
|
}
|
|
175
|
+
if (isBlankRow(row) && columnCount !== 1)
|
|
176
|
+
return;
|
|
170
177
|
if (rows.length >= maxDataRows) {
|
|
178
|
+
truncated = true;
|
|
171
179
|
done = true;
|
|
172
180
|
return;
|
|
173
181
|
}
|
|
174
182
|
rows.push(row);
|
|
175
|
-
if (rows.length >= maxDataRows)
|
|
176
|
-
done = true;
|
|
177
183
|
};
|
|
178
184
|
try {
|
|
179
185
|
while (true) {
|
|
@@ -183,15 +189,73 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
|
183
189
|
}
|
|
184
190
|
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
185
191
|
if (remainingMs <= 0)
|
|
186
|
-
throw
|
|
187
|
-
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs);
|
|
192
|
+
throw buildStreamTimeoutError(timeoutMs, "preview");
|
|
193
|
+
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "preview");
|
|
188
194
|
if (result.done)
|
|
189
195
|
break;
|
|
190
196
|
consumeTsvChunk(Buffer.from(result.value).toString("utf-8"), state, handleRow);
|
|
191
197
|
}
|
|
192
198
|
if (!done)
|
|
193
199
|
flushTsvStream(state, handleRow);
|
|
194
|
-
return { columns: columns ?? [], rows, };
|
|
200
|
+
return { columns: columns ?? [], rows, truncated, };
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
reader.releaseLock();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Count data rows in a streamed TSV response (header row excluded), stopping
|
|
208
|
+
* as soon as `stopAfter` rows have been seen so a mismatched dataset is never
|
|
209
|
+
* fully scanned. Returns the exact count when the stream ends before the cap,
|
|
210
|
+
* or `stopAfter` — a lower bound — once the cap is reached.
|
|
211
|
+
*/
|
|
212
|
+
async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
213
|
+
const state = createTsvStreamState();
|
|
214
|
+
let headerSeen = false;
|
|
215
|
+
let count = 0;
|
|
216
|
+
let bounded = false;
|
|
217
|
+
const startedAt = Date.now();
|
|
218
|
+
const reader = body.getReader();
|
|
219
|
+
let columnCount = 0;
|
|
220
|
+
const handleRow = (row) => {
|
|
221
|
+
if (bounded)
|
|
222
|
+
return;
|
|
223
|
+
if (!headerSeen) {
|
|
224
|
+
if (isBlankRow(row))
|
|
225
|
+
return;
|
|
226
|
+
headerSeen = true;
|
|
227
|
+
columnCount = row.length;
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
// An empty line is structurally invalid in a multi-column TSV (blank
|
|
231
|
+
// rows are tolerated and skipped), but for single-column datasets an
|
|
232
|
+
// empty line is a real data row carrying an empty string value and
|
|
233
|
+
// must be counted. The stream parser only emits rows for real line
|
|
234
|
+
// terminators, so the synthetic trailing newline never produces one.
|
|
235
|
+
if (isBlankRow(row) && columnCount !== 1)
|
|
236
|
+
return;
|
|
237
|
+
count += 1;
|
|
238
|
+
if (count >= stopAfter)
|
|
239
|
+
bounded = true;
|
|
240
|
+
};
|
|
241
|
+
try {
|
|
242
|
+
while (true) {
|
|
243
|
+
if (bounded) {
|
|
244
|
+
void reader.cancel().catch(() => { });
|
|
245
|
+
break;
|
|
246
|
+
}
|
|
247
|
+
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
248
|
+
if (remainingMs <= 0) {
|
|
249
|
+
throw buildStreamTimeoutError(timeoutMs, "row count assertion");
|
|
250
|
+
}
|
|
251
|
+
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "row count assertion");
|
|
252
|
+
if (result.done)
|
|
253
|
+
break;
|
|
254
|
+
consumeTsvChunk(Buffer.from(result.value).toString("utf-8"), state, handleRow);
|
|
255
|
+
}
|
|
256
|
+
if (!bounded)
|
|
257
|
+
flushTsvStream(state, handleRow);
|
|
258
|
+
return { count, bounded, };
|
|
195
259
|
}
|
|
196
260
|
finally {
|
|
197
261
|
reader.releaseLock();
|
|
@@ -202,15 +266,21 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader) {
|
|
|
202
266
|
const maxRows = Math.max(1, maxDataRows);
|
|
203
267
|
let headerSeen = false;
|
|
204
268
|
let done = false;
|
|
269
|
+
let columnCount = 0;
|
|
205
270
|
const handleRow = (row, push) => {
|
|
206
|
-
if (done
|
|
271
|
+
if (done)
|
|
207
272
|
return;
|
|
208
273
|
if (!headerSeen) {
|
|
274
|
+
if (isBlankRow(row))
|
|
275
|
+
return;
|
|
209
276
|
headerSeen = true;
|
|
277
|
+
columnCount = row.length;
|
|
210
278
|
onHeader?.(row);
|
|
211
279
|
push(`${rowToCsv(row)}\n`);
|
|
212
280
|
return;
|
|
213
281
|
}
|
|
282
|
+
if (isBlankRow(row) && columnCount !== 1)
|
|
283
|
+
return;
|
|
214
284
|
if (stats.rows >= maxRows) {
|
|
215
285
|
stats.truncated = true;
|
|
216
286
|
done = true;
|
|
@@ -337,6 +407,21 @@ function cloneDatasetParams(params) {
|
|
|
337
407
|
}
|
|
338
408
|
return cloned;
|
|
339
409
|
}
|
|
410
|
+
function parseUploadedFiles(raw, datasetName) {
|
|
411
|
+
if (!Array.isArray(raw)) {
|
|
412
|
+
throw new ClientValidationError(`DSS returned an invalid uploaded-files list for dataset ${datasetName}.`, "validation_failed", "Expected an array from the UploadedFiles dataset file endpoint.");
|
|
413
|
+
}
|
|
414
|
+
return raw.map((item, index) => {
|
|
415
|
+
if (!item
|
|
416
|
+
|| typeof item !== "object"
|
|
417
|
+
|| Array.isArray(item)
|
|
418
|
+
|| typeof item.filename !== "string"
|
|
419
|
+
|| item.filename === "") {
|
|
420
|
+
throw new ClientValidationError(`DSS returned invalid metadata for uploaded file ${String(index)} in dataset ${datasetName}.`, "validation_failed", "Expected every uploaded file to include a non-empty filename.");
|
|
421
|
+
}
|
|
422
|
+
return item;
|
|
423
|
+
});
|
|
424
|
+
}
|
|
340
425
|
export function buildDatasetCloneSettings(source, targetName, projectKey, opts) {
|
|
341
426
|
const params = {
|
|
342
427
|
...cloneDatasetParams(source.params),
|
|
@@ -395,6 +480,14 @@ export class DatasetsResource extends BaseResource {
|
|
|
395
480
|
const dsEnc = encodeURIComponent(datasetName);
|
|
396
481
|
await this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`, { columns, });
|
|
397
482
|
}
|
|
483
|
+
/**
|
|
484
|
+
* Get the raw dataset schema object exactly as the DSS schema endpoint
|
|
485
|
+
* returns it, without schema validation — used for deterministic comparison.
|
|
486
|
+
*/
|
|
487
|
+
async getSchemaObject(datasetName, projectKey) {
|
|
488
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
489
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`);
|
|
490
|
+
}
|
|
398
491
|
/**
|
|
399
492
|
* Preview dataset rows as structured data: column names plus row arrays,
|
|
400
493
|
* mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
|
|
@@ -407,7 +500,9 @@ export class DatasetsResource extends BaseResource {
|
|
|
407
500
|
const maxRows = Math.max(1, Math.min(opts?.maxRows ?? 50, 500));
|
|
408
501
|
const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
|
|
409
502
|
const dsEnc = encodeURIComponent(datasetName);
|
|
410
|
-
|
|
503
|
+
// Probe one row past the cap so callers learn whether the dataset holds
|
|
504
|
+
// more rows than requested without materializing the full dataset.
|
|
505
|
+
const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${maxRows + 1}`);
|
|
411
506
|
const onHeader = opts?.validateColumns
|
|
412
507
|
? (headerRow) => {
|
|
413
508
|
const warnings = validateStreamColumns(headerRow, opts.validateColumns);
|
|
@@ -416,10 +511,39 @@ export class DatasetsResource extends BaseResource {
|
|
|
416
511
|
}
|
|
417
512
|
}
|
|
418
513
|
: undefined;
|
|
419
|
-
if (!res.body)
|
|
420
|
-
return { columns: [], rows: [], rowCount: 0, };
|
|
421
|
-
|
|
422
|
-
|
|
514
|
+
if (!res.body) {
|
|
515
|
+
return { columns: [], rows: [], rowCount: 0, truncated: false, limit: maxRows, };
|
|
516
|
+
}
|
|
517
|
+
const { columns, rows, truncated, } = await collectPreviewRows(res.body, maxRows, timeoutMs, onHeader);
|
|
518
|
+
return {
|
|
519
|
+
columns: columns.map((name) => ({ name, })),
|
|
520
|
+
rows,
|
|
521
|
+
rowCount: rows.length,
|
|
522
|
+
truncated,
|
|
523
|
+
limit: maxRows,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Count dataset rows by streaming the TSV data endpoint and stopping as soon
|
|
528
|
+
* as `expected + 1` rows have been seen, so a mismatched dataset is never
|
|
529
|
+
* fully scanned. Returns the observed count together with `exact` (false when
|
|
530
|
+
* the probe cap was reached, i.e. the count is a lower bound) and whether the
|
|
531
|
+
* count satisfies the assertion.
|
|
532
|
+
*/
|
|
533
|
+
async assertRowCount(datasetName, expected, projectKey) {
|
|
534
|
+
if (!Number.isSafeInteger(expected) || expected < 0) {
|
|
535
|
+
throw new ClientValidationError("expected row count must be a non-negative safe integer", "validation_failed");
|
|
536
|
+
}
|
|
537
|
+
const target = expected;
|
|
538
|
+
const probe = target + 1;
|
|
539
|
+
const timeoutMs = Math.max(1, this.client.getRequestTimeoutMs());
|
|
540
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
541
|
+
const res = await this.client.stream(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${probe}`);
|
|
542
|
+
if (!res.body) {
|
|
543
|
+
return { expected: target, count: 0, exact: true, satisfied: target === 0, };
|
|
544
|
+
}
|
|
545
|
+
const { count, bounded, } = await collectTsvRowCount(res.body, probe, timeoutMs);
|
|
546
|
+
return { expected: target, count, exact: !bounded, satisfied: !bounded && count === target, };
|
|
423
547
|
}
|
|
424
548
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
425
549
|
async metadata(datasetName, projectKey) {
|
|
@@ -559,6 +683,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
559
683
|
? params.table
|
|
560
684
|
: null;
|
|
561
685
|
const normalizedType = (type ?? "").toLowerCase();
|
|
686
|
+
const uploadedFiles = normalizedType.includes("uploaded");
|
|
562
687
|
const fileBacked = !table
|
|
563
688
|
&& (normalizedType.includes("filesystem")
|
|
564
689
|
|| normalizedType.includes("uploaded")
|
|
@@ -566,7 +691,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
566
691
|
|| path !== null);
|
|
567
692
|
const formatType = details.formatType ?? null;
|
|
568
693
|
const warnings = [];
|
|
569
|
-
if (fileBacked && !path) {
|
|
694
|
+
if (fileBacked && !uploadedFiles && !path) {
|
|
570
695
|
warnings.push("File-backed dataset has no writable storage path configured.");
|
|
571
696
|
}
|
|
572
697
|
if (fileBacked && !formatType) {
|
|
@@ -585,6 +710,52 @@ export class DatasetsResource extends BaseResource {
|
|
|
585
710
|
note: "Validates build configuration only; it does not verify storage file existence or materialized data.",
|
|
586
711
|
};
|
|
587
712
|
}
|
|
713
|
+
/** List files stored by an UploadedFiles dataset. */
|
|
714
|
+
async listUploadedFiles(datasetName, projectKey) {
|
|
715
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/uploaded/files`);
|
|
716
|
+
return parseUploadedFiles(raw, datasetName);
|
|
717
|
+
}
|
|
718
|
+
/** Add a new file to an UploadedFiles dataset and verify its byte length. */
|
|
719
|
+
async uploadDatasetFile(datasetName, localPath, opts) {
|
|
720
|
+
const fileName = opts.fileName.trim();
|
|
721
|
+
if (fileName === "") {
|
|
722
|
+
throw new ClientValidationError("fileName must be a non-empty uploaded filename.");
|
|
723
|
+
}
|
|
724
|
+
const localFile = await fs.promises.stat(localPath);
|
|
725
|
+
if (!localFile.isFile()) {
|
|
726
|
+
throw new ClientValidationError(`${localPath} is not a regular file.`);
|
|
727
|
+
}
|
|
728
|
+
const projectKey = this.resolveProjectKey(opts.projectKey);
|
|
729
|
+
const details = await this.get(datasetName, projectKey);
|
|
730
|
+
if (details.type?.toLowerCase() !== "uploadedfiles") {
|
|
731
|
+
throw new ClientValidationError(`Dataset ${datasetName} is ${details.type ?? "an unknown type"}, not UploadedFiles.`, "validation_failed", "Use dataset upload-file only with an UploadedFiles dataset.");
|
|
732
|
+
}
|
|
733
|
+
const existing = (await this.listUploadedFiles(datasetName, projectKey))
|
|
734
|
+
.filter((file) => file.filename === fileName);
|
|
735
|
+
if (existing.length !== 0) {
|
|
736
|
+
throw new ClientValidationError(`Uploaded file ${fileName} already exists in dataset ${datasetName}.`, "validation_failed", "DSS 14.7's public API can add uploaded files but cannot replace or delete them. Use a new filename or import a successor project archive containing the replacement.");
|
|
737
|
+
}
|
|
738
|
+
const endpoint = `/public/api/projects/${encodeURIComponent(projectKey)}/datasets/${encodeURIComponent(datasetName)}/uploaded/files`;
|
|
739
|
+
await this.client.upload(endpoint, localPath, fileName);
|
|
740
|
+
const afterMatches = (await this.listUploadedFiles(datasetName, projectKey))
|
|
741
|
+
.filter((file) => file.filename === fileName);
|
|
742
|
+
const after = afterMatches[0];
|
|
743
|
+
if (afterMatches.length !== 1
|
|
744
|
+
|| after === undefined
|
|
745
|
+
|| after.length !== localFile.size) {
|
|
746
|
+
throw new ClientValidationError(`DSS did not verify upload of ${fileName} to dataset ${datasetName}.`, "ambiguous_outcome", "The upload returned successfully, but the uploaded-files listing did not show exactly one target with the local byte length. Inspect the dataset before retrying.", {
|
|
747
|
+
expectedBytes: localFile.size,
|
|
748
|
+
matchingFiles: afterMatches,
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
return {
|
|
752
|
+
datasetName,
|
|
753
|
+
projectKey,
|
|
754
|
+
fileName,
|
|
755
|
+
bytes: localFile.size,
|
|
756
|
+
after,
|
|
757
|
+
};
|
|
758
|
+
}
|
|
588
759
|
/** Clone dataset settings, preserving connection/storage, format, and schema fields. */
|
|
589
760
|
async clone(sourceName, targetName, opts = {}) {
|
|
590
761
|
const pk = this.resolveProjectKey(opts.projectKey);
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { resolve, } from "node:path";
|
|
3
|
-
import { Readable, } from "node:stream";
|
|
4
|
-
import { pipeline, } from "node:stream/promises";
|
|
1
|
+
import * as nodePath from "node:path";
|
|
5
2
|
import { FolderDetailsSchema, FolderItemArraySchema, FolderSummaryArraySchema, } from "../schemas.js";
|
|
6
3
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
7
4
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
@@ -72,10 +69,11 @@ export class FoldersResource extends BaseResource {
|
|
|
72
69
|
const normalizedPath = normalizeRemotePath(path);
|
|
73
70
|
const pEnc = encodeURIComponent(normalizedPath);
|
|
74
71
|
const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/managedfolders/${fEnc}/contents/${pEnc}`);
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const
|
|
78
|
-
|
|
72
|
+
if (!res.body)
|
|
73
|
+
throw new Error("folders.download response did not include a body");
|
|
74
|
+
const dest = opts?.localPath
|
|
75
|
+
?? nodePath.resolve(process.cwd(), inferDownloadFileName(normalizedPath));
|
|
76
|
+
await Bun.write(dest, new Response(res.body), { createPath: false, });
|
|
79
77
|
return dest;
|
|
80
78
|
}
|
|
81
79
|
upload(folderId, path, localPath, projectKey) {
|
|
@@ -64,10 +64,30 @@ export interface ProjectImportUploadResult {
|
|
|
64
64
|
id: string;
|
|
65
65
|
[key: string]: unknown;
|
|
66
66
|
}
|
|
67
|
+
export interface ProjectImportSettings {
|
|
68
|
+
targetProjectKey?: string;
|
|
69
|
+
remapping?: Record<string, unknown>;
|
|
70
|
+
[key: string]: unknown;
|
|
71
|
+
}
|
|
72
|
+
export interface ProjectImportProcessResult {
|
|
73
|
+
/** Present on definitive responses; a missing or non-boolean value makes the outcome indeterminate. */
|
|
74
|
+
success?: boolean;
|
|
75
|
+
/** Project key DSS actually created (after remapping). Definitive success includes it. */
|
|
76
|
+
usedProjectKey?: string;
|
|
77
|
+
[key: string]: unknown;
|
|
78
|
+
}
|
|
79
|
+
export interface ProjectImportResult extends ProjectImportProcessResult {
|
|
80
|
+
importId: string;
|
|
81
|
+
/** Key the import was bound to: the explicit settings target, else the archive manifest's key. Known on success. */
|
|
82
|
+
requestedProjectKey?: string;
|
|
83
|
+
/** True when usedProjectKey differs from requestedProjectKey (set only when both are known). */
|
|
84
|
+
remapped?: boolean;
|
|
85
|
+
/** SHA-256 fingerprint of the landed project's creationTag. Present on verified success. */
|
|
86
|
+
projectIncarnationHash?: string;
|
|
87
|
+
}
|
|
67
88
|
export declare class ProjectsResource extends BaseResource {
|
|
68
89
|
private httpInternals;
|
|
69
90
|
private postStream;
|
|
70
|
-
private uploadJson;
|
|
71
91
|
/** Create a new project. */
|
|
72
92
|
createProject(projectKey: string, name: string, ownerLogin: string, settings?: ProjectLifecycleSettings | null): Promise<Record<string, unknown>>;
|
|
73
93
|
/** Delete a project using the lifecycle endpoint. */
|
|
@@ -76,8 +96,21 @@ export declare class ProjectsResource extends BaseResource {
|
|
|
76
96
|
duplicate(projectKey: string, targetProjectKey: string, targetProjectName: string, options?: ProjectDuplicateOptions): Promise<Record<string, unknown>>;
|
|
77
97
|
/** Export a project archive to a local file. */
|
|
78
98
|
exportArchive(projectKey: string, filePath: string, options?: ProjectExportOptions): Promise<void>;
|
|
79
|
-
/** Upload a project archive and return
|
|
80
|
-
|
|
99
|
+
/** Upload a project archive and return its temporary import handle. */
|
|
100
|
+
prepareProjectImport(filePath: string): Promise<ProjectImportUploadResult>;
|
|
101
|
+
/** Process an uploaded archive into a DSS project. */
|
|
102
|
+
processProjectImport(importId: string, settings?: ProjectImportSettings): Promise<ProjectImportProcessResult>;
|
|
103
|
+
/**
|
|
104
|
+
* Upload and process a project archive.
|
|
105
|
+
*
|
|
106
|
+
* The archive is validated locally before anything is uploaded. Definitive
|
|
107
|
+
* failures surface the process response verbatim (with `importId`); every
|
|
108
|
+
* indeterminate path throws ClientValidationError(`ambiguous_outcome`).
|
|
109
|
+
* Verified success carries the actually-used project key, the requested
|
|
110
|
+
* key when known (with explicit `remapped` reporting), and the landed
|
|
111
|
+
* project's incarnation hash for identity-bound cleanup and guarded deletes.
|
|
112
|
+
*/
|
|
113
|
+
importProjectFromArchive(filePath: string, settings?: ProjectImportSettings): Promise<ProjectImportResult>;
|
|
81
114
|
/** Get project permissions. */
|
|
82
115
|
getPermissions(projectKey?: string): Promise<ProjectPermissions>;
|
|
83
116
|
/** Set project permissions. */
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { Writable, } from "node:stream";
|
|
1
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
3
2
|
import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, } from "../schemas.js";
|
|
4
3
|
import { normalizeFlowGraph, } from "../utils/flow-map.js";
|
|
4
|
+
import { assertImportableProjectArchive, } from "../utils/project-archive.js";
|
|
5
|
+
import { projectIncarnationHash, } from "../utils/project-incarnation.js";
|
|
5
6
|
import { BaseResource, } from "./base.js";
|
|
6
7
|
function fetchWithTimeout(label, timeoutMs, fetcher) {
|
|
7
8
|
return new Promise((resolve) => {
|
|
@@ -100,6 +101,7 @@ function truncateFlowMap(normalized, maxNodes, maxEdges) {
|
|
|
100
101
|
const DEFAULT_MAX_NODES = 300;
|
|
101
102
|
const DEFAULT_MAX_EDGES = 600;
|
|
102
103
|
const DEFAULT_METADATA_TIMEOUT_MS = 1_500;
|
|
104
|
+
const PROJECT_IMPORT_AMBIGUOUS_REMEDIATION = "Import processing left DSS state indeterminate. Inspect the archive with `dss project inspect-archive <file>` and the live projects with `dss project list` before retrying the import.";
|
|
103
105
|
export class ProjectsResource extends BaseResource {
|
|
104
106
|
httpInternals() {
|
|
105
107
|
return this.client;
|
|
@@ -116,18 +118,6 @@ export class ProjectsResource extends BaseResource {
|
|
|
116
118
|
body: JSON.stringify(body),
|
|
117
119
|
});
|
|
118
120
|
}
|
|
119
|
-
async uploadJson(path, filePath, fileName) {
|
|
120
|
-
const http = this.httpInternals();
|
|
121
|
-
const fileBlob = await openAsBlob(filePath);
|
|
122
|
-
const formData = new FormData();
|
|
123
|
-
formData.append("file", fileBlob, fileName);
|
|
124
|
-
const res = await http.fetchWithRetry(`${http.baseUrl}${path}`, {
|
|
125
|
-
method: "POST",
|
|
126
|
-
headers: { Authorization: `Bearer ${http.apiKey}`, },
|
|
127
|
-
body: formData,
|
|
128
|
-
});
|
|
129
|
-
return res.json();
|
|
130
|
-
}
|
|
131
121
|
/** Create a new project. */
|
|
132
122
|
async createProject(projectKey, name, ownerLogin, settings) {
|
|
133
123
|
return this.client.post("/public/api/projects/", {
|
|
@@ -167,18 +157,130 @@ export class ProjectsResource extends BaseResource {
|
|
|
167
157
|
const res = await this.postStream(`/public/api/projects/${encodeURIComponent(projectKey)}/export`, options ?? {});
|
|
168
158
|
if (!res.body)
|
|
169
159
|
throw new Error("projects.exportArchive response did not include a body");
|
|
170
|
-
|
|
160
|
+
await Bun.write(filePath, new Response(res.body), { createPath: false, });
|
|
161
|
+
}
|
|
162
|
+
/** Upload a project archive and return its temporary import handle. */
|
|
163
|
+
async prepareProjectImport(filePath) {
|
|
164
|
+
const upload = await this.client.uploadJson("/public/api/projects/import/upload", filePath, "tmp-import.zip");
|
|
165
|
+
if (!upload || typeof upload.id !== "string" || upload.id.trim() === "") {
|
|
166
|
+
throw new ClientValidationError("DSS accepted the project archive upload but did not return a temporary import id.", "ambiguous_outcome", "The archive may remain in temporary DSS storage, but no project import was started. Inspect DSS before retrying.");
|
|
167
|
+
}
|
|
168
|
+
return upload;
|
|
169
|
+
}
|
|
170
|
+
/** Process an uploaded archive into a DSS project. */
|
|
171
|
+
async processProjectImport(importId, settings = {}) {
|
|
172
|
+
const body = Object.keys(settings).length === 0 ? { _: "_", } : settings;
|
|
171
173
|
try {
|
|
172
|
-
await
|
|
174
|
+
const response = await this.client.post(`/public/api/projects/import/${encodeURIComponent(importId)}/process`, body);
|
|
175
|
+
// An empty or null 2xx body carries no definitive outcome: DSS may
|
|
176
|
+
// have created the project, partially processed it, or done nothing.
|
|
177
|
+
if (!response) {
|
|
178
|
+
throw new ClientValidationError("DSS accepted the import process request but returned no response body: the project may or may not have been created.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
179
|
+
importId,
|
|
180
|
+
...(typeof settings.targetProjectKey === "string"
|
|
181
|
+
&& settings.targetProjectKey.trim() !== ""
|
|
182
|
+
? { targetProjectKey: settings.targetProjectKey, }
|
|
183
|
+
: {}),
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
return response;
|
|
173
187
|
}
|
|
174
188
|
catch (error) {
|
|
175
|
-
|
|
189
|
+
// A transport timeout or server-side 5xx leaves the import outcome
|
|
190
|
+
// unknowable: the project may or may not exist. Surface that as an
|
|
191
|
+
// ambiguous outcome — never as a definitive failure. A 2xx
|
|
192
|
+
// response that carried a non-JSON body has the same property.
|
|
193
|
+
if (error instanceof DataikuError
|
|
194
|
+
&& (error.status === 0
|
|
195
|
+
|| error.status >= 500
|
|
196
|
+
|| (error.status >= 200 && error.status < 300))) {
|
|
197
|
+
throw new ClientValidationError(error.status >= 200 && error.status < 300
|
|
198
|
+
? "DSS returned a non-JSON process response: the import outcome is indeterminate and the project may or may not have been created."
|
|
199
|
+
: "DSS returned no definitive import result: the project may or may not have been created.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
200
|
+
importId,
|
|
201
|
+
...(typeof settings.targetProjectKey === "string"
|
|
202
|
+
&& settings.targetProjectKey.trim() !== ""
|
|
203
|
+
? { targetProjectKey: settings.targetProjectKey, }
|
|
204
|
+
: {}),
|
|
205
|
+
});
|
|
206
|
+
}
|
|
176
207
|
throw error;
|
|
177
208
|
}
|
|
178
209
|
}
|
|
179
|
-
/**
|
|
180
|
-
|
|
181
|
-
|
|
210
|
+
/**
|
|
211
|
+
* Upload and process a project archive.
|
|
212
|
+
*
|
|
213
|
+
* The archive is validated locally before anything is uploaded. Definitive
|
|
214
|
+
* failures surface the process response verbatim (with `importId`); every
|
|
215
|
+
* indeterminate path throws ClientValidationError(`ambiguous_outcome`).
|
|
216
|
+
* Verified success carries the actually-used project key, the requested
|
|
217
|
+
* key when known (with explicit `remapped` reporting), and the landed
|
|
218
|
+
* project's incarnation hash for identity-bound cleanup and guarded deletes.
|
|
219
|
+
*/
|
|
220
|
+
async importProjectFromArchive(filePath, settings = {}) {
|
|
221
|
+
// Fatal archive problems abort before any DSS call or upload.
|
|
222
|
+
const inspection = await assertImportableProjectArchive(filePath);
|
|
223
|
+
const requestedProjectKey = typeof settings.targetProjectKey === "string"
|
|
224
|
+
&& settings.targetProjectKey.trim() !== ""
|
|
225
|
+
? settings.targetProjectKey
|
|
226
|
+
: inspection.sourceProjectKey;
|
|
227
|
+
const upload = await this.prepareProjectImport(filePath);
|
|
228
|
+
const result = await this.processProjectImport(upload.id, settings);
|
|
229
|
+
const importId = upload.id;
|
|
230
|
+
if (typeof result.success !== "boolean") {
|
|
231
|
+
throw new ClientValidationError("DSS returned a process response without a boolean success field: the import outcome is indeterminate.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
232
|
+
importId,
|
|
233
|
+
...(requestedProjectKey !== undefined
|
|
234
|
+
? { targetProjectKey: requestedProjectKey, }
|
|
235
|
+
: {}),
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
if (result.success === false) {
|
|
239
|
+
// Definitive DSS-side failure: keep the response authoritative and
|
|
240
|
+
// let the command layer report it as a plain command failure.
|
|
241
|
+
return { ...result, success: false, importId, };
|
|
242
|
+
}
|
|
243
|
+
const usedProjectKey = typeof result.usedProjectKey === "string"
|
|
244
|
+
&& result.usedProjectKey.trim() !== ""
|
|
245
|
+
? result.usedProjectKey
|
|
246
|
+
: undefined;
|
|
247
|
+
if (usedProjectKey === undefined) {
|
|
248
|
+
throw new ClientValidationError("DSS reported import success but did not name the project that was created.", "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, {
|
|
249
|
+
importId,
|
|
250
|
+
...(requestedProjectKey !== undefined
|
|
251
|
+
? { targetProjectKey: requestedProjectKey, }
|
|
252
|
+
: {}),
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
// Verify the claimed landing and capture identity for cleanup binding.
|
|
256
|
+
let details;
|
|
257
|
+
try {
|
|
258
|
+
details = await this.get(usedProjectKey);
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
if (error instanceof DataikuError
|
|
262
|
+
&& (error.status === 404 || error.status === 0 || error.status >= 500)) {
|
|
263
|
+
throw new ClientValidationError(`DSS reported import success for project ${usedProjectKey}, but the project could not be re-read for verification.`, "ambiguous_outcome", PROJECT_IMPORT_AMBIGUOUS_REMEDIATION, { importId, usedProjectKey, });
|
|
264
|
+
}
|
|
265
|
+
throw error;
|
|
266
|
+
}
|
|
267
|
+
const incarnation = projectIncarnationHash(usedProjectKey, details);
|
|
268
|
+
if (incarnation === undefined) {
|
|
269
|
+
throw new ClientValidationError(`DSS imported project ${usedProjectKey}, but returned no creationTag identity.`, "ambiguous_outcome", "Refusing to claim identity-bound success without a project-incarnation hash. Verify the project identity before any cleanup or guarded delete.", { importId, usedProjectKey, });
|
|
270
|
+
}
|
|
271
|
+
return {
|
|
272
|
+
...result,
|
|
273
|
+
success: true,
|
|
274
|
+
importId,
|
|
275
|
+
usedProjectKey,
|
|
276
|
+
projectIncarnationHash: incarnation,
|
|
277
|
+
...(requestedProjectKey === undefined
|
|
278
|
+
? {}
|
|
279
|
+
: {
|
|
280
|
+
requestedProjectKey,
|
|
281
|
+
remapped: requestedProjectKey !== usedProjectKey,
|
|
282
|
+
}),
|
|
283
|
+
};
|
|
182
284
|
}
|
|
183
285
|
/** Get project permissions. */
|
|
184
286
|
async getPermissions(projectKey) {
|
package/dist/src/skill.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { execFileSync, } from "node:child_process";
|
|
2
1
|
import * as fs from "node:fs";
|
|
3
2
|
import * as os from "node:os";
|
|
4
3
|
import * as path from "node:path";
|
|
@@ -61,24 +60,11 @@ export const AGENTS = {
|
|
|
61
60
|
content: skillContent,
|
|
62
61
|
},
|
|
63
62
|
};
|
|
64
|
-
// ---------------------------------------------------------------------------
|
|
65
|
-
// Agent detection
|
|
66
|
-
// ---------------------------------------------------------------------------
|
|
67
|
-
function binaryExists(name) {
|
|
68
|
-
const cmd = process.platform === "win32" ? "where" : "which";
|
|
69
|
-
try {
|
|
70
|
-
execFileSync(cmd, [name,], { stdio: "pipe", });
|
|
71
|
-
return true;
|
|
72
|
-
}
|
|
73
|
-
catch {
|
|
74
|
-
return false;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
63
|
export function detectAgents() {
|
|
78
64
|
const home = os.homedir();
|
|
79
65
|
const found = [];
|
|
80
66
|
for (const [id, def,] of Object.entries(AGENTS)) {
|
|
81
|
-
const hasBinary =
|
|
67
|
+
const hasBinary = Bun.which(def.binary) !== null;
|
|
82
68
|
const hasConfigDir = fs.existsSync(path.join(home, def.configDir));
|
|
83
69
|
if (hasBinary && (!def.configDirRequired || hasConfigDir)) {
|
|
84
70
|
found.push({ id, def, via: "binary", });
|
|
@@ -90,9 +76,6 @@ export function detectAgents() {
|
|
|
90
76
|
return found;
|
|
91
77
|
}
|
|
92
78
|
// ---------------------------------------------------------------------------
|
|
93
|
-
// Skill installation
|
|
94
|
-
// ---------------------------------------------------------------------------
|
|
95
|
-
// ---------------------------------------------------------------------------
|
|
96
79
|
// Workspace root detection
|
|
97
80
|
// ---------------------------------------------------------------------------
|
|
98
81
|
const WORKSPACE_MARKERS = [".git",];
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export type ProjectArchiveIssueSeverity = "error" | "warning";
|
|
2
|
+
export interface ProjectArchiveIssue {
|
|
3
|
+
severity: ProjectArchiveIssueSeverity;
|
|
4
|
+
code: string;
|
|
5
|
+
message: string;
|
|
6
|
+
member?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ProjectArchiveInspection {
|
|
9
|
+
filePath: string;
|
|
10
|
+
sizeBytes: number;
|
|
11
|
+
memberCount: number;
|
|
12
|
+
sourceProjectKey?: string;
|
|
13
|
+
valid: boolean;
|
|
14
|
+
issues: ProjectArchiveIssue[];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Inspect a DSS project export ZIP archive without extracting anything to
|
|
18
|
+
* disk: enumerate every central-directory record, validate member paths
|
|
19
|
+
* independently, stream every file member through a size/CRC-32 check with
|
|
20
|
+
* bounded memory (manual `Bun.hash.crc32`, covering data-descriptor
|
|
21
|
+
* records), parse `export-manifest.json` and the source project key, and run
|
|
22
|
+
* the generic dangling-reference/orphan-member graph checks. Identity,
|
|
23
|
+
* history and machine-specific residue is reported only as warnings carrying
|
|
24
|
+
* member paths and pattern counts, never matched values.
|
|
25
|
+
*/
|
|
26
|
+
export declare function inspectProjectArchive(filePath: string): Promise<ProjectArchiveInspection>;
|
|
27
|
+
/**
|
|
28
|
+
* Like `inspectProjectArchive`, but throws a `ClientValidationError` with the
|
|
29
|
+
* `validation_failed` code when the inspection contains any error-severity
|
|
30
|
+
* issue, so the archive cannot be imported.
|
|
31
|
+
*/
|
|
32
|
+
export declare function assertImportableProjectArchive(filePath: string): Promise<ProjectArchiveInspection>;
|