dataiku-sdk 0.12.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -15
- package/bin/dss.js +21 -2
- package/dist/build-metadata.json +1 -0
- package/dist/packages/types/src/index.d.ts +128 -3
- package/dist/packages/types/src/index.js +106 -3
- package/dist/src/cli/coerce.d.ts +2 -1
- package/dist/src/cli/coerce.js +84 -26
- package/dist/src/cli/commands/app.js +6 -6
- package/dist/src/cli/commands/code.js +2 -2
- package/dist/src/cli/commands/data-quality.js +50 -8
- package/dist/src/cli/commands/dataset.d.ts +23 -0
- package/dist/src/cli/commands/dataset.js +197 -12
- package/dist/src/cli/commands/folder.js +5 -4
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/index.js +2 -0
- package/dist/src/cli/commands/job.js +26 -13
- package/dist/src/cli/commands/notebook.js +1 -1
- package/dist/src/cli/commands/project-git.d.ts +2 -0
- package/dist/src/cli/commands/project-git.js +563 -0
- package/dist/src/cli/commands/project.js +118 -14
- package/dist/src/cli/commands/recipe.js +14 -17
- package/dist/src/cli/commands/scenario.js +4 -4
- package/dist/src/cli/contract.d.ts +6 -4
- package/dist/src/cli/contract.js +595 -68
- package/dist/src/cli/doctor.d.ts +12 -2
- package/dist/src/cli/doctor.js +50 -19
- package/dist/src/cli/flags.js +23 -2
- package/dist/src/cli/helpers/cleanup.js +30 -1
- package/dist/src/cli/helpers/dataset.js +1 -1
- package/dist/src/cli/output.d.ts +5 -2
- package/dist/src/cli/output.js +13 -12
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +1 -1
- package/dist/src/cli/usage.js +2 -1
- package/dist/src/cli/version.d.ts +48 -6
- package/dist/src/cli/version.js +177 -11
- package/dist/src/cli.js +94 -39
- package/dist/src/client.d.ts +31 -1
- package/dist/src/client.js +39 -7
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/index.d.ts +6 -4
- package/dist/src/index.js +3 -1
- package/dist/src/resources/connections.d.ts +3 -1
- package/dist/src/resources/connections.js +82 -21
- package/dist/src/resources/datasets.d.ts +43 -0
- package/dist/src/resources/datasets.js +241 -30
- package/dist/src/resources/folders.js +7 -3
- package/dist/src/resources/project-git.d.ts +154 -0
- package/dist/src/resources/project-git.js +427 -0
- package/dist/src/resources/projects.d.ts +36 -3
- package/dist/src/resources/projects.js +127 -16
- package/dist/src/resources/recipes.js +4 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/project-archive.d.ts +32 -0
- package/dist/src/utils/project-archive.js +624 -0
- package/package.json +6 -5
- package/packages/types/dist/index.d.ts +128 -3
- package/packages/types/dist/index.js +106 -3
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +26 -19
|
@@ -3,11 +3,12 @@ import * as nodePath from "node:path";
|
|
|
3
3
|
import { Readable, Transform, } from "node:stream";
|
|
4
4
|
import { pipeline, } from "node:stream/promises";
|
|
5
5
|
import { createGzip, } from "node:zlib";
|
|
6
|
-
import { DataikuError, } from "../errors.js";
|
|
6
|
+
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
7
7
|
import { DatasetDetailsSchema, DatasetSchemaSchema, DatasetSummaryArraySchema, } from "../schemas.js";
|
|
8
8
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
9
9
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
10
10
|
import { BaseResource, } from "./base.js";
|
|
11
|
+
import { resolveAdminManagedStorageConnection, } from "./connections.js";
|
|
11
12
|
// ---------------------------------------------------------------------------
|
|
12
13
|
// Helpers: TSV → CSV streaming conversion
|
|
13
14
|
// ---------------------------------------------------------------------------
|
|
@@ -133,46 +134,52 @@ export function validateStreamColumns(headerRow, expectedColumns) {
|
|
|
133
134
|
}
|
|
134
135
|
return warnings;
|
|
135
136
|
}
|
|
136
|
-
function
|
|
137
|
-
return new DataikuError(0, "Request Timeout", `Dataset
|
|
137
|
+
function buildStreamTimeoutError(timeoutMs, action) {
|
|
138
|
+
return new DataikuError(0, "Request Timeout", `Dataset ${action} timed out after ${timeoutMs}ms while waiting for rows.`);
|
|
138
139
|
}
|
|
139
|
-
async function readChunkWithTimeout(reader, remainingMs, timeoutMs) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
});
|
|
140
|
+
async function readChunkWithTimeout(reader, remainingMs, timeoutMs, action) {
|
|
141
|
+
const { promise, resolve, reject, } = Promise.withResolvers();
|
|
142
|
+
const timer = setTimeout(() => {
|
|
143
|
+
void reader.cancel(buildStreamTimeoutError(timeoutMs, action)).catch(() => { });
|
|
144
|
+
reject(buildStreamTimeoutError(timeoutMs, action));
|
|
145
|
+
}, remainingMs);
|
|
146
|
+
reader.read().then((result) => {
|
|
147
|
+
clearTimeout(timer);
|
|
148
|
+
resolve(result);
|
|
149
|
+
}, (error) => {
|
|
150
|
+
clearTimeout(timer);
|
|
151
|
+
reject(error);
|
|
152
152
|
});
|
|
153
|
+
return promise;
|
|
153
154
|
}
|
|
154
155
|
async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
155
156
|
const state = createTsvStreamState();
|
|
156
157
|
let columns;
|
|
157
158
|
const rows = [];
|
|
158
159
|
let done = false;
|
|
160
|
+
let truncated = false;
|
|
159
161
|
const startedAt = Date.now();
|
|
160
162
|
const reader = body.getReader();
|
|
163
|
+
let columnCount = 0;
|
|
161
164
|
const handleRow = (row) => {
|
|
162
|
-
if (done
|
|
165
|
+
if (done)
|
|
163
166
|
return;
|
|
164
167
|
if (columns === undefined) {
|
|
168
|
+
if (isBlankRow(row))
|
|
169
|
+
return;
|
|
165
170
|
columns = row;
|
|
171
|
+
columnCount = row.length;
|
|
166
172
|
onHeader?.(row);
|
|
167
173
|
return;
|
|
168
174
|
}
|
|
175
|
+
if (isBlankRow(row) && columnCount !== 1)
|
|
176
|
+
return;
|
|
169
177
|
if (rows.length >= maxDataRows) {
|
|
178
|
+
truncated = true;
|
|
170
179
|
done = true;
|
|
171
180
|
return;
|
|
172
181
|
}
|
|
173
182
|
rows.push(row);
|
|
174
|
-
if (rows.length >= maxDataRows)
|
|
175
|
-
done = true;
|
|
176
183
|
};
|
|
177
184
|
try {
|
|
178
185
|
while (true) {
|
|
@@ -182,15 +189,73 @@ async function collectPreviewRows(body, maxDataRows, timeoutMs, onHeader) {
|
|
|
182
189
|
}
|
|
183
190
|
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
184
191
|
if (remainingMs <= 0)
|
|
185
|
-
throw
|
|
186
|
-
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs);
|
|
192
|
+
throw buildStreamTimeoutError(timeoutMs, "preview");
|
|
193
|
+
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "preview");
|
|
187
194
|
if (result.done)
|
|
188
195
|
break;
|
|
189
196
|
consumeTsvChunk(Buffer.from(result.value).toString("utf-8"), state, handleRow);
|
|
190
197
|
}
|
|
191
198
|
if (!done)
|
|
192
199
|
flushTsvStream(state, handleRow);
|
|
193
|
-
return { columns: columns ?? [], rows, };
|
|
200
|
+
return { columns: columns ?? [], rows, truncated, };
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
reader.releaseLock();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Count data rows in a streamed TSV response (header row excluded), stopping
|
|
208
|
+
* as soon as `stopAfter` rows have been seen so a mismatched dataset is never
|
|
209
|
+
* fully scanned. Returns the exact count when the stream ends before the cap,
|
|
210
|
+
* or `stopAfter` — a lower bound — once the cap is reached.
|
|
211
|
+
*/
|
|
212
|
+
async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
213
|
+
const state = createTsvStreamState();
|
|
214
|
+
let headerSeen = false;
|
|
215
|
+
let count = 0;
|
|
216
|
+
let bounded = false;
|
|
217
|
+
const startedAt = Date.now();
|
|
218
|
+
const reader = body.getReader();
|
|
219
|
+
let columnCount = 0;
|
|
220
|
+
const handleRow = (row) => {
|
|
221
|
+
if (bounded)
|
|
222
|
+
return;
|
|
223
|
+
if (!headerSeen) {
|
|
224
|
+
if (isBlankRow(row))
|
|
225
|
+
return;
|
|
226
|
+
headerSeen = true;
|
|
227
|
+
columnCount = row.length;
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
// An empty line is structurally invalid in a multi-column TSV (blank
|
|
231
|
+
// rows are tolerated and skipped), but for single-column datasets an
|
|
232
|
+
// empty line is a real data row carrying an empty string value and
|
|
233
|
+
// must be counted. The stream parser only emits rows for real line
|
|
234
|
+
// terminators, so the synthetic trailing newline never produces one.
|
|
235
|
+
if (isBlankRow(row) && columnCount !== 1)
|
|
236
|
+
return;
|
|
237
|
+
count += 1;
|
|
238
|
+
if (count >= stopAfter)
|
|
239
|
+
bounded = true;
|
|
240
|
+
};
|
|
241
|
+
try {
|
|
242
|
+
while (true) {
|
|
243
|
+
if (bounded) {
|
|
244
|
+
void reader.cancel().catch(() => { });
|
|
245
|
+
break;
|
|
246
|
+
}
|
|
247
|
+
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
248
|
+
if (remainingMs <= 0) {
|
|
249
|
+
throw buildStreamTimeoutError(timeoutMs, "row count assertion");
|
|
250
|
+
}
|
|
251
|
+
const result = await readChunkWithTimeout(reader, remainingMs, timeoutMs, "row count assertion");
|
|
252
|
+
if (result.done)
|
|
253
|
+
break;
|
|
254
|
+
consumeTsvChunk(Buffer.from(result.value).toString("utf-8"), state, handleRow);
|
|
255
|
+
}
|
|
256
|
+
if (!bounded)
|
|
257
|
+
flushTsvStream(state, handleRow);
|
|
258
|
+
return { count, bounded, };
|
|
194
259
|
}
|
|
195
260
|
finally {
|
|
196
261
|
reader.releaseLock();
|
|
@@ -201,15 +266,21 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader) {
|
|
|
201
266
|
const maxRows = Math.max(1, maxDataRows);
|
|
202
267
|
let headerSeen = false;
|
|
203
268
|
let done = false;
|
|
269
|
+
let columnCount = 0;
|
|
204
270
|
const handleRow = (row, push) => {
|
|
205
|
-
if (done
|
|
271
|
+
if (done)
|
|
206
272
|
return;
|
|
207
273
|
if (!headerSeen) {
|
|
274
|
+
if (isBlankRow(row))
|
|
275
|
+
return;
|
|
208
276
|
headerSeen = true;
|
|
277
|
+
columnCount = row.length;
|
|
209
278
|
onHeader?.(row);
|
|
210
279
|
push(`${rowToCsv(row)}\n`);
|
|
211
280
|
return;
|
|
212
281
|
}
|
|
282
|
+
if (isBlankRow(row) && columnCount !== 1)
|
|
283
|
+
return;
|
|
213
284
|
if (stats.rows >= maxRows) {
|
|
214
285
|
stats.truncated = true;
|
|
215
286
|
done = true;
|
|
@@ -253,15 +324,25 @@ function shouldRetryWithConnectionInferredType(error) {
|
|
|
253
324
|
|| detail.includes("invalid type")
|
|
254
325
|
|| detail.includes("illegal argument"));
|
|
255
326
|
}
|
|
327
|
+
function isMissingUploadedFilesTargetConnection(error) {
|
|
328
|
+
if (!(error instanceof DataikuError))
|
|
329
|
+
return false;
|
|
330
|
+
const detail = error.body.toLowerCase();
|
|
331
|
+
return (detail.includes("without a target connection")
|
|
332
|
+
|| detail.includes("target connection is required"));
|
|
333
|
+
}
|
|
256
334
|
function buildDatasetCreateBody(opts) {
|
|
257
335
|
if (opts.dsType.toLowerCase() === "uploadedfiles") {
|
|
258
336
|
return {
|
|
259
337
|
projectKey: opts.projectKey,
|
|
260
338
|
name: opts.datasetName,
|
|
261
339
|
type: opts.dsType,
|
|
262
|
-
params: { uploadConnection: opts.connection, },
|
|
340
|
+
params: opts.connection ? { uploadConnection: opts.connection, } : {},
|
|
263
341
|
};
|
|
264
342
|
}
|
|
343
|
+
if (!opts.connection) {
|
|
344
|
+
throw new ClientValidationError("connection is required unless dsType is UploadedFiles.");
|
|
345
|
+
}
|
|
265
346
|
if (opts.table) {
|
|
266
347
|
const params = {
|
|
267
348
|
connection: opts.connection,
|
|
@@ -326,6 +407,21 @@ function cloneDatasetParams(params) {
|
|
|
326
407
|
}
|
|
327
408
|
return cloned;
|
|
328
409
|
}
|
|
410
|
+
function parseUploadedFiles(raw, datasetName) {
|
|
411
|
+
if (!Array.isArray(raw)) {
|
|
412
|
+
throw new ClientValidationError(`DSS returned an invalid uploaded-files list for dataset ${datasetName}.`, "validation_failed", "Expected an array from the UploadedFiles dataset file endpoint.");
|
|
413
|
+
}
|
|
414
|
+
return raw.map((item, index) => {
|
|
415
|
+
if (!item
|
|
416
|
+
|| typeof item !== "object"
|
|
417
|
+
|| Array.isArray(item)
|
|
418
|
+
|| typeof item.filename !== "string"
|
|
419
|
+
|| item.filename === "") {
|
|
420
|
+
throw new ClientValidationError(`DSS returned invalid metadata for uploaded file ${String(index)} in dataset ${datasetName}.`, "validation_failed", "Expected every uploaded file to include a non-empty filename.");
|
|
421
|
+
}
|
|
422
|
+
return item;
|
|
423
|
+
});
|
|
424
|
+
}
|
|
329
425
|
export function buildDatasetCloneSettings(source, targetName, projectKey, opts) {
|
|
330
426
|
const params = {
|
|
331
427
|
...cloneDatasetParams(source.params),
|
|
@@ -384,6 +480,14 @@ export class DatasetsResource extends BaseResource {
|
|
|
384
480
|
const dsEnc = encodeURIComponent(datasetName);
|
|
385
481
|
await this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`, { columns, });
|
|
386
482
|
}
|
|
483
|
+
/**
|
|
484
|
+
* Get the raw dataset schema object exactly as the DSS schema endpoint
|
|
485
|
+
* returns it, without schema validation — used for deterministic comparison.
|
|
486
|
+
*/
|
|
487
|
+
async getSchemaObject(datasetName, projectKey) {
|
|
488
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
489
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`);
|
|
490
|
+
}
|
|
387
491
|
/**
|
|
388
492
|
* Preview dataset rows as structured data: column names plus row arrays,
|
|
389
493
|
* mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
|
|
@@ -396,7 +500,9 @@ export class DatasetsResource extends BaseResource {
|
|
|
396
500
|
const maxRows = Math.max(1, Math.min(opts?.maxRows ?? 50, 500));
|
|
397
501
|
const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
|
|
398
502
|
const dsEnc = encodeURIComponent(datasetName);
|
|
399
|
-
|
|
503
|
+
// Probe one row past the cap so callers learn whether the dataset holds
|
|
504
|
+
// more rows than requested without materializing the full dataset.
|
|
505
|
+
const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${maxRows + 1}`);
|
|
400
506
|
const onHeader = opts?.validateColumns
|
|
401
507
|
? (headerRow) => {
|
|
402
508
|
const warnings = validateStreamColumns(headerRow, opts.validateColumns);
|
|
@@ -405,10 +511,39 @@ export class DatasetsResource extends BaseResource {
|
|
|
405
511
|
}
|
|
406
512
|
}
|
|
407
513
|
: undefined;
|
|
408
|
-
if (!res.body)
|
|
409
|
-
return { columns: [], rows: [], rowCount: 0, };
|
|
410
|
-
|
|
411
|
-
|
|
514
|
+
if (!res.body) {
|
|
515
|
+
return { columns: [], rows: [], rowCount: 0, truncated: false, limit: maxRows, };
|
|
516
|
+
}
|
|
517
|
+
const { columns, rows, truncated, } = await collectPreviewRows(res.body, maxRows, timeoutMs, onHeader);
|
|
518
|
+
return {
|
|
519
|
+
columns: columns.map((name) => ({ name, })),
|
|
520
|
+
rows,
|
|
521
|
+
rowCount: rows.length,
|
|
522
|
+
truncated,
|
|
523
|
+
limit: maxRows,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Count dataset rows by streaming the TSV data endpoint and stopping as soon
|
|
528
|
+
* as `expected + 1` rows have been seen, so a mismatched dataset is never
|
|
529
|
+
* fully scanned. Returns the observed count together with `exact` (false when
|
|
530
|
+
* the probe cap was reached, i.e. the count is a lower bound) and whether the
|
|
531
|
+
* count satisfies the assertion.
|
|
532
|
+
*/
|
|
533
|
+
async assertRowCount(datasetName, expected, projectKey) {
|
|
534
|
+
if (!Number.isSafeInteger(expected) || expected < 0) {
|
|
535
|
+
throw new ClientValidationError("expected row count must be a non-negative safe integer", "validation_failed");
|
|
536
|
+
}
|
|
537
|
+
const target = expected;
|
|
538
|
+
const probe = target + 1;
|
|
539
|
+
const timeoutMs = Math.max(1, this.client.getRequestTimeoutMs());
|
|
540
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
541
|
+
const res = await this.client.stream(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${probe}`);
|
|
542
|
+
if (!res.body) {
|
|
543
|
+
return { expected: target, count: 0, exact: true, satisfied: target === 0, };
|
|
544
|
+
}
|
|
545
|
+
const { count, bounded, } = await collectTsvRowCount(res.body, probe, timeoutMs);
|
|
546
|
+
return { expected: target, count, exact: !bounded, satisfied: !bounded && count === target, };
|
|
412
547
|
}
|
|
413
548
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
414
549
|
async metadata(datasetName, projectKey) {
|
|
@@ -481,6 +616,35 @@ export class DatasetsResource extends BaseResource {
|
|
|
481
616
|
return await this.client.post(`/public/api/projects/${enc}/datasets/`, body);
|
|
482
617
|
}
|
|
483
618
|
catch (error) {
|
|
619
|
+
if (explicitType?.toLowerCase() === "uploadedfiles"
|
|
620
|
+
&& !opts.connection
|
|
621
|
+
&& isMissingUploadedFilesTargetConnection(error)) {
|
|
622
|
+
let uploadConnection = await resolveAdminManagedStorageConnection(this.client, "allowManagedDatasets", "filesystem_managed");
|
|
623
|
+
if (!uploadConnection) {
|
|
624
|
+
const existing = await this.list(pk);
|
|
625
|
+
uploadConnection = [
|
|
626
|
+
...new Set(existing
|
|
627
|
+
.filter((dataset) => dataset.managed === true)
|
|
628
|
+
.map((dataset) => dataset.params?.connection)
|
|
629
|
+
.filter((connection) => typeof connection === "string" && connection.length > 0)),
|
|
630
|
+
].sort()[0];
|
|
631
|
+
}
|
|
632
|
+
if (!uploadConnection)
|
|
633
|
+
throw error;
|
|
634
|
+
body = buildDatasetCreateBody({
|
|
635
|
+
projectKey: pk,
|
|
636
|
+
datasetName: opts.datasetName,
|
|
637
|
+
connection: uploadConnection,
|
|
638
|
+
dsType,
|
|
639
|
+
table: opts.table,
|
|
640
|
+
dbSchema: opts.dbSchema,
|
|
641
|
+
catalog: opts.catalog,
|
|
642
|
+
formatType: opts.formatType,
|
|
643
|
+
formatParams: opts.formatParams,
|
|
644
|
+
managed: opts.managed,
|
|
645
|
+
});
|
|
646
|
+
return this.client.post(`/public/api/projects/${enc}/datasets/`, body);
|
|
647
|
+
}
|
|
484
648
|
if (explicitType || !shouldRetryWithConnectionInferredType(error)) {
|
|
485
649
|
throw error;
|
|
486
650
|
}
|
|
@@ -519,6 +683,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
519
683
|
? params.table
|
|
520
684
|
: null;
|
|
521
685
|
const normalizedType = (type ?? "").toLowerCase();
|
|
686
|
+
const uploadedFiles = normalizedType.includes("uploaded");
|
|
522
687
|
const fileBacked = !table
|
|
523
688
|
&& (normalizedType.includes("filesystem")
|
|
524
689
|
|| normalizedType.includes("uploaded")
|
|
@@ -526,7 +691,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
526
691
|
|| path !== null);
|
|
527
692
|
const formatType = details.formatType ?? null;
|
|
528
693
|
const warnings = [];
|
|
529
|
-
if (fileBacked && !path) {
|
|
694
|
+
if (fileBacked && !uploadedFiles && !path) {
|
|
530
695
|
warnings.push("File-backed dataset has no writable storage path configured.");
|
|
531
696
|
}
|
|
532
697
|
if (fileBacked && !formatType) {
|
|
@@ -545,6 +710,52 @@ export class DatasetsResource extends BaseResource {
|
|
|
545
710
|
note: "Validates build configuration only; it does not verify storage file existence or materialized data.",
|
|
546
711
|
};
|
|
547
712
|
}
|
|
713
|
+
/** List files stored by an UploadedFiles dataset. */
|
|
714
|
+
async listUploadedFiles(datasetName, projectKey) {
|
|
715
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/uploaded/files`);
|
|
716
|
+
return parseUploadedFiles(raw, datasetName);
|
|
717
|
+
}
|
|
718
|
+
/** Add a new file to an UploadedFiles dataset and verify its byte length. */
|
|
719
|
+
async uploadDatasetFile(datasetName, localPath, opts) {
|
|
720
|
+
const fileName = opts.fileName.trim();
|
|
721
|
+
if (fileName === "") {
|
|
722
|
+
throw new ClientValidationError("fileName must be a non-empty uploaded filename.");
|
|
723
|
+
}
|
|
724
|
+
const localFile = await fs.promises.stat(localPath);
|
|
725
|
+
if (!localFile.isFile()) {
|
|
726
|
+
throw new ClientValidationError(`${localPath} is not a regular file.`);
|
|
727
|
+
}
|
|
728
|
+
const projectKey = this.resolveProjectKey(opts.projectKey);
|
|
729
|
+
const details = await this.get(datasetName, projectKey);
|
|
730
|
+
if (details.type?.toLowerCase() !== "uploadedfiles") {
|
|
731
|
+
throw new ClientValidationError(`Dataset ${datasetName} is ${details.type ?? "an unknown type"}, not UploadedFiles.`, "validation_failed", "Use dataset upload-file only with an UploadedFiles dataset.");
|
|
732
|
+
}
|
|
733
|
+
const existing = (await this.listUploadedFiles(datasetName, projectKey))
|
|
734
|
+
.filter((file) => file.filename === fileName);
|
|
735
|
+
if (existing.length !== 0) {
|
|
736
|
+
throw new ClientValidationError(`Uploaded file ${fileName} already exists in dataset ${datasetName}.`, "validation_failed", "DSS 14.7's public API can add uploaded files but cannot replace or delete them. Use a new filename or import a successor project archive containing the replacement.");
|
|
737
|
+
}
|
|
738
|
+
const endpoint = `/public/api/projects/${encodeURIComponent(projectKey)}/datasets/${encodeURIComponent(datasetName)}/uploaded/files`;
|
|
739
|
+
await this.client.upload(endpoint, localPath, fileName);
|
|
740
|
+
const afterMatches = (await this.listUploadedFiles(datasetName, projectKey))
|
|
741
|
+
.filter((file) => file.filename === fileName);
|
|
742
|
+
const after = afterMatches[0];
|
|
743
|
+
if (afterMatches.length !== 1
|
|
744
|
+
|| after === undefined
|
|
745
|
+
|| after.length !== localFile.size) {
|
|
746
|
+
throw new ClientValidationError(`DSS did not verify upload of ${fileName} to dataset ${datasetName}.`, "ambiguous_outcome", "The upload returned successfully, but the uploaded-files listing did not show exactly one target with the local byte length. Inspect the dataset before retrying.", {
|
|
747
|
+
expectedBytes: localFile.size,
|
|
748
|
+
matchingFiles: afterMatches,
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
return {
|
|
752
|
+
datasetName,
|
|
753
|
+
projectKey,
|
|
754
|
+
fileName,
|
|
755
|
+
bytes: localFile.size,
|
|
756
|
+
after,
|
|
757
|
+
};
|
|
758
|
+
}
|
|
548
759
|
/** Clone dataset settings, preserving connection/storage, format, and schema fields. */
|
|
549
760
|
async clone(sourceName, targetName, opts = {}) {
|
|
550
761
|
const pk = this.resolveProjectKey(opts.projectKey);
|
|
@@ -6,6 +6,7 @@ import { FolderDetailsSchema, FolderItemArraySchema, FolderSummaryArraySchema, }
|
|
|
6
6
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
7
7
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
8
8
|
import { BaseResource, } from "./base.js";
|
|
9
|
+
import { resolveAdminManagedStorageConnection, } from "./connections.js";
|
|
9
10
|
function normalizeRemotePath(path) {
|
|
10
11
|
return path.replace(/\\/g, "/");
|
|
11
12
|
}
|
|
@@ -20,14 +21,17 @@ function inferDownloadFileName(remotePath) {
|
|
|
20
21
|
export class FoldersResource extends BaseResource {
|
|
21
22
|
async create(opts) {
|
|
22
23
|
const pk = this.resolveProjectKey(opts.projectKey);
|
|
23
|
-
const path = opts.path?.trim() ||
|
|
24
|
+
const path = opts.path?.trim() || "/${projectKey}/${odbId}";
|
|
25
|
+
const connection = opts.connection
|
|
26
|
+
?? await resolveAdminManagedStorageConnection(this.client, "allowManagedFolders", "filesystem_folders")
|
|
27
|
+
?? "filesystem_folders";
|
|
24
28
|
const raw = await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/managedfolders/`, {
|
|
25
29
|
name: opts.name,
|
|
26
30
|
projectKey: pk,
|
|
27
|
-
type: opts.type,
|
|
31
|
+
type: opts.type ?? null,
|
|
28
32
|
params: {
|
|
29
33
|
...opts.params,
|
|
30
|
-
connection:
|
|
34
|
+
connection: connection,
|
|
31
35
|
path,
|
|
32
36
|
},
|
|
33
37
|
});
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import type { ProjectGitActionResult, ProjectGitDiffResult, ProjectGitFutureResponse, ProjectGitFutureState, ProjectGitLibraries, ProjectGitLogResult, ProjectGitRemote, ProjectGitStatus, ProjectGitTags } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export interface ProjectGitSetRemoteOptions {
|
|
4
|
+
url: string;
|
|
5
|
+
name?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface ProjectGitCreateBranchOptions {
|
|
8
|
+
commit?: string;
|
|
9
|
+
duplicateProject?: boolean;
|
|
10
|
+
targetProjectKey?: string;
|
|
11
|
+
targetProjectFolderId?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface ProjectGitDeleteBranchOptions {
|
|
14
|
+
remote?: boolean;
|
|
15
|
+
deleteRemotely?: boolean;
|
|
16
|
+
forceDelete?: boolean;
|
|
17
|
+
}
|
|
18
|
+
export interface ProjectGitCreateTagOptions {
|
|
19
|
+
reference?: string;
|
|
20
|
+
message?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ProjectGitPullOptions {
|
|
23
|
+
branchName?: string;
|
|
24
|
+
}
|
|
25
|
+
export interface ProjectGitPushOptions {
|
|
26
|
+
branchName?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface ProjectGitLogOptions {
|
|
29
|
+
path?: string;
|
|
30
|
+
startCommit?: string;
|
|
31
|
+
count?: number;
|
|
32
|
+
}
|
|
33
|
+
export interface ProjectGitDiffOptions {
|
|
34
|
+
commitFrom?: string;
|
|
35
|
+
commitTo?: string;
|
|
36
|
+
}
|
|
37
|
+
export interface ProjectGitCommitOptions {
|
|
38
|
+
message: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* `dropAndRebuild` permanently destroys the project's Git history, so the
|
|
42
|
+
* caller must acknowledge it explicitly before the request is issued.
|
|
43
|
+
*/
|
|
44
|
+
export interface ProjectGitDropAndRebuildOptions {
|
|
45
|
+
confirmed?: boolean;
|
|
46
|
+
}
|
|
47
|
+
export interface ProjectGitAddLibraryOptions {
|
|
48
|
+
repository: string;
|
|
49
|
+
localTargetPath: string;
|
|
50
|
+
checkout: string;
|
|
51
|
+
pathInGitRepository?: string;
|
|
52
|
+
addToPythonPath?: boolean;
|
|
53
|
+
login?: string;
|
|
54
|
+
password?: string;
|
|
55
|
+
}
|
|
56
|
+
export interface ProjectGitSetLibraryOptions {
|
|
57
|
+
repository: string;
|
|
58
|
+
pathInGitRepository?: string;
|
|
59
|
+
checkout: string;
|
|
60
|
+
login?: string;
|
|
61
|
+
password?: string;
|
|
62
|
+
}
|
|
63
|
+
export interface ProjectGitFutureStateOptions {
|
|
64
|
+
peek?: boolean;
|
|
65
|
+
}
|
|
66
|
+
export interface ProjectGitFutureWaitOptions {
|
|
67
|
+
pollIntervalMs?: number;
|
|
68
|
+
timeoutMs?: number;
|
|
69
|
+
}
|
|
70
|
+
export declare class ProjectGitResource extends BaseResource {
|
|
71
|
+
/** Working-copy status: current branch, remotes, tracking counts, changes. */
|
|
72
|
+
status(projectKey: string): Promise<ProjectGitStatus>;
|
|
73
|
+
/**
|
|
74
|
+
* Read a Git remote. DSS answers `{}` when the remote is not configured.
|
|
75
|
+
* The remote name defaults to `origin` like the Python client.
|
|
76
|
+
*/
|
|
77
|
+
getRemote(projectKey: string, name?: string): Promise<ProjectGitRemote>;
|
|
78
|
+
/** Create or replace a Git remote. */
|
|
79
|
+
setRemote(projectKey: string, options: ProjectGitSetRemoteOptions): Promise<ProjectGitActionResult>;
|
|
80
|
+
/** Delete a Git remote. */
|
|
81
|
+
removeRemote(projectKey: string, name?: string): Promise<void>;
|
|
82
|
+
/** Plain list of branch names, local by default. */
|
|
83
|
+
listBranches(projectKey: string, remote?: boolean): Promise<string[]>;
|
|
84
|
+
/**
|
|
85
|
+
* Create a branch and switch to it. With `duplicateProject`, DSS creates a
|
|
86
|
+
* duplicate project for the branch instead of moving this one.
|
|
87
|
+
*/
|
|
88
|
+
createBranch(projectKey: string, name: string, options?: ProjectGitCreateBranchOptions): Promise<ProjectGitActionResult>;
|
|
89
|
+
/** Delete a branch locally and, with `deleteRemotely`, on the remote too. */
|
|
90
|
+
deleteBranch(projectKey: string, name: string, options?: ProjectGitDeleteBranchOptions): Promise<ProjectGitActionResult>;
|
|
91
|
+
/** Current branch name, or null when DSS reports none. */
|
|
92
|
+
currentBranch(projectKey: string): Promise<string | null>;
|
|
93
|
+
/** Switch the project to another branch. */
|
|
94
|
+
switchBranch(projectKey: string, branchName: string): Promise<ProjectGitActionResult>;
|
|
95
|
+
listTags(projectKey: string): Promise<ProjectGitTags>;
|
|
96
|
+
createTag(projectKey: string, name: string, options?: ProjectGitCreateTagOptions): Promise<ProjectGitActionResult>;
|
|
97
|
+
deleteTag(projectKey: string, name: string): Promise<ProjectGitActionResult>;
|
|
98
|
+
/** Fetch from the remote. Failures arrive as `{success: false}` with HTTP 200. */
|
|
99
|
+
fetch(projectKey: string): Promise<ProjectGitActionResult>;
|
|
100
|
+
/** Pull. DSS only ever exposes a rebase pull; there is no merge variant. */
|
|
101
|
+
pull(projectKey: string, options?: ProjectGitPullOptions): Promise<ProjectGitActionResult>;
|
|
102
|
+
push(projectKey: string, options?: ProjectGitPushOptions): Promise<ProjectGitActionResult>;
|
|
103
|
+
/** Commit log page. Paginate by passing the previous `nextCommit` as `startCommit`. */
|
|
104
|
+
log(projectKey: string, options?: ProjectGitLogOptions): Promise<ProjectGitLogResult>;
|
|
105
|
+
diff(projectKey: string, options?: ProjectGitDiffOptions): Promise<ProjectGitDiffResult>;
|
|
106
|
+
/** Commit the project. DSS stages untracked files before committing. */
|
|
107
|
+
commit(projectKey: string, options: ProjectGitCommitOptions): Promise<ProjectGitActionResult>;
|
|
108
|
+
/** Restore the project to the state of a past commit. */
|
|
109
|
+
revertToRevision(projectKey: string, commit: string): Promise<ProjectGitActionResult>;
|
|
110
|
+
/** Revert one commit, keeping later history. */
|
|
111
|
+
revertCommit(projectKey: string, commit: string): Promise<ProjectGitActionResult>;
|
|
112
|
+
/** Hard reset to local HEAD, dropping uncommitted changes. */
|
|
113
|
+
resetToHead(projectKey: string): Promise<ProjectGitActionResult>;
|
|
114
|
+
/** Hard reset to the tracked upstream branch, dropping local commits. */
|
|
115
|
+
resetToUpstream(projectKey: string): Promise<ProjectGitActionResult>;
|
|
116
|
+
/**
|
|
117
|
+
* Wipe the project's entire Git history and rebuild a fresh repository.
|
|
118
|
+
* Irreversible, so nothing is sent unless the caller acknowledges it.
|
|
119
|
+
*/
|
|
120
|
+
dropAndRebuild(projectKey: string, options?: ProjectGitDropAndRebuildOptions): Promise<ProjectGitActionResult>;
|
|
121
|
+
listLibraries(projectKey: string): Promise<ProjectGitLibraries>;
|
|
122
|
+
/** Attach an external Git library. Runs as a DSS future. */
|
|
123
|
+
addLibrary(projectKey: string, options: ProjectGitAddLibraryOptions): Promise<ProjectGitFutureResponse>;
|
|
124
|
+
/** Update an attached library's repository, sub-path, or checkout. */
|
|
125
|
+
setLibrary(projectKey: string, gitReferencePath: string, options: ProjectGitSetLibraryOptions): Promise<string>;
|
|
126
|
+
/** Detach a library, optionally deleting its local directory. */
|
|
127
|
+
removeLibrary(projectKey: string, gitReferencePath: string, deleteDirectory?: boolean): Promise<void>;
|
|
128
|
+
/** Reset one library to its remote state. Runs as a DSS future. */
|
|
129
|
+
resetLibrary(projectKey: string, gitReferencePath: string): Promise<ProjectGitFutureResponse>;
|
|
130
|
+
/** Commit and push one library. Runs as a DSS future. */
|
|
131
|
+
pushLibrary(projectKey: string, gitReferencePath: string, commitMessage: string): Promise<ProjectGitFutureResponse>;
|
|
132
|
+
/** Commit and push every attached library. Runs as a DSS future. */
|
|
133
|
+
pushAllLibraries(projectKey: string, commitMessage: string): Promise<ProjectGitFutureResponse>;
|
|
134
|
+
/** Reset every attached library to its remote state. Runs as a DSS future. */
|
|
135
|
+
resetAllLibraries(projectKey: string): Promise<ProjectGitFutureResponse>;
|
|
136
|
+
/**
|
|
137
|
+
* Read a future's state. `peek` leaves the future's log unconsumed, which is
|
|
138
|
+
* what polling loops want; the default consumes it like the Python client.
|
|
139
|
+
*/
|
|
140
|
+
getFutureState(jobId: string, options?: ProjectGitFutureStateOptions): Promise<ProjectGitFutureState>;
|
|
141
|
+
/** Abort a running future. */
|
|
142
|
+
abortFuture(jobId: string): Promise<void>;
|
|
143
|
+
/**
|
|
144
|
+
* Poll until the future carries a result and return that result. Throws on
|
|
145
|
+
* an aborted or failed future and on an exhausted budget, so a caller never
|
|
146
|
+
* mistakes an unfinished future for a successful one.
|
|
147
|
+
*/
|
|
148
|
+
waitForFuture(jobId: string, options?: ProjectGitFutureWaitOptions): Promise<unknown>;
|
|
149
|
+
private gitPath;
|
|
150
|
+
private actionPath;
|
|
151
|
+
private libraryPath;
|
|
152
|
+
private query;
|
|
153
|
+
private request;
|
|
154
|
+
}
|