dataiku-sdk 3.6.1 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -6
- package/bin/bun-version.js +14 -0
- package/bin/dss.js +11 -15
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +114 -5
- package/dist/packages/types/src/index.js +75 -2
- package/dist/src/cli/coerce.d.ts +2 -0
- package/dist/src/cli/coerce.js +20 -16
- package/dist/src/cli/command-syntax.json +3369 -144
- package/dist/src/cli/commands/analysis.js +19 -1
- package/dist/src/cli/commands/bundle.js +22 -0
- package/dist/src/cli/commands/connection.js +26 -2
- package/dist/src/cli/commands/data-quality.js +33 -1
- package/dist/src/cli/commands/dataset.js +76 -9
- package/dist/src/cli/commands/discussion.js +15 -0
- package/dist/src/cli/commands/doctor.js +1 -1
- package/dist/src/cli/commands/flow-zone.js +241 -46
- package/dist/src/cli/commands/flow.d.ts +5 -0
- package/dist/src/cli/commands/flow.js +57 -0
- package/dist/src/cli/commands/future.js +8 -0
- package/dist/src/cli/commands/index.js +6 -0
- package/dist/src/cli/commands/job.js +44 -6
- package/dist/src/cli/commands/metrics.js +21 -1
- package/dist/src/cli/commands/ml-task.d.ts +5 -0
- package/dist/src/cli/commands/ml-task.js +86 -16
- package/dist/src/cli/commands/mlflow.d.ts +11 -0
- package/dist/src/cli/commands/mlflow.js +121 -0
- package/dist/src/cli/commands/plugin.js +5 -11
- package/dist/src/cli/commands/project-deployer.js +70 -0
- package/dist/src/cli/commands/project-library.js +2 -5
- package/dist/src/cli/commands/project.js +8 -0
- package/dist/src/cli/commands/recipe.d.ts +13 -0
- package/dist/src/cli/commands/recipe.js +130 -7
- package/dist/src/cli/commands/saved-model.d.ts +1 -1
- package/dist/src/cli/commands/saved-model.js +9 -76
- package/dist/src/cli/commands/streaming-endpoint.js +30 -1
- package/dist/src/cli/commands/webapp.js +8 -0
- package/dist/src/cli/commands/wiki.js +78 -0
- package/dist/src/cli/commands/workspace.js +8 -0
- package/dist/src/cli/contract.d.ts +16 -15
- package/dist/src/cli/contract.js +150 -3407
- package/dist/src/cli/doctor.d.ts +13 -0
- package/dist/src/cli/doctor.js +43 -1
- package/dist/src/cli/flags.d.ts +14 -1
- package/dist/src/cli/flags.js +106 -8
- package/dist/src/cli/helpers/app-successor.js +1 -2
- package/dist/src/cli/helpers/dataset.d.ts +12 -0
- package/dist/src/cli/helpers/dataset.js +36 -0
- package/dist/src/cli/helpers/documentation.d.ts +21 -0
- package/dist/src/cli/helpers/documentation.js +53 -0
- package/dist/src/cli/helpers/flow-zone.d.ts +21 -0
- package/dist/src/cli/helpers/flow-zone.js +144 -4
- package/dist/src/cli/helpers/trained-model.d.ts +26 -0
- package/dist/src/cli/helpers/trained-model.js +144 -0
- package/dist/src/cli/list-output.d.ts +7 -0
- package/dist/src/cli/list-output.js +77 -0
- package/dist/src/cli/list-shapes.d.ts +21 -0
- package/dist/src/cli/list-shapes.js +111 -0
- package/dist/src/cli/output-schemas.d.ts +0 -14
- package/dist/src/cli/output-schemas.js +4 -7
- package/dist/src/cli/output.d.ts +0 -1
- package/dist/src/cli/output.js +33 -19
- package/dist/src/cli/plans.d.ts +14 -0
- package/dist/src/cli/plans.js +3740 -0
- package/dist/src/cli/types.d.ts +2 -0
- package/dist/src/cli/usage.d.ts +7 -0
- package/dist/src/cli/usage.js +20 -1
- package/dist/src/cli/version.d.ts +2 -2
- package/dist/src/cli/version.js +2 -2
- package/dist/src/cli.js +137 -115
- package/dist/src/client.d.ts +52 -43
- package/dist/src/client.js +267 -96
- package/dist/src/errors.d.ts +22 -2
- package/dist/src/errors.js +49 -50
- package/dist/src/generated/action-output-schemas.json +345 -89
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +3 -0
- package/dist/src/resources/analyses.d.ts +8 -0
- package/dist/src/resources/analyses.js +13 -1
- package/dist/src/resources/bundles.d.ts +34 -0
- package/dist/src/resources/bundles.js +67 -0
- package/dist/src/resources/connections.d.ts +9 -0
- package/dist/src/resources/connections.js +15 -0
- package/dist/src/resources/data-quality.d.ts +20 -0
- package/dist/src/resources/data-quality.js +30 -0
- package/dist/src/resources/dataset-create.d.ts +38 -0
- package/dist/src/resources/dataset-create.js +135 -0
- package/dist/src/resources/datasets.d.ts +46 -15
- package/dist/src/resources/datasets.js +120 -112
- package/dist/src/resources/discussions.d.ts +5 -0
- package/dist/src/resources/discussions.js +7 -0
- package/dist/src/resources/documentation.d.ts +25 -0
- package/dist/src/resources/documentation.js +16 -0
- package/dist/src/resources/flow-zones.d.ts +29 -3
- package/dist/src/resources/flow-zones.js +53 -3
- package/dist/src/resources/flow.d.ts +48 -0
- package/dist/src/resources/flow.js +65 -0
- package/dist/src/resources/futures.d.ts +23 -0
- package/dist/src/resources/futures.js +32 -0
- package/dist/src/resources/jobs.d.ts +34 -0
- package/dist/src/resources/jobs.js +170 -3
- package/dist/src/resources/metrics.d.ts +10 -0
- package/dist/src/resources/metrics.js +10 -0
- package/dist/src/resources/ml-tasks.d.ts +65 -2
- package/dist/src/resources/ml-tasks.js +102 -28
- package/dist/src/resources/mlflow-extension.d.ts +73 -0
- package/dist/src/resources/mlflow-extension.js +105 -0
- package/dist/src/resources/projects.d.ts +15 -0
- package/dist/src/resources/projects.js +18 -3
- package/dist/src/resources/recipe-create.d.ts +79 -0
- package/dist/src/resources/recipe-create.js +454 -0
- package/dist/src/resources/recipes.d.ts +28 -29
- package/dist/src/resources/recipes.js +201 -267
- package/dist/src/resources/saved-models.d.ts +10 -6
- package/dist/src/resources/saved-models.js +16 -14
- package/dist/src/resources/scenarios.js +8 -14
- package/dist/src/resources/sql.js +52 -13
- package/dist/src/resources/streaming-endpoints.d.ts +12 -0
- package/dist/src/resources/streaming-endpoints.js +15 -0
- package/dist/src/resources/trained-model.d.ts +60 -0
- package/dist/src/resources/trained-model.js +98 -0
- package/dist/src/resources/webapps.d.ts +8 -0
- package/dist/src/resources/webapps.js +10 -0
- package/dist/src/resources/wiki.d.ts +52 -1
- package/dist/src/resources/wiki.js +96 -0
- package/dist/src/resources/workspaces.d.ts +2 -0
- package/dist/src/resources/workspaces.js +4 -0
- package/dist/src/utils/flow-analysis.js +4 -0
- package/dist/src/utils/response-file.js +20 -9
- package/dist/src/utils/secret-sanitize.d.ts +7 -0
- package/dist/src/utils/secret-sanitize.js +46 -0
- package/package.json +25 -26
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +1 -1
- package/skills/dataiku-dss/references/authentication.md +3 -1
- package/skills/dataiku-dss/references/coding.md +1 -0
- package/skills/dataiku-dss/references/discovery.md +4 -6
- package/skills/dataiku-dss/references/flow-maps.md +2 -2
- package/skills/dataiku-dss/references/troubleshooting.md +3 -2
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as nodePath from "node:path";
|
|
3
|
-
import { Readable, Transform, } from "node:stream";
|
|
4
|
-
import { pipeline, } from "node:stream/promises";
|
|
5
3
|
import { StringDecoder, } from "node:string_decoder";
|
|
6
|
-
import { createGzip, } from "node:zlib";
|
|
7
4
|
import { ClientValidationError, DataikuError, } from "../errors.js";
|
|
8
5
|
import { DatasetDetailsSchema, DatasetSchemaSchema, DatasetSummaryArraySchema, } from "../schemas.js";
|
|
9
6
|
import { deepMerge, } from "../utils/deep-merge.js";
|
|
7
|
+
import { isRecord, } from "../utils/records.js";
|
|
10
8
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
11
9
|
import { BaseResource, } from "./base.js";
|
|
12
10
|
import { resolveAdminManagedStorageConnection, } from "./connections.js";
|
|
11
|
+
import { buildDatasetCreateBody, } from "./dataset-create.js";
|
|
13
12
|
// ---------------------------------------------------------------------------
|
|
14
13
|
// Helpers: TSV → CSV streaming conversion
|
|
15
14
|
// ---------------------------------------------------------------------------
|
|
@@ -404,13 +403,14 @@ const CSV_BATCH_BYTES = 64 * 1024;
|
|
|
404
403
|
function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
|
|
405
404
|
const state = createTsvStreamState();
|
|
406
405
|
const decoder = new StringDecoder("utf-8");
|
|
406
|
+
const encoder = new TextEncoder();
|
|
407
407
|
const maxRows = Math.max(1, maxDataRows);
|
|
408
408
|
let headerSeen = false;
|
|
409
409
|
let done = false;
|
|
410
410
|
// Bounded CSV output batching: emitted rows accumulate here and are joined
|
|
411
|
-
// into a single
|
|
412
|
-
// bounded by the batch budget plus one escaped row (the join briefly
|
|
413
|
-
// second copy of the batch) while per-row
|
|
411
|
+
// into a single enqueue once the batch reaches CSV_BATCH_BYTES, so memory
|
|
412
|
+
// stays bounded by the batch budget plus one escaped row (the join briefly
|
|
413
|
+
// holds a second copy of the batch) while per-row chunk overhead disappears.
|
|
414
414
|
let batchLines = [];
|
|
415
415
|
let batchBytes = 0;
|
|
416
416
|
const flushBatch = (target) => {
|
|
@@ -419,7 +419,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
419
419
|
const csv = batchLines.join("");
|
|
420
420
|
batchLines = [];
|
|
421
421
|
batchBytes = 0;
|
|
422
|
-
target.
|
|
422
|
+
target.enqueue(encoder.encode(csv));
|
|
423
423
|
};
|
|
424
424
|
const queueLine = (target, line) => {
|
|
425
425
|
batchLines.push(line);
|
|
@@ -450,48 +450,27 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
450
450
|
push(`${rowToCsv(row, spreadsheetSafe)}\n`);
|
|
451
451
|
stats.rows += 1;
|
|
452
452
|
};
|
|
453
|
-
|
|
454
|
-
|
|
453
|
+
// A byte-limit overflow throws out of transform/flush: the stream errors,
|
|
454
|
+
// which cancels the response and makes download() remove the partial file.
|
|
455
|
+
return new TransformStream({
|
|
456
|
+
transform(chunk, controller) {
|
|
457
|
+
consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(controller, line)));
|
|
455
458
|
if (done) {
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
461
|
-
}
|
|
462
|
-
catch (error) {
|
|
463
|
-
// Byte-limit overflow: destroy the transform so the pipeline
|
|
464
|
-
// cancels the response and the partial file is removed.
|
|
465
|
-
callback(error instanceof Error ? error : new ClientValidationError(String(error), "internal_error"));
|
|
466
|
-
return;
|
|
459
|
+
// Truncation reached: emit the batch queued so far, end the output,
|
|
460
|
+
// and cancel the rest of the response.
|
|
461
|
+
flushBatch(controller);
|
|
462
|
+
controller.terminate();
|
|
467
463
|
}
|
|
468
|
-
if (done) {
|
|
469
|
-
// Truncation reached: emit the batch queued so far before EOF.
|
|
470
|
-
flushBatch(this);
|
|
471
|
-
this.push(null);
|
|
472
|
-
}
|
|
473
|
-
callback();
|
|
474
464
|
},
|
|
475
|
-
flush(
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
}
|
|
480
|
-
try {
|
|
481
|
-
const tail = decoder.end();
|
|
482
|
-
if (tail !== "") {
|
|
483
|
-
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
484
|
-
}
|
|
485
|
-
flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
486
|
-
}
|
|
487
|
-
catch (error) {
|
|
488
|
-
callback(error instanceof Error ? error : new ClientValidationError(String(error), "internal_error"));
|
|
489
|
-
return;
|
|
465
|
+
flush(controller) {
|
|
466
|
+
const tail = decoder.end();
|
|
467
|
+
if (tail !== "") {
|
|
468
|
+
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(controller, line)));
|
|
490
469
|
}
|
|
470
|
+
flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(controller, line)));
|
|
491
471
|
// Emit the trailing partial batch; without this the last rows of a
|
|
492
472
|
// download (below the batch budget) would be silently dropped.
|
|
493
|
-
flushBatch(
|
|
494
|
-
callback();
|
|
473
|
+
flushBatch(controller);
|
|
495
474
|
},
|
|
496
475
|
});
|
|
497
476
|
}
|
|
@@ -518,60 +497,6 @@ function isMissingUploadedFilesTargetConnection(error) {
|
|
|
518
497
|
return (detail.includes("without a target connection")
|
|
519
498
|
|| detail.includes("target connection is required"));
|
|
520
499
|
}
|
|
521
|
-
/** Wire body for POST /datasets/; shared with `dataset create --plan` so the plan matches the request. */
|
|
522
|
-
export function buildDatasetCreateBody(opts) {
|
|
523
|
-
if (opts.dsType.toLowerCase() === "uploadedfiles") {
|
|
524
|
-
return {
|
|
525
|
-
projectKey: opts.projectKey,
|
|
526
|
-
name: opts.datasetName,
|
|
527
|
-
type: opts.dsType,
|
|
528
|
-
params: opts.connection ? { uploadConnection: opts.connection, } : {},
|
|
529
|
-
};
|
|
530
|
-
}
|
|
531
|
-
if (!opts.connection) {
|
|
532
|
-
throw new ClientValidationError("connection is required unless dsType is UploadedFiles.");
|
|
533
|
-
}
|
|
534
|
-
if (opts.table) {
|
|
535
|
-
const params = {
|
|
536
|
-
connection: opts.connection,
|
|
537
|
-
mode: "table",
|
|
538
|
-
table: opts.table,
|
|
539
|
-
};
|
|
540
|
-
if (opts.dbSchema)
|
|
541
|
-
params.schema = opts.dbSchema;
|
|
542
|
-
if (opts.catalog)
|
|
543
|
-
params.catalog = opts.catalog;
|
|
544
|
-
return {
|
|
545
|
-
projectKey: opts.projectKey,
|
|
546
|
-
name: opts.datasetName,
|
|
547
|
-
type: opts.dsType,
|
|
548
|
-
params,
|
|
549
|
-
managed: opts.managed ?? false,
|
|
550
|
-
};
|
|
551
|
-
}
|
|
552
|
-
return {
|
|
553
|
-
projectKey: opts.projectKey,
|
|
554
|
-
name: opts.datasetName,
|
|
555
|
-
type: opts.dsType,
|
|
556
|
-
params: {
|
|
557
|
-
connection: opts.connection,
|
|
558
|
-
path: `/dataiku/${opts.projectKey}/${opts.datasetName}`,
|
|
559
|
-
},
|
|
560
|
-
formatType: opts.formatType ?? "csv",
|
|
561
|
-
formatParams: opts.formatParams ?? {
|
|
562
|
-
style: "excel",
|
|
563
|
-
charset: "utf8",
|
|
564
|
-
separator: "\t",
|
|
565
|
-
quoteChar: '"',
|
|
566
|
-
escapeChar: "\\",
|
|
567
|
-
dateSerializationFormat: "ISO",
|
|
568
|
-
arrayMapFormat: "json",
|
|
569
|
-
parseHeaderRow: true,
|
|
570
|
-
compress: "gz",
|
|
571
|
-
},
|
|
572
|
-
managed: opts.managed ?? true,
|
|
573
|
-
};
|
|
574
|
-
}
|
|
575
500
|
const DATASET_CLONE_PARAM_KEYS = [
|
|
576
501
|
"connection",
|
|
577
502
|
"uploadConnection",
|
|
@@ -595,6 +520,28 @@ function cloneDatasetParams(params) {
|
|
|
595
520
|
}
|
|
596
521
|
return cloned;
|
|
597
522
|
}
|
|
523
|
+
/** Explain an empty detection result from the first error DSS put in the response body. */
|
|
524
|
+
function describeSqlDetectionFailure(raw) {
|
|
525
|
+
if (!isRecord(raw))
|
|
526
|
+
return "DSS returned no schema detection result.";
|
|
527
|
+
for (const [key, value,] of Object.entries(raw)) {
|
|
528
|
+
if (!key.endsWith("Error"))
|
|
529
|
+
continue;
|
|
530
|
+
if (typeof value === "string" && value !== "")
|
|
531
|
+
return value;
|
|
532
|
+
if (isRecord(value)) {
|
|
533
|
+
const message = value["detailedMessage"] ?? value["message"];
|
|
534
|
+
if (typeof message === "string" && message !== "")
|
|
535
|
+
return message;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
const failedChecks = Object.entries(raw)
|
|
539
|
+
.filter(([key, value,]) => key.endsWith("OK") && value === false)
|
|
540
|
+
.map(([key,]) => key);
|
|
541
|
+
return failedChecks.length > 0
|
|
542
|
+
? `DSS detected no columns (failed checks: ${failedChecks.join(", ")}).`
|
|
543
|
+
: "DSS detected no columns.";
|
|
544
|
+
}
|
|
598
545
|
function parseUploadedFiles(raw, datasetName) {
|
|
599
546
|
if (!Array.isArray(raw)) {
|
|
600
547
|
throw new ClientValidationError(`DSS returned an invalid uploaded-files list for dataset ${datasetName}.`, "validation_failed", "Expected an array from the UploadedFiles dataset file endpoint.");
|
|
@@ -668,6 +615,40 @@ export class DatasetsResource extends BaseResource {
|
|
|
668
615
|
const dsEnc = encodeURIComponent(datasetName);
|
|
669
616
|
await this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`, { columns, });
|
|
670
617
|
}
|
|
618
|
+
/**
|
|
619
|
+
* Detect the schema of a SQL dataset (table or query mode) with DSS and
|
|
620
|
+
* store it: `POST /actions/testAndDetectSettings/externalSQL` (what
|
|
621
|
+
* dataikuapi's `autodetect_settings` calls), then `PUT /schema` with the
|
|
622
|
+
* suggested schema. DSS reports connection, table, and query problems in the
|
|
623
|
+
* response body (HTTP 200), so a failed detection is returned with
|
|
624
|
+
* `detected: false` and the reason instead of thrown.
|
|
625
|
+
*/
|
|
626
|
+
async detectSqlSchema(datasetName, projectKey) {
|
|
627
|
+
const base = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}`;
|
|
628
|
+
try {
|
|
629
|
+
const raw = await this.client.post(`${base}/actions/testAndDetectSettings/externalSQL`);
|
|
630
|
+
const detection = isRecord(raw) ? raw["schemaDetection"] : undefined;
|
|
631
|
+
const newSchema = isRecord(detection) ? detection["newSchema"] : undefined;
|
|
632
|
+
const rawColumns = isRecord(newSchema) && Array.isArray(newSchema["columns"])
|
|
633
|
+
? newSchema["columns"]
|
|
634
|
+
: [];
|
|
635
|
+
const columns = rawColumns.flatMap((column) => isRecord(column) && typeof column["name"] === "string"
|
|
636
|
+
&& typeof column["type"] === "string"
|
|
637
|
+
? [{ name: column["name"], type: column["type"], },]
|
|
638
|
+
: []);
|
|
639
|
+
if (!isRecord(newSchema) || columns.length === 0) {
|
|
640
|
+
return { detected: false, columns: [], error: describeSqlDetectionFailure(raw), };
|
|
641
|
+
}
|
|
642
|
+
await this.client.put(`${base}/schema`, newSchema);
|
|
643
|
+
return { detected: true, columns, };
|
|
644
|
+
}
|
|
645
|
+
catch (error) {
|
|
646
|
+
if (error instanceof DataikuError) {
|
|
647
|
+
return { detected: false, columns: [], error: error.message.split("\n")[0] ?? "", };
|
|
648
|
+
}
|
|
649
|
+
throw error;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
671
652
|
/**
|
|
672
653
|
* Get the raw dataset schema object exactly as the DSS schema endpoint
|
|
673
654
|
* returns it, without schema validation — used for deterministic comparison.
|
|
@@ -676,6 +657,23 @@ export class DatasetsResource extends BaseResource {
|
|
|
676
657
|
const dsEnc = encodeURIComponent(datasetName);
|
|
677
658
|
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/schema`);
|
|
678
659
|
}
|
|
660
|
+
/**
|
|
661
|
+
* TSV data stream of at most `limit` rows. A column or partition selection
|
|
662
|
+
* uses the documented POST variant of the data endpoint (body `columns`,
|
|
663
|
+
* `partitions`, `format`, `sampling`); otherwise the GET endpoint.
|
|
664
|
+
*/
|
|
665
|
+
async dataStream(datasetName, limit, projectKey, selection) {
|
|
666
|
+
const path = `/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/data/`;
|
|
667
|
+
if (!selection?.columns?.length && !selection?.partitions) {
|
|
668
|
+
return this.client.stream(`${path}?format=tsv-excel-header&limit=${limit}`);
|
|
669
|
+
}
|
|
670
|
+
return this.client.postStream(path, {
|
|
671
|
+
format: "tsv-excel-header",
|
|
672
|
+
...(selection.columns?.length ? { columns: selection.columns, } : {}),
|
|
673
|
+
...(selection.partitions ? { partitions: selection.partitions, } : {}),
|
|
674
|
+
sampling: { samplingMethod: "HEAD_SEQUENTIAL", maxRecords: limit, },
|
|
675
|
+
});
|
|
676
|
+
}
|
|
679
677
|
/**
|
|
680
678
|
* Preview dataset rows as structured data: column names plus row arrays,
|
|
681
679
|
* mirroring the sql query shape ({ columns, rows, rowCount }). Streams TSV
|
|
@@ -691,10 +689,9 @@ export class DatasetsResource extends BaseResource {
|
|
|
691
689
|
async preview(datasetName, opts) {
|
|
692
690
|
const maxRows = Math.max(1, Math.min(opts?.maxRows ?? 50, 500));
|
|
693
691
|
const timeoutMs = Math.max(1, opts?.timeoutMs ?? this.client.getRequestTimeoutMs());
|
|
694
|
-
const dsEnc = encodeURIComponent(datasetName);
|
|
695
692
|
// Probe one row past the cap so callers learn whether the dataset holds
|
|
696
693
|
// more rows than requested without materializing the full dataset.
|
|
697
|
-
const res = await this.
|
|
694
|
+
const res = await this.dataStream(datasetName, maxRows + 1, opts?.projectKey, opts);
|
|
698
695
|
const onHeader = opts?.validateColumns
|
|
699
696
|
? (headerRow) => {
|
|
700
697
|
const warnings = validateStreamColumns(headerRow, opts.validateColumns);
|
|
@@ -761,6 +758,20 @@ export class DatasetsResource extends BaseResource {
|
|
|
761
758
|
const dsEnc = encodeURIComponent(datasetName);
|
|
762
759
|
return this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/metadata`, metadata);
|
|
763
760
|
}
|
|
761
|
+
/**
|
|
762
|
+
* Create or update the Hive table of an HDFS dataset so it matches the
|
|
763
|
+
* dataset schema (POST /actions/synchronizeHiveMetastore).
|
|
764
|
+
*/
|
|
765
|
+
async synchronizeHiveMetastore(datasetName, projectKey) {
|
|
766
|
+
await this.client.post(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/actions/synchronizeHiveMetastore`);
|
|
767
|
+
}
|
|
768
|
+
/**
|
|
769
|
+
* Update an HDFS dataset (path, schema, partitioning, format) from its
|
|
770
|
+
* Hive table (POST /actions/updateFromHive).
|
|
771
|
+
*/
|
|
772
|
+
async updateFromHive(datasetName, projectKey) {
|
|
773
|
+
await this.client.post(`/public/api/projects/${this.enc(projectKey)}/datasets/${encodeURIComponent(datasetName)}/actions/updateFromHive`);
|
|
774
|
+
}
|
|
764
775
|
/**
|
|
765
776
|
* Create a managed dataset through the dedicated
|
|
766
777
|
* `POST /projects/{projectKey}/datasets/managed` endpoint. DSS handles the
|
|
@@ -862,8 +873,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
862
873
|
*/
|
|
863
874
|
async download(datasetName, opts) {
|
|
864
875
|
const limit = Math.max(1, opts?.limit ?? 100_000);
|
|
865
|
-
const
|
|
866
|
-
const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${limit + 1}`);
|
|
876
|
+
const res = await this.dataStream(datasetName, limit + 1, opts?.projectKey, opts);
|
|
867
877
|
const safeDatasetName = sanitizeFileName(datasetName, "dataset");
|
|
868
878
|
const filePath = opts?.outputPath?.endsWith(".gz") || opts?.outputPath?.endsWith(".csv")
|
|
869
879
|
? nodePath.resolve(opts.outputPath)
|
|
@@ -878,18 +888,13 @@ export class DatasetsResource extends BaseResource {
|
|
|
878
888
|
: undefined;
|
|
879
889
|
const shouldGzip = filePath.endsWith(".gz");
|
|
880
890
|
const stats = { rows: 0, truncated: false, };
|
|
881
|
-
|
|
882
|
-
|
|
891
|
+
let csv = res.body.pipeThrough(tsvToCsvTransform(limit, stats, onHeader, opts?.rawData !== true));
|
|
892
|
+
if (shouldGzip)
|
|
893
|
+
csv = csv.pipeThrough(new CompressionStream("gzip"));
|
|
883
894
|
fs.mkdirSync(nodePath.dirname(filePath), { recursive: true, });
|
|
884
|
-
const fileOut = fs.createWriteStream(filePath);
|
|
885
895
|
try {
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
await pipeline(nodeStream, csvTransform, gzip, fileOut);
|
|
889
|
-
}
|
|
890
|
-
else {
|
|
891
|
-
await pipeline(nodeStream, csvTransform, fileOut);
|
|
892
|
-
}
|
|
896
|
+
// Native streaming write with backpressure; no Node stream layer.
|
|
897
|
+
await Bun.write(filePath, new Response(csv));
|
|
893
898
|
}
|
|
894
899
|
catch (error) {
|
|
895
900
|
// A failed export (byte-limit overflow, network error) must never
|
|
@@ -911,7 +916,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
911
916
|
const enc = encodeURIComponent(pk);
|
|
912
917
|
const explicitType = opts.dsType;
|
|
913
918
|
let dsType = explicitType
|
|
914
|
-
?? (opts.table ? DEFAULT_DATABASE_DATASET_TYPE : DEFAULT_FILESYSTEM_DATASET_TYPE);
|
|
919
|
+
?? (opts.table || opts.query ? DEFAULT_DATABASE_DATASET_TYPE : DEFAULT_FILESYSTEM_DATASET_TYPE);
|
|
915
920
|
let body = buildDatasetCreateBody({
|
|
916
921
|
projectKey: pk,
|
|
917
922
|
datasetName: opts.datasetName,
|
|
@@ -920,6 +925,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
920
925
|
table: opts.table,
|
|
921
926
|
dbSchema: opts.dbSchema,
|
|
922
927
|
catalog: opts.catalog,
|
|
928
|
+
query: opts.query,
|
|
923
929
|
formatType: opts.formatType,
|
|
924
930
|
formatParams: opts.formatParams,
|
|
925
931
|
managed: opts.managed,
|
|
@@ -951,6 +957,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
951
957
|
table: opts.table,
|
|
952
958
|
dbSchema: opts.dbSchema,
|
|
953
959
|
catalog: opts.catalog,
|
|
960
|
+
query: opts.query,
|
|
954
961
|
formatType: opts.formatType,
|
|
955
962
|
formatParams: opts.formatParams,
|
|
956
963
|
managed: opts.managed,
|
|
@@ -975,6 +982,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
975
982
|
table: opts.table,
|
|
976
983
|
dbSchema: opts.dbSchema,
|
|
977
984
|
catalog: opts.catalog,
|
|
985
|
+
query: opts.query,
|
|
978
986
|
formatType: opts.formatType,
|
|
979
987
|
formatParams: opts.formatParams,
|
|
980
988
|
managed: opts.managed,
|
|
@@ -21,6 +21,11 @@ export declare class DiscussionsResource extends BaseResource {
|
|
|
21
21
|
create(objectType: string, objectId: string, topic: string, reply: string, projectKey?: string): Promise<Discussion>;
|
|
22
22
|
/** Add a reply to an existing discussion. */
|
|
23
23
|
reply(objectType: string, objectId: string, discussionId: string, text: string, projectKey?: string): Promise<Discussion>;
|
|
24
|
+
/**
|
|
25
|
+
* Update a discussion (topic, or the object it is attached to within the
|
|
26
|
+
* project): PUT the full discussion object obtained from `get`, edited.
|
|
27
|
+
*/
|
|
28
|
+
update(objectType: string, objectId: string, discussionId: string, discussion: Record<string, unknown>, projectKey?: string): Promise<Discussion>;
|
|
24
29
|
private objectDiscussionsPath;
|
|
25
30
|
private discussionPath;
|
|
26
31
|
}
|
|
@@ -16,6 +16,13 @@ export class DiscussionsResource extends BaseResource {
|
|
|
16
16
|
async reply(objectType, objectId, discussionId, text, projectKey) {
|
|
17
17
|
return this.client.post(`${this.discussionPath(objectType, objectId, discussionId, projectKey)}/replies/`, { reply: text, });
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Update a discussion (topic, or the object it is attached to within the
|
|
21
|
+
* project): PUT the full discussion object obtained from `get`, edited.
|
|
22
|
+
*/
|
|
23
|
+
async update(objectType, objectId, discussionId, discussion, projectKey) {
|
|
24
|
+
return this.client.put(this.discussionPath(objectType, objectId, discussionId, projectKey), discussion);
|
|
25
|
+
}
|
|
19
26
|
objectDiscussionsPath(objectType, objectId, projectKey) {
|
|
20
27
|
const pk = this.enc(projectKey);
|
|
21
28
|
const encodedType = encodeURIComponent(objectType);
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { DataikuClient } from "../client.js";
|
|
2
|
+
import { type DssTask } from "./futures.js";
|
|
3
|
+
/** Documentation template: the DSS default, an uploaded docx, or a docx in a managed folder. */
|
|
4
|
+
export type DocumentationTemplate = {
|
|
5
|
+
kind: "default";
|
|
6
|
+
} | {
|
|
7
|
+
kind: "file";
|
|
8
|
+
filePath: string;
|
|
9
|
+
} | {
|
|
10
|
+
kind: "folder";
|
|
11
|
+
folderId: string;
|
|
12
|
+
path: string;
|
|
13
|
+
};
|
|
14
|
+
/** The three generation endpoints of one documented object (flow, lab model, saved-model version). */
|
|
15
|
+
export interface DocumentationRoutes {
|
|
16
|
+
default: string;
|
|
17
|
+
file: string;
|
|
18
|
+
folder: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Start a DSS documentation export (docx): POST the default route, upload the
|
|
22
|
+
* template as multipart `file`, or POST the folder route with `folderId` and
|
|
23
|
+
* `path`. The finished task's result carries the `exportId` to download.
|
|
24
|
+
*/
|
|
25
|
+
export declare function startDocumentation(client: DataikuClient, routes: DocumentationRoutes, template: DocumentationTemplate, operation: string): Promise<DssTask>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { dssTask, } from "./futures.js";
|
|
2
|
+
/**
|
|
3
|
+
* Start a DSS documentation export (docx): POST the default route, upload the
|
|
4
|
+
* template as multipart `file`, or POST the folder route with `folderId` and
|
|
5
|
+
* `path`. The finished task's result carries the `exportId` to download.
|
|
6
|
+
*/
|
|
7
|
+
export async function startDocumentation(client, routes, template, operation) {
|
|
8
|
+
if (template.kind === "file") {
|
|
9
|
+
return dssTask(await client.uploadJson(routes.file, template.filePath), operation);
|
|
10
|
+
}
|
|
11
|
+
if (template.kind === "folder") {
|
|
12
|
+
const query = new URLSearchParams({ folderId: template.folderId, path: template.path, });
|
|
13
|
+
return dssTask(await client.post(`${routes.folder}?${query}`), operation);
|
|
14
|
+
}
|
|
15
|
+
return dssTask(await client.post(routes.default), operation);
|
|
16
|
+
}
|
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import type { DataikuGetOptions } from "../client.js";
|
|
2
|
-
import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZoneUpdateOptions } from "../schemas.js";
|
|
2
|
+
import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZonePosition, FlowZoneUpdateOptions } from "../schemas.js";
|
|
3
3
|
import { BaseResource } from "./base.js";
|
|
4
4
|
export type FlowZoneItemInput = {
|
|
5
5
|
objectId: string;
|
|
6
6
|
objectType: FlowZoneObjectType;
|
|
7
7
|
projectKey?: string;
|
|
8
8
|
};
|
|
9
|
+
export type FlowZoneRecreateOptions = {
|
|
10
|
+
position: FlowZonePosition;
|
|
11
|
+
name?: string;
|
|
12
|
+
color?: string;
|
|
13
|
+
projectKey?: string;
|
|
14
|
+
};
|
|
15
|
+
export type FlowZoneRecreateResult = {
|
|
16
|
+
zone: FlowZone;
|
|
17
|
+
replacedZoneId: string;
|
|
18
|
+
};
|
|
9
19
|
export declare class FlowZonesResource extends BaseResource {
|
|
10
20
|
/**
|
|
11
21
|
* List all flow zones in a project. Pass `timeoutMs` to bound the whole
|
|
@@ -14,10 +24,26 @@ export declare class FlowZonesResource extends BaseResource {
|
|
|
14
24
|
list(projectKey?: string, options?: DataikuGetOptions): Promise<FlowZone[]>;
|
|
15
25
|
/** Get one flow zone by id. */
|
|
16
26
|
get(zoneId: string, projectKey?: string): Promise<FlowZone>;
|
|
17
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* Create a flow zone. DSS stores `position` only while the project's manual
|
|
29
|
+
* zone positioning is off; with it on, DSS auto-places the zone.
|
|
30
|
+
*/
|
|
18
31
|
create(opts: FlowZoneCreateOptions): Promise<FlowZone>;
|
|
19
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Update flow zone name and color. DSS's public API ignores `position` on
|
|
34
|
+
* this endpoint; use {@link recreate} to move an existing zone.
|
|
35
|
+
*/
|
|
20
36
|
update(zoneId: string, opts: FlowZoneUpdateOptions): Promise<FlowZone>;
|
|
37
|
+
/**
|
|
38
|
+
* Replace a zone with a new one at `position`: DSS honors a zone position
|
|
39
|
+
* only on creation, and only while the project's manual zone positioning is
|
|
40
|
+
* off (otherwise it auto-places the new zone). The replacement receives the
|
|
41
|
+
* old zone's name, color, metadata (tags, custom fields, checklists, ...),
|
|
42
|
+
* items, and shared items; the old zone is deleted once it is empty. The zone
|
|
43
|
+
* id changes, so anything referencing the old id (for example scenario zone
|
|
44
|
+
* builds) must be updated.
|
|
45
|
+
*/
|
|
46
|
+
recreate(zoneId: string, opts: FlowZoneRecreateOptions): Promise<FlowZoneRecreateResult>;
|
|
21
47
|
/** Delete a flow zone. DSS moves its items back to the default zone. */
|
|
22
48
|
delete(zoneId: string, projectKey?: string): Promise<void>;
|
|
23
49
|
/** Move items into a flow zone. */
|
|
@@ -23,7 +23,10 @@ export class FlowZonesResource extends BaseResource {
|
|
|
23
23
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`);
|
|
24
24
|
return this.client.safeParse(FlowZoneSchema, raw, "flowZones.get");
|
|
25
25
|
}
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Create a flow zone. DSS stores `position` only while the project's manual
|
|
28
|
+
* zone positioning is off; with it on, DSS auto-places the zone.
|
|
29
|
+
*/
|
|
27
30
|
async create(opts) {
|
|
28
31
|
const raw = await this.client.post(`/public/api/projects/${this.enc(opts.projectKey)}/flow/zones`, {
|
|
29
32
|
name: opts.name,
|
|
@@ -32,17 +35,64 @@ export class FlowZonesResource extends BaseResource {
|
|
|
32
35
|
});
|
|
33
36
|
return this.client.safeParse(FlowZoneSchema, raw, "flowZones.create");
|
|
34
37
|
}
|
|
35
|
-
/**
|
|
38
|
+
/**
|
|
39
|
+
* Update flow zone name and color. DSS's public API ignores `position` on
|
|
40
|
+
* this endpoint; use {@link recreate} to move an existing zone.
|
|
41
|
+
*/
|
|
36
42
|
async update(zoneId, opts) {
|
|
37
43
|
const current = await this.get(zoneId, opts.projectKey);
|
|
38
44
|
const merged = deepMerge(current, {
|
|
39
45
|
...(opts.name !== undefined ? { name: opts.name, } : {}),
|
|
40
46
|
...(opts.color !== undefined ? { color: opts.color, } : {}),
|
|
41
|
-
...(opts.position !== undefined ? { position: opts.position, } : {}),
|
|
42
47
|
});
|
|
43
48
|
await this.client.putVoid(`/public/api/projects/${this.enc(opts.projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`, merged);
|
|
44
49
|
return this.get(zoneId, opts.projectKey);
|
|
45
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* Replace a zone with a new one at `position`: DSS honors a zone position
|
|
53
|
+
* only on creation, and only while the project's manual zone positioning is
|
|
54
|
+
* off (otherwise it auto-places the new zone). The replacement receives the
|
|
55
|
+
* old zone's name, color, metadata (tags, custom fields, checklists, ...),
|
|
56
|
+
* items, and shared items; the old zone is deleted once it is empty. The zone
|
|
57
|
+
* id changes, so anything referencing the old id (for example scenario zone
|
|
58
|
+
* builds) must be updated.
|
|
59
|
+
*/
|
|
60
|
+
async recreate(zoneId, opts) {
|
|
61
|
+
const base = `/public/api/projects/${this.enc(opts.projectKey)}/flow/zones`;
|
|
62
|
+
const old = await this.get(zoneId, opts.projectKey);
|
|
63
|
+
const created = await this.create({
|
|
64
|
+
name: opts.name ?? old.name,
|
|
65
|
+
color: opts.color ?? old.color,
|
|
66
|
+
position: opts.position,
|
|
67
|
+
projectKey: opts.projectKey,
|
|
68
|
+
});
|
|
69
|
+
const completed = ["create",];
|
|
70
|
+
try {
|
|
71
|
+
const { id: _id, items, shared, position: _position, ...metadata } = old;
|
|
72
|
+
await this.client.putVoid(`${base}/${encodeURIComponent(created.id)}`, {
|
|
73
|
+
...created,
|
|
74
|
+
...metadata,
|
|
75
|
+
id: created.id,
|
|
76
|
+
name: created.name,
|
|
77
|
+
color: created.color,
|
|
78
|
+
position: created.position,
|
|
79
|
+
});
|
|
80
|
+
completed.push("metadata");
|
|
81
|
+
if (items && items.length > 0) {
|
|
82
|
+
await this.moveItems(created.id, items, opts.projectKey);
|
|
83
|
+
}
|
|
84
|
+
completed.push("items");
|
|
85
|
+
for (const item of shared ?? []) {
|
|
86
|
+
await this.client.post(`${base}/${encodeURIComponent(created.id)}/shared`, normalizeZoneItem(item));
|
|
87
|
+
}
|
|
88
|
+
completed.push("shared");
|
|
89
|
+
await this.delete(old.id, opts.projectKey);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
throw new ClientValidationError(`Recreating flow zone ${old.id} at a new position stopped after ${completed.at(-1)}; both zones may now exist.`, "ambiguous_outcome", "Inspect both zones with dss flow-zone list, then move remaining items and delete the zone you no longer need.", { replacedZoneId: old.id, zoneId: created.id, completed, }, { cause: error, });
|
|
93
|
+
}
|
|
94
|
+
return { zone: await this.get(created.id, opts.projectKey), replacedZoneId: old.id, };
|
|
95
|
+
}
|
|
46
96
|
/** Delete a flow zone. DSS moves its items back to the default zone. */
|
|
47
97
|
async delete(zoneId, projectKey) {
|
|
48
98
|
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`);
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { FutureWaitResult } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
import { type DocumentationTemplate } from "./documentation.js";
|
|
4
|
+
import { type DssTask, type FutureWaitOptions } from "./futures.js";
|
|
5
|
+
/** Schema propagation options (dataikuapi `DSSSchemaPropagationRunBuilder` settings). */
|
|
6
|
+
export interface SchemaPropagationOptions {
|
|
7
|
+
projectKey?: string;
|
|
8
|
+
/** Rebuild datasets when propagation needs their data (DSS default: true). */
|
|
9
|
+
autoRebuild?: boolean;
|
|
10
|
+
/** Recipes where propagation stops (dataikuapi `stop_at`). */
|
|
11
|
+
excludedRecipes?: string[];
|
|
12
|
+
/** Recipes always considered OK (dataikuapi `mark_recipe_as_ok`). */
|
|
13
|
+
markAsOkRecipes?: string[];
|
|
14
|
+
/** Partitioned flows: default partition value per dimension when rebuilding. */
|
|
15
|
+
defaultPartitionValuesByDimension?: Record<string, string>;
|
|
16
|
+
/** Partitioned flows: partition per computable, keyed `PROJECTKEY.id`. */
|
|
17
|
+
partitionsByComputable?: Record<string, string>;
|
|
18
|
+
/** Per-recipe-type and per-recipe update options, as dataikuapi sends them. */
|
|
19
|
+
recipeUpdateOptions?: {
|
|
20
|
+
byType?: Record<string, unknown>;
|
|
21
|
+
byName?: Record<string, unknown>;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Flow-wide tools: schema propagation from a dataset through every downstream
|
|
26
|
+
* recipe, and flow documentation export. Zones and the flow graph live on
|
|
27
|
+
* `flowZones`; per-recipe schema updates on `recipes.updateSchema`.
|
|
28
|
+
*/
|
|
29
|
+
export declare class FlowResource extends BaseResource {
|
|
30
|
+
private flowPath;
|
|
31
|
+
/**
|
|
32
|
+
* Start propagating `datasetName`'s schema through the flow
|
|
33
|
+
* (POST /flow/tools/propagate-schema/, dataikuapi
|
|
34
|
+
* `new_schema_propagation(...).start()`). Returns the DSS future.
|
|
35
|
+
*/
|
|
36
|
+
propagateSchema(datasetName: string, opts?: SchemaPropagationOptions): Promise<DssTask>;
|
|
37
|
+
propagateSchemaAndWait(datasetName: string, opts?: SchemaPropagationOptions & FutureWaitOptions): Promise<FutureWaitResult | DssTask>;
|
|
38
|
+
/**
|
|
39
|
+
* Start generating the flow documentation (docx) from the DSS default
|
|
40
|
+
* template, an uploaded docx template, or a docx in a managed folder.
|
|
41
|
+
* The finished future's result carries the `exportId` to download.
|
|
42
|
+
*/
|
|
43
|
+
generateDocumentation(template?: DocumentationTemplate, projectKey?: string): Promise<DssTask>;
|
|
44
|
+
/** GET a generated flow documentation (docx) as a binary stream. */
|
|
45
|
+
downloadDocumentation(exportId: string, projectKey?: string): Promise<Response>;
|
|
46
|
+
}
|
|
47
|
+
/** Request body of POST /flow/tools/propagate-schema/ (pure; shared with `--plan`). */
|
|
48
|
+
export declare function schemaPropagationBody(projectKey: string, datasetName: string, opts: SchemaPropagationOptions): Record<string, unknown>;
|