dataiku-sdk 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -46
- package/dist/build-metadata.json +1 -1
- package/dist/packages/types/src/index.d.ts +102 -0
- package/dist/packages/types/src/index.js +76 -0
- package/dist/src/cli/commands/app.js +57 -15
- package/dist/src/cli/commands/bundle.js +1 -1
- package/dist/src/cli/commands/code-env.js +13 -7
- package/dist/src/cli/commands/code.js +1 -1
- package/dist/src/cli/commands/connection.d.ts +9 -0
- package/dist/src/cli/commands/connection.js +267 -1
- package/dist/src/cli/commands/data-collection.d.ts +2 -0
- package/dist/src/cli/commands/data-collection.js +164 -0
- package/dist/src/cli/commands/dataset.js +162 -33
- package/dist/src/cli/commands/future.js +2 -2
- package/dist/src/cli/commands/group.d.ts +2 -0
- package/dist/src/cli/commands/group.js +109 -0
- package/dist/src/cli/commands/index.js +151 -78
- package/dist/src/cli/commands/job.js +4 -2
- package/dist/src/cli/commands/knowledge-bank.d.ts +2 -0
- package/dist/src/cli/commands/knowledge-bank.js +80 -0
- package/dist/src/cli/commands/llm.d.ts +2 -0
- package/dist/src/cli/commands/llm.js +93 -0
- package/dist/src/cli/commands/macro.d.ts +2 -0
- package/dist/src/cli/commands/macro.js +221 -0
- package/dist/src/cli/commands/ml-task.js +18 -6
- package/dist/src/cli/commands/notebook.js +6 -2
- package/dist/src/cli/commands/plugin.d.ts +2 -0
- package/dist/src/cli/commands/plugin.js +799 -0
- package/dist/src/cli/commands/project-folder.d.ts +2 -0
- package/dist/src/cli/commands/project-folder.js +197 -0
- package/dist/src/cli/commands/project-git.js +15 -55
- package/dist/src/cli/commands/project.js +52 -6
- package/dist/src/cli/commands/recipe.js +36 -1
- package/dist/src/cli/commands/saved-model.d.ts +5 -0
- package/dist/src/cli/commands/saved-model.js +388 -1
- package/dist/src/cli/commands/scenario.js +139 -2
- package/dist/src/cli/commands/user.d.ts +8 -0
- package/dist/src/cli/commands/user.js +296 -0
- package/dist/src/cli/contract.d.ts +1 -2
- package/dist/src/cli/contract.js +1098 -297
- package/dist/src/cli/flags.js +30 -1
- package/dist/src/cli/helpers/app-successor.d.ts +19 -4
- package/dist/src/cli/helpers/app-successor.js +90 -30
- package/dist/src/cli/helpers/cleanup.js +3 -1
- package/dist/src/cli/helpers/flow-zone.js +1 -2
- package/dist/src/cli/output-schemas.d.ts +24 -0
- package/dist/src/cli/output-schemas.js +233 -0
- package/dist/src/cli/output.d.ts +1 -1
- package/dist/src/cli/output.js +23 -10
- package/dist/src/cli/usage.d.ts +1 -0
- package/dist/src/cli/usage.js +47 -1
- package/dist/src/cli.js +147 -41
- package/dist/src/client.d.ts +92 -5
- package/dist/src/client.js +202 -31
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +8 -4
- package/dist/src/generated/action-output-schemas.json +1735 -0
- package/dist/src/index.d.ts +16 -7
- package/dist/src/index.js +11 -2
- package/dist/src/resources/analyses.js +1 -8
- package/dist/src/resources/applications.d.ts +8 -6
- package/dist/src/resources/applications.js +21 -21
- package/dist/src/resources/base.d.ts +4 -0
- package/dist/src/resources/base.js +15 -0
- package/dist/src/resources/bundles.d.ts +9 -4
- package/dist/src/resources/bundles.js +10 -5
- package/dist/src/resources/connections.d.ts +78 -1
- package/dist/src/resources/connections.js +132 -1
- package/dist/src/resources/data-collections.d.ts +118 -0
- package/dist/src/resources/data-collections.js +49 -0
- package/dist/src/resources/datasets.d.ts +69 -0
- package/dist/src/resources/datasets.js +136 -5
- package/dist/src/resources/flow-zones.d.ts +6 -2
- package/dist/src/resources/flow-zones.js +6 -3
- package/dist/src/resources/futures.d.ts +6 -1
- package/dist/src/resources/futures.js +68 -22
- package/dist/src/resources/groups.d.ts +47 -0
- package/dist/src/resources/groups.js +56 -0
- package/dist/src/resources/jobs.d.ts +8 -13
- package/dist/src/resources/jobs.js +71 -34
- package/dist/src/resources/knowledge-banks.d.ts +58 -0
- package/dist/src/resources/knowledge-banks.js +61 -0
- package/dist/src/resources/llms.d.ts +92 -0
- package/dist/src/resources/llms.js +62 -0
- package/dist/src/resources/macros.d.ts +121 -0
- package/dist/src/resources/macros.js +231 -0
- package/dist/src/resources/ml-tasks.d.ts +37 -2
- package/dist/src/resources/ml-tasks.js +109 -20
- package/dist/src/resources/model-evaluation-stores.js +1 -8
- package/dist/src/resources/notebooks.d.ts +43 -7
- package/dist/src/resources/notebooks.js +53 -13
- package/dist/src/resources/plugins.d.ts +263 -0
- package/dist/src/resources/plugins.js +455 -0
- package/dist/src/resources/project-folders.d.ts +71 -0
- package/dist/src/resources/project-folders.js +57 -0
- package/dist/src/resources/project-git.d.ts +12 -12
- package/dist/src/resources/project-git.js +84 -82
- package/dist/src/resources/project-library.d.ts +0 -1
- package/dist/src/resources/project-library.js +1 -17
- package/dist/src/resources/projects.d.ts +23 -3
- package/dist/src/resources/projects.js +60 -34
- package/dist/src/resources/recipes.d.ts +15 -1
- package/dist/src/resources/recipes.js +73 -6
- package/dist/src/resources/saved-models.d.ts +134 -1
- package/dist/src/resources/saved-models.js +214 -6
- package/dist/src/resources/scenarios.d.ts +105 -0
- package/dist/src/resources/scenarios.js +232 -106
- package/dist/src/resources/sql.js +5 -1
- package/dist/src/resources/users.d.ts +129 -0
- package/dist/src/resources/users.js +128 -0
- package/dist/src/resources/webapps.d.ts +15 -2
- package/dist/src/resources/webapps.js +18 -4
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/deep-merge.d.ts +1 -1
- package/dist/src/utils/git-reference.d.ts +12 -0
- package/dist/src/utils/git-reference.js +38 -0
- package/dist/src/utils/polling.d.ts +20 -0
- package/dist/src/utils/polling.js +27 -0
- package/dist/src/utils/response-file.js +1 -1
- package/dist/src/utils/secret-sanitize.d.ts +54 -0
- package/dist/src/utils/secret-sanitize.js +101 -0
- package/package.json +9 -8
- package/packages/types/dist/index.d.ts +102 -0
- package/packages/types/dist/index.js +76 -0
- package/plugin.json +1 -1
- package/skills/dataiku-dss/SKILL.md +11 -7
- package/skills/dataiku-dss/references/app-releases.md +8 -8
- package/skills/dataiku-dss/references/authentication.md +6 -6
- package/skills/dataiku-dss/references/discovery.md +4 -4
- package/skills/dataiku-dss/references/mutations.md +1 -1
- package/skills/dataiku-dss/references/troubleshooting.md +2 -2
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { BaseResource } from "./base.js";
|
|
2
|
+
/**
|
|
3
|
+
* Data collections are instance-level curated groupings of DSS objects
|
|
4
|
+
* (datasets, saved models, agents, fine-tuned models, agent tools, semantic
|
|
5
|
+
* models) shared across projects.
|
|
6
|
+
*
|
|
7
|
+
* @see https://doc.dataiku.com/dss/api/15/rest/ (Data Collections section)
|
|
8
|
+
*/
|
|
9
|
+
export type DataCollectionObjectReferenceType = "DATASET" | "SAVED_MODEL" | "FINE_TUNED_MODEL" | "AGENT" | "AGENT_TOOL" | "SEMANTIC_MODEL" | (string & {});
|
|
10
|
+
export interface DataCollectionPermission extends Record<string, unknown> {
|
|
11
|
+
/** Name of the user this permission applies to (exclusive with group). */
|
|
12
|
+
user?: string;
|
|
13
|
+
/** Name of the group this permission applies to (exclusive with user). */
|
|
14
|
+
group?: string;
|
|
15
|
+
/** true if the permission grants administrator access. */
|
|
16
|
+
admin?: boolean;
|
|
17
|
+
/** true if the permission grants contributor access. */
|
|
18
|
+
write?: boolean;
|
|
19
|
+
/** true if the permission grants member access. */
|
|
20
|
+
read?: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface DataCollectionSummary extends Record<string, unknown> {
|
|
23
|
+
/** Name of the collection. */
|
|
24
|
+
displayName?: string;
|
|
25
|
+
/** Color of the collection (#RRGBB syntax). */
|
|
26
|
+
color?: string | null;
|
|
27
|
+
/** The description of the collection (markdown). */
|
|
28
|
+
description?: string | null;
|
|
29
|
+
/** List of tags of the collection. */
|
|
30
|
+
tags?: string[] | null;
|
|
31
|
+
/** Id of the collection. */
|
|
32
|
+
id?: string;
|
|
33
|
+
/** Number of objects in the collection. */
|
|
34
|
+
itemCount?: number;
|
|
35
|
+
/** Number of datasets in the collection. */
|
|
36
|
+
datasetsCount?: number;
|
|
37
|
+
/** Number of saved models in the collection, excluding agents and fine tuned models. */
|
|
38
|
+
savedModelsCount?: number;
|
|
39
|
+
/** Number of fine tuned models in the collection. */
|
|
40
|
+
fineTunedModelsCount?: number;
|
|
41
|
+
/** Number of agents in the collection. */
|
|
42
|
+
agentsCount?: number;
|
|
43
|
+
/** Number of agent tools in the collection. */
|
|
44
|
+
agentToolsCount?: number;
|
|
45
|
+
/** Number of semantic models in the collection. */
|
|
46
|
+
semanticModelsCount?: number;
|
|
47
|
+
/** Timestamp of the last collection modification. */
|
|
48
|
+
lastModifiedOn?: number;
|
|
49
|
+
}
|
|
50
|
+
export interface DataCollectionSettings extends Record<string, unknown> {
|
|
51
|
+
/** Id of the collection. */
|
|
52
|
+
id?: string;
|
|
53
|
+
/** Name of the collection. */
|
|
54
|
+
displayName: string;
|
|
55
|
+
/** Color of the collection (#RRGGBB syntax). */
|
|
56
|
+
color?: string | null;
|
|
57
|
+
/** The description of the collection (markdown). */
|
|
58
|
+
description?: string | null;
|
|
59
|
+
/** List of tags of the collection. */
|
|
60
|
+
tags?: string[] | null;
|
|
61
|
+
/** List of permissions of the collection (READER, CONTRIBUTOR or ADMIN). */
|
|
62
|
+
permissions?: DataCollectionPermission[] | null;
|
|
63
|
+
}
|
|
64
|
+
export interface DataCollectionCreateRequest extends Omit<DataCollectionSettings, "id"> {
|
|
65
|
+
/**
|
|
66
|
+
* Id of the Collection (a random id is generated if not provided).
|
|
67
|
+
* Must match the [A-Za-z0-9]{8} server-side format when provided.
|
|
68
|
+
*/
|
|
69
|
+
id?: string;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* An object inside a collection: a reference to a DSS object hosted by a
|
|
73
|
+
* project. The documented shape covers datasets; `type` accepts the other
|
|
74
|
+
* collection member kinds the instance may report.
|
|
75
|
+
*/
|
|
76
|
+
export interface DataCollectionObject extends Record<string, unknown> {
|
|
77
|
+
/** The type of this object (e.g. "DATASET"). */
|
|
78
|
+
type: DataCollectionObjectReferenceType;
|
|
79
|
+
/** The key of the project hosting the object. */
|
|
80
|
+
projectKey: string;
|
|
81
|
+
/** The id (name) of the object in its project. */
|
|
82
|
+
id: string;
|
|
83
|
+
}
|
|
84
|
+
export declare class DataCollectionsResource extends BaseResource {
|
|
85
|
+
/** List the collections on which the API key has READ privilege. */
|
|
86
|
+
list(): Promise<DataCollectionSummary[]>;
|
|
87
|
+
/**
|
|
88
|
+
* Create a collection. The creating user is added as an administrator.
|
|
89
|
+
* Requires the mayCreateDataCollections global permission. DSS returns
|
|
90
|
+
* `{ msg, id }`; the created id is surfaced to the caller.
|
|
91
|
+
*/
|
|
92
|
+
create(body: DataCollectionCreateRequest): Promise<{
|
|
93
|
+
msg?: string;
|
|
94
|
+
id?: string;
|
|
95
|
+
}>;
|
|
96
|
+
/**
|
|
97
|
+
* Get the settings of a collection. Requires READ on the collection; the
|
|
98
|
+
* permissions array is omitted unless the caller is an ADMIN.
|
|
99
|
+
*/
|
|
100
|
+
get(dataCollectionId: string): Promise<DataCollectionSettings>;
|
|
101
|
+
/**
|
|
102
|
+
* Update the settings of a collection. Only set a settings object obtained
|
|
103
|
+
* through {@link get}. Requires ADMIN on the collection.
|
|
104
|
+
*/
|
|
105
|
+
update(dataCollectionId: string, settings: DataCollectionSettings): Promise<DataCollectionSettings>;
|
|
106
|
+
/** Permanently delete a collection. Requires ADMIN on the collection. */
|
|
107
|
+
delete(dataCollectionId: string): Promise<void>;
|
|
108
|
+
/** List the objects in a collection. Requires READ on the collection. */
|
|
109
|
+
listObjects(dataCollectionId: string): Promise<DataCollectionObject[]>;
|
|
110
|
+
/**
|
|
111
|
+
* Add an object to a collection. Requires WRITE (Contributor) on the
|
|
112
|
+
* collection, the mayPublishToDataCollections global group permission, and
|
|
113
|
+
* publishToDataCollections on the project hosting the object (if relevant).
|
|
114
|
+
*/
|
|
115
|
+
addObject(dataCollectionId: string, object: DataCollectionObject): Promise<void>;
|
|
116
|
+
/** Remove a dataset from a collection. Requires WRITE on the collection. */
|
|
117
|
+
removeDataset(dataCollectionId: string, projectKey: string, datasetName: string): Promise<void>;
|
|
118
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { BaseResource, } from "./base.js";
|
|
2
|
+
export class DataCollectionsResource extends BaseResource {
|
|
3
|
+
/** List the collections on which the API key has READ privilege. */
|
|
4
|
+
async list() {
|
|
5
|
+
return this.client.get("/public/api/data-collections/");
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Create a collection. The creating user is added as an administrator.
|
|
9
|
+
* Requires the mayCreateDataCollections global permission. DSS returns
|
|
10
|
+
* `{ msg, id }`; the created id is surfaced to the caller.
|
|
11
|
+
*/
|
|
12
|
+
async create(body) {
|
|
13
|
+
return this.client.post("/public/api/data-collections/", body);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Get the settings of a collection. Requires READ on the collection; the
|
|
17
|
+
* permissions array is omitted unless the caller is an ADMIN.
|
|
18
|
+
*/
|
|
19
|
+
async get(dataCollectionId) {
|
|
20
|
+
return this.client.get(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}`);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Update the settings of a collection. Only set a settings object obtained
|
|
24
|
+
* through {@link get}. Requires ADMIN on the collection.
|
|
25
|
+
*/
|
|
26
|
+
async update(dataCollectionId, settings) {
|
|
27
|
+
return this.client.put(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}`, settings);
|
|
28
|
+
}
|
|
29
|
+
/** Permanently delete a collection. Requires ADMIN on the collection. */
|
|
30
|
+
async delete(dataCollectionId) {
|
|
31
|
+
await this.client.del(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}`);
|
|
32
|
+
}
|
|
33
|
+
/** List the objects in a collection. Requires READ on the collection. */
|
|
34
|
+
async listObjects(dataCollectionId) {
|
|
35
|
+
return this.client.get(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}/objects`);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Add an object to a collection. Requires WRITE (Contributor) on the
|
|
39
|
+
* collection, the mayPublishToDataCollections global group permission, and
|
|
40
|
+
* publishToDataCollections on the project hosting the object (if relevant).
|
|
41
|
+
*/
|
|
42
|
+
async addObject(dataCollectionId, object) {
|
|
43
|
+
await this.client.post(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}/objects`, object);
|
|
44
|
+
}
|
|
45
|
+
/** Remove a dataset from a collection. Requires WRITE on the collection. */
|
|
46
|
+
async removeDataset(dataCollectionId, projectKey, datasetName) {
|
|
47
|
+
await this.client.del(`/public/api/data-collections/${encodeURIComponent(dataCollectionId)}/objects/dataset/${encodeURIComponent(projectKey)}/${encodeURIComponent(datasetName)}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -17,6 +17,37 @@ export interface DatasetSchemaColumnInput {
|
|
|
17
17
|
type: string;
|
|
18
18
|
comment?: string;
|
|
19
19
|
}
|
|
20
|
+
/** Options for creating a managed dataset through the dedicated endpoint. */
|
|
21
|
+
export interface DatasetManagedCreateOptions {
|
|
22
|
+
/** Name of the new dataset (unique in the project). */
|
|
23
|
+
name: string;
|
|
24
|
+
/** Connection where the dataset is stored. */
|
|
25
|
+
connection: string;
|
|
26
|
+
/** When the connection accepts several dataset types, the type to use. */
|
|
27
|
+
typeOptionId?: string;
|
|
28
|
+
/** Optional identifier of a file format option (e.g. CSV_EXCEL_GZIP, PARQUET_HIVE). */
|
|
29
|
+
formatOptionId?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Partitioning option id. Use {@link DatasetManagedCreateOptions.copyPartitioningFrom}
|
|
32
|
+
* to build one from an existing dataset or folder reference.
|
|
33
|
+
*/
|
|
34
|
+
partitioningOptionId?: string;
|
|
35
|
+
/**
|
|
36
|
+
* Copy partitioning from an existing dataset or folder instead of passing a
|
|
37
|
+
* raw `partitioningOptionId`. `type` defaults to `DATASET` and may be
|
|
38
|
+
* `FOLDER`.
|
|
39
|
+
*/
|
|
40
|
+
copyPartitioningFrom?: {
|
|
41
|
+
ref: string;
|
|
42
|
+
type?: "DATASET" | "FOLDER";
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** Result of creating a managed dataset (the endpoint answers 204 with no body). */
|
|
46
|
+
export interface DatasetManagedCreateResult {
|
|
47
|
+
datasetName: string;
|
|
48
|
+
projectKey: string;
|
|
49
|
+
creationSettings: Record<string, unknown>;
|
|
50
|
+
}
|
|
20
51
|
export interface DatasetCloneOptions {
|
|
21
52
|
projectKey?: string;
|
|
22
53
|
path?: string;
|
|
@@ -120,6 +151,44 @@ export declare class DatasetsResource extends BaseResource {
|
|
|
120
151
|
}>;
|
|
121
152
|
/** Get dataset metadata (tags, custom fields, checklists). */
|
|
122
153
|
metadata(datasetName: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
154
|
+
/**
|
|
155
|
+
* Replace dataset metadata with the given object (faithful PUT per the DSS
|
|
156
|
+
* API contract: the caller GETs the metadata, edits it, and PUTs the full
|
|
157
|
+
* object back — no partial merge is applied client-side). `label`,
|
|
158
|
+
* `description`, `tags`, and `custom` are the documented writable fields;
|
|
159
|
+
* any additional server-provided metadata fields present in the object are
|
|
160
|
+
* preserved verbatim.
|
|
161
|
+
*/
|
|
162
|
+
updateMetadata(datasetName: string, metadata: Record<string, unknown>, projectKey?: string): Promise<Record<string, unknown>>;
|
|
163
|
+
/**
|
|
164
|
+
* Create a managed dataset through the dedicated
|
|
165
|
+
* `POST /projects/{projectKey}/datasets/managed` endpoint. DSS handles the
|
|
166
|
+
* format/storage details: you select the name, the connection, and optional
|
|
167
|
+
* type/format/partitioning options. The endpoint answers `204` with no
|
|
168
|
+
* body; the returned settings echo what was sent.
|
|
169
|
+
*/
|
|
170
|
+
createManaged(opts: DatasetManagedCreateOptions, projectKey?: string): Promise<DatasetManagedCreateResult>;
|
|
171
|
+
/**
|
|
172
|
+
* Get the full info object for a dataset (type, parameters, last build
|
|
173
|
+
* information, schema, etc.). The server returns a complex opaque object;
|
|
174
|
+
* it is surfaced without validation.
|
|
175
|
+
*/
|
|
176
|
+
info(datasetName: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
177
|
+
/**
|
|
178
|
+
* Get the full column lineage (auto-computed and manual) of a column in
|
|
179
|
+
* this dataset. Relations with datasets from both local and foreign
|
|
180
|
+
* projects are included.
|
|
181
|
+
*
|
|
182
|
+
* Route per the official Python client:
|
|
183
|
+
* `GET /projects/{projectKey}/datasets/{datasetName}/column-lineage`
|
|
184
|
+
* with query `columnName` (required) and `maxDatasetCount` (optional,
|
|
185
|
+
* positive integer — the docs page renders a merged project-scoped path,
|
|
186
|
+
* but DSS's own client and route registry use this per-dataset form).
|
|
187
|
+
*/
|
|
188
|
+
getColumnLineage(datasetName: string, column: string, opts?: {
|
|
189
|
+
maxDatasetCount?: number;
|
|
190
|
+
projectKey?: string;
|
|
191
|
+
}): Promise<unknown>;
|
|
123
192
|
/**
|
|
124
193
|
* Download dataset rows as a gzipped (or .csv) file, capped at `limit` rows
|
|
125
194
|
* (default 100k). Returns the written path, the number of data rows written,
|
|
@@ -394,12 +394,39 @@ async function collectTsvRowCount(body, stopAfter, timeoutMs) {
|
|
|
394
394
|
reader.releaseLock();
|
|
395
395
|
}
|
|
396
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* Encoded CSV bytes accumulated before one stream push. Rows are joined into
|
|
399
|
+
* batch-sized chunks so large downloads do not pay per-row stream overhead; a
|
|
400
|
+
* single escaped row may exceed the budget (rows are never split), and the
|
|
401
|
+
* batch is always flushed at EOF so trailing rows are never dropped.
|
|
402
|
+
*/
|
|
403
|
+
const CSV_BATCH_BYTES = 64 * 1024;
|
|
397
404
|
function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true) {
|
|
398
405
|
const state = createTsvStreamState();
|
|
399
406
|
const decoder = new StringDecoder("utf-8");
|
|
400
407
|
const maxRows = Math.max(1, maxDataRows);
|
|
401
408
|
let headerSeen = false;
|
|
402
409
|
let done = false;
|
|
410
|
+
// Bounded CSV output batching: emitted rows accumulate here and are joined
|
|
411
|
+
// into a single push once the batch reaches CSV_BATCH_BYTES, so memory stays
|
|
412
|
+
// bounded by the batch budget plus one escaped row (the join briefly holds a
|
|
413
|
+
// second copy of the batch) while per-row push/callback overhead disappears.
|
|
414
|
+
let batchLines = [];
|
|
415
|
+
let batchBytes = 0;
|
|
416
|
+
const flushBatch = (target) => {
|
|
417
|
+
if (batchLines.length === 0)
|
|
418
|
+
return;
|
|
419
|
+
const csv = batchLines.join("");
|
|
420
|
+
batchLines = [];
|
|
421
|
+
batchBytes = 0;
|
|
422
|
+
target.push(csv);
|
|
423
|
+
};
|
|
424
|
+
const queueLine = (target, line) => {
|
|
425
|
+
batchLines.push(line);
|
|
426
|
+
batchBytes += Buffer.byteLength(line);
|
|
427
|
+
if (batchBytes >= CSV_BATCH_BYTES)
|
|
428
|
+
flushBatch(target);
|
|
429
|
+
};
|
|
403
430
|
let columnCount = 0;
|
|
404
431
|
const handleRow = (row, push) => {
|
|
405
432
|
if (done)
|
|
@@ -430,7 +457,7 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
430
457
|
return;
|
|
431
458
|
}
|
|
432
459
|
try {
|
|
433
|
-
consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => this
|
|
460
|
+
consumeTsvChunk(decoder.write(chunk), state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
434
461
|
}
|
|
435
462
|
catch (error) {
|
|
436
463
|
// Byte-limit overflow: destroy the transform so the pipeline
|
|
@@ -438,8 +465,11 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
438
465
|
callback(error instanceof Error ? error : new Error(String(error)));
|
|
439
466
|
return;
|
|
440
467
|
}
|
|
441
|
-
if (done)
|
|
468
|
+
if (done) {
|
|
469
|
+
// Truncation reached: emit the batch queued so far before EOF.
|
|
470
|
+
flushBatch(this);
|
|
442
471
|
this.push(null);
|
|
472
|
+
}
|
|
443
473
|
callback();
|
|
444
474
|
},
|
|
445
475
|
flush(callback) {
|
|
@@ -450,14 +480,17 @@ function tsvToCsvTransform(maxDataRows, stats, onHeader, spreadsheetSafe = true)
|
|
|
450
480
|
try {
|
|
451
481
|
const tail = decoder.end();
|
|
452
482
|
if (tail !== "") {
|
|
453
|
-
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => this
|
|
483
|
+
consumeTsvChunk(tail, state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
454
484
|
}
|
|
455
|
-
flushTsvStream(state, (row) => handleRow(row, (line) => this
|
|
485
|
+
flushTsvStream(state, (row) => handleRow(row, (line) => queueLine(this, line)));
|
|
456
486
|
}
|
|
457
487
|
catch (error) {
|
|
458
488
|
callback(error instanceof Error ? error : new Error(String(error)));
|
|
459
489
|
return;
|
|
460
490
|
}
|
|
491
|
+
// Emit the trailing partial batch; without this the last rows of a
|
|
492
|
+
// download (below the batch budget) would be silently dropped.
|
|
493
|
+
flushBatch(this);
|
|
461
494
|
callback();
|
|
462
495
|
},
|
|
463
496
|
});
|
|
@@ -712,6 +745,104 @@ export class DatasetsResource extends BaseResource {
|
|
|
712
745
|
const dsEnc = encodeURIComponent(datasetName);
|
|
713
746
|
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/metadata`);
|
|
714
747
|
}
|
|
748
|
+
/**
|
|
749
|
+
* Replace dataset metadata with the given object (faithful PUT per the DSS
|
|
750
|
+
* API contract: the caller GETs the metadata, edits it, and PUTs the full
|
|
751
|
+
* object back — no partial merge is applied client-side). `label`,
|
|
752
|
+
* `description`, `tags`, and `custom` are the documented writable fields;
|
|
753
|
+
* any additional server-provided metadata fields present in the object are
|
|
754
|
+
* preserved verbatim.
|
|
755
|
+
*/
|
|
756
|
+
async updateMetadata(datasetName, metadata, projectKey) {
|
|
757
|
+
if (metadata === null || Array.isArray(metadata)) {
|
|
758
|
+
throw new ClientValidationError("metadata must be a JSON object.");
|
|
759
|
+
}
|
|
760
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
761
|
+
return this.client.put(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/metadata`, metadata);
|
|
762
|
+
}
|
|
763
|
+
/**
|
|
764
|
+
* Create a managed dataset through the dedicated
|
|
765
|
+
* `POST /projects/{projectKey}/datasets/managed` endpoint. DSS handles the
|
|
766
|
+
* format/storage details: you select the name, the connection, and optional
|
|
767
|
+
* type/format/partitioning options. The endpoint answers `204` with no
|
|
768
|
+
* body; the returned settings echo what was sent.
|
|
769
|
+
*/
|
|
770
|
+
async createManaged(opts, projectKey) {
|
|
771
|
+
const name = typeof opts.name === "string" ? opts.name.trim() : "";
|
|
772
|
+
if (name === "") {
|
|
773
|
+
throw new ClientValidationError("name must be a non-empty dataset name.");
|
|
774
|
+
}
|
|
775
|
+
const connection = typeof opts.connection === "string" ? opts.connection.trim() : "";
|
|
776
|
+
if (connection === "") {
|
|
777
|
+
throw new ClientValidationError("connection must be a non-empty connection name.");
|
|
778
|
+
}
|
|
779
|
+
if (opts.partitioningOptionId !== undefined && opts.copyPartitioningFrom !== undefined) {
|
|
780
|
+
throw new ClientValidationError("Pass either partitioningOptionId or copyPartitioningFrom, not both.");
|
|
781
|
+
}
|
|
782
|
+
if (opts.copyPartitioningFrom !== undefined
|
|
783
|
+
&& (typeof opts.copyPartitioningFrom.ref !== "string"
|
|
784
|
+
|| opts.copyPartitioningFrom.ref.trim() === "")) {
|
|
785
|
+
throw new ClientValidationError("copyPartitioningFrom.ref must be a non-empty dataset or folder name.");
|
|
786
|
+
}
|
|
787
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
788
|
+
const creationSettings = { connectionId: connection, };
|
|
789
|
+
if (opts.typeOptionId !== undefined)
|
|
790
|
+
creationSettings.typeOptionId = opts.typeOptionId;
|
|
791
|
+
if (opts.formatOptionId !== undefined || opts.partitioningOptionId !== undefined
|
|
792
|
+
|| opts.copyPartitioningFrom !== undefined) {
|
|
793
|
+
const specificSettings = {};
|
|
794
|
+
if (opts.formatOptionId !== undefined) {
|
|
795
|
+
specificSettings.formatOptionId = opts.formatOptionId;
|
|
796
|
+
}
|
|
797
|
+
if (opts.partitioningOptionId !== undefined) {
|
|
798
|
+
specificSettings.partitioningOptionId = opts.partitioningOptionId;
|
|
799
|
+
}
|
|
800
|
+
else if (opts.copyPartitioningFrom !== undefined) {
|
|
801
|
+
const code = opts.copyPartitioningFrom.type === "FOLDER" ? "folder" : "dataset";
|
|
802
|
+
specificSettings.partitioningOptionId = `copy:${code}:${opts.copyPartitioningFrom.ref.trim()}`;
|
|
803
|
+
}
|
|
804
|
+
creationSettings.specificSettings = specificSettings;
|
|
805
|
+
}
|
|
806
|
+
await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/datasets/managed`, { name, creationSettings, });
|
|
807
|
+
return { datasetName: name, projectKey: pk, creationSettings, };
|
|
808
|
+
}
|
|
809
|
+
/**
|
|
810
|
+
* Get the full info object for a dataset (type, parameters, last build
|
|
811
|
+
* information, schema, etc.). The server returns a complex opaque object;
|
|
812
|
+
* it is surfaced without validation.
|
|
813
|
+
*/
|
|
814
|
+
async info(datasetName, projectKey) {
|
|
815
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
816
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/datasets/${dsEnc}/info`);
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Get the full column lineage (auto-computed and manual) of a column in
|
|
820
|
+
* this dataset. Relations with datasets from both local and foreign
|
|
821
|
+
* projects are included.
|
|
822
|
+
*
|
|
823
|
+
* Route per the official Python client:
|
|
824
|
+
* `GET /projects/{projectKey}/datasets/{datasetName}/column-lineage`
|
|
825
|
+
* with query `columnName` (required) and `maxDatasetCount` (optional,
|
|
826
|
+
* positive integer — the docs page renders a merged project-scoped path,
|
|
827
|
+
* but DSS's own client and route registry use this per-dataset form).
|
|
828
|
+
*/
|
|
829
|
+
async getColumnLineage(datasetName, column, opts = {}) {
|
|
830
|
+
const trimmed = typeof column === "string" ? column.trim() : "";
|
|
831
|
+
if (trimmed === "") {
|
|
832
|
+
throw new ClientValidationError("column must be a non-empty column name.");
|
|
833
|
+
}
|
|
834
|
+
const maxDatasetCount = opts.maxDatasetCount;
|
|
835
|
+
if (maxDatasetCount !== undefined
|
|
836
|
+
&& (!Number.isSafeInteger(maxDatasetCount) || maxDatasetCount <= 0)) {
|
|
837
|
+
throw new ClientValidationError("maxDatasetCount must be a positive integer.");
|
|
838
|
+
}
|
|
839
|
+
const dsEnc = encodeURIComponent(datasetName);
|
|
840
|
+
const params = [`columnName=${encodeURIComponent(trimmed)}`,];
|
|
841
|
+
if (maxDatasetCount !== undefined) {
|
|
842
|
+
params.push(`maxDatasetCount=${encodeURIComponent(String(maxDatasetCount))}`);
|
|
843
|
+
}
|
|
844
|
+
return this.client.get(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/column-lineage?${params.join("&")}`);
|
|
845
|
+
}
|
|
715
846
|
/**
|
|
716
847
|
* Download dataset rows as a gzipped (or .csv) file, capped at `limit` rows
|
|
717
848
|
* (default 100k). Returns the written path, the number of data rows written,
|
|
@@ -746,7 +877,7 @@ export class DatasetsResource extends BaseResource {
|
|
|
746
877
|
: undefined;
|
|
747
878
|
const shouldGzip = filePath.endsWith(".gz");
|
|
748
879
|
const stats = { rows: 0, truncated: false, };
|
|
749
|
-
const nodeStream = Readable.
|
|
880
|
+
const nodeStream = Readable.from(res.body, { objectMode: false, });
|
|
750
881
|
const csvTransform = tsvToCsvTransform(limit, stats, onHeader, opts?.rawData !== true);
|
|
751
882
|
fs.mkdirSync(nodePath.dirname(filePath), { recursive: true, });
|
|
752
883
|
const fileOut = fs.createWriteStream(filePath);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { DataikuGetOptions } from "../client.js";
|
|
1
2
|
import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZoneUpdateOptions } from "../schemas.js";
|
|
2
3
|
import { BaseResource } from "./base.js";
|
|
3
4
|
export type FlowZoneItemInput = {
|
|
@@ -6,8 +7,11 @@ export type FlowZoneItemInput = {
|
|
|
6
7
|
projectKey?: string;
|
|
7
8
|
};
|
|
8
9
|
export declare class FlowZonesResource extends BaseResource {
|
|
9
|
-
/**
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* List all flow zones in a project. Pass `timeoutMs` to bound the whole
|
|
12
|
+
* request (attempts, backoff, and body read) by a total budget.
|
|
13
|
+
*/
|
|
14
|
+
list(projectKey?: string, options?: DataikuGetOptions): Promise<FlowZone[]>;
|
|
11
15
|
/** Get one flow zone by id. */
|
|
12
16
|
get(zoneId: string, projectKey?: string): Promise<FlowZone>;
|
|
13
17
|
/** Create a flow zone. */
|
|
@@ -9,9 +9,12 @@ function normalizeZoneItem(item) {
|
|
|
9
9
|
};
|
|
10
10
|
}
|
|
11
11
|
export class FlowZonesResource extends BaseResource {
|
|
12
|
-
/**
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
/**
|
|
13
|
+
* List all flow zones in a project. Pass `timeoutMs` to bound the whole
|
|
14
|
+
* request (attempts, backoff, and body read) by a total budget.
|
|
15
|
+
*/
|
|
16
|
+
async list(projectKey, options) {
|
|
17
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones`, options);
|
|
15
18
|
return this.client.safeParse(FlowZoneArraySchema, raw, "flowZones.list");
|
|
16
19
|
}
|
|
17
20
|
/** Get one flow zone by id. */
|
|
@@ -5,10 +5,15 @@ export interface FutureWaitOptions {
|
|
|
5
5
|
timeoutMs?: number;
|
|
6
6
|
}
|
|
7
7
|
export declare class FuturesResource extends BaseResource {
|
|
8
|
-
get(futureId: string
|
|
8
|
+
get(futureId: string, opts?: {
|
|
9
|
+
timeoutMs?: number;
|
|
10
|
+
noRetry?: boolean;
|
|
11
|
+
}): Promise<FutureState>;
|
|
9
12
|
peek(futureId: string): Promise<FutureState>;
|
|
10
13
|
state(futureId: string, opts?: {
|
|
11
14
|
peek?: boolean;
|
|
15
|
+
timeoutMs?: number;
|
|
16
|
+
noRetry?: boolean;
|
|
12
17
|
}): Promise<FutureState>;
|
|
13
18
|
abort(futureId: string): Promise<void>;
|
|
14
19
|
wait(futureId: string, opts?: FutureWaitOptions): Promise<FutureWaitResult>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { FutureStateSchema, FutureWaitResultSchema, } from "../schemas.js";
|
|
2
|
+
import { computeNextPollDelayMs, isRequestDeadlineError, } from "../utils/polling.js";
|
|
2
3
|
import { BaseResource, } from "./base.js";
|
|
3
4
|
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
4
5
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
@@ -23,8 +24,12 @@ function waitState(state) {
|
|
|
23
24
|
return "RUNNING";
|
|
24
25
|
}
|
|
25
26
|
export class FuturesResource extends BaseResource {
|
|
26
|
-
async get(futureId) {
|
|
27
|
-
return this.state(futureId, {
|
|
27
|
+
async get(futureId, opts = {}) {
|
|
28
|
+
return this.state(futureId, {
|
|
29
|
+
peek: false,
|
|
30
|
+
timeoutMs: opts.timeoutMs,
|
|
31
|
+
noRetry: opts.noRetry,
|
|
32
|
+
});
|
|
28
33
|
}
|
|
29
34
|
async peek(futureId) {
|
|
30
35
|
return this.state(futureId, { peek: true, });
|
|
@@ -32,23 +37,71 @@ export class FuturesResource extends BaseResource {
|
|
|
32
37
|
async state(futureId, opts = {}) {
|
|
33
38
|
const params = new URLSearchParams();
|
|
34
39
|
params.set("peek", String(opts.peek === true));
|
|
35
|
-
const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}
|
|
40
|
+
const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}`, { timeoutMs: opts.timeoutMs, noRetry: opts.noRetry, });
|
|
36
41
|
return this.client.safeParse(FutureStateSchema, raw, "futures.state");
|
|
37
42
|
}
|
|
38
43
|
async abort(futureId) {
|
|
39
44
|
await this.client.del(`/public/api/futures/${encodeURIComponent(futureId)}`);
|
|
40
45
|
}
|
|
41
46
|
async wait(futureId, opts = {}) {
|
|
42
|
-
|
|
47
|
+
// Adaptive backoff is the default; an explicit pollIntervalMs is an
|
|
48
|
+
// exact contract the caller chose, so it is never adapted.
|
|
49
|
+
const explicitIntervalMs = opts.pollIntervalMs;
|
|
50
|
+
const baseIntervalMs = Math.max(1, explicitIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
|
|
51
|
+
const adaptiveEnabled = explicitIntervalMs === undefined;
|
|
43
52
|
// A caller's budget is never rounded up to a whole poll interval: a 5ms
|
|
44
53
|
// wait must answer in about 5ms. One state observation always happens
|
|
45
54
|
// regardless, because the poll precedes the deadline check.
|
|
46
55
|
const timeoutMs = Math.max(0, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
47
56
|
const startedAt = Date.now();
|
|
48
57
|
let pollCount = 0;
|
|
58
|
+
let lastState;
|
|
49
59
|
while (true) {
|
|
60
|
+
const elapsedBeforeMs = Date.now() - startedAt;
|
|
61
|
+
// The first observation always happens, even when the budget is
|
|
62
|
+
// already spent. A later poll is never started after the deadline:
|
|
63
|
+
// the loop reports the structured timeout from the last observed
|
|
64
|
+
// state instead of issuing a request the budget cannot cover.
|
|
65
|
+
if (lastState !== undefined && elapsedBeforeMs >= timeoutMs) {
|
|
66
|
+
return this.client.safeParse(FutureWaitResultSchema, {
|
|
67
|
+
futureId,
|
|
68
|
+
jobId: lastState.jobId,
|
|
69
|
+
state: waitState(lastState),
|
|
70
|
+
elapsedMs: elapsedBeforeMs,
|
|
71
|
+
pollCount,
|
|
72
|
+
success: false,
|
|
73
|
+
timedOut: true,
|
|
74
|
+
hasResult: lastState.hasResult === true,
|
|
75
|
+
alive: lastState.alive,
|
|
76
|
+
aborted: lastState.aborted,
|
|
77
|
+
unknown: lastState.unknown,
|
|
78
|
+
}, "futures.wait");
|
|
79
|
+
}
|
|
50
80
|
pollCount += 1;
|
|
51
|
-
|
|
81
|
+
// Requests issued while budget remains are bounded by the remaining
|
|
82
|
+
// time; a spent budget still issues exactly one transport attempt
|
|
83
|
+
// (no retries, client requestTimeoutMs cap) so the first observation
|
|
84
|
+
// reaches the server instead of failing before any attempt.
|
|
85
|
+
const remainingMs = timeoutMs - elapsedBeforeMs;
|
|
86
|
+
let state;
|
|
87
|
+
try {
|
|
88
|
+
state = await this.get(futureId, remainingMs > 0 ? { timeoutMs: remainingMs, } : { noRetry: true, });
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
// The poll budget ran out mid-request: report the structured
|
|
92
|
+
// timeout instead of letting the transport deadline error escape.
|
|
93
|
+
if (!isRequestDeadlineError(error, startedAt + timeoutMs))
|
|
94
|
+
throw error;
|
|
95
|
+
return this.client.safeParse(FutureWaitResultSchema, {
|
|
96
|
+
futureId,
|
|
97
|
+
state: "RUNNING",
|
|
98
|
+
elapsedMs: Date.now() - startedAt,
|
|
99
|
+
pollCount,
|
|
100
|
+
success: false,
|
|
101
|
+
timedOut: true,
|
|
102
|
+
}, "futures.wait");
|
|
103
|
+
}
|
|
104
|
+
lastState = state;
|
|
52
105
|
const elapsedMs = Date.now() - startedAt;
|
|
53
106
|
const status = waitState(state);
|
|
54
107
|
if (isFinished(state)) {
|
|
@@ -67,25 +120,18 @@ export class FuturesResource extends BaseResource {
|
|
|
67
120
|
};
|
|
68
121
|
return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
|
|
69
122
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
state: status,
|
|
75
|
-
elapsedMs,
|
|
76
|
-
pollCount,
|
|
77
|
-
success: false,
|
|
78
|
-
timedOut: true,
|
|
79
|
-
hasResult: state.hasResult === true,
|
|
80
|
-
alive: state.alive,
|
|
81
|
-
aborted: state.aborted,
|
|
82
|
-
unknown: state.unknown,
|
|
83
|
-
};
|
|
84
|
-
return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
|
|
85
|
-
}
|
|
123
|
+
// Deadline reached: the guard at the top of the loop reports the
|
|
124
|
+
// timeout from this observation without issuing another request.
|
|
125
|
+
if (elapsedMs >= timeoutMs)
|
|
126
|
+
continue;
|
|
86
127
|
// Only the remaining budget is slept: a longer sleep would report an
|
|
87
128
|
// elapsed time the caller never authorized.
|
|
88
|
-
|
|
129
|
+
const nextDelayMs = computeNextPollDelayMs({
|
|
130
|
+
pollCount,
|
|
131
|
+
baseIntervalMs,
|
|
132
|
+
adaptiveEnabled,
|
|
133
|
+
});
|
|
134
|
+
await sleep(Math.min(nextDelayMs, timeoutMs - elapsedMs));
|
|
89
135
|
}
|
|
90
136
|
}
|
|
91
137
|
}
|