dataiku-sdk 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/dss.js +76 -3
- package/dist/packages/types/src/index.d.ts +275 -0
- package/dist/packages/types/src/index.js +190 -0
- package/dist/src/cli.js +4287 -862
- package/dist/src/client.d.ts +18 -0
- package/dist/src/client.js +30 -0
- package/dist/src/errors.d.ts +2 -0
- package/dist/src/errors.js +45 -1
- package/dist/src/index.d.ts +9 -3
- package/dist/src/index.js +7 -1
- package/dist/src/resources/code-envs.d.ts +9 -1
- package/dist/src/resources/code-envs.js +80 -1
- package/dist/src/resources/connections.d.ts +3 -1
- package/dist/src/resources/connections.js +4 -2
- package/dist/src/resources/dashboards.d.ts +19 -0
- package/dist/src/resources/dashboards.js +30 -0
- package/dist/src/resources/data-quality.d.ts +54 -0
- package/dist/src/resources/data-quality.js +119 -0
- package/dist/src/resources/datasets.js +1 -1
- package/dist/src/resources/flow-zones.d.ts +25 -0
- package/dist/src/resources/flow-zones.js +59 -0
- package/dist/src/resources/folders.d.ts +4 -1
- package/dist/src/resources/folders.js +27 -0
- package/dist/src/resources/futures.d.ts +15 -0
- package/dist/src/resources/futures.js +86 -0
- package/dist/src/resources/insights.d.ts +28 -0
- package/dist/src/resources/insights.js +64 -0
- package/dist/src/resources/jobs.d.ts +7 -4
- package/dist/src/resources/jobs.js +9 -7
- package/dist/src/resources/notebooks.d.ts +6 -4
- package/dist/src/resources/notebooks.js +6 -4
- package/dist/src/resources/recipes.js +115 -66
- package/dist/src/resources/wiki.d.ts +22 -0
- package/dist/src/resources/wiki.js +73 -0
- package/dist/src/schemas.d.ts +2 -2
- package/dist/src/schemas.js +1 -1
- package/dist/src/utils/cleanup-ledger.d.ts +15 -0
- package/dist/src/utils/cleanup-ledger.js +15 -0
- package/node_modules/@sinclair/typebox/package.json +2 -2
- package/node_modules/@sinclair/typebox/readme.md +7 -7
- package/package.json +11 -7
- package/packages/types/dist/index.d.ts +275 -0
- package/packages/types/dist/index.js +190 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { DataQualityComputeResultSchema, DataQualityProjectStatusSchema, DataQualityRuleArraySchema, DataQualityRuleResultArraySchema, DataQualityRuleSchema, DataQualityRulesSchema, DataQualityStatusByPartitionSchema, DataQualityStatusSchema, DataQualityTimelineSchema, } from "../schemas.js";
|
|
2
|
+
import { deepMerge, } from "../utils/deep-merge.js";
|
|
3
|
+
import { BaseResource, } from "./base.js";
|
|
4
|
+
function datasetPath(projectKey, datasetName) {
|
|
5
|
+
return `/public/api/projects/${encodeURIComponent(projectKey)}/datasets/${encodeURIComponent(datasetName)}/data-quality`;
|
|
6
|
+
}
|
|
7
|
+
function addQuery(params, name, value) {
|
|
8
|
+
if (value !== undefined)
|
|
9
|
+
params.set(name, String(value));
|
|
10
|
+
}
|
|
11
|
+
function queryString(params) {
|
|
12
|
+
const raw = params.toString();
|
|
13
|
+
return raw ? `?${raw}` : "";
|
|
14
|
+
}
|
|
15
|
+
function partitionValue(partition) {
|
|
16
|
+
return partition && partition.trim().length > 0 ? partition : "NP";
|
|
17
|
+
}
|
|
18
|
+
function projectPath(projectKey) {
|
|
19
|
+
return `/public/api/projects/${encodeURIComponent(projectKey)}/data-quality`;
|
|
20
|
+
}
|
|
21
|
+
export class DataQualityResource extends BaseResource {
|
|
22
|
+
async rules(datasetName, projectKey) {
|
|
23
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
24
|
+
const raw = await this.client.get(`${datasetPath(pk, datasetName)}/rules`);
|
|
25
|
+
return this.client.safeParse(DataQualityRulesSchema, raw, "dataQuality.rules");
|
|
26
|
+
}
|
|
27
|
+
async listRules(datasetName, projectKey) {
|
|
28
|
+
const rules = await this.rules(datasetName, projectKey);
|
|
29
|
+
return this.client.safeParse(DataQualityRuleArraySchema, rules.checks, "dataQuality.listRules");
|
|
30
|
+
}
|
|
31
|
+
async createRule(datasetName, opts) {
|
|
32
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
33
|
+
const raw = await this.client.post(`${datasetPath(pk, datasetName)}/rules`, opts.config);
|
|
34
|
+
return this.client.safeParse(DataQualityRuleSchema, raw, "dataQuality.createRule");
|
|
35
|
+
}
|
|
36
|
+
async updateRule(datasetName, ruleId, opts) {
|
|
37
|
+
const current = await this.getRule(datasetName, ruleId, opts.projectKey);
|
|
38
|
+
const next = deepMerge(current, opts.data);
|
|
39
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
40
|
+
await this.client.putVoid(`${datasetPath(pk, datasetName)}/rules/${encodeURIComponent(ruleId)}`, next);
|
|
41
|
+
return this.client.safeParse(DataQualityRuleSchema, next, "dataQuality.updateRule");
|
|
42
|
+
}
|
|
43
|
+
async deleteRule(datasetName, ruleId, projectKey) {
|
|
44
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
45
|
+
const params = new URLSearchParams();
|
|
46
|
+
params.set("ruleId", ruleId);
|
|
47
|
+
await this.client.del(`${datasetPath(pk, datasetName)}/rules/${encodeURIComponent(ruleId)}${queryString(params)}`);
|
|
48
|
+
}
|
|
49
|
+
async getRule(datasetName, ruleId, projectKey) {
|
|
50
|
+
const rules = await this.listRules(datasetName, projectKey);
|
|
51
|
+
const rule = rules.find((candidate) => candidate.id === ruleId);
|
|
52
|
+
if (!rule)
|
|
53
|
+
throw new Error(`Data quality rule not found: ${ruleId}`);
|
|
54
|
+
return rule;
|
|
55
|
+
}
|
|
56
|
+
async status(datasetName, projectKey) {
|
|
57
|
+
const pk = this.resolveProjectKey(projectKey);
|
|
58
|
+
const raw = await this.client.get(`${datasetPath(pk, datasetName)}/status`);
|
|
59
|
+
return this.client.safeParse(DataQualityStatusSchema, raw, "dataQuality.status");
|
|
60
|
+
}
|
|
61
|
+
async statusByPartition(datasetName, opts = {}) {
|
|
62
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
63
|
+
const params = new URLSearchParams();
|
|
64
|
+
addQuery(params, "includeAllPartitions", opts.includeAllPartitions);
|
|
65
|
+
const raw = await this.client.get(`${datasetPath(pk, datasetName)}/status-by-partition${queryString(params)}`);
|
|
66
|
+
return this.client.safeParse(DataQualityStatusByPartitionSchema, raw, "dataQuality.statusByPartition");
|
|
67
|
+
}
|
|
68
|
+
async lastResults(datasetName, opts = {}) {
|
|
69
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
70
|
+
const params = new URLSearchParams();
|
|
71
|
+
addQuery(params, "partition", partitionValue(opts.partition));
|
|
72
|
+
addQuery(params, "ruleId", opts.ruleId);
|
|
73
|
+
const raw = await this.client.get(`${datasetPath(pk, datasetName)}/last-rules-result${queryString(params)}`);
|
|
74
|
+
return this.client.safeParse(DataQualityRuleResultArraySchema, raw, "dataQuality.lastResults");
|
|
75
|
+
}
|
|
76
|
+
async history(datasetName, opts = {}) {
|
|
77
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
78
|
+
const params = new URLSearchParams();
|
|
79
|
+
addQuery(params, "minTimestamp", opts.minTimestamp);
|
|
80
|
+
addQuery(params, "maxTimestamp", opts.maxTimestamp);
|
|
81
|
+
addQuery(params, "resultsPerPage", opts.resultsPerPage);
|
|
82
|
+
addQuery(params, "page", opts.page);
|
|
83
|
+
addQuery(params, "ruleId", opts.ruleId);
|
|
84
|
+
const raw = await this.client.get(`${datasetPath(pk, datasetName)}/rules-history${queryString(params)}`);
|
|
85
|
+
return this.client.safeParse(DataQualityRuleResultArraySchema, raw, "dataQuality.history");
|
|
86
|
+
}
|
|
87
|
+
async computeRules(datasetName, opts = {}) {
|
|
88
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
89
|
+
const params = new URLSearchParams();
|
|
90
|
+
addQuery(params, "partition", partitionValue(opts.partition));
|
|
91
|
+
addQuery(params, "ruleId", opts.ruleId);
|
|
92
|
+
const raw = await this.client.post(`${datasetPath(pk, datasetName)}/actions/compute-rules${queryString(params)}`);
|
|
93
|
+
return this.client.safeParse(DataQualityComputeResultSchema, raw, "dataQuality.computeRules");
|
|
94
|
+
}
|
|
95
|
+
async computeRulesAndWait(datasetName, opts = {}) {
|
|
96
|
+
const future = await this.computeRules(datasetName, opts);
|
|
97
|
+
if (!future.jobId)
|
|
98
|
+
throw new Error("Data quality compute did not return a future jobId.");
|
|
99
|
+
return this.client.futures.wait(future.jobId, {
|
|
100
|
+
pollIntervalMs: opts.pollIntervalMs,
|
|
101
|
+
timeoutMs: opts.timeoutMs,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
async projectStatus(opts = {}) {
|
|
105
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
106
|
+
const params = new URLSearchParams();
|
|
107
|
+
addQuery(params, "onlyMonitored", opts.onlyMonitored);
|
|
108
|
+
const raw = await this.client.get(`${projectPath(pk)}/status${queryString(params)}`);
|
|
109
|
+
return this.client.safeParse(DataQualityProjectStatusSchema, raw, "dataQuality.projectStatus");
|
|
110
|
+
}
|
|
111
|
+
async projectTimeline(opts = {}) {
|
|
112
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
113
|
+
const params = new URLSearchParams();
|
|
114
|
+
addQuery(params, "minTimestamp", opts.minTimestamp);
|
|
115
|
+
addQuery(params, "maxTimestamp", opts.maxTimestamp);
|
|
116
|
+
const raw = await this.client.get(`${projectPath(pk)}/timeline${queryString(params)}`);
|
|
117
|
+
return this.client.safeParse(DataQualityTimelineSchema, raw, "dataQuality.projectTimeline");
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -289,7 +289,7 @@ function buildDatasetCreateBody(opts) {
|
|
|
289
289
|
type: opts.dsType,
|
|
290
290
|
params: {
|
|
291
291
|
connection: opts.connection,
|
|
292
|
-
path:
|
|
292
|
+
path: `/dataiku/${opts.projectKey}/${opts.datasetName}`,
|
|
293
293
|
},
|
|
294
294
|
formatType: opts.formatType ?? "csv",
|
|
295
295
|
formatParams: opts.formatParams ?? {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { FlowZone, FlowZoneCreateOptions, FlowZoneObjectType, FlowZoneUpdateOptions } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export type FlowZoneItemInput = {
|
|
4
|
+
objectId: string;
|
|
5
|
+
objectType: FlowZoneObjectType;
|
|
6
|
+
projectKey?: string;
|
|
7
|
+
};
|
|
8
|
+
export declare class FlowZonesResource extends BaseResource {
|
|
9
|
+
/** List all flow zones in a project. */
|
|
10
|
+
list(projectKey?: string): Promise<FlowZone[]>;
|
|
11
|
+
/** Get one flow zone by id. */
|
|
12
|
+
get(zoneId: string, projectKey?: string): Promise<FlowZone>;
|
|
13
|
+
/** Create a flow zone. */
|
|
14
|
+
create(opts: FlowZoneCreateOptions): Promise<FlowZone>;
|
|
15
|
+
/** Update flow zone settings such as name and color. */
|
|
16
|
+
update(zoneId: string, opts: FlowZoneUpdateOptions): Promise<FlowZone>;
|
|
17
|
+
/** Delete a flow zone. DSS moves its items back to the default zone. */
|
|
18
|
+
delete(zoneId: string, projectKey?: string): Promise<void>;
|
|
19
|
+
/** Move items into a flow zone. */
|
|
20
|
+
moveItems(zoneId: string, items: FlowZoneItemInput[], projectKey?: string): Promise<FlowZone>;
|
|
21
|
+
/** Move a single item into a flow zone. */
|
|
22
|
+
moveItem(zoneId: string, item: FlowZoneItemInput, projectKey?: string): Promise<FlowZone>;
|
|
23
|
+
/** Get the graph for a single flow zone. */
|
|
24
|
+
graph(zoneId: string, projectKey?: string): Promise<unknown>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { FlowZoneArraySchema, FlowZoneSchema, } from "../schemas.js";
|
|
2
|
+
import { deepMerge, } from "../utils/deep-merge.js";
|
|
3
|
+
import { BaseResource, } from "./base.js";
|
|
4
|
+
function normalizeZoneItem(item) {
|
|
5
|
+
return {
|
|
6
|
+
objectId: item.objectId,
|
|
7
|
+
objectType: item.objectType,
|
|
8
|
+
...(item.projectKey ? { projectKey: item.projectKey, } : {}),
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
export class FlowZonesResource extends BaseResource {
|
|
12
|
+
/** List all flow zones in a project. */
|
|
13
|
+
async list(projectKey) {
|
|
14
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones`);
|
|
15
|
+
return this.client.safeParse(FlowZoneArraySchema, raw, "flowZones.list");
|
|
16
|
+
}
|
|
17
|
+
/** Get one flow zone by id. */
|
|
18
|
+
async get(zoneId, projectKey) {
|
|
19
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`);
|
|
20
|
+
return this.client.safeParse(FlowZoneSchema, raw, "flowZones.get");
|
|
21
|
+
}
|
|
22
|
+
/** Create a flow zone. */
|
|
23
|
+
async create(opts) {
|
|
24
|
+
const raw = await this.client.post(`/public/api/projects/${this.enc(opts.projectKey)}/flow/zones`, {
|
|
25
|
+
name: opts.name,
|
|
26
|
+
color: opts.color ?? "#2ab1ac",
|
|
27
|
+
});
|
|
28
|
+
return this.client.safeParse(FlowZoneSchema, raw, "flowZones.create");
|
|
29
|
+
}
|
|
30
|
+
/** Update flow zone settings such as name and color. */
|
|
31
|
+
async update(zoneId, opts) {
|
|
32
|
+
const current = await this.get(zoneId, opts.projectKey);
|
|
33
|
+
const merged = deepMerge(current, {
|
|
34
|
+
...(opts.name !== undefined ? { name: opts.name, } : {}),
|
|
35
|
+
...(opts.color !== undefined ? { color: opts.color, } : {}),
|
|
36
|
+
});
|
|
37
|
+
await this.client.putVoid(`/public/api/projects/${this.enc(opts.projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`, merged);
|
|
38
|
+
return this.get(zoneId, opts.projectKey);
|
|
39
|
+
}
|
|
40
|
+
/** Delete a flow zone. DSS moves its items back to the default zone. */
|
|
41
|
+
async delete(zoneId, projectKey) {
|
|
42
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}`);
|
|
43
|
+
}
|
|
44
|
+
/** Move items into a flow zone. */
|
|
45
|
+
async moveItems(zoneId, items, projectKey) {
|
|
46
|
+
if (items.length === 0)
|
|
47
|
+
throw new Error("flowZones.moveItems requires at least one item");
|
|
48
|
+
const raw = await this.client.post(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}/add-items`, items.map(normalizeZoneItem));
|
|
49
|
+
return this.client.safeParse(FlowZoneSchema, raw, "flowZones.moveItems");
|
|
50
|
+
}
|
|
51
|
+
/** Move a single item into a flow zone. */
|
|
52
|
+
async moveItem(zoneId, item, projectKey) {
|
|
53
|
+
return this.moveItems(zoneId, [item,], projectKey);
|
|
54
|
+
}
|
|
55
|
+
/** Get the graph for a single flow zone. */
|
|
56
|
+
async graph(zoneId, projectKey) {
|
|
57
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/flow/zones/${encodeURIComponent(zoneId)}/graph`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import type { FolderDetails, FolderItem, FolderSummary } from "../schemas.js";
|
|
1
|
+
import type { FolderCreateOptions, FolderDetails, FolderItem, FolderSummary } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
3
|
export declare class FoldersResource extends BaseResource {
|
|
4
|
+
create(opts: FolderCreateOptions): Promise<FolderDetails>;
|
|
4
5
|
list(projectKey?: string): Promise<FolderSummary[]>;
|
|
5
6
|
resolveId(nameOrId: string, projectKey?: string): Promise<string>;
|
|
6
7
|
get(folderId: string, projectKey?: string): Promise<FolderDetails>;
|
|
8
|
+
update(folderId: string, data: Record<string, unknown>, projectKey?: string): Promise<void>;
|
|
7
9
|
contents(folderId: string, opts?: {
|
|
8
10
|
projectKey?: string;
|
|
9
11
|
}): Promise<FolderItem[]>;
|
|
@@ -13,4 +15,5 @@ export declare class FoldersResource extends BaseResource {
|
|
|
13
15
|
}): Promise<string>;
|
|
14
16
|
upload(folderId: string, path: string, localPath: string, projectKey?: string): Promise<void>;
|
|
15
17
|
deleteFile(folderId: string, path: string, projectKey?: string): Promise<void>;
|
|
18
|
+
delete(folderId: string, projectKey?: string): Promise<void>;
|
|
16
19
|
}
|
|
@@ -3,6 +3,7 @@ import { resolve, } from "node:path";
|
|
|
3
3
|
import { Readable, } from "node:stream";
|
|
4
4
|
import { pipeline, } from "node:stream/promises";
|
|
5
5
|
import { FolderDetailsSchema, FolderItemArraySchema, FolderSummaryArraySchema, } from "../schemas.js";
|
|
6
|
+
import { deepMerge, } from "../utils/deep-merge.js";
|
|
6
7
|
import { sanitizeFileName, } from "../utils/sanitize.js";
|
|
7
8
|
import { BaseResource, } from "./base.js";
|
|
8
9
|
function normalizeRemotePath(path) {
|
|
@@ -17,6 +18,21 @@ function inferDownloadFileName(remotePath) {
|
|
|
17
18
|
// Resource
|
|
18
19
|
// ---------------------------------------------------------------------------
|
|
19
20
|
export class FoldersResource extends BaseResource {
|
|
21
|
+
async create(opts) {
|
|
22
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
23
|
+
const path = opts.path?.trim() || `/dataiku/${pk}/${opts.name}`;
|
|
24
|
+
const raw = await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/managedfolders/`, {
|
|
25
|
+
name: opts.name,
|
|
26
|
+
projectKey: pk,
|
|
27
|
+
type: opts.type,
|
|
28
|
+
params: {
|
|
29
|
+
...opts.params,
|
|
30
|
+
connection: opts.connection,
|
|
31
|
+
path,
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
return this.client.safeParse(FolderDetailsSchema, raw, "folders.create");
|
|
35
|
+
}
|
|
20
36
|
async list(projectKey) {
|
|
21
37
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/managedfolders/`);
|
|
22
38
|
return this.client.safeParse(FolderSummaryArraySchema, raw, "folders.list");
|
|
@@ -34,6 +50,13 @@ export class FoldersResource extends BaseResource {
|
|
|
34
50
|
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/managedfolders/${fEnc}`);
|
|
35
51
|
return this.client.safeParse(FolderDetailsSchema, raw, "folders.get");
|
|
36
52
|
}
|
|
53
|
+
async update(folderId, data, projectKey) {
|
|
54
|
+
const fEnc = encodeURIComponent(folderId);
|
|
55
|
+
const pkEnc = this.enc(projectKey);
|
|
56
|
+
const current = await this.client.get(`/public/api/projects/${pkEnc}/managedfolders/${fEnc}`);
|
|
57
|
+
const merged = deepMerge(current, data);
|
|
58
|
+
await this.client.put(`/public/api/projects/${pkEnc}/managedfolders/${fEnc}`, merged);
|
|
59
|
+
}
|
|
37
60
|
async contents(folderId, opts) {
|
|
38
61
|
const fEnc = encodeURIComponent(folderId);
|
|
39
62
|
const response = await this.client.get(`/public/api/projects/${this.enc(opts?.projectKey)}/managedfolders/${fEnc}/contents/`);
|
|
@@ -63,4 +86,8 @@ export class FoldersResource extends BaseResource {
|
|
|
63
86
|
const pEnc = encodeURIComponent(normalizedPath);
|
|
64
87
|
return this.client.del(`/public/api/projects/${this.enc(projectKey)}/managedfolders/${fEnc}/contents/${pEnc}`);
|
|
65
88
|
}
|
|
89
|
+
delete(folderId, projectKey) {
|
|
90
|
+
const fEnc = encodeURIComponent(folderId);
|
|
91
|
+
return this.client.del(`/public/api/projects/${this.enc(projectKey)}/managedfolders/${fEnc}`);
|
|
92
|
+
}
|
|
66
93
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { FutureState, FutureWaitResult } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export interface FutureWaitOptions {
|
|
4
|
+
pollIntervalMs?: number;
|
|
5
|
+
timeoutMs?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class FuturesResource extends BaseResource {
|
|
8
|
+
get(futureId: string): Promise<FutureState>;
|
|
9
|
+
peek(futureId: string): Promise<FutureState>;
|
|
10
|
+
state(futureId: string, opts?: {
|
|
11
|
+
peek?: boolean;
|
|
12
|
+
}): Promise<FutureState>;
|
|
13
|
+
abort(futureId: string): Promise<void>;
|
|
14
|
+
wait(futureId: string, opts?: FutureWaitOptions): Promise<FutureWaitResult>;
|
|
15
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { FutureStateSchema, FutureWaitResultSchema, } from "../schemas.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
4
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
5
|
+
function sleep(ms) {
|
|
6
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
7
|
+
}
|
|
8
|
+
function isFinished(state) {
|
|
9
|
+
return state.hasResult === true
|
|
10
|
+
|| state.aborted === true
|
|
11
|
+
|| state.alive === false
|
|
12
|
+
|| state.unknown === true;
|
|
13
|
+
}
|
|
14
|
+
function waitState(state) {
|
|
15
|
+
if (state.hasResult === true)
|
|
16
|
+
return "DONE";
|
|
17
|
+
if (state.aborted === true)
|
|
18
|
+
return "ABORTED";
|
|
19
|
+
if (state.unknown === true)
|
|
20
|
+
return "UNKNOWN";
|
|
21
|
+
if (state.alive === false)
|
|
22
|
+
return "FAILED";
|
|
23
|
+
return "RUNNING";
|
|
24
|
+
}
|
|
25
|
+
export class FuturesResource extends BaseResource {
|
|
26
|
+
async get(futureId) {
|
|
27
|
+
return this.state(futureId, { peek: false, });
|
|
28
|
+
}
|
|
29
|
+
async peek(futureId) {
|
|
30
|
+
return this.state(futureId, { peek: true, });
|
|
31
|
+
}
|
|
32
|
+
async state(futureId, opts = {}) {
|
|
33
|
+
const params = new URLSearchParams();
|
|
34
|
+
params.set("peek", String(opts.peek === true));
|
|
35
|
+
const raw = await this.client.get(`/public/api/futures/${encodeURIComponent(futureId)}?${params.toString()}`);
|
|
36
|
+
return this.client.safeParse(FutureStateSchema, raw, "futures.state");
|
|
37
|
+
}
|
|
38
|
+
async abort(futureId) {
|
|
39
|
+
await this.client.del(`/public/api/futures/${encodeURIComponent(futureId)}`);
|
|
40
|
+
}
|
|
41
|
+
async wait(futureId, opts = {}) {
|
|
42
|
+
const baseIntervalMs = Math.max(1, opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
|
|
43
|
+
const timeoutMs = Math.max(baseIntervalMs, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
44
|
+
const startedAt = Date.now();
|
|
45
|
+
let pollCount = 0;
|
|
46
|
+
while (true) {
|
|
47
|
+
pollCount += 1;
|
|
48
|
+
const state = await this.get(futureId);
|
|
49
|
+
const elapsedMs = Date.now() - startedAt;
|
|
50
|
+
const status = waitState(state);
|
|
51
|
+
if (isFinished(state)) {
|
|
52
|
+
const result = {
|
|
53
|
+
futureId,
|
|
54
|
+
jobId: state.jobId,
|
|
55
|
+
state: status,
|
|
56
|
+
elapsedMs,
|
|
57
|
+
pollCount,
|
|
58
|
+
success: state.hasResult === true,
|
|
59
|
+
hasResult: state.hasResult === true,
|
|
60
|
+
alive: state.alive,
|
|
61
|
+
aborted: state.aborted,
|
|
62
|
+
unknown: state.unknown,
|
|
63
|
+
...(state.result !== undefined ? { result: state.result, } : {}),
|
|
64
|
+
};
|
|
65
|
+
return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
|
|
66
|
+
}
|
|
67
|
+
if (elapsedMs >= timeoutMs) {
|
|
68
|
+
const result = {
|
|
69
|
+
futureId,
|
|
70
|
+
jobId: state.jobId,
|
|
71
|
+
state: status,
|
|
72
|
+
elapsedMs,
|
|
73
|
+
pollCount,
|
|
74
|
+
success: false,
|
|
75
|
+
timedOut: true,
|
|
76
|
+
hasResult: state.hasResult === true,
|
|
77
|
+
alive: state.alive,
|
|
78
|
+
aborted: state.aborted,
|
|
79
|
+
unknown: state.unknown,
|
|
80
|
+
};
|
|
81
|
+
return this.client.safeParse(FutureWaitResultSchema, result, "futures.wait");
|
|
82
|
+
}
|
|
83
|
+
await sleep(baseIntervalMs);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { InsightDetails, InsightSummary } from "../schemas.js";
|
|
2
|
+
import { BaseResource } from "./base.js";
|
|
3
|
+
export interface InsightCreateOptions {
|
|
4
|
+
name?: string;
|
|
5
|
+
type?: string;
|
|
6
|
+
listed?: boolean;
|
|
7
|
+
params?: Record<string, unknown>;
|
|
8
|
+
data?: Record<string, unknown>;
|
|
9
|
+
contentType?: string;
|
|
10
|
+
payload?: string;
|
|
11
|
+
projectKey?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface InsightUpdateOptions {
|
|
14
|
+
name?: string;
|
|
15
|
+
listed?: boolean;
|
|
16
|
+
params?: Record<string, unknown>;
|
|
17
|
+
data?: Record<string, unknown>;
|
|
18
|
+
contentType?: string;
|
|
19
|
+
payload?: string;
|
|
20
|
+
projectKey?: string;
|
|
21
|
+
}
|
|
22
|
+
export declare class InsightsResource extends BaseResource {
|
|
23
|
+
list(projectKey?: string): Promise<InsightSummary[]>;
|
|
24
|
+
get(insightId: string, projectKey?: string): Promise<InsightDetails>;
|
|
25
|
+
create(opts: InsightCreateOptions): Promise<InsightDetails>;
|
|
26
|
+
update(insightId: string, opts: InsightUpdateOptions): Promise<InsightDetails>;
|
|
27
|
+
delete(insightId: string, projectKey?: string): Promise<void>;
|
|
28
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { InsightDetailsSchema, InsightSummaryArraySchema, } from "../schemas.js";
|
|
2
|
+
import { deepMerge, } from "../utils/deep-merge.js";
|
|
3
|
+
import { BaseResource, } from "./base.js";
|
|
4
|
+
function hasCreateShape(data) {
|
|
5
|
+
return typeof data.name === "string" && typeof data.type === "string";
|
|
6
|
+
}
|
|
7
|
+
function applyInsightFields(base, fields) {
|
|
8
|
+
const next = { ...base, };
|
|
9
|
+
if (fields.name !== undefined)
|
|
10
|
+
next.name = fields.name;
|
|
11
|
+
if (fields.type !== undefined)
|
|
12
|
+
next.type = fields.type;
|
|
13
|
+
if (fields.listed !== undefined)
|
|
14
|
+
next.listed = fields.listed;
|
|
15
|
+
if (fields.params !== undefined) {
|
|
16
|
+
const currentParams = next.params;
|
|
17
|
+
next.params = currentParams && typeof currentParams === "object" && !Array.isArray(currentParams)
|
|
18
|
+
? deepMerge(currentParams, fields.params)
|
|
19
|
+
: fields.params;
|
|
20
|
+
}
|
|
21
|
+
return next;
|
|
22
|
+
}
|
|
23
|
+
export class InsightsResource extends BaseResource {
|
|
24
|
+
async list(projectKey) {
|
|
25
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/insights/`);
|
|
26
|
+
return this.client.safeParse(InsightSummaryArraySchema, raw, "insights.list");
|
|
27
|
+
}
|
|
28
|
+
async get(insightId, projectKey) {
|
|
29
|
+
const raw = await this.client.get(`/public/api/projects/${this.enc(projectKey)}/insights/${encodeURIComponent(insightId)}/`);
|
|
30
|
+
return this.client.safeParse(InsightDetailsSchema, raw, "insights.get");
|
|
31
|
+
}
|
|
32
|
+
async create(opts) {
|
|
33
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
34
|
+
const prototype = applyInsightFields(opts.data ?? {}, opts);
|
|
35
|
+
if (!hasCreateShape(prototype)) {
|
|
36
|
+
throw new Error("Insight create requires name and type, either as options or in data.");
|
|
37
|
+
}
|
|
38
|
+
if (prototype.projectKey === undefined)
|
|
39
|
+
prototype.projectKey = pk;
|
|
40
|
+
const created = await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/insights/`, {
|
|
41
|
+
insightPrototype: prototype,
|
|
42
|
+
...(opts.contentType !== undefined ? { contentType: opts.contentType, } : {}),
|
|
43
|
+
...(opts.payload !== undefined ? { payload: opts.payload, } : {}),
|
|
44
|
+
});
|
|
45
|
+
if (typeof created.id !== "string" || created.id.length === 0) {
|
|
46
|
+
throw new Error("Insight create response did not include an id.");
|
|
47
|
+
}
|
|
48
|
+
return this.get(created.id, pk);
|
|
49
|
+
}
|
|
50
|
+
async update(insightId, opts) {
|
|
51
|
+
const current = await this.get(insightId, opts.projectKey);
|
|
52
|
+
const next = applyInsightFields(deepMerge(current, opts.data ?? {}), opts);
|
|
53
|
+
const pk = this.resolveProjectKey(opts.projectKey);
|
|
54
|
+
await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/insights/${encodeURIComponent(insightId)}/`, {
|
|
55
|
+
insight: next,
|
|
56
|
+
...(opts.contentType !== undefined ? { contentType: opts.contentType, } : {}),
|
|
57
|
+
...(opts.payload !== undefined ? { payload: opts.payload, } : {}),
|
|
58
|
+
});
|
|
59
|
+
return this.get(insightId, pk);
|
|
60
|
+
}
|
|
61
|
+
async delete(insightId, projectKey) {
|
|
62
|
+
await this.client.del(`/public/api/projects/${this.enc(projectKey)}/insights/${encodeURIComponent(insightId)}/`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { BuildMode, JobSummary, JobWaitResult } from "../schemas.js";
|
|
2
2
|
import { BaseResource } from "./base.js";
|
|
3
|
+
export type JobBuildTargetType = "DATASET" | "MANAGED_FOLDER";
|
|
3
4
|
interface ComputeNextPollDelayMsOptions {
|
|
4
5
|
pollCount: number;
|
|
5
6
|
baseIntervalMs: number;
|
|
@@ -27,21 +28,22 @@ export declare class JobsResource extends BaseResource {
|
|
|
27
28
|
projectKey?: string;
|
|
28
29
|
}): Promise<string>;
|
|
29
30
|
/**
|
|
30
|
-
* Start a dataset
|
|
31
|
+
* Start a build job for a dataset or managed folder.
|
|
31
32
|
* Returns the new job's ID.
|
|
32
33
|
*/
|
|
33
|
-
build(
|
|
34
|
+
build(targetId: string, opts?: {
|
|
34
35
|
buildMode?: BuildMode;
|
|
35
36
|
autoUpdateSchema?: boolean;
|
|
36
37
|
projectKey?: string;
|
|
38
|
+
targetType?: JobBuildTargetType;
|
|
37
39
|
}): Promise<{
|
|
38
40
|
jobId: string;
|
|
39
41
|
}>;
|
|
40
42
|
/**
|
|
41
|
-
* Build a dataset and wait for the job to reach a terminal state.
|
|
43
|
+
* Build a dataset or managed folder and wait for the job to reach a terminal state.
|
|
42
44
|
* Combines {@link build} then {@link wait}.
|
|
43
45
|
*/
|
|
44
|
-
buildAndWait(
|
|
46
|
+
buildAndWait(targetId: string, opts?: {
|
|
45
47
|
buildMode?: BuildMode;
|
|
46
48
|
autoUpdateSchema?: boolean;
|
|
47
49
|
activity?: string;
|
|
@@ -50,6 +52,7 @@ export declare class JobsResource extends BaseResource {
|
|
|
50
52
|
pollIntervalMs?: number;
|
|
51
53
|
timeoutMs?: number;
|
|
52
54
|
projectKey?: string;
|
|
55
|
+
targetType?: JobBuildTargetType;
|
|
53
56
|
}): Promise<JobWaitResult>;
|
|
54
57
|
/**
|
|
55
58
|
* Poll a job until it reaches a terminal state or times out.
|
|
@@ -69,31 +69,33 @@ export class JobsResource extends BaseResource {
|
|
|
69
69
|
return log;
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
|
-
* Start a dataset
|
|
72
|
+
* Start a build job for a dataset or managed folder.
|
|
73
73
|
* Returns the new job's ID.
|
|
74
74
|
*/
|
|
75
|
-
async build(
|
|
75
|
+
async build(targetId, opts) {
|
|
76
76
|
const pk = this.resolveProjectKey(opts?.projectKey);
|
|
77
77
|
const enc = encodeURIComponent(pk);
|
|
78
|
+
const targetType = opts?.targetType ?? "DATASET";
|
|
78
79
|
const jobDef = {
|
|
79
|
-
outputs: [{ projectKey: pk, id:
|
|
80
|
+
outputs: [{ projectKey: pk, id: targetId, type: targetType, },],
|
|
80
81
|
type: opts?.buildMode ?? "NON_RECURSIVE_FORCED_BUILD",
|
|
81
82
|
};
|
|
82
|
-
if (opts?.autoUpdateSchema) {
|
|
83
|
+
if (opts?.autoUpdateSchema && targetType === "DATASET") {
|
|
83
84
|
jobDef.autoUpdateSchemaBeforeEachRecipeRun = true;
|
|
84
85
|
}
|
|
85
86
|
const job = await this.client.post(`/public/api/projects/${enc}/jobs/`, jobDef);
|
|
86
87
|
return { jobId: job.id, };
|
|
87
88
|
}
|
|
88
89
|
/**
|
|
89
|
-
* Build a dataset and wait for the job to reach a terminal state.
|
|
90
|
+
* Build a dataset or managed folder and wait for the job to reach a terminal state.
|
|
90
91
|
* Combines {@link build} then {@link wait}.
|
|
91
92
|
*/
|
|
92
|
-
async buildAndWait(
|
|
93
|
-
const { jobId, } = await this.build(
|
|
93
|
+
async buildAndWait(targetId, opts) {
|
|
94
|
+
const { jobId, } = await this.build(targetId, {
|
|
94
95
|
buildMode: opts?.buildMode,
|
|
95
96
|
autoUpdateSchema: opts?.autoUpdateSchema,
|
|
96
97
|
projectKey: opts?.projectKey,
|
|
98
|
+
targetType: opts?.targetType,
|
|
97
99
|
});
|
|
98
100
|
return this.wait(jobId, {
|
|
99
101
|
activity: opts?.activity,
|
|
@@ -10,8 +10,9 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
10
10
|
/**
|
|
11
11
|
* Delete a Jupyter notebook.
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
13
|
+
* The SDK intentionally does not expose notebook creation as a first-class
|
|
14
|
+
* resource method yet; integration tests create disposable notebooks through
|
|
15
|
+
* the documented project-level create endpoint.
|
|
15
16
|
*/
|
|
16
17
|
deleteJupyter(name: string, projectKey?: string): Promise<void>;
|
|
17
18
|
/**
|
|
@@ -26,8 +27,9 @@ export declare class NotebooksResource extends BaseResource {
|
|
|
26
27
|
/**
|
|
27
28
|
* Unload (stop) a running Jupyter notebook session.
|
|
28
29
|
*
|
|
29
|
-
* DSS public APIs
|
|
30
|
-
*
|
|
30
|
+
* DSS public APIs expose session listing and unloading, but this SDK has no
|
|
31
|
+
* path to start a disposable kernel session; live unload coverage therefore
|
|
32
|
+
* requires an externally-started disposable session fixture.
|
|
31
33
|
*/
|
|
32
34
|
unloadJupyter(name: string, sessionId: string, projectKey?: string): Promise<void>;
|
|
33
35
|
/** List all SQL notebooks in a project. */
|
|
@@ -21,8 +21,9 @@ export class NotebooksResource extends BaseResource {
|
|
|
21
21
|
/**
|
|
22
22
|
* Delete a Jupyter notebook.
|
|
23
23
|
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
24
|
+
* The SDK intentionally does not expose notebook creation as a first-class
|
|
25
|
+
* resource method yet; integration tests create disposable notebooks through
|
|
26
|
+
* the documented project-level create endpoint.
|
|
26
27
|
*/
|
|
27
28
|
async deleteJupyter(name, projectKey) {
|
|
28
29
|
const nameEnc = encodeURIComponent(name);
|
|
@@ -52,8 +53,9 @@ export class NotebooksResource extends BaseResource {
|
|
|
52
53
|
/**
|
|
53
54
|
* Unload (stop) a running Jupyter notebook session.
|
|
54
55
|
*
|
|
55
|
-
* DSS public APIs
|
|
56
|
-
*
|
|
56
|
+
* DSS public APIs expose session listing and unloading, but this SDK has no
|
|
57
|
+
* path to start a disposable kernel session; live unload coverage therefore
|
|
58
|
+
* requires an externally-started disposable session fixture.
|
|
57
59
|
*/
|
|
58
60
|
async unloadJupyter(name, sessionId, projectKey) {
|
|
59
61
|
const nameEnc = encodeURIComponent(name);
|