dataiku-sdk 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/packages/types/src/index.d.ts +4 -0
- package/dist/packages/types/src/index.js +4 -0
- package/dist/src/cli/commands/analysis.d.ts +2 -0
- package/dist/src/cli/commands/analysis.js +66 -0
- package/dist/src/cli/commands/api-service.js +19 -0
- package/dist/src/cli/commands/index.js +8 -0
- package/dist/src/cli/commands/ml-task.d.ts +2 -0
- package/dist/src/cli/commands/ml-task.js +171 -0
- package/dist/src/cli/commands/model-evaluation-store.d.ts +2 -0
- package/dist/src/cli/commands/model-evaluation-store.js +83 -0
- package/dist/src/cli/commands/project.js +2 -1
- package/dist/src/cli/commands/recipe.js +29 -2
- package/dist/src/cli/commands/saved-model.d.ts +2 -0
- package/dist/src/cli/commands/saved-model.js +85 -0
- package/dist/src/cli/contract.js +11 -1
- package/dist/src/cli/flags.js +17 -0
- package/dist/src/cli/helpers/cleanup.js +47 -1
- package/dist/src/cli/usage.js +4 -0
- package/dist/src/client.d.ts +12 -0
- package/dist/src/client.js +20 -0
- package/dist/src/index.d.ts +4 -0
- package/dist/src/index.js +4 -0
- package/dist/src/resources/analyses.d.ts +34 -0
- package/dist/src/resources/analyses.js +31 -0
- package/dist/src/resources/api-services.d.ts +2 -0
- package/dist/src/resources/api-services.js +10 -0
- package/dist/src/resources/datasets.js +7 -5
- package/dist/src/resources/ml-tasks.d.ts +91 -0
- package/dist/src/resources/ml-tasks.js +130 -0
- package/dist/src/resources/model-evaluation-stores.d.ts +41 -0
- package/dist/src/resources/model-evaluation-stores.js +36 -0
- package/dist/src/resources/recipes.js +148 -31
- package/dist/src/resources/saved-models.d.ts +41 -0
- package/dist/src/resources/saved-models.js +40 -0
- package/package.json +1 -1
- package/packages/types/dist/index.d.ts +4 -0
- package/packages/types/dist/index.js +4 -0
package/dist/src/cli/flags.js
CHANGED
|
@@ -37,6 +37,7 @@ export const BOOLEAN_FLAGS = new Set([
|
|
|
37
37
|
"keep",
|
|
38
38
|
"full-log",
|
|
39
39
|
"drop-data",
|
|
40
|
+
"normalize",
|
|
40
41
|
]);
|
|
41
42
|
export const SHORT_FLAGS = {
|
|
42
43
|
v: "verbose",
|
|
@@ -66,6 +67,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
66
67
|
"api-key",
|
|
67
68
|
"build-mode",
|
|
68
69
|
"backup-dir",
|
|
70
|
+
"backend-type",
|
|
69
71
|
"backup",
|
|
70
72
|
"ca-cert",
|
|
71
73
|
"catalog",
|
|
@@ -86,12 +88,19 @@ export const VALUE_FLAGS = new Set([
|
|
|
86
88
|
"file",
|
|
87
89
|
"env",
|
|
88
90
|
"install-core-packages",
|
|
91
|
+
"fuzzy-on",
|
|
92
|
+
"fuzzy-distance",
|
|
93
|
+
"guess-policy",
|
|
94
|
+
"fuzzy-threshold",
|
|
89
95
|
"folder",
|
|
90
96
|
"input",
|
|
97
|
+
"input-dataset",
|
|
91
98
|
"from",
|
|
92
99
|
"knowledge-bank",
|
|
93
100
|
"labeling-task",
|
|
94
101
|
"lang",
|
|
102
|
+
"join-on",
|
|
103
|
+
"join-type",
|
|
95
104
|
"package",
|
|
96
105
|
"packages",
|
|
97
106
|
"local",
|
|
@@ -110,6 +119,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
110
119
|
"log-filter",
|
|
111
120
|
"log-id",
|
|
112
121
|
"model-evaluation-store",
|
|
122
|
+
"model-name",
|
|
113
123
|
"name",
|
|
114
124
|
"object",
|
|
115
125
|
"metastore-table",
|
|
@@ -122,6 +132,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
122
132
|
"parent",
|
|
123
133
|
"path",
|
|
124
134
|
"preview",
|
|
135
|
+
"prediction-type",
|
|
125
136
|
"project-key",
|
|
126
137
|
"recipe",
|
|
127
138
|
"request-timeout",
|
|
@@ -138,6 +149,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
138
149
|
"replace-payload-text",
|
|
139
150
|
"retain",
|
|
140
151
|
"saved-model",
|
|
152
|
+
"session-name",
|
|
141
153
|
"sql",
|
|
142
154
|
"schema",
|
|
143
155
|
"sql-file",
|
|
@@ -145,6 +157,9 @@ export const VALUE_FLAGS = new Set([
|
|
|
145
157
|
"state",
|
|
146
158
|
"streaming-endpoint",
|
|
147
159
|
"target",
|
|
160
|
+
"task-type",
|
|
161
|
+
"test-dataset",
|
|
162
|
+
"train-dataset",
|
|
148
163
|
"target-type",
|
|
149
164
|
"timeout",
|
|
150
165
|
"table",
|
|
@@ -158,7 +173,9 @@ export const VALUE_FLAGS = new Set([
|
|
|
158
173
|
export const REPEATABLE_VALUE_FLAGS = new Set([
|
|
159
174
|
"dataset",
|
|
160
175
|
"folder",
|
|
176
|
+
"fuzzy-on",
|
|
161
177
|
"input",
|
|
178
|
+
"join-on",
|
|
162
179
|
"object",
|
|
163
180
|
"package",
|
|
164
181
|
"recipe",
|
|
@@ -3,7 +3,8 @@ export function projectArg(projectKey) {
|
|
|
3
3
|
return projectKey ? ["--project-key", projectKey,] : [];
|
|
4
4
|
}
|
|
5
5
|
export function cleanupLedgerEntry(resource, action, args, flags, result, projectKey) {
|
|
6
|
-
if (!(action.startsWith("create") || action === "clone" || action === "
|
|
6
|
+
if (!(action.startsWith("create") || action === "clone" || action === "duplicate"
|
|
7
|
+
|| action === "upload"))
|
|
7
8
|
return undefined;
|
|
8
9
|
const record = resultRecord(result);
|
|
9
10
|
if (record.skipped !== undefined)
|
|
@@ -122,6 +123,39 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
122
123
|
cleanup: { argv: ["insight", "delete", id, "--if-exists", ...withProject,], },
|
|
123
124
|
};
|
|
124
125
|
}
|
|
126
|
+
case "analysis.create": {
|
|
127
|
+
const id = stringField(record, ["created", "id",]);
|
|
128
|
+
if (!id)
|
|
129
|
+
return undefined;
|
|
130
|
+
return {
|
|
131
|
+
...base,
|
|
132
|
+
id,
|
|
133
|
+
cleanup: { argv: ["analysis", "delete", id, "--if-exists", ...withProject,], },
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
case "model-evaluation-store.create": {
|
|
137
|
+
const id = stringField(record, ["created", "id",]);
|
|
138
|
+
if (!id)
|
|
139
|
+
return undefined;
|
|
140
|
+
return {
|
|
141
|
+
...base,
|
|
142
|
+
id,
|
|
143
|
+
cleanup: {
|
|
144
|
+
argv: ["model-evaluation-store", "delete", id, "--if-exists", ...withProject,],
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
case "ml-task.create": {
|
|
149
|
+
const analysisId = args[0] || stringField(record, ["analysisId",]);
|
|
150
|
+
const taskId = stringField(record, ["created", "mlTaskId", "id",]);
|
|
151
|
+
if (!analysisId || !taskId)
|
|
152
|
+
return undefined;
|
|
153
|
+
return {
|
|
154
|
+
...base,
|
|
155
|
+
id: taskId,
|
|
156
|
+
cleanup: { argv: ["ml-task", "delete", analysisId, taskId, ...withProject,], },
|
|
157
|
+
};
|
|
158
|
+
}
|
|
125
159
|
case "data-quality.create-rule": {
|
|
126
160
|
const ruleId = stringField(record, ["id", "created",]);
|
|
127
161
|
if (!ruleId)
|
|
@@ -145,6 +179,18 @@ export function cleanupLedgerEntry(resource, action, args, flags, result, projec
|
|
|
145
179
|
cleanup: { argv: ["code-env", "delete", lang, name, "--if-exists",], },
|
|
146
180
|
};
|
|
147
181
|
}
|
|
182
|
+
case "project.duplicate": {
|
|
183
|
+
const targetKey = args[1];
|
|
184
|
+
if (!targetKey)
|
|
185
|
+
return undefined;
|
|
186
|
+
return {
|
|
187
|
+
ts,
|
|
188
|
+
action,
|
|
189
|
+
resource,
|
|
190
|
+
name: targetKey,
|
|
191
|
+
cleanup: { argv: ["project", "delete", targetKey, "--drop-data",], },
|
|
192
|
+
};
|
|
193
|
+
}
|
|
148
194
|
case "folder.upload":
|
|
149
195
|
return {
|
|
150
196
|
...base,
|
package/dist/src/cli/usage.js
CHANGED
package/dist/src/client.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Static, type TSchema } from "@sinclair/typebox";
|
|
2
|
+
import { AnalysesResource } from "./resources/analyses.js";
|
|
2
3
|
import { ApiDeployerResource } from "./resources/api-deployer.js";
|
|
3
4
|
import { ApiServicesResource } from "./resources/api-services.js";
|
|
4
5
|
import { ApplicationsResource } from "./resources/applications.js";
|
|
@@ -17,10 +18,13 @@ import { InsightsResource } from "./resources/insights.js";
|
|
|
17
18
|
import { JobsResource } from "./resources/jobs.js";
|
|
18
19
|
import { MeaningsResource } from "./resources/meanings.js";
|
|
19
20
|
import { MetricsResource } from "./resources/metrics.js";
|
|
21
|
+
import { MlTasksResource } from "./resources/ml-tasks.js";
|
|
22
|
+
import { ModelEvaluationStoresResource } from "./resources/model-evaluation-stores.js";
|
|
20
23
|
import { NotebooksResource } from "./resources/notebooks.js";
|
|
21
24
|
import { ProjectLibraryResource } from "./resources/project-library.js";
|
|
22
25
|
import { ProjectsResource } from "./resources/projects.js";
|
|
23
26
|
import { RecipesResource } from "./resources/recipes.js";
|
|
27
|
+
import { SavedModelsResource } from "./resources/saved-models.js";
|
|
24
28
|
import { ScenariosResource } from "./resources/scenarios.js";
|
|
25
29
|
import { SqlResource } from "./resources/sql.js";
|
|
26
30
|
import { StatisticsResource } from "./resources/statistics.js";
|
|
@@ -107,6 +111,10 @@ export declare class DataikuClient {
|
|
|
107
111
|
private workspacesResource?;
|
|
108
112
|
private metricsResource?;
|
|
109
113
|
private meaningsResource?;
|
|
114
|
+
private analysesResource?;
|
|
115
|
+
private mlTasksResource?;
|
|
116
|
+
private savedModelsResource?;
|
|
117
|
+
private modelEvaluationStoresResource?;
|
|
110
118
|
get projects(): ProjectsResource;
|
|
111
119
|
get datasets(): DatasetsResource;
|
|
112
120
|
get dashboards(): DashboardsResource;
|
|
@@ -138,6 +146,10 @@ export declare class DataikuClient {
|
|
|
138
146
|
get workspaces(): WorkspacesResource;
|
|
139
147
|
get metrics(): MetricsResource;
|
|
140
148
|
get meanings(): MeaningsResource;
|
|
149
|
+
get analyses(): AnalysesResource;
|
|
150
|
+
get mlTasks(): MlTasksResource;
|
|
151
|
+
get savedModels(): SavedModelsResource;
|
|
152
|
+
get modelEvaluationStores(): ModelEvaluationStoresResource;
|
|
141
153
|
constructor(config?: DataikuClientConfig);
|
|
142
154
|
getRequestTimeoutMs(): number;
|
|
143
155
|
resolveProjectKey(paramValue?: string): string;
|
package/dist/src/client.js
CHANGED
|
@@ -3,6 +3,7 @@ import { getCACertificates, } from "node:tls";
|
|
|
3
3
|
import { Value, } from "@sinclair/typebox/value";
|
|
4
4
|
import { safeParseSchema, } from "./schemas.js";
|
|
5
5
|
import { classifyDataikuError, DataikuError, } from "./errors.js";
|
|
6
|
+
import { AnalysesResource, } from "./resources/analyses.js";
|
|
6
7
|
import { ApiDeployerResource, } from "./resources/api-deployer.js";
|
|
7
8
|
import { ApiServicesResource, } from "./resources/api-services.js";
|
|
8
9
|
import { ApplicationsResource, } from "./resources/applications.js";
|
|
@@ -21,10 +22,13 @@ import { InsightsResource, } from "./resources/insights.js";
|
|
|
21
22
|
import { JobsResource, } from "./resources/jobs.js";
|
|
22
23
|
import { MeaningsResource, } from "./resources/meanings.js";
|
|
23
24
|
import { MetricsResource, } from "./resources/metrics.js";
|
|
25
|
+
import { MlTasksResource, } from "./resources/ml-tasks.js";
|
|
26
|
+
import { ModelEvaluationStoresResource, } from "./resources/model-evaluation-stores.js";
|
|
24
27
|
import { NotebooksResource, } from "./resources/notebooks.js";
|
|
25
28
|
import { ProjectLibraryResource, } from "./resources/project-library.js";
|
|
26
29
|
import { ProjectsResource, } from "./resources/projects.js";
|
|
27
30
|
import { RecipesResource, } from "./resources/recipes.js";
|
|
31
|
+
import { SavedModelsResource, } from "./resources/saved-models.js";
|
|
28
32
|
import { ScenariosResource, } from "./resources/scenarios.js";
|
|
29
33
|
import { SqlResource, } from "./resources/sql.js";
|
|
30
34
|
import { StatisticsResource, } from "./resources/statistics.js";
|
|
@@ -176,6 +180,10 @@ export class DataikuClient {
|
|
|
176
180
|
workspacesResource;
|
|
177
181
|
metricsResource;
|
|
178
182
|
meaningsResource;
|
|
183
|
+
analysesResource;
|
|
184
|
+
mlTasksResource;
|
|
185
|
+
savedModelsResource;
|
|
186
|
+
modelEvaluationStoresResource;
|
|
179
187
|
get projects() {
|
|
180
188
|
return (this.projectsResource ??= new ProjectsResource(this));
|
|
181
189
|
}
|
|
@@ -269,6 +277,18 @@ export class DataikuClient {
|
|
|
269
277
|
get meanings() {
|
|
270
278
|
return (this.meaningsResource ??= new MeaningsResource(this));
|
|
271
279
|
}
|
|
280
|
+
get analyses() {
|
|
281
|
+
return (this.analysesResource ??= new AnalysesResource(this));
|
|
282
|
+
}
|
|
283
|
+
get mlTasks() {
|
|
284
|
+
return (this.mlTasksResource ??= new MlTasksResource(this));
|
|
285
|
+
}
|
|
286
|
+
get savedModels() {
|
|
287
|
+
return (this.savedModelsResource ??= new SavedModelsResource(this));
|
|
288
|
+
}
|
|
289
|
+
get modelEvaluationStores() {
|
|
290
|
+
return (this.modelEvaluationStoresResource ??= new ModelEvaluationStoresResource(this));
|
|
291
|
+
}
|
|
272
292
|
constructor(config) {
|
|
273
293
|
const envUrl = process.env["DATAIKU_URL"]?.trim();
|
|
274
294
|
const envApiKey = process.env["DATAIKU_API_KEY"]?.trim();
|
package/dist/src/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ export { DataikuClient, type DataikuClientConfig, type DataikuClientTraceEvent,
|
|
|
2
2
|
export { type CredentialValidationOptions, type CredentialValidationResult, validateCredentials, } from "./auth.js";
|
|
3
3
|
export { type DssCredentials, getConfigDir, getCredentialsPath, loadCredentials, saveCredentials, } from "./config.js";
|
|
4
4
|
export { ClientValidationError, DataikuError, type DataikuErrorCategory, type DataikuErrorTaxonomy, type DataikuRetryMetadata, } from "./errors.js";
|
|
5
|
+
export * from "./resources/analyses.js";
|
|
5
6
|
export { ApiDeployerResource, } from "./resources/api-deployer.js";
|
|
6
7
|
export { ApiServicesResource, } from "./resources/api-services.js";
|
|
7
8
|
export { ApplicationsResource, } from "./resources/applications.js";
|
|
@@ -20,10 +21,13 @@ export { InsightsResource, } from "./resources/insights.js";
|
|
|
20
21
|
export { computeNextPollDelayMs, type JobBuildAndWaitOptions, type JobBuildOptions, type JobBuildTarget, type JobBuildTargetType, type JobLogFilter, type JobLogProgress, type JobLogSummary, JobsResource, parseJobLogProgress, } from "./resources/jobs.js";
|
|
21
22
|
export { MeaningsResource, } from "./resources/meanings.js";
|
|
22
23
|
export { MetricsResource, } from "./resources/metrics.js";
|
|
24
|
+
export * from "./resources/ml-tasks.js";
|
|
25
|
+
export * from "./resources/model-evaluation-stores.js";
|
|
23
26
|
export { NotebooksResource, } from "./resources/notebooks.js";
|
|
24
27
|
export { ProjectLibraryResource, } from "./resources/project-library.js";
|
|
25
28
|
export { type FlowMapResult, ProjectsResource, } from "./resources/projects.js";
|
|
26
29
|
export { type RecipeCloneOptions, type RecipeCloneResult, type RecipeGraphReference, type RecipeGraphValidationResult, type RecipeRunOptions, type RecipeRunOutput, type RecipeRunResult, RecipesResource, } from "./resources/recipes.js";
|
|
30
|
+
export * from "./resources/saved-models.js";
|
|
27
31
|
export { normalizeScenarioUpdateData, SCENARIO_CANONICAL_EDITABLE_FIELDS, type ScenarioFieldChange, type ScenarioFieldMismatch, type ScenarioScriptRunResult, ScenariosResource, type ScenarioUpdateNormalization, type ScenarioUpdatePreview, scenarioUpdatePreview, type ScenarioUpdateResult, } from "./resources/scenarios.js";
|
|
28
32
|
export { SqlResource, } from "./resources/sql.js";
|
|
29
33
|
export { StatisticsResource, } from "./resources/statistics.js";
|
package/dist/src/index.js
CHANGED
|
@@ -6,6 +6,7 @@ export { getConfigDir, getCredentialsPath, loadCredentials, saveCredentials, } f
|
|
|
6
6
|
// Errors
|
|
7
7
|
export { ClientValidationError, DataikuError, } from "./errors.js";
|
|
8
8
|
// Resources (for advanced use / extension)
|
|
9
|
+
export * from "./resources/analyses.js";
|
|
9
10
|
export { ApiDeployerResource, } from "./resources/api-deployer.js";
|
|
10
11
|
export { ApiServicesResource, } from "./resources/api-services.js";
|
|
11
12
|
export { ApplicationsResource, } from "./resources/applications.js";
|
|
@@ -24,10 +25,13 @@ export { InsightsResource, } from "./resources/insights.js";
|
|
|
24
25
|
export { computeNextPollDelayMs, JobsResource, parseJobLogProgress, } from "./resources/jobs.js";
|
|
25
26
|
export { MeaningsResource, } from "./resources/meanings.js";
|
|
26
27
|
export { MetricsResource, } from "./resources/metrics.js";
|
|
28
|
+
export * from "./resources/ml-tasks.js";
|
|
29
|
+
export * from "./resources/model-evaluation-stores.js";
|
|
27
30
|
export { NotebooksResource, } from "./resources/notebooks.js";
|
|
28
31
|
export { ProjectLibraryResource, } from "./resources/project-library.js";
|
|
29
32
|
export { ProjectsResource, } from "./resources/projects.js";
|
|
30
33
|
export { RecipesResource, } from "./resources/recipes.js";
|
|
34
|
+
export * from "./resources/saved-models.js";
|
|
31
35
|
export { normalizeScenarioUpdateData, SCENARIO_CANONICAL_EDITABLE_FIELDS, ScenariosResource, scenarioUpdatePreview, } from "./resources/scenarios.js";
|
|
32
36
|
export { SqlResource, } from "./resources/sql.js";
|
|
33
37
|
export { StatisticsResource, } from "./resources/statistics.js";
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { BaseResource } from "./base.js";
|
|
2
|
+
export interface AnalysisListItem extends Record<string, unknown> {
|
|
3
|
+
id?: string;
|
|
4
|
+
analysisId?: string;
|
|
5
|
+
name?: string;
|
|
6
|
+
inputDataset?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface AnalysisDefinition extends Record<string, unknown> {
|
|
9
|
+
id?: string;
|
|
10
|
+
analysisId?: string;
|
|
11
|
+
name?: string;
|
|
12
|
+
projectKey?: string;
|
|
13
|
+
inputDataset?: string;
|
|
14
|
+
script?: Record<string, unknown>;
|
|
15
|
+
}
|
|
16
|
+
export interface AnalysisCreateOptions {
|
|
17
|
+
inputDataset: string;
|
|
18
|
+
projectKey?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface AnalysisCreateResult extends Record<string, unknown> {
|
|
21
|
+
id?: string;
|
|
22
|
+
analysisId?: string;
|
|
23
|
+
}
|
|
24
|
+
export declare class AnalysesResource extends BaseResource {
|
|
25
|
+
/** List visual analyses in a project. */
|
|
26
|
+
list(projectKey?: string): Promise<AnalysisListItem[]>;
|
|
27
|
+
/** Get the definition of a visual analysis. */
|
|
28
|
+
get(analysisId: string, projectKey?: string): Promise<AnalysisDefinition>;
|
|
29
|
+
/** Create a visual analysis for an input dataset. */
|
|
30
|
+
create(opts: AnalysisCreateOptions): Promise<AnalysisCreateResult>;
|
|
31
|
+
/** Delete a visual analysis. */
|
|
32
|
+
delete(analysisId: string, projectKey?: string): Promise<void>;
|
|
33
|
+
private analysisPath;
|
|
34
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { UsageError, } from "../cli/usage.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
function requireNonEmpty(value, name) {
|
|
4
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
5
|
+
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
6
|
+
}
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
export class AnalysesResource extends BaseResource {
|
|
10
|
+
/** List visual analyses in a project. */
|
|
11
|
+
async list(projectKey) {
|
|
12
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/lab/`);
|
|
13
|
+
}
|
|
14
|
+
/** Get the definition of a visual analysis. */
|
|
15
|
+
async get(analysisId, projectKey) {
|
|
16
|
+
return this.client.get(`${this.analysisPath(analysisId, projectKey)}/`);
|
|
17
|
+
}
|
|
18
|
+
/** Create a visual analysis for an input dataset. */
|
|
19
|
+
async create(opts) {
|
|
20
|
+
const inputDataset = requireNonEmpty(opts.inputDataset, "inputDataset");
|
|
21
|
+
return this.client.post(`/public/api/projects/${this.enc(opts.projectKey)}/lab/`, { inputDataset, });
|
|
22
|
+
}
|
|
23
|
+
/** Delete a visual analysis. */
|
|
24
|
+
async delete(analysisId, projectKey) {
|
|
25
|
+
await this.client.del(`${this.analysisPath(analysisId, projectKey)}/`);
|
|
26
|
+
}
|
|
27
|
+
analysisPath(analysisId, projectKey) {
|
|
28
|
+
const id = encodeURIComponent(requireNonEmpty(analysisId, "analysisId"));
|
|
29
|
+
return `/public/api/projects/${this.enc(projectKey)}/lab/${id}`;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -19,6 +19,8 @@ export declare class ApiServicesResource extends BaseResource {
|
|
|
19
19
|
getSettings(serviceId: string, projectKey?: string): Promise<Record<string, unknown>>;
|
|
20
20
|
/** Save API service settings. */
|
|
21
21
|
saveSettings(serviceId: string, body: Record<string, unknown>, projectKey?: string): Promise<ApiServiceActionResult>;
|
|
22
|
+
/** Add a standard prediction endpoint backed by a saved model. */
|
|
23
|
+
addPredictionEndpoint(serviceId: string, endpointId: string, savedModelId: string, projectKey?: string): Promise<ApiServiceActionResult>;
|
|
22
24
|
/** List deployable packages for an API service. */
|
|
23
25
|
listPackages(serviceId: string, projectKey?: string): Promise<ApiServicePackageListItem[]>;
|
|
24
26
|
/** Get an API service package summary. */
|
|
@@ -16,6 +16,16 @@ export class ApiServicesResource extends BaseResource {
|
|
|
16
16
|
async saveSettings(serviceId, body, projectKey) {
|
|
17
17
|
return this.client.put(`${this.servicePath(serviceId, projectKey)}/settings`, body);
|
|
18
18
|
}
|
|
19
|
+
/** Add a standard prediction endpoint backed by a saved model. */
|
|
20
|
+
async addPredictionEndpoint(serviceId, endpointId, savedModelId, projectKey) {
|
|
21
|
+
const settings = await this.getSettings(serviceId, projectKey);
|
|
22
|
+
const endpoints = Array.isArray(settings.endpoints) ? settings.endpoints : [];
|
|
23
|
+
settings.endpoints = [
|
|
24
|
+
...endpoints,
|
|
25
|
+
{ id: endpointId, type: "STD_PREDICTION", modelRef: savedModelId, },
|
|
26
|
+
];
|
|
27
|
+
return this.saveSettings(serviceId, settings, projectKey);
|
|
28
|
+
}
|
|
19
29
|
/** List deployable packages for an API service. */
|
|
20
30
|
async listPackages(serviceId, projectKey) {
|
|
21
31
|
return this.client.get(`${this.servicePath(serviceId, projectKey)}/packages`);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as nodePath from "node:path";
|
|
3
3
|
import { Readable, Transform, } from "node:stream";
|
|
4
4
|
import { pipeline, } from "node:stream/promises";
|
|
5
5
|
import { createGzip, } from "node:zlib";
|
|
@@ -297,6 +297,7 @@ function buildDatasetCreateBody(opts) {
|
|
|
297
297
|
}
|
|
298
298
|
const DATASET_CLONE_PARAM_KEYS = [
|
|
299
299
|
"connection",
|
|
300
|
+
"uploadConnection",
|
|
300
301
|
"path",
|
|
301
302
|
"table",
|
|
302
303
|
"schema",
|
|
@@ -418,8 +419,8 @@ export class DatasetsResource extends BaseResource {
|
|
|
418
419
|
const res = await this.client.stream(`/public/api/projects/${this.enc(opts?.projectKey)}/datasets/${dsEnc}/data/?format=tsv-excel-header&limit=${limit + 1}`);
|
|
419
420
|
const safeDatasetName = sanitizeFileName(datasetName, "dataset");
|
|
420
421
|
const filePath = opts?.outputPath?.endsWith(".gz") || opts?.outputPath?.endsWith(".csv")
|
|
421
|
-
? resolve(opts.outputPath)
|
|
422
|
-
: resolve(opts?.outputPath ?? process.cwd(), `${safeDatasetName}.csv.gz`);
|
|
422
|
+
? nodePath.resolve(opts.outputPath)
|
|
423
|
+
: nodePath.resolve(opts?.outputPath ?? process.cwd(), `${safeDatasetName}.csv.gz`);
|
|
423
424
|
const onHeader = opts?.validateColumns
|
|
424
425
|
? (headerRow) => {
|
|
425
426
|
const warnings = validateStreamColumns(headerRow, opts.validateColumns);
|
|
@@ -432,7 +433,8 @@ export class DatasetsResource extends BaseResource {
|
|
|
432
433
|
const stats = { rows: 0, truncated: false, };
|
|
433
434
|
const nodeStream = Readable.fromWeb(res.body);
|
|
434
435
|
const csvTransform = tsvToCsvTransform(limit, stats, onHeader);
|
|
435
|
-
|
|
436
|
+
fs.mkdirSync(nodePath.dirname(filePath), { recursive: true, });
|
|
437
|
+
const fileOut = fs.createWriteStream(filePath);
|
|
436
438
|
if (shouldGzip) {
|
|
437
439
|
const gzip = createGzip();
|
|
438
440
|
await pipeline(nodeStream, csvTransform, gzip, fileOut);
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { BaseResource } from "./base.js";
|
|
2
|
+
export type MlTaskType = "PREDICTION" | "CLUSTERING";
|
|
3
|
+
export interface MlTaskCreateOptions {
|
|
4
|
+
analysisId: string;
|
|
5
|
+
taskType: MlTaskType;
|
|
6
|
+
targetVariable?: string;
|
|
7
|
+
predictionType?: string;
|
|
8
|
+
backendType?: string;
|
|
9
|
+
guessPolicy?: string;
|
|
10
|
+
projectKey?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface MlTaskCreateResult extends Record<string, unknown> {
|
|
13
|
+
analysisId?: string;
|
|
14
|
+
mlTaskId?: string;
|
|
15
|
+
}
|
|
16
|
+
export interface MlTaskFullModelId extends Record<string, unknown> {
|
|
17
|
+
id?: string;
|
|
18
|
+
fullModelId?: {
|
|
19
|
+
sessionId?: string;
|
|
20
|
+
[key: string]: unknown;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export interface MlTaskStatus extends Record<string, unknown> {
|
|
24
|
+
guessing?: boolean;
|
|
25
|
+
training?: boolean;
|
|
26
|
+
fullModelIds?: MlTaskFullModelId[];
|
|
27
|
+
}
|
|
28
|
+
export interface MlTaskSettings extends Record<string, unknown> {
|
|
29
|
+
taskType?: MlTaskType;
|
|
30
|
+
predictionType?: string;
|
|
31
|
+
}
|
|
32
|
+
export interface MlTaskActionResult extends Record<string, unknown> {
|
|
33
|
+
message?: string;
|
|
34
|
+
}
|
|
35
|
+
export interface MlTaskTrainOptions {
|
|
36
|
+
analysisId: string;
|
|
37
|
+
mlTaskId: string;
|
|
38
|
+
sessionName?: string;
|
|
39
|
+
sessionDescription?: string;
|
|
40
|
+
runQueue?: boolean;
|
|
41
|
+
wait?: boolean;
|
|
42
|
+
projectKey?: string;
|
|
43
|
+
}
|
|
44
|
+
export interface MlTaskTrainingSession extends Record<string, unknown> {
|
|
45
|
+
sessionId: string;
|
|
46
|
+
}
|
|
47
|
+
export interface MlTaskTrainCompletedResult extends Record<string, unknown> {
|
|
48
|
+
sessionId: string;
|
|
49
|
+
trainedModelIds: string[];
|
|
50
|
+
}
|
|
51
|
+
export type MlTaskTrainResult = MlTaskTrainingSession | MlTaskTrainCompletedResult;
|
|
52
|
+
export interface MlTrainedModelDetails extends Record<string, unknown> {
|
|
53
|
+
id?: string;
|
|
54
|
+
algorithm?: string;
|
|
55
|
+
predictionType?: string;
|
|
56
|
+
}
|
|
57
|
+
export interface MlTaskDeployOptions {
|
|
58
|
+
analysisId: string;
|
|
59
|
+
mlTaskId: string;
|
|
60
|
+
modelId: string;
|
|
61
|
+
trainDatasetRef: string;
|
|
62
|
+
testDatasetRef?: string;
|
|
63
|
+
modelName: string;
|
|
64
|
+
redoOptimization?: boolean;
|
|
65
|
+
projectKey?: string;
|
|
66
|
+
}
|
|
67
|
+
export interface MlTaskDeployResult extends Record<string, unknown> {
|
|
68
|
+
savedModelId?: string;
|
|
69
|
+
trainRecipeName?: string;
|
|
70
|
+
}
|
|
71
|
+
export declare class MlTasksResource extends BaseResource {
|
|
72
|
+
/** Create a prediction or clustering task in an existing visual analysis. */
|
|
73
|
+
create(opts: MlTaskCreateOptions): Promise<MlTaskCreateResult>;
|
|
74
|
+
/** Get the current guessing, training, and trained-model status of a task. */
|
|
75
|
+
status(analysisId: string, mlTaskId: string, projectKey?: string): Promise<MlTaskStatus>;
|
|
76
|
+
/** Get the editable settings of an ML task. */
|
|
77
|
+
getSettings(analysisId: string, mlTaskId: string, projectKey?: string): Promise<MlTaskSettings>;
|
|
78
|
+
/** Save the full settings object of an ML task. */
|
|
79
|
+
saveSettings(analysisId: string, mlTaskId: string, settings: MlTaskSettings, projectKey?: string): Promise<MlTaskActionResult | undefined>;
|
|
80
|
+
/** Start training, optionally waiting for the task to finish. */
|
|
81
|
+
train(opts: MlTaskTrainOptions): Promise<MlTaskTrainResult>;
|
|
82
|
+
/** List identifiers for every trained model currently present on the task. */
|
|
83
|
+
listTrainedModels(analysisId: string, mlTaskId: string, projectKey?: string): Promise<string[]>;
|
|
84
|
+
/** Get full details for a trained model. */
|
|
85
|
+
trainedModelDetails(analysisId: string, mlTaskId: string, modelId: string, projectKey?: string): Promise<MlTrainedModelDetails>;
|
|
86
|
+
/** Deploy a trained model to the Flow as a saved model and training recipe. */
|
|
87
|
+
deployToFlow(opts: MlTaskDeployOptions): Promise<MlTaskDeployResult>;
|
|
88
|
+
/** Delete an ML task and its trained models. */
|
|
89
|
+
delete(analysisId: string, mlTaskId: string, projectKey?: string): Promise<void>;
|
|
90
|
+
private taskPath;
|
|
91
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { UsageError, } from "../cli/usage.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
const TRAIN_POLL_INTERVAL_MS = 5_000;
|
|
4
|
+
function requireNonEmpty(value, name) {
|
|
5
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
6
|
+
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
7
|
+
}
|
|
8
|
+
return value;
|
|
9
|
+
}
|
|
10
|
+
function delay(ms) {
|
|
11
|
+
const { promise, resolve, } = Promise.withResolvers();
|
|
12
|
+
setTimeout(resolve, ms);
|
|
13
|
+
return promise;
|
|
14
|
+
}
|
|
15
|
+
function modelIdsFromStatus(status) {
|
|
16
|
+
if (!Array.isArray(status.fullModelIds))
|
|
17
|
+
return [];
|
|
18
|
+
const modelIds = [];
|
|
19
|
+
for (const model of status.fullModelIds) {
|
|
20
|
+
if (typeof model?.id === "string" && model.id.length > 0)
|
|
21
|
+
modelIds.push(model.id);
|
|
22
|
+
}
|
|
23
|
+
return modelIds;
|
|
24
|
+
}
|
|
25
|
+
export class MlTasksResource extends BaseResource {
|
|
26
|
+
/** Create a prediction or clustering task in an existing visual analysis. */
|
|
27
|
+
async create(opts) {
|
|
28
|
+
if (opts.taskType !== "PREDICTION" && opts.taskType !== "CLUSTERING") {
|
|
29
|
+
throw new UsageError("taskType must be PREDICTION or CLUSTERING.", "invalid_enum");
|
|
30
|
+
}
|
|
31
|
+
if (opts.taskType === "PREDICTION"
|
|
32
|
+
&& (typeof opts.targetVariable !== "string" || opts.targetVariable.trim().length === 0)) {
|
|
33
|
+
throw new UsageError("targetVariable is required for PREDICTION ML tasks.", "missing_required_arg");
|
|
34
|
+
}
|
|
35
|
+
const analysisId = encodeURIComponent(requireNonEmpty(opts.analysisId, "analysisId"));
|
|
36
|
+
const backendType = opts.backendType ?? "PY_MEMORY";
|
|
37
|
+
const guessPolicy = opts.guessPolicy
|
|
38
|
+
?? (opts.taskType === "CLUSTERING" ? "KMEANS" : "DEFAULT");
|
|
39
|
+
requireNonEmpty(backendType, "backendType");
|
|
40
|
+
requireNonEmpty(guessPolicy, "guessPolicy");
|
|
41
|
+
if (opts.targetVariable !== undefined)
|
|
42
|
+
requireNonEmpty(opts.targetVariable, "targetVariable");
|
|
43
|
+
if (opts.predictionType !== undefined)
|
|
44
|
+
requireNonEmpty(opts.predictionType, "predictionType");
|
|
45
|
+
return this.client.post(`/public/api/projects/${this.enc(opts.projectKey)}/lab/${analysisId}/models/`, {
|
|
46
|
+
taskType: opts.taskType,
|
|
47
|
+
...(opts.targetVariable !== undefined
|
|
48
|
+
? { targetVariable: opts.targetVariable, }
|
|
49
|
+
: {}),
|
|
50
|
+
...(opts.predictionType !== undefined
|
|
51
|
+
? { predictionType: opts.predictionType, }
|
|
52
|
+
: {}),
|
|
53
|
+
backendType,
|
|
54
|
+
guessPolicy,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
/** Get the current guessing, training, and trained-model status of a task. */
|
|
58
|
+
async status(analysisId, mlTaskId, projectKey) {
|
|
59
|
+
return this.client.get(`${this.taskPath(analysisId, mlTaskId, projectKey)}/status`);
|
|
60
|
+
}
|
|
61
|
+
/** Get the editable settings of an ML task. */
|
|
62
|
+
async getSettings(analysisId, mlTaskId, projectKey) {
|
|
63
|
+
return this.client.get(`${this.taskPath(analysisId, mlTaskId, projectKey)}/settings`);
|
|
64
|
+
}
|
|
65
|
+
/** Save the full settings object of an ML task. */
|
|
66
|
+
async saveSettings(analysisId, mlTaskId, settings, projectKey) {
|
|
67
|
+
if (settings === null || typeof settings !== "object" || Array.isArray(settings)) {
|
|
68
|
+
throw new UsageError("settings must be an object.", "validation_failed");
|
|
69
|
+
}
|
|
70
|
+
return this.client.post(`${this.taskPath(analysisId, mlTaskId, projectKey)}/settings`, settings);
|
|
71
|
+
}
|
|
72
|
+
/** Start training, optionally waiting for the task to finish. */
|
|
73
|
+
async train(opts) {
|
|
74
|
+
if (opts.sessionName !== undefined)
|
|
75
|
+
requireNonEmpty(opts.sessionName, "sessionName");
|
|
76
|
+
if (opts.sessionDescription !== undefined) {
|
|
77
|
+
requireNonEmpty(opts.sessionDescription, "sessionDescription");
|
|
78
|
+
}
|
|
79
|
+
const taskPath = this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
80
|
+
const session = await this.client.post(`${taskPath}/train`, {
|
|
81
|
+
sessionName: opts.sessionName,
|
|
82
|
+
sessionDescription: opts.sessionDescription,
|
|
83
|
+
runQueue: opts.runQueue ?? false,
|
|
84
|
+
});
|
|
85
|
+
if (opts.wait !== true)
|
|
86
|
+
return session;
|
|
87
|
+
let status = await this.status(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
88
|
+
while (status.training !== false) {
|
|
89
|
+
await delay(TRAIN_POLL_INTERVAL_MS);
|
|
90
|
+
status = await this.status(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
sessionId: session.sessionId,
|
|
94
|
+
trainedModelIds: modelIdsFromStatus(status),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/** List identifiers for every trained model currently present on the task. */
|
|
98
|
+
async listTrainedModels(analysisId, mlTaskId, projectKey) {
|
|
99
|
+
return modelIdsFromStatus(await this.status(analysisId, mlTaskId, projectKey));
|
|
100
|
+
}
|
|
101
|
+
/** Get full details for a trained model. */
|
|
102
|
+
async trainedModelDetails(analysisId, mlTaskId, modelId, projectKey) {
|
|
103
|
+
const id = encodeURIComponent(requireNonEmpty(modelId, "modelId"));
|
|
104
|
+
return this.client.get(`${this.taskPath(analysisId, mlTaskId, projectKey)}/models/${id}/details`);
|
|
105
|
+
}
|
|
106
|
+
/** Deploy a trained model to the Flow as a saved model and training recipe. */
|
|
107
|
+
async deployToFlow(opts) {
|
|
108
|
+
const modelId = encodeURIComponent(requireNonEmpty(opts.modelId, "modelId"));
|
|
109
|
+
const trainDatasetRef = requireNonEmpty(opts.trainDatasetRef, "trainDatasetRef");
|
|
110
|
+
const modelName = requireNonEmpty(opts.modelName, "modelName");
|
|
111
|
+
if (opts.testDatasetRef !== undefined) {
|
|
112
|
+
requireNonEmpty(opts.testDatasetRef, "testDatasetRef");
|
|
113
|
+
}
|
|
114
|
+
return this.client.post(`${this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey)}/models/${modelId}/actions/deployToFlow`, {
|
|
115
|
+
trainDatasetRef,
|
|
116
|
+
testDatasetRef: opts.testDatasetRef,
|
|
117
|
+
modelName,
|
|
118
|
+
redoOptimization: opts.redoOptimization ?? true,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
/** Delete an ML task and its trained models. */
|
|
122
|
+
async delete(analysisId, mlTaskId, projectKey) {
|
|
123
|
+
await this.client.del(`${this.taskPath(analysisId, mlTaskId, projectKey)}/`);
|
|
124
|
+
}
|
|
125
|
+
taskPath(analysisId, mlTaskId, projectKey) {
|
|
126
|
+
const analysis = encodeURIComponent(requireNonEmpty(analysisId, "analysisId"));
|
|
127
|
+
const task = encodeURIComponent(requireNonEmpty(mlTaskId, "mlTaskId"));
|
|
128
|
+
return `/public/api/projects/${this.enc(projectKey)}/models/lab/${analysis}/${task}`;
|
|
129
|
+
}
|
|
130
|
+
}
|