dataiku-sdk 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/packages/types/src/index.d.ts +14 -0
- package/dist/packages/types/src/index.js +9 -0
- package/dist/src/cli/commands/analysis.d.ts +2 -0
- package/dist/src/cli/commands/analysis.js +66 -0
- package/dist/src/cli/commands/api-service.js +19 -0
- package/dist/src/cli/commands/index.js +8 -0
- package/dist/src/cli/commands/ml-task.d.ts +2 -0
- package/dist/src/cli/commands/ml-task.js +171 -0
- package/dist/src/cli/commands/model-evaluation-store.d.ts +2 -0
- package/dist/src/cli/commands/model-evaluation-store.js +83 -0
- package/dist/src/cli/commands/project.js +2 -1
- package/dist/src/cli/commands/recipe.js +29 -2
- package/dist/src/cli/commands/saved-model.d.ts +2 -0
- package/dist/src/cli/commands/saved-model.js +85 -0
- package/dist/src/cli/contract.js +11 -1
- package/dist/src/cli/flags.js +17 -0
- package/dist/src/cli/helpers/cleanup.js +47 -1
- package/dist/src/cli/usage.js +4 -0
- package/dist/src/client.d.ts +12 -0
- package/dist/src/client.js +20 -0
- package/dist/src/index.d.ts +4 -0
- package/dist/src/index.js +4 -0
- package/dist/src/resources/analyses.d.ts +34 -0
- package/dist/src/resources/analyses.js +31 -0
- package/dist/src/resources/api-services.d.ts +2 -0
- package/dist/src/resources/api-services.js +10 -0
- package/dist/src/resources/datasets.js +7 -5
- package/dist/src/resources/ml-tasks.d.ts +91 -0
- package/dist/src/resources/ml-tasks.js +130 -0
- package/dist/src/resources/model-evaluation-stores.d.ts +41 -0
- package/dist/src/resources/model-evaluation-stores.js +36 -0
- package/dist/src/resources/recipes.js +148 -31
- package/dist/src/resources/saved-models.d.ts +41 -0
- package/dist/src/resources/saved-models.js +40 -0
- package/dist/src/resources/scenarios.js +29 -0
- package/package.json +1 -1
- package/packages/types/dist/index.d.ts +14 -0
- package/packages/types/dist/index.js +9 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { UsageError, } from "../cli/usage.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
const TRAIN_POLL_INTERVAL_MS = 5_000;
|
|
4
|
+
function requireNonEmpty(value, name) {
|
|
5
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
6
|
+
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
7
|
+
}
|
|
8
|
+
return value;
|
|
9
|
+
}
|
|
10
|
+
function delay(ms) {
|
|
11
|
+
const { promise, resolve, } = Promise.withResolvers();
|
|
12
|
+
setTimeout(resolve, ms);
|
|
13
|
+
return promise;
|
|
14
|
+
}
|
|
15
|
+
function modelIdsFromStatus(status) {
|
|
16
|
+
if (!Array.isArray(status.fullModelIds))
|
|
17
|
+
return [];
|
|
18
|
+
const modelIds = [];
|
|
19
|
+
for (const model of status.fullModelIds) {
|
|
20
|
+
if (typeof model?.id === "string" && model.id.length > 0)
|
|
21
|
+
modelIds.push(model.id);
|
|
22
|
+
}
|
|
23
|
+
return modelIds;
|
|
24
|
+
}
|
|
25
|
+
export class MlTasksResource extends BaseResource {
|
|
26
|
+
/** Create a prediction or clustering task in an existing visual analysis. */
|
|
27
|
+
async create(opts) {
|
|
28
|
+
if (opts.taskType !== "PREDICTION" && opts.taskType !== "CLUSTERING") {
|
|
29
|
+
throw new UsageError("taskType must be PREDICTION or CLUSTERING.", "invalid_enum");
|
|
30
|
+
}
|
|
31
|
+
if (opts.taskType === "PREDICTION"
|
|
32
|
+
&& (typeof opts.targetVariable !== "string" || opts.targetVariable.trim().length === 0)) {
|
|
33
|
+
throw new UsageError("targetVariable is required for PREDICTION ML tasks.", "missing_required_arg");
|
|
34
|
+
}
|
|
35
|
+
const analysisId = encodeURIComponent(requireNonEmpty(opts.analysisId, "analysisId"));
|
|
36
|
+
const backendType = opts.backendType ?? "PY_MEMORY";
|
|
37
|
+
const guessPolicy = opts.guessPolicy
|
|
38
|
+
?? (opts.taskType === "CLUSTERING" ? "KMEANS" : "DEFAULT");
|
|
39
|
+
requireNonEmpty(backendType, "backendType");
|
|
40
|
+
requireNonEmpty(guessPolicy, "guessPolicy");
|
|
41
|
+
if (opts.targetVariable !== undefined)
|
|
42
|
+
requireNonEmpty(opts.targetVariable, "targetVariable");
|
|
43
|
+
if (opts.predictionType !== undefined)
|
|
44
|
+
requireNonEmpty(opts.predictionType, "predictionType");
|
|
45
|
+
return this.client.post(`/public/api/projects/${this.enc(opts.projectKey)}/lab/${analysisId}/models/`, {
|
|
46
|
+
taskType: opts.taskType,
|
|
47
|
+
...(opts.targetVariable !== undefined
|
|
48
|
+
? { targetVariable: opts.targetVariable, }
|
|
49
|
+
: {}),
|
|
50
|
+
...(opts.predictionType !== undefined
|
|
51
|
+
? { predictionType: opts.predictionType, }
|
|
52
|
+
: {}),
|
|
53
|
+
backendType,
|
|
54
|
+
guessPolicy,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
/** Get the current guessing, training, and trained-model status of a task. */
|
|
58
|
+
async status(analysisId, mlTaskId, projectKey) {
|
|
59
|
+
return this.client.get(`${this.taskPath(analysisId, mlTaskId, projectKey)}/status`);
|
|
60
|
+
}
|
|
61
|
+
/** Get the editable settings of an ML task. */
|
|
62
|
+
async getSettings(analysisId, mlTaskId, projectKey) {
|
|
63
|
+
return this.client.get(`${this.taskPath(analysisId, mlTaskId, projectKey)}/settings`);
|
|
64
|
+
}
|
|
65
|
+
/** Save the full settings object of an ML task. */
|
|
66
|
+
async saveSettings(analysisId, mlTaskId, settings, projectKey) {
|
|
67
|
+
if (settings === null || typeof settings !== "object" || Array.isArray(settings)) {
|
|
68
|
+
throw new UsageError("settings must be an object.", "validation_failed");
|
|
69
|
+
}
|
|
70
|
+
return this.client.post(`${this.taskPath(analysisId, mlTaskId, projectKey)}/settings`, settings);
|
|
71
|
+
}
|
|
72
|
+
/** Start training, optionally waiting for the task to finish. */
|
|
73
|
+
async train(opts) {
|
|
74
|
+
if (opts.sessionName !== undefined)
|
|
75
|
+
requireNonEmpty(opts.sessionName, "sessionName");
|
|
76
|
+
if (opts.sessionDescription !== undefined) {
|
|
77
|
+
requireNonEmpty(opts.sessionDescription, "sessionDescription");
|
|
78
|
+
}
|
|
79
|
+
const taskPath = this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
80
|
+
const session = await this.client.post(`${taskPath}/train`, {
|
|
81
|
+
sessionName: opts.sessionName,
|
|
82
|
+
sessionDescription: opts.sessionDescription,
|
|
83
|
+
runQueue: opts.runQueue ?? false,
|
|
84
|
+
});
|
|
85
|
+
if (opts.wait !== true)
|
|
86
|
+
return session;
|
|
87
|
+
let status = await this.status(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
88
|
+
while (status.training !== false) {
|
|
89
|
+
await delay(TRAIN_POLL_INTERVAL_MS);
|
|
90
|
+
status = await this.status(opts.analysisId, opts.mlTaskId, opts.projectKey);
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
sessionId: session.sessionId,
|
|
94
|
+
trainedModelIds: modelIdsFromStatus(status),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/** List identifiers for every trained model currently present on the task. */
|
|
98
|
+
async listTrainedModels(analysisId, mlTaskId, projectKey) {
|
|
99
|
+
return modelIdsFromStatus(await this.status(analysisId, mlTaskId, projectKey));
|
|
100
|
+
}
|
|
101
|
+
/** Get full details for a trained model. */
|
|
102
|
+
async trainedModelDetails(analysisId, mlTaskId, modelId, projectKey) {
|
|
103
|
+
const id = encodeURIComponent(requireNonEmpty(modelId, "modelId"));
|
|
104
|
+
return this.client.get(`${this.taskPath(analysisId, mlTaskId, projectKey)}/models/${id}/details`);
|
|
105
|
+
}
|
|
106
|
+
/** Deploy a trained model to the Flow as a saved model and training recipe. */
|
|
107
|
+
async deployToFlow(opts) {
|
|
108
|
+
const modelId = encodeURIComponent(requireNonEmpty(opts.modelId, "modelId"));
|
|
109
|
+
const trainDatasetRef = requireNonEmpty(opts.trainDatasetRef, "trainDatasetRef");
|
|
110
|
+
const modelName = requireNonEmpty(opts.modelName, "modelName");
|
|
111
|
+
if (opts.testDatasetRef !== undefined) {
|
|
112
|
+
requireNonEmpty(opts.testDatasetRef, "testDatasetRef");
|
|
113
|
+
}
|
|
114
|
+
return this.client.post(`${this.taskPath(opts.analysisId, opts.mlTaskId, opts.projectKey)}/models/${modelId}/actions/deployToFlow`, {
|
|
115
|
+
trainDatasetRef,
|
|
116
|
+
testDatasetRef: opts.testDatasetRef,
|
|
117
|
+
modelName,
|
|
118
|
+
redoOptimization: opts.redoOptimization ?? true,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
/** Delete an ML task and its trained models. */
|
|
122
|
+
async delete(analysisId, mlTaskId, projectKey) {
|
|
123
|
+
await this.client.del(`${this.taskPath(analysisId, mlTaskId, projectKey)}/`);
|
|
124
|
+
}
|
|
125
|
+
taskPath(analysisId, mlTaskId, projectKey) {
|
|
126
|
+
const analysis = encodeURIComponent(requireNonEmpty(analysisId, "analysisId"));
|
|
127
|
+
const task = encodeURIComponent(requireNonEmpty(mlTaskId, "mlTaskId"));
|
|
128
|
+
return `/public/api/projects/${this.enc(projectKey)}/models/lab/${analysis}/${task}`;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { BaseResource } from "./base.js";
|
|
2
|
+
export interface ModelEvaluationStoreListItem extends Record<string, unknown> {
|
|
3
|
+
id?: string;
|
|
4
|
+
name?: string;
|
|
5
|
+
projectKey?: string;
|
|
6
|
+
mesFlavor?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ModelEvaluationStore extends Record<string, unknown> {
|
|
9
|
+
id?: string;
|
|
10
|
+
name?: string;
|
|
11
|
+
projectKey?: string;
|
|
12
|
+
mesFlavor?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface ModelEvaluationStoreCreateOptions {
|
|
15
|
+
name: string;
|
|
16
|
+
projectKey?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ModelEvaluationStoreCreateResult extends Record<string, unknown> {
|
|
19
|
+
id?: string;
|
|
20
|
+
name?: string;
|
|
21
|
+
projectKey?: string;
|
|
22
|
+
}
|
|
23
|
+
export interface ModelEvaluationListItem extends Record<string, unknown> {
|
|
24
|
+
id?: string;
|
|
25
|
+
evaluationId?: string;
|
|
26
|
+
createdOn?: number;
|
|
27
|
+
label?: string;
|
|
28
|
+
}
|
|
29
|
+
export declare class ModelEvaluationStoresResource extends BaseResource {
|
|
30
|
+
/** List model evaluation stores in a project. */
|
|
31
|
+
list(projectKey?: string): Promise<ModelEvaluationStoreListItem[]>;
|
|
32
|
+
/** Get model evaluation store settings and metadata. */
|
|
33
|
+
get(modelEvaluationStoreId: string, projectKey?: string): Promise<ModelEvaluationStore>;
|
|
34
|
+
/** Create a tabular model evaluation store. */
|
|
35
|
+
create(opts: ModelEvaluationStoreCreateOptions): Promise<ModelEvaluationStoreCreateResult>;
|
|
36
|
+
/** List evaluations in a model evaluation store. */
|
|
37
|
+
listEvaluations(modelEvaluationStoreId: string, projectKey?: string): Promise<ModelEvaluationListItem[]>;
|
|
38
|
+
/** Delete a model evaluation store and its evaluations. */
|
|
39
|
+
delete(modelEvaluationStoreId: string, projectKey?: string): Promise<void>;
|
|
40
|
+
private storePath;
|
|
41
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { UsageError, } from "../cli/usage.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
function requireNonEmpty(value, name) {
|
|
4
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
5
|
+
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
6
|
+
}
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
export class ModelEvaluationStoresResource extends BaseResource {
|
|
10
|
+
/** List model evaluation stores in a project. */
|
|
11
|
+
async list(projectKey) {
|
|
12
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/evaluationstores/`);
|
|
13
|
+
}
|
|
14
|
+
/** Get model evaluation store settings and metadata. */
|
|
15
|
+
async get(modelEvaluationStoreId, projectKey) {
|
|
16
|
+
return this.client.get(this.storePath(modelEvaluationStoreId, projectKey));
|
|
17
|
+
}
|
|
18
|
+
/** Create a tabular model evaluation store. */
|
|
19
|
+
async create(opts) {
|
|
20
|
+
const name = requireNonEmpty(opts.name, "name");
|
|
21
|
+
const projectKey = this.resolveProjectKey(opts.projectKey);
|
|
22
|
+
return this.client.post(`/public/api/projects/${encodeURIComponent(projectKey)}/evaluationstores/?flavor=TABULAR`, { projectKey, name, });
|
|
23
|
+
}
|
|
24
|
+
/** List evaluations in a model evaluation store. */
|
|
25
|
+
async listEvaluations(modelEvaluationStoreId, projectKey) {
|
|
26
|
+
return this.client.get(`${this.storePath(modelEvaluationStoreId, projectKey)}/evaluations/`);
|
|
27
|
+
}
|
|
28
|
+
/** Delete a model evaluation store and its evaluations. */
|
|
29
|
+
async delete(modelEvaluationStoreId, projectKey) {
|
|
30
|
+
await this.client.del(this.storePath(modelEvaluationStoreId, projectKey));
|
|
31
|
+
}
|
|
32
|
+
storePath(modelEvaluationStoreId, projectKey) {
|
|
33
|
+
const id = encodeURIComponent(requireNonEmpty(modelEvaluationStoreId, "modelEvaluationStoreId"));
|
|
34
|
+
return `/public/api/projects/${this.enc(projectKey)}/evaluationstores/${id}`;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -22,6 +22,67 @@ function asRecord(value) {
|
|
|
22
22
|
return undefined;
|
|
23
23
|
return value;
|
|
24
24
|
}
|
|
25
|
+
function parseRecipePayload(payload) {
|
|
26
|
+
if (!payload)
|
|
27
|
+
return {};
|
|
28
|
+
const parsed = JSON.parse(payload);
|
|
29
|
+
return asRecord(parsed) ?? {};
|
|
30
|
+
}
|
|
31
|
+
function parseJoinKeyPairs(values, optionName = "joinOn") {
|
|
32
|
+
const pairs = [];
|
|
33
|
+
for (const raw of values) {
|
|
34
|
+
const token = raw.trim();
|
|
35
|
+
if (!token)
|
|
36
|
+
continue;
|
|
37
|
+
const eq = token.indexOf("=");
|
|
38
|
+
if (eq === -1) {
|
|
39
|
+
pairs.push({ left: token, right: token, });
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
const left = token.slice(0, eq).trim();
|
|
43
|
+
const right = token.slice(eq + 1).trim();
|
|
44
|
+
if (!left || !right)
|
|
45
|
+
throw new Error(`${optionName} values must use COL or LEFT=RIGHT.`);
|
|
46
|
+
pairs.push({ left, right, });
|
|
47
|
+
}
|
|
48
|
+
return pairs;
|
|
49
|
+
}
|
|
50
|
+
function normalizeFuzzyDistance(value) {
|
|
51
|
+
const normalized = typeof value === "string" && value.trim().length > 0
|
|
52
|
+
? value.trim().toUpperCase()
|
|
53
|
+
: "DAMERAU_LEVENSHTEIN";
|
|
54
|
+
if (normalized === "DAMERAU_LEVENSHTEIN")
|
|
55
|
+
return "LEVENSHTEIN";
|
|
56
|
+
if (normalized === "HAMMING" || normalized === "JACCARD" || normalized === "COSINE"
|
|
57
|
+
|| normalized === "EUCLIDEAN") {
|
|
58
|
+
return normalized;
|
|
59
|
+
}
|
|
60
|
+
throw new Error("fuzzyDistance must be one of DAMERAU_LEVENSHTEIN, HAMMING, JACCARD, COSINE, or EUCLIDEAN.");
|
|
61
|
+
}
|
|
62
|
+
function normalizeJoinType(value) {
|
|
63
|
+
const normalized = typeof value === "string" && value.trim().length > 0
|
|
64
|
+
? value.trim().toUpperCase()
|
|
65
|
+
: "LEFT";
|
|
66
|
+
if (normalized === "LEFT" || normalized === "INNER" || normalized === "RIGHT"
|
|
67
|
+
|| normalized === "FULL") {
|
|
68
|
+
return normalized;
|
|
69
|
+
}
|
|
70
|
+
throw new Error("joinType must be one of LEFT, INNER, RIGHT, or FULL.");
|
|
71
|
+
}
|
|
72
|
+
function recipeVirtualInputs(payload, inputCount) {
|
|
73
|
+
const current = Array.isArray(payload.virtualInputs) ? payload.virtualInputs : [];
|
|
74
|
+
const virtualInputs = [];
|
|
75
|
+
for (let i = 0; i < inputCount; i++) {
|
|
76
|
+
virtualInputs.push({
|
|
77
|
+
preFilter: {},
|
|
78
|
+
outputColumnsSelectionMode: "AUTO_NON_CONFLICTING",
|
|
79
|
+
computedColumns: [],
|
|
80
|
+
...asRecord(current[i]),
|
|
81
|
+
index: i,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
return virtualInputs;
|
|
85
|
+
}
|
|
25
86
|
const RECIPE_DEFINITION_FIELDS = new Set(["params", "inputs", "outputs", "scriptSettings",]);
|
|
26
87
|
function rootRecipeDefinitionFields(data) {
|
|
27
88
|
return Object.keys(data).filter((key) => RECIPE_DEFINITION_FIELDS.has(key));
|
|
@@ -467,17 +528,37 @@ export class RecipesResource extends BaseResource {
|
|
|
467
528
|
if (!type || !name || !inputs || !outputs) {
|
|
468
529
|
throw new Error("type and (inputDatasets + outputDataset/outputFolder) or (name + inputs + outputs) are required for create.");
|
|
469
530
|
}
|
|
531
|
+
const joinCols = typeof opts.joinOn === "string" ? [opts.joinOn,] : asStringArray(opts.joinOn);
|
|
532
|
+
const joinKeys = type === "join" && joinCols?.length
|
|
533
|
+
? parseJoinKeyPairs(joinCols)
|
|
534
|
+
: undefined;
|
|
535
|
+
const fuzzyOn = opts.fuzzyOn ?? (type === "fuzzyjoin" ? opts.joinOn : undefined);
|
|
536
|
+
const fuzzyCols = typeof fuzzyOn === "string" ? [fuzzyOn,] : asStringArray(fuzzyOn);
|
|
537
|
+
const fuzzyKeys = type === "fuzzyjoin" && fuzzyCols?.length
|
|
538
|
+
? parseJoinKeyPairs(fuzzyCols, "fuzzyOn")
|
|
539
|
+
: undefined;
|
|
540
|
+
const normalizedJoinType = joinKeys?.length || fuzzyKeys?.length
|
|
541
|
+
? normalizeJoinType(rawJoinType)
|
|
542
|
+
: undefined;
|
|
543
|
+
const fuzzyDistance = fuzzyKeys?.length ? normalizeFuzzyDistance(opts.fuzzyDistance) : undefined;
|
|
544
|
+
const fuzzyThreshold = opts.fuzzyThreshold ?? 1;
|
|
545
|
+
if (!Number.isFinite(fuzzyThreshold)) {
|
|
546
|
+
throw new Error("fuzzyThreshold must be a finite number.");
|
|
547
|
+
}
|
|
470
548
|
const recipePrototype = {
|
|
471
549
|
type,
|
|
472
550
|
name,
|
|
473
551
|
projectKey: pk,
|
|
474
|
-
inputs,
|
|
552
|
+
...(type === "fuzzyjoin" ? {} : { inputs, }),
|
|
475
553
|
outputs,
|
|
476
554
|
};
|
|
477
555
|
const creationSettings = {};
|
|
478
556
|
if (payload !== undefined) {
|
|
479
557
|
creationSettings.script = payload;
|
|
480
558
|
}
|
|
559
|
+
if (type === "fuzzyjoin") {
|
|
560
|
+
creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
|
|
561
|
+
}
|
|
481
562
|
const createRecipe = () => this.client.post(`/public/api/projects/${enc}/recipes/`, {
|
|
482
563
|
recipePrototype,
|
|
483
564
|
creationSettings,
|
|
@@ -597,37 +678,29 @@ export class RecipesResource extends BaseResource {
|
|
|
597
678
|
}
|
|
598
679
|
// For join recipes: configure join conditions after creation
|
|
599
680
|
let joinConfigured = false;
|
|
600
|
-
|
|
601
|
-
const joinType = asString(rawJoinType) ?? "LEFT";
|
|
602
|
-
if (type === "join" && joinCols?.length) {
|
|
681
|
+
if (type === "join" && joinKeys?.length) {
|
|
603
682
|
const rnEnc = encodeURIComponent(name);
|
|
604
683
|
const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
|
|
605
|
-
|
|
606
|
-
const inputCount = inputDatasets?.length
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
684
|
+
const joinPayload = parseRecipePayload(full.payload);
|
|
685
|
+
const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
|
|
686
|
+
const joinType = normalizedJoinType ?? "LEFT";
|
|
687
|
+
joinPayload.virtualInputs = recipeVirtualInputs(joinPayload, inputCount);
|
|
688
|
+
joinPayload.joins = Array.from({ length: Math.max(0, inputCount - 1), }, (_, index) => {
|
|
689
|
+
const table2 = index + 1;
|
|
690
|
+
return {
|
|
691
|
+
table1: 0,
|
|
692
|
+
table2,
|
|
693
|
+
conditionsMode: "AND",
|
|
694
|
+
type: joinType,
|
|
695
|
+
outerJoinOnTheLeft: joinType === "LEFT",
|
|
696
|
+
on: joinKeys.map((key) => ({
|
|
697
|
+
column1: { name: key.left, table: 0, },
|
|
698
|
+
column2: { name: key.right, table: table2, },
|
|
699
|
+
type: "EQ",
|
|
619
700
|
})),
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
}
|
|
624
|
-
const joinPayload = {
|
|
625
|
-
virtualInputs,
|
|
626
|
-
computedColumns: [],
|
|
627
|
-
postFilter: {},
|
|
628
|
-
};
|
|
629
|
-
// Ensure inputs/outputs are set — DSS may not persist them from the POST for join recipes
|
|
630
|
-
const updatedFull = {
|
|
701
|
+
};
|
|
702
|
+
});
|
|
703
|
+
await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, {
|
|
631
704
|
...full,
|
|
632
705
|
recipe: {
|
|
633
706
|
...full.recipe,
|
|
@@ -635,8 +708,52 @@ export class RecipesResource extends BaseResource {
|
|
|
635
708
|
outputs,
|
|
636
709
|
},
|
|
637
710
|
payload: JSON.stringify(joinPayload),
|
|
638
|
-
};
|
|
639
|
-
|
|
711
|
+
});
|
|
712
|
+
joinConfigured = true;
|
|
713
|
+
}
|
|
714
|
+
if (type === "fuzzyjoin" && fuzzyKeys?.length) {
|
|
715
|
+
const rnEnc = encodeURIComponent(name);
|
|
716
|
+
const full = await this.client.get(`/public/api/projects/${enc}/recipes/${rnEnc}`);
|
|
717
|
+
const fuzzyPayload = parseRecipePayload(full.payload);
|
|
718
|
+
const inputCount = recipeInputItems({ inputs, }).length || inputDatasets?.length || 2;
|
|
719
|
+
const joinType = normalizedJoinType ?? "LEFT";
|
|
720
|
+
fuzzyPayload.virtualInputs = recipeVirtualInputs(fuzzyPayload, inputCount);
|
|
721
|
+
fuzzyPayload.joins = Array.from({ length: Math.max(0, inputCount - 1), }, (_, index) => {
|
|
722
|
+
const table2 = index + 1;
|
|
723
|
+
return {
|
|
724
|
+
table1: 0,
|
|
725
|
+
table2,
|
|
726
|
+
conditionsMode: "AND",
|
|
727
|
+
type: joinType,
|
|
728
|
+
on: fuzzyKeys.map((key) => ({
|
|
729
|
+
column1: { name: key.left, table: 0, },
|
|
730
|
+
column2: { name: key.right, table: table2, },
|
|
731
|
+
type: "EQ",
|
|
732
|
+
fuzzyMatchDesc: {
|
|
733
|
+
distanceType: fuzzyDistance ?? "LEVENSHTEIN",
|
|
734
|
+
threshold: fuzzyThreshold,
|
|
735
|
+
},
|
|
736
|
+
...(opts.fuzzyNormalize === true
|
|
737
|
+
? {
|
|
738
|
+
normaliseDesc: {
|
|
739
|
+
caseInsensitive: true,
|
|
740
|
+
normaliseText: true,
|
|
741
|
+
unicodeCasting: true,
|
|
742
|
+
},
|
|
743
|
+
}
|
|
744
|
+
: {}),
|
|
745
|
+
})),
|
|
746
|
+
};
|
|
747
|
+
});
|
|
748
|
+
await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, {
|
|
749
|
+
...full,
|
|
750
|
+
recipe: {
|
|
751
|
+
...full.recipe,
|
|
752
|
+
inputs,
|
|
753
|
+
outputs,
|
|
754
|
+
},
|
|
755
|
+
payload: JSON.stringify(fuzzyPayload),
|
|
756
|
+
});
|
|
640
757
|
joinConfigured = true;
|
|
641
758
|
}
|
|
642
759
|
let temporaryOutputDatasetDeleted;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { BaseResource } from "./base.js";
|
|
2
|
+
export interface SavedModelListItem extends Record<string, unknown> {
|
|
3
|
+
id?: string;
|
|
4
|
+
name?: string;
|
|
5
|
+
projectKey?: string;
|
|
6
|
+
activeVersion?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface SavedModel extends Record<string, unknown> {
|
|
9
|
+
id?: string;
|
|
10
|
+
name?: string;
|
|
11
|
+
projectKey?: string;
|
|
12
|
+
activeVersion?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface SavedModelVersionListItem extends Record<string, unknown> {
|
|
15
|
+
id?: string;
|
|
16
|
+
active?: boolean;
|
|
17
|
+
createdOn?: number;
|
|
18
|
+
}
|
|
19
|
+
export interface SavedModelVersionDetails extends Record<string, unknown> {
|
|
20
|
+
id?: string;
|
|
21
|
+
algorithm?: string;
|
|
22
|
+
predictionType?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface SavedModelActionResult extends Record<string, unknown> {
|
|
25
|
+
message?: string;
|
|
26
|
+
}
|
|
27
|
+
export declare class SavedModelsResource extends BaseResource {
|
|
28
|
+
/** List saved models in a project. */
|
|
29
|
+
list(projectKey?: string): Promise<SavedModelListItem[]>;
|
|
30
|
+
/** Get saved-model settings and metadata. */
|
|
31
|
+
get(savedModelId: string, projectKey?: string): Promise<SavedModel>;
|
|
32
|
+
/** List all versions of a saved model. */
|
|
33
|
+
listVersions(savedModelId: string, projectKey?: string): Promise<SavedModelVersionListItem[]>;
|
|
34
|
+
/** Get full details for one saved-model version. */
|
|
35
|
+
versionDetails(savedModelId: string, versionId: string, projectKey?: string): Promise<SavedModelVersionDetails>;
|
|
36
|
+
/** Make one saved-model version active. */
|
|
37
|
+
setActiveVersion(savedModelId: string, versionId: string, projectKey?: string): Promise<SavedModelActionResult | undefined>;
|
|
38
|
+
/** Delete a saved model and all of its versions. */
|
|
39
|
+
delete(savedModelId: string, projectKey?: string): Promise<void>;
|
|
40
|
+
private savedModelPath;
|
|
41
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { UsageError, } from "../cli/usage.js";
|
|
2
|
+
import { BaseResource, } from "./base.js";
|
|
3
|
+
function requireNonEmpty(value, name) {
|
|
4
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
5
|
+
throw new UsageError(`${name} must be a non-empty string.`, "validation_failed");
|
|
6
|
+
}
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
export class SavedModelsResource extends BaseResource {
|
|
10
|
+
/** List saved models in a project. */
|
|
11
|
+
async list(projectKey) {
|
|
12
|
+
return this.client.get(`/public/api/projects/${this.enc(projectKey)}/savedmodels/`);
|
|
13
|
+
}
|
|
14
|
+
/** Get saved-model settings and metadata. */
|
|
15
|
+
async get(savedModelId, projectKey) {
|
|
16
|
+
return this.client.get(this.savedModelPath(savedModelId, projectKey));
|
|
17
|
+
}
|
|
18
|
+
/** List all versions of a saved model. */
|
|
19
|
+
async listVersions(savedModelId, projectKey) {
|
|
20
|
+
return this.client.get(`${this.savedModelPath(savedModelId, projectKey)}/versions`);
|
|
21
|
+
}
|
|
22
|
+
/** Get full details for one saved-model version. */
|
|
23
|
+
async versionDetails(savedModelId, versionId, projectKey) {
|
|
24
|
+
const version = encodeURIComponent(requireNonEmpty(versionId, "versionId"));
|
|
25
|
+
return this.client.get(`${this.savedModelPath(savedModelId, projectKey)}/versions/${version}/details`);
|
|
26
|
+
}
|
|
27
|
+
/** Make one saved-model version active. */
|
|
28
|
+
async setActiveVersion(savedModelId, versionId, projectKey) {
|
|
29
|
+
const version = encodeURIComponent(requireNonEmpty(versionId, "versionId"));
|
|
30
|
+
return this.client.post(`${this.savedModelPath(savedModelId, projectKey)}/versions/${version}/actions/setActive`, {});
|
|
31
|
+
}
|
|
32
|
+
/** Delete a saved model and all of its versions. */
|
|
33
|
+
async delete(savedModelId, projectKey) {
|
|
34
|
+
await this.client.del(this.savedModelPath(savedModelId, projectKey));
|
|
35
|
+
}
|
|
36
|
+
savedModelPath(savedModelId, projectKey) {
|
|
37
|
+
const id = encodeURIComponent(requireNonEmpty(savedModelId, "savedModelId"));
|
|
38
|
+
return `/public/api/projects/${this.enc(projectKey)}/savedmodels/${id}`;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -86,6 +86,34 @@ function extractCodeRunOutput(log) {
|
|
|
86
86
|
function isRecord(value) {
|
|
87
87
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
88
88
|
}
|
|
89
|
+
function scenarioStepWarningSummary(stepResult) {
|
|
90
|
+
const container = stepResult?.warnings;
|
|
91
|
+
if (!isRecord(container))
|
|
92
|
+
return {};
|
|
93
|
+
const rawWarnings = container.warnings;
|
|
94
|
+
const warnings = isRecord(rawWarnings)
|
|
95
|
+
? Object.entries(rawWarnings).flatMap(([key, value,]) => {
|
|
96
|
+
if (!isRecord(value))
|
|
97
|
+
return [];
|
|
98
|
+
const rawType = value.type;
|
|
99
|
+
const type = typeof rawType === "string" && rawType.length > 0 ? rawType : key;
|
|
100
|
+
const rawCount = value.count;
|
|
101
|
+
return [{
|
|
102
|
+
type,
|
|
103
|
+
...(typeof rawCount === "number" && Number.isFinite(rawCount)
|
|
104
|
+
? { count: rawCount, }
|
|
105
|
+
: {}),
|
|
106
|
+
},];
|
|
107
|
+
})
|
|
108
|
+
: [];
|
|
109
|
+
const rawTotal = container.totalCount;
|
|
110
|
+
return {
|
|
111
|
+
...(typeof rawTotal === "number" && Number.isFinite(rawTotal)
|
|
112
|
+
? { warningCount: rawTotal, }
|
|
113
|
+
: {}),
|
|
114
|
+
...(warnings.length > 0 ? { warnings, } : {}),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
89
117
|
function jsonValueEqual(left, right) {
|
|
90
118
|
if (Object.is(left, right))
|
|
91
119
|
return true;
|
|
@@ -352,6 +380,7 @@ export class ScenariosResource extends BaseResource {
|
|
|
352
380
|
name: stepDef?.name,
|
|
353
381
|
type: stepDef?.type,
|
|
354
382
|
outcome: stepResult?.outcome ?? "UNKNOWN",
|
|
383
|
+
...scenarioStepWarningSummary(stepResult),
|
|
355
384
|
};
|
|
356
385
|
});
|
|
357
386
|
if (mapped.length > 0)
|
package/package.json
CHANGED
|
@@ -147,6 +147,10 @@ export declare const RecipeCreateOptionsSchema: import("@sinclair/typebox").TObj
|
|
|
147
147
|
outputConnection: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
148
148
|
joinOn: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TString, import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>]>>;
|
|
149
149
|
joinType: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
150
|
+
fuzzyOn: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TString, import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>]>>;
|
|
151
|
+
fuzzyDistance: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
152
|
+
fuzzyThreshold: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
153
|
+
fuzzyNormalize: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
|
|
150
154
|
projectKey: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
151
155
|
outputFolder: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
152
156
|
}>;
|
|
@@ -269,6 +273,11 @@ export declare const ScenarioStepRunSchema: import("@sinclair/typebox").TObject<
|
|
|
269
273
|
name: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
270
274
|
type: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
271
275
|
outcome: import("@sinclair/typebox").TString;
|
|
276
|
+
warningCount: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
277
|
+
warnings: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
278
|
+
type: import("@sinclair/typebox").TString;
|
|
279
|
+
count: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
280
|
+
}>>>;
|
|
272
281
|
}>;
|
|
273
282
|
export type ScenarioStepRun = Static<typeof ScenarioStepRunSchema>;
|
|
274
283
|
export declare const ScenarioWaitResultSchema: import("@sinclair/typebox").TObject<{
|
|
@@ -284,6 +293,11 @@ export declare const ScenarioWaitResultSchema: import("@sinclair/typebox").TObje
|
|
|
284
293
|
name: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
285
294
|
type: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
286
295
|
outcome: import("@sinclair/typebox").TString;
|
|
296
|
+
warningCount: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
297
|
+
warnings: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
|
|
298
|
+
type: import("@sinclair/typebox").TString;
|
|
299
|
+
count: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
300
|
+
}>>>;
|
|
287
301
|
}>>>;
|
|
288
302
|
}>;
|
|
289
303
|
export type ScenarioWaitResult = Static<typeof ScenarioWaitResultSchema>;
|
|
@@ -195,6 +195,10 @@ export const RecipeCreateOptionsSchema = Type.Object({
|
|
|
195
195
|
outputConnection: Type.Optional(Type.String()),
|
|
196
196
|
joinOn: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String()),])),
|
|
197
197
|
joinType: Type.Optional(Type.String()),
|
|
198
|
+
fuzzyOn: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String()),])),
|
|
199
|
+
fuzzyDistance: Type.Optional(Type.String()),
|
|
200
|
+
fuzzyThreshold: Type.Optional(Type.Number()),
|
|
201
|
+
fuzzyNormalize: Type.Optional(Type.Boolean()),
|
|
198
202
|
projectKey: Type.Optional(Type.String()),
|
|
199
203
|
outputFolder: Type.Optional(Type.String()),
|
|
200
204
|
});
|
|
@@ -318,6 +322,11 @@ export const ScenarioStepRunSchema = Type.Object({
|
|
|
318
322
|
name: Type.Optional(Type.String()),
|
|
319
323
|
type: Type.Optional(Type.String()),
|
|
320
324
|
outcome: Type.String(),
|
|
325
|
+
warningCount: Type.Optional(Type.Number()),
|
|
326
|
+
warnings: Type.Optional(Type.Array(Type.Object({
|
|
327
|
+
type: Type.String(),
|
|
328
|
+
count: Type.Optional(Type.Number()),
|
|
329
|
+
}, { additionalProperties: false, }))),
|
|
321
330
|
}, { additionalProperties: false, });
|
|
322
331
|
export const ScenarioWaitResultSchema = Type.Object({
|
|
323
332
|
scenarioId: Type.String(),
|