dataiku-sdk 3.6.1 → 3.7.0

package/dist/src/resources/recipe-create.js

@@ -0,0 +1,338 @@
+import { ClientValidationError, } from "../errors.js";
+import { asRecord, } from "../utils/records.js";
+/*
+ * Pure, schema-free recipe-create request construction. `recipe create --plan`
+ * imports this without loading the SDK schema graph.
+ */
+export function asString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+export function asStringArray(value) {
+    if (!Array.isArray(value))
+        return undefined;
+    const out = value.filter((v) => typeof v === "string" && v.length > 0);
+    return out.length > 0 ? out : undefined;
+}
+export function parseJoinKeyPairs(values, optionName = "joinOn") {
+    const pairs = [];
+    for (const raw of values) {
+        const token = raw.trim();
+        if (!token)
+            continue;
+        const eq = token.indexOf("=");
+        if (eq === -1) {
+            pairs.push({ left: token, right: token, });
+            continue;
+        }
+        const left = token.slice(0, eq).trim();
+        const right = token.slice(eq + 1).trim();
+        if (!left || !right) {
+            throw new ClientValidationError(`${optionName} values must use COL or LEFT=RIGHT.`, "validation_failed");
+        }
+        pairs.push({ left, right, });
+    }
+    return pairs;
+}
+export function normalizeFuzzyDistance(value) {
+    const normalized = typeof value === "string" && value.trim().length > 0
+        ? value.trim().toUpperCase()
+        : "DAMERAU_LEVENSHTEIN";
+    if (normalized === "DAMERAU_LEVENSHTEIN")
+        return "LEVENSHTEIN";
+    if (normalized === "HAMMING" || normalized === "JACCARD" || normalized === "COSINE"
+        || normalized === "EUCLIDEAN") {
+        return normalized;
+    }
+    throw new ClientValidationError("fuzzyDistance must be one of DAMERAU_LEVENSHTEIN, HAMMING, JACCARD, COSINE, or EUCLIDEAN.", "validation_failed");
+}
+export function normalizeJoinType(value) {
+    const normalized = typeof value === "string" && value.trim().length > 0
+        ? value.trim().toUpperCase()
+        : "LEFT";
+    if (normalized === "LEFT" || normalized === "INNER" || normalized === "RIGHT"
+        || normalized === "FULL") {
+        return normalized;
+    }
+    throw new ClientValidationError("joinType must be one of LEFT, INNER, RIGHT, or FULL.", "validation_failed");
+}
+export function recipeInputItems(recipe) {
+    const inputs = asRecord(recipe.inputs);
+    if (!inputs)
+        return [];
+    const result = [];
+    const seen = new Set();
+    for (const [role, roleValue,] of Object.entries(inputs)) {
+        const items = asRecord(roleValue)?.items;
+        if (!Array.isArray(items))
+            continue;
+        for (const itemValue of items) {
+            const item = asRecord(itemValue);
+            const ref = asString(item?.ref);
+            if (!ref || seen.has(ref))
+                continue;
+            seen.add(ref);
+            result.push({ ref, role, });
+        }
+    }
+    return result;
+}
+/**
+ * Recipe types whose output schema DSS refuses to compute ("Output schema can't
+ * be automatically computed on 'python' recipes"): the code sets the schema
+ * when it runs. Plugin recipes (`CustomCode_*`) are code too.
+ */
+const CODE_RECIPE_TYPES = {
+    python: true,
+    r: true,
+    julia: true,
+    pyspark: true,
+    sparkr: true,
+    spark_scala: true,
+    shell: true,
+    sql_script: true,
+};
+/** Whether DSS can compute this recipe type's output schema (visual and SQL query recipes). */
+export function recipeOutputSchemaIsComputable(recipeType) {
+    const normalized = recipeType.trim().toLowerCase();
+    return CODE_RECIPE_TYPES[normalized] !== true && !normalized.startsWith("customcode_");
+}
+/** The id DSS expects in actions/updateOutputSchema (dataikuapi RequiredSchemaUpdates.apply). */
+export function schemaUpdateComputableId(computable) {
+    return (computable.type === "DATASET" ? computable.datasetName : computable.id) ?? "";
+}
+/** A computed schema is worth writing when it has columns and differs from (or fills) the current one. */
+export function schemaUpdateIsPending(computable) {
+    const columns = computable.newSchema?.columns ?? [];
+    if (columns.length === 0)
+        return false;
+    return computable.previousSchemaWasEmpty === true
+        || (computable.incompatibilities?.length ?? 0) > 0;
+}
+/** DSS grouping aggregate flags on each `values[]` entry, by lowercase name. */
+const GROUPING_AGGREGATES = {
+    count: "count",
+    countdistinct: "countDistinct",
+    sum: "sum",
+    avg: "avg",
+    min: "min",
+    max: "max",
+    median: "median",
+    stddev: "stddev",
+    first: "first",
+    last: "last",
+    concat: "concat",
+    concatdistinct: "concatDistinct",
+    sum2: "sum2",
+};
+function splitPair(raw, flag) {
+    const eq = raw.indexOf("=");
+    const left = eq === -1 ? "" : raw.slice(0, eq).trim();
+    const right = eq === -1 ? "" : raw.slice(eq + 1).trim();
+    if (left.length === 0 || right.length === 0) {
+        throw new ClientValidationError(`${flag} expects NAME=VALUE, got "${raw}".`, "validation_failed");
+    }
+    return [left, right,];
+}
+/** Grouping configuration from create options, or undefined when none was asked for. */
+export function groupingPayloadConfig(opts) {
+    const keys = opts.groupBy ?? [];
+    const specs = opts.aggregate ?? [];
+    if (keys.length === 0 && specs.length === 0)
+        return undefined;
+    const aggregates = {};
+    for (const spec of specs) {
+        const colon = spec.lastIndexOf(":");
+        const column = colon === -1 ? "" : spec.slice(0, colon).trim();
+        const fns = colon === -1
+            ? []
+            : spec.slice(colon + 1).split("+").map((fn) => fn.trim().toLowerCase());
+        if (column.length === 0 || fns.length === 0
+            || fns.some((fn) => GROUPING_AGGREGATES[fn] === undefined)) {
+            throw new ClientValidationError(`--aggregate expects COL:FN[+FN] with FN one of ${Object.values(GROUPING_AGGREGATES).join(", ")}; got "${spec}".`, "validation_failed");
+        }
+        aggregates[column] = [
+            ...(aggregates[column] ?? []),
+            ...fns.map((fn) => GROUPING_AGGREGATES[fn]),
+        ];
+    }
+    const ordered = Object.values(aggregates).some((fns) => fns.includes("first") || fns.includes("last"));
+    if (ordered && !opts.orderBy) {
+        throw new ClientValidationError("first/last aggregates need an ordering column: pass --order-by COL.", "validation_failed");
+    }
+    return { keys, aggregates, ...(opts.orderBy ? { orderBy: opts.orderBy, } : {}), };
+}
+/**
+ * Apply grouping keys and aggregates to the payload DSS created: `keys` lists
+ * the group columns; each aggregated column's `values[]` entry gets its
+ * aggregate flags (and `orderColumn` for first/last).
+ */
+export function applyGroupingPayload(payload, config) {
+    const values = Array.isArray(payload.values)
+        ? payload.values.map((value) => asRecord(value) ?? {})
+        : [];
+    for (const [column, fns,] of Object.entries(config.aggregates)) {
+        let entry = values.find((value) => value.column === column);
+        if (!entry) {
+            entry = { column, };
+            values.push(entry);
+        }
+        for (const fn of fns)
+            entry[fn] = true;
+        if (config.orderBy && (fns.includes("first") || fns.includes("last"))) {
+            entry.orderColumn = config.orderBy;
+        }
+    }
+    return { ...payload, keys: config.keys.map((column) => ({ column, })), values, };
+}
+function processorStep(type, params) {
+    return {
+        metaType: "PROCESSOR",
+        type,
+        params,
+        disabled: false,
+        preview: false,
+        alwaysShowComment: false,
+    };
+}
+/**
+ * Prepare (shaker) steps from create options, in a fixed order: rename,
+ * fill empty, formula column, row filter, drop columns, keep columns. Later
+ * steps see renamed columns. Empty when no prepare option was given.
+ */
+export function prepareSteps(opts) {
+    const steps = [];
+    const renamings = (opts.rename ?? []).map((raw) => {
+        const [from, to,] = splitPair(raw, "--rename");
+        return { from, to, };
+    });
+    if (renamings.length > 0)
+        steps.push(processorStep("ColumnRenamer", { renamings, }));
+    for (const raw of opts.fillEmpty ?? []) {
+        const [column, value,] = splitPair(raw, "--fill-empty");
+        steps.push(processorStep("FillEmptyWithValue", { appliesTo: "SINGLE_COLUMN", columns: [column,], value, }));
+    }
+    if (opts.formula) {
+        const [column, expression,] = splitPair(opts.formula, "--formula");
+        steps.push(processorStep("CreateColumnWithGREL", { column, expression, }));
+    }
+    if (opts.filter) {
+        steps.push(processorStep("FilterOnCustomFormula", { expression: opts.filter, action: "KEEP_ROW", }));
+    }
+    if (opts.dropColumns?.length) {
+        steps.push(processorStep("ColumnsSelector", {
+            appliesTo: "COLUMNS",
+            columns: opts.dropColumns,
+            keep: false,
+        }));
+    }
+    if (opts.keepColumns?.length) {
+        steps.push(processorStep("ColumnsSelector", {
+            appliesTo: "COLUMNS",
+            columns: opts.keepColumns,
+            keep: true,
+        }));
+    }
+    return steps;
+}
+/**
+ * Pure request construction for POST /recipes/ (no DSS calls). Shared with
+ * `recipe create --plan` so the plan equals the request the command sends.
+ */
+export function buildRecipeCreateRequest(opts, pk) {
+    const { payload, outputConnection: rawConnection, joinType: rawJoinType, } = opts;
+    // dataikuapi and the UI call the shaker recipe "prepare".
+    const type = opts.type === "prepare" ? "shaker" : opts.type;
+    const grouping = groupingPayloadConfig(opts);
+    const steps = prepareSteps(opts);
+    if (grouping && type !== "grouping") {
+        throw new ClientValidationError("--group-by/--aggregate/--order-by need --type grouping.", "validation_failed");
+    }
+    if (steps.length > 0 && type !== "shaker") {
+        throw new ClientValidationError("--rename/--fill-empty/--formula/--filter/--drop-columns/--keep-columns need --type prepare.", "validation_failed");
+    }
+    const outputFolder = asString(opts.outputFolder);
+    // Build inputs/outputs from simple form (inputDatasets + outputDataset) or
+    // advanced form (inputs + outputs); both may coexist — simple form wins when
+    // the advanced form is absent.
+    const inputDatasets = asStringArray(opts.inputDatasets);
+    const requestedOutputDataset = asString(opts.outputDataset);
+    let inputs = asRecord(opts.inputs);
+    let outputs = asRecord(opts.outputs);
+    if (!inputs) {
+        inputs = {
+            main: {
+                items: inputDatasets?.map((ref) => ({ ref, deps: [], })) ?? [],
+            },
+        };
+    }
+    // Auto-generate name if not provided
+    const outputNameForDefaultRecipe = requestedOutputDataset ?? outputFolder;
+    const name = opts.name ?? (type && outputNameForDefaultRecipe
+        ? `${type}_${outputNameForDefaultRecipe}`
+        : undefined);
+    const temporaryOutputDataset = outputFolder && !requestedOutputDataset && name
+        ? `${name}_folder_output_marker`
+        : undefined;
+    const outputDataset = requestedOutputDataset ?? temporaryOutputDataset;
+    if (!outputs && outputDataset) {
+        outputs = {
+            main: {
+                items: [{ ref: outputDataset, appendMode: false, },],
+            },
+        };
+    }
+    if (!type || !name || !outputs) {
+        throw new ClientValidationError("type and outputDataset/outputFolder or (name + outputs) are required for create.", "validation_failed");
+    }
+    const joinCols = typeof opts.joinOn === "string" ? [opts.joinOn,] : asStringArray(opts.joinOn);
+    const joinKeys = type === "join" && joinCols?.length
+        ? parseJoinKeyPairs(joinCols)
+        : undefined;
+    const fuzzyOn = opts.fuzzyOn ?? (type === "fuzzyjoin" ? opts.joinOn : undefined);
+    const fuzzyCols = typeof fuzzyOn === "string" ? [fuzzyOn,] : asStringArray(fuzzyOn);
+    const fuzzyKeys = type === "fuzzyjoin" && fuzzyCols?.length
+        ? parseJoinKeyPairs(fuzzyCols, "fuzzyOn")
+        : undefined;
+    const normalizedJoinType = joinKeys?.length || fuzzyKeys?.length
+        ? normalizeJoinType(rawJoinType)
+        : undefined;
+    const fuzzyDistance = fuzzyKeys?.length ? normalizeFuzzyDistance(opts.fuzzyDistance) : undefined;
+    const fuzzyThreshold = opts.fuzzyThreshold ?? 1;
+    if (!Number.isFinite(fuzzyThreshold)) {
+        throw new ClientValidationError("fuzzyThreshold must be a finite number.", "validation_failed");
+    }
+    const recipePrototype = {
+        type,
+        name,
+        projectKey: pk,
+        ...(type === "fuzzyjoin" ? {} : { inputs, }),
+        outputs,
+    };
+    const creationSettings = {};
+    if (payload !== undefined) {
+        creationSettings.script = payload;
+    }
+    if (type === "fuzzyjoin") {
+        creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
+    }
+    return {
+        creationSettings,
+        fuzzyDistance,
+        fuzzyKeys,
+        fuzzyThreshold,
+        grouping,
+        inputDatasets,
+        inputs,
+        joinKeys,
+        name,
+        normalizedJoinType,
+        outputFolder,
+        outputs,
+        payload,
+        prepareSteps: steps,
+        rawConnection,
+        recipePrototype,
+        temporaryOutputDataset,
+        type,
+    };
+}

package/dist/src/resources/recipes.d.ts

@@ -1,11 +1,6 @@
-import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
+import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSchemaUpdates, RecipeSummary, RecipeUpdateSchemaResult } from "../schemas.js";
 import { BaseResource } from "./base.js";
 import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
-type JoinType = "LEFT" | "INNER" | "RIGHT" | "FULL";
-type JoinKeyPair = {
-    left: string;
-    right: string;
-};
 export interface RecipeRunOutput extends JobBuildTarget {
     ref: string;
     role: string;
@@ -70,28 +65,6 @@ export interface RecipeCloneResult {
     payloadTextRewrites: Record<string, string>;
     copiedOutputDatasets: string[];
 }
-/**
- * Pure request construction for POST /recipes/ (no DSS calls). Shared with
- * `recipe create --plan` so the plan equals the request the command sends.
- */
-export declare function buildRecipeCreateRequest(opts: RecipeCreateOptions, pk: string): {
-    creationSettings: Record<string, unknown>;
-    fuzzyDistance: string | undefined;
-    fuzzyKeys: JoinKeyPair[] | undefined;
-    fuzzyThreshold: number;
-    inputDatasets: string[] | undefined;
-    inputs: Record<string, unknown>;
-    joinKeys: JoinKeyPair[] | undefined;
-    name: string;
-    normalizedJoinType: JoinType | undefined;
-    outputFolder: string | undefined;
-    outputs: Record<string, unknown>;
-    payload: string | undefined;
-    rawConnection: string | undefined;
-    recipePrototype: Record<string, unknown>;
-    temporaryOutputDataset: string | undefined;
-    type: string;
-};
 export declare class RecipesResource extends BaseResource {
     /** List all recipes in a project. */
     list(projectKey?: string): Promise<RecipeSummary[]>;
@@ -134,6 +107,33 @@ export declare class RecipesResource extends BaseResource {
     run(recipeName: string, opts?: RecipeRunOptions): Promise<RecipeRunResult>;
     /** Create a recipe, with optional output dataset provisioning and join configuration. */
     create(opts: RecipeCreateOptions): Promise<RecipeCreateResult>;
+    /**
+     * Compute the output schemas DSS derives from the recipe's current settings
+     * (GET /recipes/{name}/schema-update, dataikuapi `compute_schema_updates`).
+     * DSS rejects code recipes, whose code sets the schema at run time.
+     */
+    computeSchemaUpdates(recipeName: string, projectKey?: string): Promise<RecipeSchemaUpdates>;
+    /**
+     * Write each pending computed schema accepted by `select` through
+     * actions/updateOutputSchema. Pending outputs `select` rejects are reported,
+     * not written.
+     */
+    private applySchemaUpdates;
+    /**
+     * Replace each output schema that differs from the DSS-computed one
+     * (dataikuapi `compute_schema_updates().apply()`, the UI's "Update schema").
+     * With `derivedOutputs`, only outputs whose schema is empty or listed there
+     * are written: callers pass the outputs whose schema matched the computed
+     * one before an edit, so DSS-derived schemas follow the edit while
+     * hand-edited ones are left alone. `dryRun` writes nothing. Skipped
+     * differing outputs are listed under `pending`. Code recipes are rejected
+     * before the schema computation: DSS cannot compute their output schema.
+     */
+    updateSchema(recipeName: string, opts?: {
+        projectKey?: string;
+        derivedOutputs?: string[];
+        dryRun?: boolean;
+    }): Promise<RecipeUpdateSchemaResult>;
     /**
      * Update a recipe by merging the patch into the current definition.
      * The `recipe` sub-object is deep-merged to preserve nested fields.
@@ -171,4 +171,3 @@ export declare class RecipesResource extends BaseResource {
         projectKey?: string;
     }): Promise<string>;
 }
-export {};