dataiku-sdk 3.6.1 → 3.8.0

package/dist/src/resources/recipe-create.js

@@ -0,0 +1,454 @@
+import { ClientValidationError, } from "../errors.js";
+import { asRecord, } from "../utils/records.js";
+/*
+ * Pure, schema-free recipe-create request construction. `recipe create --plan`
+ * imports this without loading the SDK schema graph.
+ */
+export function asString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+export function asStringArray(value) {
+    if (!Array.isArray(value))
+        return undefined;
+    const out = value.filter((v) => typeof v === "string" && v.length > 0);
+    return out.length > 0 ? out : undefined;
+}
+export function parseJoinKeyPairs(values, optionName = "joinOn") {
+    const pairs = [];
+    for (const raw of values) {
+        const token = raw.trim();
+        if (!token)
+            continue;
+        const eq = token.indexOf("=");
+        if (eq === -1) {
+            pairs.push({ left: token, right: token, });
+            continue;
+        }
+        const left = token.slice(0, eq).trim();
+        const right = token.slice(eq + 1).trim();
+        if (!left || !right) {
+            throw new ClientValidationError(`${optionName} values must use COL or LEFT=RIGHT.`, "validation_failed");
+        }
+        pairs.push({ left, right, });
+    }
+    return pairs;
+}
+export function normalizeFuzzyDistance(value) {
+    const normalized = typeof value === "string" && value.trim().length > 0
+        ? value.trim().toUpperCase()
+        : "DAMERAU_LEVENSHTEIN";
+    if (normalized === "DAMERAU_LEVENSHTEIN")
+        return "LEVENSHTEIN";
+    if (normalized === "HAMMING" || normalized === "JACCARD" || normalized === "COSINE"
+        || normalized === "EUCLIDEAN") {
+        return normalized;
+    }
+    throw new ClientValidationError("fuzzyDistance must be one of DAMERAU_LEVENSHTEIN, HAMMING, JACCARD, COSINE, or EUCLIDEAN.", "validation_failed");
+}
+export function normalizeJoinType(value) {
+    const normalized = typeof value === "string" && value.trim().length > 0
+        ? value.trim().toUpperCase()
+        : "LEFT";
+    if (normalized === "LEFT" || normalized === "INNER" || normalized === "RIGHT"
+        || normalized === "FULL") {
+        return normalized;
+    }
+    throw new ClientValidationError("joinType must be one of LEFT, INNER, RIGHT, or FULL.", "validation_failed");
+}
+export function recipeInputItems(recipe) {
+    const inputs = asRecord(recipe.inputs);
+    if (!inputs)
+        return [];
+    const result = [];
+    const seen = new Set();
+    for (const [role, roleValue,] of Object.entries(inputs)) {
+        const items = asRecord(roleValue)?.items;
+        if (!Array.isArray(items))
+            continue;
+        for (const itemValue of items) {
+            const item = asRecord(itemValue);
+            const ref = asString(item?.ref);
+            if (!ref || seen.has(ref))
+                continue;
+            seen.add(ref);
+            result.push({ ref, role, });
+        }
+    }
+    return result;
+}
+/**
+ * Recipe types whose output schema DSS refuses to compute ("Output schema can't
+ * be automatically computed on 'python' recipes"): the code sets the schema
+ * when it runs. Plugin recipes (`CustomCode_*`) are code too.
+ */
+const CODE_RECIPE_TYPES = {
+    python: true,
+    r: true,
+    julia: true,
+    pyspark: true,
+    sparkr: true,
+    spark_scala: true,
+    shell: true,
+    sql_script: true,
+};
+/** Whether DSS can compute this recipe type's output schema (visual and SQL query recipes). */
+export function recipeOutputSchemaIsComputable(recipeType) {
+    const normalized = recipeType.trim().toLowerCase();
+    return CODE_RECIPE_TYPES[normalized] !== true && !normalized.startsWith("customcode_");
+}
+/** The id DSS expects in actions/updateOutputSchema (dataikuapi RequiredSchemaUpdates.apply). */
+export function schemaUpdateComputableId(computable) {
+    return (computable.type === "DATASET" ? computable.datasetName : computable.id) ?? "";
+}
+/** A computed schema is worth writing when it has columns and differs from (or fills) the current one. */
+export function schemaUpdateIsPending(computable) {
+    const columns = computable.newSchema?.columns ?? [];
+    if (columns.length === 0)
+        return false;
+    return computable.previousSchemaWasEmpty === true
+        || (computable.incompatibilities?.length ?? 0) > 0;
+}
+/** DSS grouping aggregate flags on each `values[]` entry, by lowercase name. */
+const GROUPING_AGGREGATES = {
+    count: "count",
+    countdistinct: "countDistinct",
+    sum: "sum",
+    avg: "avg",
+    min: "min",
+    max: "max",
+    median: "median",
+    stddev: "stddev",
+    first: "first",
+    last: "last",
+    concat: "concat",
+    concatdistinct: "concatDistinct",
+    sum2: "sum2",
+};
+function splitPair(raw, flag) {
+    const eq = raw.indexOf("=");
+    const left = eq === -1 ? "" : raw.slice(0, eq).trim();
+    const right = eq === -1 ? "" : raw.slice(eq + 1).trim();
+    if (left.length === 0 || right.length === 0) {
+        throw new ClientValidationError(`${flag} expects NAME=VALUE, got "${raw}".`, "validation_failed");
+    }
+    return [left, right,];
+}
+/** Grouping configuration from create options, or undefined when none was asked for. */
+export function groupingPayloadConfig(opts) {
+    const keys = opts.groupBy ?? [];
+    const specs = opts.aggregate ?? [];
+    if (keys.length === 0 && specs.length === 0)
+        return undefined;
+    const aggregates = {};
+    for (const spec of specs) {
+        const colon = spec.lastIndexOf(":");
+        const column = colon === -1 ? "" : spec.slice(0, colon).trim();
+        const fns = colon === -1
+            ? []
+            : spec.slice(colon + 1).split("+").map((fn) => fn.trim().toLowerCase());
+        if (column.length === 0 || fns.length === 0
+            || fns.some((fn) => GROUPING_AGGREGATES[fn] === undefined)) {
+            throw new ClientValidationError(`--aggregate expects COL:FN[+FN] with FN one of ${Object.values(GROUPING_AGGREGATES).join(", ")}; got "${spec}".`, "validation_failed");
+        }
+        aggregates[column] = [
+            ...(aggregates[column] ?? []),
+            ...fns.map((fn) => GROUPING_AGGREGATES[fn]),
+        ];
+    }
+    const ordered = Object.values(aggregates).some((fns) => fns.includes("first") || fns.includes("last"));
+    if (ordered && !opts.orderBy) {
+        throw new ClientValidationError("first/last aggregates need an ordering column: pass --order-by COL.", "validation_failed");
+    }
+    return { keys, aggregates, ...(opts.orderBy ? { orderBy: opts.orderBy, } : {}), };
+}
+/**
+ * Apply grouping keys and aggregates to the payload DSS created: `keys` lists
+ * the group columns; each aggregated column's `values[]` entry gets its
+ * aggregate flags (and `orderColumn` for first/last).
+ */
+export function applyGroupingPayload(payload, config) {
+    const values = Array.isArray(payload.values)
+        ? payload.values.map((value) => asRecord(value) ?? {})
+        : [];
+    for (const [column, fns,] of Object.entries(config.aggregates)) {
+        let entry = values.find((value) => value.column === column);
+        if (!entry) {
+            entry = { column, };
+            values.push(entry);
+        }
+        for (const fn of fns)
+            entry[fn] = true;
+        if (config.orderBy && (fns.includes("first") || fns.includes("last"))) {
+            entry.orderColumn = config.orderBy;
+        }
+    }
+    return { ...payload, keys: config.keys.map((column) => ({ column, })), values, };
+}
+function processorStep(type, params) {
+    return {
+        metaType: "PROCESSOR",
+        type,
+        params,
+        disabled: false,
+        preview: false,
+        alwaysShowComment: false,
+    };
+}
+/** Prepare option groups, in the default step order. */
+export const PREPARE_STEP_KINDS = [
+    "rename",
+    "fillEmpty",
+    "formula",
+    "filter",
+    "dropColumns",
+    "keepColumns",
+];
+function isPrepareStepKind(value) {
+    return PREPARE_STEP_KINDS.includes(value);
+}
+/**
+ * Order the prepare option groups run in: `stepOrder` (the CLI passes the order
+ * the flags were typed in) first, then any group it leaves out in the default order.
+ */
+function prepareStepOrder(opts) {
+    const order = [];
+    for (const kind of opts.stepOrder ?? []) {
+        if (!isPrepareStepKind(kind)) {
+            throw new ClientValidationError(`Unknown stepOrder entry "${kind}"; expected one of ${PREPARE_STEP_KINDS.join(", ")}.`, "invalid_flag_value");
+        }
+        if (!order.includes(kind))
+            order.push(kind);
+    }
+    for (const kind of PREPARE_STEP_KINDS) {
+        if (!order.includes(kind))
+            order.push(kind);
+    }
+    return order;
+}
+/** Prepare steps for one option group (empty when the option was not given). */
+function prepareStepsFor(kind, opts) {
+    switch (kind) {
+        case "rename": {
+            const renamings = (opts.rename ?? []).map((raw) => {
+                const [from, to,] = splitPair(raw, "--rename");
+                return { from, to, };
+            });
+            return renamings.length > 0 ? [processorStep("ColumnRenamer", { renamings, }),] : [];
+        }
+        case "fillEmpty":
+            return (opts.fillEmpty ?? []).map((raw) => {
+                const [column, value,] = splitPair(raw, "--fill-empty");
+                return processorStep("FillEmptyWithValue", {
+                    appliesTo: "SINGLE_COLUMN",
+                    columns: [column,],
+                    value,
+                });
+            });
+        case "formula": {
+            if (!opts.formula)
+                return [];
+            const [column, expression,] = splitPair(opts.formula, "--formula");
+            return [processorStep("CreateColumnWithGREL", { column, expression, }),];
+        }
+        case "filter":
+            return opts.filter
+                ? [processorStep("FilterOnCustomFormula", { expression: opts.filter, action: "KEEP_ROW", }),]
+                : [];
+        case "dropColumns":
+            return opts.dropColumns?.length
+                ? [processorStep("ColumnsSelector", {
+                        appliesTo: "COLUMNS",
+                        columns: opts.dropColumns,
+                        keep: false,
+                    }),]
+                : [];
+        case "keepColumns":
+            return opts.keepColumns?.length
+                ? [processorStep("ColumnsSelector", {
+                        appliesTo: "COLUMNS",
+                        columns: opts.keepColumns,
+                        keep: true,
+                    }),]
+                : [];
+    }
+}
+/**
+ * Prepare (shaker) steps from create options: rename, fill empty, formula
+ * column, row filter, drop columns, keep columns. They run in `stepOrder`
+ * (flag order on the CLI), defaulting to that sequence, so a step sees the
+ * columns the steps before it produced. Empty when no prepare option was given.
+ */
+export function prepareSteps(opts) {
+    return prepareStepOrder(opts).flatMap((kind) => prepareStepsFor(kind, opts));
+}
+/**
+ * Columns an expression refers to, conservatively: `val('name')` arguments
+ * plus bare identifiers outside string literals (not method names, calls, or
+ * members of another object).
+ */
+function expressionColumnReferences(expression) {
+    const references = new Set();
+    for (const match of expression.matchAll(/\bval\(\s*(['"])((?:\\.|(?!\1).)*)\1\s*\)/g)) {
+        references.add(match[2]);
+    }
+    const withoutStrings = expression.replace(/(['"])(?:\\.|(?!\1).)*\1/g, " ");
+    for (const match of withoutStrings.matchAll(/(?<![\w.$])([A-Za-z_]\w*)(?!\w)(?!\s*\()/g)) {
+        references.add(match[1]);
+    }
+    return references;
+}
+/**
+ * Warnings about the prepare options that DSS cannot report itself: a
+ * `--formula`/`--filter` that runs after a `--rename` but still names a
+ * column that rename removed (it would fail or read an empty column).
+ */
+export function recipeCreateWarnings(opts) {
+    const warnings = [];
+    const removed = new Map();
+    for (const kind of prepareStepOrder(opts)) {
+        if (kind === "rename") {
+            const renamings = (opts.rename ?? []).map((raw) => splitPair(raw, "--rename"));
+            const targets = new Set(renamings.map(([, to,]) => to));
+            for (const [from, to,] of renamings) {
+                if (!targets.has(from))
+                    removed.set(from, to);
+            }
+        }
+        else if ((kind === "formula" && opts.formula) || (kind === "filter" && opts.filter)) {
+            const expression = kind === "formula"
+                ? splitPair(opts.formula, "--formula")[1]
+                : opts.filter;
+            const stale = [...expressionColumnReferences(expression),].filter((name) => removed.has(name));
+            if (stale.length === 0)
+                continue;
+            const flag = kind === "formula" ? "--formula" : "--filter";
+            warnings.push({
+                code: "recipe_formula_uses_renamed_column",
+                flag,
+                columns: stale.map((name) => ({ renamed: name, to: removed.get(name), })),
+                message: `${flag} runs after --rename and refers to ${stale.map((name) => `"${name}"`).join(", ")}, which the rename removed.`,
+                hint: `Use the new name (${stale.map((name) => removed.get(name)).join(", ")}), or put ${flag} before --rename on the command line: steps run in flag order.`,
+            });
+        }
+    }
+    return warnings;
+}
+/**
+ * Recipe types created from `creationSettings.virtualInputs` (dataikuapi's
+ * VirtualInputsSingleOutputRecipeCreator subclasses: join, fuzzyjoin, geojoin,
+ * vstack, generate_features). Live DSS 15: without it these are created with
+ * no inputs (and `generate_features` fails with a NullPointerException).
+ */
+const VIRTUAL_INPUT_RECIPE_TYPES = {
+    vstack: true,
+    join: true,
+    fuzzyjoin: true,
+    geojoin: true,
+    generate_features: true,
+};
+/**
+ * Pure request construction for POST /recipes/ (no DSS calls). Shared with
+ * `recipe create --plan` so the plan equals the request the command sends.
+ */
+export function buildRecipeCreateRequest(opts, pk) {
+    const { payload, outputConnection: rawConnection, joinType: rawJoinType, } = opts;
+    // dataikuapi and the UI call the shaker recipe "prepare".
+    const type = opts.type === "prepare" ? "shaker" : opts.type;
+    const grouping = groupingPayloadConfig(opts);
+    const steps = prepareSteps(opts);
+    if (grouping && type !== "grouping") {
+        throw new ClientValidationError("--group-by/--aggregate/--order-by need --type grouping.", "validation_failed");
+    }
+    if (steps.length > 0 && type !== "shaker") {
+        throw new ClientValidationError("--rename/--fill-empty/--formula/--filter/--drop-columns/--keep-columns need --type prepare.", "validation_failed");
+    }
+    const outputFolder = asString(opts.outputFolder);
+    // Build inputs/outputs from simple form (inputDatasets + outputDataset) or
+    // advanced form (inputs + outputs); both may coexist — simple form wins when
+    // the advanced form is absent.
+    const inputDatasets = asStringArray(opts.inputDatasets);
+    const requestedOutputDataset = asString(opts.outputDataset);
+    let inputs = asRecord(opts.inputs);
+    let outputs = asRecord(opts.outputs);
+    if (!inputs) {
+        inputs = {
+            main: {
+                items: inputDatasets?.map((ref) => ({ ref, deps: [], })) ?? [],
+            },
+        };
+    }
+    // Auto-generate name if not provided
+    const outputNameForDefaultRecipe = requestedOutputDataset ?? outputFolder;
+    const name = opts.name ?? (type && outputNameForDefaultRecipe
+        ? `${type}_${outputNameForDefaultRecipe}`
+        : undefined);
+    const temporaryOutputDataset = outputFolder && !requestedOutputDataset && name
+        ? `${name}_folder_output_marker`
+        : undefined;
+    const outputDataset = requestedOutputDataset ?? temporaryOutputDataset;
+    if (!outputs && outputDataset) {
+        outputs = {
+            main: {
+                items: [{ ref: outputDataset, appendMode: false, },],
+            },
+        };
+    }
+    if (!type || !name || !outputs) {
+        throw new ClientValidationError("type and outputDataset/outputFolder or (name + outputs) are required for create.", "validation_failed");
+    }
+    const joinCols = typeof opts.joinOn === "string" ? [opts.joinOn,] : asStringArray(opts.joinOn);
+    const joinKeys = type === "join" && joinCols?.length
+        ? parseJoinKeyPairs(joinCols)
+        : undefined;
+    const fuzzyOn = opts.fuzzyOn ?? (type === "fuzzyjoin" ? opts.joinOn : undefined);
+    const fuzzyCols = typeof fuzzyOn === "string" ? [fuzzyOn,] : asStringArray(fuzzyOn);
+    const fuzzyKeys = type === "fuzzyjoin" && fuzzyCols?.length
+        ? parseJoinKeyPairs(fuzzyCols, "fuzzyOn")
+        : undefined;
+    const normalizedJoinType = joinKeys?.length || fuzzyKeys?.length
+        ? normalizeJoinType(rawJoinType)
+        : undefined;
+    const fuzzyDistance = fuzzyKeys?.length ? normalizeFuzzyDistance(opts.fuzzyDistance) : undefined;
+    const fuzzyThreshold = opts.fuzzyThreshold ?? 1;
+    if (!Number.isFinite(fuzzyThreshold)) {
+        throw new ClientValidationError("fuzzyThreshold must be a finite number.", "validation_failed");
+    }
+    // Virtual-input recipes: DSS builds `inputs` from creationSettings.virtualInputs
+    // (dataikuapi VirtualInputsSingleOutputRecipeCreator) and ignores or drops the
+    // prototype's inputs, creating the recipe with `inputs: {}` and an empty payload.
+    const hasVirtualInputs = VIRTUAL_INPUT_RECIPE_TYPES[type] === true;
+    const recipePrototype = {
+        type,
+        name,
+        projectKey: pk,
+        ...(type === "fuzzyjoin" ? {} : { inputs, }),
+        outputs,
+    };
+    const creationSettings = {};
+    if (payload !== undefined) {
+        creationSettings.script = payload;
+    }
+    if (hasVirtualInputs) {
+        creationSettings.virtualInputs = recipeInputItems({ inputs, }).map((item) => item.ref);
+    }
+    return {
+        creationSettings,
+        fuzzyDistance,
+        fuzzyKeys,
+        fuzzyThreshold,
+        grouping,
+        inputDatasets,
+        inputs,
+        joinKeys,
+        name,
+        normalizedJoinType,
+        outputFolder,
+        outputs,
+        payload,
+        prepareSteps: steps,
+        rawConnection,
+        recipePrototype,
+        temporaryOutputDataset,
+        type,
+    };
+}

package/dist/src/resources/recipes.d.ts

@@ -1,11 +1,6 @@
-import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
+import type { BuildMode, JobWaitResult, ProjectMetadata, RecipeCreateOptions, RecipeCreateResult, RecipeSchemaUpdates, RecipeSummary, RecipeUpdateSchemaResult } from "../schemas.js";
 import { BaseResource } from "./base.js";
 import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
-type JoinType = "LEFT" | "INNER" | "RIGHT" | "FULL";
-type JoinKeyPair = {
-    left: string;
-    right: string;
-};
 export interface RecipeRunOutput extends JobBuildTarget {
     ref: string;
     role: string;
@@ -70,28 +65,6 @@ export interface RecipeCloneResult {
     payloadTextRewrites: Record<string, string>;
     copiedOutputDatasets: string[];
 }
-/**
- * Pure request construction for POST /recipes/ (no DSS calls). Shared with
- * `recipe create --plan` so the plan equals the request the command sends.
- */
-export declare function buildRecipeCreateRequest(opts: RecipeCreateOptions, pk: string): {
-    creationSettings: Record<string, unknown>;
-    fuzzyDistance: string | undefined;
-    fuzzyKeys: JoinKeyPair[] | undefined;
-    fuzzyThreshold: number;
-    inputDatasets: string[] | undefined;
-    inputs: Record<string, unknown>;
-    joinKeys: JoinKeyPair[] | undefined;
-    name: string;
-    normalizedJoinType: JoinType | undefined;
-    outputFolder: string | undefined;
-    outputs: Record<string, unknown>;
-    payload: string | undefined;
-    rawConnection: string | undefined;
-    recipePrototype: Record<string, unknown>;
-    temporaryOutputDataset: string | undefined;
-    type: string;
-};
 export declare class RecipesResource extends BaseResource {
     /** List all recipes in a project. */
     list(projectKey?: string): Promise<RecipeSummary[]>;
@@ -134,6 +107,33 @@ export declare class RecipesResource extends BaseResource {
     run(recipeName: string, opts?: RecipeRunOptions): Promise<RecipeRunResult>;
     /** Create a recipe, with optional output dataset provisioning and join configuration. */
     create(opts: RecipeCreateOptions): Promise<RecipeCreateResult>;
+    /**
+     * Compute the output schemas DSS derives from the recipe's current settings
+     * (GET /recipes/{name}/schema-update, dataikuapi `compute_schema_updates`).
+     * DSS rejects code recipes, whose code sets the schema at run time.
+     */
+    computeSchemaUpdates(recipeName: string, projectKey?: string): Promise<RecipeSchemaUpdates>;
+    /**
+     * Write each pending computed schema accepted by `select` through
+     * actions/updateOutputSchema. Pending outputs `select` rejects are reported,
+     * not written.
+     */
+    private applySchemaUpdates;
+    /**
+     * Replace each output schema that differs from the DSS-computed one
+     * (dataikuapi `compute_schema_updates().apply()`, the UI's "Update schema").
+     * With `derivedOutputs`, only outputs whose schema is empty or listed there
+     * are written: callers pass the outputs whose schema matched the computed
+     * one before an edit, so DSS-derived schemas follow the edit while
+     * hand-edited ones are left alone. `dryRun` writes nothing. Skipped
+     * differing outputs are listed under `pending`. Code recipes are rejected
+     * before the schema computation: DSS cannot compute their output schema.
+     */
+    updateSchema(recipeName: string, opts?: {
+        projectKey?: string;
+        derivedOutputs?: string[];
+        dryRun?: boolean;
+    }): Promise<RecipeUpdateSchemaResult>;
     /**
      * Update a recipe by merging the patch into the current definition.
      * The `recipe` sub-object is deep-merged to preserve nested fields.
@@ -171,4 +171,3 @@ export declare class RecipesResource extends BaseResource {
         projectKey?: string;
     }): Promise<string>;
 }
-export {};