dataiku-sdk 0.5.1 → 0.6.1

package/dist/src/resources/recipes.d.ts

@@ -1,5 +1,69 @@
-import type { RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
+import type { BuildMode, JobWaitResult, RecipeCreateOptions, RecipeCreateResult, RecipeSummary } from "../schemas.js";
 import { BaseResource } from "./base.js";
+import type { JobBuildTarget, JobBuildTargetType, JobLogFilter, JobLogSummary } from "./jobs.js";
+export interface RecipeRunOutput extends JobBuildTarget {
+    ref: string;
+    role: string;
+}
+export interface RecipeGraphReference {
+    ref: string;
+    role: string;
+    type?: JobBuildTargetType;
+    exists: boolean;
+    id?: string;
+}
+export interface RecipeGraphValidationResult {
+    valid: boolean;
+    recipeName: string;
+    projectKey: string;
+    inputs: RecipeGraphReference[];
+    outputs: RecipeGraphReference[];
+    missingInputs: RecipeGraphReference[];
+    missingOutputs: RecipeGraphReference[];
+    ambiguousOutputs: string[];
+    warnings: string[];
+}
+export interface RecipeRunOptions {
+    buildMode?: BuildMode;
+    includeLogs?: boolean;
+    maxLogLines?: number;
+    partition?: string;
+    pollIntervalMs?: number;
+    projectKey?: string;
+    wait?: boolean;
+    timeoutMs?: number;
+    logFilter?: JobLogFilter;
+    summary?: boolean;
+}
+export type RecipeRunResult = {
+    logSummary?: JobLogSummary;
+    recipeName: string;
+    outputs: RecipeRunOutput[];
+} & ({
+    jobId: string;
+} | JobWaitResult);
+export interface RecipeCloneOptions {
+    projectKey?: string;
+    name: string;
+    outputDataset?: string;
+    outputRewrites?: Record<string, string>;
+    inputRewrites?: Record<string, string>;
+    payloadRewrites?: Record<string, string>;
+    payloadTextRewrites?: Record<string, string>;
+    copyOutputSettings?: boolean;
+    outputPath?: string;
+    metastoreTableName?: string;
+}
+export interface RecipeCloneResult {
+    sourceRecipeName: string;
+    recipeName: string;
+    projectKey: string;
+    outputRewrites: Record<string, string>;
+    inputRewrites: Record<string, string>;
+    payloadRewrites: Record<string, string>;
+    payloadTextRewrites: Record<string, string>;
+    copiedOutputDatasets: string[];
+}
 export declare class RecipesResource extends BaseResource {
     /** List all recipes in a project. */
     list(projectKey?: string): Promise<RecipeSummary[]>;
@@ -15,6 +79,17 @@ export declare class RecipesResource extends BaseResource {
         recipe: Record<string, unknown>;
         payload?: string;
     }>;
+    /** Validate declared recipe graph references before running/building. */
+    validateGraph(recipeName: string, opts?: {
+        projectKey?: string;
+    }): Promise<RecipeGraphValidationResult>;
+    /** Resolve recipe outputs to job-build targets. */
+    resolveRunOutputs(recipeName: string, opts?: {
+        partition?: string;
+        projectKey?: string;
+    }): Promise<RecipeRunOutput[]>;
+    /** Run a recipe by building its resolved outputs. */
+    run(recipeName: string, opts?: RecipeRunOptions): Promise<RecipeRunResult>;
     /** Create a recipe, with optional output dataset provisioning and join configuration. */
     create(opts: RecipeCreateOptions): Promise<RecipeCreateResult>;
     /**
@@ -22,6 +97,10 @@ export declare class RecipesResource extends BaseResource {
      * The `recipe` sub-object is deep-merged to preserve nested fields.
      */
     update(recipeName: string, data: Record<string, unknown>, projectKey?: string): Promise<void>;
+    /** Replace a full recipe API document. */
+    replace(recipeName: string, document: Record<string, unknown>, projectKey?: string): Promise<void>;
+    /** Clone recipe graph/settings and optionally clone a dataset output. */
+    clone(sourceName: string, opts: RecipeCloneOptions): Promise<RecipeCloneResult>;
     /**
     * Download a recipe code payload to a local file.
 

package/dist/src/resources/recipes.js

@@ -26,6 +26,114 @@ const RECIPE_DEFINITION_FIELDS = new Set(["params", "inputs", "outputs", "script
 function rootRecipeDefinitionFields(data) {
     return Object.keys(data).filter((key) => RECIPE_DEFINITION_FIELDS.has(key));
 }
+function normalizeRecipeOutputType(value) {
+    if (typeof value !== "string")
+        return undefined;
+    const normalized = value.trim().toUpperCase().replace(/-/g, "_");
+    if (normalized === "DATASET")
+        return "DATASET";
+    if (normalized === "MANAGED_FOLDER" || normalized === "FOLDER")
+        return "MANAGED_FOLDER";
+    return undefined;
+}
+function recipeOutputItems(recipe) {
+    const outputs = asRecord(recipe.outputs);
+    if (!outputs)
+        return [];
+    const result = [];
+    const seen = new Set();
+    for (const [role, roleValue,] of Object.entries(outputs)) {
+        const items = asRecord(roleValue)?.items;
+        if (!Array.isArray(items))
+            continue;
+        for (const itemValue of items) {
+            const item = asRecord(itemValue);
+            const ref = asString(item?.ref);
+            if (!ref)
+                continue;
+            const seenKey = ref;
+            if (seen.has(seenKey))
+                continue;
+            seen.add(seenKey);
+            const type = normalizeRecipeOutputType(item?.type ?? item?.targetType ?? item?.objectType);
+            result.push({
+                ref,
+                role,
+                ...(type ? { type, } : {}),
+            });
+        }
+    }
+    return result;
+}
+function recipeInputItems(recipe) {
+    const inputs = asRecord(recipe.inputs);
+    if (!inputs)
+        return [];
+    const result = [];
+    const seen = new Set();
+    for (const [role, roleValue,] of Object.entries(inputs)) {
+        const items = asRecord(roleValue)?.items;
+        if (!Array.isArray(items))
+            continue;
+        for (const itemValue of items) {
+            const item = asRecord(itemValue);
+            const ref = asString(item?.ref);
+            if (!ref || seen.has(ref))
+                continue;
+            seen.add(ref);
+            result.push({ ref, role, });
+        }
+    }
+    return result;
+}
+function rewriteRefs(value, rewrites) {
+    if (Object.keys(rewrites).length === 0)
+        return value;
+    if (Array.isArray(value))
+        return value.map((item) => rewriteRefs(item, rewrites));
+    const record = asRecord(value);
+    if (!record)
+        return value;
+    const next = {};
+    for (const [key, item,] of Object.entries(record)) {
+        if (key === "ref" && typeof item === "string" && rewrites[item]) {
+            next[key] = rewrites[item];
+        }
+        else {
+            next[key] = rewriteRefs(item, rewrites);
+        }
+    }
+    return next;
+}
+function escapedRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function rewritePayload(payload, rewrites, payloadTextRewrites = {}) {
+    if (payload === undefined
+        || (Object.keys(rewrites).length === 0 && Object.keys(payloadTextRewrites).length === 0)) {
+        return payload;
+    }
+    let next = payload;
+    for (const [from, to,] of Object.entries(rewrites)) {
+        if (!from)
+            continue;
+        const escaped = escapedRegExp(from);
+        next = next.replace(new RegExp(`\\bdataiku\\.(Dataset|Folder)\\(\\s*(['"])${escaped}\\2\\s*\\)`, "g"), (_match, kind, quote) => `dataiku.${kind}(${quote}${to}${quote})`);
+    }
+    for (const [from, to,] of Object.entries(payloadTextRewrites)) {
+        if (from.length > 0)
+            next = next.split(from).join(to);
+    }
+    return next;
+}
+function cloneRecipeDefinition(recipe, targetName, projectKey, rewrites) {
+    const cloned = rewriteRefs(structuredClone(recipe), rewrites);
+    delete cloned.versionTag;
+    delete cloned.neverBuilt;
+    cloned.name = targetName;
+    cloned.projectKey = projectKey;
+    return cloned;
+}
 function inferRecipeCodeExtension(recipeType) {
     const normalized = typeof recipeType === "string" ? recipeType.trim().toLowerCase() : "";
     if (!normalized)
@@ -92,6 +200,181 @@ export class RecipesResource extends BaseResource {
         }
         return opts?.includePayload ? { ...result, recipe, } : { recipe, };
     }
+    /** Validate declared recipe graph references before running/building. */
+    async validateGraph(recipeName, opts) {
+        const pk = this.resolveProjectKey(opts?.projectKey);
+        const { recipe, } = await this.get(recipeName, { projectKey: pk, });
+        const inputItems = recipeInputItems(recipe);
+        const outputItems = recipeOutputItems(recipe);
+        const [datasets, folders,] = await Promise.all([
+            this.client.datasets.list(pk),
+            this.client.folders.list(pk),
+        ]);
+        const datasetNames = new Set(datasets.map((dataset) => dataset.name));
+        const folderIdByRef = new Map();
+        for (const folder of folders) {
+            folderIdByRef.set(folder.id, folder.id);
+            if (folder.name)
+                folderIdByRef.set(folder.name, folder.id);
+        }
+        const resolveReference = (item, requireExplicitOutputType) => {
+            const folderId = folderIdByRef.get(item.ref);
+            const isDataset = datasetNames.has(item.ref);
+            if (item.type === "DATASET") {
+                return { ref: item.ref, role: item.role, type: "DATASET", exists: isDataset, id: item.ref, };
+            }
+            if (item.type === "MANAGED_FOLDER") {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    type: "MANAGED_FOLDER",
+                    exists: folderId !== undefined,
+                    id: folderId ?? item.ref,
+                };
+            }
+            if (isDataset && (!folderId || !requireExplicitOutputType)) {
+                return { ref: item.ref, role: item.role, type: "DATASET", exists: true, id: item.ref, };
+            }
+            if (folderId && !isDataset) {
+                return { ref: item.ref, role: item.role, type: "MANAGED_FOLDER", exists: true, id: folderId, };
+            }
+            return { ref: item.ref, role: item.role, exists: false, };
+        };
+        const inputs = inputItems.map((item) => resolveReference(item, false));
+        const outputs = outputItems.map((item) => resolveReference(item, true));
+        const ambiguousOutputs = outputItems
+            .filter((item) => !item.type && datasetNames.has(item.ref) && folderIdByRef.has(item.ref))
+            .map((item) => item.ref);
+        const missingInputs = inputs.filter((item) => !item.exists);
+        const missingOutputs = outputs.filter((item) => !item.exists);
+        const warnings = [];
+        if (outputItems.length === 0)
+            warnings.push("Recipe has no declared outputs to build.");
+        for (const ref of ambiguousOutputs) {
+            warnings.push(`Output "${ref}" matches both a dataset and a managed folder; declare an explicit output type.`);
+        }
+        for (const output of outputs) {
+            if (!output.exists) {
+                warnings.push(`Declared output "${output.ref}" was not found in project "${pk}".`);
+            }
+        }
+        for (const input of missingInputs) {
+            warnings.push(`Declared input "${input.ref}" was not found in project "${pk}".`);
+        }
+        return {
+            valid: missingInputs.length === 0
+                && missingOutputs.length === 0
+                && ambiguousOutputs.length === 0
+                && outputItems.length > 0,
+            recipeName,
+            projectKey: pk,
+            inputs,
+            outputs,
+            missingInputs,
+            missingOutputs,
+            ambiguousOutputs,
+            warnings,
+        };
+    }
+    /** Resolve recipe outputs to job-build targets. */
+    async resolveRunOutputs(recipeName, opts) {
+        const pk = this.resolveProjectKey(opts?.projectKey);
+        const { recipe, } = await this.get(recipeName, { projectKey: pk, });
+        const outputItems = recipeOutputItems(recipe);
+        if (outputItems.length === 0) {
+            throw new Error(`Recipe "${recipeName}" has no output items to build.`);
+        }
+        const [datasets, folders,] = await Promise.all([
+            this.client.datasets.list(pk),
+            this.client.folders.list(pk),
+        ]);
+        const datasetNames = new Set(datasets.map((dataset) => dataset.name));
+        const folderIdByRef = new Map();
+        for (const folder of folders) {
+            folderIdByRef.set(folder.id, folder.id);
+            if (folder.name)
+                folderIdByRef.set(folder.name, folder.id);
+        }
+        return outputItems.map((item) => {
+            if (item.type === "DATASET") {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: item.ref,
+                    type: "DATASET",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            const folderId = folderIdByRef.get(item.ref);
+            if (item.type === "MANAGED_FOLDER") {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: folderId ?? item.ref,
+                    type: "MANAGED_FOLDER",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            const isDataset = datasetNames.has(item.ref);
+            if (isDataset && folderId) {
+                throw new Error(`Recipe "${recipeName}" output "${item.ref}" matches both a dataset and a managed folder. Add an explicit output type to the recipe definition or build the target directly with --target-type.`);
+            }
+            if (folderId) {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: folderId,
+                    type: "MANAGED_FOLDER",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            if (isDataset) {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: item.ref,
+                    type: "DATASET",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            throw new Error(`Recipe "${recipeName}" output "${item.ref}" was not found as a dataset or managed folder in project "${pk}".`);
+        });
+    }
+    /** Run a recipe by building its resolved outputs. */
+    async run(recipeName, opts) {
+        const pk = this.resolveProjectKey(opts?.projectKey);
+        const outputs = await this.resolveRunOutputs(recipeName, {
+            partition: opts?.partition,
+            projectKey: pk,
+        });
+        const shouldWait = opts?.wait === true
+            || opts?.includeLogs === true
+            || opts?.summary === true
+            || opts?.timeoutMs !== undefined
+            || opts?.pollIntervalMs !== undefined;
+        if (shouldWait) {
+            const waitResult = await this.client.jobs.buildAndWaitOutputs(outputs, {
+                buildMode: opts?.buildMode,
+                includeLogs: opts?.includeLogs,
+                maxLogLines: opts?.maxLogLines,
+                logFilter: opts?.logFilter,
+                pollIntervalMs: opts?.pollIntervalMs,
+                projectKey: pk,
+                timeoutMs: opts?.timeoutMs,
+                summary: opts?.summary,
+            });
+            return { recipeName, outputs, ...waitResult, };
+        }
+        const started = await this.client.jobs.buildOutputs(outputs, {
+            buildMode: opts?.buildMode,
+            projectKey: pk,
+        });
+        return { recipeName, outputs, ...started, };
+    }
     /** Create a recipe, with optional output dataset provisioning and join configuration. */
     async create(opts) {
         const pk = this.resolveProjectKey(opts.projectKey);
@@ -327,6 +610,72 @@ export class RecipesResource extends BaseResource {
         const merged = { ...current, ...data, recipe: mergedRecipe, };
         await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, merged);
     }
+    /** Replace a full recipe API document. */
+    async replace(recipeName, document, projectKey) {
+        const enc = this.enc(projectKey);
+        const rnEnc = encodeURIComponent(recipeName);
+        await this.client.put(`/public/api/projects/${enc}/recipes/${rnEnc}`, document);
+    }
+    /** Clone recipe graph/settings and optionally clone a dataset output. */
+    async clone(sourceName, opts) {
+        const pk = this.resolveProjectKey(opts.projectKey);
+        const source = await this.get(sourceName, { includePayload: true, projectKey: pk, });
+        const outputRewrites = {};
+        if (opts.outputRewrites)
+            Object.assign(outputRewrites, opts.outputRewrites);
+        if (opts.outputDataset !== undefined) {
+            const outputs = recipeOutputItems(source.recipe).filter((item) => item.type !== "MANAGED_FOLDER");
+            if (outputs.length !== 1 && Object.keys(outputRewrites).length === 0) {
+                throw new Error(`Recipe "${sourceName}" has ${outputs.length} dataset outputs; pass explicit outputRewrites instead of outputDataset.`);
+            }
+            if (outputs[0])
+                outputRewrites[outputs[0].ref] = opts.outputDataset;
+        }
+        const inputRewrites = {};
+        if (opts.inputRewrites)
+            Object.assign(inputRewrites, opts.inputRewrites);
+        const graphRewrites = { ...inputRewrites, ...outputRewrites, };
+        const payloadRewrites = { ...graphRewrites, };
+        if (opts.payloadRewrites)
+            Object.assign(payloadRewrites, opts.payloadRewrites);
+        if (opts.copyOutputSettings === true
+            && Object.keys(outputRewrites).length > 1
+            && (opts.outputPath !== undefined || opts.metastoreTableName !== undefined)) {
+            throw new Error("Cannot reuse --path or --metastore-table for multiple cloned output datasets; pass per-output settings in a separate step.");
+        }
+        const payloadTextRewrites = {};
+        if (opts.payloadTextRewrites)
+            Object.assign(payloadTextRewrites, opts.payloadTextRewrites);
+        const recipe = cloneRecipeDefinition(source.recipe, opts.name, pk, graphRewrites);
+        const payload = rewritePayload(source.payload, payloadRewrites, payloadTextRewrites);
+        const copiedOutputDatasets = [];
+        if (opts.copyOutputSettings) {
+            for (const [from, to,] of Object.entries(outputRewrites)) {
+                await this.client.datasets.clone(from, to, {
+                    projectKey: pk,
+                    path: opts.outputPath,
+                    metastoreTableName: opts.metastoreTableName,
+                });
+                copiedOutputDatasets.push(to);
+            }
+        }
+        const rnEnc = encodeURIComponent(opts.name);
+        await this.client.post(`/public/api/projects/${encodeURIComponent(pk)}/recipes/`, {
+            recipePrototype: recipe,
+            creationSettings: payload !== undefined ? { script: payload, } : {},
+        });
+        await this.client.put(`/public/api/projects/${encodeURIComponent(pk)}/recipes/${rnEnc}`, { recipe, ...(payload !== undefined ? { payload, } : {}), });
+        return {
+            sourceRecipeName: sourceName,
+            recipeName: opts.name,
+            projectKey: pk,
+            outputRewrites,
+            inputRewrites,
+            payloadRewrites,
+            payloadTextRewrites,
+            copiedOutputDatasets,
+        };
+    }
     /**
     * Download a recipe code payload to a local file.
 

package/dist/src/resources/scenarios.d.ts

@@ -1,5 +1,41 @@
 import type { ScenarioDetails, ScenarioStatus, ScenarioSummary, ScenarioWaitResult } from "../schemas.js";
 import { BaseResource } from "./base.js";
+export declare const SCENARIO_CANONICAL_EDITABLE_FIELDS: readonly ["params.steps", "params.triggers", "params.reporters", "params.customScript", "active", "name"];
+export interface ScenarioUpdateNormalization {
+    from: string;
+    to: string;
+    action: "promoted" | "ignored";
+    message: string;
+}
+export interface ScenarioFieldChange {
+    path: string;
+    before: unknown;
+    after: unknown;
+}
+export interface ScenarioFieldMismatch {
+    path: string;
+    expected: unknown;
+    actual: unknown;
+}
+export interface ScenarioUpdatePreview {
+    canonicalEditableFields: typeof SCENARIO_CANONICAL_EDITABLE_FIELDS;
+    normalization: ScenarioUpdateNormalization[];
+    normalizedData: Record<string, unknown>;
+    current: Record<string, unknown>;
+    next: Record<string, unknown>;
+    changes: ScenarioFieldChange[];
+    unchangedPaths: string[];
+}
+export interface ScenarioUpdateResult extends ScenarioUpdatePreview {
+    after: Record<string, unknown>;
+    verified: true;
+    mismatches: [];
+}
+export declare function normalizeScenarioUpdateData(data: Record<string, unknown>): {
+    normalizedData: Record<string, unknown>;
+    normalization: ScenarioUpdateNormalization[];
+};
+export declare function scenarioUpdatePreview(current: Record<string, unknown>, data: Record<string, unknown>): ScenarioUpdatePreview;
 export declare class ScenariosResource extends BaseResource {
     /** List all scenarios in a project. */
     list(projectKey?: string): Promise<ScenarioSummary[]>;
@@ -19,8 +55,8 @@ export declare class ScenariosResource extends BaseResource {
     }>;
     /** Get the light/status view of a scenario. */
     status(scenarioId: string, projectKey?: string): Promise<ScenarioStatus>;
-    /** Merge-update a scenario's definition. */
-    update(scenarioId: string, data: Record<string, unknown>, projectKey?: string): Promise<void>;
+    /** Merge-update a scenario's definition, then refetch and verify requested fields persisted. */
+    update(scenarioId: string, data: Record<string, unknown>, projectKey?: string): Promise<ScenarioUpdateResult>;
     /** Delete a scenario. */
     delete(scenarioId: string, projectKey?: string): Promise<void>;
     /**