dataiku-sdk 0.5.1 → 0.6.0

package/dist/src/resources/recipes.js

@@ -26,6 +26,66 @@ const RECIPE_DEFINITION_FIELDS = new Set(["params", "inputs", "outputs", "script
 function rootRecipeDefinitionFields(data) {
     return Object.keys(data).filter((key) => RECIPE_DEFINITION_FIELDS.has(key));
 }
+function normalizeRecipeOutputType(value) {
+    if (typeof value !== "string")
+        return undefined;
+    const normalized = value.trim().toUpperCase().replace(/-/g, "_");
+    if (normalized === "DATASET")
+        return "DATASET";
+    if (normalized === "MANAGED_FOLDER" || normalized === "FOLDER")
+        return "MANAGED_FOLDER";
+    return undefined;
+}
+function recipeOutputItems(recipe) {
+    const outputs = asRecord(recipe.outputs);
+    if (!outputs)
+        return [];
+    const result = [];
+    const seen = new Set();
+    for (const [role, roleValue,] of Object.entries(outputs)) {
+        const items = asRecord(roleValue)?.items;
+        if (!Array.isArray(items))
+            continue;
+        for (const itemValue of items) {
+            const item = asRecord(itemValue);
+            const ref = asString(item?.ref);
+            if (!ref)
+                continue;
+            const seenKey = ref;
+            if (seen.has(seenKey))
+                continue;
+            seen.add(seenKey);
+            const type = normalizeRecipeOutputType(item?.type ?? item?.targetType ?? item?.objectType);
+            result.push({
+                ref,
+                role,
+                ...(type ? { type, } : {}),
+            });
+        }
+    }
+    return result;
+}
+function recipeInputItems(recipe) {
+    const inputs = asRecord(recipe.inputs);
+    if (!inputs)
+        return [];
+    const result = [];
+    const seen = new Set();
+    for (const [role, roleValue,] of Object.entries(inputs)) {
+        const items = asRecord(roleValue)?.items;
+        if (!Array.isArray(items))
+            continue;
+        for (const itemValue of items) {
+            const item = asRecord(itemValue);
+            const ref = asString(item?.ref);
+            if (!ref || seen.has(ref))
+                continue;
+            seen.add(ref);
+            result.push({ ref, role, });
+        }
+    }
+    return result;
+}
 function inferRecipeCodeExtension(recipeType) {
     const normalized = typeof recipeType === "string" ? recipeType.trim().toLowerCase() : "";
     if (!normalized)
@@ -92,6 +152,181 @@ export class RecipesResource extends BaseResource {
         }
         return opts?.includePayload ? { ...result, recipe, } : { recipe, };
     }
+    /** Validate declared recipe graph references before running/building. */
+    async validateGraph(recipeName, opts) {
+        const pk = this.resolveProjectKey(opts?.projectKey);
+        const { recipe, } = await this.get(recipeName, { projectKey: pk, });
+        const inputItems = recipeInputItems(recipe);
+        const outputItems = recipeOutputItems(recipe);
+        const [datasets, folders,] = await Promise.all([
+            this.client.datasets.list(pk),
+            this.client.folders.list(pk),
+        ]);
+        const datasetNames = new Set(datasets.map((dataset) => dataset.name));
+        const folderIdByRef = new Map();
+        for (const folder of folders) {
+            folderIdByRef.set(folder.id, folder.id);
+            if (folder.name)
+                folderIdByRef.set(folder.name, folder.id);
+        }
+        const resolveReference = (item, requireExplicitOutputType) => {
+            const folderId = folderIdByRef.get(item.ref);
+            const isDataset = datasetNames.has(item.ref);
+            if (item.type === "DATASET") {
+                return { ref: item.ref, role: item.role, type: "DATASET", exists: isDataset, id: item.ref, };
+            }
+            if (item.type === "MANAGED_FOLDER") {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    type: "MANAGED_FOLDER",
+                    exists: folderId !== undefined,
+                    id: folderId ?? item.ref,
+                };
+            }
+            if (isDataset && (!folderId || !requireExplicitOutputType)) {
+                return { ref: item.ref, role: item.role, type: "DATASET", exists: true, id: item.ref, };
+            }
+            if (folderId && !isDataset) {
+                return { ref: item.ref, role: item.role, type: "MANAGED_FOLDER", exists: true, id: folderId, };
+            }
+            return { ref: item.ref, role: item.role, exists: false, };
+        };
+        const inputs = inputItems.map((item) => resolveReference(item, false));
+        const outputs = outputItems.map((item) => resolveReference(item, true));
+        const ambiguousOutputs = outputItems
+            .filter((item) => !item.type && datasetNames.has(item.ref) && folderIdByRef.has(item.ref))
+            .map((item) => item.ref);
+        const missingInputs = inputs.filter((item) => !item.exists);
+        const missingOutputs = outputs.filter((item) => !item.exists);
+        const warnings = [];
+        if (outputItems.length === 0)
+            warnings.push("Recipe has no declared outputs to build.");
+        for (const ref of ambiguousOutputs) {
+            warnings.push(`Output "${ref}" matches both a dataset and a managed folder; declare an explicit output type.`);
+        }
+        for (const output of outputs) {
+            if (!output.exists) {
+                warnings.push(`Declared output "${output.ref}" was not found in project "${pk}".`);
+            }
+        }
+        for (const input of missingInputs) {
+            warnings.push(`Declared input "${input.ref}" was not found in project "${pk}".`);
+        }
+        return {
+            valid: missingInputs.length === 0
+                && missingOutputs.length === 0
+                && ambiguousOutputs.length === 0
+                && outputItems.length > 0,
+            recipeName,
+            projectKey: pk,
+            inputs,
+            outputs,
+            missingInputs,
+            missingOutputs,
+            ambiguousOutputs,
+            warnings,
+        };
+    }
+    /** Resolve recipe outputs to job-build targets. */
+    async resolveRunOutputs(recipeName, opts) {
+        const pk = this.resolveProjectKey(opts?.projectKey);
+        const { recipe, } = await this.get(recipeName, { projectKey: pk, });
+        const outputItems = recipeOutputItems(recipe);
+        if (outputItems.length === 0) {
+            throw new Error(`Recipe "${recipeName}" has no output items to build.`);
+        }
+        const [datasets, folders,] = await Promise.all([
+            this.client.datasets.list(pk),
+            this.client.folders.list(pk),
+        ]);
+        const datasetNames = new Set(datasets.map((dataset) => dataset.name));
+        const folderIdByRef = new Map();
+        for (const folder of folders) {
+            folderIdByRef.set(folder.id, folder.id);
+            if (folder.name)
+                folderIdByRef.set(folder.name, folder.id);
+        }
+        return outputItems.map((item) => {
+            if (item.type === "DATASET") {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: item.ref,
+                    type: "DATASET",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            const folderId = folderIdByRef.get(item.ref);
+            if (item.type === "MANAGED_FOLDER") {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: folderId ?? item.ref,
+                    type: "MANAGED_FOLDER",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            const isDataset = datasetNames.has(item.ref);
+            if (isDataset && folderId) {
+                throw new Error(`Recipe "${recipeName}" output "${item.ref}" matches both a dataset and a managed folder. Add an explicit output type to the recipe definition or build the target directly with --target-type.`);
+            }
+            if (folderId) {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: folderId,
+                    type: "MANAGED_FOLDER",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            if (isDataset) {
+                return {
+                    ref: item.ref,
+                    role: item.role,
+                    id: item.ref,
+                    type: "DATASET",
+                    projectKey: pk,
+                    partition: opts?.partition,
+                };
+            }
+            throw new Error(`Recipe "${recipeName}" output "${item.ref}" was not found as a dataset or managed folder in project "${pk}".`);
+        });
+    }
+    /** Run a recipe by building its resolved outputs. */
+    async run(recipeName, opts) {
+        const pk = this.resolveProjectKey(opts?.projectKey);
+        const outputs = await this.resolveRunOutputs(recipeName, {
+            partition: opts?.partition,
+            projectKey: pk,
+        });
+        const shouldWait = opts?.wait === true
+            || opts?.includeLogs === true
+            || opts?.summary === true
+            || opts?.timeoutMs !== undefined
+            || opts?.pollIntervalMs !== undefined;
+        if (shouldWait) {
+            const waitResult = await this.client.jobs.buildAndWaitOutputs(outputs, {
+                buildMode: opts?.buildMode,
+                includeLogs: opts?.includeLogs,
+                maxLogLines: opts?.maxLogLines,
+                logFilter: opts?.logFilter,
+                pollIntervalMs: opts?.pollIntervalMs,
+                projectKey: pk,
+                timeoutMs: opts?.timeoutMs,
+                summary: opts?.summary,
+            });
+            return { recipeName, outputs, ...waitResult, };
+        }
+        const started = await this.client.jobs.buildOutputs(outputs, {
+            buildMode: opts?.buildMode,
+            projectKey: pk,
+        });
+        return { recipeName, outputs, ...started, };
+    }
     /** Create a recipe, with optional output dataset provisioning and join configuration. */
     async create(opts) {
         const pk = this.resolveProjectKey(opts.projectKey);

package/dist/src/resources/sql.js

@@ -24,6 +24,82 @@ function asString(value) {
     const trimmed = value.trim();
     return trimmed.length > 0 ? trimmed : undefined;
 }
+function firstStringField(record, fields) {
+    for (const field of fields) {
+        const value = record[field];
+        if (typeof value === "string" && value.trim().length > 0)
+            return value.trim();
+    }
+    return undefined;
+}
+function isLikelySqlErrorDetail(detail) {
+    const lower = detail.toLowerCase();
+    return /\b[A-Z_]+(?:_ERROR|_NOT_FOUND|_MISMATCH|_DENIED|_EXCEEDED)\b/.test(detail)
+        || lower.includes("athena")
+        || lower.includes("sql")
+        || lower.includes("query")
+        || lower.includes("column")
+        || lower.includes("table")
+        || lower.includes("line ");
+}
+function sqlErrorDetailFromBody(body) {
+    const trimmed = body.trim();
+    if (!trimmed)
+        return undefined;
+    try {
+        const parsed = JSON.parse(trimmed);
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+            const record = parsed;
+            const nested = asRecord(record.details) ?? asRecord(record.error);
+            const nestedDetail = nested
+                ? firstStringField(nested, ["message", "errorMessage", "error", "reason", "cause",])
+                : undefined;
+            if (nestedDetail && isLikelySqlErrorDetail(nestedDetail))
+                return nestedDetail;
+            const direct = firstStringField(record, [
+                "message",
+                "detailedMessage",
+                "errorMessage",
+                "error",
+                "reason",
+                "cause",
+            ]);
+            if (direct && isLikelySqlErrorDetail(direct))
+                return direct;
+        }
+    }
+    catch {
+        // Fall back to regex extraction from raw DSS/Athena text below.
+    }
+    const match = trimmed.match(/\b[A-Z_]+(?:_ERROR|_NOT_FOUND|_MISMATCH|_DENIED|_EXCEEDED)\b[:\s-]+[^\n\r]+/);
+    return match?.[0] ?? (isLikelySqlErrorDetail(trimmed) ? trimmed.slice(0, 500) : undefined);
+}
+function withSqlErrorContext(error) {
+    if (error instanceof DataikuError) {
+        const detail = sqlErrorDetailFromBody(error.body);
+        if (detail) {
+            let body = error.body;
+            try {
+                const parsed = JSON.parse(error.body);
+                if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                    body = JSON.stringify({
+                        ...parsed,
+                        message: `SQL query failed: ${detail}`,
+                        sqlError: detail,
+                    });
+                }
+                else {
+                    body = `SQL query failed: ${detail}\n${error.body}`;
+                }
+            }
+            catch {
+                body = `SQL query failed: ${detail}\n${error.body}`;
+            }
+            throw new DataikuError(error.status, error.statusText, body, error.retry);
+        }
+    }
+    throw error;
+}
 function splitDatasetIdentifier(datasetFullName, fallbackProjectKey) {
     const trimmed = datasetFullName.trim();
     const dotIndex = trimmed.indexOf(".");
@@ -79,7 +155,7 @@ export class SqlResource extends BaseResource {
         const { queryId, schema, } = await this.startQuery(opts);
         const rows = await this.streamResults(queryId);
         await this.finishStreaming(queryId);
-        return { queryId, schema, rows, };
+        return { queryId, schema, columns: schema, rows, };
     }
     async resolveDatasetQueryFallback(opts) {
         const datasetFullName = opts.datasetFullName;
@@ -119,14 +195,19 @@ export class SqlResource extends BaseResource {
         }
         catch (error) {
             if (!isUnsupportedSqlDatasetConnectionError(error))
-                throw error;
+                withSqlErrorContext(error);
             const retryOpts = await this.resolveDatasetQueryFallback(queryOpts);
             if (!retryOpts) {
                 throw new Error(buildUnsupportedSqlDatasetConnectionMessage(queryOpts.datasetFullName), {
                     cause: error,
                 });
             }
-            return this.executeQuery(retryOpts);
+            try {
+                return await this.executeQuery(retryOpts);
+            }
+            catch (retryError) {
+                withSqlErrorContext(retryError);
+            }
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "dataiku-sdk",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
 	"type": "module",
 	"workspaces": [

package/packages/types/dist/index.d.ts

@@ -611,6 +611,10 @@ export declare const SqlQueryResponseSchema: import("@sinclair/typebox").TObject
         name: import("@sinclair/typebox").TString;
         type: import("@sinclair/typebox").TString;
     }>>;
+    columns: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TObject<{
+        name: import("@sinclair/typebox").TString;
+        type: import("@sinclair/typebox").TString;
+    }>>>;
     rows: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnknown>>;
 }>;
 export type SqlQueryResponse = Static<typeof SqlQueryResponseSchema>;

package/packages/types/dist/index.js

@@ -601,6 +601,7 @@ export const SqlQueryResultSchema = Type.Object({
 export const SqlQueryResponseSchema = Type.Object({
     queryId: Type.String(),
     schema: Type.Array(SqlQuerySchemaSchema),
+    columns: Type.Optional(Type.Array(SqlQuerySchemaSchema)),
     rows: Type.Array(Type.Array(Type.Unknown())),
 });
 // ---------------------------------------------------------------------------