dataiku-sdk 4.0.0 → 4.1.0

package/README.md

@@ -291,6 +291,7 @@ Use environment variables for ephemeral runs. For disposable agent tests, set `D
 Credential precedence is flags first, then `DATAIKU_*` environment variables, then saved credentials in `DSS_CONFIG_DIR` or the platform config directory.
 Set `DATAIKU_DISABLE_ENV=1` when a test must ignore both `.env` files and `DATAIKU_*` environment variables.
 When `.env` loading is enabled, the CLI reads `.env` from the CLI build/root directory and from the command's current working directory; put test-specific `.env` files in the directory where you invoke `dss`. Only `DATAIKU_*`, `NODE_TLS_REJECT_UNAUTHORIZED`, and `NODE_EXTRA_CA_CERTS` are read from `.env`; other keys are ignored.
+Set `DSS_PRETTY=1` to indent stdout JSON (still one JSON value). Use it when the shell or agent harness cuts long lines (omp's bash tool cuts each line at 768 bytes), so a one-line result is not truncated. It is read from the process environment only, not from `.env`.
 
 POSIX shell:
 

package/dist/build-metadata.json

@@ -1 +1 @@
-{"format":1,"buildRevision":"ab176cb1de853bfbdc58627b38a131ce1fc1681d"}
+{"format":1,"buildRevision":"88fd13012a34de8687737c530fbe86469d8e3f5d"}

package/dist/packages/types/src/index.d.ts

@@ -286,8 +286,10 @@ export declare const JobWaitResultSchema: import("@sinclair/typebox").TObject<{
     timedOut: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
     /** Present on a wait timeout: the wait never cancels the job, so this is `false`. */
     cancelled: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TBoolean>;
-    /** CLI only, on a wait timeout: the command that resumes waiting for the still-running job. */
+    /** CLI only, on a wait timeout: POSIX display form of the command that resumes waiting for the still-running job. */
     resume: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
+    /** CLI only, on a wait timeout: the resume command as argv, runnable without a shell on any OS. */
+    resumeArgv: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TArray<import("@sinclair/typebox").TString>>;
     progress: import("@sinclair/typebox").TObject<{
         done: import("@sinclair/typebox").TNumber;
         failed: import("@sinclair/typebox").TNumber;

package/dist/packages/types/src/index.js

@@ -310,8 +310,10 @@ export const JobWaitResultSchema = Type.Object({
     timedOut: Type.Optional(Type.Boolean()),
     /** Present on a wait timeout: the wait never cancels the job, so this is `false`. */
     cancelled: Type.Optional(Type.Boolean()),
-    /** CLI only, on a wait timeout: the command that resumes waiting for the still-running job. */
+    /** CLI only, on a wait timeout: POSIX display form of the command that resumes waiting for the still-running job. */
     resume: Type.Optional(Type.String()),
+    /** CLI only, on a wait timeout: the resume command as argv, runnable without a shell on any OS. */
+    resumeArgv: Type.Optional(Type.Array(Type.String())),
     progress: Type.Object({
         done: Type.Number(),
         failed: Type.Number(),

package/dist/src/cli/command-syntax.json

@@ -87,6 +87,15 @@
             {
                 "arg": "analysisId"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -1699,6 +1708,15 @@
             "get",
             {
                 "arg": "id"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "business-app.install-from-archive": [
@@ -1956,6 +1974,15 @@
             },
             {
                 "arg": "name"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "code-env.get-definition": [
@@ -3106,6 +3133,15 @@
             {
                 "arg": "id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -3320,6 +3356,15 @@
             "get",
             {
                 "arg": "dataCollectionId"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "data-collection.list": [
@@ -5033,6 +5078,15 @@
             {
                 "arg": "discussionId"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -5373,6 +5427,15 @@
             {
                 "arg": "id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -6092,6 +6155,15 @@
             {
                 "arg": "name-or-id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -6409,6 +6481,15 @@
             "get",
             {
                 "arg": "name"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "group.list": [
@@ -6634,6 +6715,15 @@
             {
                 "arg": "id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -7900,6 +7990,15 @@
             {
                 "arg": "macro_id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -8209,6 +8308,15 @@
             "get",
             {
                 "arg": "meaningId"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "meaning.list": [
@@ -9758,6 +9866,15 @@
             {
                 "arg": "storeId"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -11627,6 +11744,15 @@
             "get",
             {
                 "arg": "folderId"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "project-folder.move": [
@@ -13940,6 +14066,15 @@
                     ]
                 ]
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -15058,6 +15193,15 @@
             {
                 "arg": "modelId"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -15612,6 +15756,15 @@
             {
                 "arg": "id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -15633,6 +15786,15 @@
             {
                 "arg": "runId"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -15661,6 +15823,15 @@
                     ]
                 ]
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -16525,6 +16696,15 @@
             {
                 "arg": "id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -16784,6 +16964,15 @@
             "get",
             {
                 "arg": "login"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "user.list": [
@@ -16941,6 +17130,15 @@
             "dss",
             "variable",
             "get",
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -17446,6 +17644,15 @@
             {
                 "arg": "id-or-name"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -17697,6 +17904,15 @@
             "get",
             {
                 "arg": "workspaceKey"
+            },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
             }
         ],
         "workspace.list": [

package/dist/src/cli/commands/code-env.js

@@ -79,13 +79,15 @@ export const codeEnvCommands = withUsage("code-env", {
         examples: ["dss code-env list", "dss code-env list --lang PYTHON",],
     },
     get: {
-        handler: (c, a) => {
+        handler: (c, a, f) => {
             const usage = commandUsage("code-env", "get");
             requireArgs(a, 2, usage);
             const envLang = requireEnvLang(a[0], usage);
-            return c.codeEnvs.get(envLang, a[1]);
+            return f["full"] === true
+                ? c.codeEnvs.getDefinition(envLang, a[1])
+                : c.codeEnvs.get(envLang, a[1]);
         },
-        description: "Get code environment details: requested vs installed packages, Python interpreter, deployment mode, and the stable definition hash (definitionHash) used by set-definition --expect-hash.",
+        description: "Get code environment details: requested vs installed packages, Python interpreter, deployment mode, and the stable definition hash (definitionHash) used by set-definition --expect-hash. --full returns the raw DSS definition (same as get-definition).",
         examples: ["dss code-env get PYTHON my_env",],
     },
     "get-definition": {

package/dist/src/cli/commands/dataset.js

@@ -225,11 +225,13 @@ export const datasetCommands = withUsage("dataset", {
         handler: async (c, a, f) => {
             const previewUsage = commandUsage("dataset", "preview");
             requireArgs(a, 1, previewUsage);
-            const maxRows = num(f["max-rows"], "--max-rows");
-            if (maxRows !== undefined && (!Number.isSafeInteger(maxRows) || maxRows <= 0 || maxRows > 500)) {
-                throw new UsageError(`--max-rows must be a positive integer no greater than 500 (got "${String(f["max-rows"])}"). `
-                    + `Usage: ${previewUsage}`, "validation_failed");
+            const requestedRows = num(f["max-rows"], "--max-rows");
+            if (requestedRows !== undefined && (!Number.isSafeInteger(requestedRows) || requestedRows <= 0)) {
+                throw new UsageError(`--max-rows must be a positive integer (got "${String(f["max-rows"])}"). Usage: ${previewUsage}`, "validation_failed");
             }
+            // Agents ask for "all rows" with a large number; serve the 500-row public
+            // maximum instead of failing, and let the truncation warning say if more exist.
+            const maxRows = requestedRows === undefined ? undefined : Math.min(requestedRows, 500);
             const result = await c.datasets.preview(a[0], {
                 maxRows,
                 projectKey: f["project-key"],

package/dist/src/cli/commands/job.js

@@ -1,5 +1,5 @@
 import { resolve, } from "node:path";
-import { ClientValidationError, DataikuError, dataikuErrorCode, } from "../../errors.js";
+import { ClientValidationError, DataikuError, dataikuErrorCode, RESPONSE_TOO_LARGE_STATUS_TEXT, } from "../../errors.js";
 import { asRecord, } from "../../utils/records.js";
 import { jobBuildTargetTypeFromFlags, jobLogFilterFromFlag, maxLogLinesFromFlags, num, } from "../coerce.js";
 import { executionMode, } from "../flags.js";
@@ -34,23 +34,26 @@ function tailJobLogForStdout(log, limit) {
     const tail = lines.slice(-shown).join("\n");
     return hasTrailingLineBreak ? `${tail}\n` : tail;
 }
-const RESPONSE_TOO_LARGE_STATUS_TEXT = "Response Too Large";
-/** POSIX-shell quoting for a resume command argument. */
+/** POSIX-shell quoting for the display form of a resume command. */
 function shellArg(value) {
     return /^[\w.:@%+=,/-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
 }
-/** Command that resumes waiting for a job whose wait timed out. */
-function jobWaitResumeCommand(jobId, projectKey) {
-    return `dss job wait ${shellArg(jobId)}${projectKey ? ` --project-key ${shellArg(projectKey)}` : ""}`;
-}
 /**
  * A wait that timed out only stopped polling: the job is still running in DSS.
- * Attach the command that resumes waiting; other results pass through.
+ * Attach the command that resumes waiting: `resumeArgv` (run it without a
+ * shell, on any OS) and `resume` (POSIX display form); other results pass through.
  */
 function withWaitResume(result, projectKey) {
     if (result.timedOut !== true)
         return result;
-    return { ...result, resume: jobWaitResumeCommand(result.jobId, projectKey), };
+    const resumeArgv = [
+        "dss",
+        "job",
+        "wait",
+        result.jobId,
+        ...(projectKey ? ["--project-key", projectKey,] : []),
+    ];
+    return { ...result, resume: resumeArgv.map(shellArg).join(" "), resumeArgv, };
 }
 /**
  * With an empty output schema DSS ends a dataset build DONE after writing no

package/dist/src/cli/commands/recipe.js

@@ -412,9 +412,30 @@ export const recipeCommands = withUsage("recipe", {
                     hint: `Fix the recipe settings, then run dss recipe update-schema ${createdName}.`,
                 });
             }
-            return { created: createdName, resource: "recipe", ...created, ...moved, };
+            // A visual recipe whose payload came from flags is ready to run: point at the build
+            // (recursive, so outputs of recipes created just before are built too).
+            const output = f["output"];
+            const next = typeof output === "string" && (created.payloadConfigured || created.joinConfigured)
+                ? [[
+                        "dss",
+                        "job",
+                        "build-and-wait",
+                        output,
+                        "--build-mode",
+                        "RECURSIVE_BUILD",
+                        "--summary",
+                        ...(pk ? ["--project-key", pk,] : []),
+                    ],]
+                : undefined;
+            return {
+                created: createdName,
+                resource: "recipe",
+                ...created,
+                ...moved,
+                ...(next ? { next, } : {}),
+            };
         },
-        description: "Create a recipe with optional inputs and a dataset or managed-folder output. Missing outputs are created on --output-connection as DSS managed datasets. Join, grouping (--group-by/--aggregate) and prepare (--rename/--formula/--filter/...) payloads come from flags. Visual recipe outputs get the DSS-computed schema (outputSchemaUpdated).",
+        description: "Create a recipe; missing outputs become managed datasets on --output-connection. Join, grouping (--group-by/--aggregate: keys plus those aggregates, no implicit count) and prepare (--rename/--formula/--filter/...) payloads come from flags, then `next` is the build. Visual outputs get the DSS-computed schema.",
         examples: [
             "dss recipe create --type python --input raw_orders,lookup --output orders_clean",
             "dss recipe create --type grouping --input orders --output orders_by_cat --group-by category --aggregate amount:sum+avg,name:first --order-by day",

package/dist/src/cli/commands/scenario.js

@@ -1,8 +1,69 @@
+import { asRecord, } from "../../utils/records.js";
 import { jsonInput, num, parseBooleanOption, } from "../coerce.js";
 import { executionMode, } from "../flags.js";
 import { encodedProjectEndpoint, readIfExists, skipResult, } from "../output.js";
+import { scenarioRunDiagnosis, shapeCommandResult, } from "../result-shapes.js";
 import { commandUsage, withUsage, } from "../syntax.js";
 import { requireArgs, UsageError, } from "../usage.js";
+/**
+ * The recipe a failed job was running and the datasets it read, from one
+ * `job get` (its FAILED activity). Undefined when the job cannot be read or
+ * names no failed recipe: the diagnosis then stays job-level.
+ */
+async function failedActivity(c, jobId, projectKey) {
+    if (typeof jobId !== "string")
+        return undefined;
+    try {
+        const job = asRecord(await c.jobs.get(jobId, projectKey));
+        const activities = asRecord(asRecord(job?.["baseStatus"])?.["activities"]) ?? {};
+        for (const value of Object.values(activities)) {
+            const activity = asRecord(value);
+            if (activity?.["state"] !== "FAILED" || typeof activity["recipeName"] !== "string")
+                continue;
+            const sources = Array.isArray(activity["sources"]) ? activity["sources"] : [];
+            const inputs = sources.flatMap((source) => {
+                const id = asRecord(source)?.["id"];
+                if (typeof id !== "string")
+                    return [];
+                // Same-project inputs are named bare; foreign ones keep PROJECT.name.
+                const dot = id.indexOf(".");
+                return [dot > 0 && id.slice(0, dot) === projectKey ? id.slice(dot + 1) : id,];
+            });
+            return { recipe: activity["recipeName"], inputs, };
+        }
+    }
+    catch {
+        // A diagnosis aid only: the run report itself is already in hand.
+    }
+    return undefined;
+}
+/** Diagnosis of a raw run report, located at the failed recipe when its job names one. */
+async function locatedDiagnosis(c, details, projectKey) {
+    const plain = scenarioRunDiagnosis(details);
+    const runProject = asRecord(asRecord(asRecord(details)?.["scenarioRun"])?.["trigger"])?.["projectKey"];
+    const pk = projectKey ?? (typeof runProject === "string" ? runProject : undefined);
+    const activity = await failedActivity(c, plain.failure?.["jobId"], pk);
+    return activity ? scenarioRunDiagnosis(details, activity) : plain;
+}
+/**
+ * Run a scenario and wait; a failed (not timed-out) run gains the first
+ * failure, the recipe that failed, and the next commands. Extra reads happen
+ * only on failure; a read error leaves the wait result untouched.
+ */
+async function runAndWaitWithFailure(c, scenarioId, options) {
+    const result = await c.scenarios.runAndWait(scenarioId, options);
+    if (result.success || result.timedOut === true || !result.runId)
+        return result;
+    try {
+        const details = await c.scenarios.getRunDetails(scenarioId, result.runId, {
+            projectKey: options.projectKey,
+        });
+        return { ...result, ...await locatedDiagnosis(c, details, options.projectKey), };
+    }
+    catch {
+        return result;
+    }
+}
 export const scenarioCommands = withUsage("scenario", {
     list: {
         handler: (c, _a, f) => c.scenarios.list(f["project-key"]),
@@ -37,7 +98,7 @@ export const scenarioCommands = withUsage("scenario", {
                 };
             }
             if (f["wait"] === true) {
-                return c.scenarios.runAndWait(a[0], { ...options, projectKey: pk, });
+                return runAndWaitWithFailure(c, a[0], { ...options, projectKey: pk, });
             }
             return c.scenarios.run(a[0], pk);
         },
@@ -63,7 +124,7 @@ export const scenarioCommands = withUsage("scenario", {
                     method: "POST",
                 };
             }
-            return c.scenarios.runAndWait(a[0], { ...options, projectKey: pk, });
+            return runAndWaitWithFailure(c, a[0], { ...options, projectKey: pk, });
         },
         description: "Run a scenario and wait for completion.",
         examples: [
@@ -209,17 +270,24 @@ export const scenarioCommands = withUsage("scenario", {
                 projectKey: f["project-key"],
             });
         },
-        description: "List the most recent runs of a scenario (run id, start/end, outcome).",
+        description: "List the most recent runs of a scenario (compact by default: run id, outcome, start/end, trigger, failure message; --full returns the DSS run objects).",
         examples: ["dss scenario last-runs my_scenario --limit 10",],
     },
     "get-run": {
-        handler: (c, a, f) => {
+        handler: async (c, a, f) => {
             requireArgs(a, 2, commandUsage("scenario", "get-run"));
-            return c.scenarios.getRunDetails(a[0], a[1], {
-                projectKey: f["project-key"],
-            });
+            const projectKey = f["project-key"];
+            const details = await c.scenarios.getRunDetails(a[0], a[1], { projectKey, });
+            if (f["full"] === true || typeof f["fields"] === "string")
+                return details;
+            // Compact view, located at the failed recipe (one extra job read, only on failure).
+            // The shaped object has no `scenarioRun`, so the dispatcher's shaping passes it through.
+            const shaped = asRecord(shapeCommandResult("scenario", "get-run", details, f));
+            if (!shaped?.["failure"])
+                return shaped ?? details;
+            return { ...shaped, ...await locatedDiagnosis(c, details, projectKey), };
         },
-        description: "Get the details of a specific scenario run, including per-step outcomes.",
+        description: "Get a scenario run: outcome, timing, trigger, per-step datasets and jobs, plus `failure` {stepId, jobId, dataset, recipe, inputs, message} and `next` argv (recipe code, input schemas, job errors) for the first failure (compact by default; --full returns the raw DSS run report).",
         examples: ["dss scenario get-run my_scenario 2024-09-01-01-02-03-123",],
     },
     log: {

package/dist/src/cli/contract.js

@@ -779,6 +779,7 @@ const STRING_OUTPUT_ACTIONS = new Set([
  */
 const SCOPED_OUTPUT_SHAPES = {
     "folder.contents": "array",
+    "scenario.last-runs": "array",
     "plugin.usages": "object",
     "plugin.download": "object",
     "dataset.download": "object",
@@ -1518,7 +1519,7 @@ export function buildAgentContract() {
                     "dss commands run --fields dataset",
                     "dss commands run --fields dataset.create",
                 ],
-                scopedDiscoveryHint: "Default: resource/action summary. --contains TEXT: top 5 actions by concept. --fields RESOURCE: all resource entries; RESOURCE.ACTION: one entry keyed by that path; append .FIELD for nested metadata. Comma-separate paths. --output PATH exports the full registry.",
+                scopedDiscoveryHint: "Default: resource/action summary. --contains TEXT: top 5 actions by concept. --fields RESOURCE: each action's usage, description, and side effect; RESOURCE.ACTION: one full entry keyed by that path; append .FIELD for nested metadata. Comma-separate paths. --output PATH exports full entries (the full registry without --fields).",
                 // Accepted by every command and absent from per-entry `flags`; the
                 // connection flags matter only for commands that call DSS.
                 globalFlags: [...GLOBAL_AGENT_FLAGS, ...AUTHENTICATED_AGENT_FLAGS,].map((name) => {
@@ -1545,7 +1546,7 @@ export function buildAgentContract() {
                 failure: "structured-error-object",
                 failureResultDetailLimitBytes: 65_536,
                 fieldProjection: "Missing --fields paths: null on stdout; field_projection_missing on stderr.",
-                compactResults: "*.list, job get, dataset get/info, project get: compact; --full/--fields use DSS objects; lists take --contains/--limit (list_truncated warning).",
+                compactResults: "*.list, job get, dataset get/info, project get, scenario get-run/last-runs: compact; --full/--fields use DSS objects; lists take --contains/--limit (list_truncated warning).",
                 richFailureResults: "doctor/batch/cleanup failures: own compact {ok:false,...} result on stdout and command exit code; no wrapper.",
             },
             stderr: {