dataiku-sdk 3.8.0 → 3.9.0

package/README.md

@@ -520,8 +520,10 @@ UI-click smoke test. The CLI does not guess private endpoints for those operatio
 
 `project map` returns one agent-oriented JSON view of the Flow: normalized nodes and edges,
 joined zone metadata, cycle-safe topological layers, weakly connected components, diagnostics, and
-a stable `topologyFingerprint`. Zone joins add `zoneId`/`zoneName` to each node; the top-level
-`zones` array preserves zone color, position, item count, and visible node IDs.
+a stable `topologyFingerprint`. Output is deduplicated: node `name` appears only when it differs from
+`id`; each node carries `zoneId`/`componentId` (zone names live in `zones[]`, membership is derived
+from those ids, so `zones[]`/`components[]` omit `nodeIds`); `warnings` appears only when non-empty and
+`truncation` only when the map was cut. `zones` preserves zone color, position, and item count.
 
 ```bash
 dss project map --project-key MYPROJ

package/dist/build-metadata.json

@@ -1 +1 @@
-{"format":1,"buildRevision":"6814ed4e3da988eccd85f9f50d79e1da66253b0a"}
+{"format":1,"buildRevision":"3f9f0582b0fa45ee3708215954761d75f6edf9b7"}

package/dist/src/cli/command-syntax.json

@@ -4513,6 +4513,15 @@
             {
                 "arg": "name"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -4531,6 +4540,15 @@
             {
                 "arg": "name"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -7108,6 +7126,15 @@
             {
                 "arg": "id"
             },
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [
@@ -12890,6 +12917,15 @@
             "dss",
             "project",
             "get",
+            {
+                "optional": [
+                    [
+                        {
+                            "flag": "full"
+                        }
+                    ]
+                ]
+            },
             {
                 "optional": [
                     [

package/dist/src/cli/commands/dataset.js

@@ -143,7 +143,7 @@ export const datasetCommands = withUsage("dataset", {
             requireArgs(a, 1, commandUsage("dataset", "get"));
             return c.datasets.get(a[0], f["project-key"]);
         },
-        description: "Get full settings for a dataset.",
+        description: "Get the settings for a dataset (compact by default: empty values and metrics/version metadata are dropped, structure kept for get-edit-update; --full returns the raw DSS object).",
         examples: ["dss dataset get orders", "dss dataset get orders --project-key MYPROJ",],
     },
     schema: {
@@ -506,7 +506,7 @@ export const datasetCommands = withUsage("dataset", {
             requireArgs(a, 1, commandUsage("dataset", "info"));
             return c.datasets.info(a[0], f["project-key"]);
         },
-        description: "Get the full info object for a dataset (type, parameters, last build information, schema, etc.).",
+        description: "Get dataset info: type, connection, format, schema columns, flow recipes, buildability, data quality, and last modification (compact by default; --full returns the raw DSS info object).",
         examples: ["dss dataset info orders", "dss dataset info orders --project-key MYPROJ",],
     },
     "column-lineage": {

package/dist/src/cli/commands/job.js

@@ -7,6 +7,32 @@ import { executionMode, } from "../flags.js";
 import { encodedProjectEndpoint, enqueueCliWarning, } from "../output.js";
 import { commandUsage, withUsage, } from "../syntax.js";
 import { requireArgs, UsageError, } from "../usage.js";
+/** Lines `job log` prints to stdout when no `--max-lines` is given. */
+const DEFAULT_STDOUT_LOG_LINES = 100;
+/**
+ * Tail a (filtered) job log for stdout. `0`/`-1` return everything; a
+ * truncated tail enqueues a `log_truncated` warning carrying the shown and
+ * total line counts.
+ */
+function tailJobLogForStdout(log, limit) {
+    if (!log || limit === 0 || limit === -1)
+        return log;
+    const lines = log.split(/\r?\n/);
+    const hasTrailingLineBreak = lines[lines.length - 1] === "";
+    if (hasTrailingLineBreak)
+        lines.pop();
+    const shown = Math.max(1, limit);
+    if (lines.length <= shown)
+        return log;
+    enqueueCliWarning({
+        code: "log_truncated",
+        shown,
+        total: lines.length,
+        hint: "Raise --max-lines (0 = all), add --errors-only, or use --output PATH.",
+    });
+    const tail = lines.slice(-shown).join("\n");
+    return hasTrailingLineBreak ? `${tail}\n` : tail;
+}
 /**
  * With an empty output schema DSS ends a dataset build DONE after writing no
  * columns and no rows, without an error. Flag that after a successful build so
@@ -235,7 +261,7 @@ export const jobCommands = withUsage("job", {
             requireArgs(a, 1, commandUsage("job", "get"));
             return c.jobs.get(a[0], f["project-key"]);
         },
-        description: "Get job details.",
+        description: "Get job state, outputs, progress, error, and per-activity status (compact by default; --full returns the raw DSS job object). The log tail is never included: use `dss job log ID --errors-only`.",
         examples: ["dss job get JOB_ID",],
     },
     summary: {
@@ -252,23 +278,25 @@ export const jobCommands = withUsage("job", {
             const logFilter = f["errors-only"] === true
                 ? "errors"
                 : jobLogFilterFromFlag(f["log-filter"]);
+            const outputFile = f["output"]
+                ?? f["output-file"];
+            const requestedLines = maxLogLinesFromFlags(f);
+            // Stdout is tailed here (not in the SDK) so the total can be reported.
             const log = await c.jobs.log(a[0], {
                 activity: f["activity"],
                 logId: f["log-id"],
                 logFilter,
-                maxLogLines: maxLogLinesFromFlags(f),
+                maxLogLines: outputFile ? requestedLines : -1,
                 projectKey: f["project-key"],
             });
-            const outputFile = f["output"]
-                ?? f["output-file"];
             if (!outputFile)
-                return log;
+                return tailJobLogForStdout(log, requestedLines ?? DEFAULT_STDOUT_LOG_LINES);
             const outputPath = resolve(outputFile);
             await mkdir(dirname(outputPath), { recursive: true, });
             await writeFile(outputPath, log.endsWith("\n") ? log : `${log}\n`, "utf-8");
             return outputPath;
         },
-        description: "Get public API job log output; find why a job failed. Use --errors-only (or --log-filter errors) to surface just error/traceback lines, and --output PATH to write the log to a file (stdout returns the path). --log-id is accepted for UI parity but DSS API-key auth cannot select browser-only cat-activity-log files.",
+        description: "Get public API job log output (last 100 lines by default; --max-lines N, 0 = all); find why a job failed. Use --errors-only for error/traceback lines and --output PATH to write it to a file (stdout returns the path). --log-id is accepted for UI parity; API keys cannot read browser-only cat-activity-log files.",
         examples: [
             "dss job log JOB_ID",
             "dss job log JOB_ID --activity main --max-log-lines 200",

package/dist/src/cli/commands/project.js

@@ -5,6 +5,7 @@ import { projectIncarnationHash, } from "../../utils/project-incarnation.js";
 import { jsonInput, num, requiredJsonInput, requiredStringFlag, } from "../coerce.js";
 import { executionMode, } from "../flags.js";
 import { CommandResultFailure, } from "../output.js";
+import { compactProjectMapOutput, } from "../project-map-output.js";
 import { commandUsage, withUsage, } from "../syntax.js";
 import { requireArgs, UsageError, } from "../usage.js";
 const INSPECT_ARCHIVE_USAGE = commandUsage("project", "inspect-archive");
@@ -28,7 +29,7 @@ export const projectCommands = withUsage("project", {
     },
     get: {
         handler: (c, _a, f) => c.projects.get(f["project-key"]),
-        description: "Get project settings and metadata.",
+        description: "Get project settings and metadata (compact by default: empty values, metrics/version/image metadata dropped, permission flags collapsed to deniedPermissions; --full returns the raw DSS object).",
         examples: ["dss project get", "dss project get --project-key MYPROJ",],
     },
     metadata: {
@@ -42,13 +43,13 @@ export const projectCommands = withUsage("project", {
         examples: ["dss project flow", "dss project flow --project-key MYPROJ",],
     },
     map: {
-        handler: (c, _a, f) => c.projects.map({
+        handler: async (c, _a, f) => compactProjectMapOutput(await c.projects.map({
             projectKey: f["project-key"],
             maxNodes: num(f["max-nodes"], "--max-nodes"),
             maxEdges: num(f["max-edges"], "--max-edges"),
             includeRaw: f["include-raw"] === true,
             render: flowRenderFormat(f["render"]),
-        }),
+        })),
         validate: (_a, f) => {
             flowRenderFormat(f["render"]);
             num(f["max-nodes"], "--max-nodes");

package/dist/src/cli/contract.js

@@ -1400,11 +1400,10 @@ function commandRegistryJsonSchema() {
 function errorEnvelopeJsonSchema() {
     return {
         type: "object",
-        required: ["type", "ok", "error", "code", "category", "exitCode",],
+        required: ["type", "error", "code", "category", "exitCode",],
         additionalProperties: true,
         properties: {
             type: { const: "error", },
-            ok: { const: false, },
             error: { type: "string", },
             code: { type: "string", },
             category: { enum: ["usage", "permission_or_environment", "dss", "internal",], },
@@ -1414,8 +1413,6 @@ function errorEnvelopeJsonSchema() {
             retryable: { type: "boolean", },
             requestId: { type: "string", },
             details: { type: "object", additionalProperties: true, },
-            resource: { type: "string", },
-            action: { type: "string", },
             projectKey: { type: "string", },
         },
     };
@@ -1526,7 +1523,7 @@ export function buildAgentContract() {
                 failure: "structured-error-object",
                 failureResultDetailLimitBytes: 65_536,
                 fieldProjection: "Missing --fields paths: null on stdout; field_projection_missing on stderr.",
-                lists: "*.list: compact items; --full/--fields use DSS objects; --contains/--limit filter (list_truncated warning).",
+                compactResults: "*.list, job get, dataset get/info, project get: compact; --full/--fields use DSS objects; lists take --contains/--limit (list_truncated warning).",
                 richFailureResults: "doctor/batch/cleanup failures: own compact {ok:false,...} result on stdout and command exit code; no wrapper.",
             },
             stderr: {

package/dist/src/cli/output-schemas.js

@@ -13,6 +13,7 @@
  * Imported by the generator and the discovery parity test only.
  */
 import { compactListOutputSchemas, } from "./list-shapes.js";
+import { compactReadOutputSchemas, DATASET_GET_DROPPED_KEYS, PROJECT_GET_DROPPED_KEYS, } from "./read-shapes.js";
 const PROJECT_LIBRARY_ITEM_OUTPUT_SCHEMA = {
     type: "object",
     additionalProperties: false,
@@ -50,6 +51,21 @@ const NOTEBOOK_SAVE_SQL_OUTPUT_SCHEMA = {
         requested: { type: "string", },
     },
 };
+/**
+ * Discovery schema of a compact settings object (`dataset get`, `project get`):
+ * the SDK TypeBox schema minus the keys the CLI drops, with only the
+ * identifiers that survive pruning required. SDK validation keeps the
+ * unmodified TypeBox schema.
+ */
+function settingsOutputSchema(schema, droppedKeys, extraProperties = {}) {
+    const properties = Object.fromEntries(Object.entries(schema["properties"]).filter(([key,]) => !droppedKeys.includes(key)));
+    const required = (schema["required"] ?? []).filter((key) => !droppedKeys.includes(key));
+    return {
+        ...schema,
+        properties: { ...properties, ...extraProperties, },
+        required: [...required, ...Object.keys(extraProperties),],
+    };
+}
 let commandOutputSchemas;
 /**
  * Canonical TypeBox-derived output schemas, keyed by `resource.action`. This is
@@ -196,9 +212,15 @@ export function typeBoxCommandOutputSchemas() {
                 maxLines: { type: "integer", minimum: 0, },
             },
         },
-        "project.get": ProjectDetailsSchema,
+        "project.get": settingsOutputSchema(ProjectDetailsSchema, PROJECT_GET_DROPPED_KEYS, {
+            deniedPermissions: {
+                type: "array",
+                items: { type: "string", },
+                description: "Permission flags DSS reports as false; absent flags are granted.",
+            },
+        }),
         "project.metadata": ProjectMetadataSchema,
-        "dataset.get": DatasetDetailsSchema,
+        "dataset.get": settingsOutputSchema(DatasetDetailsSchema, DATASET_GET_DROPPED_KEYS),
         "dataset.schema": DatasetSchemaSchema,
         "job.wait": JobWaitResultSchema,
         "job.monitor": JobWaitResultSchema,
@@ -224,7 +246,9 @@ export function typeBoxCommandOutputSchemas() {
                 },
             ],
         },
-        // `*.list` defaults to compact items; `--full` returns the DSS objects.
+        // `*.list` and the four large reads default to compact shapes; `--full`
+        // returns the DSS objects.
         ...compactListOutputSchemas(),
+        ...compactReadOutputSchemas(),
     };
 }

package/dist/src/cli/project-map-output.d.ts

@@ -0,0 +1,24 @@
+import type { FlowMapResult } from "../resources/projects.js";
+import type { AnalyzedFlowMap, AnalyzedFlowNode, FlowMapComponent, FlowMapZone } from "../utils/flow-analysis.js";
+/**
+ * `dss project map` output: the SDK `FlowMapResult` with derivable duplicates
+ * removed at the CLI boundary. Lossless — every dropped field can be rebuilt:
+ * node `name` defaults to `id`, `zoneName` comes from `zones[]`, zone/component
+ * membership comes from each node's `zoneId`/`componentId`, an absent
+ * `warnings` means none, and an absent `truncation` means nothing was cut.
+ * `topologyFingerprint` is passed through untouched.
+ */
+export type CompactFlowMapNode = Omit<AnalyzedFlowNode, "zoneName">;
+export type CompactFlowMapZone = Omit<FlowMapZone, "nodeIds">;
+export type CompactFlowMapComponent = Omit<FlowMapComponent, "nodeIds">;
+export interface CompactFlowMap extends Omit<AnalyzedFlowMap, "nodes" | "zones" | "components" | "warnings"> {
+    nodes: CompactFlowMapNode[];
+    zones: CompactFlowMapZone[];
+    components: CompactFlowMapComponent[];
+    warnings?: string[];
+}
+export interface CompactFlowMapResult extends Omit<FlowMapResult, "map" | "truncation"> {
+    map: CompactFlowMap;
+    truncation?: FlowMapResult["truncation"];
+}
+export declare function compactProjectMapOutput(result: FlowMapResult): CompactFlowMapResult;

package/dist/src/cli/project-map-output.js

@@ -0,0 +1,21 @@
+function compactNode(node) {
+    const { zoneName: _zoneName, name, ...rest } = node;
+    return name === undefined || name === node.id ? rest : { ...rest, name, };
+}
+export function compactProjectMapOutput(result) {
+    const { nodes, zones, components, warnings, ...mapRest } = result.map;
+    const map = {
+        ...mapRest,
+        nodes: nodes.map(compactNode),
+        zones: zones.map(({ nodeIds: _nodeIds, ...zone }) => zone),
+        components: components.map(({ nodeIds: _nodeIds, ...component }) => component),
+    };
+    if (warnings.length > 0)
+        map.warnings = warnings;
+    const { map: _map, truncation, ...resultRest } = result;
+    return {
+        map,
+        ...(truncation.truncated ? { truncation, } : {}),
+        ...resultRest,
+    };
+}

package/dist/src/cli/read-shapes.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Plain data shared by the compact read shapers (src/cli/result-shapes.ts) and
+ * the discovery output schemas (src/cli/output-schemas.ts), so the advertised
+ * schema is the emitted shape. Imports nothing: the schema generator loads it
+ * without the CLI runtime.
+ */
+/**
+ * Settings-object noise `dataset get` / `project get` drop: metrics and check
+ * configuration, version stamps and checklists that agents never edit.
+ */
+export declare const DATASET_GET_DROPPED_KEYS: readonly ["metrics", "metricsChecks", "versionTag", "creationTag", "checklists"];
+export declare const PROJECT_GET_DROPPED_KEYS: readonly ["metrics", "metricsChecks", "versionTag", "creationTag", "checklists", "objectImgHash", "isProjectImg", "defaultImgColor", "imgPattern"];
+/** Discovery output schemas of the `job get` and `dataset info` compact defaults. */
+export declare function compactReadOutputSchemas(): Record<string, Record<string, unknown>>;

package/dist/src/cli/read-shapes.js

@@ -0,0 +1,114 @@
+/**
+ * Plain data shared by the compact read shapers (src/cli/result-shapes.ts) and
+ * the discovery output schemas (src/cli/output-schemas.ts), so the advertised
+ * schema is the emitted shape. Imports nothing: the schema generator loads it
+ * without the CLI runtime.
+ */
+/**
+ * Settings-object noise `dataset get` / `project get` drop: metrics and check
+ * configuration, version stamps and checklists that agents never edit.
+ */
+export const DATASET_GET_DROPPED_KEYS = [
+    "metrics",
+    "metricsChecks",
+    "versionTag",
+    "creationTag",
+    "checklists",
+];
+export const PROJECT_GET_DROPPED_KEYS = [
+    ...DATASET_GET_DROPPED_KEYS,
+    "objectImgHash",
+    "isProjectImg",
+    "defaultImgColor",
+    "imgPattern",
+];
+const STRING = { type: "string", };
+const NUMBER = { type: "number", };
+const BOOLEAN = { type: "boolean", };
+/** Discovery output schemas of the `job get` and `dataset info` compact defaults. */
+export function compactReadOutputSchemas() {
+    const errorSchema = {
+        type: "object",
+        additionalProperties: false,
+        properties: { type: STRING, message: STRING, },
+    };
+    return {
+        "job.get": {
+            type: "object",
+            additionalProperties: false,
+            required: ["id", "state",],
+            properties: {
+                id: STRING,
+                name: STRING,
+                type: STRING,
+                state: STRING,
+                initiator: STRING,
+                triggeredFrom: STRING,
+                startTime: NUMBER,
+                endTime: NUMBER,
+                outputs: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        properties: { type: STRING, id: STRING, projectKey: STRING, partition: STRING, },
+                    },
+                },
+                progress: { type: "object", additionalProperties: NUMBER, },
+                error: errorSchema,
+                activities: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        properties: {
+                            id: STRING,
+                            recipe: STRING,
+                            recipeType: STRING,
+                            state: STRING,
+                            totalTime: NUMBER,
+                            warnings: NUMBER,
+                            error: errorSchema,
+                            message: STRING,
+                        },
+                    },
+                },
+            },
+        },
+        "dataset.info": {
+            type: "object",
+            additionalProperties: false,
+            required: ["name",],
+            properties: {
+                name: STRING,
+                type: STRING,
+                managed: BOOLEAN,
+                connection: STRING,
+                formatType: STRING,
+                partitioned: BOOLEAN,
+                schema: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        required: ["name", "type",],
+                        properties: { name: STRING, type: STRING, meaning: STRING, comment: STRING, },
+                    },
+                },
+                recipes: { type: "array", items: STRING, },
+                buildable: BOOLEAN,
+                upstreamBuildable: BOOLEAN,
+                downstreamBuildable: BOOLEAN,
+                tags: { type: "array", items: STRING, },
+                dataQuality: { type: "object", additionalProperties: true, },
+                lastBuild: {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: { jobId: STRING, success: BOOLEAN, startTime: NUMBER, endTime: NUMBER, },
+                },
+                lastModifiedOn: NUMBER,
+                lastModifiedBy: STRING,
+            },
+        },
+    };
+}

package/dist/src/cli/result-shapes.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Compact a command result for agents: `*.list` items (filter, cap, compact
+ * fields) and the four large read commands. `--full` and `--fields` bypass
+ * the read shapers so the DSS object (or a projection of it) is returned.
+ */
+export declare function shapeCommandResult(resource: string, action: string, result: unknown, flags: Record<string, string | boolean>): unknown;