dataiku-sdk 3.8.0 → 4.0.0

package/dist/src/cli/output-schemas.js

@@ -13,6 +13,7 @@
  * Imported by the generator and the discovery parity test only.
  */
 import { compactListOutputSchemas, } from "./list-shapes.js";
+import { compactReadOutputSchemas, DATASET_GET_DROPPED_KEYS, PROJECT_GET_DROPPED_KEYS, } from "./read-shapes.js";
 const PROJECT_LIBRARY_ITEM_OUTPUT_SCHEMA = {
     type: "object",
     additionalProperties: false,
@@ -50,13 +51,28 @@ const NOTEBOOK_SAVE_SQL_OUTPUT_SCHEMA = {
         requested: { type: "string", },
     },
 };
+/**
+ * Discovery schema of a compact settings object (`dataset get`, `project get`):
+ * the SDK TypeBox schema minus the keys the CLI drops, with only the
+ * identifiers that survive pruning required. SDK validation keeps the
+ * unmodified TypeBox schema.
+ */
+function settingsOutputSchema(schema, droppedKeys, extraProperties = {}) {
+    const properties = Object.fromEntries(Object.entries(schema["properties"]).filter(([key,]) => !droppedKeys.includes(key)));
+    const required = (schema["required"] ?? []).filter((key) => !droppedKeys.includes(key));
+    return {
+        ...schema,
+        properties: { ...properties, ...extraProperties, },
+        required: [...required, ...Object.keys(extraProperties),],
+    };
+}
 let commandOutputSchemas;
 /**
  * Canonical TypeBox-derived output schemas, keyed by `resource.action`. This is
  * the single source of truth for `src/generated/action-output-schemas.json`,
  * which discovery reads as plain data so no TypeBox schema graph loads on the
  * discovery path. Regenerate with
- * `bun scripts/generate-action-output-schemas.mjs`; `bun run check` fails when
+ * `bun scripts/generate-action-output-schemas.mjs`; `bun run verify` fails when
  * the generated file is stale. SDK validation keeps using the TypeBox schema
  * objects from `packages/types` directly and never depends on the generated file.
  */
@@ -196,9 +212,15 @@ export function typeBoxCommandOutputSchemas() {
                 maxLines: { type: "integer", minimum: 0, },
             },
         },
-        "project.get": ProjectDetailsSchema,
+        "project.get": settingsOutputSchema(ProjectDetailsSchema, PROJECT_GET_DROPPED_KEYS, {
+            deniedPermissions: {
+                type: "array",
+                items: { type: "string", },
+                description: "Permission flags DSS reports as false; absent flags are granted.",
+            },
+        }),
         "project.metadata": ProjectMetadataSchema,
-        "dataset.get": DatasetDetailsSchema,
+        "dataset.get": settingsOutputSchema(DatasetDetailsSchema, DATASET_GET_DROPPED_KEYS),
         "dataset.schema": DatasetSchemaSchema,
         "job.wait": JobWaitResultSchema,
         "job.monitor": JobWaitResultSchema,
@@ -224,7 +246,9 @@ export function typeBoxCommandOutputSchemas() {
                 },
             ],
         },
-        // `*.list` defaults to compact items; `--full` returns the DSS objects.
+        // `*.list` and the four large reads default to compact shapes; `--full`
+        // returns the DSS objects.
         ...compactListOutputSchemas(),
+        ...compactReadOutputSchemas(),
     };
 }

package/dist/src/cli/output.js

@@ -193,6 +193,13 @@ export function commandFailureMessage(result) {
             if (typeof message === "string" && message.length > 0)
                 return message;
         }
+        if (record.timedOut === true && typeof record.jobId === "string"
+            && typeof record.progress === "object" && typeof record.state === "string") {
+            // A job wait result: the timeout only stopped polling, the DSS job is
+            // still running (and was not cancelled).
+            const resume = typeof record.resume === "string" ? ` Resume with: ${record.resume}` : "";
+            return `Timed out waiting for job ${record.jobId} after ${record.elapsedMs}ms (last known state: ${record.state}); the job is still running in DSS and was not cancelled.${resume}`;
+        }
         const state = typeof record.state === "string"
             ? record.state
             : typeof record.outcome === "string"

package/dist/src/cli/project-map-output.d.ts

@@ -0,0 +1,24 @@
+import type { FlowMapResult } from "../resources/projects.js";
+import type { AnalyzedFlowMap, AnalyzedFlowNode, FlowMapComponent, FlowMapZone } from "../utils/flow-analysis.js";
+/**
+ * `dss project map` output: the SDK `FlowMapResult` with derivable duplicates
+ * removed at the CLI boundary. Lossless — every dropped field can be rebuilt:
+ * node `name` defaults to `id`, `zoneName` comes from `zones[]`, zone/component
+ * membership comes from each node's `zoneId`/`componentId`, an absent
+ * `warnings` means none, and an absent `truncation` means nothing was cut.
+ * `topologyFingerprint` is passed through untouched.
+ */
+export type CompactFlowMapNode = Omit<AnalyzedFlowNode, "zoneName">;
+export type CompactFlowMapZone = Omit<FlowMapZone, "nodeIds">;
+export type CompactFlowMapComponent = Omit<FlowMapComponent, "nodeIds">;
+export interface CompactFlowMap extends Omit<AnalyzedFlowMap, "nodes" | "zones" | "components" | "warnings"> {
+    nodes: CompactFlowMapNode[];
+    zones: CompactFlowMapZone[];
+    components: CompactFlowMapComponent[];
+    warnings?: string[];
+}
+export interface CompactFlowMapResult extends Omit<FlowMapResult, "map" | "truncation"> {
+    map: CompactFlowMap;
+    truncation?: FlowMapResult["truncation"];
+}
+export declare function compactProjectMapOutput(result: FlowMapResult): CompactFlowMapResult;

package/dist/src/cli/project-map-output.js

@@ -0,0 +1,21 @@
+function compactNode(node) {
+    const { zoneName: _zoneName, name, ...rest } = node;
+    return name === undefined || name === node.id ? rest : { ...rest, name, };
+}
+export function compactProjectMapOutput(result) {
+    const { nodes, zones, components, warnings, ...mapRest } = result.map;
+    const map = {
+        ...mapRest,
+        nodes: nodes.map(compactNode),
+        zones: zones.map(({ nodeIds: _nodeIds, ...zone }) => zone),
+        components: components.map(({ nodeIds: _nodeIds, ...component }) => component),
+    };
+    if (warnings.length > 0)
+        map.warnings = warnings;
+    const { map: _map, truncation, ...resultRest } = result;
+    return {
+        map,
+        ...(truncation.truncated ? { truncation, } : {}),
+        ...resultRest,
+    };
+}

package/dist/src/cli/read-shapes.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Plain data shared by the compact read shapers (src/cli/result-shapes.ts) and
+ * the discovery output schemas (src/cli/output-schemas.ts), so the advertised
+ * schema is the emitted shape. Imports nothing: the schema generator loads it
+ * without the CLI runtime.
+ */
+/**
+ * Settings-object noise `dataset get` / `project get` drop: metrics and check
+ * configuration, version stamps and checklists that agents never edit.
+ */
+export declare const DATASET_GET_DROPPED_KEYS: readonly ["metrics", "metricsChecks", "versionTag", "creationTag", "checklists"];
+export declare const PROJECT_GET_DROPPED_KEYS: readonly ["metrics", "metricsChecks", "versionTag", "creationTag", "checklists", "objectImgHash", "isProjectImg", "defaultImgColor", "imgPattern"];
+/** Discovery output schemas of the `job get` and `dataset info` compact defaults. */
+export declare function compactReadOutputSchemas(): Record<string, Record<string, unknown>>;

package/dist/src/cli/read-shapes.js

@@ -0,0 +1,114 @@
+/**
+ * Plain data shared by the compact read shapers (src/cli/result-shapes.ts) and
+ * the discovery output schemas (src/cli/output-schemas.ts), so the advertised
+ * schema is the emitted shape. Imports nothing: the schema generator loads it
+ * without the CLI runtime.
+ */
+/**
+ * Settings-object noise `dataset get` / `project get` drop: metrics and check
+ * configuration, version stamps and checklists that agents never edit.
+ */
+export const DATASET_GET_DROPPED_KEYS = [
+    "metrics",
+    "metricsChecks",
+    "versionTag",
+    "creationTag",
+    "checklists",
+];
+export const PROJECT_GET_DROPPED_KEYS = [
+    ...DATASET_GET_DROPPED_KEYS,
+    "objectImgHash",
+    "isProjectImg",
+    "defaultImgColor",
+    "imgPattern",
+];
+const STRING = { type: "string", };
+const NUMBER = { type: "number", };
+const BOOLEAN = { type: "boolean", };
+/** Discovery output schemas of the `job get` and `dataset info` compact defaults. */
+export function compactReadOutputSchemas() {
+    const errorSchema = {
+        type: "object",
+        additionalProperties: false,
+        properties: { type: STRING, message: STRING, },
+    };
+    return {
+        "job.get": {
+            type: "object",
+            additionalProperties: false,
+            required: ["id", "state",],
+            properties: {
+                id: STRING,
+                name: STRING,
+                type: STRING,
+                state: STRING,
+                initiator: STRING,
+                triggeredFrom: STRING,
+                startTime: NUMBER,
+                endTime: NUMBER,
+                outputs: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        properties: { type: STRING, id: STRING, projectKey: STRING, partition: STRING, },
+                    },
+                },
+                progress: { type: "object", additionalProperties: NUMBER, },
+                error: errorSchema,
+                activities: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        properties: {
+                            id: STRING,
+                            recipe: STRING,
+                            recipeType: STRING,
+                            state: STRING,
+                            totalTime: NUMBER,
+                            warnings: NUMBER,
+                            error: errorSchema,
+                            message: STRING,
+                        },
+                    },
+                },
+            },
+        },
+        "dataset.info": {
+            type: "object",
+            additionalProperties: false,
+            required: ["name",],
+            properties: {
+                name: STRING,
+                type: STRING,
+                managed: BOOLEAN,
+                connection: STRING,
+                formatType: STRING,
+                partitioned: BOOLEAN,
+                schema: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        required: ["name", "type",],
+                        properties: { name: STRING, type: STRING, meaning: STRING, comment: STRING, },
+                    },
+                },
+                recipes: { type: "array", items: STRING, },
+                buildable: BOOLEAN,
+                upstreamBuildable: BOOLEAN,
+                downstreamBuildable: BOOLEAN,
+                tags: { type: "array", items: STRING, },
+                dataQuality: { type: "object", additionalProperties: true, },
+                lastBuild: {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: { jobId: STRING, success: BOOLEAN, startTime: NUMBER, endTime: NUMBER, },
+                },
+                lastModifiedOn: NUMBER,
+                lastModifiedBy: STRING,
+            },
+        },
+    };
+}

package/dist/src/cli/result-shapes.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Compact a command result for agents: `*.list` items (filter, cap, compact
+ * fields) and the four large read commands. `--full` and `--fields` bypass
+ * the read shapers so the DSS object (or a projection of it) is returned.
+ */
+export declare function shapeCommandResult(resource: string, action: string, result: unknown, flags: Record<string, string | boolean>): unknown;

package/dist/src/cli/result-shapes.js

@@ -0,0 +1,235 @@
+/**
+ * Compact default results of the four large read commands (`job get`,
+ * `dataset info`, `dataset get`, `project get`), plus the single dispatch
+ * point for compacting command results (`shapeCommandResult`).
+ *
+ * The SDK resources keep returning the raw DSS objects; only the CLI compacts.
+ * `--full` returns the DSS object unchanged, and `--fields` projects DSS paths
+ * out of it (exactly like the compact `*.list` shapes in list-output.ts).
+ *
+ * Imports no schema graph: the discovery schemas below are plain data.
+ */
+import { isRecord, } from "../utils/records.js";
+import { shapeListResult, } from "./list-output.js";
+import { DATASET_GET_DROPPED_KEYS, PROJECT_GET_DROPPED_KEYS, } from "./read-shapes.js";
+function record(value) {
+    return isRecord(value) ? value : {};
+}
+function nonEmptyString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+/** Drop `undefined` members so absent optional fields are omitted, never null. */
+function defined(fields) {
+    return Object.fromEntries(Object.entries(fields).filter(([, value,]) => value !== undefined));
+}
+/** `{ type, message }` of a DSS error object (`error`, `firstFailure`); undefined when empty. */
+function errorSummary(value) {
+    if (!isRecord(value))
+        return undefined;
+    const shaped = defined({
+        type: nonEmptyString(value["errorType"]),
+        message: nonEmptyString(value["detailedMessage"]) ?? nonEmptyString(value["message"]),
+    });
+    return Object.keys(shaped).length > 0 ? shaped : undefined;
+}
+/**
+ * Output id/project of a job output across DSS shapes: `targetDataset` +
+ * `targetDatasetProjectKey`, `targetManagedFolder`, `targetSavedModel`, ...
+ * and the plain `{ id, projectKey }` form.
+ */
+function outputTarget(output) {
+    const targetKey = Object.keys(output).find((key) => /^target[A-Z]/.test(key) && !key.endsWith("ProjectKey") && key !== "targetPartition"
+        && typeof output[key] === "string");
+    return {
+        id: targetKey === undefined ? output["id"] : output[targetKey],
+        projectKey: targetKey === undefined ? output["projectKey"] : output[`${targetKey}ProjectKey`],
+    };
+}
+function shapeJobOutput(output, jobProjectKey) {
+    const fields = record(output);
+    const { id, projectKey, } = outputTarget(fields);
+    const partition = fields["targetPartition"] ?? fields["partition"];
+    return defined({
+        type: fields["type"],
+        id,
+        projectKey: typeof projectKey === "string" && projectKey !== jobProjectKey
+            ? projectKey
+            : undefined,
+        partition: typeof partition === "string" && partition !== "NP" ? partition : undefined,
+    });
+}
+function shapeJobActivity(activity) {
+    const fields = record(activity);
+    return defined({
+        id: fields["activityId"],
+        recipe: fields["recipeName"],
+        recipeType: fields["recipeType"],
+        state: fields["state"],
+        totalTime: fields["totalTime"],
+        warnings: record(fields["warnings"])["totalCount"],
+        error: errorSummary(fields["firstFailure"]),
+        message: nonEmptyString(fields["message"]),
+    });
+}
+/** `job get`: state, outputs, progress, and the failure; never the log tail. */
+function shapeJobGet(result) {
+    if (!isRecord(result))
+        return result;
+    const base = record(result["baseStatus"]);
+    const def = record(base["def"]);
+    const activities = record(base["activities"]);
+    return defined({
+        id: def["id"],
+        name: def["name"],
+        type: def["type"],
+        state: base["state"],
+        initiator: nonEmptyString(def["initiator"])
+            ?? nonEmptyString(record(result["initiator"])["login"]),
+        triggeredFrom: def["triggeredFrom"],
+        startTime: base["jobStartTime"],
+        endTime: base["jobEndTime"],
+        outputs: Array.isArray(def["outputs"])
+            ? def["outputs"].map((output) => shapeJobOutput(output, def["projectKey"]))
+            : undefined,
+        progress: result["globalState"],
+        error: errorSummary(result["error"]),
+        activities: Object.values(activities).map(shapeJobActivity),
+    });
+}
+/** Extra column keys worth keeping when set (DSS sends most as ""/defaults). */
+const COLUMN_EXTRA_KEYS = ["meaning", "comment",];
+function shapeColumn(column) {
+    const fields = record(column);
+    const shaped = { name: fields["name"], type: fields["type"], };
+    for (const key of COLUMN_EXTRA_KEYS) {
+        const value = nonEmptyString(fields[key]);
+        if (value !== undefined)
+            shaped[key] = value;
+    }
+    return shaped;
+}
+function shapeLastBuild(value) {
+    if (!isRecord(value))
+        return undefined;
+    const shaped = defined({
+        jobId: value["jobId"],
+        success: value["buildSuccess"],
+        startTime: value["buildStartTime"],
+        endTime: value["buildEndTime"],
+    });
+    return Object.keys(shaped).length > 0 ? shaped : undefined;
+}
+/** `dataset info`: what the dataset is, its schema, flow neighbours, and freshness. */
+function shapeDatasetInfo(result) {
+    if (!isRecord(result))
+        return result;
+    const dataset = record(result["dataset"]);
+    const params = record(dataset["params"]);
+    const columns = record(dataset["schema"])["columns"];
+    const tags = dataset["tags"];
+    const quality = record(result["dataQualityStatus"]);
+    const versioning = record(result["versioning"]);
+    const timeline = record(result["timeline"]);
+    return defined({
+        name: result["name"] ?? dataset["name"],
+        type: result["type"] ?? dataset["type"],
+        managed: dataset["managed"],
+        connection: params["connection"] ?? params["uploadConnection"],
+        formatType: dataset["formatType"],
+        partitioned: result["partitioned"],
+        schema: Array.isArray(columns) ? columns.map(shapeColumn) : undefined,
+        recipes: Array.isArray(result["recipes"])
+            ? result["recipes"].map((recipe) => record(recipe)["name"])
+            : undefined,
+        buildable: result["buildable"],
+        upstreamBuildable: result["upstreamBuildable"],
+        downstreamBuildable: result["downstreamBuildable"],
+        tags: Array.isArray(tags) && tags.length > 0 ? tags : undefined,
+        dataQuality: Object.keys(quality).length > 0 && quality["numberItems"] !== 0
+            ? result["dataQualityStatus"]
+            : undefined,
+        lastBuild: shapeLastBuild(result["lastBuild"]),
+        lastModifiedOn: versioning["lastModifiedOn"] ?? timeline["lastModifiedOn"],
+        lastModifiedBy: record(versioning["lastModifiedBy"] ?? timeline["lastModifiedBy"])["login"],
+    });
+}
+/**
+ * Recursively remove empty strings, arrays and objects (including objects that
+ * become empty). `false`, `0` and `null` are kept. Array positions are
+ * preserved: an element that empties out stays as its empty value.
+ */
+function pruneEmpty(value) {
+    if (typeof value === "string")
+        return value.length === 0 ? undefined : value;
+    if (Array.isArray(value)) {
+        if (value.length === 0)
+            return undefined;
+        return value.map((item) => pruneEmpty(item) ?? (typeof item === "string" ? "" : Array.isArray(item) ? [] : {}));
+    }
+    if (isRecord(value)) {
+        const pruned = {};
+        for (const [key, member,] of Object.entries(value)) {
+            const kept = pruneEmpty(member);
+            if (kept !== undefined)
+                pruned[key] = kept;
+        }
+        return Object.keys(pruned).length === 0 ? undefined : pruned;
+    }
+    return value;
+}
+function withoutKeys(source, keys) {
+    return Object.fromEntries(Object.entries(source).filter(([key,]) => !keys.includes(key)));
+}
+/** `dataset get`: the settings object minus metadata and empty/default noise. */
+function shapeDatasetGet(result) {
+    if (!isRecord(result))
+        return result;
+    const settings = withoutKeys(result, DATASET_GET_DROPPED_KEYS);
+    const formatParams = settings["formatParams"];
+    if (isRecord(formatParams)) {
+        settings["formatParams"] = withoutKeys(formatParams, ["hiveSeparators",]);
+    }
+    return pruneEmpty(settings) ?? {};
+}
+const PERMISSION_KEY = /^can[A-Z]/;
+/**
+ * `project get`: as `dataset get`, with the boolean permission flags
+ * (`can*`, `isProjectAdmin`) collapsed to the denied ones. Absent from the
+ * list means granted, so nothing is lost.
+ */
+function shapeProjectGet(result) {
+    if (!isRecord(result))
+        return result;
+    const settings = withoutKeys(result, PROJECT_GET_DROPPED_KEYS);
+    const deniedPermissions = [];
+    for (const [key, value,] of Object.entries(settings)) {
+        if (typeof value !== "boolean" || !(PERMISSION_KEY.test(key) || key === "isProjectAdmin")) {
+            continue;
+        }
+        if (!value)
+            deniedPermissions.push(key);
+        delete settings[key];
+    }
+    const pruned = record(pruneEmpty(settings));
+    return { ...pruned, deniedPermissions, };
+}
+/** Compact read shapers by `resource.action`. */
+const READ_SHAPERS = {
+    "job.get": shapeJobGet,
+    "dataset.info": shapeDatasetInfo,
+    "dataset.get": shapeDatasetGet,
+    "project.get": shapeProjectGet,
+};
+/**
+ * Compact a command result for agents: `*.list` items (filter, cap, compact
+ * fields) and the four large read commands. `--full` and `--fields` bypass
+ * the read shapers so the DSS object (or a projection of it) is returned.
+ */
+export function shapeCommandResult(resource, action, result, flags) {
+    if (action === "list")
+        return shapeListResult(resource, result, flags);
+    const shaper = READ_SHAPERS[`${resource}.${action}`];
+    if (!shaper || flags["full"] === true || typeof flags["fields"] === "string")
+        return result;
+    return shaper(result);
+}

package/dist/src/cli/version.d.ts

@@ -49,7 +49,7 @@ export declare function buildVersionPayload(input: {
 export declare const PACKAGE_ROOT: string | undefined;
 export declare const CLI_VERSION: string;
 export declare const CLI_GIT_REVISION: string | undefined;
-export declare const AGENT_CONTRACT_VERSION = 3;
+export declare const AGENT_CONTRACT_VERSION = 5;
 export declare const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
-export declare const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v3.json";
+export declare const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v5.json";
 export declare function cliVersionResult(): CliVersionPayload;

package/dist/src/cli/version.js

@@ -1,8 +1,7 @@
 import { readdirSync, readFileSync, statSync, } from "node:fs";
 import { dirname, resolve, } from "node:path";
-import { fileURLToPath, } from "node:url";
 export function findPackageRoot() {
-    let dir = dirname(fileURLToPath(import.meta.url));
+    let dir = import.meta.dir;
     for (let i = 0; i < 5; i++) {
         try {
             readFileSync(resolve(dir, "package.json"), "utf-8");
@@ -133,7 +132,7 @@ export function gitRevision(packageRoot) {
     const full = gitFullRevision(packageRoot);
     return full ? full.slice(0, 7) : undefined;
 }
-export function detectLoadSource(modulePath = fileURLToPath(import.meta.url)) {
+export function detectLoadSource(modulePath = import.meta.path) {
     return modulePath.replaceAll("\\", "/").includes("/dist/") ? "dist" : "source";
 }
 export function buildMetadataRevision(packageRoot) {
@@ -204,9 +203,9 @@ export function buildVersionPayload(input) {
 export const PACKAGE_ROOT = findPackageRoot();
 export const CLI_VERSION = packageVersion(PACKAGE_ROOT);
 export const CLI_GIT_REVISION = gitRevision(PACKAGE_ROOT);
-export const AGENT_CONTRACT_VERSION = 3;
+export const AGENT_CONTRACT_VERSION = 5;
 export const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
-export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v3.json";
+export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v5.json";
 export function cliVersionResult() {
     // Only code actually loaded from dist/ may report dist provenance; an
     // inherited DSS_LOAD_SOURCE on a source run is ignored.