dataiku-sdk 3.7.0 → 3.9.0

package/dist/src/cli/plans.js

@@ -16,11 +16,12 @@ import { schemaPropagationOptionsFromFlags, } from "./commands/flow.js";
 import { mlTaskFieldsFromFlags, mlTaskReguessOptionsFromFlags, } from "./commands/ml-task.js";
 import { deployRunOptionsFromFlags, experimentsDatasetOptionsFromFlags, inferenceInfoFromFlags, } from "./commands/mlflow.js";
 import { projectLibraryPutPayload, } from "./commands/project-library.js";
-import { recipeCreateOptionsFromFlags, } from "./commands/recipe.js";
+import { enqueueRecipeCreateWarnings, recipeCreateOptionsFromFlags, } from "./commands/recipe.js";
 import { resolveSqlQueryInvocation, } from "./commands/sql.js";
 import { buildRegistryEntry, splitPackageSpec, } from "./contract.js";
 import { ambientProjectKey, } from "./env.js";
 import { executionMode, } from "./flags.js";
+import { datasetSqlSourceFromFlags, } from "./helpers/dataset.js";
 import { flowZoneColor, flowZoneMoveItems, flowZoneName, } from "./helpers/flow-zone.js";
 import { recipeBackupPath, recipeRunShouldWait, } from "./helpers/recipe.js";
 import { diagnosticsComputationFromFlags, } from "./helpers/trained-model.js";
@@ -419,6 +420,50 @@ export function commandPlanShape(resource, action, args, flags, entry, projectKe
                 endpoint: projectEndpoint(`/wiki/${encodeURIComponent(id)}`),
                 identifiers: { article: id, },
             };
+        case "wiki.attach": {
+            const articleEndpoint = projectEndpoint(`/wiki/${encodeURIComponent(id)}`);
+            const upload = {
+                method: "POST",
+                endpoint: `${articleEndpoint}/upload`,
+                payload: uploadPayload(args[1]),
+            };
+            return {
+                ...upload,
+                identifiers: { article: id, localPath: args[1], },
+                requests: [
+                    {
+                        sequence: 1,
+                        method: "GET",
+                        endpoint: articleEndpoint,
+                        purpose: "read current attachments",
+                    },
+                    { sequence: 2, ...upload, },
+                    {
+                        sequence: 3,
+                        method: "GET",
+                        endpoint: articleEndpoint,
+                        assert: { newAttachments: 1, },
+                    },
+                ],
+            };
+        }
+        case "wiki.detach": {
+            const articleEndpoint = projectEndpoint(`/wiki/${encodeURIComponent(id)}`);
+            return {
+                method: "PUT",
+                endpoint: articleEndpoint,
+                identifiers: { article: id, smartId: args[1], },
+                requests: [
+                    { sequence: 1, method: "GET", endpoint: articleEndpoint, },
+                    {
+                        sequence: 2,
+                        method: "PUT",
+                        endpoint: articleEndpoint,
+                        payload: "<article data without the attachment with the given smartId>",
+                    },
+                ],
+            };
+        }
         case "dashboard.create": {
             const data = jsonInput(flags);
             const flagName = flags["name"];
@@ -730,8 +775,16 @@ export function commandPlanShape(resource, action, args, flags, entry, projectKe
             if (!connection && dsType.toLowerCase() !== "uploadedfiles") {
                 throw new UsageError("--connection is required unless --type is UploadedFiles.");
             }
+            const sqlSource = datasetSqlSourceFromFlags(flags, dsType);
+            const hasSqlSource = sqlSource.table !== undefined || sqlSource.query !== undefined;
             const endpoint = projectEndpoint("/datasets/"); // throws without a project key
             return {
+                ...(hasSqlSource
+                    ? {
+                        exact: false,
+                        reason: "The POST is exact; apply then detects the schema (POST /actions/testAndDetectSettings/externalSQL) and stores it (PUT /schema).",
+                    }
+                    : {}),
                 method: "POST",
                 endpoint,
                 identifiers: { name, },
@@ -740,6 +793,7 @@ export function commandPlanShape(resource, action, args, flags, entry, projectKe
                     datasetName: name,
                     connection,
                     dsType,
+                    ...sqlSource,
                 }),
             };
         }
@@ -924,7 +978,9 @@ export function commandPlanShape(resource, action, args, flags, entry, projectKe
             }
             const endpoint = projectEndpoint("/recipes/"); // throws without a project key
             // Same flag mapping and body construction as the handler and RecipesResource.create.
-            const { recipePrototype, creationSettings, grouping, prepareSteps, } = buildRecipeCreateRequest(recipeCreateOptionsFromFlags(flags), projectKey);
+            const createOptions = recipeCreateOptionsFromFlags(flags);
+            enqueueRecipeCreateWarnings(createOptions);
+            const { recipePrototype, creationSettings, grouping, prepareSteps, } = buildRecipeCreateRequest(createOptions, projectKey);
             const edits = grouping
                 ? { grouping, }
                 : prepareSteps.length > 0

package/dist/src/cli/project-map-output.d.ts

@@ -0,0 +1,24 @@
+import type { FlowMapResult } from "../resources/projects.js";
+import type { AnalyzedFlowMap, AnalyzedFlowNode, FlowMapComponent, FlowMapZone } from "../utils/flow-analysis.js";
+/**
+ * `dss project map` output: the SDK `FlowMapResult` with derivable duplicates
+ * removed at the CLI boundary. Lossless — every dropped field can be rebuilt:
+ * node `name` defaults to `id`, `zoneName` comes from `zones[]`, zone/component
+ * membership comes from each node's `zoneId`/`componentId`, an absent
+ * `warnings` means none, and an absent `truncation` means nothing was cut.
+ * `topologyFingerprint` is passed through untouched.
+ */
+export type CompactFlowMapNode = Omit<AnalyzedFlowNode, "zoneName">;
+export type CompactFlowMapZone = Omit<FlowMapZone, "nodeIds">;
+export type CompactFlowMapComponent = Omit<FlowMapComponent, "nodeIds">;
+export interface CompactFlowMap extends Omit<AnalyzedFlowMap, "nodes" | "zones" | "components" | "warnings"> {
+    nodes: CompactFlowMapNode[];
+    zones: CompactFlowMapZone[];
+    components: CompactFlowMapComponent[];
+    warnings?: string[];
+}
+export interface CompactFlowMapResult extends Omit<FlowMapResult, "map" | "truncation"> {
+    map: CompactFlowMap;
+    truncation?: FlowMapResult["truncation"];
+}
+export declare function compactProjectMapOutput(result: FlowMapResult): CompactFlowMapResult;

package/dist/src/cli/project-map-output.js

@@ -0,0 +1,21 @@
+function compactNode(node) {
+    const { zoneName: _zoneName, name, ...rest } = node;
+    return name === undefined || name === node.id ? rest : { ...rest, name, };
+}
+export function compactProjectMapOutput(result) {
+    const { nodes, zones, components, warnings, ...mapRest } = result.map;
+    const map = {
+        ...mapRest,
+        nodes: nodes.map(compactNode),
+        zones: zones.map(({ nodeIds: _nodeIds, ...zone }) => zone),
+        components: components.map(({ nodeIds: _nodeIds, ...component }) => component),
+    };
+    if (warnings.length > 0)
+        map.warnings = warnings;
+    const { map: _map, truncation, ...resultRest } = result;
+    return {
+        map,
+        ...(truncation.truncated ? { truncation, } : {}),
+        ...resultRest,
+    };
+}

package/dist/src/cli/read-shapes.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Plain data shared by the compact read shapers (src/cli/result-shapes.ts) and
+ * the discovery output schemas (src/cli/output-schemas.ts), so the advertised
+ * schema is the emitted shape. Imports nothing: the schema generator loads it
+ * without the CLI runtime.
+ */
+/**
+ * Settings-object noise `dataset get` / `project get` drop: metrics and check
+ * configuration, version stamps and checklists that agents never edit.
+ */
+export declare const DATASET_GET_DROPPED_KEYS: readonly ["metrics", "metricsChecks", "versionTag", "creationTag", "checklists"];
+export declare const PROJECT_GET_DROPPED_KEYS: readonly ["metrics", "metricsChecks", "versionTag", "creationTag", "checklists", "objectImgHash", "isProjectImg", "defaultImgColor", "imgPattern"];
+/** Discovery output schemas of the `job get` and `dataset info` compact defaults. */
+export declare function compactReadOutputSchemas(): Record<string, Record<string, unknown>>;

package/dist/src/cli/read-shapes.js

@@ -0,0 +1,114 @@
+/**
+ * Plain data shared by the compact read shapers (src/cli/result-shapes.ts) and
+ * the discovery output schemas (src/cli/output-schemas.ts), so the advertised
+ * schema is the emitted shape. Imports nothing: the schema generator loads it
+ * without the CLI runtime.
+ */
+/**
+ * Settings-object noise `dataset get` / `project get` drop: metrics and check
+ * configuration, version stamps and checklists that agents never edit.
+ */
+export const DATASET_GET_DROPPED_KEYS = [
+    "metrics",
+    "metricsChecks",
+    "versionTag",
+    "creationTag",
+    "checklists",
+];
+export const PROJECT_GET_DROPPED_KEYS = [
+    ...DATASET_GET_DROPPED_KEYS,
+    "objectImgHash",
+    "isProjectImg",
+    "defaultImgColor",
+    "imgPattern",
+];
+const STRING = { type: "string", };
+const NUMBER = { type: "number", };
+const BOOLEAN = { type: "boolean", };
+/** Discovery output schemas of the `job get` and `dataset info` compact defaults. */
+export function compactReadOutputSchemas() {
+    const errorSchema = {
+        type: "object",
+        additionalProperties: false,
+        properties: { type: STRING, message: STRING, },
+    };
+    return {
+        "job.get": {
+            type: "object",
+            additionalProperties: false,
+            required: ["id", "state",],
+            properties: {
+                id: STRING,
+                name: STRING,
+                type: STRING,
+                state: STRING,
+                initiator: STRING,
+                triggeredFrom: STRING,
+                startTime: NUMBER,
+                endTime: NUMBER,
+                outputs: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        properties: { type: STRING, id: STRING, projectKey: STRING, partition: STRING, },
+                    },
+                },
+                progress: { type: "object", additionalProperties: NUMBER, },
+                error: errorSchema,
+                activities: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        properties: {
+                            id: STRING,
+                            recipe: STRING,
+                            recipeType: STRING,
+                            state: STRING,
+                            totalTime: NUMBER,
+                            warnings: NUMBER,
+                            error: errorSchema,
+                            message: STRING,
+                        },
+                    },
+                },
+            },
+        },
+        "dataset.info": {
+            type: "object",
+            additionalProperties: false,
+            required: ["name",],
+            properties: {
+                name: STRING,
+                type: STRING,
+                managed: BOOLEAN,
+                connection: STRING,
+                formatType: STRING,
+                partitioned: BOOLEAN,
+                schema: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        additionalProperties: false,
+                        required: ["name", "type",],
+                        properties: { name: STRING, type: STRING, meaning: STRING, comment: STRING, },
+                    },
+                },
+                recipes: { type: "array", items: STRING, },
+                buildable: BOOLEAN,
+                upstreamBuildable: BOOLEAN,
+                downstreamBuildable: BOOLEAN,
+                tags: { type: "array", items: STRING, },
+                dataQuality: { type: "object", additionalProperties: true, },
+                lastBuild: {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: { jobId: STRING, success: BOOLEAN, startTime: NUMBER, endTime: NUMBER, },
+                },
+                lastModifiedOn: NUMBER,
+                lastModifiedBy: STRING,
+            },
+        },
+    };
+}

package/dist/src/cli/result-shapes.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Compact a command result for agents: `*.list` items (filter, cap, compact
+ * fields) and the four large read commands. `--full` and `--fields` bypass
+ * the read shapers so the DSS object (or a projection of it) is returned.
+ */
+export declare function shapeCommandResult(resource: string, action: string, result: unknown, flags: Record<string, string | boolean>): unknown;

package/dist/src/cli/result-shapes.js

@@ -0,0 +1,235 @@
+/**
+ * Compact default results of the four large read commands (`job get`,
+ * `dataset info`, `dataset get`, `project get`), plus the single dispatch
+ * point for compacting command results (`shapeCommandResult`).
+ *
+ * The SDK resources keep returning the raw DSS objects; only the CLI compacts.
+ * `--full` returns the DSS object unchanged, and `--fields` projects DSS paths
+ * out of it (exactly like the compact `*.list` shapes in list-output.ts).
+ *
+ * Imports no schema graph: the discovery schemas below are plain data.
+ */
+import { isRecord, } from "../utils/records.js";
+import { shapeListResult, } from "./list-output.js";
+import { DATASET_GET_DROPPED_KEYS, PROJECT_GET_DROPPED_KEYS, } from "./read-shapes.js";
+function record(value) {
+    return isRecord(value) ? value : {};
+}
+function nonEmptyString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+/** Drop `undefined` members so absent optional fields are omitted, never null. */
+function defined(fields) {
+    return Object.fromEntries(Object.entries(fields).filter(([, value,]) => value !== undefined));
+}
+/** `{ type, message }` of a DSS error object (`error`, `firstFailure`); undefined when empty. */
+function errorSummary(value) {
+    if (!isRecord(value))
+        return undefined;
+    const shaped = defined({
+        type: nonEmptyString(value["errorType"]),
+        message: nonEmptyString(value["detailedMessage"]) ?? nonEmptyString(value["message"]),
+    });
+    return Object.keys(shaped).length > 0 ? shaped : undefined;
+}
+/**
+ * Output id/project of a job output across DSS shapes: `targetDataset` +
+ * `targetDatasetProjectKey`, `targetManagedFolder`, `targetSavedModel`, ...
+ * and the plain `{ id, projectKey }` form.
+ */
+function outputTarget(output) {
+    const targetKey = Object.keys(output).find((key) => /^target[A-Z]/.test(key) && !key.endsWith("ProjectKey") && key !== "targetPartition"
+        && typeof output[key] === "string");
+    return {
+        id: targetKey === undefined ? output["id"] : output[targetKey],
+        projectKey: targetKey === undefined ? output["projectKey"] : output[`${targetKey}ProjectKey`],
+    };
+}
+function shapeJobOutput(output, jobProjectKey) {
+    const fields = record(output);
+    const { id, projectKey, } = outputTarget(fields);
+    const partition = fields["targetPartition"] ?? fields["partition"];
+    return defined({
+        type: fields["type"],
+        id,
+        projectKey: typeof projectKey === "string" && projectKey !== jobProjectKey
+            ? projectKey
+            : undefined,
+        partition: typeof partition === "string" && partition !== "NP" ? partition : undefined,
+    });
+}
+function shapeJobActivity(activity) {
+    const fields = record(activity);
+    return defined({
+        id: fields["activityId"],
+        recipe: fields["recipeName"],
+        recipeType: fields["recipeType"],
+        state: fields["state"],
+        totalTime: fields["totalTime"],
+        warnings: record(fields["warnings"])["totalCount"],
+        error: errorSummary(fields["firstFailure"]),
+        message: nonEmptyString(fields["message"]),
+    });
+}
+/** `job get`: state, outputs, progress, and the failure; never the log tail. */
+function shapeJobGet(result) {
+    if (!isRecord(result))
+        return result;
+    const base = record(result["baseStatus"]);
+    const def = record(base["def"]);
+    const activities = record(base["activities"]);
+    return defined({
+        id: def["id"],
+        name: def["name"],
+        type: def["type"],
+        state: base["state"],
+        initiator: nonEmptyString(def["initiator"])
+            ?? nonEmptyString(record(result["initiator"])["login"]),
+        triggeredFrom: def["triggeredFrom"],
+        startTime: base["jobStartTime"],
+        endTime: base["jobEndTime"],
+        outputs: Array.isArray(def["outputs"])
+            ? def["outputs"].map((output) => shapeJobOutput(output, def["projectKey"]))
+            : undefined,
+        progress: result["globalState"],
+        error: errorSummary(result["error"]),
+        activities: Object.values(activities).map(shapeJobActivity),
+    });
+}
+/** Extra column keys worth keeping when set (DSS sends most as ""/defaults). */
+const COLUMN_EXTRA_KEYS = ["meaning", "comment",];
+function shapeColumn(column) {
+    const fields = record(column);
+    const shaped = { name: fields["name"], type: fields["type"], };
+    for (const key of COLUMN_EXTRA_KEYS) {
+        const value = nonEmptyString(fields[key]);
+        if (value !== undefined)
+            shaped[key] = value;
+    }
+    return shaped;
+}
+function shapeLastBuild(value) {
+    if (!isRecord(value))
+        return undefined;
+    const shaped = defined({
+        jobId: value["jobId"],
+        success: value["buildSuccess"],
+        startTime: value["buildStartTime"],
+        endTime: value["buildEndTime"],
+    });
+    return Object.keys(shaped).length > 0 ? shaped : undefined;
+}
+/** `dataset info`: what the dataset is, its schema, flow neighbours, and freshness. */
+function shapeDatasetInfo(result) {
+    if (!isRecord(result))
+        return result;
+    const dataset = record(result["dataset"]);
+    const params = record(dataset["params"]);
+    const columns = record(dataset["schema"])["columns"];
+    const tags = dataset["tags"];
+    const quality = record(result["dataQualityStatus"]);
+    const versioning = record(result["versioning"]);
+    const timeline = record(result["timeline"]);
+    return defined({
+        name: result["name"] ?? dataset["name"],
+        type: result["type"] ?? dataset["type"],
+        managed: dataset["managed"],
+        connection: params["connection"] ?? params["uploadConnection"],
+        formatType: dataset["formatType"],
+        partitioned: result["partitioned"],
+        schema: Array.isArray(columns) ? columns.map(shapeColumn) : undefined,
+        recipes: Array.isArray(result["recipes"])
+            ? result["recipes"].map((recipe) => record(recipe)["name"])
+            : undefined,
+        buildable: result["buildable"],
+        upstreamBuildable: result["upstreamBuildable"],
+        downstreamBuildable: result["downstreamBuildable"],
+        tags: Array.isArray(tags) && tags.length > 0 ? tags : undefined,
+        dataQuality: Object.keys(quality).length > 0 && quality["numberItems"] !== 0
+            ? result["dataQualityStatus"]
+            : undefined,
+        lastBuild: shapeLastBuild(result["lastBuild"]),
+        lastModifiedOn: versioning["lastModifiedOn"] ?? timeline["lastModifiedOn"],
+        lastModifiedBy: record(versioning["lastModifiedBy"] ?? timeline["lastModifiedBy"])["login"],
+    });
+}
+/**
+ * Recursively remove empty strings, arrays and objects (including objects that
+ * become empty). `false`, `0` and `null` are kept. Array positions are
+ * preserved: an element that empties out stays as its empty value.
+ */
+function pruneEmpty(value) {
+    if (typeof value === "string")
+        return value.length === 0 ? undefined : value;
+    if (Array.isArray(value)) {
+        if (value.length === 0)
+            return undefined;
+        return value.map((item) => pruneEmpty(item) ?? (typeof item === "string" ? "" : Array.isArray(item) ? [] : {}));
+    }
+    if (isRecord(value)) {
+        const pruned = {};
+        for (const [key, member,] of Object.entries(value)) {
+            const kept = pruneEmpty(member);
+            if (kept !== undefined)
+                pruned[key] = kept;
+        }
+        return Object.keys(pruned).length === 0 ? undefined : pruned;
+    }
+    return value;
+}
+function withoutKeys(source, keys) {
+    return Object.fromEntries(Object.entries(source).filter(([key,]) => !keys.includes(key)));
+}
+/** `dataset get`: the settings object minus metadata and empty/default noise. */
+function shapeDatasetGet(result) {
+    if (!isRecord(result))
+        return result;
+    const settings = withoutKeys(result, DATASET_GET_DROPPED_KEYS);
+    const formatParams = settings["formatParams"];
+    if (isRecord(formatParams)) {
+        settings["formatParams"] = withoutKeys(formatParams, ["hiveSeparators",]);
+    }
+    return pruneEmpty(settings) ?? {};
+}
+const PERMISSION_KEY = /^can[A-Z]/;
+/**
+ * `project get`: as `dataset get`, with the boolean permission flags
+ * (`can*`, `isProjectAdmin`) collapsed to the denied ones. Absent from the
+ * list means granted, so nothing is lost.
+ */
+function shapeProjectGet(result) {
+    if (!isRecord(result))
+        return result;
+    const settings = withoutKeys(result, PROJECT_GET_DROPPED_KEYS);
+    const deniedPermissions = [];
+    for (const [key, value,] of Object.entries(settings)) {
+        if (typeof value !== "boolean" || !(PERMISSION_KEY.test(key) || key === "isProjectAdmin")) {
+            continue;
+        }
+        if (!value)
+            deniedPermissions.push(key);
+        delete settings[key];
+    }
+    const pruned = record(pruneEmpty(settings));
+    return { ...pruned, deniedPermissions, };
+}
+/** Compact read shapers by `resource.action`. */
+const READ_SHAPERS = {
+    "job.get": shapeJobGet,
+    "dataset.info": shapeDatasetInfo,
+    "dataset.get": shapeDatasetGet,
+    "project.get": shapeProjectGet,
+};
+/**
+ * Compact a command result for agents: `*.list` items (filter, cap, compact
+ * fields) and the four large read commands. `--full` and `--fields` bypass
+ * the read shapers so the DSS object (or a projection of it) is returned.
+ */
+export function shapeCommandResult(resource, action, result, flags) {
+    if (action === "list")
+        return shapeListResult(resource, result, flags);
+    const shaper = READ_SHAPERS[`${resource}.${action}`];
+    if (!shaper || flags["full"] === true || typeof flags["fields"] === "string")
+        return result;
+    return shaper(result);
+}

package/dist/src/cli/types.d.ts

@@ -7,6 +7,8 @@ export interface CommandPayloadSchema {
     dataFileFlag?: boolean;
     contentType?: "application/json" | "text/plain";
     jsonShape?: "object" | "array";
+    /** Precise JSON Schema of the payload, published as the contract's `schemas.input`. */
+    jsonSchema?: Record<string, unknown>;
 }
 export interface CommandFlagChoice {
     oneOf: string[][];

package/dist/src/cli/version.d.ts

@@ -49,7 +49,7 @@ export declare function buildVersionPayload(input: {
 export declare const PACKAGE_ROOT: string | undefined;
 export declare const CLI_VERSION: string;
 export declare const CLI_GIT_REVISION: string | undefined;
-export declare const AGENT_CONTRACT_VERSION = 3;
+export declare const AGENT_CONTRACT_VERSION = 4;
 export declare const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
-export declare const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v3.json";
+export declare const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v4.json";
 export declare function cliVersionResult(): CliVersionPayload;

package/dist/src/cli/version.js

@@ -204,9 +204,9 @@ export function buildVersionPayload(input) {
 export const PACKAGE_ROOT = findPackageRoot();
 export const CLI_VERSION = packageVersion(PACKAGE_ROOT);
 export const CLI_GIT_REVISION = gitRevision(PACKAGE_ROOT);
-export const AGENT_CONTRACT_VERSION = 3;
+export const AGENT_CONTRACT_VERSION = 4;
 export const JSON_SCHEMA_DRAFT = "https://json-schema.org/draft/2020-12/schema";
-export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v3.json";
+export const AGENT_CONTRACT_SCHEMA_ID = "https://clssck.github.io/dataiku-sdk/schemas/agent-contract-v4.json";
 export function cliVersionResult() {
     // Only code actually loaded from dist/ may report dist provenance; an
     // inherited DSS_LOAD_SOURCE on a source run is ignored.