dataiku-sdk 3.6.1 → 3.8.0

package/dist/src/utils/response-file.js

@@ -1,9 +1,6 @@
-import { randomUUID, } from "node:crypto";
-import { createWriteStream, } from "node:fs";
+import { closeSync, openSync, } from "node:fs";
 import { chmod, rename, rm, stat, } from "node:fs/promises";
 import { basename, dirname, join, } from "node:path";
-import { Readable, } from "node:stream";
-import { pipeline, } from "node:stream/promises";
 /** Stream with backpressure; publish only a complete download, preserving existing output on failure. */
 export async function writeResponseToFile(path, response, permissions = "private") {
     let mode;
@@ -17,16 +14,30 @@ export async function writeResponseToFile(path, response, permissions = "private
             mode = 0o666 & ~process.umask();
         }
     }
-    const input = Readable.from(response.body, { objectMode: false, });
-    const temporaryPath = join(dirname(path), `.${basename(path)}.tmp-${randomUUID()}`);
-    const output = createWriteStream(temporaryPath, { flags: "wx", mode: 0o600, });
+    const temporaryPath = join(dirname(path), `.${basename(path)}.tmp-${crypto.randomUUID()}`);
+    // Exclusive, private temp file; Bun.write streams the body into it natively.
+    let fd;
     try {
-        await pipeline(input, output);
+        fd = openSync(temporaryPath, "wx", 0o600);
+    }
+    catch (error) {
+        // Nothing will read the body; release the connection before surfacing the error.
+        await response.body?.cancel().catch(() => { });
+        throw error;
+    }
+    try {
+        let bytes;
+        try {
+            bytes = await Bun.write(Bun.file(fd), response);
+        }
+        finally {
+            closeSync(fd);
+        }
         // Keep partial contents private; restore publication permissions only after completion.
         if (mode !== undefined)
             await chmod(temporaryPath, mode);
         await rename(temporaryPath, path);
-        return output.bytesWritten;
+        return bytes;
     }
     finally {
         await rm(temporaryPath, { force: true, });

package/dist/src/utils/secret-sanitize.d.ts

@@ -46,6 +46,13 @@ export declare function redactUrlUserinfo(text: string): string;
  * recurse, arrays recurse. Non-string scalars pass through unchanged.
  */
 export declare function sanitizeSecrets<T>(value: T, options: SecretSanitizeOptions): T;
+/**
+ * Every string value under a credential-like key (any depth), plus bare
+ * `key` fields such as `authRealm.key`. Collected from a command's own JSON
+ * input so that DSS error text echoing the payload can be scrubbed; error
+ * redaction only, so the broad key match errs toward hiding.
+ */
+export declare function credentialValues(value: unknown): string[];
 /**
  * Scrub secret material from an error's message, stack, and string body.
  * `DataikuError.message` is built at construction, so all three carriers need

package/dist/src/utils/secret-sanitize.js

@@ -77,6 +77,52 @@ export function sanitizeSecrets(value, options) {
     }
     return value;
 }
+/** Credential-like key fragments (normalized); values under them are secrets. */
+const CREDENTIAL_KEY_FRAGMENTS = [
+    "password",
+    "passwd",
+    "passphrase",
+    "secret",
+    "token",
+    "apikey",
+    "accesskey",
+    "privatekey",
+    "credential",
+    "authorization",
+];
+/** Keys naming how credentials work (`credentialsMode`, `passwordType`), not secrets. */
+const CREDENTIAL_SETTING_SUFFIXES = ["mode", "type", "policy",];
+/**
+ * Every string value under a credential-like key (any depth), plus bare
+ * `key` fields such as `authRealm.key`. Collected from a command's own JSON
+ * input so that DSS error text echoing the payload can be scrubbed; error
+ * redaction only, so the broad key match errs toward hiding.
+ */
+export function credentialValues(value) {
+    const found = [];
+    const visit = (node, sensitive) => {
+        if (typeof node === "string") {
+            if (sensitive && node.length > 0)
+                found.push(node);
+            return;
+        }
+        if (Array.isArray(node)) {
+            for (const item of node)
+                visit(item, sensitive);
+            return;
+        }
+        if (node === null || typeof node !== "object")
+            return;
+        for (const [key, item,] of Object.entries(node)) {
+            const normalized = normalizeSecretKey(key);
+            const setting = CREDENTIAL_SETTING_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
+            visit(item, sensitive || (!setting && (normalized === "key"
+                || CREDENTIAL_KEY_FRAGMENTS.some((fragment) => normalized.includes(fragment)))));
+        }
+    };
+    visit(value, false);
+    return found;
+}
 /**
  * Scrub secret material from an error's message, stack, and string body.
  * `DataikuError.message` is built at construction, so all three carriers need

package/package.json

@@ -1,11 +1,13 @@
 {
 	"name": "dataiku-sdk",
-	"version": "3.6.1",
+	"version": "3.8.0",
 	"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/clssck/dataiku-sdk.git"
+	},
+	"license": "SEE LICENSE IN LICENSE",
 	"type": "module",
-	"workspaces": [
-		"packages/*"
-	],
 	"main": "dist/src/index.js",
 	"types": "dist/src/index.d.ts",
 	"bin": {
@@ -15,11 +17,12 @@
 		"dist/",
 		"plugin.json",
 		"skills/",
-		"bin/dss.js"
+		"bin/dss.js",
+		"bin/bun-version.js"
+	],
+	"workspaces": [
+		"packages/*"
 	],
-	"engines": {
-		"bun": ">=1.4.0"
-	},
 	"scripts": {
 		"build": "bun scripts/generate-action-output-schemas.mjs && tsc && bun scripts/generate-build-metadata.mjs",
 		"prepack": "bun scripts/sync-plugin-version.mjs --check && bun scripts/generate-action-output-schemas.mjs --check && tsc && bun scripts/generate-build-metadata.mjs",
@@ -40,30 +43,26 @@
 		"test:integration:rigorous": "cross-env RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-rigorous.test.ts",
 		"test:integration:rigorous:mutating": "cross-env RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-rigorous.test.ts"
 	},
+	"dependencies": {
+		"@sinclair/typebox": "^0.34.52",
+		"node-stream-zip": "^1.16.0"
+	},
 	"devDependencies": {
-		"@oxlint/plugins": "1.81.0",
+		"@oxlint/plugins": "1.86.0",
 		"@types/bun": "^1.4.2",
 		"cross-env": "^10.1.0",
-		"dprint": "^0.57.0",
+		"dprint": "^0.60.1",
 		"js-tiktoken": "1.0.21",
-		"oxlint": "1.81.0",
-		"typescript": "^7.0.2",
-		"typescript-language-server": "6.0.0",
-		"typescript-language-service": "npm:typescript@5.9.3"
-	},
-	"dependencies": {
-		"@sinclair/typebox": "^0.34.52",
-		"node-stream-zip": "^1.16.0"
+		"oxlint": "1.86.0",
+		"typescript": "^7.0.2"
 	},
-	"license": "SEE LICENSE IN LICENSE",
-	"repository": {
-		"type": "git",
-		"url": "https://github.com/clssck/dataiku-sdk.git"
+	"bundleDependencies": [
+		"@sinclair/typebox"
+	],
+	"engines": {
+		"bun": ">=1.4.2"
 	},
 	"publishConfig": {
 		"access": "public"
-	},
-	"bundleDependencies": [
-		"@sinclair/typebox"
-	]
+	}
 }

package/plugin.json

@@ -1,7 +1,7 @@
 {
 	"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
 	"name": "dataiku-sdk",
-	"version": "3.6.1",
+	"version": "3.8.0",
 	"description": "Dataiku DSS SDK and agent-oriented CLI for programmatic access to DSS REST APIs.",
 	"homepage": "https://github.com/clssck/dataiku-sdk",
 	"repository": "https://github.com/clssck/dataiku-sdk",

package/skills/dataiku-dss/SKILL.md

@@ -6,7 +6,7 @@ description: >-
   Discover syntax: dss commands run --fields RESOURCE.ACTION.
 license: LicenseRef-Dataiku-SDK-Limited-Use-1.0
 compatibility: >-
-  Requires Bun >= 1.4.0, the CLI, DSS network access, and credentials.
+  Requires Bun >= 1.4.2, the CLI, DSS network access, and credentials.
 ---
 
 # Dataiku DSS

package/skills/dataiku-dss/references/authentication.md

@@ -36,4 +36,6 @@ dss auth login
 ```
 `auth login` lists projects before saving: the key needs project-list permission. Returns `{"saved":true,"path":"..."}`. Storage: `credentials.json` in `DSS_CONFIG_DIR`, else `XDG_CONFIG_HOME/dataiku`, Windows `APPDATA/dataiku`, or `~/.config/dataiku`.
 
-TLS: `--insecure` disables verification; `--ca-cert PATH` adds a PEM CA bundle. Environment equivalents: `NODE_TLS_REJECT_UNAUTHORIZED`, `NODE_EXTRA_CA_CERTS`.
+TLS: `--insecure` disables verification; `--ca-cert PATH` adds a PEM CA bundle. Environment equivalents: `NODE_TLS_REJECT_UNAUTHORIZED`, `NODE_EXTRA_CA_CERTS`. A subprocess sees only its own env and cwd `.env`; `dss auth login --ca-cert PATH` saves the CA for all processes.
+
+Connection `403` the UI allows: check the key's identity (`dss doctor`: `authSource`, login, groups). `connection get` needs admin; `connection info NAME` covers usable connections.

package/skills/dataiku-dss/references/coding.md

@@ -3,6 +3,7 @@
 - Git: UAT=`create-branch --duplicate-project`; pull=rebase; checkout=switch. Remotes use DSS Git rules and SSH/server credentials. `--password-env`; `future-wait`.
 - Internal `lib/`: `project-library`. External Git: `project-git list-libraries`. Safe edit: `get-bytes` → `diff` → `put --expect-sha256 HASH`; create with `--if-not-exists`. Paths reject traversal.
 - Python: `code run --file`. Inspect/diff/backup recipes before `recipe run --dry-run`.
+- Grouping/prepare: build with `recipe create` flags (`--group-by`, `--aggregate`; `--rename`, `--formula`, `--filter`, ...). Other visual payloads (join): no published schema; template = `recipe get-payload` after create. Join `MANUAL` selection uses top-level `selectedColumns: [{name, table, alias?}]` (`table` = input index). DSS-engine computed columns run as H2 SQL: no GREL `+` concat, use `SQL` mode `||`. Grouping `first` needs an ordering column.
 - Notebook saves return hashes; guard with `--expect-hash`. Clear outputs via DSS DELETE; `unload-jupyter --all` composes session deletes.
 - Inspect code-env definitions/logs before updates; guard webapp/API settings with `--expect-hash`. `exact:false`: live state not guessed; use command `--dry-run`.
 - Plugin authoring: `dss plugin` create-dev, move-to-dev, contents-put, push/pull. No notebook execute/checkpoints or webapp/API delete; use the DSS UI.

package/skills/dataiku-dss/references/discovery.md

@@ -8,11 +8,13 @@
 - Exits: 0 success, 1 usage/configuration, 2 DSS/internal/permission-or-environment, 3 transient/retryable DSS error, 4 failed long-running result/assertion.
 - Recipe payload stdout: JSON string. With `--output PATH`: exact bytes to file, JSON string equal to `PATH` on stdout.
 - General `--fields a,b,c`: object projection, element-wise for object arrays. Dotted paths traverse nested objects; missing fields become `null`; strings/scalars pass through.
+- `*.list`: compact items (ids, kind, next-step fields such as recipe `inputs`/`outputs`); `--full`/`--fields` use DSS objects. `--contains TEXT` filters ids; `--limit N` caps (`list_truncated` warning).
 
 ## Discovery
 
 ```text
 dss commands run
+dss commands run --contains "propagate schema"
 dss agent contract --fields commands.actions.dataset
 dss commands run --fields dataset.create
 dss commands run --fields dataset.preview.usage,dataset.preview.description,dataset.preview.flags,dataset.preview.examples
@@ -21,10 +23,6 @@ dss agent contract --fields protocol,agentContractVersion,cli,stdio,planning,com
 dss agent contract --fields commands.actions
 ```
 
-`commands run` defaults to a resource→action-name summary (~1.6k tokens), never full entries. For a known resource, project `commands.actions.RESOURCE` from `agent contract` (~64 tokens for dataset). Bootstrap once with the six-field projection above (~250 tokens); fetch `schemas` or `commands` only as needed.
+`commands run`: resource→action summary (~1.6k tokens). Unknown name: `--contains TEXT` gives the top 5 actions with usage. Known resource: `agent contract --fields commands.actions.RESOURCE`. Bootstrap once with the six-field projection (~250 tokens). Global flags (`--fields`, `--verbose`, connection/TLS) appear once in `commands.globalFlags`, not per entry.
 
-Look up syntax before invoking. `--fields RESOURCE` selects every full entry (potentially large); `RESOURCE.ACTION` one complete entry; append `.FIELD` for nested metadata. Reads: prefer `usage,description,flags,examples`. Writes: fetch the full entry directly once, not compact-then-full. It includes flags, positionals, side effects, auth, output, idempotency, dry-run, examples, payload schemas, unsafe outputs, cleanup, and exits.
-
-Comma-separate selectors to batch lookups. Keys echo selectors: `--fields dataset.create` → `{"dataset.create":{...}}`. Empty `--fields` fails usage validation, never dumps everything. Unknown resources/actions exit 1 with compact JSON errors listing valid options.
-
-Full registry: `commands run --output PATH` only. With `--fields`, export that subset instead. Files contain compact JSON; stdout is `{"path":"PATH"}`, never registry content.
+Look up syntax before invoking: `RESOURCE.ACTION` is one entry, `.FIELD` nests, commas batch, keys echo selectors. Reads: `usage,description,flags,examples`. Writes: the full entry once (flags, side effects, idempotency, dry-run, payload schemas, cleanup, exits). Unknown flags, resources, and actions exit 1 with the usage line or valid options. Full registry: `commands run --output PATH` only; stdout is `{"path":"PATH"}`.

package/skills/dataiku-dss/references/flow-maps.md

@@ -7,6 +7,6 @@ dss flow-zone organize --file flow-zones.json --dry-run --project-key MYPROJ
 dss flow-zone organize --file flow-zones.json --project-key MYPROJ
 ```
 
-`project map`: zones, SCC-safe layers, weak components, diagnostics, full-flow fingerprint; rendering in `rendering.content`. DSS exposes zone positions, not node pixels.
+`project map`: zones, SCC-safe layers, weak components, diagnostics, fingerprint; rendering in `rendering.content`.
 
-Feed `flow-zone plan` to `flow-zone organize`; inspect dry-run first. Organize checks topology before/after writes and skips unchanged moves. Audit recipe payloads separately: layout commands neither fetch nor analyze code.
+Feed `flow-zone plan` (`unassigned`: to place) to `flow-zone organize`; dry-run first. Recipe outputs follow their recipe. DSS positions a zone only at creation; `--recreate-on-move` moves it (new id). Layout commands never read recipe code.

package/skills/dataiku-dss/references/troubleshooting.md

@@ -4,7 +4,8 @@
 
 - Code/SQL: use `--file`/`--sql-file`; shells, especially PowerShell, mangle quotes, `$`, and multiline text. Non-UTF-8 consoles (e.g. Windows cp1252): write non-ASCII results to UTF-8 files or `--output PATH`, not the console.
 - Build errors: `dss job log <id> --errors-only` selects errors/tracebacks. For full logs, use `--output PATH` without `--errors-only`. Logs may arrive as one JVM-noisy line; `Error in Python process: At line <N>` identifies the payload source line.
-- After python/SQL/prepare output-column changes, supply updated columns with `dss dataset refresh-schema NAME --data-file columns.json` or rebuild before downstream reads. Run `dss dataset validate-build` before builds to catch file-backed misconfiguration. Exception: `dss recipe create --type sync` copies input schema to schemaless output at creation (`syncOutputSchemaPropagated`); fresh sync outputs build populated without manual refresh.
+- Schemas: code recipes set output schemas at run time. Visual/SQL query recipes: `recipe create` fills created or empty outputs (`outputSchemaUpdated`); `recipe set-payload` updates empty or DSS-derived ones, warns `recipe_output_schema_outdated` for hand-edited ones. Apply with `dss recipe update-schema NAME` (one recipe), `dss flow propagate-schema DATASET --wait` (every downstream recipe), or build `--auto-update-schema`. `built_dataset_has_no_columns`: the DONE build wrote nothing. `nothing_to_build` (exit 4): no recipe writes the managed target. Manual columns: `dss dataset refresh-schema NAME --data-file columns.json`. `dss dataset validate-build` catches file-backed misconfiguration.
+- `Inline` (editable) datasets: the public API cannot write rows, so `dataset create` rejects them; use `UploadedFiles` + `dataset upload-file`, or a python recipe output. `code run` is a scenario step; some DSS versions refuse dataset writes there.
 - `dss dataset download`: default 100k rows; result `{ path, rows, truncated, limit }`. If truncated, raise `--limit`. Specify `--output` or read the `dataset_download_default_location` path warning. CSV is spreadsheet-safe; `--raw-data` retains formula prefixes. For large tables, aggregate in SQL or use a recipe.
 - `dss dashboard create`/`update`: before POST/PUT, every `INSIGHT` tile needs agreeing/resolvable `insightId` and `targetInsightId`; any `datasetSmartName` must resolve. Missing/stale references: `validation_failed`. Cross-project `PROJECT.DATASET` needs dataset-project read permission; `403` blocks saving an unverifiable dashboard.
 - `dss api-service list-packages SERVICE` checks parent settings first. `not_found`: missing service; verify service ID/project key. Empty array: existing service, no deployable packages. Never convert `not_found` to an empty list or retry the lower-level route.
@@ -26,4 +27,4 @@ Dispatch/runtime failure: one compact stdout error:
 
 `doctor`/`batch`/`cleanup` command failures return direct stdout result objects; check nonzero exit before interpreting. Recover from structured `code`, `category`, `exitCode`, `retryable`, `status`, `details`, never message scraping.
 
-`details.body` is sanitized metadata, **not the DSS response body**: at most `requestId`/`request_id`/`errorId`/`elapsedMs` plus locally trusted target/timing. `details.statusText` derives from numeric status, not remote reason phrases. Use top-level `code`, `category`, `status`, `retryable`, `requestId`, `hint`, and `details.dssCategory`; never assume server-body fields.
+DSS failures: `error` is `STATUS Text: DSS message` (credentials redacted, bounded); `hint` names the next check (resource, project). `details` appears only with content: `elapsedMs`/`target`, `dssErrorType`, `bodyTruncated`, and `retry` when a retry or timeout happened. Refused connections, unknown hosts, bad URLs, and untrusted certificates fail at once as `validation_failed` (exit 2); `transient` (exit 3) means automatic retries already ran.