dataiku-sdk 3.9.0 → 4.0.0

package/dist/src/utils/project-archive.js

@@ -345,6 +345,190 @@ function residueIssue(member, counts) {
         member,
     };
 }
+/** Cap on credential findings recorded per member, bounding inventory size. */
+const MAX_CREDENTIAL_FINDINGS_PER_MEMBER = 200;
+/** Deepest JSON nesting the credential walker follows. */
+const MAX_CREDENTIAL_WALK_DEPTH = 32;
+const PROJECT_CONFIG_ROOT = "project_config";
+/** Credential-store member names under `project_config/` (api keys, secrets, credentials files). */
+const CREDENTIAL_MEMBER_RE = /^(?:api[-_]?keys?|secrets?|credentials?)\.json$/i;
+const CREDENTIAL_SINGLE_TOKENS = {
+    secret: true,
+    secrets: true,
+    token: true,
+    tokens: true,
+    password: true,
+    passwords: true,
+    passwd: true,
+    passphrase: true,
+    credential: true,
+    credentials: true,
+    apikey: true,
+    apikeys: true,
+    auth: true,
+    oauth: true,
+    authorization: true,
+};
+/**
+ * Trailing key words that describe a credential rather than hold one
+ * (`tokenType`, `authMode`, `secretName`, `passwordColumn`, ...).
+ */
+const NON_SECRET_SUFFIX_TOKENS = {
+    type: true,
+    mode: true,
+    method: true,
+    kind: true,
+    provider: true,
+    column: true,
+    columns: true,
+    field: true,
+    fields: true,
+    name: true,
+    names: true,
+    label: true,
+    description: true,
+    url: true,
+    uri: true,
+    path: true,
+    scope: true,
+    scopes: true,
+    format: true,
+    policy: true,
+    enabled: true,
+    required: true,
+};
+/** Split `DHL_API_KEY`, `apiKey`, `x-api-key` and `APIKey` into lower-case words. */
+function keyTokens(key) {
+    return key
+        .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+        .split(/[^A-Za-z0-9]+/)
+        .filter((token) => token.length > 0)
+        .map((token) => token.toLowerCase());
+}
+function isCredentialName(key) {
+    const tokens = keyTokens(key);
+    const last = tokens.at(-1);
+    if (last !== undefined && NON_SECRET_SUFFIX_TOKENS[last] === true)
+        return false;
+    for (let i = 0; i < tokens.length; i++) {
+        const token = tokens[i];
+        if (CREDENTIAL_SINGLE_TOKENS[token] === true)
+            return true;
+        const next = tokens[i + 1];
+        if ((token === "api" || token === "private") && (next === "key" || next === "keys"))
+            return true;
+    }
+    return false;
+}
+/** Non-empty string that is not a pure `${VARIABLE}` reference. */
+function isLiteralSecretValue(value) {
+    if (typeof value !== "string")
+        return false;
+    const trimmed = value.trim();
+    return trimmed.length > 0 && !/^\$\{[^}]*\}$/.test(trimmed);
+}
+/** True for null, empty strings and containers holding only such values. */
+function isEmptyJsonValue(value) {
+    if (value === null || value === undefined)
+        return true;
+    if (typeof value === "string")
+        return value.trim().length === 0;
+    if (Array.isArray(value))
+        return value.every(isEmptyJsonValue);
+    if (typeof value === "object")
+        return Object.values(value).every(isEmptyJsonValue);
+    return false;
+}
+function jsonPathSegment(key) {
+    const safe = key.length > 80 ? `${key.slice(0, 80)}...` : key;
+    return /^[A-Za-z_$][\w$]*$/.test(safe) ? `.${safe}` : `[${JSON.stringify(safe)}]`;
+}
+/**
+ * Structural credential detection for one `project_config/` member: the
+ * member-name rule plus a JSON-aware key walk. Only member paths and key
+ * paths/field names are returned, never values.
+ */
+function collectCredentialFindings(member, text) {
+    const findings = [];
+    const full = () => findings.length >= MAX_CREDENTIAL_FINDINGS_PER_MEMBER;
+    let parsed;
+    let isJson = false;
+    try {
+        parsed = JSON.parse(text);
+        isJson = true;
+    }
+    catch {
+        // Not JSON: only the member-name rule can apply.
+    }
+    const basename = member.slice(member.lastIndexOf("/") + 1);
+    if (CREDENTIAL_MEMBER_RE.test(basename)) {
+        const empty = isJson ? isEmptyJsonValue(parsed) : text.trim().length === 0;
+        if (!empty)
+            findings.push({ member, rule: "member_name", });
+    }
+    if (!isJson)
+        return findings;
+    const visit = (node, path, inCredential, field, depth) => {
+        if (full() || depth > MAX_CREDENTIAL_WALK_DEPTH)
+            return;
+        if (typeof node === "string") {
+            if (inCredential && isLiteralSecretValue(node)) {
+                findings.push({ member, rule: "json_key", keyPath: path, field, });
+            }
+            return;
+        }
+        if (Array.isArray(node)) {
+            node.forEach((item, index) => visit(item, `${path}[${index}]`, inCredential, field, depth + 1));
+            return;
+        }
+        if (node === null || typeof node !== "object")
+            return;
+        const record = node;
+        const name = record["name"];
+        if (typeof name === "string"
+            && isCredentialName(name)
+            && isLiteralSecretValue(record["value"])
+            && !full()) {
+            const shownName = name.length > 80 ? `${name.slice(0, 80)}...` : name;
+            findings.push({ member, rule: "list_variable", keyPath: path, field: shownName, });
+        }
+        for (const [key, value,] of Object.entries(record)) {
+            if (full())
+                return;
+            const childPath = `${path}${jsonPathSegment(key)}`;
+            const credentialKey = isCredentialName(key);
+            const leafInContext = inCredential
+                && NON_SECRET_SUFFIX_TOKENS[keyTokens(key).at(-1) ?? ""] !== true;
+            if (typeof value === "string") {
+                if ((credentialKey || leafInContext) && isLiteralSecretValue(value)) {
+                    findings.push({ member, rule: "json_key", keyPath: childPath, field: key, });
+                }
+            }
+            else {
+                visit(value, childPath, inCredential || credentialKey, key, depth + 1);
+            }
+        }
+    };
+    visit(parsed, "$", false, "", 0);
+    return findings;
+}
+/** Visible warning for a scannable member that could not be scanned for residue and credentials. */
+function residueScanSkippedIssue(member) {
+    return {
+        severity: "warning",
+        code: "residue_scan_skipped",
+        message: `member exceeds the ${MAX_RESIDUE_SCAN_BYTES}-byte residue scan limit and was not`
+            + " scanned for credentials, identity or machine-specific residue",
+        member,
+    };
+}
+/** Deterministic inventory ordering: member, then key path. */
+function sortCredentialFindings(findings) {
+    findings.sort((a, b) => a.member.localeCompare(b.member)
+        || (a.keyPath ?? "").localeCompare(b.keyPath ?? "")
+        || a.rule.localeCompare(b.rule));
+}
 /** Deterministic issue ordering: severity, then code, member, message. */
 function sortIssues(issues) {
     issues.sort((a, b) => {
@@ -413,6 +597,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
             memberCount: 0,
             valid: false,
             issues,
+            credentialFindings: [],
         };
     }
     let zip;
@@ -434,6 +619,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
             memberCount: 0,
             valid: false,
             issues,
+            credentialFindings: [],
         };
     }
     try {
@@ -452,6 +638,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
                 memberCount: records.length,
                 valid: false,
                 issues,
+                credentialFindings: [],
             };
         }
         const memberCounts = new Map();
@@ -481,6 +668,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
         const recipes = new Map();
         const tally = { expandedBytes: 0, };
         let aggregateAborted = false;
+        const credentialFindings = [];
         for (const entry of records) {
             if (!entry.isFile)
                 continue;
@@ -504,11 +692,14 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
                 });
             }
             const parseCandidate = (isManifest || isRecipeJson) && entry.size <= MAX_JSON_MEMBER_BYTES;
-            const scanCandidate = (isManifest
-                || name.startsWith("project_config/")
+            const matchesScanPrefix = isManifest
+                || name.startsWith(`${PROJECT_CONFIG_ROOT}/`)
                 || name.startsWith(`${BUNDLED_DATA_ROOT}/`)
-                || name.startsWith(`${UPLOADS_ROOT}/`))
-                && entry.size <= MAX_RESIDUE_SCAN_BYTES;
+                || name.startsWith(`${UPLOADS_ROOT}/`);
+            const scanCandidate = matchesScanPrefix && entry.size <= MAX_RESIDUE_SCAN_BYTES;
+            if (matchesScanPrefix && !scanCandidate) {
+                issues.push(residueScanSkippedIssue(name));
+            }
             // Budgets computed from the central-directory record abort before
             // any decompression work: a member whose declared compressed size
             // or expanded/compressed ratio is above budget is never streamed.
@@ -621,6 +812,8 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
                     message: `member ${name}: data exceeds the inspection byte limit and was not buffered for parsing`,
                     member: name,
                 });
+                if (scanCandidate)
+                    issues.push(residueScanSkippedIssue(name));
             }
             const content = streamed.content;
             if (!content)
@@ -649,10 +842,24 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
                 }
             }
             if (scanCandidate && buffer.length <= MAX_RESIDUE_SCAN_BYTES) {
-                const counts = countResidue(buffer.toString("utf8"));
+                const text = buffer.toString("utf8");
+                const counts = countResidue(text);
                 if (counts.credential + counts.email + counts.homePath + counts.metadataField > 0) {
                     issues.push(residueIssue(name, counts));
                 }
+                if (name.startsWith(`${PROJECT_CONFIG_ROOT}/`)) {
+                    const found = collectCredentialFindings(name, text);
+                    if (found.length > 0) {
+                        credentialFindings.push(...found);
+                        issues.push({
+                            severity: "warning",
+                            code: "credential_structure",
+                            message: `member contains ${found.length} credential-bearing structure(s);`
+                                + " see the credential inventory for key paths (values are never reported)",
+                            member: name,
+                        });
+                    }
+                }
             }
         }
         // Once the aggregate expanded-byte budget is crossed, remaining
@@ -660,6 +867,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
         // manifest/graph conclusion is drawn from the partial inspection.
         if (aggregateAborted) {
             sortIssues(issues);
+            sortCredentialFindings(credentialFindings);
             return {
                 filePath,
                 sizeBytes,
@@ -667,6 +875,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
                 sourceProjectKey: undefined,
                 valid: false,
                 issues,
+                credentialFindings,
             };
         }
         let sourceProjectKey;
@@ -726,6 +935,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
         checkRecipeReferences(recipes, datasetNames, nonDatasetNames, issues);
         checkOrphanMembers(records, recipeNames, anchored, issues);
         sortIssues(issues);
+        sortCredentialFindings(credentialFindings);
         return {
             filePath,
             sizeBytes,
@@ -733,6 +943,7 @@ export async function inspectProjectArchive(filePath, overrides = {}) {
             sourceProjectKey,
             valid: !issues.some((issue) => issue.severity === "error"),
             issues,
+            credentialFindings,
         };
     }
     finally {

package/dist/src/utils/response-file.js

@@ -1,5 +1,4 @@
-import { closeSync, openSync, } from "node:fs";
-import { chmod, rename, rm, stat, } from "node:fs/promises";
+import { chmod, mkdir, open, rename, rm, stat, } from "node:fs/promises";
 import { basename, dirname, join, } from "node:path";
 /** Stream with backpressure; publish only a complete download, preserving existing output on failure. */
 export async function writeResponseToFile(path, response, permissions = "private") {
@@ -15,10 +14,13 @@ export async function writeResponseToFile(path, response, permissions = "private
         }
     }
     const temporaryPath = join(dirname(path), `.${basename(path)}.tmp-${crypto.randomUUID()}`);
-    // Exclusive, private temp file; Bun.write streams the body into it natively.
-    let fd;
+    // Exclusive, private temp file. The body is copied chunk by chunk with an
+    // awaited write per chunk (backpressure). Not Bun.write(Bun.file(fd), response):
+    // on Windows, Bun 1.4.3 let the process exit 0 mid-copy with nothing written.
+    let handle;
     try {
-        fd = openSync(temporaryPath, "wx", 0o600);
+        await mkdir(dirname(path), { recursive: true, });
+        handle = await open(temporaryPath, "wx", 0o600);
     }
     catch (error) {
         // Nothing will read the body; release the connection before surfacing the error.
@@ -26,12 +28,17 @@ export async function writeResponseToFile(path, response, permissions = "private
         throw error;
     }
     try {
-        let bytes;
+        let bytes = 0;
         try {
-            bytes = await Bun.write(Bun.file(fd), response);
+            if (response.body) {
+                for await (const chunk of response.body) {
+                    await handle.write(chunk);
+                    bytes += chunk.byteLength;
+                }
+            }
         }
         finally {
-            closeSync(fd);
+            await handle.close();
         }
         // Keep partial contents private; restore publication permissions only after completion.
         if (mode !== undefined)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "dataiku-sdk",
-	"version": "3.9.0",
+	"version": "4.0.0",
 	"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
 	"repository": {
 		"type": "git",
@@ -27,21 +27,21 @@
 		"build": "bun scripts/generate-action-output-schemas.mjs && tsc && bun scripts/generate-build-metadata.mjs",
 		"prepack": "bun scripts/sync-plugin-version.mjs --check && bun scripts/generate-action-output-schemas.mjs --check && tsc && bun scripts/generate-build-metadata.mjs",
 		"prepublishOnly": "bun run prepack",
-		"check": "tsc --noEmit && tsc -p tools/oxlint/tsconfig.json && bun run check:live && bun scripts/generate-action-output-schemas.mjs --check && bun scripts/sync-plugin-version.mjs --check && bun scripts/cut-changelog.mjs --check",
+		"verify": "bun check -p tsconfig.json && bun check -p tools/oxlint/tsconfig.json && bun run check:live && bun scripts/generate-action-output-schemas.mjs --check && bun scripts/sync-plugin-version.mjs --check && bun scripts/cut-changelog.mjs --check",
 		"version": "bun scripts/sync-plugin-version.mjs && bun scripts/cut-changelog.mjs && git add plugin.json CHANGELOG.md",
 		"lint": "oxlint src/ tools/oxlint/ scripts/ tests/",
 		"lint:fix": "oxlint --fix src/ tools/oxlint/ scripts/ tests/",
 		"format": "dprint fmt",
 		"format:check": "dprint check",
-		"test": "bun test",
+		"test": "bun test --parallel --timeout 30000",
 		"test:live": "bun --no-env-file scripts/live-suite.ts",
-		"check:live": "tsc --noEmit -p tsconfig.live.json",
+		"check:live": "bun check -p tsconfig.live.json",
 		"test:tokens": "bun test tests/cli/agent-token-budget.test.ts",
 		"test:platform": "bun scripts/platform-smoke.mjs",
-		"test:integration": "cross-env RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-playground.test.ts",
-		"test:integration:mutating": "cross-env RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-playground.test.ts",
-		"test:integration:rigorous": "cross-env RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-rigorous.test.ts",
-		"test:integration:rigorous:mutating": "cross-env RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-rigorous.test.ts"
+		"test:integration": "RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-playground.test.ts",
+		"test:integration:mutating": "RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-playground.test.ts",
+		"test:integration:rigorous": "RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-rigorous.test.ts",
+		"test:integration:rigorous:mutating": "RUN_DATAIKU_INTEGRATION=1 RUN_DATAIKU_INTEGRATION_MUTATING=1 bun test tests/integration-rigorous.test.ts"
 	},
 	"dependencies": {
 		"@sinclair/typebox": "^0.34.52",
@@ -49,9 +49,8 @@
 	},
 	"devDependencies": {
 		"@oxlint/plugins": "1.87.0",
-		"@types/bun": "^1.4.2",
-		"cross-env": "^10.1.0",
-		"dprint": "^0.61.1",
+		"@types/bun": "^1.4.3",
+		"dprint": "^0.63.0",
 		"js-tiktoken": "1.0.21",
 		"oxlint": "1.87.0",
 		"typescript": "^7.0.2"
@@ -60,7 +59,7 @@
 		"@sinclair/typebox"
 	],
 	"engines": {
-		"bun": ">=1.4.2"
+		"bun": ">=1.4.3"
 	},
 	"publishConfig": {
 		"access": "public"

package/plugin.json

@@ -1,7 +1,7 @@
 {
 	"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
 	"name": "dataiku-sdk",
-	"version": "3.9.0",
+	"version": "4.0.0",
 	"description": "Dataiku DSS SDK and agent-oriented CLI for programmatic access to DSS REST APIs.",
 	"homepage": "https://github.com/clssck/dataiku-sdk",
 	"repository": "https://github.com/clssck/dataiku-sdk",

package/skills/dataiku-dss/SKILL.md

@@ -6,7 +6,7 @@ description: >-
   Discover syntax: dss commands run --fields RESOURCE.ACTION.
 license: LicenseRef-Dataiku-SDK-Limited-Use-1.0
 compatibility: >-
-  Requires Bun >= 1.4.2, the CLI, DSS network access, and credentials.
+  Requires Bun >= 1.4.3, the CLI, DSS network access, and credentials.
 ---
 
 # Dataiku DSS