dataiku-sdk 3.1.0 → 3.2.0

package/dist/src/client.js

@@ -76,15 +76,6 @@ function isTransientError(status, body) {
 function shouldRetryMethod(method) {
     return method.toUpperCase() === "GET";
 }
-function concatBytes(chunks, total) {
-    const out = new Uint8Array(total);
-    let offset = 0;
-    for (const chunk of chunks) {
-        out.set(chunk, offset);
-        offset += chunk.byteLength;
-    }
-    return out;
-}
 /**
  * Drop a trailing incomplete UTF-8 multibyte sequence so the bytes end on a
  * character boundary. Guarantees the decoded string re-encodes to no more bytes
@@ -133,7 +124,7 @@ function buildBodyReadTimeoutError(timeoutMs) {
  * Read the next chunk of a response body under a hard deadline.
  * When `remainingMs` elapses, the stream is cancelled and the caller's
  * promise rejects with a DataikuError, so a stalled body cannot hang the
- * buffered consumers after headers already arrived.
+ * consumers after headers already arrived.
  */
 async function readChunkWithDeadline(reader, remainingMs, timeoutMs) {
     const { promise, resolve, reject, } = Promise.withResolvers();
@@ -506,11 +497,12 @@ export class DataikuClient {
         return text;
     }
     async postStream(path, body) {
-        return this.fetchWithRetry(`${this.baseUrl}${path}`, {
+        const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
             method: "POST",
             headers: { ...this.getAnyHeaders(), "Content-Type": "application/json", },
             body: body !== undefined ? JSON.stringify(body) : undefined,
         });
+        return this.withBodyDeadline(res);
     }
     async put(path, body) {
         const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
@@ -551,10 +543,51 @@ export class DataikuClient {
         return this.parseJsonResponse(response);
     }
     async stream(path) {
-        return this.fetchWithRetry(`${this.baseUrl}${path}`, {
+        const res = await this.fetchWithRetry(`${this.baseUrl}${path}`, {
             method: "GET",
             headers: this.getAnyHeaders(),
         });
+        return this.withBodyDeadline(res);
+    }
+    /** Bound each pending read, not total transfer time; preserve large downloads and backpressure. */
+    withBodyDeadline(res) {
+        if (!res.body)
+            return res;
+        const reader = res.body.getReader();
+        const timeoutMs = this.requestTimeoutMs;
+        const body = new ReadableStream({
+            async pull(controller) {
+                try {
+                    const { done, value, } = await readChunkWithDeadline(reader, timeoutMs, timeoutMs);
+                    if (done) {
+                        controller.close();
+                        reader.releaseLock();
+                    }
+                    else {
+                        controller.enqueue(value);
+                    }
+                }
+                catch (error) {
+                    controller.error(error);
+                    void reader.cancel(error).catch(() => { });
+                }
+            },
+            cancel(reason) {
+                void reader.cancel(reason).catch(() => { });
+            },
+        });
+        const response = new Response(body, {
+            status: res.status,
+            statusText: res.statusText,
+            headers: res.headers,
+        });
+        // ResponseInit has no fields for these fetch response properties.
+        Object.defineProperties(response, {
+            url: { value: res.url, },
+            redirected: { value: res.redirected, },
+            type: { value: res.type, },
+        });
+        return response;
     }
     /* ---- private: headers ---- */
     getHeaders() {
@@ -658,7 +691,34 @@ export class DataikuClient {
         if (!res.body)
             return { text: "", truncated: false, };
         const reader = res.body.getReader();
-        const chunks = [];
+        const decoder = new TextDecoder();
+        const parts = [];
+        // Keep the final UTF-8 sequence undecoded until truncation is known.
+        // Four bytes also cover malformed sequences handled by trimToUtf8Boundary.
+        const tail = new Uint8Array(4);
+        let tailLength = 0;
+        const append = (chunk) => {
+            if (chunk.byteLength >= tail.length) {
+                if (tailLength > 0) {
+                    parts.push(decoder.decode(tail.subarray(0, tailLength), { stream: true, }));
+                }
+                const end = chunk.byteLength - tail.length;
+                if (end > 0)
+                    parts.push(decoder.decode(chunk.subarray(0, end), { stream: true, }));
+                tail.set(chunk.subarray(end));
+                tailLength = tail.length;
+            }
+            else {
+                const excess = Math.max(0, tailLength + chunk.byteLength - tail.length);
+                if (excess > 0) {
+                    parts.push(decoder.decode(tail.subarray(0, excess), { stream: true, }));
+                    tail.copyWithin(0, excess, tailLength);
+                    tailLength -= excess;
+                }
+                tail.set(chunk, tailLength);
+                tailLength += chunk.byteLength;
+            }
+        };
         let bytesRead = 0;
         let truncated = false;
         const startedAt = Date.now();
@@ -672,12 +732,12 @@ export class DataikuClient {
                     break;
                 const room = limit - bytesRead;
                 if (value.byteLength > room) {
-                    chunks.push(value.slice(0, room));
+                    append(value.subarray(0, room));
                     bytesRead += room;
                     truncated = true;
                     break;
                 }
-                chunks.push(value);
+                append(value);
                 bytesRead += value.byteLength;
             }
             if (!truncated && bytesRead >= limit) {
@@ -693,12 +753,9 @@ export class DataikuClient {
         finally {
             void reader.cancel().catch(() => { });
         }
-        const collected = concatBytes(chunks, bytesRead);
-        // On truncation, drop any trailing partial UTF-8 character so the decoded
-        // text never re-encodes beyond the byte cap (a stream-flush would otherwise
-        // emit a 3-byte replacement char for a split multibyte sequence).
-        const usable = truncated ? trimToUtf8Boundary(collected) : collected;
-        return { text: new TextDecoder().decode(usable), truncated, };
+        const remaining = tail.subarray(0, tailLength);
+        parts.push(decoder.decode(truncated ? trimToUtf8Boundary(remaining) : remaining));
+        return { text: parts.join(""), truncated, };
     }
     /**
      * Read a 2xx response body keeping only the last `maxBytes` bytes.

package/dist/src/errors.js

@@ -240,6 +240,8 @@ export function classifyDataikuError(status, body) {
         && (lowerBody.includes("column_not_found")
             || lowerBody.includes("table_not_found")
             || lowerBody.includes("no_such_table")
+            || lowerBody.includes("no such column")
+            || lowerBody.includes("no such table")
             || lowerBody.includes("column does not exist"));
     if (isSqlEngineValidation) {
         return {

package/dist/src/resources/folders.js

@@ -1,6 +1,7 @@
 import * as nodePath from "node:path";
 import { FolderDetailsSchema, FolderItemArraySchema, FolderSummaryArraySchema, } from "../schemas.js";
 import { deepMerge, } from "../utils/deep-merge.js";
+import { writeResponseToFile, } from "../utils/response-file.js";
 import { sanitizeFileName, } from "../utils/sanitize.js";
 import { BaseResource, } from "./base.js";
 import { resolveAdminManagedStorageConnection, } from "./connections.js";
@@ -73,7 +74,7 @@ export class FoldersResource extends BaseResource {
             throw new Error("folders.download response did not include a body");
         const dest = opts?.localPath
             ?? nodePath.resolve(process.cwd(), inferDownloadFileName(normalizedPath));
-        await Bun.write(dest, new Response(res.body), { createPath: false, });
+        await writeResponseToFile(dest, res);
         return dest;
     }
     upload(folderId, path, localPath, projectKey) {

package/dist/src/resources/projects.d.ts

@@ -87,8 +87,6 @@ export interface ProjectImportResult extends ProjectImportProcessResult {
     projectIncarnationHash?: string;
 }
 export declare class ProjectsResource extends BaseResource {
-    private httpInternals;
-    private postStream;
     /** Create a new project. */
     createProject(projectKey: string, name: string, ownerLogin: string, settings?: ProjectLifecycleSettings | null): Promise<Record<string, unknown>>;
     /** Delete a project using the lifecycle endpoint. */

package/dist/src/resources/projects.js

@@ -1,4 +1,5 @@
 import { ClientValidationError, DataikuError, } from "../errors.js";
+import { writeResponseToFile, } from "../utils/response-file.js";
 import { ProjectDetailsSchema, ProjectMetadataSchema, ProjectSummaryArraySchema, } from "../schemas.js";
 import { analyzeFlowMap, flowTopologyFingerprint, renderFlowMap, } from "../utils/flow-analysis.js";
 import { normalizeFlowGraph, } from "../utils/flow-map.js";
@@ -104,21 +105,6 @@ const DEFAULT_MAX_EDGES = 600;
 const DEFAULT_METADATA_TIMEOUT_MS = 1_500;
 const PROJECT_IMPORT_AMBIGUOUS_REMEDIATION = "Import processing left DSS state indeterminate. Inspect the archive with `dss project inspect-archive <file>` and the live projects with `dss project list` before retrying the import.";
 export class ProjectsResource extends BaseResource {
-    httpInternals() {
-        return this.client;
-    }
-    async postStream(path, body) {
-        const http = this.httpInternals();
-        return http.fetchWithRetry(`${http.baseUrl}${path}`, {
-            method: "POST",
-            headers: {
-                Authorization: `Bearer ${http.apiKey}`,
-                Accept: "*/*",
-                "Content-Type": "application/json",
-            },
-            body: JSON.stringify(body),
-        });
-    }
     /** Create a new project. */
     async createProject(projectKey, name, ownerLogin, settings) {
         return this.client.post("/public/api/projects/", {
@@ -155,10 +141,10 @@ export class ProjectsResource extends BaseResource {
     }
     /** Export a project archive to a local file. */
     async exportArchive(projectKey, filePath, options) {
-        const res = await this.postStream(`/public/api/projects/${encodeURIComponent(projectKey)}/export`, options ?? {});
+        const res = await this.client.postStream(`/public/api/projects/${encodeURIComponent(projectKey)}/export`, options ?? {});
         if (!res.body)
             throw new Error("projects.exportArchive response did not include a body");
-        await Bun.write(filePath, new Response(res.body), { createPath: false, });
+        await writeResponseToFile(filePath, res);
     }
     /** Upload a project archive and return its temporary import handle. */
     async prepareProjectImport(filePath) {

package/dist/src/resources/sql.js

@@ -101,7 +101,7 @@ function withSqlErrorContext(error) {
             catch {
                 body = `SQL query failed: ${detail}\n${error.body}`;
             }
-            throw new DataikuError(error.status, error.statusText, body, error.retry);
+            throw new DataikuError(error.status, error.statusText, body, error.retry, error.requestId, { target: error.trustedTarget, elapsedMs: error.trustedElapsedMs, });
         }
     }
     throw error;

package/dist/src/skill.d.ts

@@ -29,18 +29,27 @@ export declare function detectAgents(): DetectedAgent[];
  */
 export declare function findWorkspaceRoot(startDir: string): string;
 export type SkillStatus = "missing" | "stale" | "current";
+export interface InstalledSkillFile {
+    relativePath: string;
+    path: string;
+    status: SkillStatus;
+    changed: boolean;
+    expectedSha256: string;
+    actualSha256?: string;
+}
 export interface InstallResult {
     agent: string;
     path: string;
     via: DetectedAgent["via"];
-    /** Deterministic state of the destination file relative to the canonical skill content. */
+    /** Missing entrypoint, incomplete/different bundle, or entirely current bundle. */
     status: SkillStatus;
-    /** Whether an install run would (or did) write the destination file. */
+    /** Whether any bundled file would (or did) change. Unmanaged files are preserved. */
     changed: boolean;
-    /** SHA-256 hex of the canonical skill bytes this installation writes. */
+    /** SHA-256 of the canonical SKILL.md entrypoint, not the whole bundle. */
     expectedSha256: string;
-    /** SHA-256 hex of the destination file when it exists; absent when status is "missing". */
+    /** Hash of the existing entrypoint; absent when that file is missing. */
     actualSha256?: string;
+    files: InstalledSkillFile[];
 }
 export declare function planSkillInstalls(agents: DetectedAgent[], opts: {
     global: boolean;

package/dist/src/skill.js

@@ -2,17 +2,21 @@ import { createHash, } from "node:crypto";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
+import { fileURLToPath, } from "node:url";
 const SKILL_URLS = [
     new URL("../skills/dataiku-dss/SKILL.md", import.meta.url),
     new URL("../../skills/dataiku-dss/SKILL.md", import.meta.url),
 ];
-function skillContent() {
+function skillDirectory() {
     for (const skillUrl of SKILL_URLS) {
         if (fs.existsSync(skillUrl))
-            return fs.readFileSync(skillUrl, "utf-8");
+            return new URL("./", skillUrl);
     }
     throw new Error(`Bundled Dataiku skill not found. Checked: ${SKILL_URLS.map((url) => url.pathname).join(", ")}`);
 }
+function skillContent() {
+    return fs.readFileSync(new URL("SKILL.md", skillDirectory()), "utf-8");
+}
 export const AGENTS = {
     claude: {
         name: "Claude Code",
@@ -110,7 +114,27 @@ function skillState(target, expectedSha256) {
         actualSha256,
     };
 }
-export function planSkillInstalls(agents, opts) {
+/** Load the packaged tree, keeping portable relative paths in the report. */
+function skillFiles() {
+    const root = fileURLToPath(skillDirectory());
+    const files = new Map();
+    function visit(relativeDir) {
+        const dir = path.join(root, relativeDir);
+        for (const entry of fs.readdirSync(dir, { withFileTypes: true, }).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) {
+            const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
+            if (entry.isDirectory())
+                visit(relativePath);
+            else if (entry.isFile()) {
+                files.set(relativePath, fs.readFileSync(path.join(root, relativePath)));
+            }
+            else
+                throw new Error(`Unsupported bundled skill entry: ${relativePath}`);
+        }
+    }
+    visit("");
+    return files;
+}
+function skillInstallPlan(agents, opts, bundle) {
     const home = os.homedir();
     const results = [];
     for (const { id, def, via, } of agents) {
@@ -121,38 +145,65 @@ export function planSkillInstalls(agents, opts) {
                 : undefined;
         if (!dir)
             continue;
-        const target = path.join(dir, def.filename);
-        const expectedSha256 = sha256Hex(def.content());
-        const state = skillState(target, expectedSha256);
+        const files = [];
+        for (const [sourcePath, source,] of bundle) {
+            const relativePath = sourcePath === "SKILL.md" ? def.filename : sourcePath;
+            const target = path.join(dir, relativePath);
+            const expectedSha256 = sha256Hex(sourcePath === "SKILL.md" ? def.content() : source);
+            const state = skillState(target, expectedSha256);
+            files.push({
+                relativePath,
+                path: target,
+                ...state,
+                changed: state.status !== "current",
+                expectedSha256,
+            });
+        }
+        const entrypoint = files.find((file) => file.relativePath === def.filename);
+        const changed = files.some((file) => file.changed);
         results.push({
             agent: id,
-            path: target,
+            path: entrypoint.path,
             via,
-            ...state,
-            changed: state.status !== "current",
-            expectedSha256,
+            status: entrypoint.status === "missing" ? "missing" : changed ? "stale" : "current",
+            changed,
+            expectedSha256: entrypoint.expectedSha256,
+            ...(entrypoint.actualSha256 !== undefined ? { actualSha256: entrypoint.actualSha256, } : {}),
+            files,
         });
     }
     return results;
 }
+export function planSkillInstalls(agents, opts) {
+    return skillInstallPlan(agents, opts, skillFiles());
+}
+function writeSkillFile(file, content) {
+    if (!file.changed)
+        return;
+    const dir = path.dirname(file.path);
+    fs.mkdirSync(dir, { recursive: true, });
+    const tmpPath = path.join(dir, `.${path.basename(file.path)}.tmp-${process.pid}-${Date.now().toString(36)}`);
+    try {
+        fs.writeFileSync(tmpPath, content);
+        fs.renameSync(tmpPath, file.path);
+    }
+    finally {
+        fs.rmSync(tmpPath, { force: true, });
+    }
+}
 export function installSkill(agents, opts) {
-    const results = planSkillInstalls(agents, opts);
+    const bundle = skillFiles();
+    const results = skillInstallPlan(agents, opts, bundle);
     for (const result of results) {
-        const def = AGENTS[result.agent];
-        if (!def || !result.changed)
+        if (!result.changed)
             continue;
-        const content = def.content();
-        const dir = path.dirname(result.path);
-        fs.mkdirSync(dir, { recursive: true, });
-        const tmpPath = path.join(dir, `.${path.basename(result.path)}.tmp-${process.pid}-${Date.now().toString(36)}`);
-        fs.writeFileSync(tmpPath, content, "utf-8");
-        try {
-            fs.renameSync(tmpPath, result.path);
-        }
-        catch (error) {
-            fs.rmSync(tmpPath, { force: true, });
-            throw error;
+        const def = agents.find((agent) => agent.id === result.agent).def;
+        // Publish referenced files first; never expose a new entrypoint with missing references.
+        for (const file of result.files) {
+            if (file.relativePath !== def.filename)
+                writeSkillFile(file, bundle.get(file.relativePath));
         }
+        writeSkillFile(result.files.find((file) => file.relativePath === def.filename), def.content());
     }
     return results;
 }

package/dist/src/utils/response-file.d.ts

@@ -0,0 +1,2 @@
+/** Stream with backpressure; publish only a complete download, preserving existing output on failure. */
+export declare function writeResponseToFile(path: string, response: Response, permissions?: "private" | "preserve"): Promise<number>;

package/dist/src/utils/response-file.js

@@ -0,0 +1,34 @@
+import { randomUUID, } from "node:crypto";
+import { createWriteStream, } from "node:fs";
+import { chmod, rename, rm, stat, } from "node:fs/promises";
+import { basename, dirname, join, } from "node:path";
+import { Readable, } from "node:stream";
+import { pipeline, } from "node:stream/promises";
+/** Stream with backpressure; publish only a complete download, preserving existing output on failure. */
+export async function writeResponseToFile(path, response, permissions = "private") {
+    let mode;
+    if (permissions === "preserve") {
+        try {
+            mode = (await stat(path)).mode & 0o777;
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+            mode = 0o666 & ~process.umask();
+        }
+    }
+    const input = Readable.fromWeb(response.body);
+    const temporaryPath = join(dirname(path), `.${basename(path)}.tmp-${randomUUID()}`);
+    const output = createWriteStream(temporaryPath, { flags: "wx", mode: 0o600, });
+    try {
+        await pipeline(input, output);
+        // Keep partial contents private; restore publication permissions only after completion.
+        if (mode !== undefined)
+            await chmod(temporaryPath, mode);
+        await rename(temporaryPath, path);
+        return output.bytesWritten;
+    }
+    finally {
+        await rm(temporaryPath, { force: true, });
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "dataiku-sdk",
-	"version": "3.1.0",
+	"version": "3.2.0",
 	"description": "Dataiku DSS SDK and CLI for programmatic access to DSS REST APIs",
 	"type": "module",
 	"workspaces": [
@@ -25,13 +25,15 @@
 		"build": "tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
 		"prepack": "bun scripts/sync-plugin-version.mjs --check && tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
 		"prepublishOnly": "bun scripts/sync-plugin-version.mjs --check && tsc && bun scripts/generate-build-metadata.mjs && tsc -p packages/types/tsconfig.json",
-		"check": "tsc --noEmit && bun scripts/sync-plugin-version.mjs --check",
+		"check": "tsc --noEmit && bun run check:live && bun scripts/sync-plugin-version.mjs --check",
 		"version": "bun scripts/sync-plugin-version.mjs && git add plugin.json",
-		"lint": "oxlint src/",
-		"lint:fix": "oxlint --fix src/",
+		"lint": "oxlint src/ scripts/live-suite.ts tests/live-*.ts",
+		"lint:fix": "oxlint --fix src/ scripts/live-suite.ts tests/live-*.ts",
 		"format": "dprint fmt",
 		"format:check": "dprint check",
 		"test": "bun test",
+		"test:live": "bun --no-env-file scripts/live-suite.ts",
+		"check:live": "tsc --noEmit -p tsconfig.live.json",
 		"test:tokens": "bun test tests/cli/agent-token-budget.test.ts",
 		"test:platform": "bun scripts/platform-smoke.mjs",
 		"test:integration": "cross-env RUN_DATAIKU_INTEGRATION=1 bun test tests/integration-playground.test.ts",

package/plugin.json

@@ -1,7 +1,7 @@
 {
 	"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
 	"name": "dataiku-sdk",
-	"version": "3.1.0",
+	"version": "3.2.0",
 	"description": "Dataiku DSS SDK and agent-oriented CLI for programmatic access to DSS REST APIs.",
 	"homepage": "https://github.com/clssck/dataiku-sdk",
 	"repository": "https://github.com/clssck/dataiku-sdk",