@ekairos/dataset 1.22.56-beta.development.0 → 1.22.58-beta.development.0

package/dist/completeDataset.tool.d.ts

@@ -49,4 +49,9 @@ export declare function didCompleteDatasetSucceed(event: {
         parts?: any[];
     };
 }): boolean;
+export declare function getDatasetFatalFailure(event: {
+    content?: {
+        parts?: any[];
+    };
+}): string | null;
 export {};

package/dist/completeDataset.tool.js

@@ -1,6 +1,6 @@
 import { tool } from "ai";
 import { z } from "zod";
-import { readDatasetSandboxFileStep, runDatasetSandboxCommandStep } from "./sandbox/steps.js";
+import { readDatasetSandboxFileStep, readDatasetSandboxTextFileStep, runDatasetSandboxCommandStep } from "./sandbox/steps.js";
 import Ajv from "ajv";
 import { getDatasetOutputPath, } from "./datasetFiles.js";
 import { datasetGetByIdStep, datasetUpdateStatusStep, datasetUploadOutputFileStep } from "./dataset/steps.js";
@@ -113,9 +113,12 @@ export function createCompleteDatasetTool({ datasetId, sandboxId, runtime }) {
                     message: "Empty file content",
                 };
             }
-            const fileBuffer = Buffer.from(fileRead.contentBase64, "base64");
             console.log(`[Dataset ${datasetId}] Uploading file to InstantDB storage`);
-            const uploadResult = await datasetUploadOutputFileStep({ runtime, datasetId, fileBuffer });
+            const uploadResult = await datasetUploadOutputFileStep({
+                runtime,
+                datasetId,
+                contentBase64: fileRead.contentBase64,
+            });
             if (!uploadResult.ok) {
                 console.error(`[Dataset ${datasetId}] File upload failed: ${uploadResult.error}`);
                 return {
@@ -176,6 +179,31 @@ export function didCompleteDatasetSucceed(event) {
         return false;
     });
 }
+export function getDatasetFatalFailure(event) {
+    const parts = Array.isArray(event?.content?.parts) ? event.content.parts : [];
+    for (const part of parts) {
+        let actionName;
+        let output;
+        if (part?.type === "action") {
+            actionName = part.content?.actionName;
+            output = part.content?.output;
+        }
+        else if (typeof part?.type === "string" && part.type.startsWith("tool-")) {
+            actionName = part.type.slice("tool-".length);
+            output = part.output ?? part.result;
+        }
+        if (!output || output.fatal !== true) {
+            continue;
+        }
+        const message = typeof output.error === "string" && output.error.trim()
+            ? output.error.trim()
+            : typeof output.message === "string" && output.message.trim()
+                ? output.message.trim()
+                : "Dataset action failed fatally";
+        return actionName ? `${actionName}: ${message}` : message;
+    }
+    return null;
+}
 async function ensureFileExists(runtime, sandboxId, path) {
     const result = await runDatasetSandboxCommandStep({
         runtime,
@@ -192,8 +220,8 @@ async function validateJsonlRows({ runtime, sandboxId, outputPath, validator, da
     let validRowCount = 0;
     let rowRecordCount = 0;
     console.log(`[Dataset ${datasetId}] Reading and validating JSONL file from sandbox`);
-    const fileRead = await readDatasetSandboxFileStep({ runtime, sandboxId, path: outputPath });
-    if (!fileRead.contentBase64) {
+    const fileRead = await readDatasetSandboxTextFileStep({ runtime, sandboxId, path: outputPath });
+    if (!fileRead.content) {
         console.log(`[Dataset ${datasetId}] Empty output file`);
         return {
             success: false,
@@ -205,8 +233,7 @@ async function validateJsonlRows({ runtime, sandboxId, outputPath, validator, da
             message: "output.jsonl is empty",
         };
     }
-    const fileContent = Buffer.from(fileRead.contentBase64, "base64").toString();
-    const lines = fileContent.split("\n");
+    const lines = fileRead.content.split("\n");
     console.log(`[Dataset ${datasetId}] Validating ${lines.length} lines`);
     for (let index = 0; index < lines.length; index++) {
         const line = lines[index];

package/dist/dataset/steps.d.ts

@@ -18,7 +18,7 @@ export declare function datasetUpdateSchemaStep(params: {
 export declare function datasetUploadOutputFileStep(params: {
     runtime: any;
     datasetId: string;
-    fileBuffer: Buffer;
+    contentBase64: string;
 }): Promise<import("../service.js").ServiceResult<{
     fileId: string;
     storagePath: string;

package/dist/dataset/steps.js

@@ -60,7 +60,7 @@ export async function datasetUploadOutputFileStep(params) {
     const service = new DatasetService(db);
     return await service.uploadDatasetOutputFile({
         datasetId: params.datasetId,
-        fileBuffer: params.fileBuffer,
+        fileBuffer: Buffer.from(params.contentBase64, "base64"),
     });
 }
 export async function datasetUpdateStatusStep(params) {

package/dist/dataset.js

@@ -1,6 +1,6 @@
-import { id as newId } from "@instantdb/admin";
 import { buildObjectOutputInstructions } from "./builder/instructions.js";
-import { getDatasetAgentMaterializers } from "./builder/agentMaterializers.js";
+import { createDatasetId } from "./id.js";
+import { materializeDerivedDataset, materializeSingleFileLikeSource, } from "./builder/materialize.js";
 import { materializeQuerySource } from "./builder/materializeQuery.js";
 import { finalizeBuildResult } from "./builder/persistence.js";
 export function dataset(runtime, options = {}) {
@@ -131,20 +131,20 @@ export function dataset(runtime, options = {}) {
                 if (!effectiveState.reactor) {
                     throw new Error("dataset_reactor_required");
                 }
-                await getDatasetAgentMaterializers().materializeSingleFileLikeSource(effectiveState, onlySource, targetDatasetId);
+                await materializeSingleFileLikeSource(effectiveState, onlySource, targetDatasetId);
                 return finalizeOutputResult(await finalizeBuildResult(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
             }
             if (!effectiveState.reactor) {
                 throw new Error("dataset_reactor_required");
             }
-            await getDatasetAgentMaterializers().materializeDerivedDataset(effectiveState, targetDatasetId);
+            await materializeDerivedDataset(effectiveState, targetDatasetId);
             return finalizeOutputResult(await finalizeBuildResult(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
         },
     };
     return api;
 }
 function normalizeDatasetId(datasetId) {
-    const normalized = String(datasetId ?? newId()).trim();
+    const normalized = String(datasetId ?? createDatasetId()).trim();
     if (!normalized) {
         throw new Error("dataset_id_required");
     }

package/dist/executeCommand.tool.d.ts

@@ -7,6 +7,20 @@ export declare function createExecuteCommandTool({ datasetId, sandboxId, runtime
     pythonCode: string;
     scriptName: string;
 }, {
+    success: boolean;
+    fatal: boolean;
+    status: string;
+    error: string;
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+    scriptPath: string;
+    stdoutTruncated: boolean;
+    stderrTruncated: boolean;
+    stdoutOriginalLength: number;
+    stderrOriginalLength: number;
+    message?: undefined;
+} | {
     success: boolean;
     exitCode: number;
     stdout: string;
@@ -17,6 +31,8 @@ export declare function createExecuteCommandTool({ datasetId, sandboxId, runtime
     stderrTruncated: boolean;
     stdoutOriginalLength: number;
     stderrOriginalLength: number;
+    fatal?: undefined;
+    status?: undefined;
     message?: undefined;
 } | {
     success: boolean;
@@ -29,6 +45,8 @@ export declare function createExecuteCommandTool({ datasetId, sandboxId, runtime
     stderrTruncated: boolean;
     stdoutOriginalLength: number;
     stderrOriginalLength: number;
+    fatal?: undefined;
+    status?: undefined;
     error?: undefined;
 }>;
 export {};

package/dist/executeCommand.tool.js

@@ -1,39 +1,81 @@
 import { tool } from "ai";
 import { z } from "zod";
-import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "./sandbox/steps.js";
+import { runDatasetSandboxCommandStep, writeDatasetSandboxTextFilesStep } from "./sandbox/steps.js";
 import { getDatasetWorkstation } from "./datasetFiles.js";
 // To keep responses predictable for big data scenarios, we cap stdout/stderr.
 // The tool's return payload exposes stdout (capped) plus the on-disk script path.
 const MAX_STDOUT_CHARS = 20000;
 const MAX_STDERR_CHARS = 5000;
+function normalizeScriptName(scriptName) {
+    const normalized = String(scriptName ?? "")
+        .trim()
+        .replace(/[^a-zA-Z0-9_.-]/g, "_")
+        .replace(/_+/g, "_")
+        .slice(0, 80);
+    return normalized || "script";
+}
+function stableScriptHash(value) {
+    let hash = 2166136261;
+    for (let index = 0; index < value.length; index++) {
+        hash ^= value.charCodeAt(index);
+        hash = Math.imul(hash, 16777619);
+    }
+    return (hash >>> 0).toString(36);
+}
 export function createExecuteCommandTool({ datasetId, sandboxId, runtime }) {
     return tool({
         description: "Execute Python scripts in the sandbox. Always saves script to a file before executing. The tool's output is EXACTLY the script's stdout and includes the script file path for traceability. CRITICAL: Print concise, human-readable summaries only; do NOT print raw large data. For big results, write artifacts to files in the workstation and print their file paths. Always include progress/result prints (e.g., 'Processing file X...', 'Found Y records', 'Generated output.csv').",
         inputSchema: z.object({
             pythonCode: z.string().describe("Python code to execute. Saved to a file before running. MANDATORY: Use print() to report progress and final results. Keep prints concise; avoid dumping rows/JSON. For large outputs, write to files in the workstation directory and print only file paths and brief summaries."),
-            scriptName: z.string().describe("Name for the script file in snake_case (e.g., 'inspect_file', 'parse_csv', 'generate_dataset'). A UUID will be appended automatically."),
+            scriptName: z.string().describe("Name for the script file in snake_case (e.g., 'inspect_file', 'parse_csv', 'generate_dataset'). A deterministic suffix will be appended automatically."),
         }),
         execute: async ({ pythonCode, scriptName }) => {
-            const uuid = `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
             const workstation = getDatasetWorkstation(datasetId);
-            const scriptFile = `${workstation}/${scriptName}-${uuid}.py`;
+            const normalizedScriptName = normalizeScriptName(scriptName);
+            const scriptHash = stableScriptHash(`${normalizedScriptName}\0${pythonCode}`);
+            const scriptFile = `${workstation}/${normalizedScriptName}-${scriptHash}.py`;
             console.log(`[Dataset ${datasetId}] ========================================`);
             console.log(`[Dataset ${datasetId}] Tool: executeCommand`);
-            console.log(`[Dataset ${datasetId}] Script: ${scriptName}`);
+            console.log(`[Dataset ${datasetId}] Script: ${normalizedScriptName}`);
             console.log(`[Dataset ${datasetId}] File: ${scriptFile}`);
             console.log(`[Dataset ${datasetId}] Code length: ${pythonCode.length} chars`);
             console.log(`[Dataset ${datasetId}] ========================================`);
             try {
-                await writeDatasetSandboxFilesStep({
+                await writeDatasetSandboxTextFilesStep({
                     runtime,
                     sandboxId,
                     files: [
                         {
                             path: scriptFile,
-                            contentBase64: Buffer.from(pythonCode, "utf-8").toString("base64"),
+                            content: pythonCode,
                         },
                     ],
                 });
+                const written = await runDatasetSandboxCommandStep({
+                    runtime,
+                    sandboxId,
+                    cmd: "test",
+                    args: ["-f", scriptFile],
+                });
+                if (written.exitCode !== 0) {
+                    const error = `Script write verification failed: ${scriptFile}`;
+                    console.error(`[Dataset ${datasetId}] ${error}`);
+                    console.error(`[Dataset ${datasetId}] ========================================`);
+                    return {
+                        success: false,
+                        fatal: true,
+                        status: "script_write_failed",
+                        error,
+                        stdout: written.stdout || "",
+                        stderr: written.stderr || "",
+                        exitCode: written.exitCode,
+                        scriptPath: scriptFile,
+                        stdoutTruncated: false,
+                        stderrTruncated: false,
+                        stdoutOriginalLength: 0,
+                        stderrOriginalLength: 0,
+                    };
+                }
                 console.log(`[Dataset ${datasetId}] Script written to: ${scriptFile}`);
                 console.log(`[Dataset ${datasetId}] Executing: python ${scriptFile}`);
                 const result = await runDatasetSandboxCommandStep({

package/dist/file/file-dataset.agent.js

@@ -1,11 +1,11 @@
 import { createContext, INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/events";
-import { id } from "@instantdb/admin";
 import { createClearDatasetTool } from "../clearDataset.tool.js";
-import { createCompleteDatasetTool, didCompleteDatasetSucceed, } from "../completeDataset.tool.js";
+import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
 import { datasetGetByIdStep } from "../dataset/steps.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
 import { createGenerateSchemaTool } from "./generateSchema.tool.js";
 import { buildFileDatasetPromptStep, generateFileParsePreviewStep, initializeFileParseSandboxStep, } from "./file-dataset.steps.js";
+import { createDatasetId } from "../id.js";
 async function awaitContextRun(run) {
     if (!run)
         return;
@@ -136,6 +136,10 @@ function createFileParseContextDefinition(params) {
         return actions;
     })
         .shouldContinue(({ reactionEvent }) => {
+        const fatalFailure = getDatasetFatalFailure(reactionEvent);
+        if (fatalFailure) {
+            throw new Error(fatalFailure);
+        }
         return !didCompleteDatasetSucceed(reactionEvent);
     });
     if (params.reactor) {
@@ -148,7 +152,7 @@ function createFileParseContextDefinition(params) {
     return { datasetId: fallbackDatasetId ?? "", context };
 }
 export function createFileParseContext(fileId, opts) {
-    const datasetId = opts?.datasetId ?? id();
+    const datasetId = opts?.datasetId ?? createDatasetId();
     const params = {
         fileId,
         instructions: opts?.instructions,
@@ -162,7 +166,7 @@ export function createFileParseContext(fileId, opts) {
         datasetId,
         async parse(runtime, options = {}) {
             const triggerEvent = {
-                id: id(),
+                id: createDatasetId(),
                 type: INPUT_TEXT_ITEM_TYPE,
                 channel: WEB_CHANNEL,
                 createdAt: new Date().toISOString(),

package/dist/id.d.ts

@@ -0,0 +1 @@
+export declare function createDatasetId(): string;

package/dist/id.js

@@ -0,0 +1,10 @@
+export function createDatasetId() {
+    const uuid = globalThis.crypto?.randomUUID?.();
+    if (typeof uuid === "string" && uuid.length > 0)
+        return uuid;
+    return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
+        const r = (Math.random() * 16) | 0;
+        const v = c === "x" ? r : (r & 0x3) | 0x8;
+        return v.toString(16);
+    });
+}

package/dist/index.d.ts

@@ -1,6 +1,7 @@
-import "./builder/materialize.js";
 export * from "./dataset.js";
 export * from "./domain.js";
 export * from "./materializeDataset.tool.js";
 export * from "./schema.js";
 export * from "./service.js";
+export { registerFileParseContext } from "./file/file-dataset.agent.js";
+export { registerTransformDatasetContext } from "./transform/index.js";

package/dist/index.js

@@ -1,6 +1,7 @@
-import "./builder/materialize.js";
 export * from "./dataset.js";
 export * from "./domain.js";
 export * from "./materializeDataset.tool.js";
 export * from "./schema.js";
 export * from "./service.js";
+export { registerFileParseContext } from "./file/file-dataset.agent.js";
+export { registerTransformDatasetContext } from "./transform/index.js";

package/dist/query/queryDomain.step.js

@@ -1,5 +1,5 @@
-import { id as newId } from "@instantdb/admin";
 import { DatasetService } from "../service.js";
+import { createDatasetId } from "../id.js";
 function normalizeRows(result) {
     if (!result || typeof result !== "object")
         return [];
@@ -56,7 +56,7 @@ export async function queryDomainStep(params) {
     "use step";
     const db = await getRuntimeDb(params.runtime);
     const service = new DatasetService(db);
-    const datasetId = params.datasetId ?? newId();
+    const datasetId = params.datasetId ?? createDatasetId();
     const queryResult = await db.query(params.query);
     const rows = normalizeRows(queryResult);
     const previewRows = rows.slice(0, 20);

package/dist/sandbox/steps.d.ts

@@ -27,6 +27,14 @@ export declare function writeDatasetSandboxFilesStep(params: {
         contentBase64: string;
     }>;
 }): Promise<void>;
+export declare function writeDatasetSandboxTextFilesStep(params: {
+    runtime: any;
+    sandboxId: DatasetSandboxId;
+    files: Array<{
+        path: string;
+        content: string;
+    }>;
+}): Promise<void>;
 export declare function readDatasetSandboxFileStep(params: {
     runtime: any;
     sandboxId: DatasetSandboxId;
@@ -34,6 +42,13 @@ export declare function readDatasetSandboxFileStep(params: {
 }): Promise<{
     contentBase64: string;
 }>;
+export declare function readDatasetSandboxTextFileStep(params: {
+    runtime: any;
+    sandboxId: DatasetSandboxId;
+    path: string;
+}): Promise<{
+    content: string;
+}>;
 export declare function stopDatasetSandboxStep(params: {
     runtime: any;
     sandboxId: DatasetSandboxId;

package/dist/sandbox/steps.js

@@ -117,6 +117,25 @@ export async function writeDatasetSandboxFilesStep(params) {
     if (!result.ok)
         throw new Error(result.error);
 }
+export async function writeDatasetSandboxTextFilesStep(params) {
+    "use step";
+    if (isLocalDatasetSandboxMode()) {
+        for (const file of params.files) {
+            await fs.mkdir(path.dirname(file.path), { recursive: true });
+            await fs.writeFile(file.path, file.content, "utf-8");
+        }
+        return;
+    }
+    const db = await getRuntimeDb(params.runtime);
+    const service = new SandboxService(db);
+    const files = params.files.map((file) => ({
+        path: file.path,
+        contentBase64: Buffer.from(file.content, "utf-8").toString("base64"),
+    }));
+    const result = await service.writeFiles(params.sandboxId, files);
+    if (!result.ok)
+        throw new Error(result.error);
+}
 export async function readDatasetSandboxFileStep(params) {
     "use step";
     if (isLocalDatasetSandboxMode()) {
@@ -130,6 +149,19 @@ export async function readDatasetSandboxFileStep(params) {
         throw new Error(result.error);
     return result.data;
 }
+export async function readDatasetSandboxTextFileStep(params) {
+    "use step";
+    if (isLocalDatasetSandboxMode()) {
+        const content = await fs.readFile(params.path, "utf-8");
+        return { content };
+    }
+    const db = await getRuntimeDb(params.runtime);
+    const service = new SandboxService(db);
+    const result = await service.readFile(params.sandboxId, params.path);
+    if (!result.ok)
+        throw new Error(result.error);
+    return { content: Buffer.from(result.data.contentBase64, "base64").toString("utf-8") };
+}
 export async function stopDatasetSandboxStep(params) {
     "use step";
     if (isLocalDatasetSandboxMode()) {

package/dist/service.d.ts

@@ -1,4 +1,4 @@
-import { InstantAdminDatabase } from "@instantdb/admin";
+import type { InstantAdminDatabase } from "@instantdb/admin";
 import { SchemaOf } from "@ekairos/domain";
 import { datasetDomain } from "./schema.js";
 export type ServiceResult<T = any> = {

package/dist/service.js

@@ -1,4 +1,4 @@
-import { id as newId } from "@instantdb/admin";
+import { createDatasetId } from "./id.js";
 export class DatasetService {
     constructor(db) {
         this.db = db;
@@ -27,9 +27,9 @@ export class DatasetService {
     }
     async createDataset(params) {
         try {
-            const datasetId = params.id ?? newId();
+            const datasetId = params.id ?? createDatasetId();
             const existing = await this.resolveDatasetEntityId(datasetId);
-            const entityId = existing.ok ? existing.data : newId();
+            const entityId = existing.ok ? existing.data : createDatasetId();
             const mutations = [];
             mutations.push(this.db.tx.dataset_datasets[entityId].update({
                 datasetId,
@@ -73,7 +73,7 @@ export class DatasetService {
                 return resolved;
             const mutations = [];
             for (const record of params.records) {
-                const recordId = newId();
+                const recordId = createDatasetId();
                 mutations.push(this.db.tx.dataset_records[recordId].update({
                     rowContent: record.rowContent,
                     order: record.order,

package/dist/transform/transform-dataset.agent.js

@@ -1,10 +1,10 @@
 import { createContext, INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/events";
-import { id } from "@instantdb/admin";
 import { createClearDatasetTool } from "../clearDataset.tool.js";
-import { createCompleteDatasetTool, didCompleteDatasetSucceed, } from "../completeDataset.tool.js";
+import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
 import { datasetUpdateSchemaStep } from "../dataset/steps.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
 import { buildTransformDatasetPromptStep, ensureTransformSourcesInSandboxStep, generateTransformSourcePreviewsStep, } from "./transform-dataset.steps.js";
+import { createDatasetId } from "../id.js";
 async function awaitContextRun(run) {
     if (!run)
         return;
@@ -130,6 +130,10 @@ function createTransformDatasetContextDefinition(params) {
         };
     })
         .shouldContinue(({ reactionEvent }) => {
+        const fatalFailure = getDatasetFatalFailure(reactionEvent);
+        if (fatalFailure) {
+            throw new Error(fatalFailure);
+        }
         return !didCompleteDatasetSucceed(reactionEvent);
     });
     if (params.reactor) {
@@ -142,7 +146,7 @@ function createTransformDatasetContextDefinition(params) {
     return { datasetId: fallbackDatasetId ?? "", context };
 }
 export function createTransformDatasetContext(params) {
-    const datasetId = params.datasetId ?? id();
+    const datasetId = params.datasetId ?? createDatasetId();
     const { context } = createTransformDatasetContextDefinition({
         sourceDatasetIds: params.sourceDatasetIds,
         outputSchema: params.outputSchema,
@@ -159,7 +163,7 @@ export function createTransformDatasetContext(params) {
                 ? "the source dataset"
                 : `${params.sourceDatasetIds.length} source datasets`;
             const triggerEvent = {
-                id: id(),
+                id: createDatasetId(),
                 type: INPUT_TEXT_ITEM_TYPE,
                 channel: WEB_CHANNEL,
                 createdAt: new Date().toISOString(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.56-beta.development.0",
+  "version": "1.22.58-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.56-beta.development.0",
-    "@ekairos/events": "^1.22.56-beta.development.0",
-    "@ekairos/sandbox": "^1.22.56-beta.development.0",
+    "@ekairos/domain": "^1.22.58-beta.development.0",
+    "@ekairos/events": "^1.22.58-beta.development.0",
+    "@ekairos/sandbox": "^1.22.58-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",