@ekairos/dataset 1.22.35-beta.development.0 → 1.22.35

package/dist/datasetFiles.d.ts

@@ -1,5 +1,9 @@
 export declare const DATASET_OUTPUT_FILE_NAME = "output.jsonl";
 export declare function getDatasetWorkdirBase(): string;
 export declare function getDatasetWorkstation(datasetId: string): string;
+export declare function getDatasetResourcesDir(datasetId: string): string;
+export declare function getDatasetScriptsDir(datasetId: string): string;
+export declare function getDatasetArtifactsDir(datasetId: string): string;
+export declare function getDatasetLogsDir(datasetId: string): string;
+export declare function getDatasetStandardDirs(datasetId: string): string[];
 export declare function getDatasetOutputPath(datasetId: string): string;
-//# sourceMappingURL=datasetFiles.d.ts.map

package/dist/datasetFiles.js

@@ -1,33 +1,35 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DATASET_OUTPUT_FILE_NAME = void 0;
-exports.getDatasetWorkdirBase = getDatasetWorkdirBase;
-exports.getDatasetWorkstation = getDatasetWorkstation;
-exports.getDatasetOutputPath = getDatasetOutputPath;
-exports.DATASET_OUTPUT_FILE_NAME = "output.jsonl";
-const DEFAULT_VERCEL_WORKDIR_BASE = "/vercel/sandbox/datasets";
-const DEFAULT_DAYTONA_WORKDIR_BASE = "/home/daytona/.ekairos/datasets";
-const DEFAULT_SPRITES_WORKDIR_BASE = "/workspace/.ekairos/datasets";
+export const DATASET_OUTPUT_FILE_NAME = "output.jsonl";
+const DATASET_WORKDIR_BASE = "/tmp/ekairos/dataset";
 function trimTrailingSlash(value) {
     return value.endsWith("/") ? value.slice(0, -1) : value;
 }
-function getDatasetWorkdirBase() {
-    const explicit = String(process.env.DATASET_SANDBOX_WORKDIR_BASE ?? "").trim();
-    if (explicit)
-        return trimTrailingSlash(explicit);
-    const provider = String(process.env.SANDBOX_PROVIDER ?? "").trim().toLowerCase();
-    if (provider === "daytona")
-        return DEFAULT_DAYTONA_WORKDIR_BASE;
-    if (provider === "vercel")
-        return DEFAULT_VERCEL_WORKDIR_BASE;
-    if (provider === "sprites")
-        return DEFAULT_SPRITES_WORKDIR_BASE;
-    return DEFAULT_VERCEL_WORKDIR_BASE;
-}
-function getDatasetWorkstation(datasetId) {
+export function getDatasetWorkdirBase() {
+    return trimTrailingSlash(DATASET_WORKDIR_BASE);
+}
+export function getDatasetWorkstation(datasetId) {
     return `${getDatasetWorkdirBase()}/${datasetId}`;
 }
-function getDatasetOutputPath(datasetId) {
-    return `${getDatasetWorkstation(datasetId)}/${exports.DATASET_OUTPUT_FILE_NAME}`;
+export function getDatasetResourcesDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/resources`;
+}
+export function getDatasetScriptsDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/scripts`;
+}
+export function getDatasetArtifactsDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/artifacts`;
+}
+export function getDatasetLogsDir(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/logs`;
+}
+export function getDatasetStandardDirs(datasetId) {
+    return [
+        getDatasetWorkstation(datasetId),
+        getDatasetResourcesDir(datasetId),
+        getDatasetScriptsDir(datasetId),
+        getDatasetArtifactsDir(datasetId),
+        getDatasetLogsDir(datasetId),
+    ];
+}
+export function getDatasetOutputPath(datasetId) {
+    return `${getDatasetWorkstation(datasetId)}/${DATASET_OUTPUT_FILE_NAME}`;
 }
-//# sourceMappingURL=datasetFiles.js.map

package/dist/domain.d.ts

@@ -1,2 +1 @@
-export { datasetDomain } from "./schema";
-//# sourceMappingURL=domain.d.ts.map
+export { datasetDomain } from "./schema.js";

package/dist/domain.js

@@ -1,6 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.datasetDomain = void 0;
-var schema_1 = require("./schema");
-Object.defineProperty(exports, "datasetDomain", { enumerable: true, get: function () { return schema_1.datasetDomain; } });
-//# sourceMappingURL=domain.js.map
+export { datasetDomain } from "./schema.js";

package/dist/executeCommand.tool.d.ts

@@ -1,35 +1,7 @@
 interface ExecuteCommandToolParams {
     datasetId: string;
     sandboxId: string;
-    env?: any;
+    runtime: any;
 }
-export declare function createExecuteCommandTool({ datasetId, sandboxId, env }: ExecuteCommandToolParams): import("ai").Tool<{
-    pythonCode: string;
-    scriptName: string;
-}, {
-    success: boolean;
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    scriptPath: string;
-    error: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    message?: undefined;
-} | {
-    success: boolean;
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    scriptPath: string;
-    message: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    error?: undefined;
-}>;
+export declare function createExecuteCommandTool({ datasetId, sandboxId, runtime }: ExecuteCommandToolParams): any;
 export {};
-//# sourceMappingURL=executeCommand.tool.d.ts.map

package/dist/executeCommand.tool.js

@@ -1,49 +1,164 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createExecuteCommandTool = createExecuteCommandTool;
-const ai_1 = require("ai");
-const zod_1 = require("zod");
-const steps_1 = require("./sandbox/steps");
-const datasetFiles_1 = require("./datasetFiles");
-// To keep responses predictable for big data scenarios, we cap stdout/stderr.
-// The tool's return payload exposes stdout (capped) plus the on-disk script path.
+import { defineAction } from "@ekairos/events";
+import { z } from "zod";
+import { materializeContextResourcesStep } from "./contextResources.js";
+import { getDatasetScriptsDir, getDatasetStandardDirs } from "./datasetFiles.js";
+import { getContextExecutionWorkspaceDirs } from "./contextWorkspace.js";
+import { runDatasetSandboxCommandStep, writeDatasetSandboxTextFilesStep, } from "./sandbox/steps.js";
 const MAX_STDOUT_CHARS = 20000;
 const MAX_STDERR_CHARS = 5000;
-function createExecuteCommandTool({ datasetId, sandboxId, env }) {
-    return (0, ai_1.tool)({
-        description: "Execute Python scripts in the sandbox. Always saves script to a file before executing. The tool's output is EXACTLY the script's stdout and includes the script file path for traceability. CRITICAL: Print concise, human-readable summaries only; do NOT print raw large data. For big results, write artifacts to files in the workstation and print their file paths. Always include progress/result prints (e.g., 'Processing file X...', 'Found Y records', 'Generated output.csv').",
-        inputSchema: zod_1.z.object({
-            pythonCode: zod_1.z.string().describe("Python code to execute. Saved to a file before running. MANDATORY: Use print() to report progress and final results. Keep prints concise; avoid dumping rows/JSON. For large outputs, write to files in the workstation directory and print only file paths and brief summaries."),
-            scriptName: zod_1.z.string().describe("Name for the script file in snake_case (e.g., 'inspect_file', 'parse_csv', 'generate_dataset'). A UUID will be appended automatically."),
-        }),
-        execute: async ({ pythonCode, scriptName }) => {
-            const uuid = `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
-            const workstation = (0, datasetFiles_1.getDatasetWorkstation)(datasetId);
-            const scriptFile = `${workstation}/${scriptName}-${uuid}.py`;
+function normalizeScriptName(scriptName) {
+    const normalized = String(scriptName ?? "")
+        .trim()
+        .replace(/[^a-zA-Z0-9_.-]/g, "_")
+        .replace(/_+/g, "_")
+        .slice(0, 80);
+    return normalized || "script";
+}
+function stableScriptHash(value) {
+    let hash = 2166136261;
+    for (let index = 0; index < value.length; index++) {
+        hash ^= value.charCodeAt(index);
+        hash = Math.imul(hash, 16777619);
+    }
+    return (hash >>> 0).toString(36);
+}
+const executeCommandInputSchema = z.object({
+    commandDescription: z
+        .string()
+        .min(1)
+        .describe("Required pre-execution description of the command. Describe the inputs/resources it will use, the operation it will perform, the expected output, and why a command is the right tool instead of direct completion. Invalid descriptions include rereading resources whose descriptor/preview already contains the needed evidence, merely formatting JSON, constructing the final object, writing output.jsonl, or making completion easier."),
+    pythonCode: z
+        .string()
+        .describe("Python code to execute. Saved to a file before running. MANDATORY: Use print() to report progress and final results. Keep prints concise; avoid dumping rows/JSON. If context resources are materialized, read os.environ['EKAIROS_CONTEXT_RESOURCES_MANIFEST'] to discover files and metadata. Do not install packages, download dependencies, use pip/npm/apt/curl/wget, or access the network. For large outputs, write to files in the workstation directory and print only file paths and brief summaries."),
+    scriptName: z
+        .string()
+        .describe("Name for the script file in snake_case (e.g., 'inspect_file', 'parse_csv', 'generate_dataset'). A deterministic suffix will be appended automatically."),
+    resourceKeys: z
+        .array(z.string())
+        .optional()
+        .describe("Optional context resource keys to materialize before running the script. Omit to materialize every context resource."),
+});
+const materializedResourceSchema = z.object({
+    key: z.string(),
+    type: z.string(),
+    status: z.string(),
+    dir: z.string(),
+    files: z.array(z.object({
+        path: z.string(),
+        role: z.string(),
+        mediaType: z.string().optional(),
+    })),
+    reason: z.string().optional(),
+});
+const executeCommandOutputSchema = z
+    .object({
+    success: z.boolean(),
+    fatal: z.boolean().optional(),
+    status: z.string().optional(),
+    exitCode: z.number().optional(),
+    stdout: z.string(),
+    stderr: z.string(),
+    scriptPath: z.string(),
+    message: z.string().optional(),
+    error: z.string().optional(),
+    resourcesDir: z.string().optional(),
+    resourcesManifestPath: z.string().optional(),
+    materializedResources: z.array(materializedResourceSchema).optional(),
+    stdoutTruncated: z.boolean(),
+    stderrTruncated: z.boolean(),
+    stdoutOriginalLength: z.number(),
+    stderrOriginalLength: z.number(),
+})
+    .passthrough();
+export function createExecuteCommandTool({ datasetId, sandboxId, runtime }) {
+    return defineAction({
+        description: "Execute Python scripts in the sandbox only when command execution is necessary to inspect, parse, aggregate, join, or compute over context resources that are not sufficiently represented in the visible context, resource descriptors, or previews. This is a high-cost computation tool, not a completion tool. Do not use it merely to reread resources whose descriptor/preview already contains the needed evidence, format JSON, build the final object, write output.jsonl, or make completion easier when completeObject or replaceRows can return the result directly. Before the script runs, requested context resources are materialized into /tmp/ekairos/contexts/{contextId}/resources and a manifest.json is written there. The Python process receives EKAIROS_CONTEXT_RESOURCES_DIR and EKAIROS_CONTEXT_RESOURCES_MANIFEST environment variables when resources are available; manifest entries expose files as resources[].files[].path. Do not install packages, download dependencies, use pip/npm/apt/curl/wget, or access the network; use only the available runtime and standard library unless a dependency is already present. Print concise progress and results only; do not dump large data.",
+        input: executeCommandInputSchema,
+        output: executeCommandOutputSchema,
+        execute: async ({ input, context, contextId, executionId, }) => {
+            const { commandDescription, pythonCode, resourceKeys, scriptName } = input;
+            const normalizedScriptName = normalizeScriptName(scriptName);
+            const scriptHash = stableScriptHash(`${normalizedScriptName}\0${pythonCode}`);
+            const scriptsDir = contextId && executionId
+                ? getContextExecutionWorkspaceDirs({ contextId, executionId }).scriptsDir
+                : getDatasetScriptsDir(datasetId);
+            const scriptFile = `${scriptsDir}/${normalizedScriptName}-${scriptHash}.py`;
+            let resourcesManifest = null;
             console.log(`[Dataset ${datasetId}] ========================================`);
-            console.log(`[Dataset ${datasetId}] Tool: executeCommand`);
-            console.log(`[Dataset ${datasetId}] Script: ${scriptName}`);
+            console.log(`[Dataset ${datasetId}] Action: executeCommand`);
+            console.log(`[Dataset ${datasetId}] Description: ${commandDescription}`);
+            console.log(`[Dataset ${datasetId}] Script: ${normalizedScriptName}`);
             console.log(`[Dataset ${datasetId}] File: ${scriptFile}`);
             console.log(`[Dataset ${datasetId}] Code length: ${pythonCode.length} chars`);
             console.log(`[Dataset ${datasetId}] ========================================`);
             try {
-                await (0, steps_1.writeDatasetSandboxFilesStep)({
-                    env,
+                if (contextId && Array.isArray(context?.resources) && context.resources.length > 0) {
+                    resourcesManifest = await materializeContextResourcesStep({
+                        runtime,
+                        sandboxId,
+                        contextId,
+                        resources: context.resources,
+                        resourceKeys,
+                    });
+                    console.log(`[Dataset ${datasetId}] Resources manifest: ${resourcesManifest.manifestPath}`);
+                }
+                await runDatasetSandboxCommandStep({
+                    runtime,
+                    sandboxId,
+                    cmd: "mkdir",
+                    args: ["-p", ...getDatasetStandardDirs(datasetId), scriptsDir],
+                });
+                await writeDatasetSandboxTextFilesStep({
+                    runtime,
+                    sandboxId,
+                    files: [{ path: scriptFile, content: pythonCode }],
+                });
+                const written = await runDatasetSandboxCommandStep({
+                    runtime,
                     sandboxId,
-                    files: [
-                        {
-                            path: scriptFile,
-                            contentBase64: Buffer.from(pythonCode, "utf-8").toString("base64"),
-                        },
-                    ],
+                    cmd: "test",
+                    args: ["-f", scriptFile],
                 });
+                if (written.exitCode !== 0) {
+                    const error = `Script write verification failed: ${scriptFile}`;
+                    console.error(`[Dataset ${datasetId}] ${error}`);
+                    console.error(`[Dataset ${datasetId}] ========================================`);
+                    return {
+                        success: false,
+                        fatal: true,
+                        status: "script_write_failed",
+                        error,
+                        stdout: written.stdout || "",
+                        stderr: written.stderr || "",
+                        exitCode: written.exitCode,
+                        scriptPath: scriptFile,
+                        resourcesDir: resourcesManifest?.resourcesDir,
+                        resourcesManifestPath: resourcesManifest?.manifestPath,
+                        materializedResources: resourcesManifest?.resources,
+                        stdoutTruncated: false,
+                        stderrTruncated: false,
+                        stdoutOriginalLength: 0,
+                        stderrOriginalLength: 0,
+                    };
+                }
+                const pythonArgs = resourcesManifest
+                    ? [
+                        "-c",
+                        [
+                            "import os, runpy",
+                            `os.environ["EKAIROS_CONTEXT_RESOURCES_DIR"] = ${JSON.stringify(resourcesManifest.resourcesDir)}`,
+                            `os.environ["EKAIROS_CONTEXT_RESOURCES_MANIFEST"] = ${JSON.stringify(resourcesManifest.manifestPath)}`,
+                            `runpy.run_path(${JSON.stringify(scriptFile)}, run_name="__main__")`,
+                        ].join("; "),
+                    ]
+                    : [scriptFile];
                 console.log(`[Dataset ${datasetId}] Script written to: ${scriptFile}`);
-                console.log(`[Dataset ${datasetId}] Executing: python ${scriptFile}`);
-                const result = await (0, steps_1.runDatasetSandboxCommandStep)({
-                    env,
+                console.log(`[Dataset ${datasetId}] Executing: python ${resourcesManifest ? "<with context resources env>" : scriptFile}`);
+                const result = await runDatasetSandboxCommandStep({
+                    runtime,
                     sandboxId,
                     cmd: "python",
-                    args: [scriptFile],
+                    args: pythonArgs,
                 });
                 const stdout = result.stdout || "";
                 const stderr = result.stderr || "";
@@ -53,7 +168,7 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
                 const stdoutCapped = isStdoutTruncated ? stdout.slice(0, MAX_STDOUT_CHARS) : stdout;
                 const stderrCapped = isStderrTruncated ? stderr.slice(0, MAX_STDERR_CHARS) : stderr;
                 if (exitCode !== 0) {
-                    console.error(`[Dataset ${datasetId}] ❌ Command failed with exit code ${exitCode}`);
+                    console.error(`[Dataset ${datasetId}] Command failed with exit code ${exitCode}`);
                     console.error(`[Dataset ${datasetId}] Stderr:`, stderrCapped.substring(0, 500));
                     console.error(`[Dataset ${datasetId}] ========================================`);
                     return {
@@ -63,6 +178,9 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
                         stderr: stderrCapped,
                         scriptPath: scriptFile,
                         error: `Command failed with exit code ${exitCode}`,
+                        resourcesDir: resourcesManifest?.resourcesDir,
+                        resourcesManifestPath: resourcesManifest?.manifestPath,
+                        materializedResources: resourcesManifest?.resources,
                         stdoutTruncated: isStdoutTruncated,
                         stderrTruncated: isStderrTruncated,
                         stdoutOriginalLength: stdout.length,
@@ -70,7 +188,7 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
                     };
                 }
                 if (stderr && (stderr.includes("Traceback") || stderr.toLowerCase().includes("error"))) {
-                    console.error(`[Dataset ${datasetId}] ❌ Python error detected`);
+                    console.error(`[Dataset ${datasetId}] Python error detected`);
                     console.error(`[Dataset ${datasetId}] Stderr:`, stderrCapped.substring(0, 500));
                     console.error(`[Dataset ${datasetId}] ========================================`);
                     return {
@@ -80,17 +198,20 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
                         stderr: stderrCapped,
                         scriptPath: scriptFile,
                         error: "Python error detected in stderr",
+                        resourcesDir: resourcesManifest?.resourcesDir,
+                        resourcesManifestPath: resourcesManifest?.manifestPath,
+                        materializedResources: resourcesManifest?.resources,
                         stdoutTruncated: isStdoutTruncated,
                         stderrTruncated: isStderrTruncated,
                         stdoutOriginalLength: stdout.length,
                         stderrOriginalLength: stderr.length,
                     };
                 }
-                console.log(`[Dataset ${datasetId}] ✅ Command executed successfully`);
+                console.log(`[Dataset ${datasetId}] Command executed successfully`);
                 if (stdout) {
                     console.log(`[Dataset ${datasetId}] Output length: ${stdout.length} chars`);
                     if (isStdoutTruncated) {
-                        console.log(`[Dataset ${datasetId}] ⚠️  Stdout truncated to ${MAX_STDOUT_CHARS} chars`);
+                        console.log(`[Dataset ${datasetId}] Stdout truncated to ${MAX_STDOUT_CHARS} chars`);
                     }
                 }
                 console.log(`[Dataset ${datasetId}] ========================================`);
@@ -101,6 +222,9 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
                     stderr: stderrCapped,
                     scriptPath: scriptFile,
                     message: "Command executed successfully",
+                    resourcesDir: resourcesManifest?.resourcesDir,
+                    resourcesManifestPath: resourcesManifest?.manifestPath,
+                    materializedResources: resourcesManifest?.resources,
                     stdoutTruncated: isStdoutTruncated,
                     stderrTruncated: isStderrTruncated,
                     stdoutOriginalLength: stdout.length,
@@ -118,6 +242,9 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
                     stderr: "",
                     exitCode: -1,
                     scriptPath: scriptFile,
+                    resourcesDir: resourcesManifest?.resourcesDir,
+                    resourcesManifestPath: resourcesManifest?.manifestPath,
+                    materializedResources: resourcesManifest?.resources,
                     stdoutTruncated: false,
                     stderrTruncated: false,
                     stdoutOriginalLength: 0,
@@ -127,4 +254,3 @@ function createExecuteCommandTool({ datasetId, sandboxId, env }) {
         },
     });
 }
-//# sourceMappingURL=executeCommand.tool.js.map

package/dist/file/file-dataset.agent.d.ts

@@ -1,56 +1,7 @@
-import { createContext, type ContextReactor } from "@ekairos/events";
-import { FilePreviewContext } from "./filepreview";
-export type FileParseStoryContext = {
-    datasetId: string;
-    fileId: string;
-    instructions: string;
-    sandboxConfig: {
-        filePath: string;
-    };
-    analysis: any[];
-    schema: any | null;
-    plan: any | null;
-    executionResult: any | null;
-    errors: string[];
-    iterationCount: number;
-    filePreview?: FilePreviewContext;
-};
-export type FileParseStoryParams = {
-    fileId: string;
-    instructions?: string;
-    sandboxId?: string;
-    datasetId?: string;
-    model?: string;
-    reactor?: ContextReactor<any, any>;
-};
-export type FileParseStoryBuilder<Env extends {
-    orgId: string;
-}> = {
-    datasetId: string;
-    story: ReturnType<ReturnType<typeof createContext<Env>>["context"]> extends any ? any : any;
-};
-export type DatasetResult = {
-    id: string;
-    status?: string;
-    title?: string;
-    schema?: any;
-    analysis?: any;
-    calculatedTotalRows?: number;
-    actualGeneratedRowCount?: number;
-    createdAt?: number;
-    updatedAt?: number;
-};
-/**
- * Factory (DX-first):
- *
- * Usage:
- *   const { datasetId } = await createFileParseStory(fileId, { instructions }).parse(env)
- *
- * - No `db` is accepted/stored (workflow-safe).
- * - All I/O happens in `"use step"` functions via Ekairos runtime (`getContextRuntime(env).db`).
- * - `parse()` is the entrypoint; it calls `story.react(...)` internally.
- */
-export declare function createFileParseStory<Env extends {
+import { type ContextReactor } from "@ekairos/events";
+import type { FileParseContext, FileParseRunOptions, SandboxState } from "./file-dataset.types.js";
+export type { DatasetResult, FileParseContext, FileParseContextBuilder, FileParseContextParams, FileParseRunOptions, SandboxState, } from "./file-dataset.types.js";
+export declare function createFileParseContext<Env extends {
     orgId: string;
 }>(fileId: string, opts?: {
     instructions?: string;
@@ -58,11 +9,23 @@ export declare function createFileParseStory<Env extends {
     datasetId?: string;
     model?: string;
     reactor?: ContextReactor<any, any>;
+    sandboxState?: SandboxState;
+    filePreview?: FileParseContext["filePreview"];
+    schema?: any | null;
+    filename?: string;
+    mediaType?: string;
 }): {
     datasetId: string;
-    parse(env?: Env, prompt?: string): Promise<{
+    parse(runtime: {
+        env: Env;
+    }, options?: FileParseRunOptions): Promise<{
         datasetId: string;
     }>;
-    story: any;
+    context: any;
 };
-//# sourceMappingURL=file-dataset.agent.d.ts.map
+export declare function registerFileParseContext<Env extends {
+    orgId: string;
+}>(opts?: {
+    model?: string;
+    reactor?: ContextReactor<any, any>;
+}): void;