@ekairos/dataset 1.22.81-beta.development.0 → 1.22.83-beta.development.0

package/dist/builder/materialize.d.ts

@@ -11,6 +11,8 @@ type PreparedFileDatasetContext = {
     sandboxState: SandboxState;
     filePreview?: FilePreviewContext;
     schema?: DatasetSchemaInput | null;
+    filename?: string;
+    mediaType?: string;
 };
 type PreparedTransformDatasetContext = {
     kind: "transform";

package/dist/builder/materialize.js

@@ -1,6 +1,5 @@
 import { createFileParseContext } from "../file/file-dataset.agent.js";
 import { readInstantFileStep } from "../file/steps.js";
-import { generateFileParsePreviewStep, initializeFileParseSandboxStep, } from "../file/file-dataset.steps.js";
 import { createTransformDatasetContext } from "../transform/transform-dataset.agent.js";
 import { ensureTransformSourcesInSandboxStep, generateTransformSourcePreviewsStep, } from "../transform/transform-dataset.steps.js";
 import { datasetGetByIdStep, datasetInferAndUpdateSchemaStep, datasetPreviewRowsStep, datasetReadOneStep, } from "../dataset/steps.js";
@@ -266,27 +265,16 @@ export async function prepareDatasetSourcesStep(params) {
         const fileId = params.source.kind === "file"
             ? params.source.fileId
             : await uploadInlineTextSource(params.runtime, params.datasetId, params.source);
-        const initialized = await initializeFileParseSandboxStep({
-            runtime: params.runtime,
-            sandboxId: params.sandboxId,
-            datasetId: params.datasetId,
-            fileId,
-            state: { initialized: false, filePath: "" },
-        });
-        const filePreview = await generateFileParsePreviewStep({
-            runtime: params.runtime,
-            sandboxId: params.sandboxId,
-            sandboxFilePath: initialized.filePath,
-            datasetId: params.datasetId,
-        });
         return {
             kind: "file",
             datasetId: params.datasetId,
             sandboxId: params.sandboxId,
             fileId,
-            sandboxState: initialized.state,
-            filePreview,
+            sandboxState: { initialized: false, filePath: "" },
+            filePreview: undefined,
             schema: params.schema ?? null,
+            filename: params.source.kind === "file" ? params.source.filename : params.source.name,
+            mediaType: params.source.kind === "file" ? params.source.mediaType : params.source.mimeType,
         };
     }
     const initialized = await ensureTransformSourcesInSandboxStep({
@@ -393,7 +381,13 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
         instructions: state.instructions,
         sources: [
             source.kind === "file"
-                ? { kind: "file", fileId: source.fileId, description: source.description }
+                ? {
+                    kind: "file",
+                    fileId: source.fileId,
+                    description: source.description,
+                    filename: source.filename,
+                    mediaType: source.mediaType,
+                }
                 : {
                     kind: "text",
                     mimeType: source.mimeType,
@@ -428,6 +422,8 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
         sandboxState: context.sandboxState,
         filePreview: context.filePreview,
         schema: context.schema,
+        filename: context.filename,
+        mediaType: context.mediaType,
     });
     await parseContext.parse(state.runtime, {
         durable: await resolveDatasetAgentDurable(state.durable),
@@ -440,6 +436,8 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
             sandboxState: context.sandboxState,
             filePreview: context.filePreview,
             schema: context.schema,
+            filename: context.filename,
+            mediaType: context.mediaType,
         },
     });
     return targetDatasetId;

package/dist/builder/types.d.ts

@@ -12,6 +12,8 @@ export type DatasetQuerySourceInput<D extends DomainSchemaResult = DomainSchemaR
 export type DatasetFileSourceInput = {
     fileId: string;
     description?: string;
+    filename?: string;
+    mediaType?: string;
 };
 export type DatasetTextSourceInput = {
     text: string;

package/dist/completeDataset.steps.d.ts

@@ -3,8 +3,9 @@ export interface PersistDatasetStepParams {
     sandboxId: string;
     runtime: any;
     summary?: string;
+    outputPath?: string;
 }
-export declare function persistDatasetStep({ runtime, datasetId, sandboxId, summary }: PersistDatasetStepParams): Promise<{
+export declare function persistDatasetStep({ runtime, datasetId, sandboxId, summary, outputPath }: PersistDatasetStepParams): Promise<{
     success: boolean;
     validation?: RowValidationEntry[];
     validationTruncated?: number;
@@ -23,18 +24,18 @@ export declare function persistDatasetStep({ runtime, datasetId, sandboxId, summ
     validation: RowValidationEntry[] | undefined;
     error: string;
     message: string;
-    fileId?: undefined;
-    storagePath?: undefined;
+    records?: undefined;
+    summary?: undefined;
 } | {
     success: boolean;
     status: string;
-    validRows: number;
-    rowRecordCount: number;
-    fileId: string;
-    storagePath: string;
-    message: string;
+    records: number;
+    summary: string;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
     validation?: undefined;
     error?: undefined;
+    message?: undefined;
 }>;
 type RowValidationEntry = {
     index: number;

package/dist/completeDataset.steps.js

@@ -13,14 +13,15 @@ function getAjv() {
     }
     return ajvInstance;
 }
-export async function persistDatasetStep({ runtime, datasetId, sandboxId, summary }) {
+export async function persistDatasetStep({ runtime, datasetId, sandboxId, summary, outputPath }) {
     "use step";
-    const outputPath = getDatasetOutputPath(datasetId);
+    const resolvedOutputPath = outputPath ?? getDatasetOutputPath(datasetId);
+    const storagePath = resolveExecutionStoragePath(resolvedOutputPath, datasetId);
     if (summary) {
         console.log(`[Dataset ${datasetId}] Persisting completed dataset: ${summary}`);
     }
     try {
-        await ensureFileExists(runtime, sandboxId, outputPath);
+        await ensureFileExists(runtime, sandboxId, resolvedOutputPath);
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -85,7 +86,7 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
     const validationResult = await validateJsonlRows({
         runtime,
         sandboxId,
-        outputPath,
+        outputPath: resolvedOutputPath,
         validator,
         schema: schemaJson,
         datasetId,
@@ -96,7 +97,7 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
     const totalValidRows = validationResult.validRowCount ?? 0;
     const rowRecordCount = validationResult.rowRecordCount ?? totalValidRows;
     console.log(`[Dataset ${datasetId}] Reading file content for upload`);
-    const fileRead = await readDatasetSandboxFileStep({ runtime, sandboxId, path: outputPath });
+    const fileRead = await readDatasetSandboxFileStep({ runtime, sandboxId, path: resolvedOutputPath });
     if (!fileRead.contentBase64) {
         console.error(`[Dataset ${datasetId}] Empty file content`);
         return {
@@ -113,6 +114,7 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
     const uploadResult = await service.uploadDatasetOutputFile({
         datasetId,
         fileBuffer: Buffer.from(fileRead.contentBase64, "base64"),
+        storagePath,
     });
     if (!uploadResult.ok) {
         console.error(`[Dataset ${datasetId}] File upload failed: ${uploadResult.error}`);
@@ -150,13 +152,18 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
     return {
         success: true,
         status: "completed",
-        validRows: totalValidRows,
-        rowRecordCount,
-        fileId: uploadResult.data.fileId,
-        storagePath: uploadResult.data.storagePath,
-        message: "Dataset creation completed and uploaded to storage",
+        records: totalValidRows,
+        summary: summary ?? `Dataset completed with ${totalValidRows} records.`,
     };
 }
+function resolveExecutionStoragePath(outputPath, datasetId) {
+    const normalized = String(outputPath ?? "").replace(/\\/g, "/");
+    const marker = "/tmp/ekairos/contexts/";
+    if (normalized.startsWith(marker)) {
+        return normalized.slice("/tmp/ekairos".length);
+    }
+    return `/dataset/${datasetId}/output.jsonl`;
+}
 async function ensureFileExists(runtime, sandboxId, path) {
     const result = await runDatasetSandboxCommandStep({
         runtime,

package/dist/completeDataset.tool.d.ts

@@ -2,8 +2,9 @@ interface CompleteDatasetToolParams {
     datasetId: string;
     sandboxId: string;
     runtime: any;
+    outputPath?: string;
 }
-export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtime }: CompleteDatasetToolParams): import("ai").Tool<{
+export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outputPath }: CompleteDatasetToolParams): import("ai").Tool<{
     summary: string;
 }, {
     success: boolean;
@@ -82,18 +83,18 @@ export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtim
     }[] | undefined;
     error: string;
     message: string;
-    fileId?: undefined;
-    storagePath?: undefined;
+    records?: undefined;
+    summary?: undefined;
 } | {
     success: boolean;
     status: string;
-    validRows: number;
-    rowRecordCount: number;
-    fileId: string;
-    storagePath: string;
-    message: string;
+    records: number;
+    summary: string;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
     validation?: undefined;
     error?: undefined;
+    message?: undefined;
 }>;
 export declare function didCompleteDatasetSucceed(event: {
     content?: {

package/dist/completeDataset.tool.js

@@ -1,7 +1,7 @@
 import { tool } from "ai";
 import { z } from "zod";
 import { persistDatasetStep } from "./completeDataset.steps.js";
-export function createCompleteDatasetTool({ datasetId, sandboxId, runtime }) {
+export function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outputPath }) {
     return tool({
         description: "Mark the dataset as completed. Use only when output.jsonl has been successfully generated and is ready for validation.",
         inputSchema: z.object({
@@ -17,6 +17,7 @@ export function createCompleteDatasetTool({ datasetId, sandboxId, runtime }) {
                 datasetId,
                 sandboxId,
                 summary,
+                outputPath,
             });
         },
     });

package/dist/contextWorkspace.d.ts

@@ -0,0 +1,72 @@
+export type ContextWorkspaceFileRole = "input" | "output" | "artifact";
+export type ContextWorkspaceFileInput = {
+    fileId: string;
+    filename?: string;
+    mediaType?: string;
+    role?: ContextWorkspaceFileRole;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+};
+export type PreparedContextWorkspaceFile = {
+    fileId: string;
+    filename: string;
+    mediaType?: string;
+    role: ContextWorkspaceFileRole;
+    path: string;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+};
+export type PreparedContextExecutionWorkspace = {
+    contextId: string;
+    executionId: string;
+    sandboxId: string;
+    root: string;
+    contextRoot: string;
+    eventsDir: string;
+    outputDir: string;
+    scriptsDir: string;
+    tmpDir: string;
+    manifestPath: string;
+    files: PreparedContextWorkspaceFile[];
+};
+export declare function getContextWorkspaceBase(): string;
+export declare function getContextExecutionWorkspaceRoot(params: {
+    contextId: string;
+    executionId: string;
+    root?: string;
+}): string;
+export declare function getContextWorkspaceRoot(params: {
+    contextId: string;
+    root?: string;
+}): string;
+export declare function getContextEventsDir(params: {
+    contextId: string;
+    root?: string;
+}): string;
+export declare function getContextExecutionWorkspaceDirs(params: {
+    contextId: string;
+    executionId: string;
+    root?: string;
+}): {
+    root: string;
+    contextRoot: string;
+    eventsDir: string;
+    outputDir: string;
+    scriptsDir: string;
+    tmpDir: string;
+    manifestPath: string;
+};
+export declare function getContextExecutionWorkspaceStandardDirs(params: {
+    contextId: string;
+    executionId: string;
+    root?: string;
+}): string[];
+export declare function extractContextWorkspaceFilesFromEventItems(eventItems: unknown[]): ContextWorkspaceFileInput[];
+export declare function prepareContextExecutionWorkspaceStep(params: {
+    runtime: any;
+    sandboxId: string;
+    contextId: string;
+    executionId: string;
+    files: ContextWorkspaceFileInput[];
+    root?: string;
+}): Promise<PreparedContextExecutionWorkspace>;

package/dist/contextWorkspace.js

@@ -0,0 +1,218 @@
+import { readInstantFileStep } from "./file/steps.js";
+import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFilesStep, } from "./sandbox/steps.js";
+const CONTEXT_WORKSPACE_BASE = "/tmp/ekairos/contexts";
+const WORKSPACE_MANIFEST_FILE_NAME = "manifest.json";
+function trimTrailingSlash(value) {
+    return value.endsWith("/") ? value.slice(0, -1) : value;
+}
+function sanitizePathSegment(value, fallback) {
+    const parts = String(value ?? "")
+        .trim()
+        .replace(/\\/g, "/")
+        .split("/")
+        .filter(Boolean);
+    const normalized = parts[parts.length - 1]
+        ?.replace(/[^a-zA-Z0-9_.-]/g, "_")
+        .replace(/_+/g, "_")
+        .slice(0, 160);
+    return normalized || fallback;
+}
+function filenameFromContentDisposition(value, fallback) {
+    const raw = String(value ?? "").trim();
+    if (!raw)
+        return fallback;
+    const filenameStar = raw.match(/filename\*=UTF-8''([^;]+)/i)?.[1];
+    if (filenameStar) {
+        return sanitizePathSegment(decodeURIComponent(filenameStar), fallback);
+    }
+    const filename = raw.match(/filename="?([^";]+)"?/i)?.[1];
+    return sanitizePathSegment(filename ?? raw, fallback);
+}
+function resolveContextEventPartDir(params) {
+    const sourceEventId = sanitizePathSegment(params.sourceEventId, "event");
+    const sourcePartIndex = Number.isFinite(params.sourcePartIndex)
+        ? Math.max(0, Math.floor(params.sourcePartIndex))
+        : 0;
+    return `${params.eventsDir}/${sourceEventId}/parts/${sourcePartIndex}`;
+}
+function resolveWorkspaceFilePath(params) {
+    return `${resolveContextEventPartDir(params)}/file`;
+}
+export function getContextWorkspaceBase() {
+    return trimTrailingSlash(CONTEXT_WORKSPACE_BASE);
+}
+export function getContextExecutionWorkspaceRoot(params) {
+    if (params.root)
+        return trimTrailingSlash(params.root);
+    const contextId = sanitizePathSegment(params.contextId, "context");
+    const executionId = sanitizePathSegment(params.executionId, "execution");
+    return `${getContextWorkspaceBase()}/${contextId}/executions/${executionId}`;
+}
+export function getContextWorkspaceRoot(params) {
+    if (params.root)
+        return trimTrailingSlash(params.root);
+    const contextId = sanitizePathSegment(params.contextId, "context");
+    return `${getContextWorkspaceBase()}/${contextId}`;
+}
+export function getContextEventsDir(params) {
+    return `${getContextWorkspaceRoot(params)}/events`;
+}
+export function getContextExecutionWorkspaceDirs(params) {
+    const root = getContextExecutionWorkspaceRoot(params);
+    const contextRoot = getContextWorkspaceRoot(params);
+    const eventsDir = getContextEventsDir(params);
+    return {
+        root,
+        contextRoot,
+        eventsDir,
+        outputDir: `${root}/output`,
+        scriptsDir: `${root}/scripts`,
+        tmpDir: `${root}/tmp`,
+        manifestPath: `${root}/${WORKSPACE_MANIFEST_FILE_NAME}`,
+    };
+}
+export function getContextExecutionWorkspaceStandardDirs(params) {
+    const dirs = getContextExecutionWorkspaceDirs(params);
+    return [dirs.contextRoot, dirs.eventsDir, dirs.root, dirs.outputDir, dirs.scriptsDir, dirs.tmpDir];
+}
+export function extractContextWorkspaceFilesFromEventItems(eventItems) {
+    const files = [];
+    for (const item of eventItems) {
+        const itemRecord = asRecord(item);
+        const parts = Array.isArray(asRecord(itemRecord?.content)?.parts)
+            ? asRecord(itemRecord?.content)?.parts
+            : [];
+        parts.forEach((part, partIndex) => {
+            collectPartFiles(part, {
+                files,
+                sourceEventId: asText(itemRecord?.id),
+                sourcePartIndex: partIndex,
+            });
+        });
+    }
+    return files;
+}
+export async function prepareContextExecutionWorkspaceStep(params) {
+    "use step";
+    const dirs = getContextExecutionWorkspaceDirs(params);
+    const filePartDirs = Array.from(new Set(params.files.map((fileInput) => resolveContextEventPartDir({
+        eventsDir: dirs.eventsDir,
+        sourceEventId: fileInput.sourceEventId ?? fileInput.fileId,
+        sourcePartIndex: fileInput.sourcePartIndex ?? 0,
+    }))));
+    await runDatasetSandboxCommandStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        cmd: "mkdir",
+        args: ["-p", ...getContextExecutionWorkspaceStandardDirs(params), ...filePartDirs],
+    });
+    const preparedFiles = [];
+    for (const fileInput of params.files) {
+        const fileId = String(fileInput.fileId ?? "").trim();
+        if (!fileId)
+            continue;
+        const file = await readInstantFileStep({ runtime: params.runtime, fileId });
+        const filename = sanitizePathSegment(fileInput.filename ??
+            filenameFromContentDisposition(file.contentDisposition, `${fileId}.bin`), `${fileId}.bin`);
+        const path = resolveWorkspaceFilePath({
+            eventsDir: dirs.eventsDir,
+            sourceEventId: fileInput.sourceEventId ?? fileId,
+            sourcePartIndex: fileInput.sourcePartIndex ?? 0,
+        });
+        const metadataPath = `${resolveContextEventPartDir({
+            eventsDir: dirs.eventsDir,
+            sourceEventId: fileInput.sourceEventId ?? fileId,
+            sourcePartIndex: fileInput.sourcePartIndex ?? 0,
+        })}/metadata.json`;
+        await writeDatasetSandboxFilesStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            files: [{ path, contentBase64: file.contentBase64 }],
+        });
+        await writeDatasetSandboxTextFilesStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            files: [
+                {
+                    path: metadataPath,
+                    content: JSON.stringify({
+                        fileId,
+                        filename,
+                        mediaType: fileInput.mediaType,
+                        role: fileInput.role ?? "input",
+                        sourceEventId: fileInput.sourceEventId,
+                        sourcePartIndex: fileInput.sourcePartIndex,
+                    }, null, 2),
+                },
+            ],
+        });
+        preparedFiles.push({
+            fileId,
+            filename,
+            mediaType: fileInput.mediaType,
+            role: fileInput.role ?? "input",
+            path,
+            sourceEventId: fileInput.sourceEventId,
+            sourcePartIndex: fileInput.sourcePartIndex,
+        });
+    }
+    const manifest = {
+        contextId: params.contextId,
+        executionId: params.executionId,
+        sandboxId: params.sandboxId,
+        ...dirs,
+        files: preparedFiles,
+    };
+    await writeDatasetSandboxTextFilesStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        files: [
+            {
+                path: dirs.manifestPath,
+                content: JSON.stringify(manifest, null, 2),
+            },
+        ],
+    });
+    return manifest;
+}
+function collectPartFiles(value, params) {
+    const record = asRecord(value);
+    if (!record)
+        return;
+    if (record.type === "file") {
+        pushFileRecord(record, params);
+        return;
+    }
+    const content = asRecord(record.content);
+    if (!content)
+        return;
+    if (Array.isArray(content.blocks)) {
+        for (const block of content.blocks) {
+            const blockRecord = asRecord(block);
+            if (blockRecord?.type === "file") {
+                pushFileRecord(blockRecord, params);
+            }
+        }
+    }
+}
+function pushFileRecord(record, params) {
+    const fileId = asText(record.fileId);
+    if (!fileId)
+        return;
+    params.files.push({
+        fileId,
+        filename: asText(record.filename),
+        mediaType: asText(record.mediaType),
+        role: "input",
+        sourceEventId: params.sourceEventId,
+        sourcePartIndex: params.sourcePartIndex,
+    });
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function asText(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}

package/dist/executeCommand.tool.d.ts

@@ -6,47 +6,5 @@ interface ExecuteCommandToolParams {
 export declare function createExecuteCommandTool({ datasetId, sandboxId, runtime }: ExecuteCommandToolParams): import("ai").Tool<{
     pythonCode: string;
     scriptName: string;
-}, {
-    success: boolean;
-    fatal: boolean;
-    status: string;
-    error: string;
-    stdout: string;
-    stderr: string;
-    exitCode: number;
-    scriptPath: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    message?: undefined;
-} | {
-    success: boolean;
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    scriptPath: string;
-    error: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    fatal?: undefined;
-    status?: undefined;
-    message?: undefined;
-} | {
-    success: boolean;
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    scriptPath: string;
-    message: string;
-    stdoutTruncated: boolean;
-    stderrTruncated: boolean;
-    stdoutOriginalLength: number;
-    stderrOriginalLength: number;
-    fatal?: undefined;
-    status?: undefined;
-    error?: undefined;
-}>;
+}, unknown>;
 export {};

package/dist/executeCommand.tool.js

@@ -2,6 +2,7 @@ import { tool } from "ai";
 import { z } from "zod";
 import { runDatasetSandboxCommandStep, writeDatasetSandboxTextFilesStep } from "./sandbox/steps.js";
 import { getDatasetScriptsDir } from "./datasetFiles.js";
+import { getContextExecutionWorkspaceDirs } from "./contextWorkspace.js";
 // To keep responses predictable for big data scenarios, we cap stdout/stderr.
 // The tool's return payload exposes stdout (capped) plus the on-disk script path.
 const MAX_STDOUT_CHARS = 20000;
@@ -29,10 +30,16 @@ export function createExecuteCommandTool({ datasetId, sandboxId, runtime }) {
             pythonCode: z.string().describe("Python code to execute. Saved to a file before running. MANDATORY: Use print() to report progress and final results. Keep prints concise; avoid dumping rows/JSON. For large outputs, write to files in the workstation directory and print only file paths and brief summaries."),
             scriptName: z.string().describe("Name for the script file in snake_case (e.g., 'inspect_file', 'parse_csv', 'generate_dataset'). A deterministic suffix will be appended automatically."),
         }),
-        execute: async ({ pythonCode, scriptName }) => {
+        execute: (async ({ pythonCode, scriptName }, actionContext) => {
             const normalizedScriptName = normalizeScriptName(scriptName);
             const scriptHash = stableScriptHash(`${normalizedScriptName}\0${pythonCode}`);
-            const scriptFile = `${getDatasetScriptsDir(datasetId)}/${normalizedScriptName}-${scriptHash}.py`;
+            const scriptsDir = actionContext?.contextId && actionContext.executionId
+                ? getContextExecutionWorkspaceDirs({
+                    contextId: actionContext.contextId,
+                    executionId: actionContext.executionId,
+                }).scriptsDir
+                : getDatasetScriptsDir(datasetId);
+            const scriptFile = `${scriptsDir}/${normalizedScriptName}-${scriptHash}.py`;
             console.log(`[Dataset ${datasetId}] ========================================`);
             console.log(`[Dataset ${datasetId}] Tool: executeCommand`);
             console.log(`[Dataset ${datasetId}] Script: ${normalizedScriptName}`);
@@ -162,6 +169,6 @@ export function createExecuteCommandTool({ datasetId, sandboxId, runtime }) {
                     stderrOriginalLength: 0,
                 };
             }
-        },
+        }),
     });
 }

package/dist/file/file-dataset.agent.d.ts

@@ -12,6 +12,8 @@ export declare function createFileParseContext<Env extends {
     sandboxState?: SandboxState;
     filePreview?: FileParseContext["filePreview"];
     schema?: any | null;
+    filename?: string;
+    mediaType?: string;
 }): {
     datasetId: string;
     parse(runtime: {

package/dist/file/file-dataset.agent.js

@@ -4,7 +4,7 @@ import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFa
 import { datasetGetByIdStep } from "../dataset/steps.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
 import { createGenerateSchemaTool } from "./generateSchema.tool.js";
-import { buildFileDatasetPromptStep, generateFileParsePreviewStep, initializeFileParseSandboxStep, } from "./file-dataset.steps.js";
+import { buildFileDatasetPromptStep, initializeFileParseSandboxStep, } from "./file-dataset.steps.js";
 import { createDatasetId } from "../id.js";
 async function awaitContextRun(run) {
     if (!run)
@@ -27,6 +27,15 @@ function createFileParseContextDefinition(params) {
         const fileId = previous?.fileId ?? params.fileId ?? "";
         const instructions = previous?.instructions ?? params.instructions ?? "";
         const sandboxId = previous?.sandboxId ?? params.sandboxId ?? "";
+        const contextRun = runtime?.__ekairosContextRun ?? {};
+        const contextId = String(contextRun.contextId ?? stored?.id ?? "").trim();
+        const executionId = String(contextRun.executionId ?? previous?.executionId ?? "").trim();
+        const sourceEventId = String(previous?.sourceEventId ?? params.sourceEventId ?? "").trim();
+        const sourcePartIndex = typeof previous?.sourcePartIndex === "number"
+            ? previous.sourcePartIndex
+            : typeof params.sourcePartIndex === "number"
+                ? params.sourcePartIndex
+                : 0;
         if (!datasetId) {
             throw new Error("dataset_id_required");
         }
@@ -36,30 +45,29 @@ function createFileParseContextDefinition(params) {
         if (!sandboxId) {
             throw new Error("dataset_sandbox_required");
         }
+        if (!contextId) {
+            throw new Error("dataset_context_id_required");
+        }
+        if (!executionId) {
+            throw new Error("dataset_execution_id_required");
+        }
         const initialized = sandboxState.initialized && sandboxState.filePath
             ? { filePath: sandboxState.filePath, state: sandboxState }
             : await initializeFileParseSandboxStep({
                 runtime,
                 sandboxId,
+                contextId,
+                executionId,
                 datasetId,
                 fileId,
+                sourceEventId,
+                sourcePartIndex,
+                filename: previous?.filename ?? params.filename,
+                mediaType: previous?.mediaType ?? params.mediaType,
                 state: sandboxState,
             });
         const sandboxFilePath = initialized.filePath;
         let filePreview = previous?.filePreview ?? previous?.ctx?.filePreview ?? params.filePreview;
-        if (!filePreview) {
-            try {
-                filePreview = await generateFileParsePreviewStep({
-                    runtime,
-                    sandboxId,
-                    sandboxFilePath,
-                    datasetId,
-                });
-            }
-            catch {
-                // Preview is optional; parsing can still proceed from the file path.
-            }
-        }
         let schema = previous?.ctx?.schema ?? previous?.schema ?? params.schema ?? null;
         const datasetResult = await datasetGetByIdStep({ runtime, datasetId });
         if (datasetResult.ok && datasetResult.data.schema) {
@@ -69,7 +77,12 @@ function createFileParseContextDefinition(params) {
             datasetId,
             fileId,
             instructions,
-            sandboxConfig: { filePath: sandboxFilePath },
+            sandboxConfig: {
+                filePath: sandboxFilePath,
+                outputPath: initialized.state.outputPath,
+                scriptsDir: initialized.state.scriptsDir,
+                manifestPath: initialized.state.manifestPath,
+            },
             analysis: [],
             schema,
             plan: null,
@@ -84,6 +97,11 @@ function createFileParseContextDefinition(params) {
             fileId,
             instructions,
             sandboxId,
+            executionId,
+            sourceEventId,
+            sourcePartIndex,
+            filename: previous?.filename ?? params.filename,
+            mediaType: previous?.mediaType ?? params.mediaType,
             sandboxState: initialized.state,
             filePreview,
             ctx,
@@ -109,6 +127,7 @@ function createFileParseContextDefinition(params) {
         const datasetId = _stored?.content?.datasetId ?? fallbackDatasetId ?? "";
         const fileId = _stored?.content?.fileId ?? params.fileId ?? "";
         const sandboxId = _stored?.content?.sandboxId ?? params.sandboxId ?? "";
+        const outputPath = _stored?.content?.ctx?.sandboxConfig?.outputPath;
         if (!datasetId)
             throw new Error("dataset_id_required");
         if (!fileId)
@@ -125,6 +144,7 @@ function createFileParseContextDefinition(params) {
                 datasetId,
                 sandboxId,
                 runtime,
+                outputPath,
             }),
             clearDataset: createClearDatasetTool({
                 datasetId,
@@ -169,6 +189,8 @@ export function createFileParseContext(fileId, opts) {
         sandboxState: opts?.sandboxState,
         filePreview: opts?.filePreview,
         schema: opts?.schema,
+        filename: opts?.filename,
+        mediaType: opts?.mediaType,
     };
     const { context } = createFileParseContextDefinition(params);
     return {
@@ -185,9 +207,19 @@ export function createFileParseContext(fileId, opts) {
                             type: "text",
                             text: options.prompt ?? "generate a dataset for this file",
                         },
+                        {
+                            type: "file",
+                            fileId,
+                            filename: opts?.filename ?? "source-file",
+                            mediaType: opts?.mediaType ?? "application/octet-stream",
+                        },
                     ],
                 },
             };
+            params.sourceEventId = triggerEvent.id;
+            params.sourcePartIndex = 1;
+            params.filename = opts?.filename ?? "source-file";
+            params.mediaType = opts?.mediaType ?? "application/octet-stream";
             const shell = await context.react(triggerEvent, {
                 runtime: runtime,
                 context: { key: `dataset:${datasetId}` },
@@ -203,6 +235,10 @@ export function createFileParseContext(fileId, opts) {
                     ...(options.initialContent ?? {}),
                     datasetId,
                     fileId,
+                    sourceEventId: triggerEvent.id,
+                    sourcePartIndex: 1,
+                    filename: opts?.filename ?? "source-file",
+                    mediaType: opts?.mediaType ?? "application/octet-stream",
                     instructions: opts?.instructions ?? "",
                     sandboxId: opts?.sandboxId ?? "",
                     sandboxState: opts?.sandboxState ?? { initialized: false, filePath: "" },

package/dist/file/file-dataset.steps.d.ts

@@ -3,8 +3,14 @@ import type { FilePreviewContext } from "./filepreview.types.js";
 export declare function initializeFileParseSandboxStep(params: {
     runtime: any;
     sandboxId: string;
+    contextId: string;
+    executionId: string;
     datasetId: string;
     fileId: string;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+    filename?: string;
+    mediaType?: string;
     state: SandboxState;
 }): Promise<{
     filePath: string;

package/dist/file/file-dataset.steps.js

@@ -1,42 +1,39 @@
-import { getDatasetSourcesDir, getDatasetStandardDirs, getDatasetWorkstation, } from "../datasetFiles.js";
-import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "../sandbox/steps.js";
+import { DATASET_OUTPUT_FILE_NAME } from "../datasetFiles.js";
+import { prepareContextExecutionWorkspaceStep } from "../contextWorkspace.js";
 import { buildFileDatasetPrompt } from "./prompts.js";
 import { generateFilePreview } from "./filepreview.js";
-import { readInstantFileStep } from "./steps.js";
 export async function initializeFileParseSandboxStep(params) {
     "use step";
     if (params.state.initialized) {
         return { filePath: params.state.filePath, state: params.state };
     }
-    console.log(`[FileParseContext ${params.datasetId}] Preparing source file in sandbox...`);
-    console.log(`[FileParseContext ${params.datasetId}] Fetching file from InstantDB...`);
-    const file = await readInstantFileStep({ runtime: params.runtime, fileId: params.fileId });
-    console.log(`[FileParseContext ${params.datasetId}] Creating dataset workstation...`);
-    const workstation = getDatasetWorkstation(params.datasetId);
-    await runDatasetSandboxCommandStep({
-        runtime: params.runtime,
-        sandboxId: params.sandboxId,
-        cmd: "mkdir",
-        args: ["-p", ...getDatasetStandardDirs(params.datasetId)],
-    });
-    const fileName = file.contentDisposition ?? "";
-    const fileExtension = fileName.includes(".") ? fileName.substring(fileName.lastIndexOf(".")) : "";
-    const sandboxFilePath = `${getDatasetSourcesDir(params.datasetId)}/${params.fileId}${fileExtension}`;
-    await writeDatasetSandboxFilesStep({
+    console.log(`[FileParseContext ${params.datasetId}] Preparing context execution workspace...`);
+    const workspace = await prepareContextExecutionWorkspaceStep({
         runtime: params.runtime,
         sandboxId: params.sandboxId,
+        contextId: params.contextId,
+        executionId: params.executionId,
         files: [
             {
-                path: sandboxFilePath,
-                contentBase64: file.contentBase64,
+                fileId: params.fileId,
+                filename: params.filename,
+                mediaType: params.mediaType,
+                sourceEventId: params.sourceEventId,
+                sourcePartIndex: params.sourcePartIndex,
             },
         ],
     });
-    console.log(`[FileParseContext ${params.datasetId}] Workstation created: ${workstation}`);
+    const sandboxFilePath = workspace.files[0]?.path ?? "";
+    if (!sandboxFilePath)
+        throw new Error("dataset_workspace_file_missing");
+    console.log(`[FileParseContext ${params.datasetId}] Context workspace created: ${workspace.root}`);
     console.log(`[FileParseContext ${params.datasetId}] File saved: ${sandboxFilePath}`);
     const state = {
         initialized: true,
         filePath: sandboxFilePath,
+        outputPath: `${workspace.outputDir}/${DATASET_OUTPUT_FILE_NAME}`,
+        scriptsDir: workspace.scriptsDir,
+        manifestPath: workspace.manifestPath,
     };
     return { filePath: sandboxFilePath, state };
 }

package/dist/file/file-dataset.types.d.ts

@@ -3,6 +3,9 @@ import type { FilePreviewContext } from "./filepreview.types.js";
 export type SandboxState = {
     initialized: boolean;
     filePath: string;
+    outputPath?: string;
+    scriptsDir?: string;
+    manifestPath?: string;
 };
 export type FileParseContext = {
     datasetId: string;
@@ -10,6 +13,9 @@ export type FileParseContext = {
     instructions: string;
     sandboxConfig: {
         filePath: string;
+        outputPath?: string;
+        scriptsDir?: string;
+        manifestPath?: string;
     };
     analysis: any[];
     schema: any | null;
@@ -29,6 +35,10 @@ export type FileParseContextParams = {
     sandboxState?: SandboxState;
     filePreview?: FilePreviewContext;
     schema?: any | null;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+    filename?: string;
+    mediaType?: string;
 };
 export type FileParseRunOptions = {
     prompt?: string;

package/dist/file/prompts.js

@@ -223,8 +223,10 @@ function buildSchemaSection(context) {
     return xml.end({ prettyPrint: true, headless: true });
 }
 function buildInstructions(context) {
-    const datasetWorkstation = getDatasetWorkstation(context.datasetId);
-    const outputPath = getDatasetOutputPath(context.datasetId);
+    const datasetWorkstation = context.sandboxConfig.scriptsDir
+        ? context.sandboxConfig.scriptsDir.replace(/\/scripts$/, "")
+        : getDatasetWorkstation(context.datasetId);
+    const outputPath = context.sandboxConfig.outputPath ?? getDatasetOutputPath(context.datasetId);
     const hasProvidedSchema = Boolean(context.schema?.schema);
     const currentTask = hasProvidedSchema
         ? "Review FilePreview section, use the provided schema as the output contract, then parse the file and generate the dataset"

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./dataset.js";
+export * from "./contextWorkspace.js";
 export * from "./domain.js";
 export * from "./materializeDataset.tool.js";
 export * from "./schema.js";

package/dist/index.js

@@ -1,4 +1,5 @@
 export * from "./dataset.js";
+export * from "./contextWorkspace.js";
 export * from "./domain.js";
 export * from "./materializeDataset.tool.js";
 export * from "./schema.js";

package/dist/service.d.ts

@@ -64,6 +64,7 @@ export declare class DatasetService {
     uploadDatasetOutputFile(params: {
         datasetId: string;
         fileBuffer: Buffer;
+        storagePath?: string;
     }): Promise<ServiceResult<{
         fileId: string;
         storagePath: string;

package/dist/service.js

@@ -308,7 +308,7 @@ export class DatasetService {
     }
     async uploadDatasetOutputFile(params) {
         try {
-            const storagePath = `/dataset/${params.datasetId}/output.jsonl`;
+            const storagePath = params.storagePath ?? `/dataset/${params.datasetId}/output.jsonl`;
             const uploadResult = await this.db.storage.uploadFile(storagePath, params.fileBuffer, {
                 contentType: "application/x-ndjson",
                 contentDisposition: "output.jsonl",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.81-beta.development.0",
+  "version": "1.22.83-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.81-beta.development.0",
-    "@ekairos/events": "^1.22.81-beta.development.0",
-    "@ekairos/sandbox": "^1.22.81-beta.development.0",
+    "@ekairos/domain": "^1.22.83-beta.development.0",
+    "@ekairos/events": "^1.22.83-beta.development.0",
+    "@ekairos/sandbox": "^1.22.83-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",
@@ -80,10 +80,10 @@
     "@ekairos/openai-reactor": "workspace:*",
     "@ekairos/tsconfig": "workspace:*",
     "@types/node": "^24.5.0",
-    "@workflow/serde": "5.0.0-beta.0",
-    "@workflow/vitest": "5.0.0-beta.1",
+    "@workflow/serde": "5.0.0-beta.1",
+    "@workflow/vitest": "5.0.0-beta.5",
     "dotenv": "^17.2.3",
     "typescript": "^5.9.2",
-    "workflow": "5.0.0-beta.1"
+    "workflow": "5.0.0-beta.5"
   }
 }