@ekairos/structure 1.21.74-beta.0 → 1.21.77-beta.0

package/dist/dataset/steps.js

@@ -63,6 +63,7 @@ export async function structurePatchContextContentStep(params) {
 }
 export async function structureUploadRowsOutputJsonlStep(params) {
     "use step";
+    const startedAt = Date.now();
     try {
         const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
         const runtime = await resolveStoryRuntime(params.env);
@@ -76,6 +77,7 @@ export async function structureUploadRowsOutputJsonlStep(params) {
         const fileId = uploadResult?.data?.id;
         if (!fileId)
             return { ok: false, error: "Failed to upload file to storage" };
+        console.log(`[structure:upload-jsonl] structureId=${params.structureId} bytes=${fileBuffer.byteLength} elapsedMs=${Date.now() - startedAt}`);
         return { ok: true, data: { fileId, storagePath } };
     }
     catch (error) {
@@ -85,6 +87,7 @@ export async function structureUploadRowsOutputJsonlStep(params) {
 }
 export async function structureLinkRowsOutputFileToContextStep(params) {
     "use step";
+    const startedAt = Date.now();
     try {
         const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
         const runtime = await resolveStoryRuntime(params.env);
@@ -95,6 +98,7 @@ export async function structureLinkRowsOutputFileToContextStep(params) {
         if (!ctxId)
             return { ok: false, error: "Context not found" };
         await db.transact([db.tx.context_contexts[ctxId].link({ structure_output_file: params.fileId })]);
+        console.log(`[structure:link-jsonl] contextKey=${params.contextKey} fileId=${params.fileId} elapsedMs=${Date.now() - startedAt}`);
         return { ok: true };
     }
     catch (error) {
@@ -145,6 +149,7 @@ export async function structureGetContextWithRowsOutputFileStep(params) {
 }
 export async function structureReadRowsOutputJsonlStep(params) {
     "use step";
+    const startedAt = Date.now();
     try {
         const contextKey = `structure:${params.structureId}`;
         const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
@@ -164,6 +169,7 @@ export async function structureReadRowsOutputJsonlStep(params) {
         if (!url)
             return { ok: false, error: "Rows output file not found" };
         const fileBuffer = await fetchArrayBufferWithRetry(url, { attempts: 4, timeoutMs: 90000 });
+        console.log(`[structure:read-jsonl] structureId=${params.structureId} bytes=${fileBuffer.byteLength} elapsedMs=${Date.now() - startedAt}`);
         return { ok: true, data: { contentBase64: Buffer.from(fileBuffer).toString("base64") } };
     }
     catch (error) {

package/dist/datasetFiles.d.ts

@@ -1,5 +1,11 @@
 export declare const DATASET_OUTPUT_FILE_NAME = "output.jsonl";
 export declare const DATASET_OUTPUT_SCHEMA_FILE_NAME = "output_schema.json";
+export declare const DEFAULT_DAYTONA_VOLUME_MOUNT_PATH = "/home/daytona/.ekairos";
+export declare const DEFAULT_DAYTONA_VOLUME_NAME = "ekairos-structure";
+export declare const DEFAULT_DATASET_WORKDIR_BASE = "/home/daytona/.ekairos/datasets";
+export declare function getDaytonaVolumeMountPath(): string;
+export declare function getDaytonaVolumeName(): string;
+export declare function getDatasetWorkdirBase(): string;
 export declare function getDatasetWorkstation(datasetId: string): string;
 export declare function getDatasetOutputPath(datasetId: string): string;
 export declare function getDatasetOutputSchemaPath(datasetId: string): string;

package/dist/datasetFiles.js

@@ -1,7 +1,31 @@
 export const DATASET_OUTPUT_FILE_NAME = "output.jsonl";
 export const DATASET_OUTPUT_SCHEMA_FILE_NAME = "output_schema.json";
+export const DEFAULT_DAYTONA_VOLUME_MOUNT_PATH = "/home/daytona/.ekairos";
+export const DEFAULT_DAYTONA_VOLUME_NAME = "ekairos-structure";
+export const DEFAULT_DATASET_WORKDIR_BASE = `${DEFAULT_DAYTONA_VOLUME_MOUNT_PATH}/datasets`;
+function trimTrailingSlash(value) {
+    return value.endsWith("/") ? value.slice(0, -1) : value;
+}
+export function getDaytonaVolumeMountPath() {
+    const fromEnv = String(process.env.STRUCTURE_DAYTONA_VOLUME_MOUNT_PATH ?? "").trim();
+    if (fromEnv)
+        return trimTrailingSlash(fromEnv);
+    return DEFAULT_DAYTONA_VOLUME_MOUNT_PATH;
+}
+export function getDaytonaVolumeName() {
+    const fromEnv = String(process.env.STRUCTURE_DAYTONA_VOLUME_NAME ?? "").trim();
+    if (fromEnv)
+        return fromEnv;
+    return DEFAULT_DAYTONA_VOLUME_NAME;
+}
+export function getDatasetWorkdirBase() {
+    const fromEnv = String(process.env.STRUCTURE_SANDBOX_WORKDIR_BASE ?? "").trim();
+    if (fromEnv)
+        return trimTrailingSlash(fromEnv);
+    return `${getDaytonaVolumeMountPath()}/datasets`;
+}
 export function getDatasetWorkstation(datasetId) {
-    return `/vercel/sandbox/datasets/${datasetId}`;
+    return `${getDatasetWorkdirBase()}/${datasetId}`;
 }
 export function getDatasetOutputPath(datasetId) {
     return `${getDatasetWorkstation(datasetId)}/${DATASET_OUTPUT_FILE_NAME}`;

package/dist/prompts.d.ts

@@ -13,5 +13,12 @@ export type StructurePromptContext = {
     sources: PreparedSource[];
     workstation: string;
     outputPath: string;
+    sandboxProvider?: string;
+    sandboxRuntime?: string;
+    sandboxEphemeral?: boolean;
+    sandboxVolumeName?: string;
+    sandboxVolumeMountPath?: string;
+    sandboxSnapshot?: string;
+    sandboxImage?: string;
 };
 export declare function buildStructurePrompt(ctx: StructurePromptContext): string;

package/dist/prompts.js

@@ -24,6 +24,19 @@ export function buildStructurePrompt(ctx) {
     }
     lines.push("");
     lines.push("## SANDBOX");
+    if (ctx.sandboxProvider)
+        lines.push(`Provider: ${ctx.sandboxProvider}`);
+    if (ctx.sandboxRuntime)
+        lines.push(`Runtime: ${ctx.sandboxRuntime}`);
+    if (ctx.sandboxEphemeral !== undefined)
+        lines.push(`Ephemeral: ${ctx.sandboxEphemeral ? "true" : "false"}`);
+    if (ctx.sandboxVolumeName || ctx.sandboxVolumeMountPath) {
+        lines.push(`Volume: ${ctx.sandboxVolumeName ?? "unknown"} -> ${ctx.sandboxVolumeMountPath ?? "unknown"}`);
+    }
+    if (ctx.sandboxSnapshot)
+        lines.push(`Snapshot: ${ctx.sandboxSnapshot}`);
+    if (ctx.sandboxImage)
+        lines.push(`Image: ${ctx.sandboxImage}`);
     lines.push(`Workstation: ${ctx.workstation}`);
     lines.push(`OutputPath: ${ctx.outputPath}`);
     lines.push("");

package/dist/rowsOutputPaging.js

@@ -28,6 +28,15 @@ export async function structureDownloadRowsOutputToSandboxStep(params) {
         cmd: "mkdir",
         args: ["-p", workstation],
     });
+    const exists = await runDatasetSandboxCommandStep({
+        env: params.env,
+        sandboxId,
+        cmd: "test",
+        args: ["-f", localPath],
+    });
+    if (exists.exitCode === 0) {
+        return { sandboxId, localPath };
+    }
     const storyRuntime = await getStoryRuntime(params.env);
     const db = storyRuntime.db;
     const contextKey = `structure:${params.structureId}`;

package/dist/sandbox/steps.d.ts

@@ -1,13 +1,8 @@
+import type { SandboxConfig } from "@ekairos/sandbox";
 export type DatasetSandboxId = string;
-export type CreateDatasetSandboxParams = {
-    runtime?: string;
-    timeoutMs?: number;
-    ports?: number[];
-    resources?: {
-        vcpus?: number;
-    };
-    purpose?: string;
-    params?: Record<string, any>;
+export type CreateDatasetSandboxParams = Omit<SandboxConfig, "provider" | "daytona"> & {
+    provider?: SandboxConfig["provider"];
+    daytona?: SandboxConfig["daytona"];
 };
 export type DatasetSandboxRunCommandResult = {
     exitCode: number;

package/dist/sandbox/steps.js

@@ -1,16 +1,186 @@
+import { getDatasetWorkdirBase, getDaytonaVolumeMountPath, getDaytonaVolumeName } from "../datasetFiles";
+function parseOptionalNumber(value) {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed))
+        return undefined;
+    return parsed;
+}
+function parseOptionalBoolean(value) {
+    const normalized = String(value ?? "").trim().toLowerCase();
+    if (!normalized)
+        return undefined;
+    if (["1", "true", "yes", "y", "on"].includes(normalized))
+        return true;
+    if (["0", "false", "no", "n", "off"].includes(normalized))
+        return false;
+    return undefined;
+}
+function shouldLogDaytonaResources() {
+    const explicit = parseOptionalBoolean(process.env.STRUCTURE_DAYTONA_LOG_RESOURCES);
+    return explicit ?? false;
+}
+function getDaytonaConfigFromEnv() {
+    const apiKey = String(process.env.DAYTONA_API_KEY ?? "").trim();
+    const apiUrl = String(process.env.DAYTONA_API_URL ?? "").trim() ||
+        String(process.env.DAYTONA_SERVER_URL ?? "").trim();
+    const jwtToken = String(process.env.DAYTONA_JWT_TOKEN ?? "").trim();
+    const organizationId = String(process.env.DAYTONA_ORGANIZATION_ID ?? "").trim();
+    const target = String(process.env.DAYTONA_TARGET ?? "").trim();
+    if (!apiUrl)
+        return null;
+    if (!apiKey && !(jwtToken && organizationId))
+        return null;
+    return {
+        apiUrl,
+        apiKey: apiKey || undefined,
+        jwtToken: jwtToken || undefined,
+        organizationId: organizationId || undefined,
+        target: target || undefined,
+    };
+}
+async function logDaytonaResources(label) {
+    if (!shouldLogDaytonaResources())
+        return;
+    const cfg = getDaytonaConfigFromEnv();
+    if (!cfg) {
+        console.log(`[daytona:${label}] missing Daytona env config`);
+        return;
+    }
+    try {
+        const moduleName = "@daytonaio/sdk";
+        const importer = new Function("m", "return import(m)");
+        const { Daytona } = (await importer(moduleName));
+        const daytona = new Daytona(cfg);
+        const list = await daytona.list(undefined, 1, 50);
+        const total = list?.total ?? list?.items?.length ?? 0;
+        console.log(`[daytona:${label}] sandboxes total=${total} page=${list?.page ?? 1}`);
+        const items = Array.isArray(list?.items) ? list.items : [];
+        for (const sb of items.slice(0, 25)) {
+            console.log(`[daytona:${label}] sandbox id=${sb?.id} state=${sb?.state} disk=${sb?.disk} cpu=${sb?.cpu} memory=${sb?.memory} autoDelete=${sb?.autoDeleteInterval} autoStop=${sb?.autoStopInterval} snapshot=${sb?.snapshot ?? ""} createdAt=${sb?.createdAt ?? ""}`);
+        }
+        try {
+            const volumes = await daytona.volume.list();
+            const names = volumes.map((v) => v?.name).filter(Boolean);
+            console.log(`[daytona:${label}] volumes count=${volumes.length} names=${names.slice(0, 20).join(",")}`);
+        }
+        catch (e) {
+            console.log(`[daytona:${label}] volumes list error: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    catch (e) {
+        console.log(`[daytona:${label}] list error: ${e instanceof Error ? e.message : String(e)}`);
+    }
+}
+function getStructureDaytonaDefaults() {
+    const snapshot = String(process.env.STRUCTURE_DAYTONA_SNAPSHOT ?? "").trim();
+    const image = String(process.env.STRUCTURE_DAYTONA_IMAGE ?? "").trim();
+    const declarative = String(process.env.STRUCTURE_DAYTONA_DECLARATIVE_IMAGE ?? "").trim();
+    const ephemeralEnv = parseOptionalBoolean(process.env.STRUCTURE_DAYTONA_EPHEMERAL);
+    const ephemeral = ephemeralEnv ?? true;
+    const autoStopIntervalMin = parseOptionalNumber(process.env.STRUCTURE_DAYTONA_AUTO_STOP_MIN);
+    const autoArchiveIntervalMin = parseOptionalNumber(process.env.STRUCTURE_DAYTONA_AUTO_ARCHIVE_MIN);
+    const autoDeleteIntervalMin = parseOptionalNumber(process.env.STRUCTURE_DAYTONA_AUTO_DELETE_MIN);
+    const volumeName = getDaytonaVolumeName();
+    const mountPath = getDaytonaVolumeMountPath();
+    const volumes = volumeName && mountPath
+        ? [
+            {
+                volumeName,
+                mountPath,
+            },
+        ]
+        : [];
+    return {
+        snapshot: snapshot || undefined,
+        image: image || (declarative ? "declarative" : undefined),
+        ephemeral,
+        autoStopIntervalMin,
+        autoArchiveIntervalMin,
+        autoDeleteIntervalMin,
+        volumes,
+    };
+}
 export async function createDatasetSandboxStep(params) {
     "use step";
+    const startedAt = Date.now();
+    const { env, ...configInput } = params;
     const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
-    const db = (await resolveStoryRuntime(params.env)).db;
+    const db = (await resolveStoryRuntime(env)).db;
     const { SandboxService } = (await import("@ekairos/sandbox"));
     const service = new SandboxService(db);
-    const created = await service.createSandbox(params);
-    if (!created.ok)
+    const daytonaDefaults = getStructureDaytonaDefaults();
+    const explicitVolumes = configInput.daytona?.volumes;
+    const mergedDaytona = {
+        ...daytonaDefaults,
+        ...(configInput.daytona ?? {}),
+        volumes: Array.isArray(explicitVolumes) ? explicitVolumes : daytonaDefaults.volumes,
+    };
+    const vcpusOverride = parseOptionalNumber(process.env.STRUCTURE_DAYTONA_VCPUS);
+    const mergedResources = configInput.resources ?? (vcpusOverride ? { vcpus: vcpusOverride } : undefined);
+    const config = {
+        ...configInput,
+        provider: "daytona",
+        resources: mergedResources,
+        daytona: mergedDaytona,
+    };
+    if (shouldLogDaytonaResources()) {
+        console.log(`[daytona:create] config runtime=${config.runtime ?? ""} purpose=${config.purpose ?? ""} params=${JSON.stringify(config.params ?? {})} snapshot=${config.daytona?.snapshot ?? ""} image=${config.daytona?.image ?? ""} ephemeral=${config.daytona?.ephemeral} autoStop=${config.daytona?.autoStopIntervalMin ?? ""} autoDelete=${config.daytona?.autoDeleteIntervalMin ?? ""} volumes=${JSON.stringify(config.daytona?.volumes ?? [])}`);
+        console.log(`[daytona:create] ts=${new Date(startedAt).toISOString()} startMs=${startedAt}`);
+    }
+    await logDaytonaResources("before_create");
+    const created = await service.createSandbox(config);
+    if (!created.ok) {
+        await logDaytonaResources("create_failed");
         throw new Error(created.error);
+    }
+    await logDaytonaResources("after_create");
+    if (shouldLogDaytonaResources()) {
+        const elapsedMs = Date.now() - startedAt;
+        console.log(`[daytona:create] doneMs=${Date.now()} elapsedMs=${elapsedMs}`);
+    }
+    if (shouldLogDaytonaResources()) {
+        try {
+            const info = await service.reconnectToSandbox(created.data.sandboxId);
+            if (info.ok && !info.data.sandbox.sandboxId) {
+                const sb = info.data.sandbox;
+                console.log(`[daytona:after_create] sandbox id=${sb?.id} state=${sb?.state} disk=${sb?.disk} cpu=${sb?.cpu} memory=${sb?.memory} autoDelete=${sb?.autoDeleteInterval} autoStop=${sb?.autoStopInterval}`);
+            }
+        }
+        catch (e) {
+            console.log(`[daytona:after_create] reconnect error: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        try {
+            const df = await service.runCommand(created.data.sandboxId, "df", ["-h"]);
+            if (df.ok) {
+                console.log(`[sandbox:${created.data.sandboxId}] df -h\n${df.data.output}`);
+            }
+            else {
+                console.log(`[sandbox:${created.data.sandboxId}] df error: ${df.error}`);
+            }
+        }
+        catch (e) {
+            console.log(`[sandbox:${created.data.sandboxId}] df error: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        try {
+            const mountPath = getDaytonaVolumeMountPath();
+            const basePath = getDatasetWorkdirBase();
+            const du = await service.runCommand(created.data.sandboxId, "du", ["-sh", mountPath, basePath]);
+            if (du.ok) {
+                console.log(`[sandbox:${created.data.sandboxId}] du -sh\n${du.data.output}`);
+            }
+            else {
+                console.log(`[sandbox:${created.data.sandboxId}] du error: ${du.error}`);
+            }
+        }
+        catch (e) {
+            console.log(`[sandbox:${created.data.sandboxId}] du error: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
     return { sandboxId: created.data.sandboxId };
 }
 export async function runDatasetSandboxCommandStep(params) {
     "use step";
+    const startedAt = Date.now();
     const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
     const db = (await resolveStoryRuntime(params.env)).db;
     const { SandboxService } = (await import("@ekairos/sandbox"));
@@ -23,10 +193,15 @@ export async function runDatasetSandboxCommandStep(params) {
         stdout: result.data.output ?? "",
         stderr: result.data.error ?? "",
     };
+    if (shouldLogDaytonaResources()) {
+        const elapsedMs = Date.now() - startedAt;
+        console.log(`[daytona:cmd] sandboxId=${params.sandboxId} cmd=${params.cmd} args=${JSON.stringify(params.args ?? [])} elapsedMs=${elapsedMs}`);
+    }
     return normalized;
 }
 export async function writeDatasetSandboxFilesStep(params) {
     "use step";
+    const startedAt = Date.now();
     const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
     const db = (await resolveStoryRuntime(params.env)).db;
     const { SandboxService } = (await import("@ekairos/sandbox"));
@@ -34,6 +209,10 @@ export async function writeDatasetSandboxFilesStep(params) {
     const result = await service.writeFiles(params.sandboxId, params.files);
     if (!result.ok)
         throw new Error(result.error);
+    if (shouldLogDaytonaResources()) {
+        const elapsedMs = Date.now() - startedAt;
+        console.log(`[daytona:write] sandboxId=${params.sandboxId} files=${params.files.length} elapsedMs=${elapsedMs}`);
+    }
 }
 /**
  * Workflow-safe helper:
@@ -53,6 +232,7 @@ export async function writeDatasetSandboxTextFileStep(params) {
 }
 export async function readDatasetSandboxFileStep(params) {
     "use step";
+    const startedAt = Date.now();
     const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
     const db = (await resolveStoryRuntime(params.env)).db;
     const { SandboxService } = (await import("@ekairos/sandbox"));
@@ -60,6 +240,10 @@ export async function readDatasetSandboxFileStep(params) {
     const result = await service.readFile(params.sandboxId, params.path);
     if (!result.ok)
         throw new Error(result.error);
+    if (shouldLogDaytonaResources()) {
+        const elapsedMs = Date.now() - startedAt;
+        console.log(`[daytona:read] sandboxId=${params.sandboxId} path=${params.path} bytes=${result.data.contentBase64?.length ?? 0} elapsedMs=${elapsedMs}`);
+    }
     return result.data;
 }
 /**
@@ -77,6 +261,7 @@ export async function readDatasetSandboxTextFileStep(params) {
 }
 export async function stopDatasetSandboxStep(params) {
     "use step";
+    const startedAt = Date.now();
     const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
     const db = (await resolveStoryRuntime(params.env)).db;
     const { SandboxService } = (await import("@ekairos/sandbox"));
@@ -84,4 +269,8 @@ export async function stopDatasetSandboxStep(params) {
     const result = await service.stopSandbox(params.sandboxId);
     if (!result.ok)
         throw new Error(result.error);
+    if (shouldLogDaytonaResources()) {
+        const elapsedMs = Date.now() - startedAt;
+        console.log(`[daytona:stop] sandboxId=${params.sandboxId} elapsedMs=${elapsedMs}`);
+    }
 }

package/dist/structure.d.ts

@@ -1,4 +1,5 @@
 import { type StructureRowsOutputPagingCursor } from "./rowsOutputPaging";
+import type { SandboxConfig } from "@ekairos/sandbox";
 export type StructureSource = {
     kind: "file";
     fileId: string;
@@ -69,6 +70,7 @@ export declare function structure<Env extends {
     orgId: string;
 }>(env: Env, opts?: {
     datasetId?: string;
+    sandboxConfig?: SandboxConfig;
 }): {
     datasetId: string;
     from(...src: StructureSource[]): /*elided*/ any;

package/dist/structure.js

@@ -1,5 +1,5 @@
 import { createStory, didToolExecute, USER_MESSAGE_TYPE, WEB_CHANNEL } from "@ekairos/story";
-import { getDatasetOutputPath, getDatasetOutputSchemaPath, getDatasetWorkstation } from "./datasetFiles";
+import { getDatasetOutputPath, getDatasetOutputSchemaPath, getDatasetWorkstation, getDaytonaVolumeMountPath, getDaytonaVolumeName, } from "./datasetFiles";
 import { structureDownloadRowsOutputToSandboxStep, structureReadRowsOutputPageFromSandboxStep, } from "./rowsOutputPaging";
 import { structureSplitRowsOutputToDatasetStep } from "./rowsOutputSplit";
 import { createDatasetSandboxStep, readDatasetSandboxFileStep, readDatasetSandboxTextFileStep, runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFileStep, } from "./sandbox/steps";
@@ -50,6 +50,88 @@ function guessTextFileExtension(mimeType, name) {
         return ".yaml";
     return ".txt";
 }
+function shouldSkipPipInstall() {
+    const explicit = String(process.env.STRUCTURE_DAYTONA_SKIP_PIP_INSTALL ?? "").trim().toLowerCase();
+    if (explicit === "1" || explicit === "true" || explicit === "yes")
+        return true;
+    const declarative = String(process.env.STRUCTURE_DAYTONA_DECLARATIVE_IMAGE ?? "").trim().toLowerCase();
+    if (declarative === "1" || declarative === "true" || declarative === "yes")
+        return true;
+    const snapshot = String(process.env.STRUCTURE_DAYTONA_SNAPSHOT ?? "").trim();
+    return Boolean(snapshot);
+}
+function getDefaultSandboxConfig(datasetId) {
+    const volumeName = getDaytonaVolumeName();
+    const mountPath = getDaytonaVolumeMountPath();
+    const volumes = volumeName && mountPath
+        ? [
+            {
+                volumeName,
+                mountPath,
+            },
+        ]
+        : [];
+    return {
+        provider: "daytona",
+        runtime: "python3.13",
+        timeoutMs: 10 * 60 * 1000,
+        purpose: "structure.dataset",
+        params: { datasetId },
+        daytona: {
+            ephemeral: true,
+            autoStopIntervalMin: 5,
+            volumes,
+        },
+    };
+}
+function mergeSandboxConfig(base, override) {
+    if (!override)
+        return base;
+    const mergedParams = {
+        ...(base.params ?? {}),
+        ...(override.params ?? {}),
+    };
+    const mergedDaytona = {
+        ...(base.daytona ?? {}),
+        ...(override.daytona ?? {}),
+    };
+    if (override.daytona && "volumes" in override.daytona) {
+        mergedDaytona.volumes = override.daytona?.volumes;
+    }
+    return {
+        ...base,
+        ...override,
+        params: mergedParams,
+        daytona: mergedDaytona,
+    };
+}
+async function sandboxFileExists(env, sandboxId, path) {
+    const res = await runDatasetSandboxCommandStep({
+        env,
+        sandboxId,
+        cmd: "test",
+        args: ["-f", path],
+    });
+    return res.exitCode === 0;
+}
+async function sandboxFindFirstMatch(env, sandboxId, pattern) {
+    const py = [
+        "import sys, glob",
+        "pattern = sys.argv[1]",
+        "matches = glob.glob(pattern)",
+        "print(matches[0] if matches else '')",
+    ].join("\n");
+    const res = await runDatasetSandboxCommandStep({
+        env,
+        sandboxId,
+        cmd: "python",
+        args: ["-c", py, pattern],
+    });
+    if (res.exitCode !== 0)
+        return null;
+    const out = String(res.stdout ?? "").trim();
+    return out ? out : null;
+}
 async function ensureSandboxPrepared(params) {
     const { env, datasetId, sandboxId, sources, state } = params;
     const workstation = getDatasetWorkstation(datasetId);
@@ -60,25 +142,33 @@ async function ensureSandboxPrepared(params) {
     const mkdirRes = await runDatasetSandboxCommandStep({ env, sandboxId, cmd: "mkdir", args: ["-p", workstation] });
     // Align with dataset sandbox behavior: install python deps up-front (once per dataset sandbox).
     // This avoids tool-level "install if used" heuristics and ensures scripts can import pandas.
-    const pipInstall = await runDatasetSandboxCommandStep({
-        env,
-        sandboxId,
-        cmd: "python",
-        // NOTE: pandas needs openpyxl to read .xlsx files.
-        args: ["-m", "pip", "install", "pandas", "openpyxl", "--quiet", "--upgrade"],
-    });
-    const installStderr = pipInstall.stderr ?? "";
-    if (installStderr && (installStderr.includes("ERROR") || installStderr.includes("FAILED"))) {
-        throw new Error(`pip install failed: ${installStderr.substring(0, 300)}`);
+    if (!shouldSkipPipInstall()) {
+        const pipInstall = await runDatasetSandboxCommandStep({
+            env,
+            sandboxId,
+            cmd: "python",
+            // NOTE: pandas needs openpyxl to read .xlsx files.
+            args: ["-m", "pip", "install", "pandas", "openpyxl", "--quiet", "--upgrade"],
+        });
+        const installStderr = pipInstall.stderr ?? "";
+        if (installStderr && (installStderr.includes("ERROR") || installStderr.includes("FAILED"))) {
+            throw new Error(`pip install failed: ${installStderr.substring(0, 300)}`);
+        }
     }
     const prepared = [];
     for (let i = 0; i < sources.length; i++) {
         const src = sources[i];
         if (src.kind === "file") {
+            const basePath = `${workstation}/file_${i}_${src.fileId}`;
+            const existingPath = await sandboxFindFirstMatch(env, sandboxId, `${basePath}*`);
+            if (existingPath) {
+                prepared.push({ kind: "file", id: src.fileId, path: existingPath });
+                continue;
+            }
             const file = await readInstantFileStep({ env, fileId: src.fileId });
             const fileName = String(file.contentDisposition ?? "");
             const ext = fileName.includes(".") ? fileName.substring(fileName.lastIndexOf(".")) : "";
-            const path = `${workstation}/file_${i}_${src.fileId}${ext}`;
+            const path = `${basePath}${ext}`;
             await writeDatasetSandboxFilesStep({
                 env,
                 sandboxId,
@@ -88,16 +178,19 @@ async function ensureSandboxPrepared(params) {
             continue;
         }
         if (src.kind === "dataset") {
-            const content = await structureReadRowsOutputJsonlStep({ env, structureId: src.datasetId });
-            if (!content.ok) {
-                throw new Error(content.error);
-            }
             const path = `${workstation}/dataset_${src.datasetId}.jsonl`;
-            await writeDatasetSandboxFilesStep({
-                env,
-                sandboxId,
-                files: [{ path, contentBase64: content.data.contentBase64 }],
-            });
+            const exists = await sandboxFileExists(env, sandboxId, path);
+            if (!exists) {
+                const content = await structureReadRowsOutputJsonlStep({ env, structureId: src.datasetId });
+                if (!content.ok) {
+                    throw new Error(content.error);
+                }
+                await writeDatasetSandboxFilesStep({
+                    env,
+                    sandboxId,
+                    files: [{ path, contentBase64: content.data.contentBase64 }],
+                });
+            }
             prepared.push({ kind: "dataset", id: src.datasetId, path });
             continue;
         }
@@ -105,7 +198,10 @@ async function ensureSandboxPrepared(params) {
             const ext = guessTextFileExtension(src.mimeType, src.name);
             const textId = `text_${i}`;
             const path = `${workstation}/${textId}${ext}`;
-            await writeDatasetSandboxTextFileStep({ env, sandboxId, path, text: String(src.text ?? "") });
+            const exists = await sandboxFileExists(env, sandboxId, path);
+            if (!exists) {
+                await writeDatasetSandboxTextFileStep({ env, sandboxId, path, text: String(src.text ?? "") });
+            }
             prepared.push({ kind: "text", id: textId, path, name: src.name, mimeType: src.mimeType });
             continue;
         }
@@ -149,6 +245,8 @@ async function readSchemaFromSandboxIfPresent(params) {
 function createStructureStoryDefinition(config) {
     const datasetId = config.datasetId;
     const model = config.model ?? "openai/gpt-5.2";
+    const defaultSandboxConfig = getDefaultSandboxConfig(datasetId);
+    const resolvedSandboxConfig = mergeSandboxConfig(defaultSandboxConfig, config.sandboxConfig);
     const story = createStory("ekairos.structure")
         .context(async (stored, env) => {
         const prev = stored?.content ?? {};
@@ -156,7 +254,10 @@ function createStructureStoryDefinition(config) {
         const existingSandboxId = prev.sandboxId ?? config.sandboxId ?? "";
         let sandboxId = existingSandboxId;
         if (!sandboxId) {
-            const created = await createDatasetSandboxStep({ env, runtime: "python3.13", timeoutMs: 10 * 60 * 1000 });
+            const created = await createDatasetSandboxStep({
+                env,
+                ...resolvedSandboxConfig,
+            });
             sandboxId = created.sandboxId;
         }
         const { preparedSources, workstation, outputPath } = await ensureSandboxPrepared({
@@ -176,6 +277,13 @@ function createStructureStoryDefinition(config) {
             sources: preparedSources,
             workstation,
             outputPath,
+            sandboxProvider: resolvedSandboxConfig.provider ?? "daytona",
+            sandboxRuntime: resolvedSandboxConfig.runtime ?? "python3.13",
+            sandboxEphemeral: resolvedSandboxConfig.daytona?.ephemeral ?? true,
+            sandboxVolumeName: resolvedSandboxConfig.daytona?.volumes?.[0]?.volumeName,
+            sandboxVolumeMountPath: resolvedSandboxConfig.daytona?.volumes?.[0]?.mountPath,
+            sandboxSnapshot: resolvedSandboxConfig.daytona?.snapshot,
+            sandboxImage: resolvedSandboxConfig.daytona?.image,
         };
         const contextKey = `structure:${datasetId}`;
         // IMPORTANT:
@@ -274,6 +382,7 @@ export function structure(env, opts) {
     let mode = "auto";
     let output = "rows";
     let outputSchema;
+    const sandboxConfig = opts?.sandboxConfig;
     const api = {
         datasetId,
         from(...src) {
@@ -314,6 +423,7 @@ export function structure(env, opts) {
                 mode,
                 output,
                 outputSchema,
+                sandboxConfig,
             };
             const { story } = createStructureStoryDefinition(storyConfig);
             function makeUserMessageEvent(text) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/structure",
-  "version": "1.21.74-beta.0",
+  "version": "1.21.77-beta.0",
   "description": "Ekairos Structure - Unified structured extraction (rows or object) from file/text/dataset inputs",
   "type": "module",
   "main": "dist/index.js",
@@ -36,7 +36,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.21.74-beta.0",
+    "@ekairos/domain": "^1.21.77-beta.0",
     "@ekairos/sandbox": "^1.21.60-beta.0",
     "@instantdb/admin": "^0.22.13",
     "@instantdb/core": "^0.22.13",

package/dist/datasetReader.d.ts

@@ -1,16 +0,0 @@
-import { type StructureRowsOutputPagingCursor, type StructureRowsOutputSandboxRef } from "./rowsOutputPaging";
-export declare function datasetReader(env: any, opts: {
-    datasetId: string;
-    sandboxId?: string;
-    runtime?: string;
-    timeoutMs?: number;
-}): {
-    datasetId: string;
-    download(): Promise<StructureRowsOutputSandboxRef>;
-    readPage(params: {
-        sandboxId: string;
-        localPath: string;
-        cursor?: Partial<StructureRowsOutputPagingCursor>;
-        limit: number;
-    }): Promise<import("./rowsOutputPaging").StructureReadRowsOutputPageFromSandboxResult>;
-};

package/dist/datasetReader.js

@@ -1,25 +0,0 @@
-import { structureDownloadRowsOutputToSandboxStep, structureReadRowsOutputPageFromSandboxStep, } from "./rowsOutputPaging";
-export function datasetReader(env, opts) {
-    const datasetId = opts.datasetId;
-    return {
-        datasetId,
-        async download() {
-            return await structureDownloadRowsOutputToSandboxStep({
-                env,
-                structureId: datasetId,
-                sandboxId: opts.sandboxId,
-                runtime: opts.runtime,
-                timeoutMs: opts.timeoutMs,
-            });
-        },
-        async readPage(params) {
-            return await structureReadRowsOutputPageFromSandboxStep({
-                env,
-                sandboxId: params.sandboxId,
-                localPath: params.localPath,
-                cursor: params.cursor,
-                limit: params.limit,
-            });
-        },
-    };
-}

package/dist/rowsOutputPaging.steps.d.ts

@@ -1,37 +0,0 @@
-export type RowsOutputSandboxDownloadResult = {
-    sandboxId: string;
-    localPath: string;
-};
-export type RowsOutputChunkResult<T = any> = {
-    rows: T[];
-    nextByteOffset: number;
-    nextRowOffset: number;
-    done: boolean;
-};
-/**
- * Step 1/2:
- * Download the rows output.jsonl from Instant storage into a sandbox file.
- *
- * This isolates network flakiness (e.g. undici `TypeError: terminated`) into a single step
- * and makes subsequent reads purely sandbox-local.
- */
-export declare function structureDownloadRowsOutputToSandboxStep(params: {
-    env: any;
-    structureId: string;
-    runtime?: string;
-    timeoutMs?: number;
-}): Promise<RowsOutputSandboxDownloadResult>;
-/**
- * Step 2/2:
- * Read the next chunk of ROW records from the sandbox-local output.jsonl, bounded by `limit`.
- *
- * Pagination state is passed explicitly via `{ byteOffset, rowOffset }` and returned as next offsets.
- */
-export declare function structureReadRowsOutputChunkStep<T = any>(params: {
-    env: any;
-    sandboxId: string;
-    localPath: string;
-    byteOffset: number;
-    rowOffset: number;
-    limit: number;
-}): Promise<RowsOutputChunkResult<T>>;

package/dist/rowsOutputPaging.steps.js

@@ -1,125 +0,0 @@
-import { getDatasetOutputPath, getDatasetWorkstation } from "./datasetFiles";
-import { createDatasetSandboxStep, runDatasetSandboxCommandStep } from "./sandbox/steps";
-import { getStoryRuntime } from "./runtime";
-/**
- * Step 1/2:
- * Download the rows output.jsonl from Instant storage into a sandbox file.
- *
- * This isolates network flakiness (e.g. undici `TypeError: terminated`) into a single step
- * and makes subsequent reads purely sandbox-local.
- */
-export async function structureDownloadRowsOutputToSandboxStep(params) {
-    "use step";
-    const runtime = params.runtime ?? "python3.13";
-    const timeoutMs = params.timeoutMs ?? 10 * 60 * 1000;
-    const { sandboxId } = await createDatasetSandboxStep({
-        env: params.env,
-        runtime,
-        timeoutMs,
-        purpose: "structure.rows-output.reader",
-        params: { structureId: params.structureId },
-    });
-    const workstation = getDatasetWorkstation(params.structureId);
-    const localPath = getDatasetOutputPath(params.structureId);
-    await runDatasetSandboxCommandStep({
-        env: params.env,
-        sandboxId,
-        cmd: "mkdir",
-        args: ["-p", workstation],
-    });
-    const storyRuntime = await getStoryRuntime(params.env);
-    const db = storyRuntime.db;
-    const contextKey = `structure:${params.structureId}`;
-    const query = (await db.query({
-        context_contexts: {
-            $: { where: { key: contextKey }, limit: 1 },
-            structure_output_file: {},
-        },
-    }));
-    const ctx = query.context_contexts?.[0];
-    const linked = Array.isArray(ctx?.structure_output_file) ? ctx.structure_output_file[0] : ctx.structure_output_file;
-    const url = linked?.url;
-    if (!url) {
-        throw new Error("Rows output file not found");
-    }
-    const py = [
-        "import sys, urllib.request",
-        "url = sys.argv[1]",
-        "out_path = sys.argv[2]",
-        "with urllib.request.urlopen(url) as r:",
-        "  data = r.read()",
-        "with open(out_path, 'wb') as f:",
-        "  f.write(data)",
-        "print('ok', len(data))",
-    ].join("\n");
-    const res = await runDatasetSandboxCommandStep({
-        env: params.env,
-        sandboxId,
-        cmd: "python",
-        args: ["-c", py, String(url), localPath],
-    });
-    if (res.exitCode !== 0) {
-        throw new Error(res.stderr || "Failed to download rows output to sandbox");
-    }
-    return { sandboxId, localPath };
-}
-/**
- * Step 2/2:
- * Read the next chunk of ROW records from the sandbox-local output.jsonl, bounded by `limit`.
- *
- * Pagination state is passed explicitly via `{ byteOffset, rowOffset }` and returned as next offsets.
- */
-export async function structureReadRowsOutputChunkStep(params) {
-    "use step";
-    const py = [
-        "import sys, json",
-        "path = sys.argv[1]",
-        "byte_offset = int(sys.argv[2])",
-        "row_offset = int(sys.argv[3])",
-        "limit = int(sys.argv[4])",
-        "rows = []",
-        "next_byte = byte_offset",
-        "next_row = row_offset",
-        "with open(path, 'rb') as f:",
-        "  f.seek(byte_offset)",
-        "  while len(rows) < limit:",
-        "    line = f.readline()",
-        "    if not line:",
-        "      break",
-        "    next_byte = f.tell()",
-        "    try:",
-        "      obj = json.loads(line.decode('utf-8'))",
-        "    except Exception:",
-        "      continue",
-        "    if obj.get('type') != 'row':",
-        "      continue",
-        "    rows.append(obj.get('data'))",
-        "    next_row += 1",
-        "done = len(rows) < limit",
-        "print(json.dumps({",
-        "  'rows': rows,",
-        "  'nextByteOffset': next_byte,",
-        "  'nextRowOffset': next_row,",
-        "  'done': done,",
-        "}))",
-    ].join("\n");
-    const res = await runDatasetSandboxCommandStep({
-        env: params.env,
-        sandboxId: params.sandboxId,
-        cmd: "python",
-        args: [
-            "-c",
-            py,
-            params.localPath,
-            String(params.byteOffset ?? 0),
-            String(params.rowOffset ?? 0),
-            String(params.limit),
-        ],
-    });
-    if (res.exitCode !== 0) {
-        throw new Error(res.stderr || "Failed to read rows chunk from sandbox");
-    }
-    const out = String(res.stdout ?? "").trim();
-    const parsed = JSON.parse(out);
-    return parsed;
-}

package/dist/rowsPagination.steps.d.ts

@@ -1,59 +0,0 @@
-import { type DatasetSandboxId } from "./sandbox/steps";
-/**
- * Step 1: Download Structure rows output file (output.jsonl) into a sandbox.
- *
- * This enables pagination by reading chunks from the local sandbox filesystem.
- */
-export declare function downloadStructureRowsOutputToSandboxStep(params: {
-    env: any;
-    sandboxId: DatasetSandboxId;
-    structureId: string;
-}): Promise<{
-    filePath: string;
-}>;
-/**
- * @deprecated Prefer `downloadStructureRowsOutputToSandboxStep` (kept for symmetry with chunk naming).
- *
- * Note: The name includes `RowsOutputJsonl` to be explicit about the file format.
- */
-export declare function structureDownloadRowsOutputJsonlToSandboxStep(params: {
-    env: any;
-    sandboxId: DatasetSandboxId;
-    structureId: string;
-}): Promise<{
-    filePath: string;
-}>;
-/**
- * Step 2: Read a chunk/page of JSONL records from the downloaded sandbox file.
- *
- * Naming (consistent):
- * - `structureReadRowsOutputJsonlStep` reads the whole file (base64)
- * - `structureReadRowsOutputJsonlChunkStep` reads a paginated chunk from sandbox
- *
- * Offset/limit are line-based (0-indexed).
- */
-export declare function structureReadRowsOutputJsonlChunkStep(params: {
-    env: any;
-    sandboxId: DatasetSandboxId;
-    structureId: string;
-    offset: number;
-    limit: number;
-}): Promise<{
-    records: any[];
-    nextOffset: number;
-    done: boolean;
-}>;
-/**
- * @deprecated Use `structureReadRowsOutputJsonlChunkStep` instead (naming consistency).
- */
-export declare function readStructureRowsChunkFromSandboxStep(params: {
-    env: any;
-    sandboxId: DatasetSandboxId;
-    structureId: string;
-    offset: number;
-    limit: number;
-}): Promise<{
-    records: any[];
-    nextOffset: number;
-    done: boolean;
-}>;

package/dist/rowsPagination.steps.js

@@ -1,190 +0,0 @@
-import { getDatasetWorkstation } from "./datasetFiles";
-import { runDatasetSandboxCommandStep, writeDatasetSandboxTextFileStep, } from "./sandbox/steps";
-async function getRowsOutputUrl(params) {
-    const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
-    const runtime = (await resolveStoryRuntime(params.env));
-    const db = runtime.db;
-    const contextKey = `structure:${params.structureId}`;
-    const query = (await db.query({
-        context_contexts: {
-            $: { where: { key: contextKey }, limit: 1 },
-            structure_output_file: {},
-        },
-    }));
-    const ctx = query.context_contexts?.[0];
-    const linked = Array.isArray(ctx?.structure_output_file) ? ctx.structure_output_file[0] : ctx?.structure_output_file;
-    const url = linked?.url;
-    if (!url) {
-        throw new Error("Rows output file not found");
-    }
-    return String(url);
-}
-/**
- * Step 1: Download Structure rows output file (output.jsonl) into a sandbox.
- *
- * This enables pagination by reading chunks from the local sandbox filesystem.
- */
-export async function downloadStructureRowsOutputToSandboxStep(params) {
-    "use step";
-    const workstation = getDatasetWorkstation(params.structureId);
-    const filePath = `${workstation}/rows_output.jsonl`;
-    const scriptPath = `${workstation}/download_rows_output.py`;
-    // Ensure directory exists
-    await runDatasetSandboxCommandStep({
-        env: params.env,
-        sandboxId: params.sandboxId,
-        cmd: "mkdir",
-        args: ["-p", workstation],
-    });
-    const url = await getRowsOutputUrl({ env: params.env, structureId: params.structureId });
-    // Write a deterministic script to the sandbox and run it (no external deps).
-    await writeDatasetSandboxTextFileStep({
-        env: params.env,
-        sandboxId: params.sandboxId,
-        path: scriptPath,
-        text: [
-            "import argparse",
-            "import urllib.request",
-            "",
-            "def main():",
-            "  p = argparse.ArgumentParser()",
-            "  p.add_argument('--url', required=True)",
-            "  p.add_argument('--out', required=True)",
-            "  args = p.parse_args()",
-            "",
-            "  # Download to local sandbox file",
-            "  with urllib.request.urlopen(args.url, timeout=60) as r:",
-            "    data = r.read()",
-            "  with open(args.out, 'wb') as f:",
-            "    f.write(data)",
-            "",
-            "  print('ok')",
-            "  print('bytes', len(data))",
-            "",
-            "if __name__ == '__main__':",
-            "  main()",
-            "",
-        ].join("\n"),
-    });
-    const res = await runDatasetSandboxCommandStep({
-        env: params.env,
-        sandboxId: params.sandboxId,
-        cmd: "python",
-        args: [scriptPath, "--url", url, "--out", filePath],
-    });
-    if (res.exitCode !== 0) {
-        throw new Error(res.stderr || "Failed to download rows output into sandbox");
-    }
-    return { filePath };
-}
-/**
- * @deprecated Prefer `downloadStructureRowsOutputToSandboxStep` (kept for symmetry with chunk naming).
- *
- * Note: The name includes `RowsOutputJsonl` to be explicit about the file format.
- */
-export async function structureDownloadRowsOutputJsonlToSandboxStep(params) {
-    "use step";
-    return await downloadStructureRowsOutputToSandboxStep(params);
-}
-/**
- * Step 2: Read a chunk/page of JSONL records from the downloaded sandbox file.
- *
- * Naming (consistent):
- * - `structureReadRowsOutputJsonlStep` reads the whole file (base64)
- * - `structureReadRowsOutputJsonlChunkStep` reads a paginated chunk from sandbox
- *
- * Offset/limit are line-based (0-indexed).
- */
-export async function structureReadRowsOutputJsonlChunkStep(params) {
-    "use step";
-    const workstation = getDatasetWorkstation(params.structureId);
-    const filePath = `${workstation}/rows_output.jsonl`;
-    const scriptPath = `${workstation}/read_rows_chunk.py`;
-    await writeDatasetSandboxTextFileStep({
-        env: params.env,
-        sandboxId: params.sandboxId,
-        path: scriptPath,
-        text: [
-            "import argparse",
-            "import json",
-            "",
-            "def main():",
-            "  p = argparse.ArgumentParser()",
-            "  p.add_argument('--path', required=True)",
-            "  p.add_argument('--offset', type=int, required=True)",
-            "  p.add_argument('--limit', type=int, required=True)",
-            "  args = p.parse_args()",
-            "",
-            "  records = []",
-            "  current = 0",
-            "  taken = 0",
-            "  done = True",
-            "",
-            "  with open(args.path, 'r', encoding='utf-8', errors='replace') as f:",
-            "    for line in f:",
-            "      if current < args.offset:",
-            "        current += 1",
-            "        continue",
-            "      if taken >= args.limit:",
-            "        done = False",
-            "        break",
-            "      line = line.strip()",
-            "      if not line:",
-            "        current += 1",
-            "        continue",
-            "      try:",
-            "        records.append(json.loads(line))",
-            "      except Exception:",
-            "        # Skip invalid JSON line",
-            "        pass",
-            "      taken += 1",
-            "      current += 1",
-            "",
-            "  out = {",
-            "    'records': records,",
-            "    'nextOffset': args.offset + taken,",
-            "    'done': done,",
-            "  }",
-            "  print(json.dumps(out, ensure_ascii=False))",
-            "",
-            "if __name__ == '__main__':",
-            "  main()",
-            "",
-        ].join("\n"),
-    });
-    const res = await runDatasetSandboxCommandStep({
-        env: params.env,
-        sandboxId: params.sandboxId,
-        cmd: "python",
-        args: [
-            scriptPath,
-            "--path",
-            filePath,
-            "--offset",
-            String(Math.max(0, params.offset ?? 0)),
-            "--limit",
-            String(Math.max(1, params.limit ?? 1)),
-        ],
-    });
-    if (res.exitCode !== 0) {
-        throw new Error(res.stderr || "Failed to read rows chunk from sandbox");
-    }
-    const text = (res.stdout ?? "").trim();
-    if (!text) {
-        return { records: [], nextOffset: params.offset, done: true };
-    }
-    // The script prints a single JSON object.
-    const parsed = JSON.parse(text);
-    return {
-        records: Array.isArray(parsed?.records) ? parsed.records : [],
-        nextOffset: Number(parsed?.nextOffset ?? params.offset),
-        done: Boolean(parsed?.done),
-    };
-}
-/**
- * @deprecated Use `structureReadRowsOutputJsonlChunkStep` instead (naming consistency).
- */
-export async function readStructureRowsChunkFromSandboxStep(params) {
-    "use step";
-    return await structureReadRowsOutputJsonlChunkStep(params);
-}