@ekairos/dataset 1.22.40-beta.development.0 → 1.22.40

package/dist/file/file-dataset.agent.js

@@ -1,103 +1,90 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createFileParseStory = createFileParseStory;
-const events_1 = require("@ekairos/events");
-const steps_1 = require("../sandbox/steps");
-const generateSchema_tool_1 = require("./generateSchema.tool");
-const completeDataset_tool_1 = require("../completeDataset.tool");
-const executeCommand_tool_1 = require("../executeCommand.tool");
-const clearDataset_tool_1 = require("../clearDataset.tool");
-const prompts_1 = require("./prompts");
-const filepreview_1 = require("./filepreview");
-const admin_1 = require("@instantdb/admin");
-const datasetFiles_1 = require("../datasetFiles");
-const steps_2 = require("./steps");
-const steps_3 = require("../dataset/steps");
-const eventsReactRuntime_1 = require("../eventsReactRuntime");
-async function initializeSandbox(env, sandboxId, datasetId, fileId, state) {
-    if (state.initialized) {
-        return state.filePath;
+import { INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/events";
+import { createContext } from "@ekairos/reactor/context";
+import { createClearDatasetTool } from "../clearDataset.tool.js";
+import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
+import { datasetGetByIdStep } from "../dataset/steps.js";
+import { createExecuteCommandTool } from "../executeCommand.tool.js";
+import { createDefineNotationTool } from "../defineNotation.tool.js";
+import { createGenerateSchemaTool } from "./generateSchema.tool.js";
+import { buildFileDatasetPromptStep, initializeFileParseSandboxStep, } from "./file-dataset.steps.js";
+import { createDatasetId } from "../id.js";
+async function awaitContextRun(run) {
+    if (!run)
+        return;
+    if (run.returnValue) {
+        await run.returnValue;
+        return;
     }
-    console.log(`[FileParseStory ${datasetId}] Initializing sandbox...`);
-    await (0, filepreview_1.ensurePreviewScriptsAvailable)(env, sandboxId);
-    console.log(`[FileParseStory ${datasetId}] Installing Python dependencies...`);
-    const pipInstall = await (0, steps_1.runDatasetSandboxCommandStep)({
-        env,
-        sandboxId,
-        cmd: "python",
-        args: ["-m", "pip", "install", "pandas", "openpyxl", "--quiet", "--upgrade"],
-    });
-    const installStderr = pipInstall.stderr;
-    if (installStderr && (installStderr.includes("ERROR") || installStderr.includes("FAILED"))) {
-        throw new Error(`pip install failed: ${installStderr.substring(0, 300)}`);
-    }
-    console.log(`[FileParseStory ${datasetId}] Fetching file from InstantDB...`);
-    const file = await (0, steps_2.readInstantFileStep)({ env, fileId });
-    console.log(`[FileParseStory ${datasetId}] Creating dataset workstation...`);
-    const workstation = (0, datasetFiles_1.getDatasetWorkstation)(datasetId);
-    await (0, steps_1.runDatasetSandboxCommandStep)({
-        env,
-        sandboxId,
-        cmd: "mkdir",
-        args: ["-p", workstation],
-    });
-    const fileName = file.contentDisposition ?? "";
-    const fileExtension = fileName.includes(".") ? fileName.substring(fileName.lastIndexOf(".")) : "";
-    const sandboxFilePath = `${workstation}/${fileId}${fileExtension}`;
-    await (0, steps_1.writeDatasetSandboxFilesStep)({
-        env,
-        sandboxId,
-        files: [
-            {
-                path: sandboxFilePath,
-                contentBase64: file.contentBase64,
-            },
-        ],
-    });
-    console.log(`[FileParseStory ${datasetId}] ✅ Workstation created: ${workstation}`);
-    console.log(`[FileParseStory ${datasetId}] ✅ File saved: ${sandboxFilePath}`);
-    state.filePath = sandboxFilePath;
-    state.initialized = true;
-    return sandboxFilePath;
+    await run;
 }
-/**
- * FileParseStory
- *
- * Uso:
- * - Crear una instancia con `fileId`, `instructions` y un `sandbox`
- * - Llamar `getDataset()` para crear un dataset nuevo (crea un datasetId interno)
- * - Llamar `followUp(datasetId, feedback)` para iterar el mismo dataset con feedback
- *
- * Internamente corre un Context (`createContext("file.parse")`) que itera hasta que se ejecuta el tool `completeDataset`.
- */
-function createFileParseStoryDefinition(params) {
-    const datasetId = params.datasetId ?? (0, admin_1.id)();
+function createFileParseContextDefinition(params) {
+    const fallbackDatasetId = params.datasetId;
     const model = params.model ?? "openai/gpt-5";
-    let storyBuilder = (0, events_1.createContext)("file.parse")
-        .context(async (stored, env) => {
+    let contextBuilder = createContext("file.parse")
+        .context(async (stored, _env, runtime) => {
         const previous = stored?.content ?? {};
-        const sandboxState = previous?.sandboxState ?? { initialized: false, filePath: "" };
+        const sandboxState = previous?.sandboxState ??
+            params.sandboxState ?? { initialized: false, filePath: "" };
+        const datasetId = previous?.datasetId ?? fallbackDatasetId ?? "";
+        const fileId = previous?.fileId ?? params.fileId ?? "";
+        const instructions = previous?.instructions ?? params.instructions ?? "";
         const sandboxId = previous?.sandboxId ?? params.sandboxId ?? "";
+        const contextRun = runtime?.__ekairosContextRun ?? {};
+        const contextId = String(contextRun.contextId ?? stored?.id ?? "").trim();
+        const executionId = String(contextRun.executionId ?? previous?.executionId ?? "").trim();
+        const sourceEventId = String(previous?.sourceEventId ?? params.sourceEventId ?? "").trim();
+        const sourcePartIndex = typeof previous?.sourcePartIndex === "number"
+            ? previous.sourcePartIndex
+            : typeof params.sourcePartIndex === "number"
+                ? params.sourcePartIndex
+                : 0;
+        if (!datasetId) {
+            throw new Error("dataset_id_required");
+        }
+        if (!fileId) {
+            throw new Error("dataset_file_id_required");
+        }
         if (!sandboxId) {
             throw new Error("dataset_sandbox_required");
         }
-        const sandboxFilePath = await initializeSandbox(env, sandboxId, datasetId, params.fileId, sandboxState);
-        let filePreview = undefined;
-        try {
-            filePreview = await (0, filepreview_1.generateFilePreview)(env, sandboxId, sandboxFilePath, datasetId);
+        if (!contextId) {
+            throw new Error("dataset_context_id_required");
         }
-        catch {
-            // optional
+        if (!executionId) {
+            throw new Error("dataset_execution_id_required");
         }
-        let schema = null;
-        const datasetResult = await (0, steps_3.datasetGetByIdStep)({ env, datasetId });
-        if (datasetResult.ok && datasetResult.data.schema)
+        const initialized = sandboxState.initialized && sandboxState.filePath
+            ? { filePath: sandboxState.filePath, state: sandboxState }
+            : await initializeFileParseSandboxStep({
+                runtime,
+                sandboxId,
+                contextId,
+                executionId,
+                datasetId,
+                fileId,
+                sourceEventId,
+                sourcePartIndex,
+                filename: previous?.filename ?? params.filename,
+                mediaType: previous?.mediaType ?? params.mediaType,
+                state: sandboxState,
+            });
+        const sandboxFilePath = initialized.filePath;
+        let filePreview = previous?.filePreview ?? previous?.ctx?.filePreview ?? params.filePreview;
+        let schema = previous?.ctx?.schema ?? previous?.schema ?? params.schema ?? null;
+        const datasetResult = await datasetGetByIdStep({ runtime, datasetId });
+        if (datasetResult.ok && datasetResult.data.schema) {
             schema = datasetResult.data.schema;
+        }
         const ctx = {
             datasetId,
-            fileId: params.fileId,
-            instructions: params.instructions ?? "",
-            sandboxConfig: { filePath: sandboxFilePath },
+            fileId,
+            instructions,
+            sandboxConfig: {
+                filePath: sandboxFilePath,
+                outputPath: initialized.state.outputPath,
+                scriptsDir: initialized.state.scriptsDir,
+                manifestPath: initialized.state.manifestPath,
+            },
             analysis: [],
             schema,
             plan: null,
@@ -109,16 +96,22 @@ function createFileParseStoryDefinition(params) {
         return {
             ...previous,
             datasetId,
-            fileId: params.fileId,
-            instructions: params.instructions ?? "",
+            fileId,
+            instructions,
             sandboxId,
-            sandboxState,
+            executionId,
+            sourceEventId,
+            sourcePartIndex,
+            filename: previous?.filename ?? params.filename,
+            mediaType: previous?.mediaType ?? params.mediaType,
+            sandboxState: initialized.state,
+            filePreview,
             ctx,
         };
     })
         .narrative(async (stored) => {
         const ctx = stored?.content?.ctx;
-        const base = (0, prompts_1.buildFileDatasetPrompt)(ctx);
+        const base = await buildFileDatasetPromptStep({ context: ctx });
         const userInstructions = String(ctx?.instructions ?? "").trim();
         if (!userInstructions)
             return base;
@@ -131,90 +124,143 @@ function createFileParseStoryDefinition(params) {
             base,
         ].join("\n");
     })
-        .actions(async (_stored, env) => {
+        .actions(async (_stored, _env, runtime) => {
         const existingSchema = _stored?.content?.ctx?.schema?.schema;
+        const datasetId = _stored?.content?.datasetId ?? fallbackDatasetId ?? "";
+        const fileId = _stored?.content?.fileId ?? params.fileId ?? "";
+        const sandboxId = _stored?.content?.sandboxId ?? params.sandboxId ?? "";
+        const outputPath = _stored?.content?.ctx?.sandboxConfig?.outputPath;
+        if (!datasetId)
+            throw new Error("dataset_id_required");
+        if (!fileId)
+            throw new Error("dataset_file_id_required");
+        if (!sandboxId)
+            throw new Error("dataset_sandbox_required");
         const actions = {
-            executeCommand: (0, executeCommand_tool_1.createExecuteCommandTool)({
+            executeCommand: createExecuteCommandTool({
                 datasetId,
-                sandboxId: _stored?.content?.sandboxId ?? params.sandboxId ?? "",
-                env,
+                sandboxId,
+                runtime,
             }),
-            completeDataset: (0, completeDataset_tool_1.createCompleteDatasetTool)({
+            completeDataset: createCompleteDatasetTool({
                 datasetId,
-                sandboxId: _stored?.content?.sandboxId ?? params.sandboxId ?? "",
-                env,
+                sandboxId,
+                runtime,
+                outputPath,
             }),
-            clearDataset: (0, clearDataset_tool_1.createClearDatasetTool)({
+            clearDataset: createClearDatasetTool({
                 datasetId,
-                sandboxId: _stored?.content?.sandboxId ?? params.sandboxId ?? "",
-                env,
+                sandboxId,
+                runtime,
+            }),
+            defineNotation: createDefineNotationTool({
+                datasetId,
+                runtime,
             }),
         };
         if (!existingSchema) {
-            actions.generateSchema = (0, generateSchema_tool_1.createGenerateSchemaTool)({
+            actions.generateSchema = createGenerateSchemaTool({
                 datasetId,
-                fileId: params.fileId,
-                env,
+                fileId,
+                runtime,
             });
         }
         return actions;
     })
         .shouldContinue(({ reactionEvent }) => {
-        return !(0, events_1.didToolExecute)(reactionEvent, "completeDataset");
+        const fatalFailure = getDatasetFatalFailure(reactionEvent);
+        if (fatalFailure) {
+            throw new Error(fatalFailure);
+        }
+        return !didCompleteDatasetSucceed(reactionEvent);
     });
     if (params.reactor) {
-        storyBuilder = storyBuilder.reactor(params.reactor);
+        contextBuilder = contextBuilder.reactor(params.reactor);
     }
     else {
-        storyBuilder = storyBuilder.model(model);
+        contextBuilder = contextBuilder.model(model);
     }
-    const story = storyBuilder.build();
-    return { datasetId, story };
+    const context = contextBuilder.build();
+    return { datasetId: fallbackDatasetId ?? "", context };
 }
-/**
- * Factory (DX-first):
- *
- * Usage:
- *   const { datasetId } = await createFileParseStory(fileId, { instructions }).parse(env)
- *
- * - No `db` is accepted/stored (workflow-safe).
- * - All I/O happens in `"use step"` functions via Ekairos runtime (`getContextRuntime(env).db`).
- * - `parse()` is the entrypoint; it calls `story.react(...)` internally.
- */
-function createFileParseStory(fileId, opts) {
+export function createFileParseContext(fileId, opts) {
+    const datasetId = opts?.datasetId ?? createDatasetId();
     const params = {
         fileId,
         instructions: opts?.instructions,
         sandboxId: opts?.sandboxId,
-        datasetId: opts?.datasetId,
+        datasetId,
         model: opts?.model,
         reactor: opts?.reactor,
+        sandboxState: opts?.sandboxState,
+        filePreview: opts?.filePreview,
+        schema: opts?.schema,
+        filename: opts?.filename,
+        mediaType: opts?.mediaType,
     };
-    const { datasetId, story } = createFileParseStoryDefinition(params);
+    const { context } = createFileParseContextDefinition(params);
     return {
         datasetId,
-        async parse(env, prompt) {
+        async parse(runtime, options = {}) {
             const triggerEvent = {
-                id: (0, admin_1.id)(),
-                type: events_1.INPUT_TEXT_ITEM_TYPE,
-                channel: events_1.WEB_CHANNEL,
+                id: createDatasetId(),
+                type: INPUT_TEXT_ITEM_TYPE,
+                channel: WEB_CHANNEL,
                 createdAt: new Date().toISOString(),
                 content: {
-                    parts: [{ type: "text", text: prompt ?? "generate a dataset for this file" }],
+                    parts: [
+                        {
+                            type: "text",
+                            text: options.prompt ?? "generate a dataset for this file",
+                        },
+                        {
+                            type: "file",
+                            fileId,
+                            filename: opts?.filename ?? "resource-file",
+                            mediaType: opts?.mediaType ?? "application/octet-stream",
+                        },
+                    ],
                 },
             };
-            const runtime = (0, eventsReactRuntime_1.createEventsReactRuntime)((env ?? {}));
-            const shell = await story.react(triggerEvent, {
-                runtime,
+            params.sourceEventId = triggerEvent.id;
+            params.sourcePartIndex = 1;
+            params.filename = opts?.filename ?? "resource-file";
+            params.mediaType = opts?.mediaType ?? "application/octet-stream";
+            const shell = await context.react(triggerEvent, {
+                runtime: runtime,
                 context: { key: `dataset:${datasetId}` },
-                durable: false,
-                options: { silent: true, preventClose: true, sendFinish: false, maxIterations: 20, maxModelSteps: 5 },
+                durable: options.durable ?? false,
+                options: {
+                    preventClose: true,
+                    sendFinish: false,
+                    maxIterations: 20,
+                    maxModelSteps: 5,
+                },
+                __initialContent: {
+                    ...(options.initialContent ?? {}),
+                    datasetId,
+                    fileId,
+                    sourceEventId: triggerEvent.id,
+                    sourcePartIndex: 1,
+                    filename: opts?.filename ?? "resource-file",
+                    mediaType: opts?.mediaType ?? "application/octet-stream",
+                    instructions: opts?.instructions ?? "",
+                    sandboxId: opts?.sandboxId ?? "",
+                    sandboxState: opts?.sandboxState ?? { initialized: false, filePath: "" },
+                    filePreview: opts?.filePreview,
+                    schema: opts?.schema,
+                },
             });
-            await shell.run;
+            await awaitContextRun(shell.run);
             return { datasetId };
         },
-        // Optional: expose the built story for advanced callers (not required for parse DX)
-        story,
+        context,
     };
 }
-//# sourceMappingURL=file-dataset.agent.js.map
+export function registerFileParseContext(opts) {
+    createFileParseContextDefinition({
+        model: opts?.model,
+        reactor: opts?.reactor,
+    }).context;
+}
+registerFileParseContext();

package/dist/file/file-dataset.steps.d.ts

@@ -0,0 +1,27 @@
+import type { FileParseContext, SandboxState } from "./file-dataset.types.js";
+import type { FilePreviewContext } from "./filepreview.types.js";
+export declare function initializeFileParseSandboxStep(params: {
+    runtime: any;
+    sandboxId: string;
+    contextId: string;
+    executionId: string;
+    datasetId: string;
+    fileId: string;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+    filename?: string;
+    mediaType?: string;
+    state: SandboxState;
+}): Promise<{
+    filePath: string;
+    state: SandboxState;
+}>;
+export declare function generateFileParsePreviewStep(params: {
+    runtime: any;
+    sandboxId: string;
+    sandboxFilePath: string;
+    datasetId: string;
+}): Promise<FilePreviewContext>;
+export declare function buildFileDatasetPromptStep(params: {
+    context: FileParseContext;
+}): Promise<string>;

package/dist/file/file-dataset.steps.js

@@ -0,0 +1,47 @@
+import { DATASET_OUTPUT_FILE_NAME } from "../datasetFiles.js";
+import { prepareContextExecutionWorkspaceStep } from "../contextWorkspace.js";
+import { buildFileDatasetPrompt } from "./prompts.js";
+import { generateFilePreview } from "./filepreview.js";
+export async function initializeFileParseSandboxStep(params) {
+    "use step";
+    if (params.state.initialized) {
+        return { filePath: params.state.filePath, state: params.state };
+    }
+    console.log(`[FileParseContext ${params.datasetId}] Preparing context execution workspace...`);
+    const workspace = await prepareContextExecutionWorkspaceStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        contextId: params.contextId,
+        executionId: params.executionId,
+        files: [
+            {
+                fileId: params.fileId,
+                filename: params.filename,
+                mediaType: params.mediaType,
+                sourceEventId: params.sourceEventId,
+                sourcePartIndex: params.sourcePartIndex,
+            },
+        ],
+    });
+    const sandboxFilePath = workspace.files[0]?.path ?? "";
+    if (!sandboxFilePath)
+        throw new Error("dataset_workspace_file_missing");
+    console.log(`[FileParseContext ${params.datasetId}] Context workspace created: ${workspace.root}`);
+    console.log(`[FileParseContext ${params.datasetId}] File saved: ${sandboxFilePath}`);
+    const state = {
+        initialized: true,
+        filePath: sandboxFilePath,
+        outputPath: `${workspace.outputDir}/${DATASET_OUTPUT_FILE_NAME}`,
+        scriptsDir: workspace.scriptsDir,
+        manifestPath: workspace.manifestPath,
+    };
+    return { filePath: sandboxFilePath, state };
+}
+export async function generateFileParsePreviewStep(params) {
+    "use step";
+    return await generateFilePreview(params.runtime, params.sandboxId, params.sandboxFilePath, params.datasetId);
+}
+export async function buildFileDatasetPromptStep(params) {
+    "use step";
+    return buildFileDatasetPrompt(params.context);
+}

package/dist/file/file-dataset.types.d.ts

@@ -0,0 +1,64 @@
+import type { ContextReactor } from "@ekairos/reactor/context";
+import type { FilePreviewContext } from "./filepreview.types.js";
+export type SandboxState = {
+    initialized: boolean;
+    filePath: string;
+    outputPath?: string;
+    scriptsDir?: string;
+    manifestPath?: string;
+};
+export type FileParseContext = {
+    datasetId: string;
+    fileId: string;
+    instructions: string;
+    sandboxConfig: {
+        filePath: string;
+        outputPath?: string;
+        scriptsDir?: string;
+        manifestPath?: string;
+    };
+    analysis: any[];
+    schema: any | null;
+    plan: any | null;
+    executionResult: any | null;
+    errors: string[];
+    iterationCount: number;
+    filePreview?: FilePreviewContext;
+};
+export type FileParseContextParams = {
+    fileId?: string;
+    instructions?: string;
+    sandboxId?: string;
+    datasetId?: string;
+    model?: string;
+    reactor?: ContextReactor<any, any>;
+    sandboxState?: SandboxState;
+    filePreview?: FilePreviewContext;
+    schema?: any | null;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+    filename?: string;
+    mediaType?: string;
+};
+export type FileParseRunOptions = {
+    prompt?: string;
+    durable?: boolean;
+    initialContent?: Record<string, any>;
+};
+export type FileParseContextBuilder<Env extends {
+    orgId: string;
+}> = {
+    datasetId: string;
+    context: any;
+};
+export type DatasetResult = {
+    id: string;
+    status?: string;
+    title?: string;
+    schema?: any;
+    analysis?: any;
+    calculatedTotalRows?: number;
+    actualGeneratedRowCount?: number;
+    createdAt?: number;
+    updatedAt?: number;
+};

package/dist/file/file-dataset.types.js

@@ -0,0 +1 @@
+export {};

package/dist/file/filepreview.d.ts

@@ -1,40 +1,10 @@
-export type FilePreviewContext = {
-    totalRows: number;
-    metadata?: {
-        description: string;
-        script: string;
-        command: string;
-        stdout: string;
-        stderr: string;
-    };
-    head?: {
-        description: string;
-        script: string;
-        command: string;
-        stdout: string;
-        stderr: string;
-    };
-    tail?: {
-        description: string;
-        script: string;
-        command: string;
-        stdout: string;
-        stderr: string;
-    };
-    mid?: {
-        description: string;
-        script: string;
-        command: string;
-        stdout: string;
-        stderr: string;
-    };
-};
+import type { FilePreviewContext } from "./filepreview.types.js";
+export type { FilePreviewContext } from "./filepreview.types.js";
 interface PreviewOptions {
     headLines?: number;
     tailLines?: number;
     midLines?: number;
 }
-export declare function ensurePreviewScriptsAvailable(env: any, sandboxId: string): Promise<void>;
-export declare function generateFilePreview(env: any, sandboxId: string, sandboxFilePath: string, datasetId: string, options?: PreviewOptions): Promise<FilePreviewContext>;
-export {};
-//# sourceMappingURL=filepreview.d.ts.map
+export declare function getEmbeddedFilePreviewScriptBase64(scriptName: string): string;
+export declare function ensurePreviewScriptsAvailable(_runtime: any, _sandboxId: string): Promise<void>;
+export declare function generateFilePreview(runtime: any, sandboxId: string, sandboxFilePath: string, datasetId: string, options?: PreviewOptions): Promise<FilePreviewContext>;