@ekairos/dataset 1.22.36-beta.development.0 → 1.22.36

package/dist/transform/transform-dataset.agent.js

@@ -1,77 +1,76 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createTransformDatasetStory = createTransformDatasetStory;
-const events_1 = require("@ekairos/events");
-const completeDataset_tool_1 = require("../completeDataset.tool");
-const executeCommand_tool_1 = require("../executeCommand.tool");
-const clearDataset_tool_1 = require("../clearDataset.tool");
-const prompts_1 = require("./prompts");
-const datasetFiles_1 = require("../datasetFiles");
-const admin_1 = require("@instantdb/admin");
-const filepreview_1 = require("./filepreview");
-const steps_1 = require("../dataset/steps");
-const steps_2 = require("../sandbox/steps");
-const eventsReactRuntime_1 = require("../eventsReactRuntime");
-async function ensureSourcesInSandbox(env, sandboxId, datasetId, sourceDatasetIds, state) {
-    if (state.initialized) {
-        return { sourcePaths: state.sourcePaths, outputPath: (0, datasetFiles_1.getDatasetOutputPath)(datasetId) };
+import { createContext, INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/events";
+import { createClearDatasetTool } from "../clearDataset.tool.js";
+import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
+import { datasetUpdateSchemaStep } from "../dataset/steps.js";
+import { getDatasetOutputPath } from "../datasetFiles.js";
+import { createExecuteCommandTool } from "../executeCommand.tool.js";
+import { createCompleteObjectTool, createReplaceRowsTool, } from "../writeDatasetRows.tool.js";
+import { buildTransformDatasetPromptStep, } from "./transform-dataset.steps.js";
+import { createDatasetId } from "../id.js";
+async function awaitContextRun(run) {
+    if (!run)
+        return;
+    if (run.returnValue) {
+        await run.returnValue;
+        return;
     }
-    const workstation = (0, datasetFiles_1.getDatasetWorkstation)(datasetId);
-    await (0, steps_2.runDatasetSandboxCommandStep)({ env, sandboxId, cmd: "mkdir", args: ["-p", workstation] });
-    const sourcePaths = [];
-    for (const sourceDatasetId of sourceDatasetIds) {
-        const sourcePath = `${workstation}/source_${sourceDatasetId}.jsonl`;
-        const source = await (0, steps_1.datasetReadOutputJsonlStep)({ env, datasetId: sourceDatasetId });
-        await (0, steps_2.writeDatasetSandboxFilesStep)({
-            env,
-            sandboxId,
-            files: [{ path: sourcePath, contentBase64: source.contentBase64 }],
-        });
-        sourcePaths.push({ datasetId: sourceDatasetId, path: sourcePath });
-    }
-    state.sourcePaths = sourcePaths;
-    state.initialized = true;
-    return { sourcePaths, outputPath: (0, datasetFiles_1.getDatasetOutputPath)(datasetId) };
+    await run;
 }
-function createTransformDatasetStoryDefinition(params) {
-    const datasetId = params.datasetId ?? (0, admin_1.id)();
+function createTransformDatasetContextDefinition(params) {
+    const fallbackDatasetId = params.datasetId;
     const model = params.model ?? "openai/gpt-5";
-    let storyBuilder = (0, events_1.createContext)("dataset.transform")
-        .context(async (stored, env) => {
+    let contextBuilder = createContext("dataset.transform")
+        .context(async (stored, _env, runtime) => {
         const previous = stored?.content ?? {};
-        const sandboxState = previous?.sandboxState ?? { initialized: false, sourcePaths: [] };
+        const sandboxState = previous?.sandboxState ??
+            params.sandboxState ?? { initialized: false, inputPaths: [] };
+        const datasetId = previous?.datasetId ?? fallbackDatasetId ?? "";
+        const inputDatasetIds = Array.isArray(previous?.inputDatasetIds)
+            ? previous.inputDatasetIds
+            : Array.isArray(params.inputDatasetIds)
+                ? params.inputDatasetIds
+                : [];
+        const outputSchema = previous?.outputSchema ?? params.outputSchema;
+        const instructions = previous?.instructions ?? params.instructions;
         const sandboxId = previous?.sandboxId ?? params.sandboxId ?? "";
+        if (!datasetId) {
+            throw new Error("dataset_id_required");
+        }
+        if (inputDatasetIds.length === 0) {
+            throw new Error("dataset_transform_inputs_required");
+        }
+        if (!outputSchema) {
+            throw new Error("dataset_transform_schema_required");
+        }
         if (!sandboxId) {
             throw new Error("dataset_sandbox_required");
         }
-        const { sourcePaths, outputPath } = await ensureSourcesInSandbox(env, sandboxId, datasetId, params.sourceDatasetIds, sandboxState);
-        const sourcePreviews = [];
-        for (const sp of sourcePaths) {
-            try {
-                const preview = await (0, filepreview_1.generateSourcePreview)(env, sandboxId, sp.path, datasetId);
-                sourcePreviews.push({ datasetId: sp.datasetId, preview });
-            }
-            catch {
-                // optional
-            }
-        }
-        // Persist output schema on the dataset record (so completeDataset validates against it)
-        await (0, steps_1.datasetUpdateSchemaStep)({
-            env,
+        const initialized = sandboxState.initialized && Array.isArray(sandboxState.inputPaths)
+            ? sandboxState
+            : { initialized: false, inputPaths: [] };
+        const inputPreviews = previous?.inputPreviews ?? params.inputPreviews ?? [];
+        await datasetUpdateSchemaStep({
+            runtime,
             datasetId,
-            schema: params.outputSchema,
+            schema: outputSchema,
             status: "schema_complete",
         });
         const promptContext = {
             datasetId,
-            sourceDatasetIds: params.sourceDatasetIds,
-            outputSchema: params.outputSchema,
-            sandboxConfig: { sourcePaths, outputPath },
-            sourcePreviews: sourcePreviews.length > 0 ? sourcePreviews : undefined,
+            inputDatasetIds,
+            outputSchema,
+            sandboxConfig: {
+                inputPaths: initialized.inputPaths,
+                outputPath: previous?.sandboxConfig?.outputPath ?? getDatasetOutputPath(datasetId),
+            },
+            inputPreviews: inputPreviews.length > 0 ? inputPreviews : undefined,
+            contextResources: previous?.contextResources ?? params.contextResources ?? [],
             errors: [],
         };
-        const basePrompt = (0, prompts_1.buildTransformDatasetPrompt)(promptContext);
-        const userInstructions = String(params.instructions ?? "").trim();
+        const basePrompt = await buildTransformDatasetPromptStep({
+            context: promptContext,
+        });
+        const userInstructions = String(instructions ?? "").trim();
         const system = userInstructions
             ? [
                 "## USER INSTRUCTIONS",
@@ -85,87 +84,143 @@ function createTransformDatasetStoryDefinition(params) {
         return {
             ...previous,
             datasetId,
+            inputDatasetIds,
+            outputSchema,
+            instructions,
             sandboxId,
-            sandboxState,
+            sandboxState: initialized,
+            contextResources: previous?.contextResources ?? params.contextResources ?? [],
             system,
-            sandboxConfig: { sourcePaths, outputPath },
+            sandboxConfig: {
+                inputPaths: initialized.inputPaths,
+                outputPath: previous?.sandboxConfig?.outputPath ?? getDatasetOutputPath(datasetId),
+            },
         };
     })
+        .resources(({ content }) => Array.isArray(content?.contextResources) ? content.contextResources : [])
         .narrative(async (stored) => {
         return String(stored?.content?.system ?? "");
     })
-        .actions(async (stored, env) => {
+        .actions(async (stored, _env, runtime) => {
+        const datasetId = stored?.content?.datasetId ?? fallbackDatasetId ?? "";
         const sandboxId = stored?.content?.sandboxId ?? params.sandboxId ?? "";
+        if (!datasetId)
+            throw new Error("dataset_id_required");
+        if (!sandboxId)
+            throw new Error("dataset_sandbox_required");
         return {
-            executeCommand: (0, executeCommand_tool_1.createExecuteCommandTool)({
+            completeObject: createCompleteObjectTool({
+                datasetId,
+                sandboxId,
+                runtime,
+                schema: stored?.content?.outputSchema,
+            }),
+            replaceRows: createReplaceRowsTool({
                 datasetId,
                 sandboxId,
-                env,
+                runtime,
+                schema: stored?.content?.outputSchema,
             }),
-            completeDataset: (0, completeDataset_tool_1.createCompleteDatasetTool)({
+            executeCommand: createExecuteCommandTool({
                 datasetId,
                 sandboxId,
-                env,
+                runtime,
             }),
-            clearDataset: (0, clearDataset_tool_1.createClearDatasetTool)({
+            completeDataset: createCompleteDatasetTool({
                 datasetId,
                 sandboxId,
-                env,
+                runtime,
+            }),
+            clearDataset: createClearDatasetTool({
+                datasetId,
+                sandboxId,
+                runtime,
             }),
         };
     })
         .shouldContinue(({ reactionEvent }) => {
-        return !(0, events_1.didToolExecute)(reactionEvent, "completeDataset");
+        const fatalFailure = getDatasetFatalFailure(reactionEvent);
+        if (fatalFailure) {
+            throw new Error(fatalFailure);
+        }
+        return !didCompleteDatasetSucceed(reactionEvent);
     });
     if (params.reactor) {
-        storyBuilder = storyBuilder.reactor(params.reactor);
+        contextBuilder = contextBuilder.reactor(params.reactor);
     }
     else {
-        storyBuilder = storyBuilder.model(model);
+        contextBuilder = contextBuilder.model(model);
     }
-    const story = storyBuilder.build();
-    return { datasetId, story };
+    const context = contextBuilder.build();
+    return { datasetId: fallbackDatasetId ?? "", context };
 }
-function createTransformDatasetStory(params) {
-    const { datasetId, story } = createTransformDatasetStoryDefinition({
-        sourceDatasetIds: params.sourceDatasetIds,
+export function createTransformDatasetContext(params) {
+    const datasetId = params.datasetId ?? createDatasetId();
+    const { context } = createTransformDatasetContextDefinition({
+        inputDatasetIds: params.inputDatasetIds,
         outputSchema: params.outputSchema,
         instructions: params.instructions,
-        datasetId: params.datasetId,
+        datasetId,
         model: params.model,
         sandboxId: params.sandboxId,
         reactor: params.reactor,
+        sandboxState: params.sandboxState,
+        inputPreviews: params.inputPreviews,
+        contextResources: params.contextResources,
     });
     return {
         datasetId,
-        async transform(env, prompt) {
-            const datasetCountText = params.sourceDatasetIds.length === 1
-                ? "the source dataset"
-                : `${params.sourceDatasetIds.length} source datasets`;
+        async transform(runtime, options = {}) {
+            const datasetCountText = params.inputDatasetIds.length === 1
+                ? "the input dataset"
+                : `${params.inputDatasetIds.length} input datasets`;
             const triggerEvent = {
-                id: (0, admin_1.id)(),
-                type: events_1.INPUT_TEXT_ITEM_TYPE,
-                channel: events_1.WEB_CHANNEL,
+                id: createDatasetId(),
+                type: INPUT_TEXT_ITEM_TYPE,
+                channel: WEB_CHANNEL,
                 createdAt: new Date().toISOString(),
                 content: {
                     parts: [
                         {
                             type: "text",
-                            text: prompt ??
+                            text: options.prompt ??
                                 `Transform ${datasetCountText} into a new dataset matching the provided output schema`,
                         },
                     ],
                 },
             };
-            const runtime = (0, eventsReactRuntime_1.createEventsReactRuntime)(env);
-            await story.react(triggerEvent, {
-                runtime,
+            const shell = await context.react(triggerEvent, {
+                runtime: runtime,
                 context: { key: `dataset:${datasetId}` },
-                options: { silent: true, preventClose: true, sendFinish: false, maxIterations: 20, maxModelSteps: 5 },
+                durable: options.durable ?? false,
+                options: {
+                    preventClose: true,
+                    sendFinish: false,
+                    maxIterations: 20,
+                    maxModelSteps: 5,
+                },
+                __initialContent: {
+                    ...(options.initialContent ?? {}),
+                    datasetId,
+                    inputDatasetIds: params.inputDatasetIds,
+                    outputSchema: params.outputSchema,
+                    instructions: params.instructions,
+                    sandboxId: params.sandboxId ?? "",
+                    sandboxState: params.sandboxState ?? { initialized: false, inputPaths: [] },
+                    inputPreviews: params.inputPreviews,
+                    contextResources: params.contextResources ?? [],
+                },
             });
+            await awaitContextRun(shell.run);
             return { datasetId };
         },
-        story,
+        context,
     };
 }
-//# sourceMappingURL=transform-dataset.agent.js.map
+export function registerTransformDatasetContext(opts) {
+    createTransformDatasetContextDefinition({
+        model: opts?.model,
+        reactor: opts?.reactor,
+    }).context;
+}
+registerTransformDatasetContext();

package/dist/transform/transform-dataset.steps.d.ts

@@ -0,0 +1,30 @@
+import type { TransformPromptContext, TransformSandboxState, TransformInputPreviewContext } from "./transform-dataset.types.js";
+export declare function ensureTransformInputsInSandboxStep(params: {
+    runtime: any;
+    sandboxId: string;
+    datasetId: string;
+    inputDatasetIds: string[];
+    state: TransformSandboxState;
+}): Promise<{
+    inputPaths: Array<{
+        datasetId: string;
+        path: string;
+    }>;
+    outputPath: string;
+    state: TransformSandboxState;
+}>;
+export declare function generateTransformInputPreviewsStep(params: {
+    runtime: any;
+    sandboxId: string;
+    datasetId: string;
+    inputPaths: Array<{
+        datasetId: string;
+        path: string;
+    }>;
+}): Promise<Array<{
+    datasetId: string;
+    preview: TransformInputPreviewContext;
+}>>;
+export declare function buildTransformDatasetPromptStep(params: {
+    context: TransformPromptContext;
+}): Promise<string>;

package/dist/transform/transform-dataset.steps.js

@@ -0,0 +1,61 @@
+import { getDatasetOutputPath, getDatasetResourcesDir, getDatasetStandardDirs, } from "../datasetFiles.js";
+import { datasetReadOutputJsonlStep } from "../dataset/steps.js";
+import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "../sandbox/steps.js";
+import { generateInputPreview } from "./filepreview.js";
+import { buildTransformDatasetPrompt } from "./prompts.js";
+export async function ensureTransformInputsInSandboxStep(params) {
+    "use step";
+    if (params.state.initialized) {
+        return {
+            inputPaths: params.state.inputPaths,
+            outputPath: getDatasetOutputPath(params.datasetId),
+            state: params.state,
+        };
+    }
+    await runDatasetSandboxCommandStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        cmd: "mkdir",
+        args: ["-p", ...getDatasetStandardDirs(params.datasetId)],
+    });
+    const inputPaths = [];
+    for (const inputDatasetId of params.inputDatasetIds) {
+        const inputPath = `${getDatasetResourcesDir(params.datasetId)}/resource_${inputDatasetId}.jsonl`;
+        const input = await datasetReadOutputJsonlStep({
+            runtime: params.runtime,
+            datasetId: inputDatasetId,
+        });
+        await writeDatasetSandboxFilesStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            files: [{ path: inputPath, contentBase64: input.contentBase64 }],
+        });
+        inputPaths.push({ datasetId: inputDatasetId, path: inputPath });
+    }
+    return {
+        inputPaths,
+        outputPath: getDatasetOutputPath(params.datasetId),
+        state: {
+            initialized: true,
+            inputPaths,
+        },
+    };
+}
+export async function generateTransformInputPreviewsStep(params) {
+    "use step";
+    const inputPreviews = [];
+    for (const inputPath of params.inputPaths) {
+        try {
+            const preview = await generateInputPreview(params.runtime, params.sandboxId, inputPath.path, params.datasetId);
+            inputPreviews.push({ datasetId: inputPath.datasetId, preview });
+        }
+        catch {
+            // Input preview is optional; transformation can still read the JSONL files.
+        }
+    }
+    return inputPreviews;
+}
+export async function buildTransformDatasetPromptStep(params) {
+    "use step";
+    return buildTransformDatasetPrompt(params.context);
+}

package/dist/transform/transform-dataset.types.d.ts

@@ -0,0 +1,95 @@
+import type { ContextReactor, StoredContextResource } from "@ekairos/events";
+import type { TransformInputPreviewContext } from "./filepreview.js";
+export type { TransformInputPreviewContext } from "./filepreview.js";
+export type TransformSandboxState = {
+    initialized: boolean;
+    inputPaths: Array<{
+        datasetId: string;
+        path: string;
+    }>;
+};
+export type TransformDatasetContext = {
+    datasetId: string;
+    inputDatasetIds: string[];
+    outputSchema: any;
+    sandboxConfig: {
+        inputPaths: Array<{
+            datasetId: string;
+            path: string;
+        }>;
+        outputPath: string;
+    };
+    inputPreviews?: Array<{
+        datasetId: string;
+        preview: TransformInputPreviewContext;
+    }>;
+    contextResources?: StoredContextResource[];
+    errors: string[];
+    iterationCount: number;
+    instructions?: string;
+};
+export type TransformDatasetAgentParams = {
+    inputDatasetIds?: string[];
+    outputSchema?: any;
+    instructions?: string;
+    datasetId?: string;
+    model?: string;
+    sandboxId?: string;
+    reactor?: ContextReactor<any, any>;
+    sandboxState?: TransformSandboxState;
+    inputPreviews?: Array<{
+        datasetId: string;
+        preview: TransformInputPreviewContext;
+    }>;
+    contextResources?: StoredContextResource[];
+};
+export type TransformDatasetRunOptions = {
+    prompt?: string;
+    durable?: boolean;
+    initialContent?: Record<string, any>;
+};
+export type TransformDatasetResult = {
+    id: string;
+    status?: string;
+    title?: string;
+    schema?: any;
+    analysis?: any;
+    calculatedTotalRows?: number;
+    actualGeneratedRowCount?: number;
+    createdAt?: number;
+    updatedAt?: number;
+};
+export type TransformPromptContext = {
+    datasetId: string;
+    inputDatasetIds: string[];
+    outputSchema: any;
+    sandboxConfig: {
+        inputPaths: Array<{
+            datasetId: string;
+            path: string;
+        }>;
+        outputPath: string;
+    };
+    inputPreviews?: Array<{
+        datasetId: string;
+        preview: {
+            totalRows: number;
+            metadata?: {
+                description: string;
+                script: string;
+                command: string;
+                stdout: string;
+                stderr: string;
+            };
+            head?: {
+                description: string;
+                script: string;
+                command: string;
+                stdout: string;
+                stderr: string;
+            };
+        };
+    }>;
+    contextResources?: StoredContextResource[];
+    errors: string[];
+};

package/dist/transform/transform-dataset.types.js

@@ -0,0 +1 @@
+export {};

package/dist/transform/transformDataset.d.ts

@@ -1,3 +1,4 @@
+import type { AnyDatasetRuntime } from "../builder/types.js";
 export type TransformDatasetInput = {
     datasets: Array<{
         id: string;
@@ -14,7 +15,6 @@ export type TransformDatasetResult = {
 };
 /**
  * Workflow-compatible dataset transform.
- * Executes the transform story and returns datasetId + preview rows.
+ * Executes the transform context and returns datasetId + preview rows.
  */
-export declare function transformDataset(input: TransformDatasetInput): Promise<TransformDatasetResult>;
-//# sourceMappingURL=transformDataset.d.ts.map
+export declare function transformDataset(runtime: AnyDatasetRuntime, input: TransformDatasetInput): Promise<TransformDatasetResult>;

package/dist/transform/transformDataset.js

@@ -1,11 +1,7 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformDataset = transformDataset;
-const runtime_1 = require("@ekairos/events/runtime");
-const steps_1 = require("../dataset/steps");
-const transform_dataset_agent_1 = require("./transform-dataset.agent");
+import { datasetPreviewRowsStep } from "../dataset/steps.js";
+import { createTransformDatasetContext } from "./transform-dataset.agent.js";
 function buildInstructions(input) {
-    const sources = input.datasets
+    const inputs = input.datasets
         .map((d, idx) => {
         const name = d.description ? ` - ${d.description}` : "";
         return `${idx + 1}. ${d.id}${name}`;
@@ -16,8 +12,8 @@ function buildInstructions(input) {
         "Use pandas when helpful. Output must be JSONL with {type:'row', data:{...}} lines.",
         "Respect the provided output schema exactly.",
         "",
-        "## Source Datasets",
-        sources || "- (none)",
+        "## Input Datasets",
+        inputs || "- (none)",
         "",
         "## Transformation Description (LaTeX + sets)",
         String(input.description ?? "").trim(),
@@ -25,19 +21,20 @@ function buildInstructions(input) {
 }
 /**
  * Workflow-compatible dataset transform.
- * Executes the transform story and returns datasetId + preview rows.
+ * Executes the transform context and returns datasetId + preview rows.
  */
-async function transformDataset(input) {
-    const env = await (0, runtime_1.getContextEnv)();
-    const { datasetId, story } = (0, transform_dataset_agent_1.createTransformDatasetStory)({
-        sourceDatasetIds: input.datasets.map((d) => d.id),
+export async function transformDataset(runtime, input) {
+    const transformContext = createTransformDatasetContext({
+        inputDatasetIds: input.datasets.map((d) => d.id),
         outputSchema: input.outputSchema,
         instructions: buildInstructions(input),
         datasetId: input.datasetId,
         model: input.model,
     });
-    await story.transform(env);
-    const preview = await (0, steps_1.datasetPreviewRowsStep)({ datasetId });
-    return { datasetId, previewRows: preview.rows };
+    await transformContext.transform(runtime);
+    const preview = await datasetPreviewRowsStep({
+        runtime,
+        datasetId: transformContext.datasetId,
+    });
+    return { datasetId: transformContext.datasetId, previewRows: preview.rows };
 }
-//# sourceMappingURL=transformDataset.js.map