@ekairos/dataset 1.22.85-beta.development.0 → 1.22.87-beta.development.0

package/dist/builder/context.d.ts

@@ -2,6 +2,14 @@ import type { AnyDatasetRuntime, InternalDatasetResource } from "./types.js";
 type DatasetContextResolution = {
     contextId: string;
     resources: InternalDatasetResource[];
+    contextResources: DatasetContextResourceRecord[];
+};
+type DatasetContextResourceRecord = {
+    key: string;
+    type: string;
+    name: string;
+    description: string;
+    [key: string]: unknown;
 };
 export declare function resolveDatasetResourceContext<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, resources: InternalDatasetResource[]): Promise<DatasetContextResolution>;
 export {};

package/dist/builder/context.js

@@ -1,5 +1,7 @@
 import { eventsDomain } from "@ekairos/events";
 import { createDatasetId } from "../id.js";
+import { datasetDomain } from "../schema.js";
+import { DatasetService } from "../service.js";
 import { getDomainDescriptor } from "./rows.js";
 function getContextWhere(context) {
     return "id" in context ? { id: context.id } : { key: context.key };
@@ -8,6 +10,10 @@ async function getEventsDb(runtime) {
     const scoped = await runtime.use(eventsDomain);
     return scoped.db;
 }
+async function getDatasetDb(runtime) {
+    const scoped = await runtime.use(datasetDomain);
+    return scoped.db;
+}
 function resourceKey(index, resource) {
     if (resource.kind === "file")
         return `file:${index}:${resource.fileId}`;
@@ -81,27 +87,63 @@ function resourceToContextResource(index, resource) {
 async function createDatasetResourceContextStep(params) {
     "use step";
     const db = await getEventsDb(params.runtime);
-    const contextId = createDatasetId();
+    const contextKey = `dataset:${params.datasetId}`;
+    const existing = await db.query({
+        event_contexts: {
+            $: { where: { key: contextKey }, limit: 1 },
+        },
+    });
+    const contextId = existing.event_contexts?.[0]?.id ?? createDatasetId();
     const now = new Date();
+    const resources = await enrichDatasetContextResources(params.runtime, params.resources);
     await db.transact([
-        db.tx.event_contexts[contextId].create({
+        db.tx.event_contexts[contextId].update({
+            key: contextKey,
             createdAt: now,
             updatedAt: now,
-            name: `Dataset ${params.datasetId} resource context`,
+            name: `Dataset ${params.datasetId}`,
             status: "open_idle",
             content: {
                 datasetId: params.datasetId,
-                resourceCount: params.resources.length,
+                resourceCount: resources.length,
             },
-            resources: params.resources,
-            description: `Dataset materialization context for ${params.datasetId}.`,
-            goal: "Materialize the dataset from the resources declared in this context.",
+            resources,
+            description: `Dataset execution context for ${params.datasetId}.`,
+            goal: "Produce the dataset output from the resources declared in this context.",
         }),
     ]);
     return {
         contextId,
     };
 }
+async function enrichDatasetContextResources(runtime, resources) {
+    const datasetResources = resources.filter((resource) => resource.type === "dataset" && typeof resource.datasetId === "string");
+    if (datasetResources.length === 0)
+        return resources;
+    const db = await getDatasetDb(runtime);
+    const service = new DatasetService(db);
+    const enriched = [];
+    for (const resource of resources) {
+        if (resource.type !== "dataset" || typeof resource.datasetId !== "string") {
+            enriched.push(resource);
+            continue;
+        }
+        const preview = await service.previewRows(resource.datasetId, 20);
+        if (!preview.ok) {
+            enriched.push({
+                ...resource,
+                previewError: preview.error,
+            });
+            continue;
+        }
+        enriched.push({
+            ...resource,
+            previewRows: preview.data,
+            previewLimit: 20,
+        });
+    }
+    return enriched;
+}
 function contextResourceToDatasetResource(resource) {
     if (resource.type === "file" && typeof resource.fileId === "string" && resource.fileId.trim()) {
         return {
@@ -163,9 +205,16 @@ async function readExistingContext(params) {
     if (resources.length === 0) {
         throw new Error("dataset_context_resources_required");
     }
+    const sourceContextId = String(row.id);
+    const copiedResources = resources.map((resource) => ({
+        ...resource,
+        sourceContextId: resource.sourceContextId ?? sourceContextId,
+        sourceResourceKey: resource.sourceResourceKey ?? resource.key,
+    }));
     return {
-        contextId: String(row.id),
+        contextId: sourceContextId,
         resources: resources.map((resource) => contextResourceToDatasetResource(resource)),
+        contextResources: copiedResources,
     };
 }
 export async function resolveDatasetResourceContext(runtime, datasetId, resources) {
@@ -177,7 +226,16 @@ export async function resolveDatasetResourceContext(runtime, datasetId, resource
         if (resources.length > 1) {
             throw new Error("dataset_context_resource_is_exclusive");
         }
-        return await readExistingContext({ runtime, context: contextRefs[0] });
+        const source = await readExistingContext({ runtime, context: contextRefs[0] });
+        const created = await createDatasetResourceContextStep({
+            runtime,
+            datasetId,
+            resources: source.contextResources,
+        });
+        return {
+            ...source,
+            contextId: created.contextId,
+        };
     }
     const contextResourceRecords = resources.map((resource, index) => resourceToContextResource(index, resource));
     const created = await createDatasetResourceContextStep({
@@ -188,5 +246,6 @@ export async function resolveDatasetResourceContext(runtime, datasetId, resource
     return {
         contextId: created.contextId,
         resources,
+        contextResources: contextResourceRecords,
     };
 }

package/dist/builder/instructions.js

@@ -29,8 +29,9 @@ export function buildObjectOutputInstructions(userInstructions) {
     const base = String(userInstructions ?? "").trim();
     const objectContract = [
         "Output mode is object.",
-        "Produce exactly one JSONL row in output.jsonl.",
-        "That row must be {\"type\":\"row\",\"data\":<the final object>}.",
+        "Produce exactly one final object.",
+        "completeObject({ data: <the final object>, summary }) is available to complete the dataset directly.",
+        "If you use output.jsonl instead, produce exactly one row: {\"type\":\"row\",\"data\":<the final object>}.",
         "Do not emit multiple rows, headers, summaries, or metadata rows.",
     ].join("\n");
     if (!base)

package/dist/builder/materialize.js

@@ -1,7 +1,6 @@
 import { createFileParseContext } from "../file/file-dataset.agent.js";
 import { readInstantFileStep } from "../file/steps.js";
 import { createTransformDatasetContext } from "../transform/transform-dataset.agent.js";
-import { ensureTransformInputsInSandboxStep, generateTransformInputPreviewsStep, } from "../transform/transform-dataset.steps.js";
 import { datasetGetByIdStep, datasetInferAndUpdateSchemaStep, datasetPreviewRowsStep, datasetReadOneStep, } from "../dataset/steps.js";
 import { getDatasetOutputPath, getDatasetScriptsDir, getDatasetResourcesDir, getDatasetStandardDirs, } from "../datasetFiles.js";
 import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
@@ -288,27 +287,14 @@ export async function prepareDatasetResourcesStep(params) {
             mediaType: params.resource.kind === "file" ? params.resource.mediaType : params.resource.mimeType,
         };
     }
-    const initialized = await ensureTransformInputsInSandboxStep({
-        runtime: params.runtime,
-        sandboxId: params.sandboxId,
-        datasetId: params.datasetId,
-        inputDatasetIds: params.inputDatasetIds,
-        state: { initialized: false, inputPaths: [] },
-    });
-    const inputPreviews = await generateTransformInputPreviewsStep({
-        runtime: params.runtime,
-        sandboxId: params.sandboxId,
-        datasetId: params.datasetId,
-        inputPaths: initialized.inputPaths,
-    });
     return {
         kind: "transform",
         datasetId: params.datasetId,
         sandboxId: params.sandboxId,
         inputDatasetIds: params.inputDatasetIds,
         outputSchema: params.outputSchema,
-        sandboxState: initialized.state,
-        inputPreviews,
+        sandboxState: { initialized: false, inputPaths: [] },
+        inputPreviews: undefined,
     };
 }
 export async function initializeDatasetContextStep(params) {
@@ -492,10 +478,7 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
     }
     const sandboxId = resolveDatasetSandboxId(state, targetDatasetId);
     const stateWithSandbox = { ...state, sandboxId };
-    const normalizedResources = [];
-    for (let index = 0; index < stateWithSandbox.resources.length; index++) {
-        normalizedResources.push(await normalizeResourceToDatasetId(stateWithSandbox, stateWithSandbox.resources[index], targetDatasetId, index));
-    }
+    const inputDatasetIds = (stateWithSandbox.contextResources ?? []).map((resource, index) => String(resource.datasetId ?? resource.key ?? `resource_${index + 1}`));
     const transformSchema = stateWithSandbox.outputSchema ??
         {
             title: "DatasetRow",
@@ -515,17 +498,18 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
         contextId: stateWithSandbox.contextId ?? "",
         schema: transformSchema,
     });
-    const prepared = await prepareDatasetResourcesStep({
+    const prepared = {
         kind: "transform",
-        runtime: stateWithSandbox.runtime,
         datasetId: targetDatasetId,
         sandboxId,
-        inputDatasetIds: normalizedResources,
+        inputDatasetIds,
         outputSchema: transformSchema,
-    });
+        sandboxState: { initialized: false, inputPaths: [] },
+        inputPreviews: undefined,
+    };
     const context = await initializeDatasetContextStep({
         prepared,
-        instructions: buildTransformInstructions(normalizedResources.length, stateWithSandbox.instructions, stateWithSandbox.outputSchema),
+        instructions: buildTransformInstructions(inputDatasetIds.length, stateWithSandbox.instructions, stateWithSandbox.outputSchema),
         outputSchema: transformSchema,
     });
     if (context.kind !== "transform") {
@@ -540,6 +524,7 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
         sandboxId: context.sandboxId,
         sandboxState: context.sandboxState,
         inputPreviews: context.inputPreviews,
+        contextResources: stateWithSandbox.contextResources ?? [],
     });
     await transformContext.transform(stateWithSandbox.runtime, {
         durable: await resolveDatasetAgentDurable(stateWithSandbox.durable),
@@ -552,6 +537,7 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
             sandboxId: context.sandboxId,
             sandboxState: context.sandboxState,
             inputPreviews: context.inputPreviews,
+            contextResources: stateWithSandbox.contextResources ?? [],
         },
     });
     return targetDatasetId;

package/dist/builder/types.d.ts

@@ -1,7 +1,7 @@
 import type { InstaQLParams, ValidQuery } from "@instantdb/core";
 import type { DomainInstantSchema, DomainSchemaResult } from "@ekairos/domain";
 import type { EkairosRuntime, RuntimeForDomain } from "@ekairos/domain/runtime";
-import type { ContextIdentifier, ContextReactor } from "@ekairos/events";
+import type { ContextIdentifier, ContextReactor, StoredContextResource } from "@ekairos/events";
 import { datasetDomain } from "../schema.js";
 export type DatasetQueryResourceInput<D extends DomainSchemaResult = DomainSchemaResult> = {
     query: InstaQLParams<DomainInstantSchema<D>>;
@@ -92,6 +92,7 @@ export type DatasetBuilderState<Runtime extends AnyDatasetRuntime> = {
     runtime: Runtime;
     env: Runtime["env"] & DatasetRuntimeEnv;
     resources: InternalDatasetResource[];
+    contextResources?: StoredContextResource[];
     title?: string;
     sandboxId?: string;
     contextId?: string;

package/dist/completeDataset.steps.d.ts

@@ -6,6 +6,9 @@ export interface PersistDatasetStepParams {
     outputPath?: string;
 }
 export declare function persistDatasetStep({ runtime, datasetId, sandboxId, summary, outputPath }: PersistDatasetStepParams): Promise<{
+    rowSource: string;
+    outputPath: string;
+    storagePath: string;
     success: boolean;
     validation?: RowValidationEntry[];
     validationTruncated?: number;
@@ -16,21 +19,47 @@ export declare function persistDatasetStep({ runtime, datasetId, sandboxId, summ
     error?: string;
     status?: string;
     message?: string;
+    validRows?: undefined;
+    dataFileId?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    validRows: number;
+    rowRecordCount: number;
+    validation: RowValidationEntry[] | undefined;
+    error: string;
+    message: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId?: undefined;
+    records?: undefined;
+    summary?: undefined;
 } | {
     success: boolean;
     status: string;
+    rowSource: string;
     validRows: number;
     rowRecordCount: number;
     validation: RowValidationEntry[] | undefined;
     error: string;
     message: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId: string;
     records?: undefined;
     summary?: undefined;
 } | {
     success: boolean;
     status: string;
+    rowSource: string;
     records: number;
     summary: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId: string;
     validRows?: undefined;
     rowRecordCount?: undefined;
     validation?: undefined;

package/dist/completeDataset.steps.js

@@ -29,11 +29,14 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "missing_output",
+            rowSource: "jsonl",
             validRows: 0,
             rowRecordCount: 0,
             validation: [],
             error: message,
             message,
+            outputPath: resolvedOutputPath,
+            storagePath,
         };
     }
     console.log(`[Dataset ${datasetId}] Validating dataset rows against schema`);
@@ -45,11 +48,14 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "dataset_not_found",
+            rowSource: "jsonl",
             validRows: 0,
             rowRecordCount: 0,
             validation: [],
             error: datasetResult.error,
             message: datasetResult.error,
+            outputPath: resolvedOutputPath,
+            storagePath,
         };
     }
     const datasetRecord = datasetResult.data;
@@ -58,11 +64,14 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "schema_missing",
+            rowSource: "jsonl",
             validRows: 0,
             rowRecordCount: 0,
             validation: [],
             error: "Schema not found in database. Please generate schema first.",
             message: "Schema not found in database. Please generate schema first.",
+            outputPath: resolvedOutputPath,
+            storagePath,
         };
     }
     const schemaJson = datasetRecord.schema.schema;
@@ -76,11 +85,14 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "schema_invalid",
+            rowSource: "jsonl",
             validRows: 0,
             rowRecordCount: 0,
             validation: [],
             error: `Failed to compile schema: ${message}`,
             message: `Failed to compile schema: ${message}`,
+            outputPath: resolvedOutputPath,
+            storagePath,
         };
     }
     const validationResult = await validateJsonlRows({
@@ -92,7 +104,12 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         datasetId,
     });
     if (!validationResult.success) {
-        return validationResult;
+        return {
+            ...validationResult,
+            rowSource: "jsonl",
+            outputPath: resolvedOutputPath,
+            storagePath,
+        };
     }
     const totalValidRows = validationResult.validRowCount ?? 0;
     const rowRecordCount = validationResult.rowRecordCount ?? totalValidRows;
@@ -103,11 +120,14 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "empty_output",
+            rowSource: "jsonl",
             validRows: 0,
             rowRecordCount: 0,
             validation: [],
             error: "Empty file content",
             message: "Empty file content",
+            outputPath: resolvedOutputPath,
+            storagePath,
         };
     }
     console.log(`[Dataset ${datasetId}] Uploading file to InstantDB storage`);
@@ -121,11 +141,14 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "upload_failed",
+            rowSource: "jsonl",
             validRows: totalValidRows,
             rowRecordCount,
             validation: validationResult.validation,
             error: uploadResult.error,
             message: uploadResult.error,
+            outputPath: resolvedOutputPath,
+            storagePath,
         };
     }
     console.log(`[Dataset ${datasetId}] File uploaded successfully: ${uploadResult.data.fileId}`);
@@ -140,11 +163,15 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
         return {
             success: false,
             status: "status_update_failed",
+            rowSource: "jsonl",
             validRows: totalValidRows,
             rowRecordCount,
             validation: validationResult.validation,
             error: statusResult.error,
             message: statusResult.error,
+            outputPath: resolvedOutputPath,
+            storagePath,
+            dataFileId: uploadResult.data.fileId,
         };
     }
     console.log(`[Dataset ${datasetId}] Dataset marked as COMPLETED (${totalValidRows} valid rows)`);
@@ -152,8 +179,12 @@ export async function persistDatasetStep({ runtime, datasetId, sandboxId, summar
     return {
         success: true,
         status: "completed",
+        rowSource: "jsonl",
         records: totalValidRows,
         summary: summary ?? `Dataset completed with ${totalValidRows} records.`,
+        outputPath: resolvedOutputPath,
+        storagePath,
+        dataFileId: uploadResult.data.fileId,
     };
 }
 function resolveExecutionStoragePath(outputPath, datasetId) {

package/dist/completeDataset.tool.d.ts

@@ -7,6 +7,9 @@ interface CompleteDatasetToolParams {
 export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outputPath }: CompleteDatasetToolParams): import("ai").Tool<{
     summary: string;
 }, {
+    rowSource: string;
+    outputPath: string;
+    storagePath: string;
     success: boolean;
     validation?: {
         index: number;
@@ -63,9 +66,40 @@ export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtim
     error?: string;
     status?: string;
     message?: string;
+    validRows?: undefined;
+    dataFileId?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    validRows: number;
+    rowRecordCount: number;
+    validation: {
+        index: number;
+        valid: boolean;
+        errors?: string[];
+        errorDetails?: Array<{
+            path: string;
+            keyword: string;
+            message: string;
+            params?: Record<string, unknown>;
+            schemaPath?: string;
+        }>;
+        dataKeys?: string[];
+    }[] | undefined;
+    error: string;
+    message: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId?: undefined;
+    records?: undefined;
+    summary?: undefined;
 } | {
     success: boolean;
     status: string;
+    rowSource: string;
     validRows: number;
     rowRecordCount: number;
     validation: {
@@ -83,13 +117,20 @@ export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtim
     }[] | undefined;
     error: string;
     message: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId: string;
     records?: undefined;
     summary?: undefined;
 } | {
     success: boolean;
     status: string;
+    rowSource: string;
     records: number;
     summary: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId: string;
     validRows?: undefined;
     rowRecordCount?: undefined;
     validation?: undefined;

package/dist/completeDataset.tool.js

@@ -3,7 +3,7 @@ import { z } from "zod";
 import { persistDatasetStep } from "./completeDataset.steps.js";
 export function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outputPath }) {
     return tool({
-        description: "Mark the dataset as completed. Use only when output.jsonl has been successfully generated and is ready for validation.",
+        description: "Validate and complete the dataset from output.jsonl. The result includes the JSONL outputPath and storagePath used for completion.",
         inputSchema: z.object({
             summary: z.string().describe("Summary of the completed dataset including record count and structure"),
         }),
@@ -25,11 +25,14 @@ export function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outpu
 export function didCompleteDatasetSucceed(event) {
     const parts = Array.isArray(event?.content?.parts) ? event.content.parts : [];
     return parts.some((part) => {
-        if (part?.type === "action" && part?.content?.actionName === "completeDataset") {
+        if (part?.type === "action" &&
+            ["completeDataset", "completeObject", "replaceRows"].includes(part?.content?.actionName)) {
             const output = part.content.output;
             return part.content.status === "completed" && output?.success === true && output?.status === "completed";
         }
-        if (part?.type === "tool-completeDataset") {
+        if (part?.type === "tool-completeDataset" ||
+            part?.type === "tool-completeObject" ||
+            part?.type === "tool-replaceRows") {
             const output = part.output ?? part.result;
             return part.state === "output-available" && output?.success === true && output?.status === "completed";
         }

package/dist/contextResources.d.ts

@@ -0,0 +1,31 @@
+import type { StoredContextResource } from "@ekairos/events";
+type MaterializedContextResource = {
+    key: string;
+    type: string;
+    name: string;
+    description: string;
+    dir: string;
+    metadataPath: string;
+    files: Array<{
+        path: string;
+        role: string;
+        mediaType?: string;
+    }>;
+    status: "materialized" | "metadata_only" | "skipped";
+    reason?: string;
+};
+export type ContextResourcesMaterialization = {
+    contextId: string;
+    sandboxId: string;
+    resourcesDir: string;
+    manifestPath: string;
+    resources: MaterializedContextResource[];
+};
+export declare function materializeContextResourcesStep(params: {
+    runtime: any;
+    sandboxId: string;
+    contextId: string;
+    resources: StoredContextResource[];
+    resourceKeys?: string[] | null;
+}): Promise<ContextResourcesMaterialization>;
+export {};