@ekairos/dataset 1.22.48-beta.development.0 → 1.22.50-beta.development.0

package/dist/agents.d.ts

@@ -0,0 +1,8 @@
+import "./builder/materialize.js";
+export * from "./dataset.js";
+export * from "./domain.js";
+export * from "./materializeDataset.tool.js";
+export * from "./schema.js";
+export * from "./service.js";
+export * from "./file/file-dataset.agent.js";
+export * from "./transform/index.js";

package/dist/agents.js

@@ -0,0 +1,8 @@
+import "./builder/materialize.js";
+export * from "./dataset.js";
+export * from "./domain.js";
+export * from "./materializeDataset.tool.js";
+export * from "./schema.js";
+export * from "./service.js";
+export * from "./file/file-dataset.agent.js";
+export * from "./transform/index.js";

package/dist/builder/agentMaterializers.d.ts

@@ -0,0 +1,9 @@
+import type { AnyDatasetRuntime, DatasetBuilderState, InternalSource } from "./types.js";
+export type DatasetAgentMaterializers = {
+    materializeSingleFileLikeSource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, source: Extract<InternalSource, {
+        kind: "file" | "text";
+    }>, targetDatasetId: string): Promise<string>;
+    materializeDerivedDataset<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, targetDatasetId: string): Promise<string>;
+};
+export declare function registerDatasetAgentMaterializers(materializers: DatasetAgentMaterializers): void;
+export declare function getDatasetAgentMaterializers(): DatasetAgentMaterializers;

package/dist/builder/agentMaterializers.js

@@ -0,0 +1,10 @@
+let agentMaterializers = null;
+export function registerDatasetAgentMaterializers(materializers) {
+    agentMaterializers = materializers;
+}
+export function getDatasetAgentMaterializers() {
+    if (!agentMaterializers) {
+        throw new Error("dataset_agent_materializers_not_registered");
+    }
+    return agentMaterializers;
+}

package/dist/builder/materialize.d.ts

@@ -1,14 +1,4 @@
-import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalSource } from "./types.js";
-export declare function materializeQuerySource<Runtime extends AnyDatasetRuntime>(runtime: DatasetBuilderState<Runtime>["runtime"], source: Extract<InternalSource, {
-    kind: "query";
-}>, params: {
-    datasetId: string;
-    sandboxId?: string;
-    schema?: DatasetSchemaInput;
-    title?: string;
-    instructions?: string;
-    first?: boolean;
-}): Promise<string>;
+import type { AnyDatasetRuntime, DatasetBuilderState, InternalSource } from "./types.js";
 export declare function materializeSingleFileLikeSource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, source: Extract<InternalSource, {
     kind: "file" | "text";
 }>, targetDatasetId: string): Promise<string>;

package/dist/builder/materialize.js

@@ -1,44 +1,14 @@
-import { createFileParseStory } from "../file/file-dataset.agent.js";
-import { DatasetService } from "../service.js";
-import { createTransformDatasetStory } from "../transform/transform-dataset.agent.js";
+import { createFileParseContext } from "../file/file-dataset.agent.js";
+import { createTransformDatasetContext } from "../transform/transform-dataset.agent.js";
+import { datasetInferAndUpdateSchemaStep, datasetReadOneStep, } from "../dataset/steps.js";
+import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
 import { buildFileDefaultInstructions, buildRawSourceInstructions, buildTransformInstructions, } from "./instructions.js";
-import { createOrUpdateDatasetMetadata, getDatasetDb, materializeRowsToDataset, uploadInlineTextSource, } from "./persistence.js";
-import { inferDatasetSchema } from "./schemaInference.js";
-import { getDomainDescriptor, normalizeQueryRows } from "./sourceRows.js";
+import { createOrUpdateDatasetMetadata, uploadInlineTextSource, } from "./persistence.js";
+import { getDomainDescriptor } from "./sourceRows.js";
+import { materializeQuerySource } from "./materializeQuery.js";
 function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
     return `${targetDatasetId}__${sourceKind}_${index}`;
 }
-export async function materializeQuerySource(runtime, source, params) {
-    const scoped = await runtime.use(source.domain);
-    const result = await scoped.db.query(source.query);
-    const rows = normalizeQueryRows(result);
-    const domainDescriptor = getDomainDescriptor(source.domain);
-    return await materializeRowsToDataset(runtime, {
-        datasetId: params.datasetId,
-        sandboxId: params.sandboxId,
-        title: params.title ?? source.title,
-        instructions: params.instructions,
-        sources: [
-            {
-                kind: "query",
-                query: source.query,
-                title: source.title,
-                explanation: source.explanation,
-                ...domainDescriptor,
-            },
-        ],
-        sourceKinds: ["query"],
-        analysis: {
-            query: source.query,
-            explanation: source.explanation,
-            ...domainDescriptor,
-        },
-        rows,
-        schema: params.schema,
-        inferSchema: !params.schema,
-        first: params.first,
-    });
-}
 export async function materializeSingleFileLikeSource(state, source, targetDatasetId) {
     if (!state.reactor) {
         throw new Error("dataset_reactor_required");
@@ -68,37 +38,23 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
         schema: state.outputSchema,
         status: "building",
     });
-    const parseStory = createFileParseStory(fileId, {
+    const parseContext = createFileParseContext(fileId, {
         datasetId: targetDatasetId,
         instructions: state.instructions ?? buildFileDefaultInstructions(state.outputSchema),
         reactor: state.reactor,
         sandboxId: state.sandboxId,
     });
-    await parseStory.parse(state.env);
+    await parseContext.parse(state.runtime, { durable: state.durable });
     if (!state.outputSchema) {
-        const db = await getDatasetDb(state.runtime);
-        const service = new DatasetService(db);
-        const readResult = await service.readRows({ datasetId: targetDatasetId, cursor: 0, limit: 1000 });
-        if (!readResult.ok) {
-            throw new Error(readResult.error);
-        }
-        const inferred = inferDatasetSchema(readResult.data.rows, `${targetDatasetId}Row`, "One dataset row");
-        const updateResult = await service.updateDatasetSchema({
+        await datasetInferAndUpdateSchemaStep({
+            runtime: state.runtime,
             datasetId: targetDatasetId,
-            schema: inferred,
-            status: "completed",
+            title: `${targetDatasetId}Row`,
+            description: "One dataset row",
         });
-        if (!updateResult.ok) {
-            throw new Error(updateResult.error);
-        }
     }
     if (state.first) {
-        const db = await getDatasetDb(state.runtime);
-        const service = new DatasetService(db);
-        const firstResult = await service.readOne(targetDatasetId);
-        if (!firstResult.ok) {
-            throw new Error(firstResult.error);
-        }
+        await datasetReadOneStep({ runtime: state.runtime, datasetId: targetDatasetId });
     }
     return targetDatasetId;
 }
@@ -163,7 +119,7 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
         schema: transformSchema,
         status: "building",
     });
-    const transformStory = createTransformDatasetStory({
+    const transformContext = createTransformDatasetContext({
         sourceDatasetIds: normalizedSources,
         outputSchema: transformSchema,
         instructions: buildTransformInstructions(normalizedSources.length, state.instructions, state.outputSchema),
@@ -171,29 +127,21 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
         reactor: state.reactor,
         sandboxId: state.sandboxId,
     });
-    await transformStory.transform(state.env);
-    const db = await getDatasetDb(state.runtime);
-    const service = new DatasetService(db);
+    await transformContext.transform(state.runtime, { durable: state.durable });
     if (!state.outputSchema) {
-        const readResult = await service.readRows({ datasetId: targetDatasetId, cursor: 0, limit: 1000 });
-        if (!readResult.ok) {
-            throw new Error(readResult.error);
-        }
-        const inferred = inferDatasetSchema(readResult.data.rows, `${targetDatasetId}Row`, "One dataset row");
-        const updateResult = await service.updateDatasetSchema({
+        await datasetInferAndUpdateSchemaStep({
+            runtime: state.runtime,
             datasetId: targetDatasetId,
-            schema: inferred,
-            status: "completed",
+            title: `${targetDatasetId}Row`,
+            description: "One dataset row",
         });
-        if (!updateResult.ok) {
-            throw new Error(updateResult.error);
-        }
     }
     if (state.first) {
-        const firstResult = await service.readOne(targetDatasetId);
-        if (!firstResult.ok) {
-            throw new Error(firstResult.error);
-        }
+        await datasetReadOneStep({ runtime: state.runtime, datasetId: targetDatasetId });
     }
     return targetDatasetId;
 }
+registerDatasetAgentMaterializers({
+    materializeSingleFileLikeSource,
+    materializeDerivedDataset,
+});

package/dist/builder/materializeQuery.d.ts

@@ -0,0 +1,11 @@
+import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalSource } from "./types.js";
+export declare function materializeQuerySource<Runtime extends AnyDatasetRuntime>(runtime: DatasetBuilderState<Runtime>["runtime"], source: Extract<InternalSource, {
+    kind: "query";
+}>, params: {
+    datasetId: string;
+    sandboxId?: string;
+    schema?: DatasetSchemaInput;
+    title?: string;
+    instructions?: string;
+    first?: boolean;
+}): Promise<string>;

package/dist/builder/materializeQuery.js

@@ -0,0 +1,40 @@
+import { materializeRowsToDataset } from "./persistence.js";
+import { getDomainDescriptor, normalizeQueryRows } from "./sourceRows.js";
+async function readQuerySourceRowsStep(params) {
+    "use step";
+    const db = await params.runtime.db();
+    const result = await db.query(params.query);
+    return { rows: normalizeQueryRows(result) };
+}
+export async function materializeQuerySource(runtime, source, params) {
+    const { rows } = await readQuerySourceRowsStep({
+        runtime,
+        query: source.query,
+    });
+    const domainDescriptor = getDomainDescriptor(source.domain);
+    return await materializeRowsToDataset(runtime, {
+        datasetId: params.datasetId,
+        sandboxId: params.sandboxId,
+        title: params.title ?? source.title,
+        instructions: params.instructions,
+        sources: [
+            {
+                kind: "query",
+                query: source.query,
+                title: source.title,
+                explanation: source.explanation,
+                ...domainDescriptor,
+            },
+        ],
+        sourceKinds: ["query"],
+        analysis: {
+            query: source.query,
+            explanation: source.explanation,
+            ...domainDescriptor,
+        },
+        rows,
+        schema: params.schema,
+        inferSchema: !params.schema,
+        first: params.first,
+    });
+}

package/dist/builder/persistence.js

@@ -1,5 +1,6 @@
 import { DatasetService } from "../service.js";
 import { datasetDomain } from "../schema.js";
+import { datasetGetByIdStep, datasetPreviewRowsStep, datasetReadOneStep, datasetReadRowsStep, } from "../dataset/steps.js";
 import { inferDatasetSchema, validateRows } from "./schemaInference.js";
 import { rowsToJsonl } from "./sourceRows.js";
 export function defaultTextSourceName(source) {
@@ -19,6 +20,7 @@ export async function getDatasetDb(runtime) {
     return scoped.db;
 }
 export async function createOrUpdateDatasetMetadata(runtime, params) {
+    "use step";
     const db = await getDatasetDb(runtime);
     const service = new DatasetService(db);
     const result = await service.createDataset({
@@ -38,6 +40,7 @@ export async function createOrUpdateDatasetMetadata(runtime, params) {
     }
 }
 export async function materializeRowsToDataset(runtime, params) {
+    "use step";
     if (params.first && params.rows.length > 1) {
         throw new Error("dataset_first_expected_zero_or_one_row");
     }
@@ -76,6 +79,7 @@ export async function materializeRowsToDataset(runtime, params) {
     return params.datasetId;
 }
 export async function uploadInlineTextSource(runtime, datasetId, source) {
+    "use step";
     const db = await getDatasetDb(runtime);
     const fileName = defaultTextSourceName(source);
     const storagePath = `/dataset/source/${datasetId}/${Date.now()}-${fileName}`;
@@ -90,49 +94,37 @@ export async function uploadInlineTextSource(runtime, datasetId, source) {
     return fileId;
 }
 export async function finalizeBuildResult(runtime, datasetId, withFirst) {
-    const db = await getDatasetDb(runtime);
-    const service = new DatasetService(db);
-    const datasetResult = await service.getDatasetById(datasetId);
-    if (!datasetResult.ok) {
+    const datasetResult = await datasetGetByIdStep({ runtime, datasetId });
+    if (!datasetResult.ok)
         throw new Error(datasetResult.error);
-    }
-    const previewResult = await service.previewRows(datasetId, 20);
-    if (!previewResult.ok) {
-        throw new Error(previewResult.error);
-    }
+    const previewResult = await datasetPreviewRowsStep({ runtime, datasetId, limit: 20 });
     const reader = {
         async read(cursorOrParams, limit) {
             const params = typeof cursorOrParams === "object" && cursorOrParams !== null
                 ? cursorOrParams
                 : { cursor: cursorOrParams, limit };
-            const rowsResult = await service.readRows({
+            return await datasetReadRowsStep({
+                runtime,
                 datasetId,
                 cursor: params.cursor,
                 limit: params.limit,
             });
-            if (!rowsResult.ok) {
-                throw new Error(rowsResult.error);
-            }
-            return rowsResult.data;
         },
     };
     if (!withFirst) {
         return {
             datasetId,
             dataset: datasetResult.data,
-            previewRows: previewResult.data,
+            previewRows: previewResult.rows,
             reader,
         };
     }
-    const firstResult = await service.readOne(datasetId);
-    if (!firstResult.ok) {
-        throw new Error(firstResult.error);
-    }
+    const firstResult = await datasetReadOneStep({ runtime, datasetId });
     return {
         datasetId,
         dataset: datasetResult.data,
-        previewRows: previewResult.data,
+        previewRows: previewResult.rows,
         reader,
-        firstRow: firstResult.data,
+        firstRow: firstResult.row,
     };
 }

package/dist/builder/types.d.ts

@@ -42,9 +42,11 @@ export type DatasetOutput = "rows" | "object";
 export type DatasetMode = "auto" | "schema";
 export type DatasetBuilderOptions = {
     datasetId?: string;
+    durable?: boolean;
 };
 export type DatasetBuildOptions = {
     datasetId?: string;
+    durable?: boolean;
 };
 export type InternalSource = DatasetFileSource | DatasetTextSource | DatasetExistingSource | ({
     kind: "query";
@@ -91,6 +93,7 @@ export type DatasetBuilderState<Runtime extends AnyDatasetRuntime> = {
     inferSchema: boolean;
     instructions?: string;
     reactor?: ContextReactor<any, any>;
+    durable?: boolean;
     first: boolean;
 };
 export type MaterializeRowsParams = {

package/dist/clearDataset.tool.d.ts

@@ -1,9 +1,9 @@
 interface ClearDatasetToolParams {
     datasetId: string;
     sandboxId: string;
-    env?: any;
+    runtime: any;
 }
-export declare function createClearDatasetTool({ datasetId, sandboxId, env }: ClearDatasetToolParams): import("ai").Tool<{
+export declare function createClearDatasetTool({ datasetId, sandboxId, runtime }: ClearDatasetToolParams): import("ai").Tool<{
     reason: string;
 }, {
     success: boolean;

package/dist/clearDataset.tool.js

@@ -3,7 +3,7 @@ import { z } from "zod";
 import { runDatasetSandboxCommandStep } from "./sandbox/steps.js";
 import { getDatasetOutputPath } from "./datasetFiles.js";
 import { datasetClearStep } from "./dataset/steps.js";
-export function createClearDatasetTool({ datasetId, sandboxId, env }) {
+export function createClearDatasetTool({ datasetId, sandboxId, runtime }) {
     return tool({
         description: "Clear all dataset records and output files. This will delete all generated data and reset the dataset to its initial state.",
         inputSchema: z.object({
@@ -18,7 +18,7 @@ export function createClearDatasetTool({ datasetId, sandboxId, env }) {
             console.log(`[Dataset ${datasetId}] Step 1: Deleting output file`);
             try {
                 const result = await runDatasetSandboxCommandStep({
-                    env,
+                    runtime,
                     sandboxId,
                     cmd: "rm",
                     args: ["-f", outputPath],
@@ -35,7 +35,7 @@ export function createClearDatasetTool({ datasetId, sandboxId, env }) {
                 console.warn(`[Dataset ${datasetId}] Error deleting output file: ${message}`);
             }
             console.log(`[Dataset ${datasetId}] Step 2: Clearing dataset records`);
-            const clearResult = await datasetClearStep({ env, datasetId });
+            const clearResult = await datasetClearStep({ runtime, datasetId });
             if (!clearResult.ok) {
                 console.error(`[Dataset ${datasetId}] Failed to clear dataset: ${clearResult.error}`);
                 return {

package/dist/completeDataset.tool.d.ts

@@ -1,9 +1,9 @@
 interface CompleteDatasetToolParams {
     datasetId: string;
     sandboxId: string;
-    env?: any;
+    runtime: any;
 }
-export declare function createCompleteDatasetTool({ datasetId, sandboxId, env }: CompleteDatasetToolParams): import("ai").Tool<{
+export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtime }: CompleteDatasetToolParams): import("ai").Tool<{
     summary: string;
 }, {
     success: boolean;
@@ -11,14 +11,42 @@ export declare function createCompleteDatasetTool({ datasetId, sandboxId, env }:
         index: number;
         valid: boolean;
         errors?: string[];
+        dataKeys?: string[];
     }>;
     validRowCount?: number;
+    rowRecordCount?: number;
     error?: string;
+    status?: string;
+    message?: string;
 } | {
     success: boolean;
-    validRows: number | undefined;
+    status: string;
+    validRows: number;
+    rowRecordCount: number;
+    validation: {
+        index: number;
+        valid: boolean;
+        errors?: string[];
+        dataKeys?: string[];
+    }[] | undefined;
+    error: string;
+    message: string;
+    fileId?: undefined;
+    storagePath?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    validRows: number;
+    rowRecordCount: number;
     fileId: string;
     storagePath: string;
     message: string;
+    validation?: undefined;
+    error?: undefined;
 }>;
+export declare function didCompleteDatasetSucceed(event: {
+    content?: {
+        parts?: any[];
+    };
+}): boolean;
 export {};