@ekairos/dataset 1.22.79-beta.development.0 → 1.22.81-beta.development.0

package/dist/builder/materialize.d.ts

@@ -1,6 +1,82 @@
-import type { AnyDatasetRuntime, DatasetBuilderState, InternalSource } from "./types.js";
+import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalSource } from "./types.js";
+import type { SandboxState } from "../file/file-dataset.types.js";
+import type { FilePreviewContext } from "../file/filepreview.types.js";
+import type { TransformSandboxState, TransformSourcePreviewContext } from "../transform/transform-dataset.types.js";
 export declare function resolveDatasetAgentDurable(requestedDurable?: boolean): Promise<boolean>;
+type PreparedFileDatasetContext = {
+    kind: "file";
+    datasetId: string;
+    sandboxId: string;
+    fileId: string;
+    sandboxState: SandboxState;
+    filePreview?: FilePreviewContext;
+    schema?: DatasetSchemaInput | null;
+};
+type PreparedTransformDatasetContext = {
+    kind: "transform";
+    datasetId: string;
+    sandboxId: string;
+    sourceDatasetIds: string[];
+    outputSchema: DatasetSchemaInput;
+    sandboxState: TransformSandboxState;
+    sourcePreviews?: Array<{
+        datasetId: string;
+        preview: TransformSourcePreviewContext;
+    }>;
+};
+type PreparedDatasetContext = PreparedFileDatasetContext | PreparedTransformDatasetContext;
+type DatasetContextInitialization = PreparedDatasetContext & {
+    prompt: string;
+    instructions?: string;
+};
+export declare function initializeDatasetStep<Runtime extends AnyDatasetRuntime>(params: {
+    runtime: Runtime;
+    datasetId: string;
+    sandboxId: string;
+    title?: string;
+    instructions?: string;
+    sources: any[];
+    sourceKinds: string[];
+    schema?: DatasetSchemaInput;
+}): Promise<{
+    datasetId: string;
+    sandboxId: string;
+}>;
+export declare function prepareDatasetSourcesStep<Runtime extends AnyDatasetRuntime>(params: {
+    kind: "file";
+    runtime: Runtime;
+    datasetId: string;
+    sandboxId: string;
+    source: Extract<InternalSource, {
+        kind: "file" | "text";
+    }>;
+    schema?: DatasetSchemaInput;
+} | {
+    kind: "transform";
+    runtime: Runtime;
+    datasetId: string;
+    sandboxId: string;
+    sourceDatasetIds: string[];
+    outputSchema: DatasetSchemaInput;
+}): Promise<PreparedDatasetContext>;
+export declare function initializeDatasetContextStep(params: {
+    prepared: PreparedDatasetContext;
+    instructions?: string;
+    outputSchema?: DatasetSchemaInput;
+}): Promise<DatasetContextInitialization>;
+export declare function completeDatasetStep<Runtime extends AnyDatasetRuntime>(params: {
+    runtime: Runtime;
+    datasetId: string;
+    schema?: DatasetSchemaInput;
+    first: boolean;
+}): Promise<{
+    datasetId: string;
+    dataset: any;
+    previewRows: any[];
+    firstRow: any;
+}>;
 export declare function materializeSingleFileLikeSource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, source: Extract<InternalSource, {
     kind: "file" | "text";
 }>, targetDatasetId: string): Promise<string>;
 export declare function materializeDerivedDataset<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, targetDatasetId: string): Promise<string>;
+export {};

package/dist/builder/materialize.js

@@ -1,14 +1,16 @@
 import { createFileParseContext } from "../file/file-dataset.agent.js";
 import { readInstantFileStep } from "../file/steps.js";
+import { generateFileParsePreviewStep, initializeFileParseSandboxStep, } from "../file/file-dataset.steps.js";
 import { createTransformDatasetContext } from "../transform/transform-dataset.agent.js";
-import { datasetInferAndUpdateSchemaStep, datasetReadOneStep, } from "../dataset/steps.js";
-import { getDatasetOutputPath, getDatasetWorkstation } from "../datasetFiles.js";
+import { ensureTransformSourcesInSandboxStep, generateTransformSourcePreviewsStep, } from "../transform/transform-dataset.steps.js";
+import { datasetGetByIdStep, datasetInferAndUpdateSchemaStep, datasetPreviewRowsStep, datasetReadOneStep, } from "../dataset/steps.js";
+import { getDatasetOutputPath, getDatasetScriptsDir, getDatasetSourcesDir, getDatasetStandardDirs, } from "../datasetFiles.js";
 import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
 import { buildFileDefaultInstructions, buildRawSourceInstructions, buildTransformInstructions, } from "./instructions.js";
 import { createOrUpdateDatasetMetadata, materializeRowsToDataset, uploadInlineTextSource, } from "./persistence.js";
 import { getDomainDescriptor } from "./sourceRows.js";
 import { materializeQuerySource } from "./materializeQuery.js";
-import { createDatasetSandboxStep, readDatasetSandboxTextFileStep, runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFilesStep, } from "../sandbox/steps.js";
+import { readDatasetSandboxTextFileStep, runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFilesStep, } from "../sandbox/steps.js";
 function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
     return `${targetDatasetId}__${sourceKind}_${index}`;
 }
@@ -101,17 +103,16 @@ async function tryMaterializeRawPdfFileSource(state, source, targetDatasetId) {
     const file = await readInstantFileStep({ runtime: state.runtime, fileId: source.fileId });
     if (!isPdfContentDisposition(file.contentDisposition))
         return null;
-    const sandboxId = await resolveDatasetSandboxId(state, targetDatasetId);
-    const workstation = getDatasetWorkstation(targetDatasetId);
+    const sandboxId = resolveDatasetSandboxId(state, targetDatasetId);
     const outputPath = getDatasetOutputPath(targetDatasetId);
     const fileName = sanitizePdfFileName(parseContentDispositionFileName(file.contentDisposition), `${source.fileId}.pdf`);
-    const sourcePath = `${workstation}/${fileName}`;
-    const scriptPath = `${workstation}/extract_pdf_text.py`;
+    const sourcePath = `${getDatasetSourcesDir(targetDatasetId)}/${fileName}`;
+    const scriptPath = `${getDatasetScriptsDir(targetDatasetId)}/extract_pdf_text.py`;
     await runDatasetSandboxCommandStep({
         runtime: state.runtime,
         sandboxId,
         cmd: "mkdir",
-        args: ["-p", workstation],
+        args: ["-p", ...getDatasetStandardDirs(targetDatasetId)],
     });
     await writeDatasetSandboxFilesStep({
         runtime: state.runtime,
@@ -222,24 +223,11 @@ async function materializeRawTextSource(state, source, targetDatasetId) {
     });
     return targetDatasetId;
 }
-async function resolveDatasetSandboxId(state, targetDatasetId) {
+function resolveDatasetSandboxId(state, _targetDatasetId) {
     const sandboxId = String(state.sandboxId ?? "").trim();
     if (sandboxId)
         return sandboxId;
-    const created = await createDatasetSandboxStep({
-        runtime: state.runtime,
-        provider: "vercel",
-        sandboxRuntime: "python3.13",
-        timeoutMs: 20 * 60 * 1000,
-        resources: { vcpus: 2 },
-        purpose: "dataset.materialize",
-        params: { datasetId: targetDatasetId },
-        vercel: {
-            profile: "ephemeral",
-            deleteOnStop: true,
-        },
-    });
-    return created.sandboxId;
+    throw new Error("dataset_sandbox_required");
 }
 export async function resolveDatasetAgentDurable(requestedDurable) {
     if (!requestedDurable)
@@ -255,20 +243,150 @@ export async function resolveDatasetAgentDurable(requestedDurable) {
     }
     return true;
 }
+export async function initializeDatasetStep(params) {
+    "use step";
+    await createOrUpdateDatasetMetadata(params.runtime, {
+        datasetId: params.datasetId,
+        sandboxId: params.sandboxId,
+        title: params.title ?? params.datasetId,
+        instructions: params.instructions,
+        sources: params.sources,
+        sourceKinds: params.sourceKinds,
+        schema: params.schema,
+        status: "building",
+    });
+    return {
+        datasetId: params.datasetId,
+        sandboxId: params.sandboxId,
+    };
+}
+export async function prepareDatasetSourcesStep(params) {
+    "use step";
+    if (params.kind === "file") {
+        const fileId = params.source.kind === "file"
+            ? params.source.fileId
+            : await uploadInlineTextSource(params.runtime, params.datasetId, params.source);
+        const initialized = await initializeFileParseSandboxStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            datasetId: params.datasetId,
+            fileId,
+            state: { initialized: false, filePath: "" },
+        });
+        const filePreview = await generateFileParsePreviewStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            sandboxFilePath: initialized.filePath,
+            datasetId: params.datasetId,
+        });
+        return {
+            kind: "file",
+            datasetId: params.datasetId,
+            sandboxId: params.sandboxId,
+            fileId,
+            sandboxState: initialized.state,
+            filePreview,
+            schema: params.schema ?? null,
+        };
+    }
+    const initialized = await ensureTransformSourcesInSandboxStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        datasetId: params.datasetId,
+        sourceDatasetIds: params.sourceDatasetIds,
+        state: { initialized: false, sourcePaths: [] },
+    });
+    const sourcePreviews = await generateTransformSourcePreviewsStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        datasetId: params.datasetId,
+        sourcePaths: initialized.sourcePaths,
+    });
+    return {
+        kind: "transform",
+        datasetId: params.datasetId,
+        sandboxId: params.sandboxId,
+        sourceDatasetIds: params.sourceDatasetIds,
+        outputSchema: params.outputSchema,
+        sandboxState: initialized.state,
+        sourcePreviews,
+    };
+}
+export async function initializeDatasetContextStep(params) {
+    "use step";
+    if (params.prepared.kind === "file") {
+        return {
+            ...params.prepared,
+            instructions: params.instructions ?? buildFileDefaultInstructions(params.outputSchema),
+            prompt: "generate a dataset for this file",
+        };
+    }
+    return {
+        ...params.prepared,
+        instructions: params.instructions,
+        prompt: params.prepared.sourceDatasetIds.length === 1
+            ? "Transform the source dataset into a new dataset matching the provided output schema"
+            : `Transform ${params.prepared.sourceDatasetIds.length} source datasets into a new dataset matching the provided output schema`,
+    };
+}
+export async function completeDatasetStep(params) {
+    "use step";
+    let datasetResult = await datasetGetByIdStep({
+        runtime: params.runtime,
+        datasetId: params.datasetId,
+    });
+    if (!datasetResult.ok)
+        throw new Error(datasetResult.error);
+    if (!params.schema && !datasetResult.data?.schema) {
+        await datasetInferAndUpdateSchemaStep({
+            runtime: params.runtime,
+            datasetId: params.datasetId,
+            title: `${params.datasetId}Row`,
+            description: "One dataset row",
+        });
+        datasetResult = await datasetGetByIdStep({
+            runtime: params.runtime,
+            datasetId: params.datasetId,
+        });
+        if (!datasetResult.ok)
+            throw new Error(datasetResult.error);
+    }
+    const previewResult = await datasetPreviewRowsStep({
+        runtime: params.runtime,
+        datasetId: params.datasetId,
+        limit: 20,
+    });
+    if (!params.first) {
+        return {
+            datasetId: params.datasetId,
+            dataset: datasetResult.data,
+            previewRows: previewResult.rows,
+            firstRow: undefined,
+        };
+    }
+    const firstResult = await datasetReadOneStep({
+        runtime: params.runtime,
+        datasetId: params.datasetId,
+    });
+    return {
+        datasetId: params.datasetId,
+        dataset: datasetResult.data,
+        previewRows: previewResult.rows,
+        firstRow: firstResult.row,
+    };
+}
 export async function materializeSingleFileLikeSource(state, source, targetDatasetId) {
     if (source.kind === "file" && !state.outputSchema) {
         const materializedPdf = await tryMaterializeRawPdfFileSource(state, source, targetDatasetId);
         if (materializedPdf)
             return materializedPdf;
     }
+    const sandboxId = resolveDatasetSandboxId(state, targetDatasetId);
     if (!state.reactor) {
         throw new Error("dataset_reactor_required");
     }
-    const sandboxId = await resolveDatasetSandboxId(state, targetDatasetId);
-    const fileId = source.kind === "file"
-        ? source.fileId
-        : await uploadInlineTextSource(state.runtime, targetDatasetId, source);
-    await createOrUpdateDatasetMetadata(state.runtime, {
+    await initializeDatasetStep({
+        runtime: state.runtime,
         datasetId: targetDatasetId,
         sandboxId,
         title: state.title ?? targetDatasetId,
@@ -285,28 +403,45 @@ export async function materializeSingleFileLikeSource(state, source, targetDatas
         ],
         sourceKinds: [source.kind],
         schema: state.outputSchema,
-        status: "building",
     });
-    const parseContext = createFileParseContext(fileId, {
+    const prepared = await prepareDatasetSourcesStep({
+        kind: "file",
+        runtime: state.runtime,
         datasetId: targetDatasetId,
-        instructions: state.instructions ?? buildFileDefaultInstructions(state.outputSchema),
-        reactor: state.reactor,
         sandboxId,
+        source,
+        schema: state.outputSchema,
+    });
+    const context = await initializeDatasetContextStep({
+        prepared,
+        instructions: state.instructions,
+        outputSchema: state.outputSchema,
+    });
+    if (context.kind !== "file") {
+        throw new Error("dataset_context_kind_mismatch:file");
+    }
+    const parseContext = createFileParseContext(context.fileId, {
+        datasetId: context.datasetId,
+        instructions: context.instructions,
+        reactor: state.reactor,
+        sandboxId: context.sandboxId,
+        sandboxState: context.sandboxState,
+        filePreview: context.filePreview,
+        schema: context.schema,
     });
     await parseContext.parse(state.runtime, {
         durable: await resolveDatasetAgentDurable(state.durable),
+        prompt: context.prompt,
+        initialContent: {
+            datasetId: context.datasetId,
+            fileId: context.fileId,
+            instructions: context.instructions ?? "",
+            sandboxId: context.sandboxId,
+            sandboxState: context.sandboxState,
+            filePreview: context.filePreview,
+            schema: context.schema,
+        },
     });
-    if (!state.outputSchema) {
-        await datasetInferAndUpdateSchemaStep({
-            runtime: state.runtime,
-            datasetId: targetDatasetId,
-            title: `${targetDatasetId}Row`,
-            description: "One dataset row",
-        });
-    }
-    if (state.first) {
-        await datasetReadOneStep({ runtime: state.runtime, datasetId: targetDatasetId });
-    }
     return targetDatasetId;
 }
 async function normalizeSourceToDatasetId(state, source, targetDatasetId, sourceIndex) {
@@ -345,7 +480,7 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
     if (!state.reactor) {
         throw new Error("dataset_reactor_required");
     }
-    const sandboxId = await resolveDatasetSandboxId(state, targetDatasetId);
+    const sandboxId = resolveDatasetSandboxId(state, targetDatasetId);
     const stateWithSandbox = { ...state, sandboxId };
     const normalizedSources = [];
     for (let index = 0; index < stateWithSandbox.sources.length; index++) {
@@ -361,7 +496,8 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
                 properties: {},
             },
         };
-    await createOrUpdateDatasetMetadata(stateWithSandbox.runtime, {
+    await initializeDatasetStep({
+        runtime: stateWithSandbox.runtime,
         datasetId: targetDatasetId,
         sandboxId,
         title: stateWithSandbox.title ?? targetDatasetId,
@@ -377,30 +513,46 @@ export async function materializeDerivedDataset(state, targetDatasetId) {
             : source),
         sourceKinds: stateWithSandbox.sources.map((source) => source.kind),
         schema: transformSchema,
-        status: "building",
     });
-    const transformContext = createTransformDatasetContext({
+    const prepared = await prepareDatasetSourcesStep({
+        kind: "transform",
+        runtime: stateWithSandbox.runtime,
+        datasetId: targetDatasetId,
+        sandboxId,
         sourceDatasetIds: normalizedSources,
         outputSchema: transformSchema,
+    });
+    const context = await initializeDatasetContextStep({
+        prepared,
         instructions: buildTransformInstructions(normalizedSources.length, stateWithSandbox.instructions, stateWithSandbox.outputSchema),
-        datasetId: targetDatasetId,
+        outputSchema: transformSchema,
+    });
+    if (context.kind !== "transform") {
+        throw new Error("dataset_context_kind_mismatch:transform");
+    }
+    const transformContext = createTransformDatasetContext({
+        sourceDatasetIds: context.sourceDatasetIds,
+        outputSchema: context.outputSchema,
+        instructions: context.instructions,
+        datasetId: context.datasetId,
         reactor: stateWithSandbox.reactor,
-        sandboxId,
+        sandboxId: context.sandboxId,
+        sandboxState: context.sandboxState,
+        sourcePreviews: context.sourcePreviews,
     });
     await transformContext.transform(stateWithSandbox.runtime, {
         durable: await resolveDatasetAgentDurable(stateWithSandbox.durable),
+        prompt: context.prompt,
+        initialContent: {
+            datasetId: context.datasetId,
+            sourceDatasetIds: context.sourceDatasetIds,
+            outputSchema: context.outputSchema,
+            instructions: context.instructions,
+            sandboxId: context.sandboxId,
+            sandboxState: context.sandboxState,
+            sourcePreviews: context.sourcePreviews,
+        },
     });
-    if (!stateWithSandbox.outputSchema) {
-        await datasetInferAndUpdateSchemaStep({
-            runtime: stateWithSandbox.runtime,
-            datasetId: targetDatasetId,
-            title: `${targetDatasetId}Row`,
-            description: "One dataset row",
-        });
-    }
-    if (stateWithSandbox.first) {
-        await datasetReadOneStep({ runtime: stateWithSandbox.runtime, datasetId: targetDatasetId });
-    }
     return targetDatasetId;
 }
 registerDatasetAgentMaterializers({

package/dist/builder/persistence.d.ts

@@ -15,3 +15,9 @@ export declare function createOrUpdateDatasetMetadata<Runtime extends AnyDataset
 export declare function materializeRowsToDataset<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: MaterializeRowsParams): Promise<string>;
 export declare function uploadInlineTextSource<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, source: DatasetTextSourceInput): Promise<string>;
 export declare function finalizeBuildResult<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, withFirst: boolean): Promise<DatasetBuildResult>;
+export declare function createDatasetBuildResult<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: {
+    datasetId: string;
+    dataset: any;
+    previewRows: any[];
+    firstRow?: any | null;
+}): DatasetBuildResult;

package/dist/builder/persistence.js

@@ -128,3 +128,25 @@ export async function finalizeBuildResult(runtime, datasetId, withFirst) {
         firstRow: firstResult.row,
     };
 }
+export function createDatasetBuildResult(runtime, params) {
+    const reader = {
+        async read(cursorOrParams, limit) {
+            const readParams = typeof cursorOrParams === "object" && cursorOrParams !== null
+                ? cursorOrParams
+                : { cursor: cursorOrParams, limit };
+            return await datasetReadRowsStep({
+                runtime,
+                datasetId: params.datasetId,
+                cursor: readParams.cursor,
+                limit: readParams.limit,
+            });
+        },
+    };
+    return {
+        datasetId: params.datasetId,
+        dataset: params.dataset,
+        previewRows: params.previewRows,
+        reader,
+        ...(params.firstRow !== undefined ? { firstRow: params.firstRow } : {}),
+    };
+}

package/dist/completeDataset.steps.d.ts

@@ -0,0 +1,87 @@
+export interface PersistDatasetStepParams {
+    datasetId: string;
+    sandboxId: string;
+    runtime: any;
+    summary?: string;
+}
+export declare function persistDatasetStep({ runtime, datasetId, sandboxId, summary }: PersistDatasetStepParams): Promise<{
+    success: boolean;
+    validation?: RowValidationEntry[];
+    validationTruncated?: number;
+    failureSummary?: ValidationFailureSummary;
+    repairInstructions?: string[];
+    validRowCount?: number;
+    rowRecordCount?: number;
+    error?: string;
+    status?: string;
+    message?: string;
+} | {
+    success: boolean;
+    status: string;
+    validRows: number;
+    rowRecordCount: number;
+    validation: RowValidationEntry[] | undefined;
+    error: string;
+    message: string;
+    fileId?: undefined;
+    storagePath?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    validRows: number;
+    rowRecordCount: number;
+    fileId: string;
+    storagePath: string;
+    message: string;
+    validation?: undefined;
+    error?: undefined;
+}>;
+type RowValidationEntry = {
+    index: number;
+    valid: boolean;
+    errors?: string[];
+    errorDetails?: Array<{
+        path: string;
+        keyword: string;
+        message: string;
+        params?: Record<string, unknown>;
+        schemaPath?: string;
+    }>;
+    dataKeys?: string[];
+};
+type ValidationFailureSummary = {
+    rowRecordCount: number;
+    validRowCount: number;
+    invalidRowCount: number;
+    expectedTopLevelKeys: string[];
+    requiredTopLevelKeys: string[];
+    requiredPaths: string[];
+    enumConstraints: Array<{
+        path: string;
+        values: unknown[];
+    }>;
+    topErrors: Array<{
+        message: string;
+        count: number;
+    }>;
+    missingRequiredProperties: Array<{
+        property: string;
+        count: number;
+    }>;
+    additionalProperties: Array<{
+        property: string;
+        count: number;
+    }>;
+    enumFailures: Array<{
+        path: string;
+        allowedValues: unknown[];
+        count: number;
+    }>;
+    observedTopLevelKeys: string[];
+    sampleInvalidRows: Array<{
+        index: number;
+        dataKeys?: string[];
+        errors?: string[];
+    }>;
+};
+export {};