@ekairos/dataset 1.22.40-beta.development.0 → 1.22.40

package/dist/completeDataset.tool.d.ts

@@ -1,25 +1,150 @@
 interface CompleteDatasetToolParams {
     datasetId: string;
     sandboxId: string;
-    env?: any;
+    runtime: any;
+    outputPath?: string;
 }
-export declare function createCompleteDatasetTool({ datasetId, sandboxId, env }: CompleteDatasetToolParams): import("ai").Tool<{
+export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outputPath }: CompleteDatasetToolParams): import("ai").Tool<{
     summary: string;
 }, {
+    rowSource: string;
+    outputPath: string;
+    storagePath: string;
     success: boolean;
-    validation?: Array<{
+    validation?: {
         index: number;
         valid: boolean;
         errors?: string[];
-    }>;
+        errorDetails?: Array<{
+            path: string;
+            keyword: string;
+            message: string;
+            params?: Record<string, unknown>;
+            schemaPath?: string;
+        }>;
+        dataKeys?: string[];
+    }[];
+    validationTruncated?: number;
+    failureSummary?: {
+        rowRecordCount: number;
+        validRowCount: number;
+        invalidRowCount: number;
+        expectedTopLevelKeys: string[];
+        requiredTopLevelKeys: string[];
+        requiredPaths: string[];
+        enumConstraints: Array<{
+            path: string;
+            values: unknown[];
+        }>;
+        topErrors: Array<{
+            message: string;
+            count: number;
+        }>;
+        missingRequiredProperties: Array<{
+            property: string;
+            count: number;
+        }>;
+        additionalProperties: Array<{
+            property: string;
+            count: number;
+        }>;
+        enumFailures: Array<{
+            path: string;
+            allowedValues: unknown[];
+            count: number;
+        }>;
+        observedTopLevelKeys: string[];
+        sampleInvalidRows: Array<{
+            index: number;
+            dataKeys?: string[];
+            errors?: string[];
+        }>;
+    };
+    repairInstructions?: string[];
     validRowCount?: number;
+    rowRecordCount?: number;
     error?: string;
+    status?: string;
+    message?: string;
+    validRows?: undefined;
+    dataFileId?: undefined;
+    records?: undefined;
+    summary?: undefined;
 } | {
     success: boolean;
-    validRows: number | undefined;
-    fileId: string;
+    status: string;
+    rowSource: string;
+    validRows: number;
+    rowRecordCount: number;
+    validation: {
+        index: number;
+        valid: boolean;
+        errors?: string[];
+        errorDetails?: Array<{
+            path: string;
+            keyword: string;
+            message: string;
+            params?: Record<string, unknown>;
+            schemaPath?: string;
+        }>;
+        dataKeys?: string[];
+    }[] | undefined;
+    error: string;
+    message: string;
+    outputPath: string;
     storagePath: string;
+    dataFileId?: undefined;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    validRows: number;
+    rowRecordCount: number;
+    validation: {
+        index: number;
+        valid: boolean;
+        errors?: string[];
+        errorDetails?: Array<{
+            path: string;
+            keyword: string;
+            message: string;
+            params?: Record<string, unknown>;
+            schemaPath?: string;
+        }>;
+        dataKeys?: string[];
+    }[] | undefined;
+    error: string;
     message: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId: string;
+    records?: undefined;
+    summary?: undefined;
+} | {
+    success: boolean;
+    status: string;
+    rowSource: string;
+    records: number;
+    summary: string;
+    outputPath: string;
+    storagePath: string;
+    dataFileId: string;
+    validRows?: undefined;
+    rowRecordCount?: undefined;
+    validation?: undefined;
+    error?: undefined;
+    message?: undefined;
 }>;
+export declare function didCompleteDatasetSucceed(event: {
+    content?: {
+        parts?: any[];
+    };
+}): boolean;
+export declare function getDatasetFatalFailure(event: {
+    content?: {
+        parts?: any[];
+    };
+}): string | null;
 export {};
-//# sourceMappingURL=completeDataset.tool.d.ts.map

package/dist/completeDataset.tool.js

@@ -1,212 +1,66 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createCompleteDatasetTool = createCompleteDatasetTool;
-const ai_1 = require("ai");
-const zod_1 = require("zod");
-const steps_1 = require("./sandbox/steps");
-const ajv_1 = __importDefault(require("ajv"));
-const datasetFiles_1 = require("./datasetFiles");
-const steps_2 = require("./dataset/steps");
-let ajvInstance = null;
-function getAjv() {
-    if (!ajvInstance) {
-        ajvInstance = new ajv_1.default({
-            allErrors: true,
-            strict: false,
-        });
-    }
-    return ajvInstance;
-}
-function createCompleteDatasetTool({ datasetId, sandboxId, env }) {
-    return (0, ai_1.tool)({
-        description: "Mark the dataset as completed. Use only when output.jsonl has been successfully generated and is ready for validation.",
-        inputSchema: zod_1.z.object({
-            summary: zod_1.z.string().describe("Summary of the completed dataset including record count and structure"),
+import { tool } from "ai";
+import { z } from "zod";
+import { persistDatasetStep } from "./completeDataset.steps.js";
+export function createCompleteDatasetTool({ datasetId, sandboxId, runtime, outputPath }) {
+    return tool({
+        description: "Validate and complete the dataset from output.jsonl. The result includes the JSONL outputPath and storagePath used for completion.",
+        inputSchema: z.object({
+            summary: z.string().describe("Summary of the completed dataset including record count and structure"),
         }),
         execute: async ({ summary }) => {
             console.log(`[Dataset ${datasetId}] ========================================`);
             console.log(`[Dataset ${datasetId}] Tool: completeDataset`);
             console.log(`[Dataset ${datasetId}] Summary: ${summary}`);
             console.log(`[Dataset ${datasetId}] ========================================`);
-            const outputPath = (0, datasetFiles_1.getDatasetOutputPath)(datasetId);
-            try {
-                await ensureFileExists(env, sandboxId, outputPath);
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                console.error(`[Dataset ${datasetId}] Missing output file:`, message);
-                return {
-                    success: false,
-                    error: message,
-                };
-            }
-            console.log(`[Dataset ${datasetId}] Validating dataset rows against schema`);
-            const datasetResult = await (0, steps_2.datasetGetByIdStep)({ env, datasetId });
-            if (!datasetResult.ok) {
-                console.error(`[Dataset ${datasetId}] ${datasetResult.error}`);
-                return {
-                    success: false,
-                    error: datasetResult.error,
-                };
-            }
-            const datasetRecord = datasetResult.data;
-            if (!datasetRecord.schema) {
-                console.error(`[Dataset ${datasetId}] Schema not found in database`);
-                return {
-                    success: false,
-                    error: "Schema not found in database. Please generate schema first.",
-                };
-            }
-            const schemaJson = datasetRecord.schema.schema;
-            let validator;
-            try {
-                validator = getAjv().compile(schemaJson);
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                console.error(`[Dataset ${datasetId}] Failed to compile schema:`, message);
-                return {
-                    success: false,
-                    error: `Failed to compile schema: ${message}`,
-                };
-            }
-            const validationResult = await validateJsonlRows({
-                env,
+            return await persistDatasetStep({
+                runtime,
+                datasetId,
                 sandboxId,
+                summary,
                 outputPath,
-                validator,
-                datasetId,
-            });
-            if (!validationResult.success) {
-                return validationResult;
-            }
-            const totalValidRows = validationResult.validRowCount;
-            console.log(`[Dataset ${datasetId}] Reading file content for upload`);
-            const fileRead = await (0, steps_1.readDatasetSandboxFileStep)({ env, sandboxId, path: outputPath });
-            if (!fileRead.contentBase64) {
-                console.error(`[Dataset ${datasetId}] Empty file content`);
-                return {
-                    success: false,
-                    error: "Empty file content",
-                };
-            }
-            const fileBuffer = Buffer.from(fileRead.contentBase64, "base64");
-            console.log(`[Dataset ${datasetId}] Uploading file to InstantDB storage`);
-            const uploadResult = await (0, steps_2.datasetUploadOutputFileStep)({ env, datasetId, fileBuffer });
-            if (!uploadResult.ok) {
-                console.error(`[Dataset ${datasetId}] File upload failed: ${uploadResult.error}`);
-                return {
-                    success: false,
-                    error: uploadResult.error,
-                };
-            }
-            console.log(`[Dataset ${datasetId}] File uploaded successfully: ${uploadResult.data.fileId}`);
-            const statusResult = await (0, steps_2.datasetUpdateStatusStep)({
-                env,
-                datasetId,
-                status: "completed",
-                calculatedTotalRows: totalValidRows,
-                actualGeneratedRowCount: totalValidRows,
             });
-            if (!statusResult.ok) {
-                console.error(`[Dataset ${datasetId}] Failed to update status: ${statusResult.error}`);
-                return {
-                    success: false,
-                    error: statusResult.error,
-                };
-            }
-            console.log(`[Dataset ${datasetId}] Dataset marked as COMPLETED (${totalValidRows} valid rows)`);
-            console.log(`[Dataset ${datasetId}] ========================================`);
-            return {
-                success: true,
-                validRows: totalValidRows,
-                fileId: uploadResult.data.fileId,
-                storagePath: uploadResult.data.storagePath,
-                message: "Dataset creation completed and uploaded to storage",
-            };
         },
     });
 }
-async function ensureFileExists(env, sandboxId, path) {
-    const result = await (0, steps_1.runDatasetSandboxCommandStep)({
-        env,
-        sandboxId,
-        cmd: "test",
-        args: ["-f", path],
-    });
-    if (result.exitCode !== 0) {
-        throw new Error(`Required file not found: ${path}`);
-    }
-}
-async function validateJsonlRows({ env, sandboxId, outputPath, validator, datasetId }) {
-    const validation = [];
-    let validRowCount = 0;
-    console.log(`[Dataset ${datasetId}] Reading and validating JSONL file from sandbox`);
-    const fileRead = await (0, steps_1.readDatasetSandboxFileStep)({ env, sandboxId, path: outputPath });
-    if (!fileRead.contentBase64) {
-        console.log(`[Dataset ${datasetId}] Empty output file`);
-        return { success: true, validation, validRowCount: 0 };
-    }
-    const fileContent = Buffer.from(fileRead.contentBase64, "base64").toString();
-    const lines = fileContent.split("\n");
-    console.log(`[Dataset ${datasetId}] Validating ${lines.length} lines`);
-    for (let index = 0; index < lines.length; index++) {
-        const line = lines[index];
-        const trimmed = line.trim();
-        if (trimmed.length === 0) {
-            continue;
+export function didCompleteDatasetSucceed(event) {
+    const parts = Array.isArray(event?.content?.parts) ? event.content.parts : [];
+    return parts.some((part) => {
+        if (part?.type === "action" &&
+            ["completeDataset", "completeObject", "replaceRows"].includes(part?.content?.actionName)) {
+            const output = part.content.output;
+            return part.content.status === "completed" && output?.success === true && output?.status === "completed";
         }
-        let record;
-        try {
-            record = JSON.parse(trimmed);
+        if (part?.type === "tool-completeDataset" ||
+            part?.type === "tool-completeObject" ||
+            part?.type === "tool-replaceRows") {
+            const output = part.output ?? part.result;
+            return part.state === "output-available" && output?.success === true && output?.status === "completed";
         }
-        catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            validation.push({
-                index,
-                valid: false,
-                errors: [`Invalid JSON: ${message}`],
-            });
-            continue;
-        }
-        if (record.type !== "row") {
-            continue;
+        return false;
+    });
+}
+export function getDatasetFatalFailure(event) {
+    const parts = Array.isArray(event?.content?.parts) ? event.content.parts : [];
+    for (const part of parts) {
+        let actionName;
+        let output;
+        if (part?.type === "action") {
+            actionName = part.content?.actionName;
+            output = part.content?.output;
         }
-        const data = record.data;
-        if (data === undefined || data === null) {
-            validation.push({
-                index,
-                valid: false,
-                errors: ["Missing 'data' field"],
-            });
-            continue;
+        else if (typeof part?.type === "string" && part.type.startsWith("tool-")) {
+            actionName = part.type.slice("tool-".length);
+            output = part.output ?? part.result;
         }
-        const valid = validator(data);
-        if (!valid) {
-            const errors = Array.isArray(validator.errors)
-                ? validator.errors.map((err) => err.message || "Unknown validation error")
-                : ["Unknown validation error"];
-            validation.push({
-                index,
-                valid: false,
-                errors,
-            });
+        if (!output || output.fatal !== true) {
             continue;
         }
-        validation.push({
-            index,
-            valid: true,
-        });
-        validRowCount++;
+        const message = typeof output.error === "string" && output.error.trim()
+            ? output.error.trim()
+            : typeof output.message === "string" && output.message.trim()
+                ? output.message.trim()
+                : "Dataset action failed fatally";
+        return actionName ? `${actionName}: ${message}` : message;
     }
-    console.log(`[Dataset ${datasetId}] Validation completed: ${validRowCount} valid rows`);
-    return {
-        success: true,
-        validation,
-        validRowCount,
-    };
+    return null;
 }
-//# sourceMappingURL=completeDataset.tool.js.map

package/dist/contextResources.d.ts

@@ -0,0 +1,31 @@
+import type { StoredContextResource } from "@ekairos/events";
+type MaterializedContextResource = {
+    key: string;
+    type: string;
+    name: string;
+    description: string;
+    dir: string;
+    metadataPath: string;
+    files: Array<{
+        path: string;
+        role: string;
+        mediaType?: string;
+    }>;
+    status: "materialized" | "metadata_only" | "skipped";
+    reason?: string;
+};
+export type ContextResourcesMaterialization = {
+    contextId: string;
+    sandboxId: string;
+    resourcesDir: string;
+    manifestPath: string;
+    resources: MaterializedContextResource[];
+};
+export declare function materializeContextResourcesStep(params: {
+    runtime: any;
+    sandboxId: string;
+    contextId: string;
+    resources: StoredContextResource[];
+    resourceKeys?: string[] | null;
+}): Promise<ContextResourcesMaterialization>;
+export {};

package/dist/contextResources.js

@@ -0,0 +1,151 @@
+import { datasetReadOutputJsonlStep } from "./dataset/steps.js";
+import { readInstantFileStep } from "./file/steps.js";
+import { getContextResourcesDir, sanitizeContextWorkspacePathSegment, } from "./contextWorkspace.js";
+import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFilesStep, } from "./sandbox/steps.js";
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function asString(value) {
+    return typeof value === "string" ? value.trim() : "";
+}
+function contentFileName(resource) {
+    const filename = asString(resource.filename);
+    if (filename) {
+        return sanitizeContextWorkspacePathSegment(filename, "resource");
+    }
+    if (resource.type === "dataset")
+        return "resource.jsonl";
+    if (resource.type === "text")
+        return "resource.txt";
+    return "resource";
+}
+function selectResources(resources, resourceKeys) {
+    const requested = new Set(Array.isArray(resourceKeys)
+        ? resourceKeys.map((key) => String(key).trim()).filter(Boolean)
+        : []);
+    if (requested.size === 0)
+        return resources;
+    return resources.filter((resource) => requested.has(resource.key));
+}
+export async function materializeContextResourcesStep(params) {
+    "use step";
+    const resourcesDir = getContextResourcesDir({ contextId: params.contextId });
+    const manifestPath = `${resourcesDir}/manifest.json`;
+    const selectedResources = selectResources(params.resources, params.resourceKeys);
+    const resourceDirs = selectedResources.map((resource, index) => {
+        const segment = sanitizeContextWorkspacePathSegment(resource.key, `resource_${index + 1}`);
+        return `${resourcesDir}/${segment}`;
+    });
+    await runDatasetSandboxCommandStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        cmd: "mkdir",
+        args: ["-p", resourcesDir, ...resourceDirs],
+    });
+    const materialized = [];
+    for (let index = 0; index < selectedResources.length; index++) {
+        const resource = selectedResources[index];
+        const resourceDir = resourceDirs[index];
+        const metadataPath = `${resourceDir}/metadata.json`;
+        const files = [];
+        let status = "metadata_only";
+        let reason;
+        if (resource.type === "file" && asString(resource.fileId)) {
+            const file = await readInstantFileStep({
+                runtime: params.runtime,
+                fileId: asString(resource.fileId),
+            });
+            const path = `${resourceDir}/${contentFileName(resource)}`;
+            await writeDatasetSandboxFilesStep({
+                runtime: params.runtime,
+                sandboxId: params.sandboxId,
+                files: [{ path, contentBase64: file.contentBase64 }],
+            });
+            files.push({ path, role: "content", mediaType: asString(resource.mediaType) || undefined });
+            status = "materialized";
+        }
+        else if (resource.type === "dataset" && asString(resource.datasetId)) {
+            const datasetId = asString(resource.datasetId);
+            const content = await datasetReadOutputJsonlStep({
+                runtime: params.runtime,
+                datasetId,
+            });
+            const path = `${resourceDir}/resource.jsonl`;
+            await writeDatasetSandboxFilesStep({
+                runtime: params.runtime,
+                sandboxId: params.sandboxId,
+                files: [{ path, contentBase64: content.contentBase64 }],
+            });
+            files.push({ path, role: "content", mediaType: "application/x-ndjson" });
+            status = "materialized";
+        }
+        else if (resource.type === "text" && typeof resource.text === "string") {
+            const path = `${resourceDir}/${contentFileName(resource)}`;
+            await writeDatasetSandboxTextFilesStep({
+                runtime: params.runtime,
+                sandboxId: params.sandboxId,
+                files: [{ path, content: String(resource.text) }],
+            });
+            files.push({
+                path,
+                role: "content",
+                mediaType: asString(resource.mimeType) || "text/plain",
+            });
+            status = "materialized";
+        }
+        else {
+            reason =
+                resource.type === "file"
+                    ? "file resource has no fileId"
+                    : resource.type === "dataset"
+                        ? "dataset resource has no datasetId"
+                        : resource.type === "link" || resource.type === "repository" || resource.type === "external"
+                            ? `${resource.type} resources are metadata-only until an approved adapter materializes them`
+                            : "resource type is metadata-only";
+        }
+        const metadata = {
+            ...(asRecord(resource) ?? {}),
+            key: resource.key,
+            type: resource.type,
+            name: resource.name,
+            description: resource.description,
+            materialized: {
+                status,
+                reason,
+                dir: resourceDir,
+                files,
+            },
+        };
+        await writeDatasetSandboxTextFilesStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            files: [{ path: metadataPath, content: JSON.stringify(metadata, null, 2) }],
+        });
+        materialized.push({
+            key: resource.key,
+            type: resource.type,
+            name: resource.name,
+            description: resource.description,
+            dir: resourceDir,
+            metadataPath,
+            files,
+            status,
+            reason,
+        });
+    }
+    const manifest = {
+        contextId: params.contextId,
+        sandboxId: params.sandboxId,
+        resourcesDir,
+        manifestPath,
+        resources: materialized,
+    };
+    await writeDatasetSandboxTextFilesStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        files: [{ path: manifestPath, content: JSON.stringify(manifest, null, 2) }],
+    });
+    return manifest;
+}

package/dist/contextWorkspace.d.ts

@@ -0,0 +1,79 @@
+export type ContextWorkspaceFileRole = "input" | "output" | "artifact";
+export type ContextWorkspaceFileInput = {
+    fileId: string;
+    filename?: string;
+    mediaType?: string;
+    role?: ContextWorkspaceFileRole;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+};
+export type PreparedContextWorkspaceFile = {
+    fileId: string;
+    filename: string;
+    mediaType?: string;
+    role: ContextWorkspaceFileRole;
+    path: string;
+    sourceEventId?: string;
+    sourcePartIndex?: number;
+};
+export type PreparedContextExecutionWorkspace = {
+    contextId: string;
+    executionId: string;
+    sandboxId: string;
+    root: string;
+    contextRoot: string;
+    eventsDir: string;
+    resourcesDir: string;
+    outputDir: string;
+    scriptsDir: string;
+    tmpDir: string;
+    manifestPath: string;
+    files: PreparedContextWorkspaceFile[];
+};
+export declare function sanitizeContextWorkspacePathSegment(value: string, fallback: string): string;
+export declare function getContextWorkspaceBase(): string;
+export declare function getContextExecutionWorkspaceRoot(params: {
+    contextId: string;
+    executionId: string;
+    root?: string;
+}): string;
+export declare function getContextWorkspaceRoot(params: {
+    contextId: string;
+    root?: string;
+}): string;
+export declare function getContextEventsDir(params: {
+    contextId: string;
+    root?: string;
+}): string;
+export declare function getContextResourcesDir(params: {
+    contextId: string;
+    root?: string;
+}): string;
+export declare function getContextExecutionWorkspaceDirs(params: {
+    contextId: string;
+    executionId: string;
+    root?: string;
+}): {
+    root: string;
+    contextRoot: string;
+    eventsDir: string;
+    resourcesDir: string;
+    outputDir: string;
+    scriptsDir: string;
+    tmpDir: string;
+    manifestPath: string;
+};
+export declare function getContextExecutionWorkspaceStandardDirs(params: {
+    contextId: string;
+    executionId: string;
+    root?: string;
+}): string[];
+export declare function extractContextWorkspaceFilesFromEventItems(eventItems: unknown[]): ContextWorkspaceFileInput[];
+export declare function prepareContextExecutionWorkspaceStep(params: {
+    runtime: any;
+    sandboxId: string;
+    contextId: string;
+    executionId: string;
+    files: ContextWorkspaceFileInput[];
+    root?: string;
+}): Promise<PreparedContextExecutionWorkspace>;