@ekairos/dataset 1.22.79-beta.development.0 → 1.22.80-beta.development.0

package/dist/completeDataset.steps.js

@@ -0,0 +1,449 @@
+import Ajv from "ajv";
+import { getDatasetOutputPath } from "./datasetFiles.js";
+import { DatasetService } from "./service.js";
+import { getDatasetRuntimeDb } from "./dataset/steps.js";
+import { readDatasetSandboxFileStep, readDatasetSandboxTextFileStep, runDatasetSandboxCommandStep, } from "./sandbox/steps.js";
+let ajvInstance = null;
+function getAjv() {
+    if (!ajvInstance) {
+        ajvInstance = new Ajv({
+            allErrors: true,
+            strict: false,
+        });
+    }
+    return ajvInstance;
+}
+export async function persistDatasetStep({ runtime, datasetId, sandboxId, summary }) {
+    "use step";
+    const outputPath = getDatasetOutputPath(datasetId);
+    if (summary) {
+        console.log(`[Dataset ${datasetId}] Persisting completed dataset: ${summary}`);
+    }
+    try {
+        await ensureFileExists(runtime, sandboxId, outputPath);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(`[Dataset ${datasetId}] Missing output file:`, message);
+        return {
+            success: false,
+            status: "missing_output",
+            validRows: 0,
+            rowRecordCount: 0,
+            validation: [],
+            error: message,
+            message,
+        };
+    }
+    console.log(`[Dataset ${datasetId}] Validating dataset rows against schema`);
+    const db = await getDatasetRuntimeDb(runtime);
+    const service = new DatasetService(db);
+    const datasetResult = await service.getDatasetById(datasetId);
+    if (!datasetResult.ok) {
+        console.error(`[Dataset ${datasetId}] ${datasetResult.error}`);
+        return {
+            success: false,
+            status: "dataset_not_found",
+            validRows: 0,
+            rowRecordCount: 0,
+            validation: [],
+            error: datasetResult.error,
+            message: datasetResult.error,
+        };
+    }
+    const datasetRecord = datasetResult.data;
+    if (!datasetRecord.schema) {
+        console.error(`[Dataset ${datasetId}] Schema not found in database`);
+        return {
+            success: false,
+            status: "schema_missing",
+            validRows: 0,
+            rowRecordCount: 0,
+            validation: [],
+            error: "Schema not found in database. Please generate schema first.",
+            message: "Schema not found in database. Please generate schema first.",
+        };
+    }
+    const schemaJson = datasetRecord.schema.schema;
+    let validator;
+    try {
+        validator = getAjv().compile(schemaJson);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(`[Dataset ${datasetId}] Failed to compile schema:`, message);
+        return {
+            success: false,
+            status: "schema_invalid",
+            validRows: 0,
+            rowRecordCount: 0,
+            validation: [],
+            error: `Failed to compile schema: ${message}`,
+            message: `Failed to compile schema: ${message}`,
+        };
+    }
+    const validationResult = await validateJsonlRows({
+        runtime,
+        sandboxId,
+        outputPath,
+        validator,
+        schema: schemaJson,
+        datasetId,
+    });
+    if (!validationResult.success) {
+        return validationResult;
+    }
+    const totalValidRows = validationResult.validRowCount ?? 0;
+    const rowRecordCount = validationResult.rowRecordCount ?? totalValidRows;
+    console.log(`[Dataset ${datasetId}] Reading file content for upload`);
+    const fileRead = await readDatasetSandboxFileStep({ runtime, sandboxId, path: outputPath });
+    if (!fileRead.contentBase64) {
+        console.error(`[Dataset ${datasetId}] Empty file content`);
+        return {
+            success: false,
+            status: "empty_output",
+            validRows: 0,
+            rowRecordCount: 0,
+            validation: [],
+            error: "Empty file content",
+            message: "Empty file content",
+        };
+    }
+    console.log(`[Dataset ${datasetId}] Uploading file to InstantDB storage`);
+    const uploadResult = await service.uploadDatasetOutputFile({
+        datasetId,
+        fileBuffer: Buffer.from(fileRead.contentBase64, "base64"),
+    });
+    if (!uploadResult.ok) {
+        console.error(`[Dataset ${datasetId}] File upload failed: ${uploadResult.error}`);
+        return {
+            success: false,
+            status: "upload_failed",
+            validRows: totalValidRows,
+            rowRecordCount,
+            validation: validationResult.validation,
+            error: uploadResult.error,
+            message: uploadResult.error,
+        };
+    }
+    console.log(`[Dataset ${datasetId}] File uploaded successfully: ${uploadResult.data.fileId}`);
+    const statusResult = await service.updateDatasetStatus({
+        datasetId,
+        status: "completed",
+        calculatedTotalRows: totalValidRows,
+        actualGeneratedRowCount: totalValidRows,
+    });
+    if (!statusResult.ok) {
+        console.error(`[Dataset ${datasetId}] Failed to update status: ${statusResult.error}`);
+        return {
+            success: false,
+            status: "status_update_failed",
+            validRows: totalValidRows,
+            rowRecordCount,
+            validation: validationResult.validation,
+            error: statusResult.error,
+            message: statusResult.error,
+        };
+    }
+    console.log(`[Dataset ${datasetId}] Dataset marked as COMPLETED (${totalValidRows} valid rows)`);
+    console.log(`[Dataset ${datasetId}] ========================================`);
+    return {
+        success: true,
+        status: "completed",
+        validRows: totalValidRows,
+        rowRecordCount,
+        fileId: uploadResult.data.fileId,
+        storagePath: uploadResult.data.storagePath,
+        message: "Dataset creation completed and uploaded to storage",
+    };
+}
+async function ensureFileExists(runtime, sandboxId, path) {
+    const result = await runDatasetSandboxCommandStep({
+        runtime,
+        sandboxId,
+        cmd: "test",
+        args: ["-f", path],
+    });
+    if (result.exitCode !== 0) {
+        throw new Error(`Required file not found: ${path}`);
+    }
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function joinSchemaPath(basePath, key) {
+    return basePath === "$" ? `$.${key}` : `${basePath}.${key}`;
+}
+function collectRequiredPaths(schema, path = "$", paths = []) {
+    const record = asRecord(schema);
+    if (!record)
+        return paths;
+    const properties = asRecord(record.properties);
+    if (properties) {
+        const required = Array.isArray(record.required)
+            ? record.required.filter((value) => typeof value === "string")
+            : [];
+        for (const key of required) {
+            paths.push(joinSchemaPath(path, key));
+        }
+        for (const [key, childSchema] of Object.entries(properties)) {
+            collectRequiredPaths(childSchema, joinSchemaPath(path, key), paths);
+        }
+    }
+    if (record.items) {
+        collectRequiredPaths(record.items, `${path}[]`, paths);
+    }
+    for (const keyword of ["oneOf", "anyOf", "allOf"]) {
+        if (Array.isArray(record[keyword])) {
+            for (const childSchema of record[keyword]) {
+                collectRequiredPaths(childSchema, path, paths);
+            }
+        }
+    }
+    return [...new Set(paths)];
+}
+function collectEnumConstraints(schema, path = "$", constraints = []) {
+    const record = asRecord(schema);
+    if (!record)
+        return constraints;
+    if (Array.isArray(record.enum)) {
+        constraints.push({ path, values: record.enum });
+    }
+    const properties = asRecord(record.properties);
+    if (properties) {
+        for (const [key, childSchema] of Object.entries(properties)) {
+            collectEnumConstraints(childSchema, joinSchemaPath(path, key), constraints);
+        }
+    }
+    if (record.items) {
+        collectEnumConstraints(record.items, `${path}[]`, constraints);
+    }
+    for (const keyword of ["oneOf", "anyOf", "allOf"]) {
+        if (Array.isArray(record[keyword])) {
+            for (const childSchema of record[keyword]) {
+                collectEnumConstraints(childSchema, path, constraints);
+            }
+        }
+    }
+    return constraints;
+}
+function countValues(values, maxItems = 20) {
+    const counts = new Map();
+    for (const value of values) {
+        counts.set(value, (counts.get(value) ?? 0) + 1);
+    }
+    return [...counts.entries()]
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .slice(0, maxItems)
+        .map(([value, count]) => ({ value, count }));
+}
+function toErrorDetails(errors) {
+    if (!Array.isArray(errors))
+        return [];
+    return errors.map((err) => ({
+        path: err.instancePath || "$",
+        keyword: err.keyword,
+        message: err.message || "Unknown validation error",
+        params: asRecord(err.params) ?? undefined,
+        schemaPath: err.schemaPath,
+    }));
+}
+function buildValidationFailureSummary(params) {
+    const rootSchema = asRecord(params.schema);
+    const rootProperties = asRecord(rootSchema?.properties);
+    const invalidRows = params.validation.filter((entry) => !entry.valid);
+    const errorMessages = invalidRows.flatMap((entry) => entry.errors ?? []);
+    const observedTopLevelKeys = [
+        ...new Set(invalidRows.flatMap((entry) => entry.dataKeys ?? [])),
+    ].sort((a, b) => a.localeCompare(b));
+    const details = invalidRows.flatMap((entry) => entry.errorDetails ?? []);
+    const missingRequiredProperties = countValues(details
+        .filter((detail) => detail.keyword === "required")
+        .map((detail) => String(detail.params?.missingProperty ?? "unknown"))).map(({ value, count }) => ({ property: value, count }));
+    const additionalProperties = countValues(details
+        .filter((detail) => detail.keyword === "additionalProperties")
+        .map((detail) => String(detail.params?.additionalProperty ?? "unknown"))).map(({ value, count }) => ({ property: value, count }));
+    const enumFailureCounts = new Map();
+    for (const detail of details.filter((entry) => entry.keyword === "enum")) {
+        const key = `${detail.path}:${JSON.stringify(detail.params?.allowedValues ?? [])}`;
+        const current = enumFailureCounts.get(key);
+        enumFailureCounts.set(key, {
+            path: detail.path,
+            allowedValues: Array.isArray(detail.params?.allowedValues)
+                ? detail.params.allowedValues
+                : [],
+            count: (current?.count ?? 0) + 1,
+        });
+    }
+    return {
+        rowRecordCount: params.rowRecordCount,
+        validRowCount: params.validRowCount,
+        invalidRowCount: invalidRows.length,
+        expectedTopLevelKeys: rootProperties ? Object.keys(rootProperties) : [],
+        requiredTopLevelKeys: Array.isArray(rootSchema?.required)
+            ? rootSchema.required.filter((value) => typeof value === "string")
+            : [],
+        requiredPaths: collectRequiredPaths(params.schema).slice(0, 120),
+        enumConstraints: collectEnumConstraints(params.schema).slice(0, 80),
+        topErrors: countValues(errorMessages, 20).map(({ value, count }) => ({
+            message: value,
+            count,
+        })),
+        missingRequiredProperties,
+        additionalProperties,
+        enumFailures: [...enumFailureCounts.values()]
+            .sort((a, b) => b.count - a.count || a.path.localeCompare(b.path))
+            .slice(0, 20),
+        observedTopLevelKeys,
+        sampleInvalidRows: invalidRows.slice(0, 10).map((entry) => ({
+            index: entry.index,
+            dataKeys: entry.dataKeys,
+            errors: entry.errors?.slice(0, 12),
+        })),
+    };
+}
+function buildRepairInstructions(summary) {
+    const instructions = [
+        "Rewrite output.jsonl using the schema as the source of truth. Do not use source file headers as JSON keys unless they exactly match schema property names.",
+        "Each non-empty line must be a JSON object shaped as {\"type\":\"row\",\"data\":{...}}.",
+        "Populate every required top-level and nested required path from failureSummary.requiredPaths.",
+        "For enum fields, emit exactly one allowed literal from failureSummary.enumConstraints or failureSummary.enumFailures.",
+    ];
+    if (summary.validRowCount === 0 && summary.rowRecordCount > 0) {
+        instructions.unshift("All produced row records failed validation; treat the previous output as structurally invalid and regenerate it from scratch.");
+    }
+    if (summary.additionalProperties.length > 0) {
+        instructions.push("Remove unexpected data keys listed in failureSummary.additionalProperties; map their values into schema keys instead.");
+    }
+    if (summary.missingRequiredProperties.length > 0) {
+        instructions.push("Add the missing required properties listed in failureSummary.missingRequiredProperties to each affected row.");
+    }
+    return instructions;
+}
+function validationOutputSample(validation) {
+    const maxEntries = 50;
+    const invalidRows = validation.filter((entry) => !entry.valid);
+    const sampleSource = invalidRows.length > 0 ? invalidRows : validation;
+    return {
+        validation: sampleSource.slice(0, maxEntries),
+        validationTruncated: Math.max(0, sampleSource.length - maxEntries),
+    };
+}
+async function validateJsonlRows({ runtime, sandboxId, outputPath, validator, schema, datasetId }) {
+    const validation = [];
+    let validRowCount = 0;
+    let rowRecordCount = 0;
+    console.log(`[Dataset ${datasetId}] Reading and validating JSONL file from sandbox`);
+    const fileRead = await readDatasetSandboxTextFileStep({ runtime, sandboxId, path: outputPath });
+    if (!fileRead.content) {
+        console.log(`[Dataset ${datasetId}] Empty output file`);
+        return {
+            success: false,
+            status: "empty_output",
+            validation,
+            validRowCount: 0,
+            rowRecordCount: 0,
+            error: "output.jsonl is empty",
+            message: "output.jsonl is empty",
+        };
+    }
+    const lines = fileRead.content.split("\n");
+    console.log(`[Dataset ${datasetId}] Validating ${lines.length} lines`);
+    for (let index = 0; index < lines.length; index++) {
+        const line = lines[index];
+        const trimmed = line.trim();
+        if (trimmed.length === 0) {
+            continue;
+        }
+        let record;
+        try {
+            record = JSON.parse(trimmed);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            validation.push({
+                index,
+                valid: false,
+                errors: [`Invalid JSON: ${message}`],
+            });
+            continue;
+        }
+        if (record.type !== "row") {
+            validation.push({
+                index,
+                valid: false,
+                errors: ["Every non-empty output line must be a JSON object with type 'row'"],
+            });
+            continue;
+        }
+        rowRecordCount++;
+        const data = record.data;
+        if (data === undefined || data === null) {
+            validation.push({
+                index,
+                valid: false,
+                errors: ["Missing 'data' field"],
+            });
+            continue;
+        }
+        const valid = validator(data);
+        if (!valid) {
+            const errorDetails = toErrorDetails(validator.errors);
+            const errors = errorDetails.length > 0
+                ? errorDetails.map((err) => err.message || "Unknown validation error")
+                : ["Unknown validation error"];
+            validation.push({
+                index,
+                valid: false,
+                errors,
+                errorDetails,
+                dataKeys: data && typeof data === "object" && !Array.isArray(data) ? Object.keys(data) : [],
+            });
+            continue;
+        }
+        validation.push({
+            index,
+            valid: true,
+        });
+        validRowCount++;
+    }
+    console.log(`[Dataset ${datasetId}] Validation completed: ${validRowCount} valid rows`);
+    const invalidRows = validation.filter((entry) => !entry.valid);
+    if (rowRecordCount === 0 || validRowCount === 0 || invalidRows.length > 0) {
+        const failureSummary = buildValidationFailureSummary({
+            schema,
+            validation,
+            rowRecordCount,
+            validRowCount,
+        });
+        const repairInstructions = buildRepairInstructions(failureSummary);
+        const sampled = validationOutputSample(validation);
+        const message = rowRecordCount === 0
+            ? "output.jsonl does not contain any type='row' records"
+            : validRowCount === 0
+                ? "No dataset rows matched the stored schema"
+                : `${invalidRows.length} dataset row(s) failed schema validation`;
+        console.error(`[Dataset ${datasetId}] Validation failed: ${message}`);
+        return {
+            success: false,
+            status: "validation_failed",
+            validation: sampled.validation,
+            validationTruncated: sampled.validationTruncated,
+            failureSummary,
+            repairInstructions,
+            validRowCount,
+            rowRecordCount,
+            error: message,
+            message: `${message}. Repair output.jsonl using repairInstructions and failureSummary.`,
+        };
+    }
+    return {
+        success: true,
+        status: "completed",
+        validation,
+        validRowCount,
+        rowRecordCount,
+    };
+}

package/dist/completeDataset.tool.d.ts

@@ -7,12 +7,56 @@ export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtim
     summary: string;
 }, {
     success: boolean;
-    validation?: Array<{
+    validation?: {
         index: number;
         valid: boolean;
         errors?: string[];
+        errorDetails?: Array<{
+            path: string;
+            keyword: string;
+            message: string;
+            params?: Record<string, unknown>;
+            schemaPath?: string;
+        }>;
         dataKeys?: string[];
-    }>;
+    }[];
+    validationTruncated?: number;
+    failureSummary?: {
+        rowRecordCount: number;
+        validRowCount: number;
+        invalidRowCount: number;
+        expectedTopLevelKeys: string[];
+        requiredTopLevelKeys: string[];
+        requiredPaths: string[];
+        enumConstraints: Array<{
+            path: string;
+            values: unknown[];
+        }>;
+        topErrors: Array<{
+            message: string;
+            count: number;
+        }>;
+        missingRequiredProperties: Array<{
+            property: string;
+            count: number;
+        }>;
+        additionalProperties: Array<{
+            property: string;
+            count: number;
+        }>;
+        enumFailures: Array<{
+            path: string;
+            allowedValues: unknown[];
+            count: number;
+        }>;
+        observedTopLevelKeys: string[];
+        sampleInvalidRows: Array<{
+            index: number;
+            dataKeys?: string[];
+            errors?: string[];
+        }>;
+    };
+    repairInstructions?: string[];
     validRowCount?: number;
     rowRecordCount?: number;
     error?: string;
@@ -27,6 +71,13 @@ export declare function createCompleteDatasetTool({ datasetId, sandboxId, runtim
         index: number;
         valid: boolean;
         errors?: string[];
+        errorDetails?: Array<{
+            path: string;
+            keyword: string;
+            message: string;
+            params?: Record<string, unknown>;
+            schemaPath?: string;
+        }>;
         dataKeys?: string[];
     }[] | undefined;
     error: string;