@ekairos/dataset 1.22.39-beta.development.0 → 1.22.39

package/dist/file/filepreview.js

@@ -1,30 +1,38 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePreviewScriptsAvailable = ensurePreviewScriptsAvailable;
-exports.generateFilePreview = generateFilePreview;
-const fs_1 = require("fs");
-const path_1 = require("path");
-const steps_1 = require("../sandbox/steps");
+import { runDatasetSandboxCommandStep } from "../sandbox/steps.js";
+import { PYTHON_SCRIPT_BASE64_BY_NAME } from "./scripts.generated.js";
 const DEFAULT_HEAD_LINES = 50;
 const DEFAULT_TAIL_LINES = 20;
 const DEFAULT_MID_LINES = 20;
-const SANDBOX_SCRIPT_DIRECTORY = "/tmp/ekairos/dataset/file/scripts";
-const PYTHON_SCRIPT_FILES = [
-    "file_metadata.py",
-    "preview_head_csv.py",
-    "preview_head_excel.py",
-    "preview_mid_csv.py",
-    "preview_mid_excel.py",
-    "preview_tail_csv.py",
-    "preview_tail_excel.py",
-];
-function resolveScriptPath(scriptName) {
-    // Prefer local scripts in src/ (tests/dev), and after build the scripts are copied to dist/
-    // at the same relative path, so this works in both environments.
-    return (0, path_1.join)(__dirname, "scripts", scriptName);
+export function getEmbeddedFilePreviewScriptBase64(scriptName) {
+    const embedded = PYTHON_SCRIPT_BASE64_BY_NAME[scriptName];
+    if (!embedded) {
+        throw new Error(`dataset_preview_script_not_embedded:${scriptName}`);
+    }
+    return embedded;
+}
+function readFilePreviewScriptText(scriptName) {
+    return Buffer.from(getEmbeddedFilePreviewScriptBase64(scriptName), "base64").toString("utf-8");
+}
+function sanitizePreviewText(value) {
+    return String(value ?? "")
+        .replace(/\u0000/g, "")
+        .replace(/[\u0001-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "");
+}
+function getPreviewKind(extension) {
+    const normalized = extension.toLowerCase();
+    if (normalized === ".xlsx" || normalized === ".xls")
+        return "excel";
+    if (normalized === ".csv" ||
+        normalized === ".tsv" ||
+        normalized === ".txt" ||
+        normalized === ".log" ||
+        normalized === ".json" ||
+        normalized === ".jsonl" ||
+        normalized === ".md") {
+        return "text";
+    }
+    return null;
 }
-const preparedSandboxIds = new Set();
-const sandboxSetupPromises = new Map();
 function validateScriptResult(result, context) {
     if (!result.stderr) {
         return;
@@ -37,75 +45,24 @@ function validateScriptResult(result, context) {
         throw new Error(`${context} failed: ${stderr.substring(0, 500)}`);
     }
 }
-async function ensurePreviewScriptsAvailable(env, sandboxId) {
-    if (preparedSandboxIds.has(sandboxId)) {
-        return;
-    }
-    const inFlight = sandboxSetupPromises.get(sandboxId);
-    if (inFlight) {
-        await inFlight;
-        return;
-    }
-    const setupPromise = (async () => {
-        try {
-            await (0, steps_1.runDatasetSandboxCommandStep)({
-                env,
-                sandboxId,
-                cmd: "mkdir",
-                args: ["-p", SANDBOX_SCRIPT_DIRECTORY],
-            });
-        }
-        catch (error) {
-            console.warn("[Dataset Scripts] Failed to create sandbox scripts directory", error);
-        }
-        const filesToWrite = [];
-        for (const scriptName of PYTHON_SCRIPT_FILES) {
-            try {
-                const scriptPath = resolveScriptPath(scriptName);
-                const fileBuffer = (0, fs_1.readFileSync)(scriptPath);
-                filesToWrite.push({
-                    path: `${SANDBOX_SCRIPT_DIRECTORY}/${scriptName}`,
-                    contentBase64: Buffer.from(fileBuffer).toString("base64"),
-                });
-            }
-            catch (error) {
-                console.error(`[Dataset Scripts] Failed to read script ${scriptName}`, error);
-                throw error;
-            }
-        }
-        if (filesToWrite.length > 0) {
-            await (0, steps_1.writeDatasetSandboxFilesStep)({
-                env,
-                sandboxId,
-                files: filesToWrite,
-            });
-        }
-    })();
-    sandboxSetupPromises.set(sandboxId, setupPromise);
-    try {
-        await setupPromise;
-        preparedSandboxIds.add(sandboxId);
-    }
-    catch (error) {
-        sandboxSetupPromises.delete(sandboxId);
-        throw error;
-    }
+export async function ensurePreviewScriptsAvailable(_runtime, _sandboxId) {
+    return;
 }
-async function generateFilePreview(env, sandboxId, sandboxFilePath, datasetId, options = {}) {
+export async function generateFilePreview(runtime, sandboxId, sandboxFilePath, datasetId, options = {}) {
     const context = {
         totalRows: 0,
     };
     try {
-        await ensurePreviewScriptsAvailable(env, sandboxId);
-        const metadataResult = await runScript(env, sandboxId, "file_metadata.py", [sandboxFilePath], "Extracts file metadata: name, extension, size, row count estimate, column count, and header preview");
+        const metadataResult = await runScript(runtime, sandboxId, "file_metadata.py", [sandboxFilePath], "Extracts file metadata: name, extension, size, row count estimate, column count, and header preview");
+        validateScriptResult(metadataResult, `preview_metadata for ${datasetId}`);
         context.metadata = metadataResult;
-        let isExcel = false;
+        let previewKind = null;
         if (metadataResult.stdout) {
             try {
                 const metadataJson = JSON.parse(metadataResult.stdout);
                 context.totalRows = metadataJson.row_count_estimate || 0;
                 const extension = metadataJson.extension || "";
-                isExcel = extension === ".xlsx" || extension === ".xls";
+                previewKind = getPreviewKind(extension);
             }
             catch {
                 console.warn(`[Dataset ${datasetId}] Failed to parse metadata JSON`);
@@ -118,28 +75,32 @@ async function generateFilePreview(env, sandboxId, sandboxFilePath, datasetId, o
             console.log(`[Dataset ${datasetId}] No rows detected, skipping preview`);
             return context;
         }
-        const headScript = isExcel ? "preview_head_excel.py" : "preview_head_csv.py";
-        const tailScript = isExcel ? "preview_tail_excel.py" : "preview_tail_csv.py";
-        const midScript = isExcel ? "preview_mid_excel.py" : "preview_mid_csv.py";
+        if (!previewKind) {
+            console.log(`[Dataset ${datasetId}] Binary or unsupported preview format, keeping metadata only`);
+            return context;
+        }
+        const headScript = previewKind === "excel" ? "preview_head_excel.py" : "preview_head_csv.py";
+        const tailScript = previewKind === "excel" ? "preview_tail_excel.py" : "preview_tail_csv.py";
+        const midScript = previewKind === "excel" ? "preview_mid_excel.py" : "preview_mid_csv.py";
         if (totalRows <= headLines) {
             console.log(`[Dataset ${datasetId}] File has ${totalRows} rows, reading all with head only`);
-            const headResult = await runScript(env, sandboxId, headScript, [sandboxFilePath, String(totalRows)], `Reads the first ${totalRows} rows (entire file)`);
+            const headResult = await runScript(runtime, sandboxId, headScript, [sandboxFilePath, String(totalRows)], `Reads the first ${totalRows} rows (entire file)`);
             validateScriptResult(headResult, `preview_head for ${datasetId}`);
             context.head = headResult;
             return context;
         }
         if (headLines + tailLines >= totalRows) {
             console.log(`[Dataset ${datasetId}] Head + tail would cover entire file (${totalRows} rows), reading all with head only`);
-            const headResult = await runScript(env, sandboxId, headScript, [sandboxFilePath, String(totalRows)], `Reads the first ${totalRows} rows (entire file)`);
+            const headResult = await runScript(runtime, sandboxId, headScript, [sandboxFilePath, String(totalRows)], `Reads the first ${totalRows} rows (entire file)`);
             validateScriptResult(headResult, `preview_head for ${datasetId}`);
             context.head = headResult;
             return context;
         }
         console.log(`[Dataset ${datasetId}] Reading head (${headLines} rows) and tail (${tailLines} rows) from ${totalRows} total rows`);
-        const headResult = await runScript(env, sandboxId, headScript, [sandboxFilePath, String(headLines)], `Reads the first ${headLines} rows of the file`);
+        const headResult = await runScript(runtime, sandboxId, headScript, [sandboxFilePath, String(headLines)], `Reads the first ${headLines} rows of the file`);
         validateScriptResult(headResult, `preview_head for ${datasetId}`);
         context.head = headResult;
-        const tailResult = await runScript(env, sandboxId, tailScript, [sandboxFilePath, String(tailLines)], `Reads the last ${tailLines} rows of the file`);
+        const tailResult = await runScript(runtime, sandboxId, tailScript, [sandboxFilePath, String(tailLines)], `Reads the last ${tailLines} rows of the file`);
         validateScriptResult(tailResult, `preview_tail for ${datasetId}`);
         context.tail = tailResult;
         const midLines = options.midLines || DEFAULT_MID_LINES;
@@ -148,40 +109,33 @@ async function generateFilePreview(env, sandboxId, sandboxFilePath, datasetId, o
             const midStart = headLines;
             const midEnd = totalRows - tailLines;
             console.log(`[Dataset ${datasetId}] Large gap (${gapSize} rows), adding mid sample (${midLines} rows)`);
-            const midResult = await runScript(env, sandboxId, midScript, [sandboxFilePath, String(midStart), String(midEnd), String(midLines)], `Samples ${midLines} rows from the middle section (rows ${midStart + 1} to ${midEnd})`);
+            const midResult = await runScript(runtime, sandboxId, midScript, [sandboxFilePath, String(midStart), String(midEnd), String(midLines)], `Samples ${midLines} rows from the middle section (rows ${midStart + 1} to ${midEnd})`);
             validateScriptResult(midResult, `preview_mid for ${datasetId}`);
             context.mid = midResult;
         }
     }
     catch (error) {
         console.error(`[Dataset ${datasetId}] Error generating file preview:`, error);
+        throw error;
     }
     return context;
 }
-async function runScript(env, sandboxId, scriptName, args, description) {
-    const scriptPath = `/vercel/sandbox/lib/domain/dataset/file/scripts/${scriptName}`;
-    const command = `python ${scriptPath} ${args.join(" ")}`;
-    let scriptContent = "";
-    try {
-        const localScriptPath = resolveScriptPath(scriptName);
-        scriptContent = (0, fs_1.readFileSync)(localScriptPath, 'utf-8');
-    }
-    catch (error) {
-        console.warn(`Failed to read script ${scriptName}:`, error);
-    }
+async function runScript(runtime, sandboxId, scriptName, args, description) {
+    const scriptContent = readFilePreviewScriptText(scriptName);
+    const command = `python -c <${scriptName}> ${args.join(" ")}`;
     try {
-        const result = await (0, steps_1.runDatasetSandboxCommandStep)({
-            env,
+        const result = await runDatasetSandboxCommandStep({
+            runtime,
             sandboxId,
             cmd: "python",
-            args: [scriptPath, ...args],
+            args: ["-c", scriptContent, ...args],
         });
         return {
             description,
             script: scriptContent,
             command,
-            stdout: result.stdout || "",
-            stderr: result.stderr || "",
+            stdout: sanitizePreviewText(result.stdout),
+            stderr: sanitizePreviewText(result.stderr),
         };
     }
     catch (error) {
@@ -190,8 +144,7 @@ async function runScript(env, sandboxId, scriptName, args, description) {
             script: scriptContent,
             command,
             stdout: "",
-            stderr: error instanceof Error ? error.message : String(error),
+            stderr: sanitizePreviewText(error instanceof Error ? error.message : String(error)),
         };
     }
 }
-//# sourceMappingURL=filepreview.js.map

package/dist/file/filepreview.types.d.ts

@@ -0,0 +1,31 @@
+export type FilePreviewContext = {
+    totalRows: number;
+    metadata?: {
+        description: string;
+        script: string;
+        command: string;
+        stdout: string;
+        stderr: string;
+    };
+    head?: {
+        description: string;
+        script: string;
+        command: string;
+        stdout: string;
+        stderr: string;
+    };
+    tail?: {
+        description: string;
+        script: string;
+        command: string;
+        stdout: string;
+        stderr: string;
+    };
+    mid?: {
+        description: string;
+        script: string;
+        command: string;
+        stdout: string;
+        stderr: string;
+    };
+};

package/dist/file/filepreview.types.js

@@ -0,0 +1 @@
+export {};

package/dist/file/generateSchema.tool.d.ts

@@ -2,9 +2,9 @@ interface GenerateSchemaToolParams {
     datasetId: string;
     isNested?: boolean;
     fileId?: string;
-    env: any;
+    runtime: any;
 }
-export declare function createGenerateSchemaTool({ datasetId, isNested, fileId, env }: GenerateSchemaToolParams): import("ai").Tool<{
+export declare function createGenerateSchemaTool({ datasetId, isNested, fileId, runtime }: GenerateSchemaToolParams): import("ai").Tool<{
     schemaTitle: string;
     schemaDescription: string;
     schemaJson: string;
@@ -25,4 +25,3 @@ export declare function createGenerateSchemaTool({ datasetId, isNested, fileId,
     error?: undefined;
 }>;
 export {};
-//# sourceMappingURL=generateSchema.tool.d.ts.map

package/dist/file/generateSchema.tool.js

@@ -1,20 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createGenerateSchemaTool = createGenerateSchemaTool;
-const ai_1 = require("ai");
-const zod_1 = require("zod");
-const steps_1 = require("../dataset/steps");
-function createGenerateSchemaTool({ datasetId, isNested, fileId, env }) {
-    return (0, ai_1.tool)({
+import { tool } from "ai";
+import { z } from "zod";
+import { datasetUpdateSchemaStep } from "../dataset/steps.js";
+export function createGenerateSchemaTool({ datasetId, isNested, fileId, runtime }) {
+    return tool({
         description: `Generate a formal JSON schema for a SINGLE RECORD (row) from the file. This schema describes the structure of ONE record, not the entire dataset or array of records. Requirements:
 1. Schema describes ONE RECORD structure only (no array wrappers)
 2. All property names MUST use lowercaseCamelCase convention (e.g., 'productName', 'unitPrice')
 3. Each property MUST have a description field
 4. The schema description must explain what one record represents and field mappings from original file`,
-        inputSchema: zod_1.z.object({
-            schemaTitle: zod_1.z.string().describe("Title for the RECORD schema in PascalCase (e.g., 'ProductRecord', 'TransactionRecord')"),
-            schemaDescription: zod_1.z.string().describe("Comprehensive description that includes: 1) what ONE record represents, 2) its purpose, 3) complete field mapping from original file fields to schema fields with explanations (e.g., 'ARTÍCULO' -> 'articleCode': normalized to camelCase)"),
-            schemaJson: zod_1.z.string().describe("Complete JSON schema as string describing ONE RECORD. Must be type 'object' with properties. All properties must be in lowercaseCamelCase and have descriptions. Do NOT use type 'array' at root level."),
+        inputSchema: z.object({
+            schemaTitle: z.string().describe("Title for the RECORD schema in PascalCase (e.g., 'ProductRecord', 'TransactionRecord')"),
+            schemaDescription: z.string().describe("Comprehensive description that includes: 1) what ONE record represents, 2) its purpose, 3) complete field mapping from original file fields to schema fields with explanations (e.g., 'ARTÍCULO' -> 'articleCode': normalized to camelCase)"),
+            schemaJson: z.string().describe("Complete JSON schema as string describing ONE RECORD. Must be type 'object' with properties. All properties must be in lowercaseCamelCase and have descriptions. Do NOT use type 'array' at root level."),
         }),
         execute: async ({ schemaTitle, schemaDescription, schemaJson, }) => {
             console.log(`[Dataset ${datasetId}] ========================================`);
@@ -74,8 +71,8 @@ function createGenerateSchemaTool({ datasetId, isNested, fileId, env }) {
                 console.log(`[Dataset ${datasetId}] Description: ${schemaDescription}`);
                 console.log(`[Dataset ${datasetId}] Schema JSON:`);
                 console.log(JSON.stringify(parsedSchema, null, 2));
-                const updateResult = await (0, steps_1.datasetUpdateSchemaStep)({
-                    env,
+                const updateResult = await datasetUpdateSchemaStep({
+                    runtime,
                     datasetId,
                     schema: schemaData,
                     status: "schema_complete",
@@ -107,4 +104,3 @@ function createGenerateSchemaTool({ datasetId, isNested, fileId, env }) {
         },
     });
 }
-//# sourceMappingURL=generateSchema.tool.js.map

package/dist/file/index.d.ts

@@ -1,2 +1 @@
-export * from "./file-dataset.agent";
-//# sourceMappingURL=index.d.ts.map
+export * from "./file-dataset.agent.js";

package/dist/file/index.js

@@ -1,18 +1 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./file-dataset.agent"), exports);
-//# sourceMappingURL=index.js.map
+export * from "./file-dataset.agent.js";

package/dist/file/prompts.d.ts

@@ -1,3 +1,2 @@
-import { FileParseStoryContext } from "./file-dataset.agent";
-export declare function buildFileDatasetPrompt(context: FileParseStoryContext): string;
-//# sourceMappingURL=prompts.d.ts.map
+import type { FileParseContext } from "./file-dataset.types.js";
+export declare function buildFileDatasetPrompt(context: FileParseContext): string;