@ekairos/dataset 1.22.59-beta.development.0 → 1.22.61-beta.development.0

package/dist/file/filepreview.js

@@ -1,5 +1,4 @@
 import { readFileSync } from "node:fs";
-import { createRequire } from "node:module";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep } from "../sandbox/steps.js";
@@ -16,19 +15,33 @@ const PYTHON_SCRIPT_FILES = [
     "preview_tail_csv.py",
     "preview_tail_excel.py",
 ];
-const require = createRequire(import.meta.url);
 function resolveScriptPath(scriptName) {
-    try {
-        return require.resolve(`@ekairos/dataset/file/scripts/${scriptName}`);
-    }
-    catch {
-        // Prefer local scripts in src/ (tests/dev), and after build the scripts are copied to dist/
-        // at the same relative path, so this works in both environments.
-        return join(dirname(fileURLToPath(import.meta.url)), "scripts", scriptName);
-    }
+    // In src and dist the scripts live beside this module. Avoid package-resolution here:
+    // Turbopack treats package-resolved Python script paths as module edges.
+    return join(dirname(fileURLToPath(import.meta.url)), "scripts", scriptName);
 }
 const preparedSandboxIds = new Set();
 const sandboxSetupPromises = new Map();
+function sanitizePreviewText(value) {
+    return String(value ?? "")
+        .replace(/\u0000/g, "")
+        .replace(/[\u0001-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "");
+}
+function getPreviewKind(extension) {
+    const normalized = extension.toLowerCase();
+    if (normalized === ".xlsx" || normalized === ".xls")
+        return "excel";
+    if (normalized === ".csv" ||
+        normalized === ".tsv" ||
+        normalized === ".txt" ||
+        normalized === ".log" ||
+        normalized === ".json" ||
+        normalized === ".jsonl" ||
+        normalized === ".md") {
+        return "text";
+    }
+    return null;
+}
 function validateScriptResult(result, context) {
     if (!result.stderr) {
         return;
@@ -103,13 +116,13 @@ export async function generateFilePreview(runtime, sandboxId, sandboxFilePath, d
         await ensurePreviewScriptsAvailable(runtime, sandboxId);
         const metadataResult = await runScript(runtime, sandboxId, "file_metadata.py", [sandboxFilePath], "Extracts file metadata: name, extension, size, row count estimate, column count, and header preview");
         context.metadata = metadataResult;
-        let isExcel = false;
+        let previewKind = null;
         if (metadataResult.stdout) {
             try {
                 const metadataJson = JSON.parse(metadataResult.stdout);
                 context.totalRows = metadataJson.row_count_estimate || 0;
                 const extension = metadataJson.extension || "";
-                isExcel = extension === ".xlsx" || extension === ".xls";
+                previewKind = getPreviewKind(extension);
             }
             catch {
                 console.warn(`[Dataset ${datasetId}] Failed to parse metadata JSON`);
@@ -122,9 +135,13 @@ export async function generateFilePreview(runtime, sandboxId, sandboxFilePath, d
             console.log(`[Dataset ${datasetId}] No rows detected, skipping preview`);
             return context;
         }
-        const headScript = isExcel ? "preview_head_excel.py" : "preview_head_csv.py";
-        const tailScript = isExcel ? "preview_tail_excel.py" : "preview_tail_csv.py";
-        const midScript = isExcel ? "preview_mid_excel.py" : "preview_mid_csv.py";
+        if (!previewKind) {
+            console.log(`[Dataset ${datasetId}] Binary or unsupported preview format, keeping metadata only`);
+            return context;
+        }
+        const headScript = previewKind === "excel" ? "preview_head_excel.py" : "preview_head_csv.py";
+        const tailScript = previewKind === "excel" ? "preview_tail_excel.py" : "preview_tail_csv.py";
+        const midScript = previewKind === "excel" ? "preview_mid_excel.py" : "preview_mid_csv.py";
         if (totalRows <= headLines) {
             console.log(`[Dataset ${datasetId}] File has ${totalRows} rows, reading all with head only`);
             const headResult = await runScript(runtime, sandboxId, headScript, [sandboxFilePath, String(totalRows)], `Reads the first ${totalRows} rows (entire file)`);
@@ -184,8 +201,8 @@ async function runScript(runtime, sandboxId, scriptName, args, description) {
             description,
             script: scriptContent,
             command,
-            stdout: result.stdout || "",
-            stderr: result.stderr || "",
+            stdout: sanitizePreviewText(result.stdout),
+            stderr: sanitizePreviewText(result.stderr),
         };
     }
     catch (error) {
@@ -194,7 +211,7 @@ async function runScript(runtime, sandboxId, scriptName, args, description) {
             script: scriptContent,
             command,
             stdout: "",
-            stderr: error instanceof Error ? error.message : String(error),
+            stderr: sanitizePreviewText(error instanceof Error ? error.message : String(error)),
         };
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.59-beta.development.0",
+  "version": "1.22.61-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.59-beta.development.0",
-    "@ekairos/events": "^1.22.59-beta.development.0",
-    "@ekairos/sandbox": "^1.22.59-beta.development.0",
+    "@ekairos/domain": "^1.22.61-beta.development.0",
+    "@ekairos/events": "^1.22.61-beta.development.0",
+    "@ekairos/sandbox": "^1.22.61-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",