@ekairos/dataset 1.22.58-beta.development.0 → 1.22.60-beta.development.0

package/dist/builder/materialize.js

@@ -3,13 +3,66 @@ import { createTransformDatasetContext } from "../transform/transform-dataset.ag
 import { datasetInferAndUpdateSchemaStep, datasetReadOneStep, } from "../dataset/steps.js";
 import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
 import { buildFileDefaultInstructions, buildRawSourceInstructions, buildTransformInstructions, } from "./instructions.js";
-import { createOrUpdateDatasetMetadata, uploadInlineTextSource, } from "./persistence.js";
+import { createOrUpdateDatasetMetadata, materializeRowsToDataset, uploadInlineTextSource, } from "./persistence.js";
 import { getDomainDescriptor } from "./sourceRows.js";
 import { materializeQuerySource } from "./materializeQuery.js";
 import { createDatasetSandboxStep } from "../sandbox/steps.js";
 function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
     return `${targetDatasetId}__${sourceKind}_${index}`;
 }
+function normalizeParsedTextRows(value) {
+    if (Array.isArray(value)) {
+        return value.map((item) => (item && typeof item === "object" ? item : { value: item }));
+    }
+    if (value && typeof value === "object")
+        return [value];
+    return [{ value }];
+}
+function materializeRawTextRows(source) {
+    const text = String(source.text ?? "");
+    const mimeType = String(source.mimeType ?? "").toLowerCase();
+    const name = String(source.name ?? "").toLowerCase();
+    const shouldParseJson = mimeType.includes("json") || name.endsWith(".json") || name.endsWith(".jsonl");
+    if (shouldParseJson) {
+        try {
+            if (name.endsWith(".jsonl")) {
+                const rows = text
+                    .split(/\r?\n/g)
+                    .map((line) => line.trim())
+                    .filter(Boolean)
+                    .map((line) => JSON.parse(line));
+                return rows.flatMap((row) => normalizeParsedTextRows(row));
+            }
+            return normalizeParsedTextRows(JSON.parse(text));
+        }
+        catch {
+            return [{ text }];
+        }
+    }
+    return [{ text }];
+}
+async function materializeRawTextSource(state, source, targetDatasetId) {
+    const rows = materializeRawTextRows(source);
+    await materializeRowsToDataset(state.runtime, {
+        datasetId: targetDatasetId,
+        sandboxId: state.sandboxId,
+        title: state.title ?? source.name ?? targetDatasetId,
+        instructions: state.instructions,
+        sources: [
+            {
+                kind: "text",
+                mimeType: source.mimeType,
+                name: source.name,
+                description: source.description,
+            },
+        ],
+        sourceKinds: ["text"],
+        rows,
+        schema: state.outputSchema,
+        first: state.first,
+    });
+    return targetDatasetId;
+}
 async function resolveDatasetSandboxId(state, targetDatasetId) {
     const sandboxId = String(state.sandboxId ?? "").trim();
     if (sandboxId)
@@ -90,6 +143,16 @@ async function normalizeSourceToDatasetId(state, source, targetDatasetId, source
         });
         return intermediateDatasetId;
     }
+    if (source.kind === "text") {
+        await materializeRawTextSource({
+            ...state,
+            outputSchema: undefined,
+            first: false,
+            instructions: buildRawSourceInstructions(source.kind),
+            title: source.name ?? state.title,
+        }, source, intermediateDatasetId);
+        return intermediateDatasetId;
+    }
     await materializeSingleFileLikeSource({
         ...state,
         outputSchema: undefined,

package/dist/file/filepreview.js

@@ -29,6 +29,26 @@ function resolveScriptPath(scriptName) {
 }
 const preparedSandboxIds = new Set();
 const sandboxSetupPromises = new Map();
+function sanitizePreviewText(value) {
+    return String(value ?? "")
+        .replace(/\u0000/g, "")
+        .replace(/[\u0001-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "");
+}
+function getPreviewKind(extension) {
+    const normalized = extension.toLowerCase();
+    if (normalized === ".xlsx" || normalized === ".xls")
+        return "excel";
+    if (normalized === ".csv" ||
+        normalized === ".tsv" ||
+        normalized === ".txt" ||
+        normalized === ".log" ||
+        normalized === ".json" ||
+        normalized === ".jsonl" ||
+        normalized === ".md") {
+        return "text";
+    }
+    return null;
+}
 function validateScriptResult(result, context) {
     if (!result.stderr) {
         return;
@@ -103,13 +123,13 @@ export async function generateFilePreview(runtime, sandboxId, sandboxFilePath, d
         await ensurePreviewScriptsAvailable(runtime, sandboxId);
         const metadataResult = await runScript(runtime, sandboxId, "file_metadata.py", [sandboxFilePath], "Extracts file metadata: name, extension, size, row count estimate, column count, and header preview");
         context.metadata = metadataResult;
-        let isExcel = false;
+        let previewKind = null;
         if (metadataResult.stdout) {
             try {
                 const metadataJson = JSON.parse(metadataResult.stdout);
                 context.totalRows = metadataJson.row_count_estimate || 0;
                 const extension = metadataJson.extension || "";
-                isExcel = extension === ".xlsx" || extension === ".xls";
+                previewKind = getPreviewKind(extension);
             }
             catch {
                 console.warn(`[Dataset ${datasetId}] Failed to parse metadata JSON`);
@@ -122,9 +142,13 @@ export async function generateFilePreview(runtime, sandboxId, sandboxFilePath, d
             console.log(`[Dataset ${datasetId}] No rows detected, skipping preview`);
             return context;
         }
-        const headScript = isExcel ? "preview_head_excel.py" : "preview_head_csv.py";
-        const tailScript = isExcel ? "preview_tail_excel.py" : "preview_tail_csv.py";
-        const midScript = isExcel ? "preview_mid_excel.py" : "preview_mid_csv.py";
+        if (!previewKind) {
+            console.log(`[Dataset ${datasetId}] Binary or unsupported preview format, keeping metadata only`);
+            return context;
+        }
+        const headScript = previewKind === "excel" ? "preview_head_excel.py" : "preview_head_csv.py";
+        const tailScript = previewKind === "excel" ? "preview_tail_excel.py" : "preview_tail_csv.py";
+        const midScript = previewKind === "excel" ? "preview_mid_excel.py" : "preview_mid_csv.py";
         if (totalRows <= headLines) {
             console.log(`[Dataset ${datasetId}] File has ${totalRows} rows, reading all with head only`);
             const headResult = await runScript(runtime, sandboxId, headScript, [sandboxFilePath, String(totalRows)], `Reads the first ${totalRows} rows (entire file)`);
@@ -184,8 +208,8 @@ async function runScript(runtime, sandboxId, scriptName, args, description) {
             description,
             script: scriptContent,
             command,
-            stdout: result.stdout || "",
-            stderr: result.stderr || "",
+            stdout: sanitizePreviewText(result.stdout),
+            stderr: sanitizePreviewText(result.stderr),
         };
     }
     catch (error) {
@@ -194,7 +218,7 @@ async function runScript(runtime, sandboxId, scriptName, args, description) {
             script: scriptContent,
             command,
             stdout: "",
-            stderr: error instanceof Error ? error.message : String(error),
+            stderr: sanitizePreviewText(error instanceof Error ? error.message : String(error)),
         };
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.58-beta.development.0",
+  "version": "1.22.60-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.58-beta.development.0",
-    "@ekairos/events": "^1.22.58-beta.development.0",
-    "@ekairos/sandbox": "^1.22.58-beta.development.0",
+    "@ekairos/domain": "^1.22.60-beta.development.0",
+    "@ekairos/events": "^1.22.60-beta.development.0",
+    "@ekairos/sandbox": "^1.22.60-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",