@ekairos/dataset 1.22.58-beta.development.0 → 1.22.59-beta.development.0

package/dist/builder/materialize.js

@@ -3,13 +3,66 @@ import { createTransformDatasetContext } from "../transform/transform-dataset.ag
 import { datasetInferAndUpdateSchemaStep, datasetReadOneStep, } from "../dataset/steps.js";
 import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
 import { buildFileDefaultInstructions, buildRawSourceInstructions, buildTransformInstructions, } from "./instructions.js";
-import { createOrUpdateDatasetMetadata, uploadInlineTextSource, } from "./persistence.js";
+import { createOrUpdateDatasetMetadata, materializeRowsToDataset, uploadInlineTextSource, } from "./persistence.js";
 import { getDomainDescriptor } from "./sourceRows.js";
 import { materializeQuerySource } from "./materializeQuery.js";
 import { createDatasetSandboxStep } from "../sandbox/steps.js";
 function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
     return `${targetDatasetId}__${sourceKind}_${index}`;
 }
+function normalizeParsedTextRows(value) {
+    if (Array.isArray(value)) {
+        return value.map((item) => (item && typeof item === "object" ? item : { value: item }));
+    }
+    if (value && typeof value === "object")
+        return [value];
+    return [{ value }];
+}
+function materializeRawTextRows(source) {
+    const text = String(source.text ?? "");
+    const mimeType = String(source.mimeType ?? "").toLowerCase();
+    const name = String(source.name ?? "").toLowerCase();
+    const shouldParseJson = mimeType.includes("json") || name.endsWith(".json") || name.endsWith(".jsonl");
+    if (shouldParseJson) {
+        try {
+            if (name.endsWith(".jsonl")) {
+                const rows = text
+                    .split(/\r?\n/g)
+                    .map((line) => line.trim())
+                    .filter(Boolean)
+                    .map((line) => JSON.parse(line));
+                return rows.flatMap((row) => normalizeParsedTextRows(row));
+            }
+            return normalizeParsedTextRows(JSON.parse(text));
+        }
+        catch {
+            return [{ text }];
+        }
+    }
+    return [{ text }];
+}
+async function materializeRawTextSource(state, source, targetDatasetId) {
+    const rows = materializeRawTextRows(source);
+    await materializeRowsToDataset(state.runtime, {
+        datasetId: targetDatasetId,
+        sandboxId: state.sandboxId,
+        title: state.title ?? source.name ?? targetDatasetId,
+        instructions: state.instructions,
+        sources: [
+            {
+                kind: "text",
+                mimeType: source.mimeType,
+                name: source.name,
+                description: source.description,
+            },
+        ],
+        sourceKinds: ["text"],
+        rows,
+        schema: state.outputSchema,
+        first: state.first,
+    });
+    return targetDatasetId;
+}
 async function resolveDatasetSandboxId(state, targetDatasetId) {
     const sandboxId = String(state.sandboxId ?? "").trim();
     if (sandboxId)
@@ -90,6 +143,16 @@ async function normalizeSourceToDatasetId(state, source, targetDatasetId, source
         });
         return intermediateDatasetId;
     }
+    if (source.kind === "text") {
+        await materializeRawTextSource({
+            ...state,
+            outputSchema: undefined,
+            first: false,
+            instructions: buildRawSourceInstructions(source.kind),
+            title: source.name ?? state.title,
+        }, source, intermediateDatasetId);
+        return intermediateDatasetId;
+    }
     await materializeSingleFileLikeSource({
         ...state,
         outputSchema: undefined,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.58-beta.development.0",
+  "version": "1.22.59-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.58-beta.development.0",
-    "@ekairos/events": "^1.22.58-beta.development.0",
-    "@ekairos/sandbox": "^1.22.58-beta.development.0",
+    "@ekairos/domain": "^1.22.59-beta.development.0",
+    "@ekairos/events": "^1.22.59-beta.development.0",
+    "@ekairos/sandbox": "^1.22.59-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",