@ekairos/dataset 1.22.57-beta.development.0 → 1.22.59-beta.development.0

package/dist/builder/materialize.js

@@ -3,13 +3,66 @@ import { createTransformDatasetContext } from "../transform/transform-dataset.ag
 import { datasetInferAndUpdateSchemaStep, datasetReadOneStep, } from "../dataset/steps.js";
 import { registerDatasetAgentMaterializers } from "./agentMaterializers.js";
 import { buildFileDefaultInstructions, buildRawSourceInstructions, buildTransformInstructions, } from "./instructions.js";
-import { createOrUpdateDatasetMetadata, uploadInlineTextSource, } from "./persistence.js";
+import { createOrUpdateDatasetMetadata, materializeRowsToDataset, uploadInlineTextSource, } from "./persistence.js";
 import { getDomainDescriptor } from "./sourceRows.js";
 import { materializeQuerySource } from "./materializeQuery.js";
 import { createDatasetSandboxStep } from "../sandbox/steps.js";
 function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
     return `${targetDatasetId}__${sourceKind}_${index}`;
 }
+function normalizeParsedTextRows(value) {
+    if (Array.isArray(value)) {
+        return value.map((item) => (item && typeof item === "object" ? item : { value: item }));
+    }
+    if (value && typeof value === "object")
+        return [value];
+    return [{ value }];
+}
+function materializeRawTextRows(source) {
+    const text = String(source.text ?? "");
+    const mimeType = String(source.mimeType ?? "").toLowerCase();
+    const name = String(source.name ?? "").toLowerCase();
+    const shouldParseJson = mimeType.includes("json") || name.endsWith(".json") || name.endsWith(".jsonl");
+    if (shouldParseJson) {
+        try {
+            if (name.endsWith(".jsonl")) {
+                const rows = text
+                    .split(/\r?\n/g)
+                    .map((line) => line.trim())
+                    .filter(Boolean)
+                    .map((line) => JSON.parse(line));
+                return rows.flatMap((row) => normalizeParsedTextRows(row));
+            }
+            return normalizeParsedTextRows(JSON.parse(text));
+        }
+        catch {
+            return [{ text }];
+        }
+    }
+    return [{ text }];
+}
+async function materializeRawTextSource(state, source, targetDatasetId) {
+    const rows = materializeRawTextRows(source);
+    await materializeRowsToDataset(state.runtime, {
+        datasetId: targetDatasetId,
+        sandboxId: state.sandboxId,
+        title: state.title ?? source.name ?? targetDatasetId,
+        instructions: state.instructions,
+        sources: [
+            {
+                kind: "text",
+                mimeType: source.mimeType,
+                name: source.name,
+                description: source.description,
+            },
+        ],
+        sourceKinds: ["text"],
+        rows,
+        schema: state.outputSchema,
+        first: state.first,
+    });
+    return targetDatasetId;
+}
 async function resolveDatasetSandboxId(state, targetDatasetId) {
     const sandboxId = String(state.sandboxId ?? "").trim();
     if (sandboxId)
@@ -90,6 +143,16 @@ async function normalizeSourceToDatasetId(state, source, targetDatasetId, source
         });
         return intermediateDatasetId;
     }
+    if (source.kind === "text") {
+        await materializeRawTextSource({
+            ...state,
+            outputSchema: undefined,
+            first: false,
+            instructions: buildRawSourceInstructions(source.kind),
+            title: source.name ?? state.title,
+        }, source, intermediateDatasetId);
+        return intermediateDatasetId;
+    }
     await materializeSingleFileLikeSource({
         ...state,
         outputSchema: undefined,

package/dist/dataset.js

@@ -1,5 +1,5 @@
-import { id as newId } from "@instantdb/admin";
 import { buildObjectOutputInstructions } from "./builder/instructions.js";
+import { createDatasetId } from "./id.js";
 import { materializeDerivedDataset, materializeSingleFileLikeSource, } from "./builder/materialize.js";
 import { materializeQuerySource } from "./builder/materializeQuery.js";
 import { finalizeBuildResult } from "./builder/persistence.js";
@@ -144,7 +144,7 @@ export function dataset(runtime, options = {}) {
     return api;
 }
 function normalizeDatasetId(datasetId) {
-    const normalized = String(datasetId ?? newId()).trim();
+    const normalized = String(datasetId ?? createDatasetId()).trim();
     if (!normalized) {
         throw new Error("dataset_id_required");
     }

package/dist/file/file-dataset.agent.js

@@ -1,11 +1,11 @@
 import { createContext, INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/events";
-import { id } from "@instantdb/admin";
 import { createClearDatasetTool } from "../clearDataset.tool.js";
 import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
 import { datasetGetByIdStep } from "../dataset/steps.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
 import { createGenerateSchemaTool } from "./generateSchema.tool.js";
 import { buildFileDatasetPromptStep, generateFileParsePreviewStep, initializeFileParseSandboxStep, } from "./file-dataset.steps.js";
+import { createDatasetId } from "../id.js";
 async function awaitContextRun(run) {
     if (!run)
         return;
@@ -152,7 +152,7 @@ function createFileParseContextDefinition(params) {
     return { datasetId: fallbackDatasetId ?? "", context };
 }
 export function createFileParseContext(fileId, opts) {
-    const datasetId = opts?.datasetId ?? id();
+    const datasetId = opts?.datasetId ?? createDatasetId();
     const params = {
         fileId,
         instructions: opts?.instructions,
@@ -166,7 +166,7 @@ export function createFileParseContext(fileId, opts) {
         datasetId,
         async parse(runtime, options = {}) {
             const triggerEvent = {
-                id: id(),
+                id: createDatasetId(),
                 type: INPUT_TEXT_ITEM_TYPE,
                 channel: WEB_CHANNEL,
                 createdAt: new Date().toISOString(),

package/dist/id.d.ts

@@ -0,0 +1 @@
+export declare function createDatasetId(): string;

package/dist/id.js

@@ -0,0 +1,10 @@
+export function createDatasetId() {
+    const uuid = globalThis.crypto?.randomUUID?.();
+    if (typeof uuid === "string" && uuid.length > 0)
+        return uuid;
+    return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c) => {
+        const r = (Math.random() * 16) | 0;
+        const v = c === "x" ? r : (r & 0x3) | 0x8;
+        return v.toString(16);
+    });
+}

package/dist/query/queryDomain.step.js

@@ -1,5 +1,5 @@
-import { id as newId } from "@instantdb/admin";
 import { DatasetService } from "../service.js";
+import { createDatasetId } from "../id.js";
 function normalizeRows(result) {
     if (!result || typeof result !== "object")
         return [];
@@ -56,7 +56,7 @@ export async function queryDomainStep(params) {
     "use step";
     const db = await getRuntimeDb(params.runtime);
     const service = new DatasetService(db);
-    const datasetId = params.datasetId ?? newId();
+    const datasetId = params.datasetId ?? createDatasetId();
     const queryResult = await db.query(params.query);
     const rows = normalizeRows(queryResult);
     const previewRows = rows.slice(0, 20);

package/dist/service.d.ts

@@ -1,4 +1,4 @@
-import { InstantAdminDatabase } from "@instantdb/admin";
+import type { InstantAdminDatabase } from "@instantdb/admin";
 import { SchemaOf } from "@ekairos/domain";
 import { datasetDomain } from "./schema.js";
 export type ServiceResult<T = any> = {

package/dist/service.js

@@ -1,4 +1,4 @@
-import { id as newId } from "@instantdb/admin";
+import { createDatasetId } from "./id.js";
 export class DatasetService {
     constructor(db) {
         this.db = db;
@@ -27,9 +27,9 @@ export class DatasetService {
     }
     async createDataset(params) {
         try {
-            const datasetId = params.id ?? newId();
+            const datasetId = params.id ?? createDatasetId();
             const existing = await this.resolveDatasetEntityId(datasetId);
-            const entityId = existing.ok ? existing.data : newId();
+            const entityId = existing.ok ? existing.data : createDatasetId();
             const mutations = [];
             mutations.push(this.db.tx.dataset_datasets[entityId].update({
                 datasetId,
@@ -73,7 +73,7 @@ export class DatasetService {
                 return resolved;
             const mutations = [];
             for (const record of params.records) {
-                const recordId = newId();
+                const recordId = createDatasetId();
                 mutations.push(this.db.tx.dataset_records[recordId].update({
                     rowContent: record.rowContent,
                     order: record.order,

package/dist/transform/transform-dataset.agent.js

@@ -1,10 +1,10 @@
 import { createContext, INPUT_TEXT_ITEM_TYPE, WEB_CHANNEL, } from "@ekairos/events";
-import { id } from "@instantdb/admin";
 import { createClearDatasetTool } from "../clearDataset.tool.js";
 import { createCompleteDatasetTool, didCompleteDatasetSucceed, getDatasetFatalFailure, } from "../completeDataset.tool.js";
 import { datasetUpdateSchemaStep } from "../dataset/steps.js";
 import { createExecuteCommandTool } from "../executeCommand.tool.js";
 import { buildTransformDatasetPromptStep, ensureTransformSourcesInSandboxStep, generateTransformSourcePreviewsStep, } from "./transform-dataset.steps.js";
+import { createDatasetId } from "../id.js";
 async function awaitContextRun(run) {
     if (!run)
         return;
@@ -146,7 +146,7 @@ function createTransformDatasetContextDefinition(params) {
     return { datasetId: fallbackDatasetId ?? "", context };
 }
 export function createTransformDatasetContext(params) {
-    const datasetId = params.datasetId ?? id();
+    const datasetId = params.datasetId ?? createDatasetId();
     const { context } = createTransformDatasetContextDefinition({
         sourceDatasetIds: params.sourceDatasetIds,
         outputSchema: params.outputSchema,
@@ -163,7 +163,7 @@ export function createTransformDatasetContext(params) {
                 ? "the source dataset"
                 : `${params.sourceDatasetIds.length} source datasets`;
             const triggerEvent = {
-                id: id(),
+                id: createDatasetId(),
                 type: INPUT_TEXT_ITEM_TYPE,
                 channel: WEB_CHANNEL,
                 createdAt: new Date().toISOString(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ekairos/dataset",
-  "version": "1.22.57-beta.development.0",
+  "version": "1.22.59-beta.development.0",
   "description": "Pulzar Dataset Tools",
   "type": "module",
   "main": "dist/index.js",
@@ -65,9 +65,9 @@
     "test:ai-sdk:instant": "vitest run -c vitest.codex.config.mts src/tests/materializeDataset.ai-sdk.instant.test.ts"
   },
   "dependencies": {
-    "@ekairos/domain": "^1.22.57-beta.development.0",
-    "@ekairos/events": "^1.22.57-beta.development.0",
-    "@ekairos/sandbox": "^1.22.57-beta.development.0",
+    "@ekairos/domain": "^1.22.59-beta.development.0",
+    "@ekairos/events": "^1.22.59-beta.development.0",
+    "@ekairos/sandbox": "^1.22.59-beta.development.0",
     "@instantdb/admin": "0.22.158",
     "@instantdb/core": "0.22.142",
     "ai": "^5.0.44",