@ekairos/dataset 1.22.37-beta.development.0 → 1.22.37

package/README.md

@@ -0,0 +1,347 @@
+# @ekairos/dataset
+
+Runtime-first dataset materialization for Ekairos domains.
+
+`@ekairos/dataset` replaces the older `@ekairos/structure` flow with a domain-owned dataset API. It persists dataset metadata and output rows in InstantDB, while file parsing and transformations run through sandbox-backed reactors when work cannot be materialized directly.
+
+## Mental Model
+
+A dataset build has two responsibilities:
+
+1. Read or produce source rows from one or more sources.
+2. Persist the resulting dataset under `datasetDomain`.
+
+The caller passes a typed `EkairosRuntime`. The runtime must include `datasetDomain`; query sources also require the runtime to include the queried source domain.
+
+```ts
+import { domain } from "@ekairos/domain"
+import { EkairosRuntime } from "@ekairos/domain/runtime"
+import { dataset, datasetDomain } from "@ekairos/dataset"
+
+import { sourceDomain } from "./source.domain"
+
+const appDomain = domain("app")
+  .includes(datasetDomain)
+  .includes(sourceDomain)
+  .withSchema({ entities: {}, links: {}, rooms: {} })
+
+class AppRuntime extends EkairosRuntime<{ orgId: string }, typeof appDomain, any> {
+  protected getDomain() {
+    return appDomain
+  }
+
+  protected resolveDb() {
+    return db
+  }
+}
+
+const runtime = new AppRuntime({ orgId: "org_1" })
+```
+
+Use `appDomain.toInstantSchema()` to provision or push the InstantDB schema. Dataset itself does not own global DB access; it uses `runtime.use(datasetDomain)` internally.
+
+## Basic Usage
+
+```ts
+const result = await dataset(runtime, { datasetId: "products_v1" })
+  .from({ kind: "text", text: "sku,price\nA1,10", mimeType: "text/csv" })
+  .auto()
+  .asRows()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build()
+
+console.log(result.datasetId)
+console.log(result.previewRows)
+```
+
+`dataset(runtime, { datasetId })` mirrors the old `structure(env, { datasetId })` style. You can also pass the id at build time:
+
+```ts
+await dataset(runtime)
+  .from({ kind: "dataset", datasetId: "source_dataset_v1" })
+  .schema(productSchema)
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build({ datasetId: "normalized_products_v1" })
+```
+
+## Sources
+
+Use `.from(...)` for structure-compatible sources:
+
+```ts
+dataset(runtime).from(
+  { kind: "file", fileId: "file_1", description: "Supplier CSV" },
+  { kind: "text", text: "sku,price\nA1,10", mimeType: "text/csv", name: "inline.csv" },
+  { kind: "dataset", datasetId: "existing_dataset_v1" },
+)
+```
+
+The builder also keeps explicit source methods:
+
+```ts
+dataset(runtime)
+  .fromFile({ fileId: "file_1" })
+  .fromText({ text: "raw input", name: "input.txt" })
+  .fromDataset({ datasetId: "existing_dataset_v1" })
+```
+
+Query sources must use `.fromQuery(sourceDomain, ...)` because they need a second domain:
+
+```ts
+const snapshot = await dataset(runtime, { datasetId: "open_items_v1" })
+  .fromQuery(sourceDomain, {
+    query: {
+      source_items: {
+        $: {
+          where: { status: "open" },
+          fields: ["title", "quantity"],
+          limit: 100,
+        },
+      },
+    },
+    title: "Open items",
+    explanation: "Snapshot of open source items",
+  })
+  .build()
+```
+
+The query is typed with the same InstantDB query types used by `db.query`. Unknown entities, fields, filters, and link shapes fail at compile time.
+
+## Runtime Compatibility
+
+The runtime check is name plus schema, not name only.
+
+```ts
+const appDomain = domain("app")
+  .includes(datasetDomain)
+  .includes(sourceDomain)
+  .withSchema({ entities: {}, links: {}, rooms: {} })
+
+dataset(runtime).fromQuery(sourceDomain, { query: { source_items: {} } })
+```
+
+Subdomains are supported transitively. If domain `B` includes domain `A`, and the runtime root includes `B`, then `.fromQuery(A, ...)` is accepted.
+
+A different domain with the same name but incompatible schema is rejected. A runtime that includes only `datasetDomain` can persist datasets but cannot query a source domain through `.fromQuery(...)`.
+
+## Output Modes
+
+Rows are the default output:
+
+```ts
+await dataset(runtime)
+  .from({ kind: "dataset", datasetId: "source_v1" })
+  .asRows()
+  .build({ datasetId: "rows_v1" })
+```
+
+Object output is represented as a single-row dataset:
+
+```ts
+const result = await dataset(runtime, { datasetId: "summary_v1" })
+  .from({ kind: "dataset", datasetId: "orders_v1" })
+  .instructions("Summarize orders by currency.")
+  .schema(summarySchema)
+  .asObject()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build()
+
+console.log(result.object)
+```
+
+`asObject()` forces a one-row output contract. The persisted dataset still uses JSONL rows, and the returned result exposes the row as both `firstRow` and `object`.
+
+## Schema Modes
+
+Use `schema(...)` when the output contract is known:
+
+```ts
+const productSchema = {
+  title: "ProductRow",
+  description: "One product row",
+  schema: {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+      sku: { type: "string" },
+      price: { type: "number" },
+    },
+    required: ["sku", "price"],
+  },
+}
+
+await dataset(runtime)
+  .from({ kind: "file", fileId })
+  .schema(productSchema)
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build({ datasetId: "products_v1" })
+```
+
+Use `auto()` or `inferSchema()` when the builder should infer a schema from the materialized rows:
+
+```ts
+await dataset(runtime)
+  .from({ kind: "text", text: csv, mimeType: "text/csv" })
+  .auto()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build({ datasetId: "auto_products_v1" })
+```
+
+`auto()` is an alias for `inferSchema()`.
+
+## Sandbox And Reactor Requirements
+
+Some builds can materialize directly:
+
+- A single query source without custom instructions can run without sandbox or reactor.
+
+Other builds require sandbox execution and a reactor:
+
+- File sources
+- Text sources
+- Existing dataset transformations
+- Multiple sources
+- Query sources with custom instructions
+- Any build that needs agent-driven parsing or transformation
+
+If these are missing, the builder throws `dataset_sandbox_required` or `dataset_reactor_required`.
+
+## Build Result
+
+```ts
+type DatasetBuildResult = {
+  datasetId: string
+  dataset: any
+  previewRows: any[]
+  reader: {
+    read(cursor?: number, limit?: number): Promise<{
+      rows: any[]
+      cursor: number
+      done: boolean
+    }>
+    read(params?: { cursor?: number; limit?: number }): Promise<{
+      rows: any[]
+      cursor: number
+      done: boolean
+    }>
+  }
+  firstRow?: any | null
+  object?: any | null
+}
+```
+
+Read more rows with the returned reader:
+
+```ts
+const page = await result.reader.read({ cursor: 0, limit: 100 })
+```
+
+Use `.first()` when the build must produce zero or one row:
+
+```ts
+const result = await dataset(runtime)
+  .fromQuery(sourceDomain, { query: { source_items: { $: { limit: 1 } } } })
+  .first()
+  .build({ datasetId: "single_item_v1" })
+
+console.log(result.firstRow)
+```
+
+If more than one row is produced, the builder throws `dataset_first_expected_zero_or_one_row`.
+
+## Declarative Tool
+
+`createMaterializeDatasetTool` exposes the same materialization contract as an AI SDK tool. It is useful when a reactor or agent needs to request dataset builds declaratively.
+
+```ts
+import { createMaterializeDatasetTool } from "@ekairos/dataset"
+
+const materializeDataset = createMaterializeDatasetTool({
+  runtime,
+  reactor,
+  queryDomain: sourceDomain,
+})
+```
+
+Tool input:
+
+```ts
+{
+  datasetId?: string
+  sandboxId?: string
+  title?: string
+  sources: Array<
+    | { kind: "file"; fileId: string; description?: string }
+    | { kind: "text"; text: string; mimeType?: string; name?: string; description?: string }
+    | { kind: "dataset"; datasetId: string; description?: string }
+    | { kind: "query"; query: Record<string, any>; title?: string; explanation?: string }
+  >
+  instructions?: string
+  mode?: "auto" | "schema"
+  output?: "rows" | "object"
+  schema?: DatasetSchemaInput
+  first?: boolean
+}
+```
+
+The tool returns only `{ datasetId }`.
+
+The tool runtime must include `datasetDomain`, and its `queryDomain` must also be compatible with that same runtime.
+
+## Replacing Structure
+
+Old structure-style chain:
+
+```ts
+await structure(env, { datasetId: "summary_v1" })
+  .from({ kind: "text", text, mimeType: "text/plain", name: "input.txt" })
+  .instructions("Return a summary object.")
+  .schema(summarySchema)
+  .asObject()
+  .build()
+```
+
+Dataset replacement:
+
+```ts
+await dataset(runtime, { datasetId: "summary_v1" })
+  .from({ kind: "text", text, mimeType: "text/plain", name: "input.txt" })
+  .instructions("Return a summary object.")
+  .schema(summarySchema)
+  .asObject()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build()
+```
+
+Key differences:
+
+- Pass `runtime`, not `env`.
+- The runtime must include `datasetDomain`.
+- Query sources must be explicit: `.fromQuery(sourceDomain, { query })`.
+- Object output is stored as a one-row dataset, not as structure context content.
+- Sandbox and reactor are explicit when parsing or transforming is required.
+
+## Exports
+
+Main exports:
+
+- `dataset`
+- `datasetDomain`
+- `createMaterializeDatasetTool`
+- `materializeDatasetToolInputSchema`
+- `DatasetBuilder`
+- `DatasetBuildResult`
+- `DatasetSourceInput`
+- `DatasetSchemaInput`
+- `DatasetOutput`
+- `DatasetMode`
+
+## Internal Notes
+
+Implementation notes live in `src/README.md`. Public callers should use the root package API from `@ekairos/dataset`.

package/dist/agents.d.ts

@@ -0,0 +1,8 @@
+import "./builder/materialize.js";
+export * from "./dataset.js";
+export * from "./domain.js";
+export * from "./materializeDataset.tool.js";
+export * from "./schema.js";
+export * from "./service.js";
+export * from "./file/file-dataset.agent.js";
+export * from "./transform/index.js";

package/dist/agents.js

@@ -0,0 +1,8 @@
+import "./builder/materialize.js";
+export * from "./dataset.js";
+export * from "./domain.js";
+export * from "./materializeDataset.tool.js";
+export * from "./schema.js";
+export * from "./service.js";
+export * from "./file/file-dataset.agent.js";
+export * from "./transform/index.js";

package/dist/builder/agentMaterializers.d.ts

@@ -0,0 +1,9 @@
+import type { AnyDatasetRuntime, DatasetBuilderState, InternalDatasetResource } from "./types.js";
+export type DatasetAgentMaterializers = {
+    materializeSingleFileLikeResource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, resource: Extract<InternalDatasetResource, {
+        kind: "file" | "text";
+    }>, targetDatasetId: string): Promise<string>;
+    materializeDerivedDataset<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, targetDatasetId: string): Promise<string>;
+};
+export declare function registerDatasetAgentMaterializers(materializers: DatasetAgentMaterializers): void;
+export declare function getDatasetAgentMaterializers(): DatasetAgentMaterializers;

package/dist/builder/agentMaterializers.js

@@ -0,0 +1,10 @@
+let agentMaterializers = null;
+export function registerDatasetAgentMaterializers(materializers) {
+    agentMaterializers = materializers;
+}
+export function getDatasetAgentMaterializers() {
+    if (!agentMaterializers) {
+        throw new Error("dataset_agent_materializers_not_registered");
+    }
+    return agentMaterializers;
+}

package/dist/builder/context.d.ts

@@ -0,0 +1,15 @@
+import type { AnyDatasetRuntime, InternalDatasetResource } from "./types.js";
+type DatasetContextResolution = {
+    contextId: string;
+    resources: InternalDatasetResource[];
+    contextResources: DatasetContextResourceRecord[];
+};
+type DatasetContextResourceRecord = {
+    key: string;
+    type: string;
+    name: string;
+    description: string;
+    [key: string]: unknown;
+};
+export declare function resolveDatasetResourceContext<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, resources: InternalDatasetResource[]): Promise<DatasetContextResolution>;
+export {};

package/dist/builder/context.js

@@ -0,0 +1,251 @@
+import { eventsDomain } from "@ekairos/events";
+import { createDatasetId } from "../id.js";
+import { datasetDomain } from "../schema.js";
+import { DatasetService } from "../service.js";
+import { getDomainDescriptor } from "./rows.js";
+function getContextWhere(context) {
+    return "id" in context ? { id: context.id } : { key: context.key };
+}
+async function getEventsDb(runtime) {
+    const scoped = await runtime.use(eventsDomain);
+    return scoped.db;
+}
+async function getDatasetDb(runtime) {
+    const scoped = await runtime.use(datasetDomain);
+    return scoped.db;
+}
+function resourceKey(index, resource) {
+    if (resource.kind === "file")
+        return `file:${index}:${resource.fileId}`;
+    if (resource.kind === "text")
+        return `text:${index}:${resource.name ?? "inline"}`;
+    if (resource.kind === "dataset")
+        return `dataset:${index}:${resource.datasetId}`;
+    if (resource.kind === "query")
+        return `query:${index}:${resource.title ?? "query"}`;
+    return `resource:${index}`;
+}
+function resourceName(index, resource) {
+    if (resource.kind === "file")
+        return resource.filename ?? `File ${index + 1}`;
+    if (resource.kind === "text")
+        return resource.name ?? `Text ${index + 1}`;
+    if (resource.kind === "dataset")
+        return resource.datasetId;
+    if (resource.kind === "query")
+        return resource.title ?? `Query ${index + 1}`;
+    return `Resource ${index + 1}`;
+}
+function resourceDescription(resource) {
+    if ("description" in resource && typeof resource.description === "string" && resource.description.trim()) {
+        return resource.description.trim();
+    }
+    if (resource.kind === "query" && typeof resource.explanation === "string" && resource.explanation.trim()) {
+        return resource.explanation.trim();
+    }
+    return `Dataset ${resource.kind} resource.`;
+}
+function resourceToContextResource(index, resource) {
+    const base = {
+        key: resourceKey(index, resource),
+        type: resource.kind,
+        name: resourceName(index, resource),
+        description: resourceDescription(resource),
+    };
+    if (resource.kind === "file") {
+        return {
+            ...base,
+            fileId: resource.fileId,
+            filename: resource.filename,
+            mediaType: resource.mediaType,
+        };
+    }
+    if (resource.kind === "text") {
+        return {
+            ...base,
+            text: resource.text,
+            mimeType: resource.mimeType,
+        };
+    }
+    if (resource.kind === "dataset") {
+        return {
+            ...base,
+            datasetId: resource.datasetId,
+        };
+    }
+    if (resource.kind === "query") {
+        return {
+            ...base,
+            query: resource.query,
+            title: resource.title,
+            explanation: resource.explanation,
+            ...getDomainDescriptor(resource.domain),
+        };
+    }
+    return base;
+}
+async function createDatasetResourceContextStep(params) {
+    "use step";
+    const db = await getEventsDb(params.runtime);
+    const contextKey = `dataset:${params.datasetId}`;
+    const existing = await db.query({
+        event_contexts: {
+            $: { where: { key: contextKey }, limit: 1 },
+        },
+    });
+    const contextId = existing.event_contexts?.[0]?.id ?? createDatasetId();
+    const now = new Date();
+    const resources = await enrichDatasetContextResources(params.runtime, params.resources);
+    await db.transact([
+        db.tx.event_contexts[contextId].update({
+            key: contextKey,
+            createdAt: now,
+            updatedAt: now,
+            name: `Dataset ${params.datasetId}`,
+            status: "open_idle",
+            content: {
+                datasetId: params.datasetId,
+                resourceCount: resources.length,
+            },
+            resources,
+            description: `Dataset execution context for ${params.datasetId}.`,
+            goal: "Produce the dataset output from the resources declared in this context.",
+        }),
+    ]);
+    return {
+        contextId,
+    };
+}
+async function enrichDatasetContextResources(runtime, resources) {
+    const datasetResources = resources.filter((resource) => resource.type === "dataset" && typeof resource.datasetId === "string");
+    if (datasetResources.length === 0)
+        return resources;
+    const db = await getDatasetDb(runtime);
+    const service = new DatasetService(db);
+    const enriched = [];
+    for (const resource of resources) {
+        if (resource.type !== "dataset" || typeof resource.datasetId !== "string") {
+            enriched.push(resource);
+            continue;
+        }
+        const preview = await service.previewRows(resource.datasetId, 20);
+        if (!preview.ok) {
+            enriched.push({
+                ...resource,
+                previewError: preview.error,
+            });
+            continue;
+        }
+        enriched.push({
+            ...resource,
+            previewRows: preview.data,
+            previewLimit: 20,
+        });
+    }
+    return enriched;
+}
+function contextResourceToDatasetResource(resource) {
+    if (resource.type === "file" && typeof resource.fileId === "string" && resource.fileId.trim()) {
+        return {
+            kind: "file",
+            fileId: resource.fileId.trim(),
+            description: resource.description,
+            filename: typeof resource.filename === "string" ? resource.filename : undefined,
+            mediaType: typeof resource.mediaType === "string" ? resource.mediaType : undefined,
+        };
+    }
+    if (resource.type === "dataset" &&
+        typeof resource.datasetId === "string" &&
+        resource.datasetId.trim()) {
+        return {
+            kind: "dataset",
+            datasetId: resource.datasetId.trim(),
+            description: resource.description,
+        };
+    }
+    if (resource.type === "text" && typeof resource.text === "string") {
+        return {
+            kind: "text",
+            text: String(resource.text),
+            mimeType: typeof resource.mimeType === "string"
+                ? String(resource.mimeType)
+                : "text/plain",
+            name: resource.name,
+            description: resource.description,
+        };
+    }
+    if (resource.type === "query") {
+        throw new Error("dataset_context_query_resource_requires_builder_shortcut");
+    }
+    return {
+        kind: "text",
+        text: JSON.stringify({ resource }, null, 2),
+        mimeType: "application/vnd.ekairos.context-resource+json",
+        name: `${resource.key}.context-resource.json`,
+        description: resource.description,
+    };
+}
+async function readExistingContext(params) {
+    "use step";
+    const db = await getEventsDb(params.runtime);
+    const res = await db.query({
+        event_contexts: {
+            $: {
+                where: getContextWhere(params.context),
+                limit: 1,
+            },
+        },
+    });
+    const row = res?.event_contexts?.[0];
+    if (!row?.id)
+        throw new Error("dataset_context_not_found");
+    const resources = Array.isArray(row.resources)
+        ? row.resources
+        : [];
+    if (resources.length === 0) {
+        throw new Error("dataset_context_resources_required");
+    }
+    const sourceContextId = String(row.id);
+    const copiedResources = resources.map((resource) => ({
+        ...resource,
+        sourceContextId: resource.sourceContextId ?? sourceContextId,
+        sourceResourceKey: resource.sourceResourceKey ?? resource.key,
+    }));
+    return {
+        contextId: sourceContextId,
+        resources: resources.map((resource) => contextResourceToDatasetResource(resource)),
+        contextResources: copiedResources,
+    };
+}
+export async function resolveDatasetResourceContext(runtime, datasetId, resources) {
+    const contextRefs = resources.filter((resource) => resource.kind === "context");
+    if (contextRefs.length > 1) {
+        throw new Error("dataset_context_resource_must_be_unique");
+    }
+    if (contextRefs.length === 1) {
+        if (resources.length > 1) {
+            throw new Error("dataset_context_resource_is_exclusive");
+        }
+        const source = await readExistingContext({ runtime, context: contextRefs[0] });
+        const created = await createDatasetResourceContextStep({
+            runtime,
+            datasetId,
+            resources: source.contextResources,
+        });
+        return {
+            ...source,
+            contextId: created.contextId,
+        };
+    }
+    const contextResourceRecords = resources.map((resource, index) => resourceToContextResource(index, resource));
+    const created = await createDatasetResourceContextStep({
+        runtime,
+        datasetId,
+        resources: contextResourceRecords,
+    });
+    return {
+        contextId: created.contextId,
+        resources,
+        contextResources: contextResourceRecords,
+    };
+}

package/dist/builder/instructions.d.ts

@@ -0,0 +1,5 @@
+import type { DatasetSchemaInput } from "./types.js";
+export declare function buildFileDefaultInstructions(schema?: DatasetSchemaInput): "Create a dataset from the resource file and ensure each output row matches the provided dataset schema exactly." | "Create a dataset representing the resource content as structured rows.";
+export declare function buildRawResourceInstructions(resourceKind: "file" | "text"): "Create a dataset representing the raw text content as structured rows without applying business transformations." | "Create a dataset representing the raw file content as structured rows without applying business transformations.";
+export declare function buildTransformInstructions(resourceCount: number, userInstructions?: string, schema?: DatasetSchemaInput): string;
+export declare function buildObjectOutputInstructions(userInstructions?: string): string;