@ekairos/dataset 1.22.39-beta.development.0 → 1.22.40-beta.development.0

package/README.md

@@ -0,0 +1,347 @@
+# @ekairos/dataset
+
+Runtime-first dataset materialization for Ekairos domains.
+
+`@ekairos/dataset` replaces the older `@ekairos/structure` flow with a domain-owned dataset API. It persists dataset metadata and output rows in InstantDB, while file parsing and transformations run through sandbox-backed reactors when work cannot be materialized directly.
+
+## Mental Model
+
+A dataset build has two responsibilities:
+
+1. Read or produce source rows from one or more sources.
+2. Persist the resulting dataset under `datasetDomain`.
+
+The caller passes a typed `EkairosRuntime`. The runtime must include `datasetDomain`; query sources also require the runtime to include the queried source domain.
+
+```ts
+import { domain } from "@ekairos/domain"
+import { EkairosRuntime } from "@ekairos/domain/runtime"
+import { dataset, datasetDomain } from "@ekairos/dataset"
+
+import { sourceDomain } from "./source.domain"
+
+const appDomain = domain("app")
+  .includes(datasetDomain)
+  .includes(sourceDomain)
+  .withSchema({ entities: {}, links: {}, rooms: {} })
+
+class AppRuntime extends EkairosRuntime<{ orgId: string }, typeof appDomain, any> {
+  protected getDomain() {
+    return appDomain
+  }
+
+  protected resolveDb() {
+    return db
+  }
+}
+
+const runtime = new AppRuntime({ orgId: "org_1" })
+```
+
+Use `appDomain.toInstantSchema()` to provision or push the InstantDB schema. Dataset itself does not own global DB access; it uses `runtime.use(datasetDomain)` internally.
+
+## Basic Usage
+
+```ts
+const result = await dataset(runtime, { datasetId: "products_v1" })
+  .from({ kind: "text", text: "sku,price\nA1,10", mimeType: "text/csv" })
+  .auto()
+  .asRows()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build()
+
+console.log(result.datasetId)
+console.log(result.previewRows)
+```
+
+`dataset(runtime, { datasetId })` mirrors the old `structure(env, { datasetId })` style. You can also pass the id at build time:
+
+```ts
+await dataset(runtime)
+  .from({ kind: "dataset", datasetId: "source_dataset_v1" })
+  .schema(productSchema)
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build({ datasetId: "normalized_products_v1" })
+```
+
+## Sources
+
+Use `.from(...)` for structure-compatible sources:
+
+```ts
+dataset(runtime).from(
+  { kind: "file", fileId: "file_1", description: "Supplier CSV" },
+  { kind: "text", text: "sku,price\nA1,10", mimeType: "text/csv", name: "inline.csv" },
+  { kind: "dataset", datasetId: "existing_dataset_v1" },
+)
+```
+
+The builder also keeps explicit source methods:
+
+```ts
+dataset(runtime)
+  .fromFile({ fileId: "file_1" })
+  .fromText({ text: "raw input", name: "input.txt" })
+  .fromDataset({ datasetId: "existing_dataset_v1" })
+```
+
+Query sources must use `.fromQuery(sourceDomain, ...)` because they need a second domain:
+
+```ts
+const snapshot = await dataset(runtime, { datasetId: "open_items_v1" })
+  .fromQuery(sourceDomain, {
+    query: {
+      source_items: {
+        $: {
+          where: { status: "open" },
+          fields: ["title", "quantity"],
+          limit: 100,
+        },
+      },
+    },
+    title: "Open items",
+    explanation: "Snapshot of open source items",
+  })
+  .build()
+```
+
+The query is typed with the same InstantDB query types used by `db.query`. Unknown entities, fields, filters, and link shapes fail at compile time.
+
+## Runtime Compatibility
+
+The runtime check is name plus schema, not name only.
+
+```ts
+const appDomain = domain("app")
+  .includes(datasetDomain)
+  .includes(sourceDomain)
+  .withSchema({ entities: {}, links: {}, rooms: {} })
+
+dataset(runtime).fromQuery(sourceDomain, { query: { source_items: {} } })
+```
+
+Subdomains are supported transitively. If domain `B` includes domain `A`, and the runtime root includes `B`, then `.fromQuery(A, ...)` is accepted.
+
+A different domain with the same name but incompatible schema is rejected. A runtime that includes only `datasetDomain` can persist datasets but cannot query a source domain through `.fromQuery(...)`.
+
+## Output Modes
+
+Rows are the default output:
+
+```ts
+await dataset(runtime)
+  .from({ kind: "dataset", datasetId: "source_v1" })
+  .asRows()
+  .build({ datasetId: "rows_v1" })
+```
+
+Object output is represented as a single-row dataset:
+
+```ts
+const result = await dataset(runtime, { datasetId: "summary_v1" })
+  .from({ kind: "dataset", datasetId: "orders_v1" })
+  .instructions("Summarize orders by currency.")
+  .schema(summarySchema)
+  .asObject()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build()
+
+console.log(result.object)
+```
+
+`asObject()` forces a one-row output contract. The persisted dataset still uses JSONL rows, and the returned result exposes the row as both `firstRow` and `object`.
+
+## Schema Modes
+
+Use `schema(...)` when the output contract is known:
+
+```ts
+const productSchema = {
+  title: "ProductRow",
+  description: "One product row",
+  schema: {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+      sku: { type: "string" },
+      price: { type: "number" },
+    },
+    required: ["sku", "price"],
+  },
+}
+
+await dataset(runtime)
+  .from({ kind: "file", fileId })
+  .schema(productSchema)
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build({ datasetId: "products_v1" })
+```
+
+Use `auto()` or `inferSchema()` when the builder should infer a schema from the materialized rows:
+
+```ts
+await dataset(runtime)
+  .from({ kind: "text", text: csv, mimeType: "text/csv" })
+  .auto()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build({ datasetId: "auto_products_v1" })
+```
+
+`auto()` is an alias for `inferSchema()`.
+
+## Sandbox And Reactor Requirements
+
+Some builds can materialize directly:
+
+- A single query source without custom instructions can run without sandbox or reactor.
+
+Other builds require sandbox execution and a reactor:
+
+- File sources
+- Text sources
+- Existing dataset transformations
+- Multiple sources
+- Query sources with custom instructions
+- Any build that needs agent-driven parsing or transformation
+
+If these are missing, the builder throws `dataset_sandbox_required` or `dataset_reactor_required`.
+
+## Build Result
+
+```ts
+type DatasetBuildResult = {
+  datasetId: string
+  dataset: any
+  previewRows: any[]
+  reader: {
+    read(cursor?: number, limit?: number): Promise<{
+      rows: any[]
+      cursor: number
+      done: boolean
+    }>
+    read(params?: { cursor?: number; limit?: number }): Promise<{
+      rows: any[]
+      cursor: number
+      done: boolean
+    }>
+  }
+  firstRow?: any | null
+  object?: any | null
+}
+```
+
+Read more rows with the returned reader:
+
+```ts
+const page = await result.reader.read({ cursor: 0, limit: 100 })
+```
+
+Use `.first()` when the build must produce zero or one row:
+
+```ts
+const result = await dataset(runtime)
+  .fromQuery(sourceDomain, { query: { source_items: { $: { limit: 1 } } } })
+  .first()
+  .build({ datasetId: "single_item_v1" })
+
+console.log(result.firstRow)
+```
+
+If more than one row is produced, the builder throws `dataset_first_expected_zero_or_one_row`.
+
+## Declarative Tool
+
+`createMaterializeDatasetTool` exposes the same materialization contract as an AI SDK tool. It is useful when a reactor or agent needs to request dataset builds declaratively.
+
+```ts
+import { createMaterializeDatasetTool } from "@ekairos/dataset"
+
+const materializeDataset = createMaterializeDatasetTool({
+  runtime,
+  reactor,
+  queryDomain: sourceDomain,
+})
+```
+
+Tool input:
+
+```ts
+{
+  datasetId?: string
+  sandboxId?: string
+  title?: string
+  sources: Array<
+    | { kind: "file"; fileId: string; description?: string }
+    | { kind: "text"; text: string; mimeType?: string; name?: string; description?: string }
+    | { kind: "dataset"; datasetId: string; description?: string }
+    | { kind: "query"; query: Record<string, any>; title?: string; explanation?: string }
+  >
+  instructions?: string
+  mode?: "auto" | "schema"
+  output?: "rows" | "object"
+  schema?: DatasetSchemaInput
+  first?: boolean
+}
+```
+
+The tool returns only `{ datasetId }`.
+
+The tool runtime must include `datasetDomain`, and its `queryDomain` must also be compatible with that same runtime.
+
+## Replacing Structure
+
+Old structure-style chain:
+
+```ts
+await structure(env, { datasetId: "summary_v1" })
+  .from({ kind: "text", text, mimeType: "text/plain", name: "input.txt" })
+  .instructions("Return a summary object.")
+  .schema(summarySchema)
+  .asObject()
+  .build()
+```
+
+Dataset replacement:
+
+```ts
+await dataset(runtime, { datasetId: "summary_v1" })
+  .from({ kind: "text", text, mimeType: "text/plain", name: "input.txt" })
+  .instructions("Return a summary object.")
+  .schema(summarySchema)
+  .asObject()
+  .sandbox({ sandboxId })
+  .reactor(reactor)
+  .build()
+```
+
+Key differences:
+
+- Pass `runtime`, not `env`.
+- The runtime must include `datasetDomain`.
+- Query sources must be explicit: `.fromQuery(sourceDomain, { query })`.
+- Object output is stored as a one-row dataset, not as structure context content.
+- Sandbox and reactor are explicit when parsing or transforming is required.
+
+## Exports
+
+Main exports:
+
+- `dataset`
+- `datasetDomain`
+- `createMaterializeDatasetTool`
+- `materializeDatasetToolInputSchema`
+- `DatasetBuilder`
+- `DatasetBuildResult`
+- `DatasetSourceInput`
+- `DatasetSchemaInput`
+- `DatasetOutput`
+- `DatasetMode`
+
+## Internal Notes
+
+Implementation notes live in `src/README.md`. Public callers should use the root package API from `@ekairos/dataset`.

package/dist/builder/instructions.d.ts

@@ -0,0 +1,6 @@
+import type { DatasetSchemaInput } from "./types";
+export declare function buildFileDefaultInstructions(schema?: DatasetSchemaInput): "Create a dataset from the source file and ensure each output row matches the provided dataset schema exactly." | "Create a dataset representing the source content as structured rows.";
+export declare function buildRawSourceInstructions(sourceKind: "file" | "text"): "Create a dataset representing the raw text content as structured rows without applying business transformations." | "Create a dataset representing the raw file content as structured rows without applying business transformations.";
+export declare function buildTransformInstructions(sourceCount: number, userInstructions?: string, schema?: DatasetSchemaInput): string;
+export declare function buildObjectOutputInstructions(userInstructions?: string): string;
+//# sourceMappingURL=instructions.d.ts.map

package/dist/builder/instructions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instructions.d.ts","sourceRoot":"","sources":["../../src/builder/instructions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAEjD,wBAAgB,4BAA4B,CAAC,MAAM,CAAC,EAAE,kBAAkB,4LAKvE;AAED,wBAAgB,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,2OAKrE;AAED,wBAAgB,0BAA0B,CAAC,WAAW,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,kBAAkB,UAarH;AAED,wBAAgB,6BAA6B,CAAC,gBAAgB,CAAC,EAAE,MAAM,UAWtE"}

package/dist/builder/instructions.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildFileDefaultInstructions = buildFileDefaultInstructions;
+exports.buildRawSourceInstructions = buildRawSourceInstructions;
+exports.buildTransformInstructions = buildTransformInstructions;
+exports.buildObjectOutputInstructions = buildObjectOutputInstructions;
+function buildFileDefaultInstructions(schema) {
+    if (schema) {
+        return "Create a dataset from the source file and ensure each output row matches the provided dataset schema exactly.";
+    }
+    return "Create a dataset representing the source content as structured rows.";
+}
+function buildRawSourceInstructions(sourceKind) {
+    if (sourceKind === "text") {
+        return "Create a dataset representing the raw text content as structured rows without applying business transformations.";
+    }
+    return "Create a dataset representing the raw file content as structured rows without applying business transformations.";
+}
+function buildTransformInstructions(sourceCount, userInstructions, schema) {
+    const explicit = String(userInstructions ?? "").trim();
+    if (explicit)
+        return explicit;
+    if (sourceCount > 1) {
+        if (schema) {
+            return "Combine the source datasets into a new dataset that matches the provided output schema exactly.";
+        }
+        return "Combine the source datasets into one coherent dataset.";
+    }
+    if (schema) {
+        return "Transform the source dataset into a new dataset that matches the provided output schema exactly.";
+    }
+    return "Transform the source dataset into a new useful dataset.";
+}
+function buildObjectOutputInstructions(userInstructions) {
+    const base = String(userInstructions ?? "").trim();
+    const objectContract = [
+        "Output mode is object.",
+        "Produce exactly one JSONL row in output.jsonl.",
+        "That row must be {\"type\":\"row\",\"data\":<the final object>}.",
+        "Do not emit multiple rows, headers, summaries, or metadata rows.",
+    ].join("\n");
+    if (!base)
+        return objectContract;
+    return [base, "", objectContract].join("\n");
+}
+//# sourceMappingURL=instructions.js.map

package/dist/builder/instructions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"instructions.js","sourceRoot":"","sources":["../../src/builder/instructions.ts"],"names":[],"mappings":";;AAEA,oEAKC;AAED,gEAKC;AAED,gEAaC;AAED,sEAWC;AAxCD,SAAgB,4BAA4B,CAAC,MAA2B;IACtE,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,+GAA+G,CAAA;IACxH,CAAC;IACD,OAAO,sEAAsE,CAAA;AAC/E,CAAC;AAED,SAAgB,0BAA0B,CAAC,UAA2B;IACpE,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,kHAAkH,CAAA;IAC3H,CAAC;IACD,OAAO,kHAAkH,CAAA;AAC3H,CAAC;AAED,SAAgB,0BAA0B,CAAC,WAAmB,EAAE,gBAAyB,EAAE,MAA2B;IACpH,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACtD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAA;IAC7B,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,iGAAiG,CAAA;QAC1G,CAAC;QACD,OAAO,wDAAwD,CAAA;IACjE,CAAC;IACD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,kGAAkG,CAAA;IAC3G,CAAC;IACD,OAAO,yDAAyD,CAAA;AAClE,CAAC;AAED,SAAgB,6BAA6B,CAAC,gBAAyB;IACrE,MAAM,IAAI,GAAG,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAClD,MAAM,cAAc,GAAG;QACrB,wBAAwB;QACxB,gDAAgD;QAChD,kEAAkE;QAClE,kEAAkE;KACnE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEZ,IAAI,CAAC,IAAI;QAAE,OAAO,cAAc,CAAA;IAChC,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC9C,CAAC"}

package/dist/builder/materialize.d.ts

@@ -0,0 +1,16 @@
+import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalSource } from "./types";
+export declare function materializeQuerySource<Runtime extends AnyDatasetRuntime>(runtime: DatasetBuilderState<Runtime>["runtime"], source: Extract<InternalSource, {
+    kind: "query";
+}>, params: {
+    datasetId: string;
+    sandboxId?: string;
+    schema?: DatasetSchemaInput;
+    title?: string;
+    instructions?: string;
+    first?: boolean;
+}): Promise<string>;
+export declare function materializeSingleFileLikeSource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, source: Extract<InternalSource, {
+    kind: "file" | "text";
+}>, targetDatasetId: string): Promise<string>;
+export declare function materializeDerivedDataset<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, targetDatasetId: string): Promise<string>;
+//# sourceMappingURL=materialize.d.ts.map

package/dist/builder/materialize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"materialize.d.ts","sourceRoot":"","sources":["../../src/builder/materialize.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,kBAAkB,EAClB,cAAc,EACf,MAAM,SAAS,CAAA;AAMhB,wBAAsB,sBAAsB,CAAC,OAAO,SAAS,iBAAiB,EAC5E,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,EAChD,MAAM,EAAE,OAAO,CAAC,cAAc,EAAE;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC,EAClD,MAAM,EAAE;IACN,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,kBAAkB,CAAA;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,KAAK,CAAC,EAAE,OAAO,CAAA;CAChB,mBAgCF;AAED,wBAAsB,+BAA+B,CAAC,OAAO,SAAS,iBAAiB,EACrF,KAAK,EAAE,mBAAmB,CAAC,OAAO,CAAC,EACnC,MAAM,EAAE,OAAO,CAAC,cAAc,EAAE;IAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,CAAC,EAC1D,eAAe,EAAE,MAAM,mBAuExB;AAqCD,wBAAsB,yBAAyB,CAAC,OAAO,SAAS,iBAAiB,EAC/E,KAAK,EAAE,mBAAmB,CAAC,OAAO,CAAC,EACnC,eAAe,EAAE,MAAM,mBAoFxB"}

package/dist/builder/materialize.js

@@ -0,0 +1,205 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.materializeQuerySource = materializeQuerySource;
+exports.materializeSingleFileLikeSource = materializeSingleFileLikeSource;
+exports.materializeDerivedDataset = materializeDerivedDataset;
+const file_dataset_agent_1 = require("../file/file-dataset.agent");
+const service_1 = require("../service");
+const transform_dataset_agent_1 = require("../transform/transform-dataset.agent");
+const instructions_1 = require("./instructions");
+const persistence_1 = require("./persistence");
+const schemaInference_1 = require("./schemaInference");
+const sourceRows_1 = require("./sourceRows");
+function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
+    return `${targetDatasetId}__${sourceKind}_${index}`;
+}
+async function materializeQuerySource(runtime, source, params) {
+    const scoped = await runtime.use(source.domain);
+    const result = await scoped.db.query(source.query);
+    const rows = (0, sourceRows_1.normalizeQueryRows)(result);
+    const domainDescriptor = (0, sourceRows_1.getDomainDescriptor)(source.domain);
+    return await (0, persistence_1.materializeRowsToDataset)(runtime, {
+        datasetId: params.datasetId,
+        sandboxId: params.sandboxId,
+        title: params.title ?? source.title,
+        instructions: params.instructions,
+        sources: [
+            {
+                kind: "query",
+                query: source.query,
+                title: source.title,
+                explanation: source.explanation,
+                ...domainDescriptor,
+            },
+        ],
+        sourceKinds: ["query"],
+        analysis: {
+            query: source.query,
+            explanation: source.explanation,
+            ...domainDescriptor,
+        },
+        rows,
+        schema: params.schema,
+        inferSchema: !params.schema,
+        first: params.first,
+    });
+}
+async function materializeSingleFileLikeSource(state, source, targetDatasetId) {
+    if (!state.reactor) {
+        throw new Error("dataset_reactor_required");
+    }
+    if (!state.sandboxId) {
+        throw new Error("dataset_sandbox_required");
+    }
+    const fileId = source.kind === "file"
+        ? source.fileId
+        : await (0, persistence_1.uploadInlineTextSource)(state.runtime, targetDatasetId, source);
+    await (0, persistence_1.createOrUpdateDatasetMetadata)(state.runtime, {
+        datasetId: targetDatasetId,
+        sandboxId: state.sandboxId,
+        title: state.title ?? targetDatasetId,
+        instructions: state.instructions,
+        sources: [
+            source.kind === "file"
+                ? { kind: "file", fileId: source.fileId, description: source.description }
+                : {
+                    kind: "text",
+                    mimeType: source.mimeType,
+                    name: source.name,
+                    description: source.description,
+                },
+        ],
+        sourceKinds: [source.kind],
+        schema: state.outputSchema,
+        status: "building",
+    });
+    const parseStory = (0, file_dataset_agent_1.createFileParseStory)(fileId, {
+        datasetId: targetDatasetId,
+        instructions: state.instructions ?? (0, instructions_1.buildFileDefaultInstructions)(state.outputSchema),
+        reactor: state.reactor,
+        sandboxId: state.sandboxId,
+    });
+    await parseStory.parse(state.env);
+    if (!state.outputSchema) {
+        const db = await (0, persistence_1.getDatasetDb)(state.runtime);
+        const service = new service_1.DatasetService(db);
+        const readResult = await service.readRows({ datasetId: targetDatasetId, cursor: 0, limit: 1000 });
+        if (!readResult.ok) {
+            throw new Error(readResult.error);
+        }
+        const inferred = (0, schemaInference_1.inferDatasetSchema)(readResult.data.rows, `${targetDatasetId}Row`, "One dataset row");
+        const updateResult = await service.updateDatasetSchema({
+            datasetId: targetDatasetId,
+            schema: inferred,
+            status: "completed",
+        });
+        if (!updateResult.ok) {
+            throw new Error(updateResult.error);
+        }
+    }
+    if (state.first) {
+        const db = await (0, persistence_1.getDatasetDb)(state.runtime);
+        const service = new service_1.DatasetService(db);
+        const firstResult = await service.readOne(targetDatasetId);
+        if (!firstResult.ok) {
+            throw new Error(firstResult.error);
+        }
+    }
+    return targetDatasetId;
+}
+async function normalizeSourceToDatasetId(state, source, targetDatasetId, sourceIndex) {
+    if (source.kind === "dataset") {
+        return source.datasetId;
+    }
+    const intermediateDatasetId = makeIntermediateDatasetId(targetDatasetId, source.kind, sourceIndex);
+    if (source.kind === "query") {
+        await materializeQuerySource(state.runtime, source, {
+            datasetId: intermediateDatasetId,
+            sandboxId: state.sandboxId,
+            title: source.title,
+            first: false,
+        });
+        return intermediateDatasetId;
+    }
+    await materializeSingleFileLikeSource({
+        ...state,
+        outputSchema: undefined,
+        first: false,
+        instructions: (0, instructions_1.buildRawSourceInstructions)(source.kind),
+    }, source, intermediateDatasetId);
+    return intermediateDatasetId;
+}
+async function materializeDerivedDataset(state, targetDatasetId) {
+    if (!state.reactor) {
+        throw new Error("dataset_reactor_required");
+    }
+    if (!state.sandboxId) {
+        throw new Error("dataset_sandbox_required");
+    }
+    const normalizedSources = [];
+    for (let index = 0; index < state.sources.length; index++) {
+        normalizedSources.push(await normalizeSourceToDatasetId(state, state.sources[index], targetDatasetId, index));
+    }
+    const transformSchema = state.outputSchema ??
+        {
+            title: "DatasetRow",
+            description: "One dataset row",
+            schema: {
+                type: "object",
+                additionalProperties: true,
+                properties: {},
+            },
+        };
+    await (0, persistence_1.createOrUpdateDatasetMetadata)(state.runtime, {
+        datasetId: targetDatasetId,
+        sandboxId: state.sandboxId,
+        title: state.title ?? targetDatasetId,
+        instructions: state.instructions,
+        sources: state.sources.map((source) => source.kind === "query"
+            ? {
+                kind: "query",
+                query: source.query,
+                title: source.title,
+                explanation: source.explanation,
+                ...(0, sourceRows_1.getDomainDescriptor)(source.domain),
+            }
+            : source),
+        sourceKinds: state.sources.map((source) => source.kind),
+        schema: transformSchema,
+        status: "building",
+    });
+    const transformStory = (0, transform_dataset_agent_1.createTransformDatasetStory)({
+        sourceDatasetIds: normalizedSources,
+        outputSchema: transformSchema,
+        instructions: (0, instructions_1.buildTransformInstructions)(normalizedSources.length, state.instructions, state.outputSchema),
+        datasetId: targetDatasetId,
+        reactor: state.reactor,
+        sandboxId: state.sandboxId,
+    });
+    await transformStory.transform(state.env);
+    const db = await (0, persistence_1.getDatasetDb)(state.runtime);
+    const service = new service_1.DatasetService(db);
+    if (!state.outputSchema) {
+        const readResult = await service.readRows({ datasetId: targetDatasetId, cursor: 0, limit: 1000 });
+        if (!readResult.ok) {
+            throw new Error(readResult.error);
+        }
+        const inferred = (0, schemaInference_1.inferDatasetSchema)(readResult.data.rows, `${targetDatasetId}Row`, "One dataset row");
+        const updateResult = await service.updateDatasetSchema({
+            datasetId: targetDatasetId,
+            schema: inferred,
+            status: "completed",
+        });
+        if (!updateResult.ok) {
+            throw new Error(updateResult.error);
+        }
+    }
+    if (state.first) {
+        const firstResult = await service.readOne(targetDatasetId);
+        if (!firstResult.ok) {
+            throw new Error(firstResult.error);
+        }
+    }
+    return targetDatasetId;
+}
+//# sourceMappingURL=materialize.js.map

package/dist/builder/materialize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"materialize.js","sourceRoot":"","sources":["../../src/builder/materialize.ts"],"names":[],"mappings":";;AA2BA,wDA0CC;AAED,0EA0EC;AAqCD,8DAsFC;AA5QD,mEAAiE;AACjE,wCAA2C;AAC3C,kFAAkF;AAClF,iDAIuB;AACvB,+CAKsB;AACtB,uDAAsD;AACtD,6CAAsE;AAQtE,SAAS,yBAAyB,CAAC,eAAuB,EAAE,UAAkB,EAAE,KAAa;IAC3F,OAAO,GAAG,eAAe,KAAK,UAAU,IAAI,KAAK,EAAE,CAAA;AACrD,CAAC;AAEM,KAAK,UAAU,sBAAsB,CAC1C,OAAgD,EAChD,MAAkD,EAClD,MAOC;IAED,MAAM,MAAM,GAAG,MAAO,OAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACxD,MAAM,MAAM,GAAG,MAAO,MAAM,CAAC,EAAU,CAAC,KAAK,CAAC,MAAM,CAAC,KAAY,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,IAAA,+BAAkB,EAAC,MAAM,CAAC,CAAA;IACvC,MAAM,gBAAgB,GAAG,IAAA,gCAAmB,EAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAE3D,OAAO,MAAM,IAAA,sCAAwB,EAAC,OAAO,EAAE;QAC7C,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK;QACnC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,GAAG,gBAAgB;aACpB;SACF;QACD,WAAW,EAAE,CAAC,OAAO,CAAC;QACtB,QAAQ,EAAE;YACR,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,GAAG,gBAAgB;SACpB;QACD,IAAI;QACJ,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC,CAAA;AACJ,CAAC;AAEM,KAAK,UAAU,+BAA+B,CACnD,KAAmC,EACnC,MAA0D,EAC1D,eAAuB;IAEvB,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAC7C,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,MAAM,GACV,MAAM,CAAC,IAAI,KAAK,MAAM;QACpB,CAAC,CAAC,MAAM,CAAC,MAAM;QACf,CAAC,CAAC,MAAM,IAAA,oCAAsB,EAAC,KAAK,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,CAAA;IAE1E,MAAM,IAAA,2CAA6B,EAAC,KAAK,CAAC,OAAO,EAAE;QACjD,SAAS,EAAE,eAAe;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,eAAe;QACrC,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,OAAO,EAAE;YACP,MAAM,CAAC,IAAI,KAAK,MAAM;gBACpB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE;gBAC1E,CAAC,CAAC;oBACE,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC;SACN;QACD,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1B,MAAM,EAAE,KAAK,CAAC,YAAY;QAC1B,MAAM,EAAE,UAAU;KACnB,CAAC,CAAA;IAEF,MAAM,UAAU,GAAG,IAAA,yCAAoB,EAAmB,MAAM,EAAE;QAChE,SAAS,EAAE,eAAe;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAA,2CAA4B,EAAC,KAAK,CAAC,YAAY,CAAC;QACpF,OAAO,EAAE,KAAK,CAAC,OAAc;QAC7B,SAAS,EAAE,KAAK,CAAC,SAAS;KAC3B,CAAC,CAAA;IAEF,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAEjC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,MAAM,IAAA,0BAAY,EAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,wBAAc,CAAC,EAAE,CAAC,CAAA;QACtC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACjG,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAA,oCAAkB,EAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,eAAe,KAAK,EAAE,iBAAiB,CAAC,CAAA;QACrG,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC;YACrD,SAAS,EAAE,eAAe;YAC1B,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,WAAW;SACpB,CAAC,CAAA;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,GAAG,MAAM,IAAA,0BAAY,EAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,wBAAc,CAAC,EAAE,CAAC,CAAA;QACtC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QAC1D,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAA;AACxB,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,KAAmC,EACnC,MAAsB,EACtB,eAAuB,EACvB,WAAmB;IAEnB,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,SAAS,CAAA;IACzB,CAAC;IAED,MAAM,qBAAqB,GAAG,yBAAyB,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAElG,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,sBAAsB,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE;YAClD,SAAS,EAAE,qBAAqB;YAChC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,KAAK;SACb,CAAC,CAAA;QACF,OAAO,qBAAqB,CAAA;IAC9B,CAAC;IAED,MAAM,+BAA+B,CACnC;QACE,GAAG,KAAK;QACR,YAAY,EAAE,SAAS;QACvB,KAAK,EAAE,KAAK;QACZ,YAAY,EAAE,IAAA,yCAA0B,EAAC,MAAM,CAAC,IAAI,CAAC;KACtD,EACD,MAAM,EACN,qBAAqB,CACtB,CAAA;IACD,OAAO,qBAAqB,CAAA;AAC9B,CAAC;AAEM,KAAK,UAAU,yBAAyB,CAC7C,KAAmC,EACnC,eAAuB;IAEvB,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAC7C,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,iBAAiB,GAAa,EAAE,CAAA;IACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1D,iBAAiB,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC,CAAA;IAC/G,CAAC;IAED,MAAM,eAAe,GACnB,KAAK,CAAC,YAAY;QACjB;YACC,KAAK,EAAE,YAAY;YACnB,WAAW,EAAE,iBAAiB;YAC9B,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,oBAAoB,EAAE,IAAI;gBAC1B,UAAU,EAAE,EAAE;aACf;SAC4B,CAAA;IAEjC,MAAM,IAAA,2CAA6B,EAAC,KAAK,CAAC,OAAO,EAAE;QACjD,SAAS,EAAE,eAAe;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,eAAe;QACrC,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACpC,MAAM,CAAC,IAAI,KAAK,OAAO;YACrB,CAAC,CAAC;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,GAAG,IAAA,gCAAmB,EAAC,MAAM,CAAC,MAAM,CAAC;aACtC;YACH,CAAC,CAAC,MAAM,CACX;QACD,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;QACvD,MAAM,EAAE,eAAe;QACvB,MAAM,EAAE,UAAU;KACnB,CAAC,CAAA;IAEF,MAAM,cAAc,GAAG,IAAA,qDAA2B,EAAmB;QACnE,gBAAgB,EAAE,iBAAiB;QACnC,YAAY,EAAE,eAAe;QAC7B,YAAY,EAAE,IAAA,yCAA0B,EAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC;QAC1G,SAAS,EAAE,eAAe;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAc;QAC7B,SAAS,EAAE,KAAK,CAAC,SAAS;KAC3B,CAAC,CAAA;IAEF,MAAM,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAEzC,MAAM,EAAE,GAAG,MAAM,IAAA,0BAAY,EAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC5C,MAAM,OAAO,GAAG,IAAI,wBAAc,CAAC,EAAE,CAAC,CAAA;IACtC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACjG,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAA,oCAAkB,EAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,eAAe,KAAK,EAAE,iBAAiB,CAAC,CAAA;QACrG,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC;YACrD,SAAS,EAAE,eAAe;YAC1B,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,WAAW;SACpB,CAAC,CAAA;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QAC1D,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAA;AACxB,CAAC"}

package/dist/builder/persistence.d.ts

@@ -0,0 +1,18 @@
+import type { AnyDatasetRuntime, DatasetBuildResult, DatasetTextSourceInput, MaterializeRowsParams } from "./types";
+export declare function defaultTextSourceName(source: DatasetTextSourceInput): string;
+export declare function getDatasetDb<Runtime extends AnyDatasetRuntime>(runtime: Runtime): Promise<any>;
+export declare function createOrUpdateDatasetMetadata<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: {
+    datasetId: string;
+    sandboxId?: string;
+    title?: string;
+    instructions?: string;
+    sources: any[];
+    sourceKinds: string[];
+    analysis?: any;
+    schema?: any;
+    status?: string;
+}): Promise<void>;
+export declare function materializeRowsToDataset<Runtime extends AnyDatasetRuntime>(runtime: Runtime, params: MaterializeRowsParams): Promise<string>;
+export declare function uploadInlineTextSource<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, source: DatasetTextSourceInput): Promise<string>;
+export declare function finalizeBuildResult<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, withFirst: boolean): Promise<DatasetBuildResult>;
+//# sourceMappingURL=persistence.d.ts.map

package/dist/builder/persistence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/builder/persistence.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAElB,sBAAsB,EACtB,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAEhB,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,CAO5E;AAED,wBAAsB,YAAY,CAAC,OAAO,SAAS,iBAAiB,EAClE,OAAO,EAAE,OAAO,gBAIjB;AAED,wBAAsB,6BAA6B,CAAC,OAAO,SAAS,iBAAiB,EACnF,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;IACN,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,GAAG,EAAE,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,CAAC,EAAE,GAAG,CAAA;IACd,MAAM,CAAC,EAAE,GAAG,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,iBAmBF;AAED,wBAAsB,wBAAwB,CAAC,OAAO,SAAS,iBAAiB,EAC9E,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,MAAM,CAAC,CAgDjB;AAED,wBAAsB,sBAAsB,CAAC,OAAO,SAAS,iBAAiB,EAC5E,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,mBAc/B;AAED,wBAAsB,mBAAmB,CAAC,OAAO,SAAS,iBAAiB,EACzE,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,OAAO,GACjB,OAAO,CAAC,kBAAkB,CAAC,CAmD7B"}