@ekairos/dataset 1.22.34-beta.development.0 → 1.22.35

package/dist/dataset.js

@@ -1,506 +1,63 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.dataset = dataset;
-const admin_1 = require("@instantdb/admin");
-const runtime_1 = require("@ekairos/domain/runtime");
-const runtime_2 = require("@ekairos/events/runtime");
-const ajv_1 = __importDefault(require("ajv"));
-const file_dataset_agent_1 = require("./file/file-dataset.agent");
-const transform_dataset_agent_1 = require("./transform/transform-dataset.agent");
-const service_1 = require("./service");
-const ajv = new ajv_1.default({ allErrors: true, strict: false });
-function defaultTextSourceName(source) {
-    if (source.name?.trim())
-        return source.name.trim();
-    const mimeType = String(source.mimeType ?? "").toLowerCase();
-    if (mimeType.includes("csv"))
-        return "source.csv";
-    if (mimeType.includes("json"))
-        return "source.json";
-    if (mimeType.includes("yaml") || mimeType.includes("yml"))
-        return "source.yaml";
-    return "source.txt";
-}
-function inferJsonSchemaType(value) {
-    if (value === null)
-        return { type: "null" };
-    if (Array.isArray(value))
-        return { type: "array" };
-    switch (typeof value) {
-        case "number":
-            return { type: "number" };
-        case "boolean":
-            return { type: "boolean" };
-        case "object":
-            return { type: "object", additionalProperties: true };
-        default:
-            return { type: "string" };
-    }
-}
-function inferDatasetSchema(rows, title = "DatasetRow", description = "One dataset row") {
-    const properties = {};
-    const required = [];
-    const keys = new Set();
-    for (const row of rows) {
-        if (!row || typeof row !== "object")
-            continue;
-        for (const key of Object.keys(row)) {
-            keys.add(key);
-        }
-    }
-    for (const key of keys) {
-        const values = rows.map((row) => (row && typeof row === "object" ? row[key] : undefined));
-        const firstDefined = values.find((value) => value !== undefined);
-        properties[key] = {
-            ...inferJsonSchemaType(firstDefined),
-            description: `${key} value`,
-        };
-        if (values.every((value) => value !== undefined)) {
-            required.push(key);
-        }
-    }
-    return {
-        title,
-        description,
-        schema: {
-            type: "object",
-            additionalProperties: false,
-            properties,
-            required,
-        },
-    };
-}
-function validateRows(rows, schema) {
-    const validator = ajv.compile(schema.schema);
-    for (const row of rows) {
-        const valid = validator(row);
-        if (!valid) {
-            const error = validator.errors?.map((entry) => entry.message || "validation_error").join("; ");
-            throw new Error(error || "dataset_schema_validation_failed");
-        }
-    }
-}
-function rowsToJsonl(rows) {
-    return rows
-        .map((row) => JSON.stringify({
-        type: "row",
-        data: row,
-    }))
-        .join("\n")
-        .concat(rows.length > 0 ? "\n" : "");
-}
-function normalizeQueryRows(result) {
-    if (!result || typeof result !== "object")
-        return [];
-    const entries = Object.entries(result);
-    if (entries.length === 0)
-        return [];
-    if (entries.length === 1) {
-        const [key, value] = entries[0];
-        if (Array.isArray(value)) {
-            return value.map((row) => (row && typeof row === "object" ? row : { value: row }));
-        }
-        if (value && typeof value === "object") {
-            return [value];
-        }
-        return [{ [key]: value }];
-    }
-    const rows = [];
-    for (const [key, value] of entries) {
-        if (Array.isArray(value)) {
-            for (const row of value) {
-                if (row && typeof row === "object") {
-                    rows.push({ __entity: key, ...row });
-                }
-                else {
-                    rows.push({ __entity: key, value: row });
-                }
-            }
-            continue;
-        }
-        if (value && typeof value === "object") {
-            rows.push({ __entity: key, ...value });
-            continue;
-        }
-        rows.push({ __entity: key, value });
-    }
-    return rows;
-}
-function getDomainDescriptor(domain) {
-    const meta = domain?.meta ?? {};
-    const context = typeof domain?.context === "function" ? domain.context() : {};
-    const name = String(meta?.name ?? context?.name ?? "domain");
-    const packageName = String(meta?.packageName ?? "");
-    return {
-        domainName: name,
-        ...(packageName ? { domainPackageName: packageName } : {}),
-    };
-}
-function makeIntermediateDatasetId(targetDatasetId, sourceKind, index) {
-    return `${targetDatasetId}__${sourceKind}_${index}`;
-}
-function buildFileDefaultInstructions(schema) {
-    if (schema) {
-        return "Create a dataset from the source file and ensure each output row matches the provided dataset schema exactly.";
-    }
-    return "Create a dataset representing the source content as structured rows.";
-}
-function buildRawSourceInstructions(sourceKind) {
-    if (sourceKind === "text") {
-        return "Create a dataset representing the raw text content as structured rows without applying business transformations.";
-    }
-    return "Create a dataset representing the raw file content as structured rows without applying business transformations.";
-}
-function buildTransformInstructions(sourceCount, userInstructions, schema) {
-    const explicit = String(userInstructions ?? "").trim();
-    if (explicit)
-        return explicit;
-    if (sourceCount > 1) {
-        if (schema) {
-            return "Combine the source datasets into a new dataset that matches the provided output schema exactly.";
-        }
-        return "Combine the source datasets into one coherent dataset.";
-    }
-    if (schema) {
-        return "Transform the source dataset into a new dataset that matches the provided output schema exactly.";
-    }
-    return "Transform the source dataset into a new useful dataset.";
-}
-async function getDatasetDb(env) {
-    const runtime = (await (0, runtime_2.getContextRuntime)(env));
-    return runtime.db;
-}
-async function createOrUpdateDatasetMetadata(env, params) {
-    const db = await getDatasetDb(env);
-    const service = new service_1.DatasetService(db);
-    const result = await service.createDataset({
-        id: params.datasetId,
-        title: params.title ?? params.datasetId,
-        instructions: params.instructions ?? "",
-        sources: params.sources,
-        sourceKinds: params.sourceKinds,
-        analysis: params.analysis,
-        schema: params.schema,
-        status: params.status ?? "building",
-        organizationId: env.orgId,
-    });
-    if (!result.ok) {
-        throw new Error(result.error);
-    }
-}
-async function materializeRowsToDataset(env, params) {
-    if (params.first && params.rows.length > 1) {
-        throw new Error("dataset_first_expected_zero_or_one_row");
-    }
-    const resolvedSchema = params.schema ??
-        inferDatasetSchema(params.rows, params.title ? `${params.title}Row` : "DatasetRow", params.title ? `One row for ${params.title}` : "One dataset row");
-    validateRows(params.rows, resolvedSchema);
-    await createOrUpdateDatasetMetadata(env, {
-        datasetId: params.datasetId,
-        sandboxId: params.sandboxId,
-        title: params.title,
-        instructions: params.instructions,
-        sources: params.sources,
-        sourceKinds: params.sourceKinds,
-        analysis: params.analysis,
-        schema: resolvedSchema,
-        status: "building",
-    });
-    const db = await getDatasetDb(env);
-    const service = new service_1.DatasetService(db);
-    const uploadResult = await service.uploadDatasetOutputFile({
-        datasetId: params.datasetId,
-        fileBuffer: Buffer.from(rowsToJsonl(params.rows), "utf-8"),
-    });
-    if (!uploadResult.ok) {
-        throw new Error(uploadResult.error);
-    }
-    const statusResult = await service.updateDatasetStatus({
-        datasetId: params.datasetId,
-        status: "completed",
-        calculatedTotalRows: params.rows.length,
-        actualGeneratedRowCount: params.rows.length,
-    });
-    if (!statusResult.ok) {
-        throw new Error(statusResult.error);
-    }
-    return params.datasetId;
-}
-async function uploadInlineTextSource(env, datasetId, source) {
-    const db = await getDatasetDb(env);
-    const fileName = defaultTextSourceName(source);
-    const storagePath = `/dataset/source/${datasetId}/${Date.now()}-${fileName}`;
-    const uploadResult = await db.storage.uploadFile(storagePath, Buffer.from(source.text, "utf-8"), {
-        contentType: source.mimeType ?? "text/plain",
-        contentDisposition: fileName,
-    });
-    const fileId = uploadResult?.data?.id;
-    if (!fileId) {
-        throw new Error("dataset_text_source_upload_failed");
-    }
-    return fileId;
-}
-async function finalizeBuildResult(env, datasetId, withFirst) {
-    const db = await getDatasetDb(env);
-    const service = new service_1.DatasetService(db);
-    const datasetResult = await service.getDatasetById(datasetId);
-    if (!datasetResult.ok) {
-        throw new Error(datasetResult.error);
-    }
-    const previewResult = await service.previewRows(datasetId, 20);
-    if (!previewResult.ok) {
-        throw new Error(previewResult.error);
-    }
-    const reader = {
-        async read(cursorOrParams, limit) {
-            const params = typeof cursorOrParams === "object" && cursorOrParams !== null
-                ? cursorOrParams
-                : { cursor: cursorOrParams, limit };
-            const rowsResult = await service.readRows({
-                datasetId,
-                cursor: params.cursor,
-                limit: params.limit,
-            });
-            if (!rowsResult.ok) {
-                throw new Error(rowsResult.error);
-            }
-            return rowsResult.data;
-        },
-    };
-    if (!withFirst) {
-        return {
-            datasetId,
-            dataset: datasetResult.data,
-            previewRows: previewResult.data,
-            reader,
-        };
-    }
-    const firstResult = await service.readOne(datasetId);
-    if (!firstResult.ok) {
-        throw new Error(firstResult.error);
-    }
-    return {
-        datasetId,
-        dataset: datasetResult.data,
-        previewRows: previewResult.data,
-        reader,
-        firstRow: firstResult.data,
-    };
-}
-async function materializeQuerySource(env, source, params) {
-    const runtime = await (0, runtime_1.resolveRuntime)(source.domain, env);
-    const result = await runtime.db.query(source.query);
-    const rows = normalizeQueryRows(result);
-    const domainDescriptor = getDomainDescriptor(source.domain);
-    return await materializeRowsToDataset(env, {
-        datasetId: params.datasetId,
-        sandboxId: params.sandboxId,
-        title: params.title ?? source.title,
-        instructions: params.instructions,
-        sources: [
-            {
-                kind: "query",
-                query: source.query,
-                title: source.title,
-                explanation: source.explanation,
-                ...domainDescriptor,
-            },
-        ],
-        sourceKinds: ["query"],
-        analysis: {
-            query: source.query,
-            explanation: source.explanation,
-            ...domainDescriptor,
-        },
-        rows,
-        schema: params.schema,
-        inferSchema: !params.schema,
-        first: params.first,
-    });
-}
-async function materializeSingleFileLikeSource(state, source, targetDatasetId) {
-    if (!state.reactor) {
-        throw new Error("dataset_reactor_required");
-    }
-    if (!state.sandboxId) {
-        throw new Error("dataset_sandbox_required");
-    }
-    const fileId = source.kind === "file"
-        ? source.fileId
-        : await uploadInlineTextSource(state.env, targetDatasetId, source);
-    await createOrUpdateDatasetMetadata(state.env, {
-        datasetId: targetDatasetId,
-        sandboxId: state.sandboxId,
-        title: state.title ?? targetDatasetId,
-        instructions: state.instructions,
-        sources: [
-            source.kind === "file"
-                ? { kind: "file", fileId: source.fileId, description: source.description }
-                : {
-                    kind: "text",
-                    mimeType: source.mimeType,
-                    name: source.name,
-                    description: source.description,
-                },
-        ],
-        sourceKinds: [source.kind],
-        schema: state.outputSchema,
-        status: "building",
-    });
-    const parseStory = (0, file_dataset_agent_1.createFileParseStory)(fileId, {
-        datasetId: targetDatasetId,
-        instructions: state.instructions ?? buildFileDefaultInstructions(state.outputSchema),
-        reactor: state.reactor,
-        sandboxId: state.sandboxId,
-    });
-    await parseStory.parse(state.env);
-    if (!state.outputSchema) {
-        const db = await getDatasetDb(state.env);
-        const service = new service_1.DatasetService(db);
-        const readResult = await service.readRows({ datasetId: targetDatasetId, cursor: 0, limit: 1000 });
-        if (!readResult.ok) {
-            throw new Error(readResult.error);
-        }
-        const inferred = inferDatasetSchema(readResult.data.rows, `${targetDatasetId}Row`, "One dataset row");
-        const updateResult = await service.updateDatasetSchema({
-            datasetId: targetDatasetId,
-            schema: inferred,
-            status: "completed",
-        });
-        if (!updateResult.ok) {
-            throw new Error(updateResult.error);
-        }
-    }
-    if (state.first) {
-        const db = await getDatasetDb(state.env);
-        const service = new service_1.DatasetService(db);
-        const firstResult = await service.readOne(targetDatasetId);
-        if (!firstResult.ok) {
-            throw new Error(firstResult.error);
-        }
-    }
-    return targetDatasetId;
-}
-async function normalizeSourceToDatasetId(state, source, targetDatasetId, sourceIndex) {
-    if (source.kind === "dataset") {
-        return source.datasetId;
-    }
-    const intermediateDatasetId = makeIntermediateDatasetId(targetDatasetId, source.kind, sourceIndex);
-    if (source.kind === "query") {
-        await materializeQuerySource(state.env, source, {
-            datasetId: intermediateDatasetId,
-            sandboxId: state.sandboxId,
-            title: source.title,
-            first: false,
-        });
-        return intermediateDatasetId;
-    }
-    await materializeSingleFileLikeSource({
-        ...state,
-        outputSchema: undefined,
-        first: false,
-        instructions: buildRawSourceInstructions(source.kind),
-    }, source, intermediateDatasetId);
-    return intermediateDatasetId;
-}
-async function materializeDerivedDataset(state, targetDatasetId) {
-    if (!state.reactor) {
-        throw new Error("dataset_reactor_required");
-    }
-    if (!state.sandboxId) {
-        throw new Error("dataset_sandbox_required");
-    }
-    const normalizedSources = [];
-    for (let index = 0; index < state.sources.length; index++) {
-        normalizedSources.push(await normalizeSourceToDatasetId(state, state.sources[index], targetDatasetId, index));
-    }
-    const transformSchema = state.outputSchema ??
-        {
-            title: "DatasetRow",
-            description: "One dataset row",
-            schema: {
-                type: "object",
-                additionalProperties: true,
-                properties: {},
-            },
-        };
-    await createOrUpdateDatasetMetadata(state.env, {
-        datasetId: targetDatasetId,
-        sandboxId: state.sandboxId,
-        title: state.title ?? targetDatasetId,
-        instructions: state.instructions,
-        sources: state.sources.map((source) => source.kind === "query"
-            ? {
-                kind: "query",
-                query: source.query,
-                title: source.title,
-                explanation: source.explanation,
-                ...getDomainDescriptor(source.domain),
-            }
-            : source),
-        sourceKinds: state.sources.map((source) => source.kind),
-        schema: transformSchema,
-        status: "building",
-    });
-    const transformStory = (0, transform_dataset_agent_1.createTransformDatasetStory)({
-        sourceDatasetIds: normalizedSources,
-        outputSchema: transformSchema,
-        instructions: buildTransformInstructions(normalizedSources.length, state.instructions, state.outputSchema),
-        datasetId: targetDatasetId,
-        reactor: state.reactor,
-        sandboxId: state.sandboxId,
-    });
-    await transformStory.transform(state.env);
-    const db = await getDatasetDb(state.env);
-    const service = new service_1.DatasetService(db);
-    if (!state.outputSchema) {
-        const readResult = await service.readRows({ datasetId: targetDatasetId, cursor: 0, limit: 1000 });
-        if (!readResult.ok) {
-            throw new Error(readResult.error);
-        }
-        const inferred = inferDatasetSchema(readResult.data.rows, `${targetDatasetId}Row`, "One dataset row");
-        const updateResult = await service.updateDatasetSchema({
-            datasetId: targetDatasetId,
-            schema: inferred,
-            status: "completed",
-        });
-        if (!updateResult.ok) {
-            throw new Error(updateResult.error);
-        }
-    }
-    if (state.first) {
-        const firstResult = await service.readOne(targetDatasetId);
-        if (!firstResult.ok) {
-            throw new Error(firstResult.error);
-        }
-    }
-    return targetDatasetId;
-}
-function dataset(env) {
+import { buildObjectOutputInstructions } from "./builder/instructions.js";
+import { resolveDatasetResourceContext } from "./builder/context.js";
+import { createDatasetId } from "./id.js";
+import { completeDatasetStep, materializeDerivedDataset, materializeSingleFileLikeResource, } from "./builder/materialize.js";
+import { materializeQueryResource } from "./builder/materializeQuery.js";
+import { createDatasetBuildResult, finalizeBuildResult, } from "./builder/persistence.js";
+export function dataset(runtime, options = {}) {
+    const datasetId = normalizeDatasetId(options.datasetId);
+    const typedRuntime = runtime;
     const state = {
-        env,
-        sources: [],
+        runtime: typedRuntime,
+        env: typedRuntime.env,
+        resources: [],
+        output: "rows",
         inferSchema: false,
+        durable: options.durable,
         first: false,
     };
     const api = {
-        fromFile(source) {
-            state.sources.push({ kind: "file", ...source });
+        datasetId,
+        fromFile(resource) {
+            state.resources.push({ kind: "file", ...resource });
             return api;
         },
-        fromText(source) {
-            state.sources.push({ kind: "text", ...source });
+        fromText(resource) {
+            state.resources.push({ kind: "text", ...resource });
             return api;
         },
-        fromDataset(source) {
-            state.sources.push({ kind: "dataset", ...source });
+        fromDataset(resource) {
+            state.resources.push({ kind: "dataset", ...resource });
             return api;
         },
-        fromQuery(domain, source) {
-            state.sources.push({ kind: "query", domain, ...source });
+        fromContext(context) {
+            state.resources.push({ kind: "context", ...context });
+            return api;
+        },
+        from(...resources) {
+            for (const resource of resources) {
+                if ("kind" in resource) {
+                    state.resources.push(resource);
+                    continue;
+                }
+                if ("fileId" in resource) {
+                    state.resources.push({ kind: "file", ...resource });
+                    continue;
+                }
+                if ("datasetId" in resource) {
+                    state.resources.push({ kind: "dataset", ...resource });
+                    continue;
+                }
+                if ("id" in resource || "key" in resource) {
+                    state.resources.push({ kind: "context", ...resource });
+                    continue;
+                }
+                state.resources.push({ kind: "text", ...resource });
+            }
+            return api;
+        },
+        fromQuery(domain, resource) {
+            state.resources.push({ kind: "query", domain, ...resource });
             return api;
         },
         title(title) {
@@ -521,6 +78,20 @@ function dataset(env) {
             state.inferSchema = true;
             return api;
         },
+        auto() {
+            state.outputSchema = undefined;
+            state.inferSchema = true;
+            return api;
+        },
+        asRows() {
+            state.output = "rows";
+            return api;
+        },
+        asObject() {
+            state.output = "object";
+            state.first = true;
+            return api;
+        },
         instructions(instructions) {
             state.instructions = instructions;
             return api;
@@ -534,44 +105,82 @@ function dataset(env) {
             return api;
         },
         async build(options) {
-            if (state.sources.length === 0) {
-                throw new Error("dataset_sources_required");
+            if (state.resources.length === 0) {
+                throw new Error("dataset_resources_required");
             }
-            const targetDatasetId = String(options?.datasetId ?? (0, admin_1.id)());
-            const onlySource = state.sources[0];
-            const isSingleSource = state.sources.length === 1;
-            const hasInstructions = Boolean(String(state.instructions ?? "").trim());
-            if (isSingleSource && onlySource.kind === "query" && !hasInstructions) {
-                await materializeQuerySource(state.env, onlySource, {
+            const targetDatasetId = options?.datasetId
+                ? normalizeDatasetId(options.datasetId)
+                : datasetId;
+            const stateWithBuildOptions = {
+                ...state,
+                durable: options?.durable ?? state.durable,
+            };
+            const context = await resolveDatasetResourceContext(typedRuntime, targetDatasetId, stateWithBuildOptions.resources);
+            stateWithBuildOptions.resources = context.resources;
+            stateWithBuildOptions.contextId = context.contextId;
+            stateWithBuildOptions.contextResources = context.contextResources;
+            const effectiveState = stateWithBuildOptions.output === "object"
+                ? {
+                    ...stateWithBuildOptions,
+                    first: true,
+                    instructions: buildObjectOutputInstructions(stateWithBuildOptions.instructions),
+                }
+                : stateWithBuildOptions;
+            const onlyResource = effectiveState.resources[0];
+            const isSingleResource = effectiveState.resources.length === 1;
+            const hasInstructions = Boolean(String(effectiveState.instructions ?? "").trim());
+            if (isSingleResource && onlyResource.kind === "query" && !hasInstructions) {
+                await materializeQueryResource(effectiveState.runtime, onlyResource, {
                     datasetId: targetDatasetId,
-                    sandboxId: state.sandboxId,
-                    schema: state.outputSchema,
-                    title: state.title ?? onlySource.title,
-                    instructions: state.instructions,
-                    first: state.first,
+                    sandboxId: effectiveState.sandboxId,
+                    schema: effectiveState.outputSchema,
+                    title: effectiveState.title ?? onlyResource.title,
+                    instructions: effectiveState.instructions,
+                    first: effectiveState.first,
+                    contextId: effectiveState.contextId ?? "",
                 });
-                return await finalizeBuildResult(state.env, targetDatasetId, state.first);
+                return finalizeOutputResult(await finalizeBuildResult(effectiveState.runtime, targetDatasetId, effectiveState.first), effectiveState.output);
             }
-            if (isSingleSource && (onlySource.kind === "file" || onlySource.kind === "text")) {
-                if (!state.sandboxId) {
-                    throw new Error("dataset_sandbox_required");
-                }
-                if (!state.reactor) {
+            if (isSingleResource && (onlyResource.kind === "file" || onlyResource.kind === "text")) {
+                if (!effectiveState.reactor) {
                     throw new Error("dataset_reactor_required");
                 }
-                await materializeSingleFileLikeSource(state, onlySource, targetDatasetId);
-                return await finalizeBuildResult(state.env, targetDatasetId, state.first);
-            }
-            if (!state.sandboxId) {
-                throw new Error("dataset_sandbox_required");
+                await materializeSingleFileLikeResource(effectiveState, onlyResource, targetDatasetId);
+                const completed = await completeDatasetStep({
+                    runtime: effectiveState.runtime,
+                    datasetId: targetDatasetId,
+                    schema: effectiveState.outputSchema,
+                    first: effectiveState.first,
+                });
+                return finalizeOutputResult(createDatasetBuildResult(effectiveState.runtime, completed), effectiveState.output);
             }
-            if (!state.reactor) {
+            if (!effectiveState.reactor) {
                 throw new Error("dataset_reactor_required");
             }
-            await materializeDerivedDataset(state, targetDatasetId);
-            return await finalizeBuildResult(state.env, targetDatasetId, state.first);
+            await materializeDerivedDataset(effectiveState, targetDatasetId);
+            const completed = await completeDatasetStep({
+                runtime: effectiveState.runtime,
+                datasetId: targetDatasetId,
+                schema: effectiveState.outputSchema,
+                first: effectiveState.first,
+            });
+            return finalizeOutputResult(createDatasetBuildResult(effectiveState.runtime, completed), effectiveState.output);
         },
     };
     return api;
 }
-//# sourceMappingURL=dataset.js.map
+function normalizeDatasetId(datasetId) {
+    const normalized = String(datasetId ?? createDatasetId()).trim();
+    if (!normalized) {
+        throw new Error("dataset_id_required");
+    }
+    return normalized;
+}
+function finalizeOutputResult(result, output) {
+    if (output !== "object")
+        return result;
+    return {
+        ...result,
+        object: result.firstRow ?? null,
+    };
+}