@ekairos/dataset 1.22.40-beta.development.0 → 1.22.40

package/dist/agents.d.ts

@@ -0,0 +1,8 @@
+import "./builder/materialize.js";
+export * from "./dataset.js";
+export * from "./domain.js";
+export * from "./materializeDataset.tool.js";
+export * from "./schema.js";
+export * from "./service.js";
+export * from "./file/file-dataset.agent.js";
+export * from "./transform/index.js";

package/dist/agents.js

@@ -0,0 +1,8 @@
+import "./builder/materialize.js";
+export * from "./dataset.js";
+export * from "./domain.js";
+export * from "./materializeDataset.tool.js";
+export * from "./schema.js";
+export * from "./service.js";
+export * from "./file/file-dataset.agent.js";
+export * from "./transform/index.js";

package/dist/builder/agentMaterializers.d.ts

@@ -0,0 +1,9 @@
+import type { AnyDatasetRuntime, DatasetBuilderState, InternalDatasetResource } from "./types.js";
+export type DatasetAgentMaterializers = {
+    materializeSingleFileLikeResource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, resource: Extract<InternalDatasetResource, {
+        kind: "file" | "text";
+    }>, targetDatasetId: string): Promise<string>;
+    materializeDerivedDataset<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, targetDatasetId: string): Promise<string>;
+};
+export declare function registerDatasetAgentMaterializers(materializers: DatasetAgentMaterializers): void;
+export declare function getDatasetAgentMaterializers(): DatasetAgentMaterializers;

package/dist/builder/agentMaterializers.js

@@ -0,0 +1,10 @@
+let agentMaterializers = null;
+export function registerDatasetAgentMaterializers(materializers) {
+    agentMaterializers = materializers;
+}
+export function getDatasetAgentMaterializers() {
+    if (!agentMaterializers) {
+        throw new Error("dataset_agent_materializers_not_registered");
+    }
+    return agentMaterializers;
+}

package/dist/builder/context.d.ts

@@ -0,0 +1,15 @@
+import type { AnyDatasetRuntime, InternalDatasetResource } from "./types.js";
+type DatasetContextResolution = {
+    contextId: string;
+    resources: InternalDatasetResource[];
+    contextResources: DatasetContextResourceRecord[];
+};
+type DatasetContextResourceRecord = {
+    key: string;
+    type: string;
+    name: string;
+    description: string;
+    [key: string]: unknown;
+};
+export declare function resolveDatasetResourceContext<Runtime extends AnyDatasetRuntime>(runtime: Runtime, datasetId: string, resources: InternalDatasetResource[]): Promise<DatasetContextResolution>;
+export {};

package/dist/builder/context.js

@@ -0,0 +1,251 @@
+import { eventsDomain } from "@ekairos/events";
+import { createDatasetId } from "../id.js";
+import { datasetDomain } from "../schema.js";
+import { DatasetService } from "../service.js";
+import { getDomainDescriptor } from "./rows.js";
+function getContextWhere(context) {
+    return "id" in context ? { id: context.id } : { key: context.key };
+}
+async function getEventsDb(runtime) {
+    const scoped = await runtime.use(eventsDomain);
+    return scoped.db;
+}
+async function getDatasetDb(runtime) {
+    const scoped = await runtime.use(datasetDomain);
+    return scoped.db;
+}
+function resourceKey(index, resource) {
+    if (resource.kind === "file")
+        return `file:${index}:${resource.fileId}`;
+    if (resource.kind === "text")
+        return `text:${index}:${resource.name ?? "inline"}`;
+    if (resource.kind === "dataset")
+        return `dataset:${index}:${resource.datasetId}`;
+    if (resource.kind === "query")
+        return `query:${index}:${resource.title ?? "query"}`;
+    return `resource:${index}`;
+}
+function resourceName(index, resource) {
+    if (resource.kind === "file")
+        return resource.filename ?? `File ${index + 1}`;
+    if (resource.kind === "text")
+        return resource.name ?? `Text ${index + 1}`;
+    if (resource.kind === "dataset")
+        return resource.datasetId;
+    if (resource.kind === "query")
+        return resource.title ?? `Query ${index + 1}`;
+    return `Resource ${index + 1}`;
+}
+function resourceDescription(resource) {
+    if ("description" in resource && typeof resource.description === "string" && resource.description.trim()) {
+        return resource.description.trim();
+    }
+    if (resource.kind === "query" && typeof resource.explanation === "string" && resource.explanation.trim()) {
+        return resource.explanation.trim();
+    }
+    return `Dataset ${resource.kind} resource.`;
+}
+function resourceToContextResource(index, resource) {
+    const base = {
+        key: resourceKey(index, resource),
+        type: resource.kind,
+        name: resourceName(index, resource),
+        description: resourceDescription(resource),
+    };
+    if (resource.kind === "file") {
+        return {
+            ...base,
+            fileId: resource.fileId,
+            filename: resource.filename,
+            mediaType: resource.mediaType,
+        };
+    }
+    if (resource.kind === "text") {
+        return {
+            ...base,
+            text: resource.text,
+            mimeType: resource.mimeType,
+        };
+    }
+    if (resource.kind === "dataset") {
+        return {
+            ...base,
+            datasetId: resource.datasetId,
+        };
+    }
+    if (resource.kind === "query") {
+        return {
+            ...base,
+            query: resource.query,
+            title: resource.title,
+            explanation: resource.explanation,
+            ...getDomainDescriptor(resource.domain),
+        };
+    }
+    return base;
+}
+async function createDatasetResourceContextStep(params) {
+    "use step";
+    const db = await getEventsDb(params.runtime);
+    const contextKey = `dataset:${params.datasetId}`;
+    const existing = await db.query({
+        event_contexts: {
+            $: { where: { key: contextKey }, limit: 1 },
+        },
+    });
+    const contextId = existing.event_contexts?.[0]?.id ?? createDatasetId();
+    const now = new Date();
+    const resources = await enrichDatasetContextResources(params.runtime, params.resources);
+    await db.transact([
+        db.tx.event_contexts[contextId].update({
+            key: contextKey,
+            createdAt: now,
+            updatedAt: now,
+            name: `Dataset ${params.datasetId}`,
+            status: "open_idle",
+            content: {
+                datasetId: params.datasetId,
+                resourceCount: resources.length,
+            },
+            resources,
+            description: `Dataset execution context for ${params.datasetId}.`,
+            goal: "Produce the dataset output from the resources declared in this context.",
+        }),
+    ]);
+    return {
+        contextId,
+    };
+}
+async function enrichDatasetContextResources(runtime, resources) {
+    const datasetResources = resources.filter((resource) => resource.type === "dataset" && typeof resource.datasetId === "string");
+    if (datasetResources.length === 0)
+        return resources;
+    const db = await getDatasetDb(runtime);
+    const service = new DatasetService(db);
+    const enriched = [];
+    for (const resource of resources) {
+        if (resource.type !== "dataset" || typeof resource.datasetId !== "string") {
+            enriched.push(resource);
+            continue;
+        }
+        const preview = await service.previewRows(resource.datasetId, 20);
+        if (!preview.ok) {
+            enriched.push({
+                ...resource,
+                previewError: preview.error,
+            });
+            continue;
+        }
+        enriched.push({
+            ...resource,
+            previewRows: preview.data,
+            previewLimit: 20,
+        });
+    }
+    return enriched;
+}
+function contextResourceToDatasetResource(resource) {
+    if (resource.type === "file" && typeof resource.fileId === "string" && resource.fileId.trim()) {
+        return {
+            kind: "file",
+            fileId: resource.fileId.trim(),
+            description: resource.description,
+            filename: typeof resource.filename === "string" ? resource.filename : undefined,
+            mediaType: typeof resource.mediaType === "string" ? resource.mediaType : undefined,
+        };
+    }
+    if (resource.type === "dataset" &&
+        typeof resource.datasetId === "string" &&
+        resource.datasetId.trim()) {
+        return {
+            kind: "dataset",
+            datasetId: resource.datasetId.trim(),
+            description: resource.description,
+        };
+    }
+    if (resource.type === "text" && typeof resource.text === "string") {
+        return {
+            kind: "text",
+            text: String(resource.text),
+            mimeType: typeof resource.mimeType === "string"
+                ? String(resource.mimeType)
+                : "text/plain",
+            name: resource.name,
+            description: resource.description,
+        };
+    }
+    if (resource.type === "query") {
+        throw new Error("dataset_context_query_resource_requires_builder_shortcut");
+    }
+    return {
+        kind: "text",
+        text: JSON.stringify({ resource }, null, 2),
+        mimeType: "application/vnd.ekairos.context-resource+json",
+        name: `${resource.key}.context-resource.json`,
+        description: resource.description,
+    };
+}
+async function readExistingContext(params) {
+    "use step";
+    const db = await getEventsDb(params.runtime);
+    const res = await db.query({
+        event_contexts: {
+            $: {
+                where: getContextWhere(params.context),
+                limit: 1,
+            },
+        },
+    });
+    const row = res?.event_contexts?.[0];
+    if (!row?.id)
+        throw new Error("dataset_context_not_found");
+    const resources = Array.isArray(row.resources)
+        ? row.resources
+        : [];
+    if (resources.length === 0) {
+        throw new Error("dataset_context_resources_required");
+    }
+    const sourceContextId = String(row.id);
+    const copiedResources = resources.map((resource) => ({
+        ...resource,
+        sourceContextId: resource.sourceContextId ?? sourceContextId,
+        sourceResourceKey: resource.sourceResourceKey ?? resource.key,
+    }));
+    return {
+        contextId: sourceContextId,
+        resources: resources.map((resource) => contextResourceToDatasetResource(resource)),
+        contextResources: copiedResources,
+    };
+}
+export async function resolveDatasetResourceContext(runtime, datasetId, resources) {
+    const contextRefs = resources.filter((resource) => resource.kind === "context");
+    if (contextRefs.length > 1) {
+        throw new Error("dataset_context_resource_must_be_unique");
+    }
+    if (contextRefs.length === 1) {
+        if (resources.length > 1) {
+            throw new Error("dataset_context_resource_is_exclusive");
+        }
+        const source = await readExistingContext({ runtime, context: contextRefs[0] });
+        const created = await createDatasetResourceContextStep({
+            runtime,
+            datasetId,
+            resources: source.contextResources,
+        });
+        return {
+            ...source,
+            contextId: created.contextId,
+        };
+    }
+    const contextResourceRecords = resources.map((resource, index) => resourceToContextResource(index, resource));
+    const created = await createDatasetResourceContextStep({
+        runtime,
+        datasetId,
+        resources: contextResourceRecords,
+    });
+    return {
+        contextId: created.contextId,
+        resources,
+        contextResources: contextResourceRecords,
+    };
+}

package/dist/builder/instructions.d.ts

@@ -1,6 +1,5 @@
-import type { DatasetSchemaInput } from "./types";
-export declare function buildFileDefaultInstructions(schema?: DatasetSchemaInput): "Create a dataset from the source file and ensure each output row matches the provided dataset schema exactly." | "Create a dataset representing the source content as structured rows.";
-export declare function buildRawSourceInstructions(sourceKind: "file" | "text"): "Create a dataset representing the raw text content as structured rows without applying business transformations." | "Create a dataset representing the raw file content as structured rows without applying business transformations.";
-export declare function buildTransformInstructions(sourceCount: number, userInstructions?: string, schema?: DatasetSchemaInput): string;
+import type { DatasetSchemaInput } from "./types.js";
+export declare function buildFileDefaultInstructions(schema?: DatasetSchemaInput): "Create a dataset from the resource file and ensure each output row matches the provided dataset schema exactly." | "Create a dataset representing the resource content as structured rows.";
+export declare function buildRawResourceInstructions(resourceKind: "file" | "text"): "Create a dataset representing the raw text content as structured rows without applying business transformations." | "Create a dataset representing the raw file content as structured rows without applying business transformations.";
+export declare function buildTransformInstructions(resourceCount: number, userInstructions?: string, schema?: DatasetSchemaInput): string;
 export declare function buildObjectOutputInstructions(userInstructions?: string): string;
-//# sourceMappingURL=instructions.d.ts.map

package/dist/builder/instructions.js

@@ -1,46 +1,40 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildFileDefaultInstructions = buildFileDefaultInstructions;
-exports.buildRawSourceInstructions = buildRawSourceInstructions;
-exports.buildTransformInstructions = buildTransformInstructions;
-exports.buildObjectOutputInstructions = buildObjectOutputInstructions;
-function buildFileDefaultInstructions(schema) {
+export function buildFileDefaultInstructions(schema) {
     if (schema) {
-        return "Create a dataset from the source file and ensure each output row matches the provided dataset schema exactly.";
+        return "Create a dataset from the resource file and ensure each output row matches the provided dataset schema exactly.";
     }
-    return "Create a dataset representing the source content as structured rows.";
+    return "Create a dataset representing the resource content as structured rows.";
 }
-function buildRawSourceInstructions(sourceKind) {
-    if (sourceKind === "text") {
+export function buildRawResourceInstructions(resourceKind) {
+    if (resourceKind === "text") {
         return "Create a dataset representing the raw text content as structured rows without applying business transformations.";
     }
     return "Create a dataset representing the raw file content as structured rows without applying business transformations.";
 }
-function buildTransformInstructions(sourceCount, userInstructions, schema) {
+export function buildTransformInstructions(resourceCount, userInstructions, schema) {
     const explicit = String(userInstructions ?? "").trim();
     if (explicit)
         return explicit;
-    if (sourceCount > 1) {
+    if (resourceCount > 1) {
         if (schema) {
-            return "Combine the source datasets into a new dataset that matches the provided output schema exactly.";
+            return "Combine the input datasets into a new dataset that matches the provided output schema exactly.";
         }
-        return "Combine the source datasets into one coherent dataset.";
+        return "Combine the input datasets into one coherent dataset.";
     }
     if (schema) {
-        return "Transform the source dataset into a new dataset that matches the provided output schema exactly.";
+        return "Transform the input dataset into a new dataset that matches the provided output schema exactly.";
     }
-    return "Transform the source dataset into a new useful dataset.";
+    return "Transform the input dataset into a new useful dataset.";
 }
-function buildObjectOutputInstructions(userInstructions) {
+export function buildObjectOutputInstructions(userInstructions) {
     const base = String(userInstructions ?? "").trim();
     const objectContract = [
         "Output mode is object.",
-        "Produce exactly one JSONL row in output.jsonl.",
-        "That row must be {\"type\":\"row\",\"data\":<the final object>}.",
+        "Produce exactly one final object.",
+        "completeObject({ data: <the final object>, summary }) is available to complete the dataset directly.",
+        "If you use output.jsonl instead, produce exactly one row: {\"type\":\"row\",\"data\":<the final object>}.",
         "Do not emit multiple rows, headers, summaries, or metadata rows.",
     ].join("\n");
     if (!base)
         return objectContract;
     return [base, "", objectContract].join("\n");
 }
-//# sourceMappingURL=instructions.js.map

package/dist/builder/materialize.d.ts

@@ -1,16 +1,83 @@
-import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalSource } from "./types";
-export declare function materializeQuerySource<Runtime extends AnyDatasetRuntime>(runtime: DatasetBuilderState<Runtime>["runtime"], source: Extract<InternalSource, {
-    kind: "query";
-}>, params: {
+import type { AnyDatasetRuntime, DatasetBuilderState, DatasetSchemaInput, InternalDatasetResource } from "./types.js";
+import type { SandboxState } from "../file/file-dataset.types.js";
+import type { FilePreviewContext } from "../file/filepreview.types.js";
+import type { TransformSandboxState, TransformInputPreviewContext } from "../transform/transform-dataset.types.js";
+export declare function resolveDatasetAgentDurable(requestedDurable?: boolean): Promise<boolean>;
+type PreparedFileDatasetContext = {
+    kind: "file";
     datasetId: string;
-    sandboxId?: string;
-    schema?: DatasetSchemaInput;
+    sandboxId: string;
+    fileId: string;
+    sandboxState: SandboxState;
+    filePreview?: FilePreviewContext;
+    schema?: DatasetSchemaInput | null;
+    filename?: string;
+    mediaType?: string;
+};
+type PreparedTransformDatasetContext = {
+    kind: "transform";
+    datasetId: string;
+    sandboxId: string;
+    inputDatasetIds: string[];
+    outputSchema: DatasetSchemaInput;
+    sandboxState: TransformSandboxState;
+    inputPreviews?: Array<{
+        datasetId: string;
+        preview: TransformInputPreviewContext;
+    }>;
+};
+type PreparedDatasetContext = PreparedFileDatasetContext | PreparedTransformDatasetContext;
+type DatasetContextInitialization = PreparedDatasetContext & {
+    prompt: string;
+    instructions?: string;
+};
+export declare function initializeDatasetStep<Runtime extends AnyDatasetRuntime>(params: {
+    runtime: Runtime;
+    datasetId: string;
+    sandboxId: string;
     title?: string;
     instructions?: string;
-    first?: boolean;
-}): Promise<string>;
-export declare function materializeSingleFileLikeSource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, source: Extract<InternalSource, {
+    contextId: string;
+    schema?: DatasetSchemaInput;
+}): Promise<{
+    datasetId: string;
+    sandboxId: string;
+}>;
+export declare function prepareDatasetResourcesStep<Runtime extends AnyDatasetRuntime>(params: {
+    kind: "file";
+    runtime: Runtime;
+    datasetId: string;
+    sandboxId: string;
+    resource: Extract<InternalDatasetResource, {
+        kind: "file" | "text";
+    }>;
+    schema?: DatasetSchemaInput;
+} | {
+    kind: "transform";
+    runtime: Runtime;
+    datasetId: string;
+    sandboxId: string;
+    inputDatasetIds: string[];
+    outputSchema: DatasetSchemaInput;
+}): Promise<PreparedDatasetContext>;
+export declare function initializeDatasetContextStep(params: {
+    prepared: PreparedDatasetContext;
+    instructions?: string;
+    outputSchema?: DatasetSchemaInput;
+}): Promise<DatasetContextInitialization>;
+export declare function completeDatasetStep<Runtime extends AnyDatasetRuntime>(params: {
+    runtime: Runtime;
+    datasetId: string;
+    schema?: DatasetSchemaInput;
+    first: boolean;
+}): Promise<{
+    datasetId: string;
+    dataset: any;
+    previewRows: any[];
+    firstRow: any;
+}>;
+export declare function materializeSingleFileLikeResource<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, resource: Extract<InternalDatasetResource, {
     kind: "file" | "text";
 }>, targetDatasetId: string): Promise<string>;
 export declare function materializeDerivedDataset<Runtime extends AnyDatasetRuntime>(state: DatasetBuilderState<Runtime>, targetDatasetId: string): Promise<string>;
-//# sourceMappingURL=materialize.d.ts.map
+export {};