@ekairos/dataset 1.22.39-beta.development.0 → 1.22.39

package/dist/contextWorkspace.js

@@ -0,0 +1,234 @@
+import { readInstantFileStep } from "./file/steps.js";
+import { runDatasetSandboxCommandStep, writeDatasetSandboxFilesStep, writeDatasetSandboxTextFilesStep, } from "./sandbox/steps.js";
+const CONTEXT_WORKSPACE_BASE = "/tmp/ekairos/contexts";
+const WORKSPACE_MANIFEST_FILE_NAME = "manifest.json";
+function trimTrailingSlash(value) {
+    return value.endsWith("/") ? value.slice(0, -1) : value;
+}
+function sanitizePathSegment(value, fallback) {
+    const parts = String(value ?? "")
+        .trim()
+        .replace(/\\/g, "/")
+        .split("/")
+        .filter(Boolean);
+    const normalized = parts[parts.length - 1]
+        ?.replace(/[^a-zA-Z0-9_.-]/g, "_")
+        .replace(/_+/g, "_")
+        .slice(0, 160);
+    return normalized || fallback;
+}
+export function sanitizeContextWorkspacePathSegment(value, fallback) {
+    return sanitizePathSegment(value, fallback);
+}
+function filenameFromContentDisposition(value, fallback) {
+    const raw = String(value ?? "").trim();
+    if (!raw)
+        return fallback;
+    const filenameStar = raw.match(/filename\*=UTF-8''([^;]+)/i)?.[1];
+    if (filenameStar) {
+        return sanitizePathSegment(decodeURIComponent(filenameStar), fallback);
+    }
+    const filename = raw.match(/filename="?([^";]+)"?/i)?.[1];
+    return sanitizePathSegment(filename ?? raw, fallback);
+}
+function resolveContextEventPartDir(params) {
+    const sourceEventId = sanitizePathSegment(params.sourceEventId, "event");
+    const sourcePartIndex = Number.isFinite(params.sourcePartIndex)
+        ? Math.max(0, Math.floor(params.sourcePartIndex))
+        : 0;
+    return `${params.eventsDir}/${sourceEventId}/parts/${sourcePartIndex}`;
+}
+function resolveWorkspaceFilePath(params) {
+    return `${resolveContextEventPartDir(params)}/file`;
+}
+export function getContextWorkspaceBase() {
+    return trimTrailingSlash(CONTEXT_WORKSPACE_BASE);
+}
+export function getContextExecutionWorkspaceRoot(params) {
+    if (params.root)
+        return trimTrailingSlash(params.root);
+    const contextId = sanitizePathSegment(params.contextId, "context");
+    const executionId = sanitizePathSegment(params.executionId, "execution");
+    return `${getContextWorkspaceBase()}/${contextId}/executions/${executionId}`;
+}
+export function getContextWorkspaceRoot(params) {
+    if (params.root)
+        return trimTrailingSlash(params.root);
+    const contextId = sanitizePathSegment(params.contextId, "context");
+    return `${getContextWorkspaceBase()}/${contextId}`;
+}
+export function getContextEventsDir(params) {
+    return `${getContextWorkspaceRoot(params)}/events`;
+}
+export function getContextResourcesDir(params) {
+    return `${getContextWorkspaceRoot(params)}/resources`;
+}
+export function getContextExecutionWorkspaceDirs(params) {
+    const root = getContextExecutionWorkspaceRoot(params);
+    const contextRoot = getContextWorkspaceRoot(params);
+    const eventsDir = getContextEventsDir(params);
+    const resourcesDir = getContextResourcesDir(params);
+    return {
+        root,
+        contextRoot,
+        eventsDir,
+        resourcesDir,
+        outputDir: `${root}/output`,
+        scriptsDir: `${root}/scripts`,
+        tmpDir: `${root}/tmp`,
+        manifestPath: `${root}/${WORKSPACE_MANIFEST_FILE_NAME}`,
+    };
+}
+export function getContextExecutionWorkspaceStandardDirs(params) {
+    const dirs = getContextExecutionWorkspaceDirs(params);
+    return [
+        dirs.contextRoot,
+        dirs.eventsDir,
+        dirs.resourcesDir,
+        dirs.root,
+        dirs.outputDir,
+        dirs.scriptsDir,
+        dirs.tmpDir,
+    ];
+}
+export function extractContextWorkspaceFilesFromEventItems(eventItems) {
+    const files = [];
+    for (const item of eventItems) {
+        const itemRecord = asRecord(item);
+        const parts = Array.isArray(asRecord(itemRecord?.content)?.parts)
+            ? asRecord(itemRecord?.content)?.parts
+            : [];
+        parts.forEach((part, partIndex) => {
+            collectPartFiles(part, {
+                files,
+                sourceEventId: asText(itemRecord?.id),
+                sourcePartIndex: partIndex,
+            });
+        });
+    }
+    return files;
+}
+export async function prepareContextExecutionWorkspaceStep(params) {
+    "use step";
+    const dirs = getContextExecutionWorkspaceDirs(params);
+    const filePartDirs = Array.from(new Set(params.files.map((fileInput) => resolveContextEventPartDir({
+        eventsDir: dirs.eventsDir,
+        sourceEventId: fileInput.sourceEventId ?? fileInput.fileId,
+        sourcePartIndex: fileInput.sourcePartIndex ?? 0,
+    }))));
+    await runDatasetSandboxCommandStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        cmd: "mkdir",
+        args: ["-p", ...getContextExecutionWorkspaceStandardDirs(params), ...filePartDirs],
+    });
+    const preparedFiles = [];
+    for (const fileInput of params.files) {
+        const fileId = String(fileInput.fileId ?? "").trim();
+        if (!fileId)
+            continue;
+        const file = await readInstantFileStep({ runtime: params.runtime, fileId });
+        const filename = sanitizePathSegment(fileInput.filename ??
+            filenameFromContentDisposition(file.contentDisposition, `${fileId}.bin`), `${fileId}.bin`);
+        const path = resolveWorkspaceFilePath({
+            eventsDir: dirs.eventsDir,
+            sourceEventId: fileInput.sourceEventId ?? fileId,
+            sourcePartIndex: fileInput.sourcePartIndex ?? 0,
+        });
+        const metadataPath = `${resolveContextEventPartDir({
+            eventsDir: dirs.eventsDir,
+            sourceEventId: fileInput.sourceEventId ?? fileId,
+            sourcePartIndex: fileInput.sourcePartIndex ?? 0,
+        })}/metadata.json`;
+        await writeDatasetSandboxFilesStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            files: [{ path, contentBase64: file.contentBase64 }],
+        });
+        await writeDatasetSandboxTextFilesStep({
+            runtime: params.runtime,
+            sandboxId: params.sandboxId,
+            files: [
+                {
+                    path: metadataPath,
+                    content: JSON.stringify({
+                        fileId,
+                        filename,
+                        mediaType: fileInput.mediaType,
+                        role: fileInput.role ?? "input",
+                        sourceEventId: fileInput.sourceEventId,
+                        sourcePartIndex: fileInput.sourcePartIndex,
+                    }, null, 2),
+                },
+            ],
+        });
+        preparedFiles.push({
+            fileId,
+            filename,
+            mediaType: fileInput.mediaType,
+            role: fileInput.role ?? "input",
+            path,
+            sourceEventId: fileInput.sourceEventId,
+            sourcePartIndex: fileInput.sourcePartIndex,
+        });
+    }
+    const manifest = {
+        contextId: params.contextId,
+        executionId: params.executionId,
+        sandboxId: params.sandboxId,
+        ...dirs,
+        files: preparedFiles,
+    };
+    await writeDatasetSandboxTextFilesStep({
+        runtime: params.runtime,
+        sandboxId: params.sandboxId,
+        files: [
+            {
+                path: dirs.manifestPath,
+                content: JSON.stringify(manifest, null, 2),
+            },
+        ],
+    });
+    return manifest;
+}
+function collectPartFiles(value, params) {
+    const record = asRecord(value);
+    if (!record)
+        return;
+    if (record.type === "file") {
+        pushFileRecord(record, params);
+        return;
+    }
+    const content = asRecord(record.content);
+    if (!content)
+        return;
+    if (Array.isArray(content.blocks)) {
+        for (const block of content.blocks) {
+            const blockRecord = asRecord(block);
+            if (blockRecord?.type === "file") {
+                pushFileRecord(blockRecord, params);
+            }
+        }
+    }
+}
+function pushFileRecord(record, params) {
+    const fileId = asText(record.fileId);
+    if (!fileId)
+        return;
+    params.files.push({
+        fileId,
+        filename: asText(record.filename),
+        mediaType: asText(record.mediaType),
+        role: "input",
+        sourceEventId: params.sourceEventId,
+        sourcePartIndex: params.sourcePartIndex,
+    });
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function asText(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}

package/dist/dataset/steps.d.ts

@@ -1,46 +1,70 @@
-export declare function getDatasetServiceDb(env?: any): Promise<any>;
+export declare function getDatasetRuntimeDb(runtime: any): Promise<any>;
+export declare function getDatasetServiceDb(runtime: any): Promise<any>;
 export declare function datasetGetByIdStep(params: {
-    env?: any;
+    runtime: any;
     datasetId: string;
-}): Promise<import("../service").ServiceResult<any>>;
+}): Promise<import("../service.js").ServiceResult<any>>;
 export declare function datasetReadOutputJsonlStep(params: {
-    env?: any;
+    runtime: any;
     datasetId: string;
 }): Promise<{
     contentBase64: string;
 }>;
 export declare function datasetUpdateSchemaStep(params: {
-    env?: any;
+    runtime: any;
     datasetId: string;
     schema: any;
     status?: string;
-}): Promise<import("../service").ServiceResult<void>>;
+}): Promise<import("../service.js").ServiceResult<void>>;
 export declare function datasetUploadOutputFileStep(params: {
-    env?: any;
+    runtime: any;
     datasetId: string;
-    fileBuffer: Buffer;
-}): Promise<import("../service").ServiceResult<{
+    contentBase64: string;
+}): Promise<import("../service.js").ServiceResult<{
     fileId: string;
     storagePath: string;
 }>>;
 export declare function datasetUpdateStatusStep(params: {
-    env?: any;
+    runtime: any;
     datasetId: string;
     status: string;
     calculatedTotalRows?: number;
     actualGeneratedRowCount?: number;
-}): Promise<import("../service").ServiceResult<void>>;
+}): Promise<import("../service.js").ServiceResult<void>>;
 export declare function datasetClearStep(params: {
-    env?: any;
+    runtime: any;
     datasetId: string;
-}): Promise<import("../service").ServiceResult<{
+}): Promise<import("../service.js").ServiceResult<{
     deletedCount: number;
 }>>;
 export declare function datasetPreviewRowsStep(params: {
-    env?: any;
+    runtime: any;
+    datasetId: string;
+    limit?: number;
+}): Promise<{
+    rows: any[];
+}>;
+export declare function datasetReadRowsStep(params: {
+    runtime: any;
     datasetId: string;
+    cursor?: number;
     limit?: number;
 }): Promise<{
     rows: any[];
+    cursor: number;
+    done: boolean;
+}>;
+export declare function datasetReadOneStep(params: {
+    runtime: any;
+    datasetId: string;
+}): Promise<{
+    row: any | null;
+}>;
+export declare function datasetInferAndUpdateSchemaStep(params: {
+    runtime: any;
+    datasetId: string;
+    title?: string;
+    description?: string;
+}): Promise<{
+    schema: import("../dataset.js").DatasetSchemaInput;
 }>;
-//# sourceMappingURL=steps.d.ts.map

package/dist/dataset/steps.js

@@ -1,32 +1,33 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDatasetServiceDb = getDatasetServiceDb;
-exports.datasetGetByIdStep = datasetGetByIdStep;
-exports.datasetReadOutputJsonlStep = datasetReadOutputJsonlStep;
-exports.datasetUpdateSchemaStep = datasetUpdateSchemaStep;
-exports.datasetUploadOutputFileStep = datasetUploadOutputFileStep;
-exports.datasetUpdateStatusStep = datasetUpdateStatusStep;
-exports.datasetClearStep = datasetClearStep;
-exports.datasetPreviewRowsStep = datasetPreviewRowsStep;
-const runtime_1 = require("@ekairos/events/runtime");
-const service_1 = require("../service");
-async function resolveEnv(env) {
-    return env ?? (await (0, runtime_1.getContextEnv)());
+import { DatasetService } from "../service.js";
+import { datasetDomain } from "../schema.js";
+import { inferDatasetSchema } from "../builder/schemaInference.js";
+import { rowsToJsonl } from "../builder/rows.js";
+export async function getDatasetRuntimeDb(runtime) {
+    if (!runtime) {
+        throw new Error("Dataset step requires runtime.");
+    }
+    if (typeof runtime.use === "function") {
+        const scoped = await runtime.use(datasetDomain);
+        const scopedDb = scoped.db;
+        return typeof scopedDb === "function" ? await scopedDb.call(scoped) : scopedDb;
+    }
+    const db = runtime.db;
+    return typeof db === "function" ? await db.call(runtime) : db;
 }
-async function getDatasetServiceDb(env) {
+export async function getDatasetServiceDb(runtime) {
     "use step";
-    const runtime = (await (0, runtime_1.getContextRuntime)(await resolveEnv(env)));
-    return runtime.db;
+    return await getDatasetRuntimeDb(runtime);
 }
-async function datasetGetByIdStep(params) {
+export async function datasetGetByIdStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
-    const service = new service_1.DatasetService(db);
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     return await service.getDatasetById(params.datasetId);
 }
-async function datasetReadOutputJsonlStep(params) {
+export async function datasetReadOutputJsonlStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     for (let attempt = 1; attempt <= 20; attempt++) {
         const query = await db.query({
             dataset_datasets: {
@@ -41,33 +42,43 @@ async function datasetReadOutputJsonlStep(params) {
             const fileBuffer = await fetch(url).then((r) => r.arrayBuffer());
             return { contentBase64: Buffer.from(fileBuffer).toString("base64") };
         }
+        const directRows = await service.readRows({
+            datasetId: params.datasetId,
+            cursor: 0,
+            limit: 100000,
+        });
+        if (directRows.ok && directRows.data.rows.length > 0) {
+            return {
+                contentBase64: Buffer.from(rowsToJsonl(directRows.data.rows), "utf-8").toString("base64"),
+            };
+        }
         await new Promise((resolve) => setTimeout(resolve, 250 * attempt));
     }
     throw new Error("Dataset output file not found");
 }
-async function datasetUpdateSchemaStep(params) {
+export async function datasetUpdateSchemaStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
-    const service = new service_1.DatasetService(db);
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     return await service.updateDatasetSchema({
         datasetId: params.datasetId,
         schema: params.schema,
         status: params.status,
     });
 }
-async function datasetUploadOutputFileStep(params) {
+export async function datasetUploadOutputFileStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
-    const service = new service_1.DatasetService(db);
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     return await service.uploadDatasetOutputFile({
         datasetId: params.datasetId,
-        fileBuffer: params.fileBuffer,
+        fileBuffer: Buffer.from(params.contentBase64, "base64"),
     });
 }
-async function datasetUpdateStatusStep(params) {
+export async function datasetUpdateStatusStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
-    const service = new service_1.DatasetService(db);
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     return await service.updateDatasetStatus({
         datasetId: params.datasetId,
         status: params.status,
@@ -75,20 +86,66 @@ async function datasetUpdateStatusStep(params) {
         actualGeneratedRowCount: params.actualGeneratedRowCount,
     });
 }
-async function datasetClearStep(params) {
+export async function datasetClearStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
-    const service = new service_1.DatasetService(db);
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     return await service.clearDataset(params.datasetId);
 }
-async function datasetPreviewRowsStep(params) {
+export async function datasetPreviewRowsStep(params) {
     "use step";
-    const db = (await (0, runtime_1.getContextRuntime)(await resolveEnv(params.env))).db;
-    const service = new service_1.DatasetService(db);
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
     const rowsResult = await service.previewRows(params.datasetId, params.limit ?? 20);
     if (!rowsResult.ok) {
         throw new Error(rowsResult.error);
     }
     return { rows: rowsResult.data };
 }
-//# sourceMappingURL=steps.js.map
+export async function datasetReadRowsStep(params) {
+    "use step";
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
+    const rowsResult = await service.readRows({
+        datasetId: params.datasetId,
+        cursor: params.cursor,
+        limit: params.limit,
+    });
+    if (!rowsResult.ok) {
+        throw new Error(rowsResult.error);
+    }
+    return rowsResult.data;
+}
+export async function datasetReadOneStep(params) {
+    "use step";
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
+    const firstResult = await service.readOne(params.datasetId);
+    if (!firstResult.ok) {
+        throw new Error(firstResult.error);
+    }
+    return { row: firstResult.data };
+}
+export async function datasetInferAndUpdateSchemaStep(params) {
+    "use step";
+    const db = await getDatasetRuntimeDb(params.runtime);
+    const service = new DatasetService(db);
+    const readResult = await service.readRows({
+        datasetId: params.datasetId,
+        cursor: 0,
+        limit: 1000,
+    });
+    if (!readResult.ok) {
+        throw new Error(readResult.error);
+    }
+    const inferred = inferDatasetSchema(readResult.data.rows, params.title ?? `${params.datasetId}Row`, params.description ?? "One dataset row");
+    const updateResult = await service.updateDatasetSchema({
+        datasetId: params.datasetId,
+        schema: inferred,
+        status: "completed",
+    });
+    if (!updateResult.ok) {
+        throw new Error(updateResult.error);
+    }
+    return { schema: inferred };
+}

package/dist/dataset.d.ts

@@ -1,67 +1,3 @@
-import type { DomainSchemaResult } from "@ekairos/domain";
-import type { ContextReactor } from "@ekairos/events";
-export type DatasetQuerySourceInput<D extends DomainSchemaResult = DomainSchemaResult> = {
-    query: Record<string, any>;
-    title?: string;
-    explanation?: string;
-    domain: D;
-};
-export type DatasetFileSourceInput = {
-    fileId: string;
-    description?: string;
-};
-export type DatasetTextSourceInput = {
-    text: string;
-    mimeType?: string;
-    name?: string;
-    description?: string;
-};
-export type DatasetExistingSourceInput = {
-    datasetId: string;
-    description?: string;
-};
-export type DatasetSchemaInput = {
-    title?: string;
-    description?: string;
-    schema: any;
-};
-export type DatasetReaderResult = {
-    rows: any[];
-    cursor: number;
-    done: boolean;
-};
-export type DatasetReader = {
-    read(cursor?: number, limit?: number): Promise<DatasetReaderResult>;
-    read(params?: {
-        cursor?: number;
-        limit?: number;
-    }): Promise<DatasetReaderResult>;
-};
-export type DatasetBuildResult = {
-    datasetId: string;
-    dataset: any;
-    previewRows: any[];
-    reader: DatasetReader;
-    firstRow?: any | null;
-};
-export declare function dataset<Env extends {
-    orgId: string;
-}>(env: Env): {
-    fromFile(source: DatasetFileSourceInput): /*elided*/ any;
-    fromText(source: DatasetTextSourceInput): /*elided*/ any;
-    fromDataset(source: DatasetExistingSourceInput): /*elided*/ any;
-    fromQuery<D extends DomainSchemaResult>(domain: D, source: Omit<DatasetQuerySourceInput<D>, "domain">): /*elided*/ any;
-    title(title: string): /*elided*/ any;
-    sandbox(input: {
-        sandboxId: string;
-    }): /*elided*/ any;
-    schema(schema: DatasetSchemaInput): /*elided*/ any;
-    inferSchema(): /*elided*/ any;
-    instructions(instructions: string): /*elided*/ any;
-    reactor(reactor: ContextReactor<any, any>): /*elided*/ any;
-    first(): /*elided*/ any;
-    build(options?: {
-        datasetId?: string;
-    }): Promise<DatasetBuildResult>;
-};
-//# sourceMappingURL=dataset.d.ts.map
+import type { AnyDatasetRuntime, DatasetBuilder, DatasetBuilderOptions, DatasetRuntimeHandle } from "./builder/types.js";
+export type { AnyDatasetRuntime, CompatibleQueryDomain, DatasetBuilder, DatasetBuilderOptions, DatasetBuildOptions, DatasetBuildResult, DatasetExistingResource, DatasetExistingResourceInput, DatasetFileResource, DatasetFileResourceInput, DatasetMode, DatasetOutput, DatasetQueryResourceInput, DatasetReader, DatasetReaderResult, DatasetRuntimeEnv, DatasetRuntimeHandle, DatasetSchemaInput, DatasetTextResource, DatasetResourceInput, DatasetTextResourceInput, } from "./builder/types.js";
+export declare function dataset<Runtime extends AnyDatasetRuntime>(runtime: Runtime & DatasetRuntimeHandle<Runtime>, options?: DatasetBuilderOptions): DatasetBuilder<Runtime>;