@ekairos/structure 1.21.54-beta.0 → 1.21.57-beta.0

package/dist/clearDataset.tool.d.ts

@@ -11,4 +11,3 @@ export declare function createClearDatasetTool({ datasetId, sandboxId, env }: Cl
     reason: string;
 }>;
 export {};
-//# sourceMappingURL=clearDataset.tool.d.ts.map

package/dist/clearDataset.tool.js

@@ -23,4 +23,3 @@ export function createClearDatasetTool({ datasetId, sandboxId, env }) {
         },
     });
 }
-//# sourceMappingURL=clearDataset.tool.js.map

package/dist/completeObject.tool.d.ts

@@ -20,4 +20,3 @@ export declare function createCompleteObjectTool({ datasetId, sandboxId, env }:
     error?: undefined;
 }>;
 export {};
-//# sourceMappingURL=completeObject.tool.d.ts.map

package/dist/completeObject.tool.js

@@ -77,4 +77,3 @@ export function createCompleteObjectTool({ datasetId, sandboxId, env }) {
         },
     });
 }
-//# sourceMappingURL=completeObject.tool.js.map

package/dist/completeRows.tool.d.ts

@@ -17,4 +17,3 @@ export declare function createCompleteRowsTool({ datasetId, sandboxId, env }: Co
     storagePath: string;
 }>;
 export {};
-//# sourceMappingURL=completeRows.tool.d.ts.map

package/dist/completeRows.tool.js

@@ -131,4 +131,3 @@ async function validateJsonlRows(params) {
     }
     return { success: true, validRowCount };
 }
-//# sourceMappingURL=completeRows.tool.js.map

package/dist/dataset/steps.d.ts

@@ -139,4 +139,3 @@ export declare function structureReadRowsOutputJsonlStep(params: {
     ok: false;
     error: string;
 }>;
-//# sourceMappingURL=steps.d.ts.map

package/dist/dataset/steps.js

@@ -1,8 +1,8 @@
-import { getStoryRuntime } from "../runtime";
 export async function structureGetOrCreateContextStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const ctx = await runtime.store.getOrCreateContext({ key: params.contextKey });
         return { ok: true, data: ctx };
     }
@@ -14,7 +14,8 @@ export async function structureGetOrCreateContextStep(params) {
 export async function structureGetContextStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const ctx = await runtime.store.getContext({ key: params.contextKey });
         if (!ctx)
             return { ok: false, error: "Context not found" };
@@ -28,7 +29,8 @@ export async function structureGetContextStep(params) {
 export async function structureUpdateContextContentStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const updated = await runtime.store.updateContextContent({ key: params.contextKey }, params.content);
         return { ok: true, data: updated };
     }
@@ -40,7 +42,8 @@ export async function structureUpdateContextContentStep(params) {
 export async function structurePatchContextContentStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const existing = await runtime.store.getOrCreateContext({ key: params.contextKey });
         const existingContent = (existing?.content ?? {});
         const existingStructure = (existingContent?.structure ?? {});
@@ -61,7 +64,8 @@ export async function structurePatchContextContentStep(params) {
 export async function structureUploadRowsOutputJsonlStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const db = runtime.db;
         const storagePath = `/structure/${params.structureId}/output.jsonl`;
         const fileBuffer = Buffer.from(params.contentBase64 ?? "", "base64");
@@ -82,7 +86,8 @@ export async function structureUploadRowsOutputJsonlStep(params) {
 export async function structureLinkRowsOutputFileToContextStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const store = runtime.store;
         const db = runtime.db;
         const ctx = await store.getOrCreateContext({ key: params.contextKey });
@@ -100,7 +105,8 @@ export async function structureLinkRowsOutputFileToContextStep(params) {
 export async function structureUnlinkRowsOutputFileFromContextStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const store = runtime.store;
         const db = runtime.db;
         const ctx = await store.getOrCreateContext({ key: params.contextKey });
@@ -118,7 +124,8 @@ export async function structureUnlinkRowsOutputFileFromContextStep(params) {
 export async function structureGetContextWithRowsOutputFileStep(params) {
     "use step";
     try {
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const db = runtime.db;
         const query = (await db.query({
             context_contexts: {
@@ -140,7 +147,8 @@ export async function structureReadRowsOutputJsonlStep(params) {
     "use step";
     try {
         const contextKey = `structure:${params.structureId}`;
-        const runtime = await getStoryRuntime(params.env);
+        const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+        const runtime = await resolveStoryRuntime(params.env);
         const db = runtime.db;
         const query = (await db.query({
             context_contexts: {
@@ -155,7 +163,7 @@ export async function structureReadRowsOutputJsonlStep(params) {
         const url = linked?.url;
         if (!url)
             return { ok: false, error: "Rows output file not found" };
-        const fileBuffer = await fetch(url).then((r) => r.arrayBuffer());
+        const fileBuffer = await fetchArrayBufferWithRetry(url, { attempts: 4, timeoutMs: 90000 });
         return { ok: true, data: { contentBase64: Buffer.from(fileBuffer).toString("base64") } };
     }
     catch (error) {
@@ -163,4 +171,32 @@ export async function structureReadRowsOutputJsonlStep(params) {
         return { ok: false, error: message };
     }
 }
-//# sourceMappingURL=steps.js.map
+async function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+async function fetchArrayBufferWithRetry(url, opts) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= opts.attempts; attempt++) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
+        try {
+            const res = await fetch(url, { signal: controller.signal });
+            if (!res.ok) {
+                throw new Error(`Failed to download rows output file (HTTP ${res.status})`);
+            }
+            return await res.arrayBuffer();
+        }
+        catch (e) {
+            lastError = e;
+            if (attempt < opts.attempts) {
+                await sleep(250 * Math.pow(2, attempt - 1));
+                continue;
+            }
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    const message = lastError instanceof Error ? lastError.message : String(lastError);
+    throw new Error(message || "Failed to download rows output file");
+}

package/dist/datasetFiles.d.ts

@@ -3,4 +3,3 @@ export declare const DATASET_OUTPUT_SCHEMA_FILE_NAME = "output_schema.json";
 export declare function getDatasetWorkstation(datasetId: string): string;
 export declare function getDatasetOutputPath(datasetId: string): string;
 export declare function getDatasetOutputSchemaPath(datasetId: string): string;
-//# sourceMappingURL=datasetFiles.d.ts.map

package/dist/datasetFiles.js

@@ -9,4 +9,3 @@ export function getDatasetOutputPath(datasetId) {
 export function getDatasetOutputSchemaPath(datasetId) {
     return `${getDatasetWorkstation(datasetId)}/${DATASET_OUTPUT_SCHEMA_FILE_NAME}`;
 }
-//# sourceMappingURL=datasetFiles.js.map

package/dist/datasetReader.d.ts

@@ -0,0 +1,16 @@
+import { type StructureRowsOutputPagingCursor, type StructureRowsOutputSandboxRef } from "./rowsOutputPaging";
+export declare function datasetReader(env: any, opts: {
+    datasetId: string;
+    sandboxId?: string;
+    runtime?: string;
+    timeoutMs?: number;
+}): {
+    datasetId: string;
+    download(): Promise<StructureRowsOutputSandboxRef>;
+    readPage(params: {
+        sandboxId: string;
+        localPath: string;
+        cursor?: Partial<StructureRowsOutputPagingCursor>;
+        limit: number;
+    }): Promise<import("./rowsOutputPaging").StructureReadRowsOutputPageFromSandboxResult>;
+};

package/dist/datasetReader.js

@@ -0,0 +1,25 @@
+import { structureDownloadRowsOutputToSandboxStep, structureReadRowsOutputPageFromSandboxStep, } from "./rowsOutputPaging";
+export function datasetReader(env, opts) {
+    const datasetId = opts.datasetId;
+    return {
+        datasetId,
+        async download() {
+            return await structureDownloadRowsOutputToSandboxStep({
+                env,
+                structureId: datasetId,
+                sandboxId: opts.sandboxId,
+                runtime: opts.runtime,
+                timeoutMs: opts.timeoutMs,
+            });
+        },
+        async readPage(params) {
+            return await structureReadRowsOutputPageFromSandboxStep({
+                env,
+                sandboxId: params.sandboxId,
+                localPath: params.localPath,
+                cursor: params.cursor,
+                limit: params.limit,
+            });
+        },
+    };
+}

package/dist/domain.d.ts

@@ -1,2 +1 @@
 export { structureDomain } from "./schema";
-//# sourceMappingURL=domain.d.ts.map

package/dist/domain.js

@@ -1,2 +1 @@
 export { structureDomain } from "./schema";
-//# sourceMappingURL=domain.js.map

package/dist/executeCommand.tool.d.ts

@@ -32,4 +32,3 @@ export declare function createExecuteCommandTool({ datasetId, sandboxId, env }:
     error?: undefined;
 }>;
 export {};
-//# sourceMappingURL=executeCommand.tool.d.ts.map

package/dist/executeCommand.tool.js

@@ -103,4 +103,3 @@ export function createExecuteCommandTool({ datasetId, sandboxId, env }) {
         },
     });
 }
-//# sourceMappingURL=executeCommand.tool.js.map

package/dist/file/steps.d.ts

@@ -5,4 +5,3 @@ export declare function readInstantFileStep(params: {
     contentBase64: string;
     contentDisposition?: string;
 }>;
-//# sourceMappingURL=steps.d.ts.map

package/dist/file/steps.js

@@ -1,9 +1,9 @@
-import { getStoryRuntime } from "../runtime";
-import { DatasetService } from "../service";
 export async function readInstantFileStep(params) {
     "use step";
-    const runtime = (await getStoryRuntime(params.env));
+    const { resolveStoryRuntime } = await import("@ekairos/story/runtime");
+    const runtime = (await resolveStoryRuntime(params.env));
     const db = runtime.db;
+    const { DatasetService } = await import("../service");
     const service = new DatasetService(db);
     const file = await service.getFileById(params.fileId);
     const fileRow = file?.$files?.[0];
@@ -17,4 +17,3 @@ export async function readInstantFileStep(params) {
         contentDisposition: fileRow["content-disposition"],
     };
 }
-//# sourceMappingURL=steps.js.map

package/dist/generateSchema.tool.d.ts

@@ -19,4 +19,3 @@ export declare function createGenerateSchemaTool({ datasetId, sandboxId, env }:
     error?: undefined;
 }>;
 export {};
-//# sourceMappingURL=generateSchema.tool.d.ts.map

package/dist/generateSchema.tool.js

@@ -58,4 +58,3 @@ export function createGenerateSchemaTool({ datasetId, sandboxId, env }) {
         },
     });
 }
-//# sourceMappingURL=generateSchema.tool.js.map

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export * from "./structure";
 export * from "./schema";
 export * from "./service";
 export * from "./datasetFiles";
-//# sourceMappingURL=index.d.ts.map
+export * from "./rowsOutputPaging";

package/dist/index.js

@@ -2,4 +2,4 @@ export * from "./structure";
 export * from "./schema";
 export * from "./service";
 export * from "./datasetFiles";
-//# sourceMappingURL=index.js.map
+export * from "./rowsOutputPaging";

package/dist/prompts.d.ts

@@ -15,4 +15,3 @@ export type StructurePromptContext = {
     outputPath: string;
 };
 export declare function buildStructurePrompt(ctx: StructurePromptContext): string;
-//# sourceMappingURL=prompts.d.ts.map

package/dist/prompts.js

@@ -59,4 +59,3 @@ export function buildStructurePrompt(ctx) {
     lines.push("- Do not leak secrets. Do not print large raw datasets to stdout.");
     return lines.join("\n");
 }
-//# sourceMappingURL=prompts.js.map

package/dist/rowsOutputPaging.d.ts

@@ -0,0 +1,57 @@
+export type StructureRowsOutputPagingCursor = {
+    byteOffset: number;
+    rowOffset: number;
+};
+export type StructureRowsOutputSandboxRef = {
+    sandboxId: string;
+    localPath: string;
+};
+/**
+ * Step 1/2:
+ * Download the rows output.jsonl from Instant storage into a sandbox file.
+ *
+ * This isolates network flakiness (e.g. undici `TypeError: terminated`) into a single step
+ * and makes subsequent reads purely sandbox-local.
+ */
+export declare function structureDownloadRowsOutputToSandboxStep(params: {
+    env: any;
+    structureId: string;
+    sandboxId?: string;
+    runtime?: string;
+    timeoutMs?: number;
+}): Promise<StructureRowsOutputSandboxRef>;
+export type StructureReadRowsOutputPageFromSandboxResult = {
+    rows: any[];
+    nextCursor: StructureRowsOutputPagingCursor;
+    done: boolean;
+};
+/**
+ * Step 2/2:
+ * Read the next page of ROW records from the sandbox-local output.jsonl, bounded by `limit`.
+ *
+ * Pagination state is passed explicitly via `cursor` and returned as `nextCursor`.
+ */
+export declare function structureReadRowsOutputPageFromSandboxStep(params: {
+    env: any;
+    sandboxId: string;
+    localPath: string;
+    cursor?: Partial<StructureRowsOutputPagingCursor>;
+    limit: number;
+}): Promise<StructureReadRowsOutputPageFromSandboxResult>;
+/**
+ * Back-compat alias (older naming).
+ * Prefer `structureReadRowsOutputPageFromSandboxStep`.
+ */
+export declare function structureReadRowsOutputChunkStep(params: {
+    env: any;
+    sandboxId: string;
+    localPath: string;
+    byteOffset?: number;
+    rowOffset?: number;
+    limit: number;
+}): Promise<{
+    rows: any[];
+    nextByteOffset: number;
+    nextRowOffset: number;
+    done: boolean;
+}>;

package/dist/rowsOutputPaging.js

@@ -0,0 +1,148 @@
+import { getDatasetOutputPath, getDatasetWorkstation } from "./datasetFiles";
+import { createDatasetSandboxStep, runDatasetSandboxCommandStep } from "./sandbox/steps";
+import { getStoryRuntime } from "./runtime";
+/**
+ * Step 1/2:
+ * Download the rows output.jsonl from Instant storage into a sandbox file.
+ *
+ * This isolates network flakiness (e.g. undici `TypeError: terminated`) into a single step
+ * and makes subsequent reads purely sandbox-local.
+ */
+export async function structureDownloadRowsOutputToSandboxStep(params) {
+    "use step";
+    const runtime = params.runtime ?? "python3.13";
+    const timeoutMs = params.timeoutMs ?? 10 * 60 * 1000;
+    const sandboxId = params.sandboxId ??
+        (await createDatasetSandboxStep({
+            env: params.env,
+            runtime,
+            timeoutMs,
+            purpose: "structure.rows-output.reader",
+            params: { structureId: params.structureId },
+        })).sandboxId;
+    const workstation = getDatasetWorkstation(params.structureId);
+    const localPath = getDatasetOutputPath(params.structureId);
+    await runDatasetSandboxCommandStep({
+        env: params.env,
+        sandboxId,
+        cmd: "mkdir",
+        args: ["-p", workstation],
+    });
+    const storyRuntime = await getStoryRuntime(params.env);
+    const db = storyRuntime.db;
+    const contextKey = `structure:${params.structureId}`;
+    const query = (await db.query({
+        context_contexts: {
+            $: { where: { key: contextKey }, limit: 1 },
+            structure_output_file: {},
+        },
+    }));
+    const ctx = query.context_contexts?.[0];
+    const linked = Array.isArray(ctx?.structure_output_file) ? ctx.structure_output_file[0] : ctx.structure_output_file;
+    const url = linked?.url;
+    if (!url) {
+        throw new Error("Rows output file not found");
+    }
+    // Download inside the sandbox runtime (python) to avoid streaming aborts in the Node step runtime.
+    const py = [
+        "import sys, urllib.request",
+        "url = sys.argv[1]",
+        "out_path = sys.argv[2]",
+        "with urllib.request.urlopen(url) as r:",
+        "  data = r.read()",
+        "with open(out_path, 'wb') as f:",
+        "  f.write(data)",
+        "print('ok', len(data))",
+    ].join("\n");
+    const res = await runDatasetSandboxCommandStep({
+        env: params.env,
+        sandboxId,
+        cmd: "python",
+        args: ["-c", py, String(url), localPath],
+    });
+    if (res.exitCode !== 0) {
+        throw new Error(res.stderr || "Failed to download rows output to sandbox");
+    }
+    return { sandboxId, localPath };
+}
+/**
+ * Step 2/2:
+ * Read the next page of ROW records from the sandbox-local output.jsonl, bounded by `limit`.
+ *
+ * Pagination state is passed explicitly via `cursor` and returned as `nextCursor`.
+ */
+export async function structureReadRowsOutputPageFromSandboxStep(params) {
+    "use step";
+    const byteOffset = params.cursor?.byteOffset ?? 0;
+    const rowOffset = params.cursor?.rowOffset ?? 0;
+    const py = [
+        "import sys, json",
+        "path = sys.argv[1]",
+        "byte_offset = int(sys.argv[2])",
+        "row_offset = int(sys.argv[3])",
+        "limit = int(sys.argv[4])",
+        "rows = []",
+        "next_byte = byte_offset",
+        "next_row = row_offset",
+        "with open(path, 'rb') as f:",
+        "  f.seek(byte_offset)",
+        "  while len(rows) < limit:",
+        "    line = f.readline()",
+        "    if not line:",
+        "      break",
+        "    next_byte = f.tell()",
+        "    try:",
+        "      obj = json.loads(line.decode('utf-8'))",
+        "    except Exception:",
+        "      continue",
+        "    if obj.get('type') != 'row':",
+        "      continue",
+        "    rows.append(obj.get('data'))",
+        "    next_row += 1",
+        "done = len(rows) < limit",
+        "print(json.dumps({",
+        "  'rows': rows,",
+        "  'nextByteOffset': next_byte,",
+        "  'nextRowOffset': next_row,",
+        "  'done': done,",
+        "}))",
+    ].join("\n");
+    const res = await runDatasetSandboxCommandStep({
+        env: params.env,
+        sandboxId: params.sandboxId,
+        cmd: "python",
+        args: ["-c", py, params.localPath, String(byteOffset), String(rowOffset), String(params.limit)],
+    });
+    if (res.exitCode !== 0) {
+        throw new Error(res.stderr || "Failed to read rows page from sandbox");
+    }
+    const out = String(res.stdout ?? "").trim();
+    const parsed = JSON.parse(out);
+    return {
+        rows: parsed.rows ?? [],
+        nextCursor: {
+            byteOffset: parsed.nextByteOffset ?? byteOffset,
+            rowOffset: parsed.nextRowOffset ?? rowOffset,
+        },
+        done: Boolean(parsed.done),
+    };
+}
+/**
+ * Back-compat alias (older naming).
+ * Prefer `structureReadRowsOutputPageFromSandboxStep`.
+ */
+export async function structureReadRowsOutputChunkStep(params) {
+    const res = await structureReadRowsOutputPageFromSandboxStep({
+        env: params.env,
+        sandboxId: params.sandboxId,
+        localPath: params.localPath,
+        cursor: { byteOffset: params.byteOffset ?? 0, rowOffset: params.rowOffset ?? 0 },
+        limit: params.limit,
+    });
+    return {
+        rows: res.rows,
+        nextByteOffset: res.nextCursor.byteOffset,
+        nextRowOffset: res.nextCursor.rowOffset,
+        done: res.done,
+    };
+}

package/dist/rowsOutputPaging.steps.d.ts

@@ -0,0 +1,37 @@
+export type RowsOutputSandboxDownloadResult = {
+    sandboxId: string;
+    localPath: string;
+};
+export type RowsOutputChunkResult<T = any> = {
+    rows: T[];
+    nextByteOffset: number;
+    nextRowOffset: number;
+    done: boolean;
+};
+/**
+ * Step 1/2:
+ * Download the rows output.jsonl from Instant storage into a sandbox file.
+ *
+ * This isolates network flakiness (e.g. undici `TypeError: terminated`) into a single step
+ * and makes subsequent reads purely sandbox-local.
+ */
+export declare function structureDownloadRowsOutputToSandboxStep(params: {
+    env: any;
+    structureId: string;
+    runtime?: string;
+    timeoutMs?: number;
+}): Promise<RowsOutputSandboxDownloadResult>;
+/**
+ * Step 2/2:
+ * Read the next chunk of ROW records from the sandbox-local output.jsonl, bounded by `limit`.
+ *
+ * Pagination state is passed explicitly via `{ byteOffset, rowOffset }` and returned as next offsets.
+ */
+export declare function structureReadRowsOutputChunkStep<T = any>(params: {
+    env: any;
+    sandboxId: string;
+    localPath: string;
+    byteOffset: number;
+    rowOffset: number;
+    limit: number;
+}): Promise<RowsOutputChunkResult<T>>;