okengine 0.11.2 → 0.13.0

package/src/runs/collect.ts

@@ -30,10 +30,17 @@ export interface CollectWideEventInput {
   readonly startedAt: number;
   /** Epoch-ms when the run ended. */
   readonly endedAt: number;
+  /**
+   * High-res handler duration. When omitted, falls back to
+   * `endedAt - startedAt` (integer ms).
+   */
+  readonly durationMs?: number;
   /** Pipeline failure when present. */
   readonly failure?: FlowFailure | null;
   /** Validated flow input (for local replay). */
   readonly input?: unknown;
+  /** Flow return value when the run completed with an output. */
+  readonly output?: unknown;
   /** Optional parent run id (causal chain). */
   readonly parentId?: string;
   /** Optional run id (generated when omitted). */
@@ -64,7 +71,10 @@ export function collectWideEvent(input: CollectWideEventInput): WideEvent {
     input.telemetry.gates.length > 0 ? [...input.telemetry.gates] : httpGates(input.trigger);
 
   const error = failureToRunError(input.failure);
-  const durationMs = Math.max(0, input.endedAt - input.startedAt);
+  const durationMs =
+    input.durationMs !== undefined && Number.isFinite(input.durationMs)
+      ? Math.max(0, input.durationMs)
+      : Math.max(0, input.endedAt - input.startedAt);
 
   const dimensions: WideEvent["dimensions"] = {
     flow: input.flow.name,
@@ -77,6 +87,8 @@ export function collectWideEvent(input: CollectWideEventInput): WideEvent {
     replica: input.telemetry.replica ?? null,
     replica_lag_ms: input.telemetry.replicaLagMs ?? null,
     cost: input.telemetry.cost || null,
+    input_tokens: input.telemetry.inputTokens || null,
+    output_tokens: input.telemetry.outputTokens || null,
     prompt_version: input.telemetry.promptVersion ?? null,
     build_version: input.buildVersion ?? null,
     error_code: error?.code ?? null,
@@ -101,12 +113,15 @@ export function collectWideEvent(input: CollectWideEventInput): WideEvent {
       ? { replicaLagMs: input.telemetry.replicaLagMs }
       : {}),
     ...(input.telemetry.cost > 0 ? { cost: input.telemetry.cost } : {}),
+    ...(input.telemetry.inputTokens > 0 ? { inputTokens: input.telemetry.inputTokens } : {}),
+    ...(input.telemetry.outputTokens > 0 ? { outputTokens: input.telemetry.outputTokens } : {}),
     ...(input.telemetry.promptVersion !== undefined
       ? { promptVersion: input.telemetry.promptVersion }
       : {}),
     ...(input.buildVersion !== undefined ? { buildVersion: input.buildVersion } : {}),
     error,
     ...(input.input !== undefined ? { input: input.input } : {}),
+    ...(input.output !== undefined ? { output: input.output } : {}),
     effects: [...input.ledger.entries],
     logs: [...input.telemetry.logs],
     durationMs,
@@ -125,6 +140,18 @@ function httpGates(trigger: Trigger): string[] {
 function failureToRunError(failure: FlowFailure | null | undefined): RunError | null {
   if (!failure) return null;
   const code = failure.error?.code ?? "failure";
-  const message = failure.error?.message;
+  const message = failure.error?.message ?? messageFromFailureData(failure.error?.data);
   return message !== undefined ? { code, message } : { code };
 }
+
+/**
+ * App flows often put the human line on `fail(code, { message })` data
+ * instead of `fail(code, data, { message })`. Lift it so Traces can show it.
+ *
+ * @param data - Failure payload
+ */
+function messageFromFailureData(data: unknown): string | undefined {
+  if (data === null || data === undefined || typeof data !== "object") return undefined;
+  const message = (data as { message?: unknown }).message;
+  return typeof message === "string" && message.trim().length > 0 ? message : undefined;
+}

package/src/runs/drivers/clickhouse.ts

@@ -84,7 +84,10 @@ export const clickhouseRunsDriver: RunsDriver = {
       async flush(): Promise<void> {
         /* inserts are immediate in the fake */
       },
-      async query(sql: string): Promise<RunsRow[]> {
+      async query(
+        sql: string,
+        _options?: import("../types.ts").RunsQueryOptions,
+      ): Promise<RunsRow[]> {
         return client.query(sql);
       },
       async all(): Promise<WideEvent[]> {

package/src/runs/drivers/files.ts

@@ -6,21 +6,37 @@
  * The user never declares archive tiers.
  */
 
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 
 import { fsDriver } from "../../drivers/fs.ts";
 import type { FilesBucket } from "../../drivers/types.ts";
-import { duckPath, duckQuery, openDuckDB, type DuckSession } from "../duckdb.ts";
 import {
+  duckPath,
+  duckQuery,
+  duckQueryWithTimeout,
+  openDuckDB,
+  type DuckSession,
+} from "../duckdb.ts";
+import {
+  parquetUnionSql,
   partitionKey,
   partitionObjectKey,
   rowToWideEvent,
   wideEventToRow,
   writeParquet,
 } from "../parquet.ts";
-import type { RunsDriver, RunsOpenOptions, RunsRow, RunsStore, WideEvent } from "../types.ts";
+import { retentionKeepMs, shouldDropPartition } from "../retention.ts";
+import {
+  DEFAULT_RUNS_LOCAL_ROOT,
+  type RunsDriver,
+  type RunsOpenOptions,
+  type RunsQueryOptions,
+  type RunsRow,
+  type RunsStore,
+  type WideEvent,
+} from "../types.ts";
 
 /** Default hot window — 7 days. Older writes go to object storage when present. */
 const DEFAULT_HOT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
@@ -32,10 +48,12 @@ export const filesRunsDriver: RunsDriver = {
   id: "files",
   async open(options: RunsOpenOptions = {}): Promise<RunsStore> {
     const hotWindowMs = options.hotWindowMs ?? DEFAULT_HOT_WINDOW_MS;
-    const clock = () => Date.now();
+    const clock = options.now ?? (() => Date.now());
 
-    const localRoot = options.localRoot ?? (await mkdtemp(join(tmpdir(), "oke-runs-")));
-    const ownsLocalRoot = options.localRoot === undefined && !options.localBucket;
+    const localRoot = options.localRoot ?? resolve(process.cwd(), DEFAULT_RUNS_LOCAL_ROOT);
+    if (!options.localBucket) {
+      await mkdir(localRoot, { recursive: true });
+    }
 
     const local: FilesBucket =
       options.localBucket ?? (await fsDriver.open({ name: "runs-local", root: localRoot }));
@@ -44,8 +62,9 @@ export const filesRunsDriver: RunsDriver = {
     const remotePrefix = options.remote?.prefix ?? "";
 
     const buffer: WideEvent[] = [];
-    const catalog: WideEvent[] = [];
     let queryScratch: string | undefined;
+    let partitionFingerprint: string | undefined;
+    let viewReady = false;
     const session: DuckSession = await openDuckDB();
 
     function chooseBucket(startedAt: number): {
@@ -94,42 +113,124 @@ export const filesRunsDriver: RunsDriver = {
           );
           const bytes = new Uint8Array(await Bun.file(tmp).arrayBuffer());
           await target.bucket.put(objectKey, bytes);
-          for (const e of events) catalog.push(e);
         } finally {
           await rm(tmpDir, { recursive: true, force: true });
         }
       }
     }
 
-    async function materialiseAllParquet(): Promise<string[]> {
+    async function listPartitionKeys(): Promise<{
+      readonly localKeys: readonly string[];
+      readonly remoteKeys: readonly string[];
+      readonly fingerprint: string;
+    }> {
+      const localKeys = (await local.list("runs/")).filter((k) => k.endsWith(".parquet")).sort();
+      const remoteKeys = remote
+        ? (await remote.list(`${remotePrefix}runs/`)).filter((k) => k.endsWith(".parquet")).sort()
+        : [];
+      return {
+        localKeys,
+        remoteKeys,
+        fingerprint: `local:${localKeys.join("|")}||remote:${remoteKeys.join("|")}`,
+      };
+    }
+
+    async function materialiseAllParquet(
+      listed: Awaited<ReturnType<typeof listPartitionKeys>>,
+    ): Promise<string[]> {
       if (queryScratch) {
         await rm(queryScratch, { recursive: true, force: true }).catch(() => undefined);
       }
       queryScratch = await mkdtemp(join(tmpdir(), "oke-runs-q-"));
       const paths: string[] = [];
+      const sizes: string[] = [];
 
-      const localKeys = (await local.list("runs/")).filter((k) => k.endsWith(".parquet"));
-      for (const key of localKeys) {
+      for (const key of listed.localKeys) {
         const data = await local.get(key);
         if (!data) continue;
         const dest = join(queryScratch, key.replaceAll("/", "__"));
         await writeFile(dest, data);
         paths.push(dest);
+        sizes.push(`${key}:${data.byteLength}`);
       }
       if (remote) {
-        const prefix = `${remotePrefix}runs/`;
-        const remoteKeys = (await remote.list(prefix)).filter((k) => k.endsWith(".parquet"));
-        for (const key of remoteKeys) {
+        for (const key of listed.remoteKeys) {
           const data = await remote.get(key);
           if (!data) continue;
           const dest = join(queryScratch, `remote__${key.replaceAll("/", "__")}`);
           await writeFile(dest, data);
           paths.push(dest);
+          sizes.push(`remote:${key}:${data.byteLength}`);
+        }
+      }
+      partitionFingerprint = `${listed.fingerprint}||sizes:${sizes.join(",")}`;
+      return paths;
+    }
+
+    async function ensureParquetView(): Promise<string[]> {
+      await flushBuffer();
+      await applyRetention();
+      const listed = await listPartitionKeys();
+      const cached =
+        viewReady &&
+        queryScratch !== undefined &&
+        partitionFingerprint !== undefined &&
+        partitionFingerprint.startsWith(`${listed.fingerprint}||sizes:`);
+      const scratch = queryScratch;
+      if (cached && scratch !== undefined) {
+        const glob = new Bun.Glob("*.parquet");
+        const paths: string[] = [];
+        for await (const match of glob.scan({ cwd: scratch, onlyFiles: true })) {
+          paths.push(join(scratch, match));
         }
+        if (paths.length > 0) return paths.sort();
+        viewReady = false;
+      }
+      const paths = await materialiseAllParquet(listed);
+      if (paths.length === 0) {
+        viewReady = false;
+        return [];
       }
+      const scan = await parquetUnionSql(session.conn, paths);
+      await session.conn.run(`CREATE OR REPLACE VIEW runs AS ${scan}`);
+      viewReady = true;
       return paths;
     }
 
+    async function runUserSql(
+      conn: DuckSession["conn"],
+      sql: string,
+      options?: RunsQueryOptions,
+    ): Promise<RunsRow[]> {
+      const timeoutMs = options?.timeoutMs;
+      const rows =
+        timeoutMs !== undefined
+          ? await duckQueryWithTimeout(conn, sql, timeoutMs)
+          : await duckQuery(conn, sql);
+      if (options?.maxRows !== undefined && rows.length > options.maxRows) {
+        return rows.slice(0, options.maxRows);
+      }
+      return rows;
+    }
+
+    async function applyRetention(): Promise<void> {
+      const keepMs = retentionKeepMs(options.retention?.keep);
+      if (keepMs == null) return;
+      const now = clock();
+      const dropFrom = async (bucket: FilesBucket, prefix: string): Promise<void> => {
+        const keys = (await bucket.list(`${prefix}runs/`)).filter((k) => k.endsWith(".parquet"));
+        for (const key of keys) {
+          if (shouldDropPartition(key, now, keepMs)) {
+            await bucket.delete(key);
+          }
+        }
+      };
+      await dropFrom(local, "");
+      if (remote) await dropFrom(remote, remotePrefix);
+    }
+
+    await applyRetention();
+
     return {
       driverId: "files",
       async append(event: WideEvent): Promise<void> {
@@ -138,28 +239,35 @@ export const filesRunsDriver: RunsDriver = {
       },
       async flush(): Promise<void> {
         await flushBuffer();
+        await applyRetention();
       },
-      async query(sql: string): Promise<RunsRow[]> {
-        await flushBuffer();
-        const paths = await materialiseAllParquet();
+      async query(sql: string, options?: RunsQueryOptions): Promise<RunsRow[]> {
+        const paths = await ensureParquetView();
         if (paths.length === 0) return [];
-        const list = paths.map((p) => `'${duckPath(p)}'`).join(", ");
-        await session.conn.run(`DROP VIEW IF EXISTS runs`);
+        if (options?.sandbox !== true) {
+          return runUserSql(session.conn, sql, options);
+        }
+        const scratch = queryScratch;
+        if (scratch === undefined) return [];
+        const snap = join(scratch, "_console_snap.parquet");
         await session.conn.run(
-          `CREATE VIEW runs AS SELECT * FROM read_parquet([${list}], union_by_name = true)`,
+          `COPY (SELECT * FROM runs) TO '${duckPath(snap)}' (FORMAT PARQUET, COMPRESSION ZSTD)`,
         );
-        return duckQuery(session.conn, sql);
+        const isolated = await openDuckDB();
+        try {
+          await isolated.conn.run(
+            `CREATE TABLE runs AS SELECT * FROM read_parquet('${duckPath(snap)}')`,
+          );
+          await isolated.conn.run("SET enable_external_access = false");
+          return await runUserSql(isolated.conn, sql, options);
+        } finally {
+          isolated.close();
+        }
       },
       async all(): Promise<WideEvent[]> {
-        await flushBuffer();
-        if (catalog.length > 0) return [...catalog];
-        const paths = await materialiseAllParquet();
+        const paths = await ensureParquetView();
         if (paths.length === 0) return [];
-        const list = paths.map((p) => `'${duckPath(p)}'`).join(", ");
-        const rows = await duckQuery(
-          session.conn,
-          `SELECT * FROM read_parquet([${list}], union_by_name = true)`,
-        );
+        const rows = await duckQuery(session.conn, "SELECT * FROM runs");
         return rows.map((r) => rowToWideEvent(r));
       },
       async close(): Promise<void> {
@@ -170,9 +278,6 @@ export const filesRunsDriver: RunsDriver = {
         }
         await local.close();
         await remote?.close();
-        if (ownsLocalRoot) {
-          await rm(localRoot, { recursive: true, force: true }).catch(() => undefined);
-        }
       },
     };
   },

package/src/runs/drivers/memory.ts

@@ -6,9 +6,22 @@
 
 import type { DuckDBConnection } from "@duckdb/node-api";
 
-import { duckLiteral, duckQuery, openDuckDB, type DuckSession } from "../duckdb.ts";
+import {
+  duckLiteral,
+  duckQuery,
+  duckQueryWithTimeout,
+  openDuckDB,
+  type DuckSession,
+} from "../duckdb.ts";
 import { wideEventToRow, type ParquetRow } from "../parquet.ts";
-import type { RunsDriver, RunsOpenOptions, RunsRow, RunsStore, WideEvent } from "../types.ts";
+import type {
+  RunsDriver,
+  RunsOpenOptions,
+  RunsQueryOptions,
+  RunsRow,
+  RunsStore,
+  WideEvent,
+} from "../types.ts";
 
 /**
  * Memory runs driver (DuckDB `:memory:`).
@@ -16,8 +29,13 @@ import type { RunsDriver, RunsOpenOptions, RunsRow, RunsStore, WideEvent } from
 export const memoryRunsDriver: RunsDriver = {
   id: "memory",
   async open(_options: RunsOpenOptions = {}): Promise<RunsStore> {
-    const session: DuckSession = await openDuckDB();
     const events: WideEvent[] = [];
+    let session: DuckSession | undefined;
+
+    async function ensureConn(): Promise<DuckSession["conn"]> {
+      session ??= await openDuckDB();
+      return session.conn;
+    }
 
     return {
       driverId: "memory",
@@ -27,16 +45,37 @@ export const memoryRunsDriver: RunsDriver = {
       async flush(): Promise<void> {
         /* nothing buffered */
       },
-      async query(sql: string): Promise<RunsRow[]> {
-        await materialiseRunsTable(session.conn, events);
-        return duckQuery(session.conn, sql);
+      async query(sql: string, options?: RunsQueryOptions): Promise<RunsRow[]> {
+        const conn = await ensureConn();
+        await materialiseRunsTable(conn, events);
+        if (options?.sandbox === true) {
+          await conn.run("SET enable_external_access = false");
+        }
+        try {
+          const rows =
+            options?.timeoutMs !== undefined
+              ? await duckQueryWithTimeout(conn, sql, options.timeoutMs)
+              : await duckQuery(conn, sql);
+          if (options?.maxRows !== undefined && rows.length > options.maxRows) {
+            return rows.slice(0, options.maxRows);
+          }
+          return rows;
+        } finally {
+          if (options?.sandbox === true) {
+            try {
+              await conn.run("SET enable_external_access = true");
+            } catch {
+              /* ignore */
+            }
+          }
+        }
       },
       async all(): Promise<WideEvent[]> {
         return [...events];
       },
       async close(): Promise<void> {
         events.length = 0;
-        session.close();
+        session?.close();
       },
     };
   },

package/src/runs/drivers/postgres.ts

@@ -80,7 +80,10 @@ export const postgresRunsDriver: RunsDriver = {
       async flush(): Promise<void> {
         /* row-level writes are immediate */
       },
-      async query(sql: string): Promise<RunsRow[]> {
+      async query(
+        sql: string,
+        _options?: import("../types.ts").RunsQueryOptions,
+      ): Promise<RunsRow[]> {
         return client.query(sql);
       },
       async all(): Promise<WideEvent[]> {

package/src/runs/duckdb.test.ts

@@ -0,0 +1,33 @@
+/**
+ * DuckDB peer is a real runtime dependency — open must succeed here
+ * (and in scaffolded apps after `bun add @duckdb/node-api`).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { duckQuery, openDuckDB } from "./duckdb.ts";
+
+describe("openDuckDB", () => {
+  test("opens an in-memory session and runs SQL", async () => {
+    const session = await openDuckDB();
+    try {
+      const rows = await duckQuery(session.conn, "SELECT 1 AS n");
+      expect(Number(rows[0]?.n)).toBe(1);
+    } finally {
+      session.close();
+    }
+  });
+
+  test("both create-oke templates declare @duckdb/node-api", () => {
+    const root = join(import.meta.dir, "../../packages/create-oke/templates");
+    for (const tier of ["standard", "advanced"] as const) {
+      const pkg = JSON.parse(readFileSync(join(root, tier, "package.json"), "utf8")) as {
+        dependencies?: { "@duckdb/node-api"?: string };
+        trustedDependencies?: readonly string[];
+      };
+      expect(pkg.dependencies?.["@duckdb/node-api"]).toMatch(/^\^1\.5\./);
+      expect(pkg.trustedDependencies).toContain("@duckdb/node-api");
+    }
+  });
+});

package/src/runs/duckdb.ts

@@ -55,6 +55,47 @@ export async function duckQuery(conn: DuckDBConnection, sql: string): Promise<Ru
   return rows.map(normalizeRow);
 }
 
+/** Thrown when {@link duckQueryWithTimeout} exceeds `timeoutMs`. */
+export class DuckQueryTimeoutError extends Error {
+  readonly timeoutMs: number;
+
+  constructor(timeoutMs: number) {
+    super(`runs query timed out after ${timeoutMs}ms`);
+    this.name = "DuckQueryTimeoutError";
+    this.timeoutMs = timeoutMs;
+  }
+}
+
+/**
+ * Run {@link duckQuery} with a wall-clock timeout and optional interrupt.
+ *
+ * @param conn - DuckDB connection
+ * @param sql - SQL text
+ * @param timeoutMs - Deadline
+ */
+export async function duckQueryWithTimeout(
+  conn: DuckDBConnection,
+  sql: string,
+  timeoutMs: number,
+): Promise<RunsRow[]> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      try {
+        (conn as { interrupt?: () => void }).interrupt?.();
+      } catch {
+        /* ignore */
+      }
+      reject(new DuckQueryTimeoutError(timeoutMs));
+    }, timeoutMs);
+  });
+  try {
+    return await Promise.race([duckQuery(conn, sql), timeout]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}
+
 /**
  * Escape a filesystem path for use inside a DuckDB string literal.
  *

package/src/runs/history.ts

@@ -0,0 +1,61 @@
+/**
+ * Read persisted WideEvents from a `files` localRoot (Console historical list).
+ */
+
+import { access } from "node:fs/promises";
+import { createRunsRuntime } from "./runtime.ts";
+import type { WideEvent } from "./types.ts";
+
+/**
+ * Load WideEvents from a Parquet `files` root. Empty / missing dir → `[]`.
+ *
+ * @param localRoot - Absolute `.oke/runs` path
+ */
+export async function readPersistedRuns(localRoot: string): Promise<WideEvent[]> {
+  try {
+    await access(localRoot);
+  } catch {
+    return [];
+  }
+  const runs = createRunsRuntime({
+    driver: "files",
+    localRoot,
+    retention: { keep: "forever" },
+  });
+  await runs.open();
+  try {
+    return await runs.all();
+  } finally {
+    await runs.close();
+  }
+}
+
+/**
+ * Union two wide-event sets. Live wins on the same id.
+ *
+ * @param live - In-process / ingest copy
+ * @param persisted - Disk snapshot
+ */
+export function mergeLiveAndPersistedEvents(
+  live: readonly WideEvent[],
+  persisted: readonly WideEvent[],
+): WideEvent[] {
+  const byId = new Map<string, WideEvent>();
+  for (const event of persisted) byId.set(event.id, event);
+  for (const event of live) byId.set(event.id, event);
+  return [...byId.values()].sort((a, b) => b.startedAt - a.startedAt);
+}
+
+/**
+ * Union live (in-process) events with disk. Live wins on the same id.
+ *
+ * @param live - Console memory / ingest copy
+ * @param localRoot - Host `files` root
+ */
+export async function mergeLiveAndPersistedRuns(
+  live: readonly WideEvent[],
+  localRoot: string,
+): Promise<WideEvent[]> {
+  const persisted = await readPersistedRuns(localRoot);
+  return mergeLiveAndPersistedEvents(live, persisted);
+}

package/src/runs/index.ts

@@ -17,6 +17,7 @@ export type {
   RunsStore,
   RunsDriver,
   RunsOpenOptions,
+  RunsQueryOptions,
 } from "./types.ts";
 
 export { createRunTelemetry, cacheDimensionOf, type RunTelemetry } from "./telemetry.ts";
@@ -53,7 +54,24 @@ export {
   type ParquetRow,
 } from "./parquet.ts";
 
-export { openDuckDB, duckQuery, duckPath, duckLiteral } from "./duckdb.ts";
+export {
+  openDuckDB,
+  duckQuery,
+  duckQueryWithTimeout,
+  DuckQueryTimeoutError,
+  duckPath,
+  duckLiteral,
+} from "./duckdb.ts";
+
+export { DEFAULT_RUNS_LOCAL_ROOT } from "./types.ts";
+
+export { retentionKeepMs, partitionDayFromKey, shouldDropPartition } from "./retention.ts";
+
+export {
+  readPersistedRuns,
+  mergeLiveAndPersistedEvents,
+  mergeLiveAndPersistedRuns,
+} from "./history.ts";
 
 export {
   createRunsRuntime,
@@ -66,6 +84,16 @@ export {
   type RunsRuntime,
 } from "./runtime.ts";
 
+export {
+  RUNS_INGEST_SECRET_HEADER,
+  RUNS_INGEST_URL_ENV,
+  RUNS_INGEST_SECRET_ENV,
+  resolveRunsConsoleBridge,
+  wrapRunsForConsoleIngest,
+  type RunsConsoleBridgeFetch,
+  type RunsConsoleBridgeTarget,
+} from "./bridge-to-console.ts";
+
 export { privacyErase, type PrivacyEraseOptions, type PrivacyEraseResult } from "./privacy.ts";
 
 export {