okengine 0.1.4

package/src/runs/collect.ts

@@ -0,0 +1,144 @@
+/**
+ * Build a {@link WideEvent} from an execution context — zero instrumentation.
+ *
+ * Dimensions come from the flow contract, trigger, principals, effect ledger,
+ * and {@link RunTelemetry} accumulated through `fx`.
+ */
+
+import type { EffectLedger } from "../kernel/effects.ts";
+import type { FlowFailure } from "../kernel/errors.ts";
+import type { AnyFlowDef } from "../kernel/flow.ts";
+import type { Fx } from "../kernel/fx.ts";
+import type { Trigger } from "../kernel/triggers.ts";
+import type { FlowPlane } from "../manifest/types.ts";
+import {
+  cacheDimensionOf,
+  type RunTelemetry,
+} from "../kernel/run-telemetry.ts";
+import type { RunError, WideEvent } from "./types.ts";
+
+/** Inputs for {@link collectWideEvent}. */
+export interface CollectWideEventInput {
+  /** Flow definition. */
+  readonly flow: AnyFlowDef;
+  /** Trigger that started the run. */
+  readonly trigger: Trigger;
+  /** Fx context (principals). */
+  readonly fx: Fx;
+  /** Effect ledger for this invocation. */
+  readonly ledger: EffectLedger;
+  /** Telemetry collector (logs, cache, gates, cost). */
+  readonly telemetry: RunTelemetry;
+  /** Epoch-ms when the run started. */
+  readonly startedAt: number;
+  /** Epoch-ms when the run ended. */
+  readonly endedAt: number;
+  /** Pipeline failure when present. */
+  readonly failure?: FlowFailure | null;
+  /** Optional parent run id (causal chain). */
+  readonly parentId?: string;
+  /** Optional run id (generated when omitted). */
+  readonly id?: string;
+  /** Build version stamp. */
+  readonly buildVersion?: string;
+  /**
+   * Personal field cleartext to archive under the subject key.
+   * Keys become ciphertext entries on {@link WideEvent.archived}.
+   */
+  readonly archiveFields?: Readonly<Record<string, string>>;
+  /** Pre-encrypted archived map (when shred already ran). */
+  readonly archived?: Readonly<Record<string, string>>;
+}
+
+/**
+ * Collect every declared / observed dimension into one wide event.
+ *
+ * @param input - Execution context
+ */
+export function collectWideEvent(input: CollectWideEventInput): WideEvent {
+  const plane: FlowPlane = input.flow.plane ?? "user";
+  const principal =
+    plane === "operator"
+      ? input.fx.operator.id
+      : input.fx.auth.userId;
+  const tenant = input.fx.tenant.id;
+  const subjectId =
+    input.telemetry.subjectId ?? principal ?? tenant ?? null;
+  const cache = cacheDimensionOf(input.telemetry);
+  const gates =
+    input.telemetry.gates.length > 0
+      ? [...input.telemetry.gates]
+      : httpGates(input.trigger);
+
+  const error = failureToRunError(input.failure);
+  const durationMs = Math.max(0, input.endedAt - input.startedAt);
+
+  const dimensions: WideEvent["dimensions"] = {
+    flow: input.flow.name,
+    unit: input.flow.unit ?? null,
+    trigger: input.trigger.kind,
+    plane,
+    tenant: tenant ?? null,
+    principal: principal ?? null,
+    cache,
+    replica: input.telemetry.replica ?? null,
+    replica_lag_ms: input.telemetry.replicaLagMs ?? null,
+    cost: input.telemetry.cost || null,
+    prompt_version: input.telemetry.promptVersion ?? null,
+    build_version: input.buildVersion ?? null,
+    error_code: error?.code ?? null,
+    duration_ms: durationMs,
+    ...input.telemetry.dimensions,
+  };
+
+  return {
+    id: input.id ?? crypto.randomUUID(),
+    ...(input.parentId !== undefined ? { parentId: input.parentId } : {}),
+    flow: input.flow.name,
+    ...(input.flow.unit !== undefined ? { unit: input.flow.unit } : {}),
+    trigger: input.trigger.kind,
+    plane,
+    tenant,
+    principal,
+    subjectId,
+    gates,
+    cache,
+    ...(input.telemetry.replica !== undefined
+      ? { replica: input.telemetry.replica }
+      : {}),
+    ...(input.telemetry.replicaLagMs !== undefined
+      ? { replicaLagMs: input.telemetry.replicaLagMs }
+      : {}),
+    ...(input.telemetry.cost > 0 ? { cost: input.telemetry.cost } : {}),
+    ...(input.telemetry.promptVersion !== undefined
+      ? { promptVersion: input.telemetry.promptVersion }
+      : {}),
+    ...(input.buildVersion !== undefined
+      ? { buildVersion: input.buildVersion }
+      : {}),
+    error,
+    effects: [...input.ledger.entries],
+    logs: [...input.telemetry.logs],
+    durationMs,
+    startedAt: input.startedAt,
+    endedAt: input.endedAt,
+    ...(input.archived !== undefined ? { archived: input.archived } : {}),
+    dimensions,
+  };
+}
+
+function httpGates(trigger: Trigger): string[] {
+  if (trigger.kind !== "http") return [];
+  return trigger.gates.map((g) =>
+    typeof g === "string" ? g : g.name,
+  );
+}
+
+function failureToRunError(
+  failure: FlowFailure | null | undefined,
+): RunError | null {
+  if (!failure) return null;
+  const code = failure.error?.code ?? "failure";
+  const message = failure.error?.message;
+  return message !== undefined ? { code, message } : { code };
+}

package/src/runs/drivers/clickhouse.ts

@@ -0,0 +1,108 @@
+/**
+ * Optional ClickHouse runs driver — sub-second over billions of rows at scale.
+ *
+ * Protocol-named (`clickhouse`). Fake HTTP client for tests; production uses
+ * the ClickHouse HTTP interface.
+ */
+
+import type {
+  RunsDriver,
+  RunsOpenOptions,
+  RunsRow,
+  RunsStore,
+  WideEvent,
+} from "../types.ts";
+
+/** Minimal ClickHouse HTTP-like client. */
+export interface RunsClickHouseClient {
+  /**
+   * Insert rows into a table.
+   *
+   * @param table - Table name
+   * @param rows - JSONEachRow rows
+   */
+  insert(table: string, rows: readonly RunsRow[]): Promise<void>;
+  /**
+   * Run a query returning JSON rows.
+   *
+   * @param sql - SQL text
+   */
+  query(sql: string): Promise<RunsRow[]>;
+}
+
+/**
+ * In-memory fake ClickHouse client for tests.
+ */
+export function createRunsClickHouseFake(): RunsClickHouseClient & {
+  readonly events: WideEvent[];
+} {
+  const events: WideEvent[] = [];
+  return {
+    events,
+    async insert(_table: string, rows: readonly RunsRow[]): Promise<void> {
+      for (const row of rows) {
+        if (row.payload && typeof row.payload === "object") {
+          events.push(row.payload as WideEvent);
+        } else if (typeof row.payload === "string") {
+          events.push(JSON.parse(row.payload) as WideEvent);
+        }
+      }
+    },
+    async query(sql: string): Promise<RunsRow[]> {
+      const lower = sql.toLowerCase();
+      if (lower.includes("count()")) {
+        return [{ count: events.length }];
+      }
+      return events.map((e) => ({
+        id: e.id,
+        flow: e.flow,
+        duration_ms: e.durationMs,
+        cache: e.cache,
+        tenant: e.tenant,
+        subject_id: e.subjectId,
+      }));
+    },
+  };
+}
+
+/**
+ * ClickHouse runs driver.
+ */
+export const clickhouseRunsDriver: RunsDriver = {
+  id: "clickhouse",
+  async open(options: RunsOpenOptions = {}): Promise<RunsStore> {
+    const client =
+      (options.client as RunsClickHouseClient | undefined) ??
+      createRunsClickHouseFake();
+    const table = options.name ?? "oke_runs";
+
+    return {
+      driverId: "clickhouse",
+      async append(event: WideEvent): Promise<void> {
+        await client.insert(table, [
+          {
+            id: event.id,
+            flow: event.flow,
+            duration_ms: event.durationMs,
+            payload: event,
+          },
+        ]);
+      },
+      async flush(): Promise<void> {
+        /* inserts are immediate in the fake */
+      },
+      async query(sql: string): Promise<RunsRow[]> {
+        return client.query(sql);
+      },
+      async all(): Promise<WideEvent[]> {
+        if ("events" in client) {
+          return [...(client as { events: WideEvent[] }).events];
+        }
+        return [];
+      },
+      async close(): Promise<void> {
+        /* fake has no resources */
+      },
+    };
+  },
+};

package/src/runs/drivers/files.ts

@@ -0,0 +1,201 @@
+/**
+ * Files runs driver — Parquet + DuckDB (default in every environment).
+ *
+ * Locality is an engine detail: recent partitions sit on a local bucket;
+ * older ones on optional object storage. Queries span both as one result set.
+ * The user never declares archive tiers.
+ */
+
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { fsDriver } from "../../drivers/fs.ts";
+import type { FilesBucket } from "../../drivers/types.ts";
+import { duckPath, duckQuery, openDuckDB, type DuckSession } from "../duckdb.ts";
+import {
+  partitionKey,
+  partitionObjectKey,
+  rowToWideEvent,
+  wideEventToRow,
+  writeParquet,
+} from "../parquet.ts";
+import type {
+  RunsDriver,
+  RunsOpenOptions,
+  RunsRow,
+  RunsStore,
+  WideEvent,
+} from "../types.ts";
+
+/** Default hot window — 7 days. Older writes go to object storage when present. */
+const DEFAULT_HOT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * Files (Parquet + DuckDB) runs driver.
+ */
+export const filesRunsDriver: RunsDriver = {
+  id: "files",
+  async open(options: RunsOpenOptions = {}): Promise<RunsStore> {
+    const hotWindowMs = options.hotWindowMs ?? DEFAULT_HOT_WINDOW_MS;
+    const clock = () => Date.now();
+
+    const localRoot =
+      options.localRoot ??
+      (await mkdtemp(join(tmpdir(), "oke-runs-")));
+    const ownsLocalRoot = options.localRoot === undefined && !options.localBucket;
+
+    const local: FilesBucket =
+      options.localBucket ??
+      (await fsDriver.open({ name: "runs-local", root: localRoot }));
+
+    const remote = options.remote?.bucket;
+    const remotePrefix = options.remote?.prefix ?? "";
+
+    const buffer: WideEvent[] = [];
+    const catalog: WideEvent[] = [];
+    let queryScratch: string | undefined;
+    const session: DuckSession = await openDuckDB();
+
+    function chooseBucket(startedAt: number): {
+      bucket: FilesBucket;
+      keyPrefix: string;
+      locality: "local" | "remote";
+    } {
+      const age = clock() - startedAt;
+      if (remote && age > hotWindowMs) {
+        return {
+          bucket: remote,
+          keyPrefix: remotePrefix,
+          locality: "remote",
+        };
+      }
+      return { bucket: local, keyPrefix: "", locality: "local" };
+    }
+
+    async function flushBuffer(): Promise<void> {
+      if (buffer.length === 0) return;
+      const groups = new Map<string, WideEvent[]>();
+      for (const event of buffer) {
+        const { locality } = chooseBucket(event.startedAt);
+        const day = partitionKey(event.startedAt);
+        const gkey = `${locality}:${day}`;
+        const list = groups.get(gkey) ?? [];
+        list.push(event);
+        groups.set(gkey, list);
+      }
+      buffer.length = 0;
+
+      for (const [gkey, events] of groups) {
+        const [locality, day] = gkey.split(":") as ["local" | "remote", string];
+        const target =
+          locality === "remote" && remote
+            ? { bucket: remote, keyPrefix: remotePrefix }
+            : { bucket: local, keyPrefix: "" };
+        const fileId = crypto.randomUUID();
+        const objectKey = `${target.keyPrefix}${partitionObjectKey(day!, fileId)}`;
+        const tmpDir = await mkdtemp(join(tmpdir(), "oke-pq-"));
+        const tmp = join(tmpDir, `${fileId}.parquet`);
+        try {
+          await writeParquet(
+            tmp,
+            events.map((e) => wideEventToRow(e)),
+          );
+          const bytes = new Uint8Array(await Bun.file(tmp).arrayBuffer());
+          await target.bucket.put(objectKey, bytes);
+          for (const e of events) catalog.push(e);
+        } finally {
+          await rm(tmpDir, { recursive: true, force: true });
+        }
+      }
+    }
+
+    async function materialiseAllParquet(): Promise<string[]> {
+      if (queryScratch) {
+        await rm(queryScratch, { recursive: true, force: true }).catch(
+          () => undefined,
+        );
+      }
+      queryScratch = await mkdtemp(join(tmpdir(), "oke-runs-q-"));
+      const paths: string[] = [];
+
+      const localKeys = (await local.list("runs/")).filter((k) =>
+        k.endsWith(".parquet"),
+      );
+      for (const key of localKeys) {
+        const data = await local.get(key);
+        if (!data) continue;
+        const dest = join(queryScratch, key.replaceAll("/", "__"));
+        await writeFile(dest, data);
+        paths.push(dest);
+      }
+      if (remote) {
+        const prefix = `${remotePrefix}runs/`;
+        const remoteKeys = (await remote.list(prefix)).filter((k) =>
+          k.endsWith(".parquet"),
+        );
+        for (const key of remoteKeys) {
+          const data = await remote.get(key);
+          if (!data) continue;
+          const dest = join(
+            queryScratch,
+            `remote__${key.replaceAll("/", "__")}`,
+          );
+          await writeFile(dest, data);
+          paths.push(dest);
+        }
+      }
+      return paths;
+    }
+
+    return {
+      driverId: "files",
+      async append(event: WideEvent): Promise<void> {
+        buffer.push(event);
+        if (buffer.length >= 256) await flushBuffer();
+      },
+      async flush(): Promise<void> {
+        await flushBuffer();
+      },
+      async query(sql: string): Promise<RunsRow[]> {
+        await flushBuffer();
+        const paths = await materialiseAllParquet();
+        if (paths.length === 0) return [];
+        const list = paths.map((p) => `'${duckPath(p)}'`).join(", ");
+        await session.conn.run(`DROP VIEW IF EXISTS runs`);
+        await session.conn.run(
+          `CREATE VIEW runs AS SELECT * FROM read_parquet([${list}], union_by_name = true)`,
+        );
+        return duckQuery(session.conn, sql);
+      },
+      async all(): Promise<WideEvent[]> {
+        await flushBuffer();
+        if (catalog.length > 0) return [...catalog];
+        const paths = await materialiseAllParquet();
+        if (paths.length === 0) return [];
+        const list = paths.map((p) => `'${duckPath(p)}'`).join(", ");
+        const rows = await duckQuery(
+          session.conn,
+          `SELECT * FROM read_parquet([${list}], union_by_name = true)`,
+        );
+        return rows.map((r) => rowToWideEvent(r));
+      },
+      async close(): Promise<void> {
+        await flushBuffer();
+        session.close();
+        if (queryScratch) {
+          await rm(queryScratch, { recursive: true, force: true }).catch(
+            () => undefined,
+          );
+        }
+        await local.close();
+        await remote?.close();
+        if (ownsLocalRoot) {
+          await rm(localRoot, { recursive: true, force: true }).catch(
+            () => undefined,
+          );
+        }
+      },
+    };
+  },
+};

package/src/runs/drivers/memory.ts

@@ -0,0 +1,94 @@
+/**
+ * In-memory runs driver — default for `test` environments.
+ *
+ * Uses DuckDB so queries match the `files` driver dialect (`FROM runs`).
+ */
+
+import type { DuckDBConnection } from "@duckdb/node-api";
+
+import {
+  duckLiteral,
+  duckQuery,
+  openDuckDB,
+  type DuckSession,
+} from "../duckdb.ts";
+import { wideEventToRow, type ParquetRow } from "../parquet.ts";
+import type {
+  RunsDriver,
+  RunsOpenOptions,
+  RunsRow,
+  RunsStore,
+  WideEvent,
+} from "../types.ts";
+
+/**
+ * Memory runs driver (DuckDB `:memory:`).
+ */
+export const memoryRunsDriver: RunsDriver = {
+  id: "memory",
+  async open(_options: RunsOpenOptions = {}): Promise<RunsStore> {
+    const session: DuckSession = await openDuckDB();
+    const events: WideEvent[] = [];
+
+    return {
+      driverId: "memory",
+      async append(event: WideEvent): Promise<void> {
+        events.push(event);
+      },
+      async flush(): Promise<void> {
+        /* nothing buffered */
+      },
+      async query(sql: string): Promise<RunsRow[]> {
+        await materialiseRunsTable(session.conn, events);
+        return duckQuery(session.conn, sql);
+      },
+      async all(): Promise<WideEvent[]> {
+        return [...events];
+      },
+      async close(): Promise<void> {
+        events.length = 0;
+        session.close();
+      },
+    };
+  },
+};
+
+/**
+ * Build / replace the `runs` table from wide events.
+ *
+ * @param conn - DuckDB connection
+ * @param events - Events to materialise
+ */
+export async function materialiseRunsTable(
+  conn: DuckDBConnection,
+  events: readonly WideEvent[],
+): Promise<void> {
+  await conn.run(`DROP TABLE IF EXISTS runs`);
+  if (events.length === 0) {
+    await conn.run(
+      `CREATE TABLE runs AS SELECT * FROM (SELECT 1 AS id WHERE FALSE)`,
+    );
+    return;
+  }
+  // JSONL path avoids VALUES type-inference issues for mixed columns.
+  const rows = events.map(wideEventToRow);
+  const cols = unionKeys(rows);
+  // For small test sets, VALUES is fine and keeps the driver dependency-light.
+  const colList = cols.map((c) => `"${c.replaceAll('"', '""')}"`).join(", ");
+  const values = rows
+    .map(
+      (r) => `(${cols.map((c) => duckLiteral(r[c] ?? null)).join(", ")})`,
+    )
+    .join(",\n");
+  await conn.run(
+    `CREATE TABLE runs AS SELECT * FROM (VALUES ${values}) AS t(${colList})`,
+  );
+}
+
+function unionKeys(rows: readonly ParquetRow[]): string[] {
+  const keys = new Set<string>();
+  for (const r of rows) {
+    for (const k of Object.keys(r)) keys.add(k);
+  }
+  return [...keys];
+}

package/src/runs/drivers/postgres.ts

@@ -0,0 +1,110 @@
+/**
+ * Optional Postgres runs driver — for teams who want a single store.
+ *
+ * Protocol-named. Uses an injected client in tests; production binds Bun.sql.
+ */
+
+import type {
+  RunsDriver,
+  RunsOpenOptions,
+  RunsRow,
+  RunsStore,
+  WideEvent,
+} from "../types.ts";
+
+/** Minimal SQL surface used by the postgres runs driver. */
+export interface RunsPostgresClient {
+  /**
+   * Run a query returning rows.
+   *
+   * @param sql - SQL text
+   * @param params - Bound parameters
+   */
+  query(sql: string, params?: readonly unknown[]): Promise<RunsRow[]>;
+  /**
+   * Execute a statement.
+   *
+   * @param sql - SQL text
+   * @param params - Bound parameters
+   */
+  exec(sql: string, params?: readonly unknown[]): Promise<void>;
+}
+
+/**
+ * In-memory fake Postgres client for tests.
+ */
+export function createRunsPostgresFake(): RunsPostgresClient & {
+  readonly events: WideEvent[];
+} {
+  const events: WideEvent[] = [];
+  return {
+    events,
+    async query(sql: string): Promise<RunsRow[]> {
+      const lower = sql.toLowerCase();
+      if (lower.includes("count(*)")) {
+        return [{ count: events.length }];
+      }
+      if (lower.includes("select payload")) {
+        return events.map((e) => ({ payload: e }));
+      }
+      return events.map((e) => ({
+        id: e.id,
+        flow: e.flow,
+        duration_ms: e.durationMs,
+        cache: e.cache,
+        tenant: e.tenant,
+        subject_id: e.subjectId,
+      }));
+    },
+    async exec(sql: string, params?: readonly unknown[]): Promise<void> {
+      const lower = sql.toLowerCase();
+      if (lower.includes("insert") && params && params.length >= 2) {
+        events.push(JSON.parse(String(params[1])) as WideEvent);
+      }
+    },
+  };
+}
+
+/**
+ * Postgres runs driver.
+ */
+export const postgresRunsDriver: RunsDriver = {
+  id: "postgres",
+  async open(options: RunsOpenOptions = {}): Promise<RunsStore> {
+    const client =
+      (options.client as RunsPostgresClient | undefined) ??
+      createRunsPostgresFake();
+    const table = options.name ?? "oke_runs";
+    await client.exec(
+      `CREATE TABLE IF NOT EXISTS ${table} (id TEXT PRIMARY KEY, payload JSONB)`,
+    );
+
+    return {
+      driverId: "postgres",
+      async append(event: WideEvent): Promise<void> {
+        await client.exec(
+          `INSERT INTO ${table} (id, payload) VALUES ($1, $2::jsonb)`,
+          [event.id, JSON.stringify(event)],
+        );
+      },
+      async flush(): Promise<void> {
+        /* row-level writes are immediate */
+      },
+      async query(sql: string): Promise<RunsRow[]> {
+        return client.query(sql);
+      },
+      async all(): Promise<WideEvent[]> {
+        const result = await client.query(`SELECT payload FROM ${table}`);
+        return result.map((r) => {
+          const p = r.payload ?? r;
+          return typeof p === "string"
+            ? (JSON.parse(p) as WideEvent)
+            : (p as WideEvent);
+        });
+      },
+      async close(): Promise<void> {
+        /* fake has no resources */
+      },
+    };
+  },
+};