okengine 0.21.1 → 0.23.0

package/src/drivers/memory.ts

@@ -503,7 +503,7 @@ function createMemorySqlConnection(role: "primary" | "replica"): SqlConnection {
       }
 
       const create =
-        /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+("?[a-zA-Z_][a-zA-Z0-9_]*"?)\s*\((.+)\)\s*$/i.exec(
+        /^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+("?[a-zA-Z_][a-zA-Z0-9_]*"?)\s*\((.+)\)\s*$/is.exec(
           text,
         );
       if (create) {
@@ -519,6 +519,26 @@ function createMemorySqlConnection(role: "primary" | "replica"): SqlConnection {
         return { changes: 0 };
       }
 
+      const addColumn =
+        /^ALTER\s+TABLE\s+("?[a-zA-Z_][a-zA-Z0-9_]*"?)\s+ADD\s+COLUMN\s+IF\s+NOT\s+EXISTS\s+("?[a-zA-Z_][a-zA-Z0-9_]*"?)\b/is.exec(
+          text,
+        );
+      if (addColumn) {
+        const name = parseIdent(addColumn[1]!);
+        const column = parseIdent(addColumn[2]!);
+        const table = tables.get(name);
+        if (table && !table.columns.includes(column)) table.columns.push(column);
+        return { changes: 0 };
+      }
+
+      // Scan results do not depend on indexes. Identity and vault DDL still
+      // issue CREATE INDEX IF NOT EXISTS on the shared memory connection.
+      const createIndex =
+        /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?:"?[a-zA-Z_][a-zA-Z0-9_]*"?)\s+ON\s+/is.exec(
+          text,
+        );
+      if (createIndex) return { changes: 0 };
+
       const insert =
         /^INSERT\s+INTO\s+("?[a-zA-Z_][a-zA-Z0-9_]*"?)\s*\(([^)]+)\)\s*VALUES\s*\(([^)]+)\)\s*$/i.exec(
           text,

package/src/drivers/pg-rls.test.ts

@@ -5,6 +5,7 @@ import {
   buildRlsIdentityPreludeSql,
   emitPgPolicySource,
   formatPolicyRole,
+  OKE_PG_STAT_STATEMENTS_SQL,
   OKE_RLS_HELPER_STATEMENTS,
   OKE_RLS_ROLE,
   parseSqlPolicySpec,
@@ -149,6 +150,36 @@ describe("OKE_RLS_HELPER_STATEMENTS", () => {
     expect(hasScope).not.toMatch(/jsonb\s+\?/);
   });
 
+  test("initial oke SQL creates pg_stat_statements before helpers", async () => {
+    const ran: string[] = [];
+    const { installOkeRlsHelpers } = await import("./pg-rls.ts");
+    await installOkeRlsHelpers(async (sql) => {
+      ran.push(sql);
+      if (sql.startsWith("CREATE EXTENSION")) return;
+      if (sql.includes("pg_vault") || sql.includes("vault.")) {
+        throw new Error("stop before vault attach");
+      }
+    }).catch(() => undefined);
+    expect(ran[0]).toBe(OKE_PG_STAT_STATEMENTS_SQL);
+    expect(ran[1]).toBe(OKE_RLS_HELPER_STATEMENTS[0]);
+  });
+
+  test("a rejected pg_stat_statements create still installs helpers", async () => {
+    const ran: string[] = [];
+    const { installOkeRlsHelpers } = await import("./pg-rls.ts");
+    await installOkeRlsHelpers(async (sql) => {
+      ran.push(sql);
+      if (sql.startsWith("CREATE EXTENSION")) {
+        throw new Error('extension "pg_stat_statements" is not available');
+      }
+      if (sql.includes("CREATE OR REPLACE FUNCTION oke.gate")) {
+        throw new Error("stop");
+      }
+    }).catch(() => undefined);
+    expect(ran[0]).toBe(OKE_PG_STAT_STATEMENTS_SQL);
+    expect(ran.some((sql) => sql.includes("CREATE SCHEMA IF NOT EXISTS oke"))).toBe(true);
+  });
+
   test("pins search_path to public so schema oke is not the table home", () => {
     expect(OKE_RLS_HELPER_STATEMENTS).toContain("SET search_path TO public, oke");
     expect(

package/src/drivers/pg-rls.ts

@@ -135,12 +135,27 @@ $oke_app_search_path$`,
  */
 export const OKE_RLS_HELPER_SQL = `${OKE_RLS_HELPER_STATEMENTS.join(";\n")};`;
 
+/**
+ * Enable query telemetry on postgres. Preload is still required
+ * (`shared_preload_libraries=pg_stat_statements`). PGlite and Cockroach
+ * reject this; callers swallow that failure so the rest of the oke SQL runs.
+ */
+export const OKE_PG_STAT_STATEMENTS_SQL = "CREATE EXTENSION IF NOT EXISTS pg_stat_statements";
+
 /**
  * Install `oke.*` helpers one statement at a time.
  *
+ * `pg_stat_statements` is created first when the engine allows it, so Store
+ * performance does not depend on a later manual `CREATE EXTENSION`.
+ *
  * @param exec - Statement runner
  */
 export async function installOkeRlsHelpers(exec: (sql: string) => Promise<unknown>): Promise<void> {
+  try {
+    await exec(OKE_PG_STAT_STATEMENTS_SQL);
+  } catch {
+    // Not preloaded, or this engine has no pg_stat_statements (pglite, cockroach).
+  }
   for (const stmt of OKE_RLS_HELPER_STATEMENTS) {
     await exec(stmt);
   }

package/src/drivers/postgres.test.ts

@@ -38,6 +38,15 @@ describe("toPostgresParams", () => {
 });
 
 describe("closeSharedPostgresClients", () => {
+  test("a different max is a different pool on the same URL", () => {
+    const url = "postgres://127.0.0.1:9/oke-pool-max-shared-test";
+    const small = sharedPostgresClient(url, 8);
+    const large = sharedPostgresClient(url, 20);
+    expect(large).not.toBe(small);
+    expect(sharedPostgresClient(url, 20)).toBe(large);
+    expect(sharedPostgresClient(url)).toBe(small);
+  });
+
   test("drops cached pools so the next checkout is a new client", async () => {
     const url = "postgres://127.0.0.1:9/oke-close-shared-test";
     const first = sharedPostgresClient(url);

package/src/drivers/postgres.ts

@@ -43,7 +43,27 @@ type SharedPostgresEntry = {
   readonly guarded: PostgresClientLike;
 };
 
-const sharedClients = new Map<string, SharedPostgresEntry>();
+/**
+ * Process-wide pool registry.
+ *
+ * `bun --hot` re-evaluates this module on each save. A module-local `Map`
+ * would drop the only handle to the previous Bun.SQL pools and leave them
+ * connected. PgDog then queues every checkout — Console login included —
+ * until its server pool times out.
+ */
+const SHARED_POSTGRES_CLIENTS = Symbol.for("oke.sharedPostgresClients");
+
+function sharedClientMap(): Map<string, SharedPostgresEntry> {
+  const g = globalThis as typeof globalThis & {
+    [SHARED_POSTGRES_CLIENTS]?: Map<string, SharedPostgresEntry>;
+  };
+  let map = g[SHARED_POSTGRES_CLIENTS];
+  if (!map) {
+    map = new Map();
+    g[SHARED_POSTGRES_CLIENTS] = map;
+  }
+  return map;
+}
 
 /**
  * When true, shared pools stay closed and every query fails fast — no Bun.SQL
@@ -160,21 +180,38 @@ function guardSharedPostgresClient(raw: PostgresClientLike): PostgresClientLike
 }
 
 /**
- * One Bun.SQL pool per URL for store / journal / clock / vault / console.
+ * Bun.SQL pool size. Unset, non-finite, or below 1 keeps {@link POSTGRES_POOL_MAX}.
+ * A configured max is not capped — callers that need a larger pool ask for it.
+ *
+ * @param poolMax - Requested `max`
+ */
+function postgresPoolMax(poolMax: number | undefined): number {
+  if (poolMax === undefined || !Number.isFinite(poolMax) || poolMax < 1) return POSTGRES_POOL_MAX;
+  return Math.floor(poolMax);
+}
+
+/**
+ * One Bun.SQL pool per URL and `max` for store / journal / clock / vault / console.
  *
  * Returns a pause-aware client — holders (including fleet store wrappers)
  * fail soft after {@link pauseSharedPostgresClients} instead of reconnecting.
  *
  * @param url - Connection URL
+ * @param poolMax - Bun.SQL `max`. Omitted uses {@link POSTGRES_POOL_MAX}. A
+ * different max is a different pool — a store pin of 20 must not reuse a
+ * pool another subsystem already opened at the default.
  */
-export function sharedPostgresClient(url?: string): PostgresClientLike {
+export function sharedPostgresClient(url?: string, poolMax?: number): PostgresClientLike {
   assertSharedPostgresReady();
-  const key = resolvePostgresUrl(url);
-  const existing = sharedClients.get(key);
+  const resolved = resolvePostgresUrl(url);
+  const max = postgresPoolMax(poolMax);
+  const key = `${resolved}\0${String(max)}`;
+  const clients = sharedClientMap();
+  const existing = clients.get(key);
   if (existing) return existing.guarded;
-  const raw = new Bun.SQL(key, { max: POSTGRES_POOL_MAX }) as unknown as PostgresClientLike;
+  const raw = new Bun.SQL(resolved, { max }) as unknown as PostgresClientLike;
   const guarded = guardSharedPostgresClient(raw);
-  sharedClients.set(key, { raw, guarded });
+  clients.set(key, { raw, guarded });
   return guarded;
 }
 
@@ -185,8 +222,8 @@ export function sharedPostgresClient(url?: string): PostgresClientLike {
  * not stop Console from opening a fresh pool on the next poll.
  */
 export async function closeSharedPostgresClients(): Promise<void> {
-  const clients = [...sharedClients.values()];
-  sharedClients.clear();
+  const clients = [...sharedClientMap().values()];
+  sharedClientMap().clear();
   await Promise.all(
     clients.map(async (entry) => {
       try {
@@ -283,12 +320,13 @@ export async function connectPostgres(options: SqlConnectOptions = {}): Promise<
   // must not share a pooled client — Bun raises ERR_POSTGRES_UNSAFE_TRANSACTION
   // when raw statements interleave with open transaction state on the shared
   // pool. Opt in explicitly; default behavior is unchanged.
-  const dedicated = !injected && options.pool?.max === 1;
+  const requestedMax = options.pool?.max;
+  const dedicated = !injected && requestedMax === 1;
   const client =
     injected ??
     (dedicated
       ? dedicatedPostgresClient(resolvePostgresUrl(options.url))
-      : sharedPostgresClient(options.url));
+      : sharedPostgresClient(options.url, requestedMax));
   return wrapPostgresClient(client, role, {
     shared: injected === undefined && !dedicated,
     ...(options.url !== undefined ? { url: options.url } : {}),

package/src/drivers/vault-builtin.test.ts

@@ -115,4 +115,14 @@ describe("builtin vault driver", () => {
       await rm(dir, { recursive: true, force: true });
     }
   }, 15_000);
+
+  test("webhook audit without a URL fails open instead of degrading", async () => {
+    await expect(
+      builtinVaultDriver.open({
+        connection: sharedConn,
+        env: {},
+        audit: { sink: "webhook" },
+      }),
+    ).rejects.toThrow(/webhookUrl/);
+  });
 });

package/src/drivers/vault-builtin.ts

@@ -18,6 +18,7 @@ import {
   sqlConnectionAsExec,
   type BuiltinVaultAdapter,
 } from "../elements/vault/builtin-adapter.ts";
+import { isVaultError } from "../elements/vault/errors.ts";
 import type { SqlConnection } from "./types.ts";
 import type { VaultBag, VaultDriver, VaultOpenOptions } from "./vault-types.ts";
 
@@ -39,6 +40,10 @@ export interface OpenBuiltinVaultOptions {
   readonly env?: Readonly<Record<string, string | undefined>>;
   /** DEKs re-wrapped per master-rotation batch. */
   readonly kekRewrapBatchSize?: number;
+  /** `vault.audit` from config. Omitted means the SQL hash chain. */
+  readonly audit?: import("../elements/vault/types.ts").VaultAuditConfig;
+  /** Fetch override for the webhook audit sink (tests). */
+  readonly fetch?: typeof globalThis.fetch;
 }
 
 /** An adapter plus the connection this module opened for it, if any. */
@@ -76,6 +81,8 @@ export async function openBuiltinVaultAdapter(
     ...(options.kekRewrapBatchSize === undefined
       ? {}
       : { kekRewrapBatchSize: options.kekRewrapBatchSize }),
+    ...(options.audit === undefined ? {} : { audit: options.audit }),
+    ...(options.fetch === undefined ? {} : { fetch: options.fetch }),
   });
 
   const masterKey = options.masterKey ?? env[VAULT_MASTER_KEY_ENV];
@@ -114,6 +121,8 @@ export const builtinVaultDriver: VaultDriver = {
         opened = await openBuiltinVaultAdapter({
           env,
           ...(options.connection !== undefined ? { connection: options.connection } : { url }),
+          ...(options.audit === undefined ? {} : { audit: options.audit }),
+          ...(options.fetch === undefined ? {} : { fetch: options.fetch }),
         });
         if (opened.adapter.getUnsealer() !== null) {
           for (const entry of await opened.adapter.list()) {
@@ -121,7 +130,10 @@ export const builtinVaultDriver: VaultDriver = {
             if (secret) map.set(entry.path, secret.value);
           }
         }
-      } catch {
+      } catch (err) {
+        // A bad audit sink is a config error — do not hide it behind the
+        // uninitialized-backend fallback.
+        if (isVaultError(err, "MISSING_PEER")) throw err;
         // Uninitialized / sealed / unreachable: fall back to the seed bag so
         // boot reports missing contracts instead of a driver stack trace.
         await opened?.close().catch(() => undefined);

package/src/drivers/vault-remote-bag.ts

@@ -8,6 +8,7 @@
  */
 
 import { VaultError } from "../elements/vault/errors.ts";
+import { requestSignal } from "../kernel/abort-scope.ts";
 import type { VaultBag } from "./vault-types.ts";
 
 /**
@@ -183,7 +184,10 @@ export async function vaultHttpJson(
 ): Promise<VaultHttpJsonResult> {
   let response: Response;
   try {
-    response = await fetchFn(url, init);
+    response = await fetchFn(url, {
+      ...init,
+      signal: requestSignal(init.signal),
+    });
   } catch (error) {
     throw asRemoteVaultError(error, message);
   }

package/src/drivers/vault-types.ts

@@ -46,8 +46,13 @@ export interface VaultOpenOptions {
   readonly region?: string;
   /** Seed map for `memory` / `managed` drivers. */
   readonly secrets?: Readonly<Record<string, string>>;
-  /** HTTP fetch override for remote drivers (tests). */
+  /** HTTP fetch override for remote drivers and the webhook audit sink (tests). */
   readonly fetch?: typeof globalThis.fetch;
+  /**
+   * Built-in vault audit block (`vault.audit`). Honored by the `vault` driver.
+   * Omitted means the SQL hash chain.
+   */
+  readonly audit?: import("../elements/vault/types.ts").VaultAuditConfig;
 }
 
 /** Resolved secret bag — names → cleartext values. Never logged. */

package/src/elements/ai/agent-event-slot.ts

@@ -0,0 +1,27 @@
+/**
+ * Installed agent event log.
+ *
+ * The follow route and the AI runtime must share one slot. The log
+ * implementation stays in `run-events.ts` and loads on first use; this file
+ * is the only static edge so a `lazyRequire` copy cannot hide the log.
+ */
+
+import type { AgentEventLog } from "./run-events.ts";
+
+let activeLog: AgentEventLog | undefined;
+
+/**
+ * Install the log the follow route reads. Boot and tests call this.
+ *
+ * @param log - Active log
+ */
+export function setAgentEventLog(log: AgentEventLog | undefined): void {
+  activeLog = log;
+}
+
+/**
+ * Log installed by the running AI runtime.
+ */
+export function getAgentEventLog(): AgentEventLog | undefined {
+  return activeLog;
+}

package/src/elements/ai/approval-http.ts

@@ -0,0 +1,207 @@
+/**
+ * Built-in HTTP approve / deny for a parked agent tool.
+ *
+ * The route is public so the request can arrive. The tool's own gate is
+ * checked inside `fx.agent`. `Idempotency-Key` applies because the flow writes.
+ */
+
+import { flow } from "../../kernel/flow.ts";
+import type { Fx } from "../../kernel/fx.ts";
+import { lazyRequire } from "../../kernel/lazy-require.ts";
+import type { Binding } from "../../kernel/on.ts";
+import { http } from "../../kernel/triggers.ts";
+import type { GateRuntime } from "../gate/runtime.ts";
+import type { StandardSchemaV1 } from "../../validation/standard-schema.ts";
+import { journalLeaseBusyResponse } from "./approval.ts";
+import { getAgentEventLog } from "./agent-event-slot.ts";
+import type { AgentEventLog } from "./run-events.ts";
+
+/** SSE frame helpers live on `fx.ts`. Load them on the first follow frame. */
+function loadSseFrames(): typeof import("../../kernel/fx.ts") {
+  return lazyRequire(`${import.meta.dir}/../../kernel`, ["f", "x"].join(""));
+}
+
+/** Body the approve route accepts. `args` replaces the tool input. */
+const approveIn: StandardSchemaV1<{ id: string; args?: unknown }> = {
+  "~standard": {
+    version: 1,
+    vendor: "okengine",
+    validate(value) {
+      if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return { issues: [{ message: "expected object", path: [] }] };
+      }
+      const body = value as { id?: unknown; args?: unknown };
+      if (typeof body.id !== "string" || body.id.length === 0) {
+        return { issues: [{ message: "id is required", path: ["id"] }] };
+      }
+      return { value: { id: body.id, ...(body.args !== undefined ? { args: body.args } : {}) } };
+    },
+  },
+};
+
+/** Body the deny route accepts. */
+const denyIn: StandardSchemaV1<{ id: string; reason?: string }> = {
+  "~standard": {
+    version: 1,
+    vendor: "okengine",
+    validate(value) {
+      if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return { issues: [{ message: "expected object", path: [] }] };
+      }
+      const body = value as { id?: unknown; reason?: unknown };
+      if (typeof body.id !== "string" || body.id.length === 0) {
+        return { issues: [{ message: "id is required", path: ["id"] }] };
+      }
+      if (body.reason !== undefined && typeof body.reason !== "string") {
+        return { issues: [{ message: "reason must be a string", path: ["reason"] }] };
+      }
+      return {
+        value: {
+          id: body.id,
+          ...(typeof body.reason === "string" ? { reason: body.reason } : {}),
+        },
+      };
+    },
+  },
+};
+
+/**
+ * Map a resolution to the HTTP failure the rest of the kernel already speaks.
+ *
+ * @param fx - Flow context
+ * @param result - Approval resolution
+ */
+function approvalHttpResult(
+  fx: Fx,
+  result:
+    | { readonly ok: true }
+    | { readonly ok: false; readonly status: 403 | 404 }
+    | { readonly ok: false; readonly status: 409; readonly reason: "resolved" }
+    | {
+        readonly ok: false;
+        readonly status: 409;
+        readonly reason: "lease";
+        readonly retryAfterSeconds: number;
+      },
+): unknown {
+  if (result.ok) return { ok: true };
+  if (result.status === 409 && result.reason === "lease") {
+    return journalLeaseBusyResponse(result.retryAfterSeconds);
+  }
+  if (result.status === 409) return fx.fail.conflict();
+  if (result.status === 403) return fx.fail.forbidden();
+  return fx.fail.notFound();
+}
+
+/**
+ * Register the approve and deny routes on an app.
+ *
+ * @param adopt - App binding adopter
+ */
+export function bindAgentApprovalFlows(adopt: (binding: Binding) => void): void {
+  adopt({
+    trigger: http.post("/agent/approvals/approve", { in: approveIn }).public(),
+    flow: flow("oke.agent.approve", {
+      effects: { writes: ["journal:runs"] },
+      do: async (input, fx) => {
+        const body = input as { id: string; args?: unknown };
+        const result = await fx.agent.approve(body.id, { args: body.args });
+        return approvalHttpResult(fx, result);
+      },
+    }),
+  });
+  adopt({
+    trigger: http.post("/agent/approvals/deny", { in: denyIn }).public(),
+    flow: flow("oke.agent.deny", {
+      effects: { writes: ["journal:runs"] },
+      do: async (input, fx) => {
+        const body = input as { id: string; reason?: string };
+        const result = await fx.agent.deny(body.id, { reason: body.reason });
+        return approvalHttpResult(fx, result);
+      },
+    }),
+  });
+  adopt({
+    trigger: http.get("/agent/runs/:runId/events").public(),
+    flow: flow("oke.agent.follow", {
+      do: async (input, fx) => followAgentRun(input, fx),
+    }),
+  });
+}
+
+let followGates: GateRuntime | undefined;
+
+/**
+ * Gate runtime the follow route uses. Boot installs the app's gates.
+ *
+ * @param gates - Runtime that checked the calling Flow
+ */
+export function setAgentFollowGates(gates: GateRuntime | undefined): void {
+  followGates = gates;
+}
+
+async function followAgentRun(input: unknown, fx: Fx): Promise<unknown> {
+  const body = (input ?? {}) as { runId?: unknown };
+  const runId = typeof body.runId === "string" ? body.runId : "";
+  const log = getAgentEventLog();
+  if (!log || !runId) return fx.fail.notFound();
+  const header = await log.header(runId);
+  if (!header) return fx.fail.notFound();
+  if ((header.tenant ?? null) !== (fx.tenant.id ?? null)) return fx.fail.notFound();
+  const followerIsOperator = fx.operator.id !== null;
+  const sameUser = (header.userId ?? null) === (fx.auth.userId ?? null);
+  const sameOperator = (header.operatorId ?? null) === (fx.operator.id ?? null);
+  const samePrincipal =
+    (header.userId === null && header.operatorId === null && sameUser && sameOperator) ||
+    (header.userId !== null && sameUser) ||
+    (header.operatorId !== null && sameOperator);
+  if (!followerIsOperator && !samePrincipal) return fx.fail.forbidden();
+  if (header.gates.length > 0) {
+    if (!followGates) return fx.fail.forbidden();
+    const checks = await followGates.check(header.gates, {
+      auth: fx.auth,
+      operator: fx.operator,
+      meta: {},
+    });
+    if (checks.some((check) => check.allowed === false)) return fx.fail.forbidden();
+  }
+  const after = Number(fx.lastEventId ?? "0");
+  const afterSeq = Number.isFinite(after) ? after : 0;
+  return fx.json.stream(frames(log, runId, afterSeq, fx.signal));
+}
+
+async function* frames(
+  log: AgentEventLog,
+  runId: string,
+  afterSeq: number,
+  signal: AbortSignal,
+): AsyncIterable<unknown> {
+  const iter = log.subscribe(runId, afterSeq, signal)[Symbol.asyncIterator]();
+  let pending = iter.next();
+  try {
+    for (;;) {
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const ping = new Promise<"ping">((resolve) => {
+        timer = setTimeout(() => resolve("ping"), 15_000);
+      });
+      const winner = await Promise.race([
+        pending.then((row) => ({ kind: "row" as const, row })),
+        ping.then((kind) => ({ kind })),
+      ]);
+      if (timer) clearTimeout(timer);
+      if (winner.kind === "ping") {
+        if (signal.aborted) return;
+        yield loadSseFrames().sseComment("keepalive");
+        continue;
+      }
+      pending = iter.next();
+      if (winner.row.done || winner.row.value === undefined) return;
+      const row = winner.row.value;
+      yield loadSseFrames().sseFrame(row.event, String(row.seq));
+      if (row.event.type === "RUN_ERROR") return;
+      if (row.event.type === "RUN_FINISHED" && row.event.outcome?.type !== "interrupt") return;
+    }
+  } finally {
+    await iter.return?.();
+  }
+}