okengine 0.17.2 → 0.18.4

package/src/cli/doctor.test.ts

@@ -5,6 +5,7 @@
 import { describe, expect, test } from "bun:test";
 import type { Manifest } from "../manifest/types.ts";
 import { runDoctor } from "./doctor.ts";
+import { estimatePeakFds } from "./doctor-fd.ts";
 
 describe("oke doctor", () => {
   test("catches a missing secret", async () => {
@@ -210,3 +211,194 @@ describe("oke doctor", () => {
     expect(findings.some((f) => f.code === "pii_ask")).toBe(false);
   });
 });
+
+describe("doctor file_descriptor_limit", () => {
+  const base = {
+    secrets: [] as string[],
+    ports: [] as number[],
+    skipDbDrift: true,
+    expectedSchemaFingerprint: "same",
+    currentSchemaFingerprint: "same",
+    write: () => {},
+  };
+
+  test("warns when soft limit is low relative to estimated need", async () => {
+    const { code, findings } = await runDoctor({
+      ...base,
+      detectFdPressure: async () => ({ softLimit: 256, estimatedNeed: 400, headroom: -144 }),
+    });
+    expect(code).toBe(0);
+    expect(findings.some((f) => f.code === "file_descriptor_limit" && f.severity === "warn")).toBe(
+      true,
+    );
+  });
+
+  test("errors when soft limit below estimated need", async () => {
+    const { code, findings } = await runDoctor({
+      ...base,
+      detectFdPressure: async () => ({ softLimit: 200, estimatedNeed: 400, headroom: -200 }),
+    });
+    expect(code).toBe(2);
+    const finding = findings.find((f) => f.code === "file_descriptor_limit");
+    expect(finding?.severity).toBe("error");
+    expect(finding?.message).toContain("file descriptor");
+  });
+
+  test("no finding with ample headroom", async () => {
+    const { code, findings } = await runDoctor({
+      ...base,
+      detectFdPressure: async () => ({ softLimit: 65536, estimatedNeed: 500, headroom: 65036 }),
+    });
+    expect(code).toBe(0);
+    expect(findings.some((f) => f.code === "file_descriptor_limit")).toBe(false);
+  });
+
+  test("skips conservatively without manifest/live signals", async () => {
+    const findings: unknown[] = [];
+    const { code } = await runDoctor({
+      ...base,
+      write: (t) => findings.push(t),
+    });
+    // Real probe runs; on typical dev limits it must not throw and must exit ok.
+    expect([0, 2]).toContain(code);
+  });
+
+  test("estimatePeakFds counts live signals and http routes", async () => {
+    const manifest = {
+      oke: "1.0",
+      app: "t",
+      signals: {
+        tick: { delivery: "live" },
+        once: { delivery: "once" },
+        fan: { delivery: "broadcast" },
+      },
+      flows: {
+        sse: { trigger: { http: { method: "GET", path: "/live" } }, live: "tick" },
+        plain: { trigger: { http: { method: "POST", path: "/x" } } },
+      },
+    } as unknown as Manifest;
+    const need = estimatePeakFds(manifest);
+    expect(need).toBeGreaterThan(estimatePeakFds(null));
+    expect(need).toBeGreaterThanOrEqual(64 * 2 * 2); // live signal + SSE route, ≥2 fds/sub
+  });
+
+  test("estimatePeakFds subscriber term is linear in live signals (G3b-calibrated)", () => {
+    const one = {
+      oke: "1.0",
+      app: "t",
+      signals: { a: { delivery: "live" } },
+      flows: {},
+    } as unknown as Manifest;
+    const two = {
+      ...one,
+      signals: { a: { delivery: "live" }, b: { delivery: "live" } },
+    } as unknown as Manifest;
+    const base = estimatePeakFds(null);
+    const deltaOne = estimatePeakFds(one) - base;
+    const deltaTwo = estimatePeakFds(two) - base;
+    // Each additional live signal adds the same subscriber-fd budget.
+    expect(deltaTwo).toBe(deltaOne * 2);
+    // G3b measured 1.53 fds/subscriber; budget must be at least 1 fd/sub.
+    expect(deltaOne).toBeGreaterThanOrEqual(64);
+  });
+});
+
+describe("doctor realtime checks", () => {
+  const base = {
+    secrets: [] as string[],
+    ports: [] as number[],
+    skipDbDrift: true,
+    write: () => {},
+    currentSchemaFingerprint: null,
+  };
+
+  test("cdc_outbox_backlog warn at >10k pending", async () => {
+    const { findings } = await runDoctor({
+      ...base,
+      detectRealtimeMetrics: async () => ({
+        subscribers: 10,
+        queueDepth: 0,
+        fanout: { eventsIn: 1, eventsShed: 0, checksRun: 0, checkFailures: 0 },
+        outbox: { pending: 20_000, dispatchedOverCap: 0 },
+      }),
+    });
+    const finding = findings.find((f) => f.code === "cdc_outbox_backlog");
+    expect(finding?.severity).toBe("warn");
+  });
+
+  test("cdc_outbox_backlog error at >100k pending", async () => {
+    const { code, findings } = await runDoctor({
+      ...base,
+      detectRealtimeMetrics: async () => ({
+        subscribers: 10,
+        queueDepth: 0,
+        fanout: { eventsIn: 1, eventsShed: 0, checksRun: 0, checkFailures: 0 },
+        outbox: { pending: 150_000, dispatchedOverCap: 0 },
+      }),
+    });
+    expect(code).toBe(2);
+    expect(findings.find((f) => f.code === "cdc_outbox_backlog")?.severity).toBe("error");
+  });
+
+  test("cdc_outbox_retention warns when delivered rows exceed maxCount cap", async () => {
+    const { findings } = await runDoctor({
+      ...base,
+      detectRealtimeMetrics: async () => ({
+        subscribers: 10,
+        queueDepth: 0,
+        fanout: { eventsIn: 1, eventsShed: 0, checksRun: 0, checkFailures: 0 },
+        outbox: { pending: 0, dispatchedOverCap: 500 },
+      }),
+    });
+    expect(findings.some((f) => f.code === "cdc_outbox_retention")).toBe(true);
+  });
+
+  test("live_subscriber_pressure warns above ~150 active subs", async () => {
+    const { findings } = await runDoctor({
+      ...base,
+      detectRealtimeMetrics: async () => ({
+        subscribers: 200,
+        queueDepth: 0,
+        fanout: { eventsIn: 1, eventsShed: 0, checksRun: 0, checkFailures: 0 },
+        outbox: { unavailable: "no_bridge" },
+      }),
+    });
+    expect(findings.some((f) => f.code === "live_subscriber_pressure")).toBe(true);
+  });
+
+  test("live_fanout_queue_saturated errors on near-cap depth plus shedding; no finding when healthy", async () => {
+    const saturated = await runDoctor({
+      ...base,
+      detectRealtimeMetrics: async () => ({
+        subscribers: 100,
+        queueDepth: 9_000,
+        fanout: { eventsIn: 50_000, eventsShed: 12, checksRun: 40_000, checkFailures: 2 },
+        outbox: { unavailable: "no_bridge" },
+      }),
+    });
+    expect(saturated.findings.some((f) => f.code === "live_fanout_queue_saturated")).toBe(true);
+
+    const healthy = await runDoctor({
+      ...base,
+      expectedSchemaFingerprint: "same",
+      currentSchemaFingerprint: "same",
+      detectRealtimeMetrics: async () => ({
+        subscribers: 100,
+        queueDepth: 4,
+        fanout: { eventsIn: 5_000, eventsShed: 0, checksRun: 4_800, checkFailures: 0 },
+        outbox: { unavailable: "no_bridge" },
+      }),
+    });
+    expect(healthy.findings).toHaveLength(0);
+  });
+
+  test("no realtime findings when no bridge is bound (metrics null)", async () => {
+    const { findings } = await runDoctor({
+      ...base,
+      detectRealtimeMetrics: async () => null,
+    });
+    expect(
+      findings.filter((f) => f.code.startsWith("cdc_") || f.code.startsWith("live_")),
+    ).toHaveLength(0);
+  });
+});

package/src/cli/doctor.ts

@@ -17,11 +17,24 @@ import { hasFlag, wantsJson } from "./args.ts";
 import { checkManifestPiiAsks } from "./doctor-pii.ts";
 import { EXIT_OK, EXIT_RUNTIME } from "./exit.ts";
 import { loadManifest, loadOkeConfig } from "./load-config.ts";
+import { OUTBOX_PRUNE_INTERVAL_MS } from "../drivers/cdc-outbox.ts";
 import { isPortInUse } from "./ports.ts";
 import { schemaFingerprint, readSchemaFingerprint } from "./schema.ts";
 
 export { isPortInUse } from "./ports.ts";
 
+/** Warn when the CDC outbox drain rate looks stalled below this backlog. */
+const OUTBOX_BACKLOG_WARN = 10_000;
+
+/** Error-severity outbox backlog — poller considered down. */
+const OUTBOX_BACKLOG_ERROR = 100_000;
+
+/** v1 documented fan-out ceiling per table (Hardening 2 / plan §Operations). */
+const LIVE_SUBSCRIBER_SOFT_CAP = 150;
+
+/** Queue-depth threshold where fan-out health findings start firing. */
+const LIVE_FANOUT_QUEUE_WARN_DEPTH = 8_000;
+
 /**
  * `label → merged EnvDriverMap` pairs, in the fixed order shown by
  * {@link formatDriversSummary} / the `drivers` JSON key.
@@ -84,7 +97,12 @@ export interface DoctorFinding {
     | "tenancy"
     | "driver"
     | "pii_ask"
-    | "vault_master_key";
+    | "vault_master_key"
+    | "file_descriptor_limit"
+    | "cdc_outbox_backlog"
+    | "cdc_outbox_retention"
+    | "live_subscriber_pressure"
+    | "live_fanout_queue_saturated";
   readonly severity: "error" | "warn";
   readonly message: string;
 }
@@ -138,8 +156,31 @@ export interface DoctorOptions {
   readonly writeErr?: (text: string) => void;
   /** Emit only JSON on stdout. */
   readonly json?: boolean;
+  /**
+   * Inject the file-descriptor pressure probe (tests). When unset, the real
+   * probe runs only when a manifest was loaded (conservative skip otherwise).
+   */
+  readonly detectFdPressure?: () => Promise<{
+    readonly softLimit: number;
+    readonly estimatedNeed: number;
+    readonly headroom: number;
+  }>;
+  /**
+   * Inject the realtime metrics probe (tests). When unset, the real probe
+   * reads the bound realtime bridge in-process (returns `null` — no findings
+   * — when this CLI process has none, e.g. the app runs in another host).
+   */
+  readonly detectRealtimeMetrics?: () => Promise<RealtimeMetricsSnapshot | null>;
 }
 
+/** Realtime metrics shape used by the doctor checks (see {@link realtimeMetrics}). */
+export type RealtimeMetricsSnapshot = {
+  subscribers: number;
+  queueDepth: number;
+  fanout: { eventsIn: number; eventsShed: number; checksRun: number; checkFailures: number };
+  outbox: { pending: number; dispatchedOverCap: number } | { unavailable: "no_bridge" };
+};
+
 /**
  * Run doctor checks.
  *
@@ -232,6 +273,93 @@ export async function runDoctor(options: DoctorOptions = {}): Promise<{
     }
   }
 
+  // File-descriptor headroom — live SSE holds one fd per open subscriber.
+  // Severity: error when the soft limit dips under ~60% of estimated peak
+  // need (exhaustion is imminent); warn when it is under 2× need.
+  const fd =
+    options.detectFdPressure !== undefined
+      ? await options.detectFdPressure()
+      : manifest
+        ? await (async () => {
+            const { checkFileDescriptorHeadroom } = await import("./doctor-fd.ts");
+            return checkFileDescriptorHeadroom(manifest);
+          })()
+        : null;
+  if (fd) {
+    if (fd.softLimit < fd.estimatedNeed * 0.6) {
+      findings.push({
+        code: "file_descriptor_limit",
+        severity: "error",
+        message: `file descriptor soft limit ${fd.softLimit} is below estimated peak need ${fd.estimatedNeed} (headroom ${fd.headroom}) — raise with \`ulimit -n\` or launchd Limits`,
+      });
+    } else if (fd.softLimit < fd.estimatedNeed * 2) {
+      findings.push({
+        code: "file_descriptor_limit",
+        severity: "warn",
+        message: `file descriptor soft limit ${fd.softLimit} leaves thin headroom over estimated peak need ${fd.estimatedNeed} (headroom ${fd.headroom}) — consider raising with \`ulimit -n\``,
+      });
+    }
+  }
+
+  // Realtime health — CDC outbox backlog/retention + live fan-out pressure.
+  // Thresholds from the Realtime plan's operations section; all probes are
+  // injectable so tests never need a bound bridge.
+  if (options.detectRealtimeMetrics !== undefined || manifest !== undefined) {
+    const rt =
+      options.detectRealtimeMetrics !== undefined
+        ? await options.detectRealtimeMetrics()
+        : await (async () => {
+            const { realtimeMetrics } = await import("../kernel/realtime-bind.ts");
+            return realtimeMetrics() as Promise<RealtimeMetricsSnapshot | null>;
+          })();
+    if (rt !== null) {
+      const outbox = rt.outbox;
+      if (!("unavailable" in outbox)) {
+        if (outbox.pending > OUTBOX_BACKLOG_ERROR) {
+          findings.push({
+            code: "cdc_outbox_backlog",
+            severity: "error",
+            message: `CDC outbox backlog ${outbox.pending} rows exceeds ${OUTBOX_BACKLOG_ERROR} — poller stalled or consumers down; check \`oke_cdc_outbox\` and runner metrics`,
+          });
+        } else if (outbox.pending > OUTBOX_BACKLOG_WARN) {
+          findings.push({
+            code: "cdc_outbox_backlog",
+            severity: "warn",
+            message: `CDC outbox backlog ${outbox.pending} rows exceeds ${OUTBOX_BACKLOG_WARN} — watch the poller drain rate`,
+          });
+        }
+        if (outbox.dispatchedOverCap > 0) {
+          findings.push({
+            code: "cdc_outbox_retention",
+            severity: "warn",
+            message: `${outbox.dispatchedOverCap} delivered outbox rows exceed the maxCount retention cap — pruner may be stuck; verify \`${OUTBOX_PRUNE_INTERVAL_MS}ms\` prune cadence`,
+          });
+        }
+      }
+      if (rt.subscribers > LIVE_SUBSCRIBER_SOFT_CAP) {
+        findings.push({
+          code: "live_subscriber_pressure",
+          severity: "warn",
+          message: `${rt.subscribers} active live subscribers exceeds the ~150 fan-out ceiling documented for v1 — expect rising p99 latency; identity dedup lands in Round 2+`,
+        });
+      }
+      const saturated = rt.queueDepth >= LIVE_FANOUT_QUEUE_WARN_DEPTH && rt.fanout.eventsShed > 0;
+      if (saturated) {
+        findings.push({
+          code: "live_fanout_queue_saturated",
+          severity: "error",
+          message: `fan-out queue depth ${rt.queueDepth} at cap with ${rt.fanout.eventsShed} events shed — subscribers are losing events now; raise pool concurrency or shed rate first`,
+        });
+      } else if (rt.queueDepth >= LIVE_FANOUT_QUEUE_WARN_DEPTH) {
+        findings.push({
+          code: "live_fanout_queue_saturated",
+          severity: "warn",
+          message: `fan-out queue depth ${rt.queueDepth} near cap (${rt.fanout.checksRun} checks in flight historically) — headroom thin before shedding begins`,
+        });
+      }
+    }
+  }
+
   const current =
     options.currentSchemaFingerprint !== undefined
       ? options.currentSchemaFingerprint

package/src/client/create.ts

@@ -24,6 +24,7 @@ import {
 } from "./live.ts";
 import type {
   Client,
+  ClientHeaders,
   ClientLive,
   ClientOptions,
   ClientRouteMap,
@@ -97,11 +98,13 @@ function buildClient(url: string, opts: ClientOptions = {}): Client {
   const routes = opts.routes ?? flattenRoutes(opts.$routes);
   const live = flattenLiveRoutes(opts.$routes);
   const transport = createTransport(base, { ...opts, routes });
+  const perCallHeaders = createPerCallHeaders();
   return proxy(transport, [], {
     base,
     opts,
     liveBySignal: live.bySignal,
     liveByFlow: live.byFlow,
+    perCallHeaders,
   }) as Client;
 }
 
@@ -110,8 +113,48 @@ type ProxyCtx = {
   readonly opts: ClientOptions;
   readonly liveBySignal: LiveRouteTable;
   readonly liveByFlow: LiveByFlow;
+  /**
+   * Extra headers attached to the next transport call, then cleared — the
+   * `X-Oke-Mutation-Id` channel for optimistic dedupe (one-shot by design;
+   * a mutated header must not leak into unrelated calls).
+   */
+  perCallHeaders: PerCallHeaders;
 };
 
+/**
+ * One-shot header bag: set before a mutation call, drained after it. Safe
+ * across concurrent calls — each `run` stages its own merge and restores the
+ * previous stage on completion (LIFO), so nested/parallel runs don't clobber.
+ */
+function createPerCallHeaders(): PerCallHeaders {
+  let extra: ClientHeaders | undefined;
+  return {
+    /** Stage headers consumed by the next transport call only. */
+    run<T>(headers: ClientHeaders | undefined, fn: () => Promise<T>): Promise<T> {
+      const prev = extra;
+      extra = headers === undefined ? prev : mergeHeaders(prev, headers);
+      return fn().finally(() => {
+        extra = prev;
+      });
+    },
+    /** Transport-side read (invoked inside `invoke`). */
+    read(): ClientHeaders | undefined {
+      return extra;
+    },
+  };
+}
+
+/** Shallow-merge two header bags (later wins on key conflicts). */
+function mergeHeaders(base: ClientHeaders | undefined, over: ClientHeaders): ClientHeaders {
+  if (base === undefined) return over;
+  if (!Array.isArray(base) && !Array.isArray(over)) {
+    return { ...base, ...over };
+  }
+  const toEntries = (h: ClientHeaders): [string, string][] =>
+    Array.isArray(h) ? h : Object.entries(h);
+  return [...toEntries(base), ...toEntries(over)];
+}
+
 /**
  * Flatten `app.$routes` into the transport REST table (`unit.flow` → method/path).
  * Entries without both method and path are omitted (RPC fallback).
@@ -162,7 +205,7 @@ function proxy(transport: Transport, path: readonly string[], ctx: ProxyCtx): un
     }
     const unit = path[0]!;
     const flow = path.slice(1).join(".");
-    const result = await transport.call(`${unit}/${flow}`, input);
+    const result = await transport.call(`${unit}/${flow}`, input, ctx.perCallHeaders.read());
     return attachPager(result, invoke, input);
   };
   const call = (a?: unknown, b?: unknown) => {
@@ -181,6 +224,13 @@ function proxy(transport: Transport, path: readonly string[], ctx: ProxyCtx): un
   return new Proxy(call, {
     get(_target, prop, receiver) {
       if (typeof prop === "symbol") {
+        if (prop === TRANSPORT_BRAND) {
+          return {
+            base: ctx.base,
+            opts: ctx.opts,
+            perCallHeaders: ctx.perCallHeaders,
+          } satisfies TransportBag;
+        }
         return Reflect.get(_target, prop, receiver);
       }
       if (prop === "then") return undefined;
@@ -192,6 +242,50 @@ function proxy(transport: Transport, path: readonly string[], ctx: ProxyCtx): un
   });
 }
 
+/** Symbol brand exposing `{ base, opts }` from a client Proxy instance. */
+const TRANSPORT_BRAND = Symbol("oke.transportBag");
+
+/** One-shot header channel surface (see {@link createPerCallHeaders}). */
+export interface PerCallHeaders {
+  /**
+   * Run `fn` with `headers` merged onto its transport call (and any pager
+   * walks the result spawns). Restores prior state afterwards.
+   */
+  run<T>(headers: ClientHeaders | undefined, fn: () => Promise<T>): Promise<T>;
+  /** Transport-side read (invoked inside `invoke`). */
+  read(): ClientHeaders | undefined;
+}
+
+/** Transport surface carried by a built client — read via {@link transportOf}. */
+export interface TransportBag {
+  readonly base: string;
+  readonly opts: ClientOptions;
+  /** One-shot extra headers for a mutation (e.g. `X-Oke-Mutation-Id`). */
+  readonly perCallHeaders: PerCallHeaders;
+}
+
+/**
+ * Read the origin + client options from any {@link createClient} instance
+ * (symbol-brand channel on the root proxy — invisible to `get` traps beyond
+ * symbol reflection). The root proxy targets a callable, so both `function`
+ * and `object` receivers are accepted.
+ *
+ * @param api - Client instance or its typed surface
+ */
+export function transportOf(api: unknown): TransportBag | undefined {
+  if (typeof api !== "object" && typeof api !== "function") return undefined;
+  const bag = Reflect.get(api as object, TRANSPORT_BRAND) as unknown;
+  if (
+    bag !== null &&
+    typeof bag === "object" &&
+    "base" in bag &&
+    typeof (bag as { base: unknown }).base === "string"
+  ) {
+    return bag as TransportBag;
+  }
+  return undefined;
+}
+
 function makeLive(ctx: ProxyCtx): ClientLive {
   return ((signalOrName: unknown, inputOrHandlers: unknown, maybeHandlers?: unknown) => {
     const name =

package/src/client/index.ts

@@ -13,11 +13,18 @@ export {
   nextResubscribeDelay,
   pickLiveExposure,
 } from "./live.ts";
-export { createClient, flattenRoutes } from "./create.ts";
-export type { AppWithRoutes } from "./create.ts";
+export { createClient, flattenRoutes, transportOf } from "./create.ts";
+export type { AppWithRoutes, TransportBag } from "./create.ts";
 export { isErrorCode, isFail, isOk, isTransportError } from "./errors.ts";
 export { isPagerMeta, pagerLink } from "./pager.ts";
 export type { ClientPager, PagerLink } from "./pager.ts";
+export {
+  applyOptimisticPatch,
+  clearOptimisticPatch,
+  isStaleUpsert,
+  reduceLiveQueryRows,
+} from "./use-live-query.ts";
+export type { LiveQueryError, LiveQueryEvent } from "./use-live-query.ts";
 export { createTransport } from "./transport.ts";
 export type { Transport } from "./transport.ts";
 export type {

package/src/client/transport.ts

@@ -5,7 +5,7 @@
  * Optional `routes` map switches to REST (`method` + path template).
  */
 
-import type { ClientEnvelope, ClientFetch, ClientOptions } from "./types.ts";
+import type { ClientEnvelope, ClientFetch, ClientHeaders, ClientOptions } from "./types.ts";
 
 /** Internal transport handle. */
 export interface Transport {
@@ -14,8 +14,9 @@ export interface Transport {
    *
    * @param key - `unit/flow`
    * @param input - JSON body / path-param source
+   * @param headers - Per-call extra headers (merged after static ones)
    */
-  call(key: string, input: unknown): Promise<ClientEnvelope>;
+  call(key: string, input: unknown, headers?: ClientHeaders): Promise<ClientEnvelope>;
 }
 
 /**
@@ -31,14 +32,14 @@ export function createTransport(base: string, opts: ClientOptions = {}): Transpo
   const backoff = opts.retry?.backoff ?? 2;
 
   return {
-    async call(key, input) {
+    async call(key, input, callHeaders) {
       let refreshed = false;
       let attempt = 0;
       let delay = delay0;
 
       for (;;) {
         try {
-          const res = await once(base, key, input, opts, fetchFn);
+          const res = await once(base, key, input, opts, fetchFn, callHeaders);
           if (res.status === 401 && opts.auth?.refresh && !refreshed) {
             refreshed = true;
             await opts.auth.refresh();
@@ -88,6 +89,7 @@ async function once(
   input: unknown,
   opts: ClientOptions,
   fetchFn: ClientFetch,
+  callHeaders?: ClientHeaders,
 ): Promise<Response> {
   const route = opts.routes?.[key.replace("/", ".")];
   const { url, method, body } = route
@@ -101,6 +103,11 @@ async function once(
   } else if (extra) {
     for (const [k, v] of Object.entries(extra)) headers.set(k, v);
   }
+  if (Array.isArray(callHeaders)) {
+    for (const [k, v] of callHeaders) headers.set(k, v);
+  } else if (callHeaders) {
+    for (const [k, v] of Object.entries(callHeaders)) headers.set(k, v);
+  }
   if (body !== undefined && !headers.has("content-type")) {
     headers.set("content-type", "application/json");
   }