okengine 0.17.2 → 0.18.3

package/src/bench/g14-graceful-shutdown.bench.ts

@@ -0,0 +1,244 @@
+/**
+ * G14 — graceful shutdown under load.
+ *
+ * Extends the `instances-chaos-child.ts` graceful-hold pattern through
+ * `load-child shutdown-test` semantics, but with the REAL app: a full
+ * `load-child serve` boot installs `installGracefulShutdown`
+ * (`src/kernel/graceful-shutdown.ts`) over its server handle.
+ *
+ * Scenario: 220 held SSE subscribers (`load-child flood-sse`) + sustained
+ * HTTP load mixing `/ping` and sharedSqlConn RLS stamp writes. Then a FROZEN
+ * in-flight batch of 50 RLS stamp writes is launched, SIGTERM lands while
+ * they are mid-handler, and their outcomes classify the drain exactly:
+ *   - completed vs dropped requests,
+ *   - time-to-exit,
+ *   - pending sharedSqlConn stamp ops completing AFTER the signal (drain).
+ *
+ * No request is initiated after SIGTERM, so every error is a genuinely
+ * dropped in-flight request — no new-connection-refusal noise.
+ *
+ * Run: OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=… OKE_TEST_REDIS_URL=… bun test ./src/bench/g14-graceful-shutdown.bench.ts --timeout 120000
+ */
+
+import { describe, expect, test } from "bun:test";
+import { LIVE_PG } from "./lib/infra.ts";
+import { DISCLAIMER, HARDWARE, writeArtifact } from "./lib/report.ts";
+
+const CAL = process.env.OKE_BENCH_CAL === "1";
+const PORT = 6679;
+const BASE = `http://127.0.0.1:${PORT}`;
+const SSE_COUNT = CAL ? 60 : 220;
+const PING_WORKERS = 20;
+const RLS_WORKERS = 40;
+const INFLIGHT_BATCH = CAL ? 20 : 50;
+const WARM_MS = CAL ? 2_000 : 4_000;
+const LOAD_MS = CAL ? 2_000 : 4_000;
+
+type Server = Bun.Subprocess<"ignore", "pipe", "ignore">;
+
+function spawnServe(): Server {
+  return Bun.spawn(["bun", "run", "src/bench/load-child.ts", "serve", String(PORT)], {
+    stdout: "pipe",
+    stderr: "ignore",
+    env: process.env,
+  });
+}
+
+function spawnFloodSse(count: number): Bun.Subprocess<"ignore", "pipe", "ignore"> {
+  return Bun.spawn(
+    ["bun", "run", "src/bench/load-child.ts", "flood-sse", BASE, "bench-live", String(count)],
+    { stdout: "pipe", stderr: "ignore", env: process.env },
+  );
+}
+
+async function readReadyPid(proc: Server): Promise<number> {
+  const reader = proc.stdout.getReader();
+  const dec = new TextDecoder();
+  let buf = "";
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) throw new Error("server died before ready");
+    buf += dec.decode(value, { stream: true });
+    const nl = buf.indexOf("\n");
+    if (nl >= 0) {
+      const line = buf.slice(0, nl).trim();
+      if (line.startsWith("{")) {
+        void (async () => {
+          try {
+            for (;;) {
+              const r = await reader.read();
+              if (r.done) break;
+            }
+          } catch {
+            /* closed */
+          }
+        })();
+        return (JSON.parse(line) as { pid: number }).pid;
+      }
+    }
+  }
+}
+
+describe.skipIf(!process.env.OKE_BENCH || !LIVE_PG)("G14 — graceful shutdown under load", () => {
+  test(
+    `${SSE_COUNT} SSE + ${PING_WORKERS + RLS_WORKERS} workers → frozen in-flight batch → SIGTERM → drain`,
+    async () => {
+      const server = spawnServe();
+      const pid = await readReadyPid(server);
+      console.log(`[G14] server ready pid=${pid}`);
+      await Bun.sleep(WARM_MS);
+
+      // Hold SSE subscribers in a separate child so the test process only
+      // runs the closing HTTP load.
+      const flood = spawnFloodSse(SSE_COUNT);
+      await Bun.sleep(1_500); // let subscribers attach
+
+      let stopLoad = false;
+      const c = { ping: 0, rls: 0, errorsBeforeSignal: 0 };
+
+      const pingLoop = async (): Promise<void> => {
+        while (!stopLoad) {
+          try {
+            const res = await fetch(`${BASE}/ping`);
+            await res.arrayBuffer();
+            if (!res.ok) c.errorsBeforeSignal++;
+            else c.ping++;
+          } catch {
+            if (!stopLoad) c.errorsBeforeSignal++;
+          }
+        }
+      };
+      const rlsLoop = async (): Promise<void> => {
+        while (!stopLoad) {
+          try {
+            const res = await fetch(`${BASE}/_/bench/rls`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: JSON.stringify({ tenantId: `g14-${Math.floor(Math.random() * 8)}`, n: 1 }),
+            });
+            await res.arrayBuffer();
+            if (!res.ok) c.errorsBeforeSignal++;
+            else c.rls++;
+          } catch {
+            if (!stopLoad) c.errorsBeforeSignal++;
+          }
+        }
+      };
+
+      const running = Promise.all([
+        ...Array.from({ length: PING_WORKERS }, pingLoop),
+        ...Array.from({ length: RLS_WORKERS }, rlsLoop),
+      ]);
+
+      await Bun.sleep(LOAD_MS);
+      // Freeze the sustained load; wait for its last requests to settle so
+      // nothing except the frozen batch is in flight at signal time.
+      stopLoad = true;
+      await running;
+      expect(c.errorsBeforeSignal).toBe(0);
+
+      // Frozen in-flight batch — multi-roundtrip sharedSqlConn stamp writes.
+      // SIGTERM lands while all of these are mid-handler.
+      let batchSettled = false;
+      let batchDrained = 0;
+      let batchDropped = 0;
+      const batchErrors: string[] = [];
+      const batch = Promise.all(
+        Array.from({ length: INFLIGHT_BATCH }, async () => {
+          try {
+            const res = await fetch(`${BASE}/_/bench/rls`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: JSON.stringify({ tenantId: "g14-inflight", n: 1 }),
+            });
+            await res.arrayBuffer();
+            if (res.ok && !batchSettled) batchDrained++;
+            else if (!batchSettled) batchDropped++;
+          } catch (err) {
+            if (!batchSettled) {
+              batchDropped++;
+              if (batchErrors.length < 3) batchErrors.push(String(err));
+            }
+          }
+        }),
+      );
+
+      await Bun.sleep(15); // batch is now mid-handler behind the RLS stamps
+      const tSignal = performance.now();
+      server.kill("SIGTERM");
+      await batch;
+      batchSettled = true;
+
+      const exitRace = await Promise.race([
+        server.exited.then(() => "exited" as const),
+        Bun.sleep(20_000).then(() => "timeout" as const),
+      ]);
+      const timeToExitMs =
+        exitRace === "exited" ? Number((performance.now() - tSignal).toFixed(1)) : -1;
+      if (exitRace === "timeout") {
+        server.kill("SIGKILL");
+        await server.exited.catch(() => {});
+      }
+
+      // Flood child exits (printing floodDone) once every SSE stream ends.
+      const floodExit = await Promise.race([
+        flood.exited.then(() => "closed" as const),
+        Bun.sleep(10_000).then(() => "timeout" as const),
+      ]);
+      if (floodExit === "timeout") flood.kill();
+
+      const metrics: Record<string, number> = {
+        sseSubscribers: SSE_COUNT,
+        sseClosedOnShutdown: floodExit === "closed" ? 1 : 0,
+        inflightBatchSize: INFLIGHT_BATCH,
+        requestsCompletedBeforeSignal: c.ping + c.rls,
+        pingCompletedBeforeSignal: c.ping,
+        rlsStampWritesBeforeSignal: c.rls,
+        inflightDrainedAfterSignal: batchDrained,
+        inflightDroppedAfterSignal: batchDropped,
+        droppedRequests: batchDropped,
+        timeToExitMs,
+      };
+
+      const issues: string[] = [
+        // Calibration-run history: the first pass classified post-SIGTERM
+        // *new* connection attempts ("unable to connect") as dropped
+        // in-flight work. Methodology was corrected to a frozen in-flight
+        // batch before the signal — measurement fix, not a product fix.
+        "note: first CAL pass used a load-loop that kept opening new requests during the SIGTERM window and miscounted connection refusals as dropped in-flight work; replaced with the frozen in-flight-batch method (no product change)",
+      ];
+      if (batchDropped > 0) {
+        issues.push(
+          `${batchDropped}/${INFLIGHT_BATCH} in-flight sharedSqlConn stamp writes dropped across SIGTERM — graceful shutdown did not fully drain: ${batchErrors.join("; ")}`,
+        );
+      }
+      if (timeToExitMs < 0) {
+        issues.push("server did not exit within 20s of SIGTERM");
+      } else if (timeToExitMs > 10_000) {
+        issues.push(`time-to-exit ${timeToExitMs}ms exceeds 10s`);
+      }
+      if (floodExit !== "closed")
+        issues.push("SSE subscribers not all closed within 10s of shutdown");
+
+      console.log("[G14] metrics:", JSON.stringify(metrics));
+      if (issues.length > 0) console.log("[G14] issues:", issues);
+      const path = await writeArtifact({
+        group: "G14-graceful-shutdown",
+        hardware: HARDWARE,
+        disclaimer: DISCLAIMER,
+        command:
+          "OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=$DATABASE_URL OKE_TEST_REDIS_URL=$OKE_TEST_REDIS_URL bun test ./src/bench/g14-graceful-shutdown.bench.ts --timeout 120000",
+        metrics,
+        issues,
+        fixes: [],
+        remeasured: null,
+      });
+      console.log(`[G14] artifact: ${path}`);
+
+      expect(c.ping + c.rls).toBeGreaterThan(100);
+      expect(c.rls).toBeGreaterThan(10);
+      expect(batchDrained).toBeGreaterThanOrEqual(INFLIGHT_BATCH - 2);
+    },
+    CAL ? 60_000 : 120_000,
+  );
+});

package/src/bench/g15-postgres-degradation.bench.ts

@@ -0,0 +1,264 @@
+/**
+ * G15 — Postgres degradation (in-process backpressure).
+ *
+ * Test-only driver wrapper honors `OKE_BENCH_PG_DELAY_MS`: every
+ * query/exec awaits `Bun.sleep(delay)` BEFORE hitting live Postgres
+ * (preferred over pgdog network delays for reproducibility). Live PG only —
+ * never PGlite.
+ *
+ * Open-loop arrivals hammer ONE sharedSqlConn through RLS-stamped sessions
+ * while SQL is stalled at 500ms/query. Watch: unbounded `withRlsStampLock`
+ * queue growth / OOM vs loud bounded errors.
+ *
+ * MANDATORY HONESTY NOTE (carried in the artifact): this is an IN-PROCESS
+ * BACKPRESSURE TEST ONLY — it does NOT simulate network jitter, disk
+ * contention, or pooler failure modes.
+ *
+ * Run: OKE_BENCH=1 OKE_BENCH_PG_DELAY_MS=500 OKE_TEST_POSTGRES=1 DATABASE_URL=… bun test ./src/bench/g15-postgres-degradation.bench.ts --timeout 300000
+ */
+
+import { describe, expect, test } from "bun:test";
+import { connectPostgres, postgresDriver } from "../drivers/postgres.ts";
+import type { SqlDriver, SqlConnection } from "../drivers/types.ts";
+import { sql } from "../elements/store/declare.ts";
+import { createSqlStoreHandle, type SqlStoreHandle } from "../elements/store/sql-session.ts";
+import { createStoreRuntime } from "../elements/store/runtime.ts";
+import { LIVE_PG } from "./lib/infra.ts";
+import { measureEventLoopLag } from "./lib/event-loop-lag.ts";
+import { DISCLAIMER, HARDWARE, percentile, writeArtifact } from "./lib/report.ts";
+
+const CAL = process.env.OKE_BENCH_CAL === "1";
+const DELAY_MS = Math.max(0, Number(process.env.OKE_BENCH_PG_DELAY_MS ?? "500"));
+const DURATION_S = CAL ? 12 : Math.max(10, Number(process.env.OKE_BENCH_G15_S ?? "30"));
+const ARRIVALS_PER_SEC = CAL ? 6 : Math.max(1, Number(process.env.OKE_BENCH_G15_RATE ?? "10"));
+const SAMPLE_MS = 250;
+
+/**
+ * Test-only driver wrapper: injects `Bun.sleep(DELAY_MS)` before every
+ * query/exec on connections this driver opens. Transaction pinning and
+ * role routing pass through untouched.
+ */
+function delayedPostgresDriver(delayMs: number): SqlDriver {
+  if (delayMs <= 0) return postgresDriver;
+  return {
+    id: "postgres",
+    facet: "sql",
+    connect: async (options): Promise<SqlConnection> => {
+      const conn = await connectPostgres(options);
+      const slowed: SqlConnection = {
+        driverId: conn.driverId,
+        role: conn.role,
+        async query(text, params = []) {
+          await Bun.sleep(delayMs);
+          return conn.query(text, params);
+        },
+        async exec(text, params = []) {
+          await Bun.sleep(delayMs);
+          return conn.exec(text, params);
+        },
+        transaction(fn) {
+          if (!conn.transaction) {
+            throw new Error("G15: underlying postgres connection lacks transaction()");
+          }
+          return conn.transaction(fn);
+        },
+        close() {
+          return conn.close();
+        },
+      };
+      return slowed;
+    },
+  };
+}
+
+describe.skipIf(!process.env.OKE_BENCH || !LIVE_PG)(
+  "G15 — Postgres degradation (in-process backpressure)",
+  () => {
+    test(
+      `open-loop ${ARRIVALS_PER_SEC}/s vs sharedSqlConn @ ${DELAY_MS}ms/query for ${DURATION_S}s`,
+      async () => {
+        const db = sql("g15degrade");
+        const runtime = createStoreRuntime({
+          drivers: { sql: delayedPostgresDriver(DELAY_MS) },
+          sql: { g15degrade: { name: "g15degrade", primary: { url: LIVE_PG! } } },
+          now: () => Date.now(),
+        });
+        runtime.register(db);
+        const maybeConn = await runtime.primarySql();
+        if (!maybeConn) throw new Error("G15: runtime.primarySql() returned undefined");
+        const c: SqlConnection = maybeConn;
+        // DDL is stamp-exempt — set up before the stall matters anyway.
+        await c.exec(`CREATE TABLE IF NOT EXISTS oke_bench_g15 (
+          id TEXT PRIMARY KEY, n BIGINT NOT NULL DEFAULT 0)`);
+        await c.exec(
+          `INSERT INTO oke_bench_g15 (id, n) VALUES ('g15-counter', 0)
+           ON CONFLICT (id) DO UPDATE SET n = 0`,
+        );
+
+        const IDENTITIES = [
+          undefined,
+          { gate: "member", userId: "g15-alice", scopes: ["member"] },
+          { gate: "member", userId: "g15-bob", scopes: ["member"] },
+        ] as const;
+        let handleIdx = 0;
+        function nextSession(): SqlStoreHandle {
+          const identity = IDENTITIES[handleIdx % IDENTITIES.length]!;
+          handleIdx++;
+          return createSqlStoreHandle(db.ref, {
+            connection: c,
+            classifications: new Map(),
+            routedRole: "primary",
+            domainDdl: "off",
+            ...(identity ? { rls: identity } : {}),
+          });
+        }
+
+        let inflight = 0;
+        let maxInflight = 0;
+        let completed = 0;
+        let failed = 0;
+        const failureSamples: string[] = [];
+
+        async function oneOp(): Promise<void> {
+          inflight++;
+          if (inflight > maxInflight) maxInflight = inflight;
+          try {
+            const session = nextSession();
+            const rows = await session
+              .select()
+              .from({ name: "oke_bench_g15" })
+              .where({ id: "g15-counter" })
+              .limit(1);
+            const cur = Number((rows[0] as { n?: number } | undefined)?.n ?? 0);
+            await session
+              .update({ name: "oke_bench_g15" })
+              .set({ n: cur + 1 })
+              .where({ id: "g15-counter" });
+            completed++;
+          } catch (err) {
+            failed++;
+            if (failureSamples.length < 5) {
+              failureSamples.push(err instanceof Error ? err.message : String(err));
+            }
+          } finally {
+            inflight--;
+          }
+        }
+
+        // Background samplers.
+        const rssSamples: { tMs: number; rssMb: number }[] = [];
+        const inflightSamples: { tMs: number; inflight: number }[] = [];
+        const t0 = performance.now();
+        const samplerId = setInterval(() => {
+          const tMs = performance.now() - t0;
+          rssSamples.push({ tMs, rssMb: process.memoryUsage.rss() / 1024 / 1024 });
+          inflightSamples.push({ tMs, inflight });
+        }, SAMPLE_MS);
+        const lag = measureEventLoopLag(100);
+
+        // Open-loop arrivals: fixed cadence regardless of completion.
+        const arrivalId = setInterval(() => void oneOp(), 1000 / ARRIVALS_PER_SEC);
+        await Bun.sleep(DURATION_S * 1000);
+        clearInterval(arrivalId);
+        clearInterval(samplerId);
+        lag.stop();
+
+        // Let the CURRENTLY ARRIVING tick settle, then freeze the picture.
+        await Bun.sleep(50);
+        const finalInflight = inflight;
+
+        const rssStart = rssSamples[0]?.rssMb ?? 0;
+        const rssEnd = rssSamples.at(-1)?.rssMb ?? 0;
+        // Least-squares MB/min over all samples.
+        const n = rssSamples.length;
+        let rssSlope = 0;
+        if (n >= 2) {
+          const sx = rssSamples.reduce((a, s) => a + s.tMs, 0);
+          const sy = rssSamples.reduce((a, s) => a + s.rssMb, 0);
+          const sxy = rssSamples.reduce((a, s) => a + s.tMs * s.rssMb, 0);
+          const sxx = rssSamples.reduce((a, s) => a + s.tMs * s.tMs, 0);
+          rssSlope = ((n * sxy - sx * sy) / (n * sxx - sx * sx || 1)) * 60_000;
+        }
+
+        const lags = lag.lags();
+        const metrics: Record<string, number> = {
+          pgDelayMs: DELAY_MS,
+          durationS: DURATION_S,
+          arrivalsPerSec: ARRIVALS_PER_SEC,
+          completedOps: completed,
+          failedOps: failed,
+          errorRatePct: Number(((failed / (completed + failed || 1)) * 100).toFixed(3)),
+          opsPerSec: Number((completed / DURATION_S).toFixed(2)),
+          maxInflight: maxInflight,
+          finalInflight: finalInflight,
+          avgInflight: Number(
+            (
+              inflightSamples.reduce((a, s) => a + s.inflight, 0) / (inflightSamples.length || 1)
+            ).toFixed(1),
+          ),
+          rssStartMb: Number(rssStart.toFixed(1)),
+          rssEndMb: Number(rssEnd.toFixed(1)),
+          rssGrowthMb: Number((rssEnd - rssStart).toFixed(1)),
+          rssSlopeMbPerMin: Number(rssSlope.toFixed(2)),
+          lagP50Ms: Number(percentile(lags, 50).toFixed(2)),
+          lagP99Ms: Number(percentile(lags, 99).toFixed(2)),
+          lagMaxMs: lags.length > 0 ? Number(Math.max(...lags).toFixed(2)) : 0,
+        };
+
+        const issues: string[] = [];
+        if (failed > 0) {
+          issues.push(
+            `${failed} op(s) failed under stall — sample errors: ${failureSamples.join(" | ")}`,
+          );
+        }
+        if (metrics.rssSlopeMbPerMin! > 100 || metrics.rssGrowthMb! > 300) {
+          issues.push(
+            `RSS grew ${metrics.rssGrowthMb}MB (${metrics.rssSlopeMbPerMin}MB/min) — OOM-shaped growth while the withRlsStampLock queue built up`,
+          );
+        }
+        if (metrics.lagP99Ms! > 250) {
+          issues.push(
+            `event-loop lag p99 ${metrics.lagP99Ms}ms — backpressure is leaking onto the loop`,
+          );
+        }
+        if (issues.length === 0 && finalInflight > ARRIVALS_PER_SEC) {
+          issues.push(
+            `note (expected physics): withRlsStampLock queue grew UNBOUNDED without any loud bounded error — ` +
+              `finalInflight=${finalInflight} at cutoff (arrivals ${ARRIVALS_PER_SEC}/s outpace serialized service ~${metrics.opsPerSec}/s at ${DELAY_MS}ms/query). ` +
+              `Memory stayed flat (+${metrics.rssGrowthMb}MB), so this is unbounded LATENCY exposure, not OOM — callers wait forever instead of failing fast`,
+          );
+        }
+
+        console.log("[G15] metrics:", JSON.stringify(metrics));
+        if (issues.length > 0) console.log("[G15] issues:", issues);
+
+        const path = await writeArtifact({
+          group: "G15-postgres-degradation",
+          hardware: HARDWARE,
+          disclaimer: DISCLAIMER,
+          command:
+            "OKE_BENCH=1 OKE_BENCH_PG_DELAY_MS=500 OKE_TEST_POSTGRES=1 DATABASE_URL=$DATABASE_URL bun test ./src/bench/g15-postgres-degradation.bench.ts --timeout 300000",
+          metrics,
+          issues: [
+            // Mandatory honesty note — in the artifact, not just the header.
+            "HONESTY NOTE: in-process backpressure test only — does not simulate network jitter, disk contention, or pooler failure modes",
+            ...issues,
+          ],
+          fixes: [],
+          remeasured: null,
+        });
+        console.log(`[G15] artifact: ${path}`);
+
+        expect(failed).toBe(0);
+        expect(completed).toBeGreaterThan(0);
+        // Backpressure must be observable: the queue builds instead of
+        // silently dropping work.
+        expect(maxInflight).toBeGreaterThan(ARRIVALS_PER_SEC);
+
+        // Do not drain the multi-minute queue — abandon it with the runtime.
+        await runtime.close().catch(() => {});
+      },
+      CAL ? 90_000 : 300_000,
+    );
+  },
+);