okengine 0.17.2 → 0.18.3

package/src/bench/g01-rls-stamp.bench.ts

@@ -0,0 +1,194 @@
+/**
+ * G1 — Serialized RLS stamp queue (PGlite).
+ *
+ * Measures stamp-frame serialization latency vs concurrency on ONE shared
+ * PGlite connection (`withRlsStampLock` physics). Two bind shapes:
+ *   • 3 binds — gate / user / scopes (no tenantId)
+ *   • 4 binds — gate / user / scopes + tenantId
+ *
+ * Expectation: roughly linear inverse degradation with N (the queue working
+ * as designed). Super-linear collapse is flagged as an issue.
+ *
+ * Run: bun test src/bench/g01-rls-stamp.bench.ts --timeout 120000
+ * Env: OKE_BENCH=1 (PGlite only — no Docker needed). OKE_BENCH_CAL=1 shrinks
+ * iterations for a calibration pass.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { connectPglite } from "../drivers/pglite.ts";
+import { installOkeRlsHelpers } from "../drivers/pg-rls.ts";
+import type { SqlConnection } from "../drivers/types.ts";
+import { createSqlStoreHandle, type SqlStoreHandle } from "../elements/store/sql-session.ts";
+import { DISCLAIMER, HARDWARE, summarize, writeArtifact } from "./lib/report.ts";
+
+const CAL = process.env.OKE_BENCH_CAL === "1";
+/** Concurrent callers per sweep point. */
+const CONCURRENCY = CAL ? [10, 50] : [10, 50, 100, 200, 500];
+/** Repeats per (N, shape) point — pooled into that point's samples. */
+const ROUNDS = CAL ? 1 : 3;
+
+const NOTES_DDL = `CREATE TABLE notes (
+  id text PRIMARY KEY,
+  owner text NOT NULL,
+  body text NOT NULL
+)`;
+
+const OWNER_POLICY = `CREATE POLICY owner_select ON notes
+  AS PERMISSIVE FOR SELECT TO public
+  USING (owner = oke.user())`;
+
+let conn: SqlConnection;
+/** 3-bind stamp (gate/user/scopes). */
+let asUser3: SqlStoreHandle;
+/** 4-bind stamp (+tenantId). */
+let asUser4: SqlStoreHandle;
+
+const PROBE_SQL = `SELECT oke.user() AS u, oke.tenant() AS t`;
+
+beforeAll(async () => {
+  conn = await connectPglite({ url: "memory://bench-g01-rls-stamp", role: "primary" });
+  await conn.exec(NOTES_DDL);
+  await conn.exec(`INSERT INTO notes VALUES ('a', 'alice', 'hi'), ('b', 'bob', 'yo')`);
+  await conn.exec(`ALTER TABLE notes ENABLE ROW LEVEL SECURITY`);
+  await installOkeRlsHelpers((sql) => conn.exec(sql));
+  await conn.exec(OWNER_POLICY);
+
+  const base = {
+    classifications: new Map(),
+    routedRole: "primary",
+    domainDdl: "off",
+  } as const;
+  asUser3 = createSqlStoreHandle("sql:app", {
+    ...base,
+    connection: conn,
+    rls: { gate: "member", userId: "alice", scopes: ["member"] },
+  });
+  asUser4 = createSqlStoreHandle("sql:app", {
+    ...base,
+    connection: conn,
+    rls: { gate: "member", userId: "bob", scopes: ["member"], tenantId: "acme" },
+  });
+}, 15_000);
+
+afterAll(async () => {
+  await conn.close();
+});
+
+interface PointResult {
+  readonly n: number;
+  readonly shape: "binds3" | "binds4";
+  readonly p50Ms: number;
+  readonly p99Ms: number;
+  readonly maxMs: number;
+  /** Batch throughput: calls / wall-clock seconds. */
+  readonly opsPerSec: number;
+}
+
+/**
+ * Fire `n` concurrent stamped probes; return per-call end-to-end latency.
+ */
+async function sweepPoint(
+  handle: SqlStoreHandle,
+  n: number,
+  rounds: number,
+): Promise<{ latencies: number[]; opsPerSec: number }> {
+  const latencies: number[] = [];
+  let wallMs = 0;
+  for (let r = 0; r < rounds; r += 1) {
+    const t0 = performance.now();
+    const times = await Promise.all(
+      Array.from({ length: n }, async () => {
+        const s = performance.now();
+        await handle.raw(PROBE_SQL);
+        return performance.now() - s;
+      }),
+    );
+    wallMs += performance.now() - t0;
+    latencies.push(...times);
+  }
+  return { latencies, opsPerSec: (n * rounds) / (wallMs / 1000) };
+}
+
+describe.skipIf(!process.env.OKE_BENCH)("G1 — RLS stamp serialization (pglite)", () => {
+  test("correctness sanity: concurrent identities do not leak across stamps", async () => {
+    const [u3, u4, u3again] = await Promise.all([
+      asUser3.raw(PROBE_SQL),
+      asUser4.raw(PROBE_SQL),
+      asUser3.raw(PROBE_SQL),
+    ]);
+    expect((u3[0] as { u: string }).u).toBe("alice");
+    expect((u4[0] as { u: string }).u).toBe("bob");
+    expect((u3[0] as { t: string | null }).t).toBeNull();
+    expect((u4[0] as { t: string }).t).toBe("acme");
+    expect((u3again[0] as { u: string }).u).toBe("alice");
+  });
+
+  test(
+    "sweep concurrency × bind shapes",
+    async () => {
+      const points: PointResult[] = [];
+      for (const n of CONCURRENCY) {
+        for (const [shape, handle] of [
+          ["binds3", asUser3],
+          ["binds4", asUser4],
+        ] as const) {
+          const { latencies, opsPerSec } = await sweepPoint(handle, n, ROUNDS);
+          const s = summarize(latencies);
+          const maxMs = Number(Math.max(...latencies).toFixed(3));
+          points.push({
+            n,
+            shape,
+            p50Ms: s.p50Ms,
+            p99Ms: s.p99Ms,
+            maxMs,
+            opsPerSec: Number(opsPerSec.toFixed(1)),
+          });
+        }
+      }
+
+      // Report + issue detection: super-linear collapse = queue physics broken.
+      const issues: string[] = [];
+      for (const shape of ["binds3", "binds4"] as const) {
+        const pts = points.filter((p) => p.shape === shape);
+        for (let i = 1; i < pts.length; i += 1) {
+          const prev = pts[i - 1]!;
+          const cur = pts[i]!;
+          const nRatio = cur.n / prev.n;
+          const p50Ratio = cur.p50Ms / Math.max(prev.p50Ms, 0.001);
+          if (p50Ratio > nRatio * 2.5) {
+            issues.push(
+              `super-linear collapse at N=${cur.n} (${shape}): p50 grew ${p50Ratio.toFixed(1)}x for ${nRatio.toFixed(1)}x concurrency`,
+            );
+          }
+        }
+      }
+
+      console.log("[G1] points:", JSON.stringify(points));
+      if (issues.length > 0) console.warn("[G1] ISSUES:", issues);
+
+      const metrics: Record<string, number> = {};
+      for (const p of points) {
+        metrics[`n${p.n}.${p.shape}.p50Ms`] = p.p50Ms;
+        metrics[`n${p.n}.${p.shape}.p99Ms`] = p.p99Ms;
+        metrics[`n${p.n}.${p.shape}.maxMs`] = p.maxMs;
+        metrics[`n${p.n}.${p.shape}.opsPerSec`] = p.opsPerSec;
+      }
+
+      const path = await writeArtifact({
+        group: "G1",
+        hardware: HARDWARE,
+        disclaimer: DISCLAIMER,
+        command: CAL
+          ? "OKE_BENCH=1 OKE_BENCH_CAL=1 bun test src/bench/g01-rls-stamp.bench.ts --timeout 120000"
+          : "OKE_BENCH=1 bun test src/bench/g01-rls-stamp.bench.ts --timeout 120000",
+        metrics,
+        issues,
+        fixes: [],
+        remeasured: null,
+      });
+      console.log(`[G1] artifact: ${path}`);
+      expect(points.length).toBe(CONCURRENCY.length * 2);
+    },
+    CAL ? 60_000 : 120_000,
+  );
+});

package/src/bench/g02-clock-per-tenant.bench.ts

@@ -0,0 +1,158 @@
+/**
+ * G2 — clock.perTenant lease contention (live Postgres via pgdog).
+ *
+ * Seeds ≥100 `oke_crons` rows named via {@link perTenantCronName}, then fires
+ * concurrent lease acquisitions through `acquireLease` (which runs
+ * `CLAIM_LEASE_SQL` = `FOR UPDATE SKIP LOCKED`). Two phases:
+ *
+ *   1. Correctness: C instances race for ONE name with a long lease —
+ *      exactly 1 winner, duplicate-fire count === 0.
+ *   2. Throughput: N tenants × K concurrent claimers sweep; histogram of
+ *      claim latencies + claims/sec.
+ *
+ * Run: OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=… bun test ./src/bench/g02-clock-per-tenant.bench.ts --timeout 180000
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { perTenantCronName } from "../elements/clock/declare.ts";
+import type { CronRow } from "../elements/clock/reconcile.ts";
+import { createPostgresCronStore, type PostgresCronSql } from "../drivers/clock-postgres.ts";
+import { LIVE_PG } from "./lib/infra.ts";
+import { DISCLAIMER, HARDWARE, percentile, writeArtifact } from "./lib/report.ts";
+
+const CAL = process.env.OKE_BENCH_CAL === "1";
+const TENANTS = CAL ? 20 : 120;
+const CLAIMERS_PER_TENANT = CAL ? [4] : [4, 16];
+const CONTENTION_LEASE_MS = CAL ? 5_000 : 30_000;
+/** Prefix so we only ever touch our own rows on cleanup. */
+const TEMPLATE = "bench-g2-cron";
+
+let store: Awaited<ReturnType<typeof createPostgresCronStore>>;
+
+function seedRow(name: string): CronRow {
+  return {
+    name,
+    declaredEvery: "1h",
+    effectiveEvery: "1h",
+    timezone: "UTC",
+    overridable: false,
+    status: "active",
+  };
+}
+
+beforeAll(async () => {
+  if (!LIVE_PG) throw new Error("G2 needs live Postgres: set OKE_TEST_POSTGRES=1 + DATABASE_URL");
+  store = await createPostgresCronStore({ url: LIVE_PG });
+}, 15_000);
+
+afterAll(async () => {
+  if (!store) return;
+  // Delete only rows this bench seeded.
+  await store.sql.exec(`DELETE FROM oke_crons WHERE name LIKE ?`, [`${TEMPLATE}#%`]);
+  await store.close();
+});
+
+describe.skipIf(!process.env.OKE_BENCH || !LIVE_PG)("G2 — perTenant lease contention", () => {
+  test("correctness: concurrent claimers → exactly one holder, zero duplicate fires", async () => {
+    const name = perTenantCronName(TEMPLATE, "dupe-check");
+    const existing = await store.get(name);
+    if (!existing) await store.put(seedRow(name));
+
+    const C = 24;
+    const results = await Promise.all(
+      Array.from({ length: C }, (_, i) =>
+        store.acquireLease(name, `g2-instance-${i}`, Date.now(), CONTENTION_LEASE_MS),
+      ),
+    );
+    const winners = results.filter(Boolean).length;
+    expect(winners).toBe(1);
+
+    const row = await store.get(name);
+    expect(row?.leaderInstanceId ?? (await rawLeader())).toBeTruthy();
+  }, 60_000);
+
+  test(
+    "throughput sweep: tenants × concurrent claimers",
+    async () => {
+      const names = Array.from({ length: TENANTS }, (_, i) => perTenantCronName(TEMPLATE, `t${i}`));
+      // Seed (idempotent).
+      const have = new Set((await store.list()).map((r) => r.name));
+      for (const n of names) if (!have.has(n)) await store.put(seedRow(n));
+
+      const metrics: Record<string, number> = {};
+      let totalDuplicateFires = 0;
+      const allLatencies: number[] = [];
+
+      for (const k of CLAIMERS_PER_TENANT) {
+        const latencies: number[] = [];
+        let wins = 0;
+        const t0 = performance.now();
+        // Wave: every tenant contested by k claimers at once.
+        const rounds = CAL ? 1 : 3;
+        for (let r = 0; r < rounds; r += 1) {
+          await Promise.all(
+            names.flatMap((name) =>
+              Array.from({ length: k }, (_, i) => {
+                const s = performance.now();
+                return store
+                  .acquireLease(name, `g2-sweep-${r}-${i}`, Date.now(), 60_000)
+                  .then((ok) => {
+                    latencies.push(performance.now() - s);
+                    if (ok) wins += 1;
+                  });
+              }),
+            ),
+          );
+        }
+        const wallS = (performance.now() - t0) / 1000;
+        // Each (tenant, round) must elect exactly one holder per lease window.
+        const expectedWins = names.length * rounds;
+        const duplicateFires = Math.max(0, wins - expectedWins);
+        totalDuplicateFires += duplicateFires;
+
+        const p50 = percentile(latencies, 50);
+        const p99 = percentile(latencies, 99);
+        metrics[`k${k}.claims`] = latencies.length;
+        metrics[`k${k}.p50Ms`] = Number(p50.toFixed(3));
+        metrics[`k${k}.p99Ms`] = Number(p99.toFixed(3));
+        metrics[`k${k}.maxMs`] = Number(Math.max(...latencies).toFixed(3));
+        metrics[`k${k}.claimsPerSec`] = Number((latencies.length / wallS).toFixed(1));
+        metrics[`k${k}.duplicateFires`] = duplicateFires;
+        allLatencies.push(...latencies);
+      }
+
+      console.log("[G2] metrics:", JSON.stringify(metrics));
+
+      const issues: string[] = [];
+      if (totalDuplicateFires > 0) {
+        issues.push(
+          `duplicate lease fires observed: ${totalDuplicateFires} (CLAIM_LEASE_SQL correctness)`,
+        );
+      }
+
+      const path = await writeArtifact({
+        group: "G2",
+        hardware: HARDWARE,
+        disclaimer: DISCLAIMER,
+        command:
+          "OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=$DATABASE_URL bun test ./src/bench/g02-clock-per-tenant.bench.ts --timeout 180000",
+        metrics,
+        issues,
+        fixes: [],
+        remeasured: null,
+      });
+      console.log(`[G2] artifact: ${path}`);
+      expect(totalDuplicateFires).toBe(0);
+    },
+    CAL ? 90_000 : 180_000,
+  );
+});
+
+/** Fallback leader read when the CronRow mapping hides the lease columns. */
+async function rawLeader(): Promise<string | null> {
+  const sql: PostgresCronSql = store.sql;
+  const rows = await sql.query(`SELECT locked_by FROM oke_crons WHERE name = ?`, [
+    perTenantCronName(TEMPLATE, "dupe-check"),
+  ]);
+  return (rows[0] as { locked_by?: string | null } | undefined)?.locked_by ?? null;
+}

package/src/bench/g03-signal-once.bench.ts

@@ -0,0 +1,157 @@
+/**
+ * G3a — competing once-consumer sweep against the live postgres signal driver.
+ *
+ * Opens a real transactional-outbox bus (`openPostgresSignal`) on live
+ * Postgres (pgdog), subscribes N competing consumers on one `once` signal,
+ * emits M messages, and measures delivery throughput + `throughputPerSec()`
+ * from `bus.inspect()` for N ∈ {1, 2, 4, 8, 16}.
+ *
+ * Honesty note: Bun.SQL has no LISTEN/NOTIFY API, so wakeups are fanned out
+ * in-process by `lib/signal-pg.ts` (real SQL outbox + real claims; only the
+ * cross-process wakeup path is not exercised).
+ *
+ * Run: OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=… bun test ./src/bench/g03-signal-once.bench.ts --timeout 180000
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { openPostgresSignal } from "../drivers/signal-postgres.ts";
+import type { SignalBus } from "../drivers/signal-types.ts";
+import { signal } from "../elements/signal/declare.ts";
+import { LIVE_PG } from "./lib/infra.ts";
+import { createBunSignalSql } from "./lib/signal-pg.ts";
+import { DISCLAIMER, HARDWARE, writeArtifact } from "./lib/report.ts";
+
+const CAL = process.env.OKE_BENCH_CAL === "1";
+const SIGNAL_NAME = "g3-once";
+const MESSAGES_PER_LEVEL = CAL ? 50 : 200;
+const LEVELS = CAL ? [1, 4] : [1, 2, 4, 8, 16];
+const DRAIN_TIMEOUT_MS = 30_000;
+
+let sql: ReturnType<typeof createBunSignalSql>;
+let bus: SignalBus;
+
+beforeAll(async () => {
+  if (!LIVE_PG) throw new Error("G3 needs live Postgres: set OKE_TEST_POSTGRES=1 + DATABASE_URL");
+  sql = createBunSignalSql(LIVE_PG);
+  const decl = signal(SIGNAL_NAME, { delivery: "once", retries: 2, deadLetter: true });
+  bus = await openPostgresSignal({
+    signals: new Map([[decl.name, decl]]),
+    sql,
+    leaseMs: 5_000,
+  });
+}, 20_000);
+
+afterAll(async () => {
+  try {
+    await sql.exec(`DELETE FROM oke_signal_messages WHERE signal LIKE 'g3-%'`);
+  } catch {
+    /* best-effort cleanup */
+  }
+  try {
+    await bus?.close();
+  } catch {
+    /* already closed */
+  }
+  try {
+    await sql.close();
+  } catch {
+    /* best-effort */
+  }
+});
+
+describe.skipIf(!process.env.OKE_BENCH || !LIVE_PG)("G3a — once-consumer sweep", () => {
+  test(`${MESSAGES_PER_LEVEL} msgs × competing consumers ${LEVELS.join("/")} — throughput + inspect()`, async () => {
+    const perLevel: Array<{
+      consumers: number;
+      delivered: number;
+      msgsPerSec: number;
+      wallMs: number;
+      throughputPerSec: number;
+    }> = [];
+    const issues: string[] = [];
+
+    for (const n of LEVELS) {
+      // Fresh consumer cohort per level.
+      const received: number[] = [];
+      const unsubs: Array<() => void | Promise<void>> = [];
+      for (let i = 0; i < n; i++) {
+        const id = `g3-c${n}-${i}`;
+        const unsub = await bus.subscribe(SIGNAL_NAME, id, async () => {
+          received.push(1);
+        });
+        unsubs.push(unsub);
+      }
+
+      const t0 = performance.now();
+      // One emit == one outbox message; fire MESSAGES_PER_LEVEL of them
+      // (small concurrent batches), then closed-loop drain until all are
+      // delivered or timeout.
+      const BATCH = 25;
+      for (let i = 0; i < MESSAGES_PER_LEVEL; i += BATCH) {
+        await Promise.all(
+          Array.from({ length: Math.min(BATCH, MESSAGES_PER_LEVEL - i) }, (_, j) =>
+            bus.emit(SIGNAL_NAME, { level: n, seq: i + j }),
+          ),
+        );
+      }
+
+      // Closed-loop drain until every message is delivered (or timeout).
+      const deadline = Date.now() + DRAIN_TIMEOUT_MS;
+      while (received.length < MESSAGES_PER_LEVEL && Date.now() < deadline) {
+        await bus.drain();
+        await Bun.sleep(2);
+      }
+      const wallMs = performance.now() - t0;
+
+      const stats = (await bus.inspect()).find((s) => s.signal === SIGNAL_NAME);
+      perLevel.push({
+        consumers: n,
+        delivered: received.length,
+        msgsPerSec: Number((received.length / (wallMs / 1000)).toFixed(1)),
+        wallMs: Number(wallMs.toFixed(0)),
+        throughputPerSec: stats?.throughputPerSec ?? -1,
+      });
+
+      for (const u of unsubs) await u();
+
+      if (received.length !== MESSAGES_PER_LEVEL) {
+        issues.push(
+          `consumers=${n}: delivered ${received.length}/${MESSAGES_PER_LEVEL} within ${DRAIN_TIMEOUT_MS}ms`,
+        );
+      }
+      // Exactly-once across competing consumers.
+      expect(received.length).toBeLessThanOrEqual(MESSAGES_PER_LEVEL);
+    }
+
+    const metrics: Record<string, number> = {};
+    for (const l of perLevel) {
+      metrics[`consumers${l.consumers}MsgsPerSec`] = l.msgsPerSec;
+      metrics[`consumers${l.consumers}ThroughputInspect`] = l.throughputPerSec;
+      metrics[`consumers${l.consumers}WallMs`] = l.wallMs;
+    }
+    metrics.totalDelivered = perLevel.reduce((a, l) => a + l.delivered, 0);
+    // Super-linear collapse check: 16 consumers should retain ≥25% of the
+    // 1-consumer rate (queue physics allow some degradation, not collapse).
+    const base = perLevel.find((l) => l.consumers === LEVELS[0])?.msgsPerSec ?? 0;
+    const top = perLevel[perLevel.length - 1]?.msgsPerSec ?? 0;
+    if (base > 0 && top > 0 && top < base * 0.25) {
+      issues.push(`super-linear collapse: ${LEVELS[0]}c=${base}/s vs ${top}c=${top}/s`);
+    }
+
+    console.log("[G3a] per-level:", JSON.stringify(perLevel));
+    const path = await writeArtifact({
+      group: "G3a-signal-once",
+      hardware: HARDWARE,
+      disclaimer: DISCLAIMER,
+      command:
+        "OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=$DATABASE_URL bun test ./src/bench/g03-signal-once.bench.ts --timeout 180000",
+      metrics,
+      issues,
+      fixes: [],
+      remeasured: null,
+    });
+    console.log(`[G3a] artifact: ${path}`);
+
+    expect(metrics.totalDelivered!).toBeGreaterThan(0);
+  }, 150_000);
+});