@humwick/server 0.130.1 → 0.130.2

package/node_modules/@humwick/kernel/timers-chaos-worker.ts

@@ -1,186 +0,0 @@
-import { relayEnv } from "@humwick/sdk/env";
-/**
- * Kernel Phase-2 P2-6 (#1409) — the TIMERS kill-9 chaos harness WORKER (spawned entrypoint). Not a test
- * file: it drives the REAL kernel timers firing + relay-dispatch outbox against a shared on-disk
- * SQLite/WAL database and can SIGKILL itself at a labeled boundary. `timers-chaos.integration.test.ts`
- * spawns it as real OS processes. It reuses the M4 chaos pattern (`workflow-chaos-worker.ts`) but points
- * it at the timers pilot specifically: fire → outbox → send-dedup + run-number stability under kill-9.
- *
- * ── The modeled send effect (why no `src` import) ─────────────────────────────────────────────────
- * The kernel must not import `src`, so the worker cannot call the real `sendMessageService`. It models
- * that effect with a harness-owned idempotency ledger keyed on the outbox row's run-scoped
- * `idempotencyKey` (`base:run:N`) — the EXACT key `sendMessageService` dedupes on. This faithfully
- * reproduces the delivery guard the real drainer relies on:
- *   - `timer_sent(idempotency_key PRIMARY KEY)` — INSERT OR IGNORE per send. A crash+re-drain re-presents
- *     the SAME key → absorbed. Proves the send was applied AT MOST once (the PRIMARY KEY makes a second
- *     row structurally impossible) AND at least once (`has(key)`).
- *   - `timer_send_attempts(idempotency_key)` — raw append-only counter, so the at-least-once re-drain is
- *     directly observable (a resumed drain re-issued the send, absorbed by the key).
- * The kernel firing itself (the outbox-row write) is the IN-TRANSACTION effect: exactly-once by
- * construction (it commits atomically with the fired flip + journal memo).
- */
-import { Database } from "bun:sqlite";
-import {
-  type Grant,
-  type InvocationIdentity,
-  type ModuleHost,
-  type WorkflowStepCommitFaults,
-  createSqliteModuleHost,
-  createRealClock,
-} from "./index";
-import { type RelaySend, timers } from "./timers.pilot";
-
-const DB_PATH = required("CHAOS_DB");
-const CMD = required("CHAOS_CMD"); // "start" | "recover"
-const KEY = process.env.CHAOS_KEY ?? "timer-1";
-const BOUNDARY = process.env.CHAOS_BOUNDARY ?? "none";
-const EVERY_MS = process.env.CHAOS_EVERY_MS ? Number(process.env.CHAOS_EVERY_MS) : undefined;
-const MAX_RUNS = process.env.CHAOS_MAX_RUNS ? Number(process.env.CHAOS_MAX_RUNS) : undefined;
-const LEASE_TTL_MS = Number(process.env.CHAOS_LEASE_TTL_MS ?? "200");
-const RECOVER_DEADLINE_MS = Number(process.env.CHAOS_RECOVER_DEADLINE_MS ?? "8000");
-const BUSY_TIMEOUT_MS = Number(process.env.CHAOS_BUSY_TIMEOUT_MS ?? "5000");
-
-const MIGRATOR: InvocationIdentity = { actor: "kernel:timers-migrator", grants: ["timers:admin"] as Grant[] };
-const DRAINER: InvocationIdentity = { actor: "kernel:timers-drainer", grants: ["timers:admin"] as Grant[] };
-const DRIVER: InvocationIdentity = { actor: "kernel:timers-driver", grants: [] as Grant[] };
-
-function required(name: string): string {
-  const value = relayEnv(process.env, name);
-  if (!value) throw new Error(`timers chaos worker requires env ${name}`);
-  return value;
-}
-
-function selfKill(): never {
-  process.kill(process.pid, "SIGKILL");
-  throw new Error("unreachable after SIGKILL");
-}
-
-/** Map a labeled fire-step COMMIT boundary to the fault hook that self-kills there. `before-journal` is
- *  the crux atomicity case: the outbox row (an in-tx entity write) exists in the buffer but the whole tx
- *  rolls back on kill → NO outbox row, NO fired flip → recover re-fires → no lost fire, no double. */
-function fireCommitFaultFor(boundary: string): WorkflowStepCommitFaults | undefined {
-  switch (boundary) {
-    case "before-commit": return { beforeCommit: selfKill };
-    case "before-journal": return { insideTxBeforeJournal: selfKill };
-    case "after-journal": return { afterJournalBeforeCommit: selfKill };
-    case "after-commit": return { afterCommit: selfKill };
-    default: return undefined; // "drain-after-send" / "none" arm nothing on the commit path.
-  }
-}
-
-/** Open the shared on-disk connection in WAL with a real busy_timeout, and install the harness-owned
- *  send ledger tables (idempotent). busy_timeout FIRST — the WAL switch + table creation need the write
- *  lock and many workers open the shared db at once. */
-function openDb(): Database {
-  const db = new Database(DB_PATH);
-  db.run(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
-  db.run("PRAGMA journal_mode = WAL");
-  db.run("CREATE TABLE IF NOT EXISTS timer_sent (idempotency_key TEXT PRIMARY KEY)");
-  db.run("CREATE TABLE IF NOT EXISTS timer_send_attempts (idempotency_key TEXT NOT NULL)");
-  return db;
-}
-
-function buildHost(db: Database): ModuleHost {
-  const host = createSqliteModuleHost({
-    db,
-    clock: createRealClock(),
-    leaseTtlMs: LEASE_TTL_MS,
-    busyTimeoutMs: BUSY_TIMEOUT_MS,
-    // Arm the labeled kill only on the timer's `fire` step commit.
-    workflowCommitFaults: (_instanceId, stepName) =>
-      stepName === "fire" ? fireCommitFaultFor(BOUNDARY) : undefined,
-  });
-  host.register(timers);
-  return host;
-}
-
-/** Seed the timer to fire via the P2-6 migrate op. `fireAt` is in the past (epoch 1000) so the first
- *  wakeup fires it immediately under the real clock; a recurring timer's next run is `now + everyMs`. */
-async function seed(host: ModuleHost): Promise<void> {
-  await host.invoke(timers, "migrate", MIGRATOR, {
-    id: KEY,
-    label: "chaos",
-    createdBy: "runner:agent-a",
-    fireAt: 1_000,
-    ...(EVERY_MS !== undefined ? { everyMs: EVERY_MS } : {}),
-    ...(MAX_RUNS !== undefined ? { maxRuns: MAX_RUNS } : {}),
-    message: {
-      from: "agent-a",
-      to: "agent-a",
-      body: "chaos wake",
-      identity: {
-        createdBy: "runner:agent-a",
-        createdByKind: "provider",
-        createdByScopes: ["message:send"],
-        createdByConstraints: { spawnRequestIds: ["sp-a"] },
-        createdAt: 500,
-      },
-    },
-  });
-}
-
-/** Model the send effect for one outbox row: a raw attempt + an idempotent apply, both keyed on the
- *  run-scoped `base:run:N` key (mirroring `sendMessageService`'s dedup). */
-function recordSend(db: Database, row: RelaySend): void {
-  db.query("INSERT INTO timer_send_attempts (idempotency_key) VALUES (?)").run(row.idempotencyKey);
-  db.query("INSERT OR IGNORE INTO timer_sent (idempotency_key) VALUES (?)").run(row.idempotencyKey);
-}
-
-/** Drain the outbox once: read pending rows, model the send (keyed dedup), optionally self-kill AFTER
- *  the send but BEFORE the `sent` mark (the at-least-once re-drain boundary), then mark sent. */
-async function drainOnce(host: ModuleHost, db: Database): Promise<number> {
-  const page = (await host.invoke(timers, "drainRelaySends", DRAINER, { limit: 100 })) as { items: RelaySend[] };
-  for (const row of page.items) {
-    recordSend(db, row);
-    if (BOUNDARY === "drain-after-send") selfKill();
-    await host.invoke(timers, "markRelaySendSent", DRAINER, { id: row.id });
-  }
-  return page.items.length;
-}
-
-const openInstances = (db: Database): number =>
-  (db.query("SELECT COUNT(*) AS c FROM workflow_instances WHERE status NOT IN ('done','failed','canceled')").get() as { c: number }).c;
-const pendingSends = (db: Database): number =>
-  (db.query(`SELECT COUNT(*) AS c FROM kernel_entity__timers__relaySend WHERE status = 'pending'`).get() as { c: number }).c;
-
-/** Fresh-process recovery: reclaim orphaned firing instances, fire due sleepers/waiters, and drain the
- *  outbox, until the timer's firing is terminal AND no pending outbox row remains (or the deadline). */
-async function recoverLoop(host: ModuleHost, db: Database): Promise<void> {
-  const driver = host.workflowDriver(timers, DRIVER);
-  const deadline = Date.now() + RECOVER_DEADLINE_MS;
-  while (Date.now() < deadline) {
-    await driver.recover(); // reclaim expired-lease running/pending instances
-    await driver.wakeup();   // fire due sleepers (arm→fire, recurring next runs) + satisfied waiters
-    await drainOnce(host, db);
-    if (openInstances(db) === 0 && pendingSends(db) === 0) return;
-    await Bun.sleep(EVERY_MS !== undefined ? Math.max(5, Math.floor(EVERY_MS / 2)) : 10);
-  }
-}
-
-async function main(): Promise<void> {
-  const db = openDb();
-  const host = buildHost(db);
-
-  if (CMD === "start") {
-    await seed(host);
-    // Drive the first firing (may self-kill at an armed fire-commit boundary and never return), then a
-    // drain (may self-kill at the drain boundary). For a recurring timer, keep pumping until its runs
-    // complete or an armed kill lands.
-    const driver = host.workflowDriver(timers, DRIVER);
-    const deadline = Date.now() + RECOVER_DEADLINE_MS;
-    do {
-      await driver.wakeup();
-      await drainOnce(host, db);
-      if (openInstances(db) === 0 && pendingSends(db) === 0) break;
-      await Bun.sleep(EVERY_MS !== undefined ? Math.max(5, Math.floor(EVERY_MS / 2)) : 10);
-    } while (Date.now() < deadline);
-    process.exit(0);
-  }
-  if (CMD === "recover") {
-    await recoverLoop(host, db);
-    process.exit(0);
-  }
-  throw new Error(`unknown CHAOS_CMD '${CMD}'`);
-}
-
-await main();