@humwick/server 0.0.0-stage → 0.130.0

package/src/services/kernel-timers-ingress.ts

@@ -0,0 +1,702 @@
+/**
+ * T-0 M-0b (#1409, T1160) — the durable INGRESS LOG + crash-recovering APPLY state machine for
+ * MESSAGE-timer intents (docs/kernel-timers-message-cutover.md §3.1). This is the reusable foundation
+ * the later authority flip sits on; it replaces the best-effort `void mirrorLegacyTimerCreate(...)`
+ * fire-and-forget seam with a durable intent that provably converges across kill -9.
+ *
+ * ── THE CONTRACT ──────────────────────────────────────────────────────────────────────────────────
+ * Every message-timer create/cancel becomes a durable INTENT:
+ *
+ *   accepted ──(applier claims lease)──► applying ──(apply target committed)──► applied
+ *      │                                     │
+ *      └──(superseded by a later intent /    └──(crash / apply error) ─► re-driven by the
+ *          tombstone-before-apply /               crash-recovering applier (idempotent re-apply)
+ *          malformed payload)──► terminal
+ *
+ *  • FULL PAYLOAD PERSISTED — a create intent stores the COMPLETE resolved `ScheduledTimer` snapshot
+ *    (not a hash), so an accepted create is fully reconstructable after a crash.
+ *  • ATOMIC INTENT BINDING (review finding #3) — `acceptCreate`/`acceptCancel` run the caller's
+ *    `bind()` (the legacy `scheduled_timers` insert / cancel) and the intent insert in ONE
+ *    `BEGIN IMMEDIATE` transaction: there is no state in which the legacy row exists without its
+ *    durable intent (the stranded-S2 gap), nor an intent without the legacy write (phantom-cancel).
+ *  • ACK-AT-APPLIED — accept resolves only once THIS intent is settled (`applied`, or `terminal` when
+ *    a concurrent later intent superseded it). If the inline drive cannot settle it (apply-target
+ *    error, cross-process contention) accept throws `TimerIngressPendingError`: the intent is durably
+ *    accepted and the applier worker + alarms own convergence — a normal return IS the ACK.
+ *  • DURABLE WATERMARK, TERMINAL-INCLUSIVE — `timer_ingress_watermark.applied_seq = W` guarantees
+ *    every log row with `seq <= W` is settled (`applied` OR `terminal`); a superseded/tombstoned/
+ *    malformed row can never pin the watermark.
+ *  • NEAR-DUE / OVERDUE ALARM — any unsettled intent whose `fire_at_hint` is inside the near-due
+ *    window, or that has sat unsettled past `overdueAfterMs`, is surfaced by `alarms()` (and warned
+ *    by the worker pass), so a near-due create can never sit unapplied and expire unfired SILENTLY;
+ *    the applier additionally orders its backlog by `fire_at_hint` so the nearest-due applies first.
+ *  • COMMAND TIMERS BYPASS THIS PATH ENTIRELY — commands stay 100% legacy (double-dispatch guard,
+ *    see `mirrorLegacyTimerCreate`). A command timer is structurally rejected at accept and the
+ *    WHOLE transaction (including the bound legacy write) rolls back: a command intent can never
+ *    exist in the log. Callers gate with `timerIngressEligible(timer)` and use the legacy-direct
+ *    path for commands.
+ *  • CRASH-SAFE, IDEMPOTENT WRITES — every transition is a single SQLite transaction; the apply
+ *    target must be idempotent (the production-shaped `kernelTimersApplyTarget` drives the kernel's
+ *    id-keyed `migrate` / `cancelMirror` ops, both idempotent by construction), so a re-drive after
+ *    a crash at ANY boundary converges without doubling. Malformed/partial payloads are ignored on
+ *    read: they settle `terminal('malformed')` (watermark advances; the applier never throws on
+ *    garbage), never applied.
+ *  • REQUEST IDENTITY / EXACTLY-ONCE-BY-KEY (#1409 hardening, finding 2) — `acceptCreate` REQUIRES a
+ *    caller-supplied `requestKey` (stable across retries of ONE logical request; `acceptCancel` takes
+ *    it optionally). The key is stored on the log row under a UNIQUE index: a retry after a lost ACK
+ *    (crash after settle/commit, before the caller heard back) finds the ORIGINAL intent and ACKs it
+ *    — `bind()` is NOT re-run, no second timer id / legacy row / entity / workflow is ever minted. A
+ *    concurrent duplicate accept loses the UNIQUE race, rolls its WHOLE transaction back (including
+ *    its bound legacy write), and converges on the winner's intent. SCOPED, NOT GLOBAL (T1175
+ *    finding 3): the stored key is namespaced per op — and for cancels per timer — via an
+ *    unambiguous JSON-array encoding (`["create",key]` / `["cancel",timerId,key]`), so two DISTINCT
+ *    logical requests sharing a raw key (a create and a cancel, or cancels of two timers) are
+ *    independent intents, never a false dedupe or a false reuse-rejection. Within one scope a key
+ *    stays one-per-logical-request; the pre-namespacing cross-op guards remain as corruption checks.
+ *  • MALFORMED-ROW QUARANTINE, KERNEL-PARITY (#1409 hardening, finding 3) — accept-time validation
+ *    enforces the SAME constraints the kernel `migrate` op enforces (integer times, `runCount >= 0`,
+ *    positive `maxRuns`/interval `everyMs`, string scopes, object payload/meta, …) AND — the T1175
+ *    drift-proof half — runs the snapshot through the kernel's ACTUAL schema objects: the `migrate`
+ *    op input plus the `fire` workflow input at migrate's own `initialRun = runCount + 1` derivation
+ *    (so `runCount = MAX_SAFE_INTEGER`, whose +1 is unrepresentable, is refused at accept, not
+ *    discovered as an eternal kernel rejection). A row the kernel would deterministically reject can
+ *    never enter the log and pin the watermark. On the applier's read path the payload must also
+ *    name the row's own `timer_id` — a mismatched payload can never arm or cancel ANOTHER timer
+ *    (quarantined `terminal('malformed')`). Belt-and-braces: a kernel apply that still rejects
+ *    deterministically — a typed `invalid_input` OpError, or a raw ZodError from ANY schema parse
+ *    inside the op path (both → `TimerIngressRejectedError`) — quarantines to `terminal('malformed')`
+ *    after `maxApplyAttempts` instead of retrying forever; TRANSIENT apply errors keep retrying +
+ *    alarming (missed-fire safety is untouched). A deterministically-invalid row can never pin the
+ *    watermark.
+ *
+ * ── SUPERSEDE / TOMBSTONE COLLAPSE (per-timer ordering) ───────────────────────────────────────────
+ * Intents are globally ordered by `seq` (AUTOINCREMENT ⇒ monotone, never reused). Per timer, only
+ * the LATEST intent needs applying: every earlier unsettled intent settles `terminal` ('tombstoned'
+ * when a later cancel exists, else 'superseded') WITHOUT applying — so a cancel that overtook its
+ * create collapses the pair to "no kernel timer" (no resurrection), and a superseded create never
+ * starts a second firing workflow. A live foreign lease on any of the timer's intents defers the
+ * whole timer to the next pass (never collapse under a mid-flight foreign apply). NOTE: payload
+ * convergence for a REPEATED create of the SAME id additionally needs the M-0e migrate-UPDATE fix
+ * (deferred; see docs §3.5) — the primitive guarantees single-workflow/no-double, not field rebase.
+ *
+ * ── APPLY FENCING (#1409 hardening, finding 1) — collapse alone is NOT enough ─────────────────────
+ * The lease only arbitrates CLAIMS; it cannot stop a live-but-lease-expired applier that already
+ * claimed and is paused MID-APPLY. Without a fence, that applier could resume `migrate` AFTER a newer
+ * cancel terminalized its create and advanced the watermark — arming a phantom timer no log row
+ * admits to. Every apply therefore carries a MONOTONIC FENCE = the intent's `seq` (per-timer strictly
+ * monotone, never reused) into the kernel ops, and the kernel keeps a durable per-timer
+ * last-applied-fence record (`applyFence` entity — the narrow owner-index/tombstone):
+ *   • a write whose fence is LOWER than the stored fence is REFUSED AT THE KERNEL WRITE (no entity
+ *     write, no workflow start, no event) — refusal is atomic with the kernel commit, so it holds
+ *     under a live slow applier, not just at claim time;
+ *   • `cancelMirror` on an ABSENT entity durably records its fence as the KERNEL TOMBSTONE — never a
+ *     silent no-op — so a stale lower-fence create is refused afterwards ("a cancel has already been
+ *     applied for this timerId at fence ≥ mine");
+ *   • equal fence = the SAME intent re-driven after a crash → proceeds idempotently (unchanged);
+ *   • a `fenced` refusal settles the intent `terminal` (tombstoned/superseded), NEVER `applied`.
+ * INVARIANT ENFORCED: a create terminalized by a later cancel can NEVER arm a kernel timer, even if
+ * a live, lease-expired applier resumes mid-apply. Unfenced `migrate`/`cancelMirror` calls (the
+ * pre-activation shadow mirror + P2-6 migration script, which run under LEGACY authority) bypass the
+ * fence record entirely; once activation routes traffic through THIS module, all applies are fenced.
+ *
+ * ── THE FENCE COVERS EVERY SIDE EFFECT (T1175 second pass, finding 1) ─────────────────────────────
+ * The read-check above refuses a fence KNOWN stale when the handler runs. The remaining hole was the
+ * race where BOTH appliers read the old fence: migrate used to commit its firing-workflow start in an
+ * own transaction DURING the handler, so a migrate that then LOST the fence CAS at flush rolled back
+ * its entity — but its already-committed workflow survived as an orphan keyed `idempotencyKey =
+ * timerId`, and a LATER legitimate create deduped onto it and fired at the ROLLED-BACK migrate's old
+ * deadline (the reviewer's requested-10000-fired-at-5000 repro). CLOSED STRUCTURALLY, option (a):
+ * `migrate`/`create` start the firing workflow with `enlist: "op-commit"` — the instance SEED
+ * executes INSIDE the op's single atomic SQLite transaction (entities → events → seeds), i.e. inside
+ * the very transaction whose applyFence version-CAS / PK-conflict arbitrates the race. Ordering
+ * argument: the fence write and the workflow seed cannot commit separately, so EITHER the op wins the
+ * CAS and commits {entity, event, fence, seed} together, OR it loses and commits NOTHING. Hence
+ *   (1) a fence-losing migrate leaves nothing that can fire — no entity, no fence write, and no
+ *       workflow instance (the seed rolled back with the CAS), and
+ *   (2) a later create can never adopt an orphan: an instance keyed to a timer id exists ONLY if the
+ *       op that seeded it committed, in which case that timer's entity + applyFence rows exist at
+ *       that op's fence — dedup can only ever land on a workflow whose deadline a COMMITTED apply
+ *       wrote. The workflow drive runs post-commit; a crash before it leaves a durable `pending`
+ *       instance the equal-fence re-drive / `recover()` converges (nothing new fires early — an
+ *       undriven seed only parks later).
+ *
+ */
+import { randomUUID } from "node:crypto";
+import { errMessage } from "@humwick/sdk";
+import {
+  acceptTimerIngressBootstrapCreateAsync,
+  acceptTimerIngressCancelAsync,
+  acceptTimerIngressCreateAsync,
+  claimTimerIngressIntentAsync,
+  getDb,
+  installTimerIngressSchema,
+  isTimerIngressRequestKeyConflict,
+  listTimerIngressIntents,
+  listTimerIngressUnsettledIntents,
+  releaseTimerIngressLeaseAfterFailureAsync,
+  settleTimerIngressAppliedAsync,
+  settleTimerIngressTerminalAsync,
+  timerIngressIntentByRequestKeyAsync,
+  timerIngressIntentsForAsync,
+  timerIngressUnsettledTimerIdsAsync,
+  timerIngressWatermark,
+  type ScheduledTimer,
+  type TimerIngressAcceptOutcome,
+  type TimerIngressCreateSource,
+  type TimerIngressIntentRow,
+  type TimerIngressOp,
+  type TimerIngressTerminalReason,
+} from "../db";
+import {
+  TimerIngressEligibilityError,
+  assertIngressCreateSnapshot,
+  parseCancelPayload,
+  parseCreatePayload,
+  type TimerIngressCancelPayload,
+} from "../db/timer-ingress-payload.ts";
+import { canonicalTimerIngressRequestKey } from "../db/timer-ingress-request-keys.ts";
+import { armWriterFault } from "../db/writer-fault-seam.ts";
+
+// The payload/eligibility layer (snapshot validation, payload parsing, the eligibility gate) lives
+// in `db/timer-ingress-payload.ts`; re-exported here so callers keep ONE import surface.
+export {
+  TimerIngressEligibilityError,
+  timerIngressEligible,
+  type TimerIngressCancelPayload,
+} from "../db/timer-ingress-payload.ts";
+
+// The `timer_ingress_log` / `timer_ingress_watermark` SQL — the row shape, the schema install, the
+// reads, and the four CAS transitions — lives in `src/db/timer-ingress.ts` (they are db tables, and
+// a registered writer op's body must be reachable from `db/write-ops.ts` without inverting the
+// db→services layering). Re-exported here so every existing caller keeps ONE import surface.
+export {
+  installTimerIngressSchema,
+  type TimerIngressIntentRow,
+  type TimerIngressOp,
+  type TimerIngressTerminalReason,
+} from "../db/timer-ingress.ts";
+
+/** The settled outcome of one apply-target invocation. `"fenced"` = the kernel REFUSED the write
+ *  because a strictly newer intent for this timer already applied (stale fence) — the effect did NOT
+ *  happen and the intent must settle `terminal`, never `applied`. */
+export type TimerIngressApplyDisposition = "applied" | "fenced";
+
+/** The idempotent apply seam. `applyCreate`/`applyCancel` MUST tolerate re-invocation for the same
+ *  intent (crash-recovering re-drive) — the production-shaped target maps onto the kernel's id-keyed
+ *  idempotent `migrate`/`cancelMirror` ops and threads the intent's `seq` as the apply FENCE (module
+ *  header). A TRANSIENT apply error (throw) leaves the intent durably unsettled for retry; a
+ *  DETERMINISTIC kernel rejection must throw {@link TimerIngressRejectedError} so the state machine
+ *  can quarantine it after bounded attempts instead of pinning the watermark forever. */
+export interface TimerIngressApplyTarget {
+  applyCreate(timer: ScheduledTimer, intent: TimerIngressIntentRow): Promise<TimerIngressApplyDisposition>;
+  applyCancel(payload: TimerIngressCancelPayload, intent: TimerIngressIntentRow): Promise<TimerIngressApplyDisposition>;
+}
+
+/** Test-only crash-boundary hooks (chaos harness SIGKILLs — or PAUSES a live process — inside them;
+ *  unit tests throw). Each maps to one durability boundary of the state machine — see the chaos
+ *  suite for the full ladder. The post-commit hooks may return a promise (awaited): that is what
+ *  lets the chaos harness model a LIVE-but-slow applier stalled mid-apply past its lease TTL
+ *  (finding 1), not just a dead one. `beforeAcceptCommit` stays synchronous — it runs INSIDE the
+ *  synchronous accept transaction. */
+export interface TimerIngressFaults {
+  /** After the request-key replay lookup found NO existing intent, before the accept transaction
+   *  opens. The T1175 concurrent-duplicate chaos scenario parks TWO processes here (both past the
+   *  replay lookup, neither inserted) and releases them together, so the partial-unique
+   *  `request_key` index — not the lookup — arbitrates a genuinely concurrent same-key accept. */
+  beforeAcceptTx?: (op: TimerIngressOp) => void | Promise<void>;
+  /** Inside the accept transaction, after bind() + the intent insert, before COMMIT. */
+  beforeAcceptCommit?: (op: TimerIngressOp) => void;
+  /** After the accept transaction committed, before the inline drive. */
+  afterAcceptCommit?: (seq: number) => void | Promise<void>;
+  /** After the claim (applying + lease) committed, before the apply target runs. */
+  afterClaim?: (seq: number) => void | Promise<void>;
+  /** After the apply target resolved, before the applied/watermark settle commits. */
+  afterApply?: (seq: number) => void | Promise<void>;
+  /** After the applied/watermark settle committed. */
+  afterSettle?: (seq: number) => void | Promise<void>;
+}
+
+export interface TimerIngressAlarms {
+  /** Unsettled intents older than `overdueAfterMs` — the apply pipeline is stuck. */
+  overdue: TimerIngressIntentRow[];
+  /** Unsettled creates whose `fire_at_hint` is within the near-due window — apply NOW or the
+   *  timer risks expiring unfired. */
+  nearDue: TimerIngressIntentRow[];
+}
+
+export interface TimerIngressAcceptResult<T> {
+  seq: number;
+  /** Settled outcome of THIS intent: `applied`, or `terminal` when a concurrent later intent for the
+   *  same timer superseded it before it applied (its net effect is subsumed). Either is an ACK. */
+  state: "applied" | "terminal";
+  terminalReason?: TimerIngressTerminalReason;
+  /** Whatever bind() returned — the durable legacy write's result. */
+  bound: T;
+}
+
+export interface TimerIngressPassResult {
+  applied: number;
+  terminal: number;
+  /** Intents whose apply attempt failed this pass (durably retained for retry). */
+  failed: number;
+  /** Timers skipped because a foreign live lease covers one of their intents. */
+  contended: number;
+}
+
+export interface TimerIngressOptions {
+  apply: TimerIngressApplyTarget;
+  /** Applier identity for lease ownership. Default: `pid-uuid` (unique per instance). */
+  ownerId?: string;
+  /** Injectable clock (tests). Default Date.now. */
+  now?: () => number;
+  /** Applier worker tick. Default 1000ms. */
+  tickMs?: number;
+  /** Apply lease TTL — an intent claimed by a crashed process becomes re-claimable after this. */
+  leaseMs?: number;
+  /** Near-due window for the missed-apply alarm. Default 2 × tickMs (plan §3.1). */
+  nearDueWindowMs?: number;
+  /** Unsettled-age threshold for the overdue alarm. Default 15s. */
+  overdueAfterMs?: number;
+  /** Apply attempts before a DETERMINISTIC kernel rejection ({@link TimerIngressRejectedError})
+   *  quarantines the intent `terminal('malformed')` instead of pinning the watermark. Transient
+   *  errors never count against this — they retry indefinitely (+ alarms). Default 3. */
+  maxApplyAttempts?: number;
+  /** Alarm sink for the worker pass. Default: console.warn a summary. */
+  onAlarm?: (alarms: TimerIngressAlarms) => void;
+  faults?: TimerIngressFaults;
+}
+
+export interface TimerIngress {
+  readonly ownerId: string;
+  /** Durably accept a MESSAGE-timer create: ONE transaction runs `bind()` (the legacy
+   *  `scheduled_timers` insert) + the full-payload intent insert, then drives the intent to settled
+   *  (ACK-at-applied). `requestKey` is REQUIRED — the stable identity of ONE logical create request:
+   *  a retry with the same key (lost ACK) skips `bind()` and ACKs the ORIGINAL intent (same seq,
+   *  same timer id reconstructed from the persisted payload) — exactly-once-by-request-key. Throws
+   *  `TimerIngressEligibilityError` (rolling the WHOLE tx back) for a command/disabled/kernel-
+   *  rejectable timer or a key already used by a different op; throws `TimerIngressPendingError`
+   *  when the intent is durable but not yet settled. */
+  acceptCreate(opts: { source: TimerIngressCreateSource; requestKey: string }): Promise<TimerIngressAcceptResult<ScheduledTimer>>;
+  /** Durably accept a cancel: ONE transaction runs `bind()` (the legacy cancel; may return null when
+   *  the row is missing — the tombstone is durable regardless) + the tombstone insert, then drives to
+   *  settled. A command-timer row from bind() rolls the whole tx back (commands bypass ingress).
+   *  `requestKey` is optional (the cancel effect is idempotent by construction): when supplied, a
+   *  retried cancel dedupes onto the original intent instead of appending a second tombstone row
+   *  (its `bound` is null on the replay path — the original bind result is not reconstructable). */
+  acceptCancel(opts: { timerId: string; cancel?: boolean; requestKey?: string }): Promise<TimerIngressAcceptResult<ScheduledTimer | null>>;
+  /** Coordinator-only Stage-9 bootstrap accept. It bypasses the deliberately closed public gate,
+   * but still commits intent + accepted legacy authority in one writer transaction. */
+  acceptBootstrapCreate(opts: { timer: ScheduledTimer; requestKey: string; epoch: number }): Promise<TimerIngressAcceptResult<ScheduledTimer>>;
+  /** One applier pass: recover expired leases, settle/collapse the backlog nearest-due first, check
+   *  alarms. Boot recovery IS this (idempotent). Apply errors are isolated per timer. */
+  applyOnce(opts?: { limit?: number }): Promise<TimerIngressPassResult>;
+  /** Start the interval applier worker (immediate recovery pass + tick). Idempotent. */
+  start(): void;
+  stop(): void;
+  /** The overdue / near-due missed-apply alarm state (plan §3.1). */
+  alarms(now?: number): TimerIngressAlarms;
+  /** The durable applied-watermark: every row with `seq <=` this is applied OR terminal. */
+  watermark(): number;
+  listIntents(filter?: { timerId?: string; unsettledOnly?: boolean }): TimerIngressIntentRow[];
+}
+
+/** Thrown when accept cannot settle the intent inline. The intent IS durably accepted (the bound
+ *  legacy write committed with it); the applier worker + alarms own convergence. Callers surface it
+ *  as "accepted, pending" — never as a lost write. */
+export class TimerIngressPendingError extends Error {
+  constructor(readonly seq: number, readonly timerId: string, cause: string) {
+    super(`timer ingress intent ${seq} (timer ${timerId}) is accepted but not yet applied: ${cause}`);
+    this.name = "TimerIngressPendingError";
+  }
+}
+
+/** Thrown by an apply target when the kernel DETERMINISTICALLY rejects the payload (validation-class
+ *  `invalid_input` — retrying can never succeed). The state machine quarantines the intent
+ *  `terminal('malformed')` after `maxApplyAttempts` so it cannot pin the watermark forever. Never
+ *  used for transient failures — those stay plain throws (retry + alarms). */
+export class TimerIngressRejectedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "TimerIngressRejectedError";
+  }
+}
+
+export function createTimerIngress(options: TimerIngressOptions): TimerIngress {
+  installTimerIngressSchema();
+  const apply = options.apply;
+  const ownerId = options.ownerId ?? `${process.pid}-${randomUUID()}`;
+  const now = options.now ?? Date.now;
+  const tickMs = options.tickMs ?? 1000;
+  const leaseMs = options.leaseMs ?? 10_000;
+  const nearDueWindowMs = options.nearDueWindowMs ?? 2 * tickMs;
+  const overdueAfterMs = options.overdueAfterMs ?? 15_000;
+  const maxApplyAttempts = options.maxApplyAttempts ?? 3;
+  const faults = options.faults ?? {};
+  let ticker: Timer | undefined;
+
+  // Per-timer in-process serialization of drives. Two same-process drivers share ownerId, so the
+  // lease CAS alone cannot arbitrate them — the chain does. Cross-process arbitration is the CAS.
+  const chains = new Map<string, Promise<unknown>>();
+  function serialized<T>(timerId: string, fn: () => Promise<T>): Promise<T> {
+    const prev = chains.get(timerId) ?? Promise.resolve();
+    const run = prev.catch(() => {}).then(fn);
+    chains.set(timerId, run);
+    void run.catch(() => {}).finally(() => {
+      if (chains.get(timerId) === run) chains.delete(timerId);
+    });
+    return run;
+  }
+
+  // The applier's reads and transitions go through the gateway: reads to the reader pool, the four
+  // CAS writes to the writer. Each is an `await`, so the drive's snapshot is now separated from the
+  // transition it feeds by a real thread hop — which is safe for exactly the reason the CAS exists
+  // (see timer-ingress-write-ops.ts): a stale snapshot can only lose the compare-and-swap.
+  const intentsFor = timerIngressIntentsForAsync;
+
+  const isUnsettled = (i: TimerIngressIntentRow) => i.applyState === "accepted" || i.applyState === "applying";
+  const foreignLiveLease = (i: TimerIngressIntentRow, at: number) =>
+    i.leaseOwner !== undefined && i.leaseOwner !== ownerId && (i.leaseExpiresAt ?? 0) > at;
+
+  // The four state transitions are CAS writes owned by `db/timer-ingress.ts`; this module supplies
+  // the applier identity and the clock and interprets the outcome (`false` ⇒ contended).
+  const settleTerminal = (seq: number, reason: TimerIngressTerminalReason, error?: string): Promise<boolean> =>
+    settleTimerIngressTerminalAsync({ seq, reason, ...(error !== undefined ? { error } : {}), ownerId, at: now() });
+
+  const claim = (seq: number): Promise<boolean> => claimTimerIngressIntentAsync({ seq, ownerId, leaseMs, at: now() });
+
+  const settleApplied = (seq: number): Promise<void> => settleTimerIngressAppliedAsync({ seq, ownerId, at: now() });
+
+  const releaseAfterFailure = (seq: number, error: string): Promise<void> =>
+    releaseTimerIngressLeaseAfterFailureAsync({ seq, error, ownerId });
+
+  interface DriveOutcome {
+    applied: number;
+    terminal: number;
+    contended: boolean;
+    /** Final state of `forSeq` when requested (accept's ACK). */
+    forState?: { state: "applied" | "terminal"; terminalReason?: TimerIngressTerminalReason };
+    failure?: { seq: number; error: string };
+  }
+
+  /**
+   * Settle everything unsettled for ONE timer: collapse every non-latest unsettled intent to
+   * `terminal`, then claim+apply+settle the latest (if unsettled). Crash-safe at every boundary:
+   * each transition is its own transaction and the apply target is idempotent. Bails `contended`
+   * (without collapsing) when any of the timer's intents is under a live foreign lease — never
+   * supersede under a mid-flight foreign apply.
+   */
+  async function driveTimerLocked(timerId: string, forSeq?: number): Promise<DriveOutcome> {
+    const outcome: DriveOutcome = { applied: 0, terminal: 0, contended: false };
+    const readState = async (seq: number): Promise<DriveOutcome["forState"]> => {
+      const row = (await intentsFor(timerId)).find((i) => i.seq === seq);
+      if (!row || isUnsettled(row)) return undefined;
+      return row.applyState === "applied"
+        ? { state: "applied" }
+        : { state: "terminal", ...(row.terminalReason ? { terminalReason: row.terminalReason } : {}) };
+    };
+
+    const intents = await intentsFor(timerId);
+    const unsettled = intents.filter(isUnsettled);
+    if (unsettled.length === 0) {
+      if (forSeq !== undefined) outcome.forState = await readState(forSeq);
+      return outcome;
+    }
+    const at = now();
+    if (intents.some((i) => isUnsettled(i) && foreignLiveLease(i, at))) {
+      outcome.contended = true;
+      return outcome;
+    }
+
+    // 1) Latest-wins collapse: every unsettled intent below the timer's max seq is superseded — a
+    //    later cancel makes it a tombstone collapse (create must NOT resurrect a cancelled timer).
+    const maxSeq = intents[intents.length - 1]!.seq;
+    for (const loser of unsettled.filter((i) => i.seq < maxSeq)) {
+      const laterCancel = intents.some((i) => i.seq > loser.seq && i.op === "cancel");
+      if (await settleTerminal(loser.seq, laterCancel ? "tombstoned" : "superseded")) outcome.terminal += 1;
+      else {
+        outcome.contended = true; // raced by a foreign claim — do not apply past it
+        return outcome;
+      }
+    }
+
+    // 2) Apply the latest intent if it is the unsettled one.
+    const latest = intents[intents.length - 1]!;
+    if (isUnsettled(latest)) {
+      if (!(await claim(latest.seq))) {
+        outcome.contended = true;
+        if (forSeq !== undefined) outcome.forState = await readState(forSeq);
+        return outcome;
+      }
+      await faults.afterClaim?.(latest.seq);
+      const applyError = await (async (): Promise<string | undefined> => {
+        try {
+          let disposition: TimerIngressApplyDisposition;
+          if (latest.op === "create") {
+            const payload = parseCreatePayload(latest.payload, latest.timerId);
+            if (!payload) {
+              if (await settleTerminal(latest.seq, "malformed", "create payload failed validation on read")) outcome.terminal += 1;
+              return undefined;
+            }
+            disposition = await apply.applyCreate(payload, latest);
+          } else {
+            const payload = parseCancelPayload(latest.payload, latest.timerId);
+            if (!payload) {
+              if (await settleTerminal(latest.seq, "malformed", "cancel payload failed validation on read")) outcome.terminal += 1;
+              return undefined;
+            }
+            disposition = await apply.applyCancel(payload, latest);
+          }
+          if (disposition === "fenced") {
+            // The kernel REFUSED the write: a strictly newer intent for this timer already applied
+            // (our snapshot went stale mid-apply — the live-slow-applier race, finding 1). The
+            // effect did NOT happen, so this intent settles terminal, never applied. Whoever drove
+            // the newer intent normally terminalized us already — then this settle is a no-op.
+            const laterCancel = (await intentsFor(timerId)).some((i) => i.seq > latest.seq && i.op === "cancel");
+            if (await settleTerminal(latest.seq, laterCancel ? "tombstoned" : "superseded", "fenced at the kernel write by a newer applied intent")) {
+              outcome.terminal += 1;
+            }
+            return undefined;
+          }
+          await faults.afterApply?.(latest.seq);
+          await settleApplied(latest.seq);
+          await faults.afterSettle?.(latest.seq);
+          outcome.applied += 1;
+          return undefined;
+        } catch (error) {
+          if (error instanceof TimerIngressRejectedError) {
+            // The kernel rejects this payload DETERMINISTICALLY (validation-class): bounded attempts
+            // (`claim` already counted this one), then quarantine — a permanently rejected row must
+            // never pin the watermark. Transient errors never take this branch.
+            const attemptsNow = latest.attempts + 1;
+            if (attemptsNow >= maxApplyAttempts) {
+              const detail = `kernel rejected the apply ${attemptsNow}x (quarantined): ${errMessage(error)}`;
+              if (await settleTerminal(latest.seq, "malformed", detail)) outcome.terminal += 1;
+              return undefined;
+            }
+          }
+          return errMessage(error);
+        }
+      })();
+      if (applyError !== undefined) {
+        await releaseAfterFailure(latest.seq, applyError);
+        outcome.failure = { seq: latest.seq, error: applyError };
+      }
+    }
+    if (forSeq !== undefined) outcome.forState = await readState(forSeq);
+    return outcome;
+  }
+
+  const driveTimer = (timerId: string, forSeq?: number) => serialized(timerId, () => driveTimerLocked(timerId, forSeq));
+
+  function assertNotInTransaction(): void {
+    if (getDb().inTransaction) {
+      throw new Error("timer ingress accept must own its transaction (do not call inside an open transaction)");
+    }
+  }
+
+  const intentByRequestKey = timerIngressIntentByRequestKeyAsync;
+
+  /**
+   * Submit one accept op and converge on the winner when a CONCURRENT same-key accept beat us.
+   *
+   * The op folds its own replay lookup into its transaction (§3.4), which closes the single-process
+   * TOCTOU that used to sit between the pre-transaction lookup and the insert. What no fold can
+   * prevent is two PROCESSES accepting the same key at once: the loser hits `UNIQUE(request_key)`,
+   * its WHOLE transaction — including the legacy row it just wrote — rolls back, and it must ACK the
+   * winner's intent rather than surface a duplicate accept as a failure. That is this catch.
+   *
+   * Eligibility rejections (`TimerIngressEligibilityError`) are thrown INSIDE the op and arrive here
+   * as real instances because the error is in the reconstruction registry (§3.7) — without that they
+   * would degrade to bare `Error` and every `instanceof` branch below would silently stop matching.
+   */
+  async function runAccept(
+    submit: () => Promise<TimerIngressAcceptOutcome>,
+    storedKey: string | undefined,
+    op: TimerIngressOp,
+  ): Promise<TimerIngressAcceptOutcome> {
+    // `beforeAcceptCommit` must fire INSIDE the accept transaction, which now runs in the op — so it
+    // is delivered through the writer-fault seam rather than captured in a closure the op could not
+    // see. Under the worker backend `armWriterFault` THROWS instead of injecting nothing, which is
+    // the whole point: a crash-recovery test must never pass having crashed nothing.
+    const disarm = faults.beforeAcceptCommit
+      ? armWriterFault(`timerIngress.accept${op === "cancel" ? "Cancel" : "Create"}.beforeCommit`, () => {
+          faults.beforeAcceptCommit?.(op);
+        })
+      : undefined;
+    try {
+      return await submit();
+    } catch (error) {
+      if (storedKey !== undefined && isTimerIngressRequestKeyConflict(error)) {
+        const winner = await intentByRequestKey(storedKey);
+        if (winner) return { kind: "replayed", intent: winner };
+      }
+      throw error;
+    } finally {
+      disarm?.();
+    }
+  }
+
+  async function settleForAck<T>(
+    seq: number,
+    timerId: string,
+    bound: T,
+    opts: { freshAccept: boolean },
+  ): Promise<TimerIngressAcceptResult<T>> {
+    // The accept-commit fault models a crash between the COMMIT and the caller receiving the ACK; a
+    // replayed accept (request-key dedupe) performed no fresh commit, so the boundary does not exist.
+    if (opts.freshAccept) await faults.afterAcceptCommit?.(seq);
+    const outcome = await driveTimer(timerId, seq);
+    if (outcome.forState) {
+      return {
+        seq,
+        state: outcome.forState.state,
+        ...(outcome.forState.terminalReason ? { terminalReason: outcome.forState.terminalReason } : {}),
+        bound,
+      };
+    }
+    throw new TimerIngressPendingError(seq, timerId, outcome.failure?.error ?? (outcome.contended ? "contended by another applier" : "unsettled"));
+  }
+
+  /** ACK a create replayed under its original request key: `bind()` never re-runs; the ORIGINAL
+   *  timer snapshot is reconstructed from the persisted full payload and the ORIGINAL intent is
+   *  driven to settled. This is what makes an ACK-lost create retry-safe (exactly-once-by-key). */
+  async function ackReplayedCreate(intent: TimerIngressIntentRow, requestKey: string): Promise<TimerIngressAcceptResult<ScheduledTimer>> {
+    if (intent.op !== "create") {
+      throw new TimerIngressEligibilityError(`requestKey '${requestKey}' was already used by a '${intent.op}' intent`);
+    }
+    const bound = parseCreatePayload(intent.payload, intent.timerId);
+    if (!bound) {
+      // Unreachable for an intent this module accepted (validated at accept) — only raw corruption.
+      throw new TimerIngressEligibilityError(`requestKey '${requestKey}' maps to a malformed persisted intent`);
+    }
+    return settleForAck(intent.seq, intent.timerId, bound, { freshAccept: false });
+  }
+
+  async function ackReplayedCancel(
+    intent: TimerIngressIntentRow,
+    requestKey: string,
+    timerId: string,
+  ): Promise<TimerIngressAcceptResult<ScheduledTimer | null>> {
+    if (intent.op !== "cancel" || intent.timerId !== timerId) {
+      throw new TimerIngressEligibilityError(`requestKey '${requestKey}' was already used by a different intent`);
+    }
+    return settleForAck<ScheduledTimer | null>(intent.seq, intent.timerId, null, { freshAccept: false });
+  }
+
+  const api: TimerIngress = {
+    ownerId,
+
+    async acceptCreate({ source, requestKey }) {
+      assertNotInTransaction();
+      if (typeof requestKey !== "string" || !requestKey) {
+        throw new TimerIngressEligibilityError("acceptCreate requires a stable requestKey (the exactly-once request identity)");
+      }
+      // Request identity is namespaced per op (T1175 finding 3): a cancel sharing this raw key is a
+      // DIFFERENT logical request and must neither dedupe onto nor be rejected against this intent.
+      const storedKey = canonicalTimerIngressRequestKey("create", requestKey);
+      await faults.beforeAcceptTx?.("create");
+      const outcome = await runAccept(() => acceptTimerIngressCreateAsync({ source, storedKey, at: now() }), storedKey, "create");
+      if (outcome.kind === "replayed") return ackReplayedCreate(outcome.intent, requestKey);
+      return settleForAck(outcome.seq, outcome.bound!.id, outcome.bound!, { freshAccept: true });
+    },
+
+    async acceptCancel({ timerId, cancel, requestKey }) {
+      assertNotInTransaction();
+      if (typeof timerId !== "string" || !timerId) throw new TimerIngressEligibilityError("cancel requires timerId");
+      if (requestKey !== undefined && (typeof requestKey !== "string" || !requestKey)) {
+        throw new TimerIngressEligibilityError("requestKey, when supplied, must be a non-empty string");
+      }
+      // Cancel identity is namespaced per op AND per timer (T1175 finding 3): the same raw key
+      // cancelling two different timers is two logical requests — independent intents, no collision.
+      const storedKey = requestKey !== undefined ? canonicalTimerIngressRequestKey("cancel", requestKey, timerId) : undefined;
+      await faults.beforeAcceptTx?.("cancel");
+      const outcome = await runAccept(
+        () => acceptTimerIngressCancelAsync({ timerId, cancel: cancel ?? false, ...(storedKey !== undefined ? { storedKey } : {}), at: now() }),
+        storedKey,
+        "cancel",
+      );
+      if (outcome.kind === "replayed") return ackReplayedCancel(outcome.intent, requestKey!, timerId);
+      return settleForAck<ScheduledTimer | null>(outcome.seq, timerId, outcome.bound, { freshAccept: true });
+    },
+
+    async acceptBootstrapCreate({ timer, requestKey, epoch }) {
+      assertNotInTransaction();
+      if (timer.command || !timer.enabled) throw new TimerIngressEligibilityError("bootstrap requires an active MESSAGE timer");
+      assertIngressCreateSnapshot(timer);
+      const storedKey = canonicalTimerIngressRequestKey("create", requestKey);
+      const outcome = await runAccept(
+        () => acceptTimerIngressBootstrapCreateAsync({ timer, storedKey, epoch, at: now() }),
+        storedKey,
+        "create",
+      );
+      if (outcome.kind === "replayed") return ackReplayedCreate(outcome.intent, requestKey);
+      return settleForAck(outcome.seq, timer.id, timer, { freshAccept: true });
+    },
+
+    async applyOnce(opts) {
+      const at = now();
+      const limit = opts?.limit ?? 100;
+      // Nearest-due first (missed-fire pressure), then log order; cancels (no hint) after hinted
+      // creates. Per-timer ordering/collapse/lease arbitration is driveTimer's job — this only picks
+      // WHICH timers (foreign-leased ones included, so contention is observed, not silently skipped).
+      const timerIds = await timerIngressUnsettledTimerIdsAsync(limit);
+      const result: TimerIngressPassResult = { applied: 0, terminal: 0, failed: 0, contended: 0 };
+      for (const timerId of timerIds) {
+        try {
+          const outcome = await driveTimer(timerId);
+          result.applied += outcome.applied;
+          result.terminal += outcome.terminal;
+          if (outcome.contended) result.contended += 1;
+          if (outcome.failure) result.failed += 1;
+        } catch (error) {
+          // Isolation: one bad timer never aborts the pass (its intents stay durably unsettled).
+          result.failed += 1;
+          console.warn(`timer ingress apply failed for ${timerId}: ${errMessage(error)}`);
+        }
+      }
+      const alarms = api.alarms(at);
+      if (alarms.overdue.length > 0 || alarms.nearDue.length > 0) {
+        if (options.onAlarm) options.onAlarm(alarms);
+        else {
+          console.warn(
+            `timer ingress missed-apply alarm: ${alarms.overdue.length} overdue, ${alarms.nearDue.length} near-due unapplied ` +
+            `(seqs ${[...alarms.overdue, ...alarms.nearDue].map((i) => i.seq).join(",")})`,
+          );
+        }
+      }
+      return result;
+    },
+
+    start() {
+      if (ticker) return;
+      const pass = () => {
+        void api.applyOnce().catch((error) => console.warn(`timer ingress applier pass failed: ${errMessage(error)}`));
+      };
+      pass(); // boot recovery: converge whatever a prior process left unsettled
+      ticker = setInterval(pass, tickMs);
+    },
+
+    stop() {
+      if (ticker) clearInterval(ticker);
+      ticker = undefined;
+    },
+
+    // `alarms`, `watermark` and `listIntents` stay SYNCHRONOUS on this interface. They are read-only
+    // diagnostics whose callers (the operator surface, the verifier, the routes) are still
+    // synchronous; converting them is a caller-side change that belongs with those modules, and the
+    // async equivalents are already registered (`timerIngress.*` read ops) for when they move.
+    alarms(atInput) {
+      const at = atInput ?? now();
+      const unsettled = listTimerIngressUnsettledIntents();
+      return {
+        overdue: unsettled.filter((i) => i.acceptedAt <= at - overdueAfterMs),
+        nearDue: unsettled.filter((i) => i.fireAtHint !== undefined && i.fireAtHint <= at + nearDueWindowMs),
+      };
+    },
+
+    watermark: timerIngressWatermark,
+
+    listIntents: listTimerIngressIntents,
+  };
+  return api;
+}