@humwick/server 0.130.2 → 0.130.3

package/src/db/timer-cutover-core.ts

@@ -1,3 +1,7 @@
+import { framed } from "../hub-metrics";
+import { getGateway } from "./gateway";
+import { normalizePhase, readCutover } from "./timer-cutover-state";
+import type { CutoverWriterState, CutoverStepArgs, CutoverStepResults } from "./timer-cutover-bodies";
 import { randomUUID } from "node:crypto";
 import {
   clearExpiredLegacyMessageClaimsInTx,
@@ -64,48 +68,9 @@ export type TimerMessageCutoverTickResult =
   | { state: "absent" | "idle" | "waiting-for-lease" }
   | { state: "advanced" | "holding" | "aborting"; phase: TimerMessageCutoverPhase; epoch: number; liveClaims?: number; guard: TimerMessageDeadlineGuard };
 
-function normalizePhase(phase: string): TimerMessageCutoverPhase {
-  return phase === "leased" || phase === "quiescing" || phase === "draining" || phase === "quiesced" || phase === "aborting"
-    ? phase
-    : "idle";
-}
-
-function journalOf(row: TimerMessageCutoverRow): TimerMessageCutoverJournalEntry[] {
-  return Array.isArray(row.phaseJournal) ? row.phaseJournal as TimerMessageCutoverJournalEntry[] : [];
-}
-
-function appendJournal(row: TimerMessageCutoverRow, entry: TimerMessageCutoverJournalEntry): string {
-  return JSON.stringify([...journalOf(row), entry]);
-}
-
-function readCutover(): TimerMessageCutoverRow | null {
-  const row = getDb().query("SELECT * FROM timer_message_cutover WHERE id = 1").get() as any;
-  return row ? {
-    id: 1,
-    protocolVersion: row.protocol_version,
-    epoch: row.epoch,
-    desiredMode: row.desired_mode,
-    observedMode: row.observed_mode,
-    phase: row.phase,
-    leaseOwner: row.lease_owner,
-    leaseExpiresAt: row.lease_expires_at,
-    quiesceStartedAt: row.quiesce_started_at,
-    quiesceDeadline: row.quiesce_deadline,
-    lastCompletedTimer: row.last_completed_timer,
-    ingressPolicy: row.ingress_policy,
-    messageMutationGate: row.message_mutation_gate,
-    gateEpoch: row.gate_epoch,
-    abortRequested: row.abort_requested === 1,
-    phaseJournal: JSON.parse(row.phase_journal),
-    updatedAt: row.updated_at,
-  } : null;
-}
-
 /** Singleton Stage-6 lease/quiesce state machine. It has no HTTP/MCP/CLI binding: production stays
  * shadow and the only boot wiring is recovery of an already-existing non-idle journal. */
-// LC-4c (#1920) owns extraction of this remaining synchronous state-machine core into a db-domain
-// writer-op module. It cannot be safely folded into this service in LC-4b: doing so breaches the
-// file-size ratchet; callers with an awaitable batch facade are already migrated here.
+// #1870 — only transaction bodies cross the writer seam; fault hooks and timers stay on main.
 export class TimerMessageCutoverCoordinator {
   readonly ownerId: string;
   readonly #now: () => number;
@@ -158,7 +123,7 @@ export class TimerMessageCutoverCoordinator {
     if (!acquired) return false;
     const guard = this.deadlineGuard();
     if (!guard.safe) {
-      this.requestAbort(`deadline guard refused entry (slack=${guard.slackMs}, required=${guard.requiredMs})`);
+      await this.requestAbort(`deadline guard refused entry (slack=${guard.slackMs}, required=${guard.requiredMs})`);
       await this.tick();
       await this.tick();
       return false;
@@ -166,15 +131,9 @@ export class TimerMessageCutoverCoordinator {
     return true;
   }
 
-  requestAbort(reason = "abort requested"): boolean {
+  async requestAbort(reason = "abort requested"): Promise<boolean> {
     const now = this.#now();
-    return writeTransaction(() => {
-      const row = readCutover();
-      if (!row || normalizePhase(row.phase) === "idle") return false;
-      return getDb().query(`UPDATE timer_message_cutover
-        SET abort_requested = 1, phase_journal = ?, updated_at = ? WHERE id = 1`)
-        .run(appendJournal(row, { epoch: row.epoch, phase: "aborting", at: now, reason }), now).changes === 1;
-    });
+    return await this.#writeStep("abort", { now, reason });
   }
 
   /** Stage 9: commit only the durable route marker while the same all-surface gate remains closed. */
@@ -182,19 +141,7 @@ export class TimerMessageCutoverCoordinator {
     const now = this.#now();
     const epoch = this.#epoch;
     if (epoch === undefined) throw new Error("cutover lease is not owned");
-    writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      if (normalizePhase(row.phase) !== "quiesced" || row.messageMutationGate !== "closed") {
-        throw new Error("ingress routing activation requires the live quiesced gate owner");
-      }
-      const changed = getDb().query(`UPDATE timer_message_cutover SET ingress_policy = 'ingress',
-        phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?
-          AND phase = 'quiesced' AND message_mutation_gate = 'closed'`)
-        .run(appendJournal(row, { epoch, phase: "quiesced", at: now, reason: "authoritative ingress routing activated under fence" }),
-          now, epoch, this.ownerId, now);
-      if (changed.changes !== 1) throw new Error("cutover lease lost before ingress routing activation");
-    });
+    await this.#writeStep("activate", { now, epoch });
     await this.#faults.afterRoutingActivation?.(epoch);
   }
 
@@ -205,23 +152,7 @@ export class TimerMessageCutoverCoordinator {
     const epoch = this.#epoch;
     if (epoch === undefined) throw new Error("cutover lease is not owned");
     await this.#faults.afterG1BeforeReopen?.(epoch);
-    writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      if (normalizePhase(row.phase) !== "quiesced" || row.messageMutationGate !== "closed" || row.ingressPolicy !== "ingress") {
-        throw new Error("G1 completion requires fenced authoritative ingress routing");
-      }
-      const changed = getDb().query(`UPDATE timer_message_cutover SET
-        protocol_version = MAX(COALESCE(protocol_version, 1), 2),
-        phase = 'idle', lease_owner = NULL, lease_expires_at = NULL,
-        quiesce_started_at = NULL, quiesce_deadline = NULL,
-        message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
-        phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?
-          AND phase = 'quiesced' AND message_mutation_gate = 'closed' AND ingress_policy = 'ingress'`)
-        .run(appendJournal(row, { epoch, phase: "idle", at: now, reason: "G1 proven; gate reopened under legacy authority" }),
-          now, epoch, this.ownerId, now);
-      if (changed.changes !== 1) throw new Error("cutover lease lost before G1 gate reopen");
-    });
+    await this.#writeStep("completeLegacy", { now, epoch });
     this.#epoch = undefined;
   }
 
@@ -236,37 +167,7 @@ export class TimerMessageCutoverCoordinator {
     const now = this.#now();
     const epoch = this.#epoch;
     if (epoch === undefined) throw new Error("cutover lease is not owned");
-    writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      if (normalizePhase(row.phase) !== "quiesced" || row.messageMutationGate !== "closed"
-        || row.ingressPolicy !== "ingress") {
-        throw new Error("authority-transfer completion requires the live quiesced ingress gate owner");
-      }
-      // Only ACTIVE (scheduled) rows must converge on the target backend — terminal rows never
-      // dispatch, so their backend is irrelevant and must not block completion (an ordinary kernel-mode
-      // cancel terminalizes a kernel-owned row that a reverse would otherwise be unable to move).
-      const wrongOwners = (getDb().query(`SELECT count(*) AS n FROM timer_message_authority
-        WHERE lifecycle='scheduled' AND backend != ?`).get(expectedBackend) as { n: number }).n;
-      if (wrongOwners !== 0) throw new Error(`${wrongOwners} scheduled MESSAGE owners are not ${expectedBackend}`);
-      const expectedFrozen = expectedBackend === "kernel" ? 1 : 0;
-      const projectionMismatch = (getDb().query(`SELECT count(*) AS n FROM timer_message_authority a
-          JOIN scheduled_timers s ON s.id=a.timer_id
-          WHERE a.lifecycle='scheduled' AND (s.command IS NOT NULL OR s.message_authority_frozen != ?)`)
-          .get(expectedFrozen) as { n: number }).n;
-      if (projectionMismatch !== 0) throw new Error(`${projectionMismatch} MESSAGE compatibility projections disagree with ${mode}`);
-      const changed = getDb().query(`UPDATE timer_message_cutover SET
-        desired_mode = ?, observed_mode = ?, phase = 'idle', lease_owner = NULL,
-        lease_expires_at = NULL, quiesce_started_at = NULL, quiesce_deadline = NULL,
-        message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
-        phase_journal = ?, updated_at = ?
-        WHERE id=1 AND epoch=? AND lease_owner=? AND lease_expires_at>?
-          AND phase='quiesced' AND message_mutation_gate='closed' AND ingress_policy='ingress'`)
-        .run(mode, mode, appendJournal(row, {
-          epoch, phase: "idle", at: now,
-          reason: `${mode} per-row ownership proven; MESSAGE mutation gate reopened`,
-        }), now, epoch, this.ownerId, now);
-      if (changed.changes !== 1) throw new Error("cutover lease lost before authority-transfer completion");
-    });
+    await this.#writeStep("completeTransfer", { now, epoch, mode, expectedBackend });
     this.#epoch = undefined;
     await this.#faults.afterModeCommit?.(epoch, mode);
   }
@@ -275,22 +176,11 @@ export class TimerMessageCutoverCoordinator {
    * benignly refuse while reverse projection owns the quiesced writer window. `targetMode` is the mode
    * the reverse restores (dual-read→shadow or kernel→dual-read); `sourceObservedMode` is the currently
    * observed mode the reverse departs from. */
-  markReverseTransferDesired(targetMode: KernelTimersDesiredMode, sourceObservedMode: KernelTimersDesiredMode): void {
+  async markReverseTransferDesired(targetMode: KernelTimersDesiredMode, sourceObservedMode: KernelTimersDesiredMode): Promise<void> {
     const now = this.#now();
     const epoch = this.#epoch;
     if (epoch === undefined) throw new Error("cutover lease is not owned");
-    writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      const changed = getDb().query(`UPDATE timer_message_cutover SET desired_mode=?,
-        phase_journal=?, updated_at=?
-        WHERE id=1 AND epoch=? AND lease_owner=? AND lease_expires_at>?
-          AND phase='quiesced' AND message_mutation_gate='closed' AND observed_mode=?`)
-        .run(targetMode, appendJournal(row, {
-          epoch, phase: "quiesced", at: now,
-          reason: `${targetMode} rollback desired; kernel decision commits quiesced for reverse projection`,
-        }), now, epoch, this.ownerId, now, sourceObservedMode);
-      if (changed.changes !== 1) throw new Error(`reverse transfer requires observed ${sourceObservedMode} under the live fence`);
-    });
+    await this.#writeStep("reverse", { now, epoch, targetMode, sourceObservedMode });
   }
 
   /** One deterministic recovery/coordination step. The deadline guard is recomputed on every call,
@@ -315,7 +205,7 @@ export class TimerMessageCutoverCoordinator {
         : boundedOut
           ? "bounded quiesce timeout reached"
           : `deadline guard tripped (slack=${guard.slackMs}, required=${guard.requiredMs})`;
-      if (normalizePhase(row.phase) !== "aborting") this.#transition(row.epoch, "aborting", reason);
+      if (normalizePhase(row.phase) !== "aborting") await this.#transition(row.epoch, "aborting", reason);
       else await this.#finishAbort(row.epoch, reason);
       return { state: "aborting", phase: "aborting", epoch: row.epoch, guard };
     }
@@ -327,20 +217,20 @@ export class TimerMessageCutoverCoordinator {
 
     switch (normalizePhase(row.phase)) {
       case "leased":
-        if (!this.#closeGate(row.epoch, now)) {
+        if (!await this.#closeGate(row.epoch, now)) {
           return { state: "aborting", phase: "aborting", epoch: row.epoch, guard: this.deadlineGuard(now) };
         }
         await this.#faults.afterGateClose?.(row.epoch);
         return { state: "advanced", phase: "quiescing", epoch: row.epoch, guard };
       case "quiescing":
-        this.#transition(row.epoch, "draining", "message claims are fenced; begin bounded drain");
+        await this.#transition(row.epoch, "draining", "message claims are fenced; begin bounded drain");
         return { state: "advanced", phase: "draining", epoch: row.epoch, guard };
       case "draining": {
-        writeTransaction((tx) => clearExpiredLegacyMessageClaimsInTx(tx, now));
+        await this.#writeStep("clearClaims", { now });
         const liveClaims = countLiveLegacyMessageClaims(now);
         await this.#faults.duringClaimDrain?.(row.epoch, liveClaims);
         if (liveClaims === 0) {
-          this.#transition(row.epoch, "quiesced", "all live legacy MESSAGE claims drained");
+          await this.#transition(row.epoch, "quiesced", "all live legacy MESSAGE claims drained");
           return { state: "advanced", phase: "quiesced", epoch: row.epoch, liveClaims, guard };
         }
         return { state: "holding", phase: "draining", epoch: row.epoch, liveClaims, guard };
@@ -358,11 +248,22 @@ export class TimerMessageCutoverCoordinator {
   /** Boot recovery is deliberately non-creative: absent/idle shadow state stays a production no-op. */
   startRecovery(): void {
     if (this.#ticker) return;
-    const pass = () => void this.tick().catch((error) => console.warn(`timer MESSAGE cutover recovery failed: ${String(error)}`));
+    const pass = framed("timer message-cutover", () => this.tick().catch((error) => console.warn(`timer MESSAGE cutover recovery failed: ${String(error)}`)));
     pass();
     this.#ticker = setInterval(pass, this.#tickMs);
   }
 
+  async #writeStep<K extends keyof CutoverStepArgs>(step: K, args: CutoverStepArgs[K]): Promise<CutoverStepResults[K]> {
+    const epoch = this.#epoch;
+    const state: CutoverWriterState = { ownerId: this.ownerId, epoch, leaseMs: this.#leaseMs,
+      quiesceTimeoutMs: this.#quiesceTimeoutMs, worstCaseMs: this.#worstCaseMs,
+      authoritativeActivation: this.#authoritativeActivation, operationLeaseFence: this.#operationLeaseFence };
+    const result = await getGateway().write("timerCutover.coordinatorStep", { step, state, args },
+      { critical: true, idempotencyKey: randomUUID() }) as unknown as { value: CutoverStepResults[K]; clearEpoch?: boolean };
+    if (result.clearEpoch && this.#epoch === epoch) this.#epoch = undefined;
+    return result.value;
+  }
+
   stop(): void {
     if (this.#ticker) clearInterval(this.#ticker);
     this.#ticker = undefined;
@@ -371,57 +272,7 @@ export class TimerMessageCutoverCoordinator {
   async #acquire(create: boolean): Promise<boolean> {
     const now = this.#now();
     let acquiredEpoch: number | undefined;
-    let takeover = false;
-    writeTransaction(() => {
-      // Stage-10 pins the resolved leg before it asks this independent coordinator to close the
-      // global gate. Check that pin's operation lease under THIS acquisition transaction: if an
-      // abort won after pin but before requestQuiesce, it cannot be followed by a stale gate close.
-      const fence = this.#operationLeaseFence;
-      if (fence) {
-        const live = getDb().query(`SELECT 1 FROM timer_message_cutover_operation
-          WHERE id=1 AND request_id=? AND state='running' AND abort_requested=0
-            AND lease_owner=? AND lease_expires_at > ?`)
-          .get(fence.requestId, fence.ownerId, now);
-        if (!live) return;
-      }
-      const row = readCutover();
-      if (!row) {
-        if (!create) return;
-        acquiredEpoch = 1;
-        const journal: TimerMessageCutoverJournalEntry[] = [{ epoch: 1, phase: "leased", at: now, reason: "initial lease acquired" }];
-        getDb().query(`INSERT INTO timer_message_cutover
-          (id, protocol_version, epoch, desired_mode, observed_mode, phase, lease_owner, lease_expires_at,
-           quiesce_started_at, quiesce_deadline, last_completed_timer, ingress_policy,
-           message_mutation_gate, gate_epoch, abort_requested, phase_journal, updated_at)
-          VALUES (1, 1, 1, 'shadow', 'shadow', 'leased', ?, ?, NULL, NULL, NULL, 'legacy', 'open', NULL, 0, ?, ?)`)
-          .run(this.ownerId, now + this.#leaseMs, JSON.stringify(journal), now);
-        return;
-      }
-      const rowPhase = normalizePhase(row.phase);
-      const liveForeign = row.leaseOwner !== null && row.leaseOwner !== this.ownerId && (row.leaseExpiresAt ?? 0) > now;
-      if (liveForeign) return;
-      if (row.leaseOwner === this.ownerId && (row.leaseExpiresAt ?? 0) > now && this.#epoch === row.epoch) {
-        acquiredEpoch = row.epoch;
-        return;
-      }
-      const nextEpoch = row.epoch + 1;
-      takeover = row.leaseOwner !== null || rowPhase !== "idle" || row.messageMutationGate === "closed";
-      const nextPhase: TimerMessageCutoverPhase = rowPhase === "idle" ? "leased" : rowPhase;
-      const entry: TimerMessageCutoverJournalEntry = {
-        epoch: nextEpoch,
-        phase: takeover ? "takeover" : "leased",
-        at: now,
-        reason: takeover ? `expired epoch ${row.epoch} taken over in phase ${rowPhase}` : "lease acquired",
-      };
-      const result = getDb().query(`UPDATE timer_message_cutover SET
-        protocol_version = COALESCE(protocol_version, 1), epoch = ?, phase = ?, lease_owner = ?, lease_expires_at = ?,
-        gate_epoch = CASE WHEN message_mutation_gate = 'closed' THEN ? ELSE NULL END,
-        phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND (lease_owner IS NULL OR lease_expires_at <= ? OR lease_owner = ?)`)
-        .run(nextEpoch, nextPhase, this.ownerId, now + this.#leaseMs, nextEpoch,
-          appendJournal(row, entry), now, now, this.ownerId);
-      if (result.changes === 1) acquiredEpoch = nextEpoch;
-    });
+    ({ acquiredEpoch } = await this.#writeStep("acquire", { now, create }));
     if (acquiredEpoch === undefined) return false;
     this.#epoch = acquiredEpoch;
     await this.#faults.afterLeaseAcquire?.(acquiredEpoch);
@@ -430,116 +281,24 @@ export class TimerMessageCutoverCoordinator {
 
   async #renew(epoch: number): Promise<boolean> {
     const now = this.#now();
-    const renewed = writeTransaction(() => {
-      const row = readCutover();
-      if (!row || row.epoch !== epoch || row.leaseOwner !== this.ownerId || (row.leaseExpiresAt ?? 0) <= now) return false;
-      const deadlineCap = row.messageMutationGate === "closed" && row.quiesceDeadline !== null
-        ? row.quiesceDeadline
-        : Number.MAX_SAFE_INTEGER;
-      const nextExpiry = Math.min(now + this.#leaseMs, deadlineCap);
-      return getDb().query(`UPDATE timer_message_cutover SET lease_expires_at = ?, phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?`)
-        .run(nextExpiry, appendJournal(row, { epoch, phase: "renewed", at: now }), now,
-          epoch, this.ownerId, now).changes === 1;
-    });
+    const renewed = await this.#writeStep("renew", { now, epoch });
     if (renewed) await this.#faults.afterRenewal?.(epoch);
     return renewed;
   }
 
-  #closeGate(epoch: number, now: number): boolean {
-    return writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      // `requestQuiesce` acquires the coordinator lease before the following tick closes the gate.
-      // Recheck the Stage-10 operation lease in this close transaction: an abort (or expiry) that
-      // won in that interval must leave this independent coordinator open and unleased, never
-      // permit stale authority to close the MESSAGE gate after the abort's release observed it open.
-      const fence = this.#operationLeaseFence;
-      if (fence) {
-        const live = getDb().query(`SELECT 1 FROM timer_message_cutover_operation
-          WHERE id=1 AND request_id=? AND state='running' AND abort_requested=0
-            AND lease_owner=? AND lease_expires_at > ?`)
-          .get(fence.requestId, fence.ownerId, now);
-        if (!live) {
-          const released = getDb().query(`UPDATE timer_message_cutover SET
-            phase = 'idle', lease_owner = NULL, lease_expires_at = NULL,
-            quiesce_started_at = NULL, quiesce_deadline = NULL,
-            message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
-            desired_mode = observed_mode, phase_journal = ?, updated_at = ?
-            WHERE id=1 AND epoch=? AND lease_owner=? AND lease_expires_at>? AND phase='leased'`)
-            .run(appendJournal(row, {
-              epoch, phase: "idle", at: now,
-              reason: "Stage-10 operation lease no longer authorizes MESSAGE gate close",
-            }), now, epoch, this.ownerId, now);
-          if (released.changes !== 1) throw new Error("cutover lease lost while releasing stale Stage-10 gate request");
-          this.#epoch = undefined;
-          return false;
-        }
-      }
-      // Re-read all three deadline sources under the SAME writer lock that closes the gate. A
-      // deadline mutation that won the race is therefore observed here and refuses entry; one that
-      // loses cannot commit until after the closed marker and is caught by the next continuous tick.
-      const guard = this.deadlineGuard(now);
-      if (!guard.safe) {
-        const changed = getDb().query(`UPDATE timer_message_cutover SET phase = 'aborting', abort_requested = 1,
-          phase_journal = ?, updated_at = ? WHERE id = 1 AND epoch = ? AND lease_owner = ? AND phase = 'leased'`)
-          .run(appendJournal(row, {
-            epoch, phase: "aborting", at: now,
-            reason: `deadline guard refused atomic gate close (slack=${guard.slackMs}, required=${guard.requiredMs})`,
-          }), now, epoch, this.ownerId);
-        if (changed.changes !== 1) throw new Error("cutover lease lost while refusing gate close");
-        return false;
-      }
-      const deadline = Math.min(
-        now + this.#quiesceTimeoutMs,
-        guard.deadlines.earliestDeadline === null ? Number.MAX_SAFE_INTEGER : guard.deadlines.earliestDeadline - 1,
-      );
-      const changed = getDb().query(`UPDATE timer_message_cutover SET phase = 'quiescing',
-        message_mutation_gate = 'closed', gate_epoch = ?, quiesce_started_at = ?, quiesce_deadline = ?,
-        ingress_policy = CASE WHEN ? THEN 'activating' ELSE ingress_policy END,
-        phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND phase = 'leased'`)
-        .run(epoch, now, deadline, this.#authoritativeActivation ? 1 : 0,
-          appendJournal(row, { epoch, phase: "quiescing", at: now, reason: "MESSAGE mutation gate closed" }),
-          now, epoch, this.ownerId);
-      if (changed.changes !== 1) throw new Error("cutover lease lost before gate close");
-      return true;
-    });
+  async #closeGate(epoch: number, now: number): Promise<boolean> {
+    return await this.#writeStep("closeGate", { epoch, now });
   }
 
-  #transition(epoch: number, phase: TimerMessageCutoverPhase, reason: string): void {
+  async #transition(epoch: number, phase: TimerMessageCutoverPhase, reason: string): Promise<void> {
     const now = this.#now();
-    writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      const changed = getDb().query(`UPDATE timer_message_cutover SET phase = ?, phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?`)
-        .run(phase, appendJournal(row, { epoch, phase, at: now, reason }), now, epoch, this.ownerId, now);
-      if (changed.changes !== 1) throw new Error(`cutover lease lost before phase ${phase}`);
-    });
+    await this.#writeStep("transition", { epoch, phase, reason, now });
   }
 
   async #finishAbort(epoch: number, reason: string): Promise<void> {
     await this.#faults.beforeLeaseRelease?.(epoch);
     const now = this.#now();
-    writeTransaction(() => {
-      const row = this.#owned(epoch, now);
-      const changed = getDb().query(`UPDATE timer_message_cutover SET
-        phase = 'idle', lease_owner = NULL, lease_expires_at = NULL,
-        quiesce_started_at = NULL, quiesce_deadline = NULL,
-        message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
-        phase_journal = ?, updated_at = ?
-        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?`)
-        .run(appendJournal(row, { epoch, phase: "idle", at: now, reason: `gate reopened: ${reason}` }),
-          now, epoch, this.ownerId, now);
-      if (changed.changes !== 1) throw new Error("cutover lease lost before abort release");
-    });
+    await this.#writeStep("finishAbort", { epoch, reason, now });
     this.#epoch = undefined;
   }
-
-  #owned(epoch: number, now: number): TimerMessageCutoverRow {
-    const row = readCutover();
-    if (!row || row.epoch !== epoch || row.leaseOwner !== this.ownerId || (row.leaseExpiresAt ?? 0) <= now) {
-      throw new Error(`cutover epoch ${epoch} is not owned by ${this.ownerId}`);
-    }
-    return row;
-  }
 }

package/src/db/timer-cutover-state.ts

@@ -0,0 +1,40 @@
+import { getDb } from "./connection";
+import type { TimerMessageCutoverRow } from "./timer-message-flip";
+import type { TimerMessageCutoverPhase, TimerMessageCutoverJournalEntry } from "./timer-cutover-core";
+
+export function normalizePhase(phase: string): TimerMessageCutoverPhase {
+  return phase === "leased" || phase === "quiescing" || phase === "draining" || phase === "quiesced" || phase === "aborting"
+    ? phase
+    : "idle";
+}
+
+export function journalOf(row: TimerMessageCutoverRow): TimerMessageCutoverJournalEntry[] {
+  return Array.isArray(row.phaseJournal) ? row.phaseJournal as TimerMessageCutoverJournalEntry[] : [];
+}
+
+export function appendJournal(row: TimerMessageCutoverRow, entry: TimerMessageCutoverJournalEntry): string {
+  return JSON.stringify([...journalOf(row), entry]);
+}
+
+export function readCutover(): TimerMessageCutoverRow | null {
+  const row = getDb().query("SELECT * FROM timer_message_cutover WHERE id = 1").get() as any;
+  return row ? {
+    id: 1,
+    protocolVersion: row.protocol_version,
+    epoch: row.epoch,
+    desiredMode: row.desired_mode,
+    observedMode: row.observed_mode,
+    phase: row.phase,
+    leaseOwner: row.lease_owner,
+    leaseExpiresAt: row.lease_expires_at,
+    quiesceStartedAt: row.quiesce_started_at,
+    quiesceDeadline: row.quiesce_deadline,
+    lastCompletedTimer: row.last_completed_timer,
+    ingressPolicy: row.ingress_policy,
+    messageMutationGate: row.message_mutation_gate,
+    gateEpoch: row.gate_epoch,
+    abortRequested: row.abort_requested === 1,
+    phaseJournal: JSON.parse(row.phase_journal),
+    updatedAt: row.updated_at,
+  } : null;
+}

package/src/db/timer-cutover-write-ops.ts

@@ -1,3 +1,4 @@
+import { readCutover, appendJournal } from "./timer-cutover-state";
 // Narrow cutover recovery write operations. The release is deliberately critical: it reopens the
 // dispatch gate after a crashed transfer; shedding it can leave both dispatchers fenced indefinitely.
 
@@ -19,15 +20,6 @@ interface BatchCheckpoint {
   done: boolean;
 }
 
-type TimerMessageCutoverPhase = "idle" | "leased" | "quiescing" | "draining" | "quiesced" | "aborting";
-
-interface TimerMessageCutoverJournalEntry {
-  epoch: number;
-  phase: TimerMessageCutoverPhase | "takeover" | "renewed";
-  at: number;
-  reason?: string;
-}
-
 function operationRow(): OperationRow | null {
   return getDb().query("SELECT failures, journal FROM timer_message_cutover_operation WHERE id=1")
     .get() as OperationRow | null;
@@ -37,37 +29,6 @@ function appendOperationEntry(row: OperationRow, entry: Record<string, unknown>)
   return JSON.stringify([...JSON.parse(row.journal) as Array<Record<string, unknown>>, entry]);
 }
 
-function readCutover(): TimerMessageCutoverRow | null {
-  const row = getDb().query("SELECT * FROM timer_message_cutover WHERE id = 1").get() as any;
-  return row ? {
-    id: 1,
-    protocolVersion: row.protocol_version,
-    epoch: row.epoch,
-    desiredMode: row.desired_mode,
-    observedMode: row.observed_mode,
-    phase: row.phase,
-    leaseOwner: row.lease_owner,
-    leaseExpiresAt: row.lease_expires_at,
-    quiesceStartedAt: row.quiesce_started_at,
-    quiesceDeadline: row.quiesce_deadline,
-    lastCompletedTimer: row.last_completed_timer,
-    ingressPolicy: row.ingress_policy,
-    messageMutationGate: row.message_mutation_gate,
-    gateEpoch: row.gate_epoch,
-    abortRequested: row.abort_requested === 1,
-    phaseJournal: JSON.parse(row.phase_journal),
-    updatedAt: row.updated_at,
-  } : null;
-}
-
-function journalOf(row: TimerMessageCutoverRow): TimerMessageCutoverJournalEntry[] {
-  return Array.isArray(row.phaseJournal) ? row.phaseJournal as TimerMessageCutoverJournalEntry[] : [];
-}
-
-function appendJournal(row: TimerMessageCutoverRow, entry: TimerMessageCutoverJournalEntry): string {
-  return JSON.stringify([...journalOf(row), entry]);
-}
-
 /**
  * Operator-abort escape hatch for an interrupted transfer. This is a writer-op body, so the
  * gateway executor already owns its transaction; keeping it here avoids a writer-registry →

package/src/db/token-db-write-ops.ts

@@ -1,3 +1,4 @@
+import { retireHandedOffPredecessorBody } from "./token-standing";
 // Token-db writer-op registry (ADR 0001 §3.1/§3.4/§3.7.1, S3 L-F #1898). Domain-owned map; the op
 // BODIES live in `../token-db.ts` and the shared `write-ops.ts` edit is one import + spread.
 //
@@ -72,6 +73,7 @@ function stripHandoffToken(result: unknown): unknown {
 
 export const TOKEN_DB_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object.freeze({
   // Non-idempotent (mints a new row each apply); the facade supplies `jti` as the ledger key.
+  "tokens.retireHandedOffPredecessor": { offLoop: true, idempotent: true, fn: (args: { jti: string }, ctx) => retireHandedOffPredecessorBody(args.jti, ctx.now) },
   "tokens.create": {
     offLoop: true,
     idempotent: false,

package/src/db/token-standing-async.ts

@@ -0,0 +1,12 @@
+import { getGateway } from "./gateway";
+import { trackRelayEvent } from "../relay-event-drain";
+
+// A concurrent authentication must see a predecessor as retired while its writer op is pending.
+const retiring = new Set<string>();
+export function tokenRetirementPending(jti: string): boolean { return retiring.has(jti); }
+export function retireHandedOffPredecessor(jti: string): void {
+  if (retiring.has(jti)) return;
+  retiring.add(jti);
+  const write = getGateway().write("tokens.retireHandedOffPredecessor", { jti });
+  trackRelayEvent(write.finally(() => retiring.delete(jti)), "token.handoff.retired");
+}

package/src/db/token-standing.ts

@@ -3,7 +3,7 @@ import { participatesInSpawnDelegation, spawnDelegationState, spawnTreeStopped }
 import { getDb } from "./connection.ts";
 import type { TokenRevokeReason } from "../security.ts";
 
-export function isTokenRevoked(jti: string): boolean {
+export function isTokenRevoked(jti: string, retire?: (predecessor: string) => void): boolean {
   let row: { revoked_at: number | null; retire_jti: string | null; agent_id: string | null } | null;
   try {
     // #1245 — the same lookup names a handed-off predecessor this token has not retired yet.
@@ -24,19 +24,16 @@ export function isTokenRevoked(jti: string): boolean {
     const state = spawnDelegationState(row.agent_id);
     if (state.participating && (state.displaced || spawnTreeStopped(row.agent_id))) return true;
   }
-  if (row.retire_jti) retireHandedOffPredecessor(row.retire_jti);
+  if (row.retire_jti) retire?.(row.retire_jti);
   return false;
 }
 
 // A successor's first authentication proves its holder has it, so the predecessor it replaced (token-db.ts
 // handoffTokenInternal) is retired as a rotation. A failed write is retried by the successor's next request.
-function retireHandedOffPredecessor(jti: string): void {
-  try {
-    getDb().query("UPDATE tokens SET revoked_at = COALESCE(revoked_at, ?), revoked_reason = 'rotation' WHERE jti = ? AND (revoked_at IS NULL OR revoked_reason = 'stale')")
-      .run(Math.floor(Date.now() / 1000), jti);
-  } catch {
-    // The successor stays valid; its next authentication retries.
-  }
+export function retireHandedOffPredecessorBody(jti: string, now: number): null {
+  getDb().query("UPDATE tokens SET revoked_at = COALESCE(revoked_at, ?), revoked_reason = 'rotation' WHERE jti = ? AND (revoked_at IS NULL OR revoked_reason = 'stale')")
+    .run(Math.floor(now / 1000), jti);
+  return null;
 }
 
 // Returns undefined if the token is not revoked, otherwise its reason string

package/src/db/token-usage-retention.ts

@@ -0,0 +1,33 @@
+import { getDb } from "./connection";
+
+export const TOKEN_USAGE_RETENTION_SCHEMA = `
+  CREATE TABLE IF NOT EXISTS retired_token_usage_totals (
+    agent_id TEXT PRIMARY KEY, tokens INTEGER NOT NULL, unknown_reports INTEGER NOT NULL);
+  CREATE TABLE IF NOT EXISTS token_usage_retention_state (
+    id INTEGER PRIMARY KEY CHECK (id = 1), expired_before INTEGER NOT NULL);
+`;
+
+/** #3619 — old retries must not resurrect reports already folded into permanent budget totals. */
+export function isTokenUsageExpired(createdAt: number): boolean {
+  const state = getDb().query("SELECT expired_before FROM token_usage_retention_state WHERE id = 1")
+    .get() as { expired_before: number } | null;
+  return state !== null && createdAt < state.expired_before;
+}
+
+/** Writer-op body: fold each deleted report into its agent's permanent spend in the SAME commit. */
+export function pruneTokenUsage(cutoff: number, limit: number): { pruned: number; moreWork: boolean } {
+  const db = getDb();
+  db.query(`INSERT INTO token_usage_retention_state VALUES (1, ?)
+    ON CONFLICT(id) DO UPDATE SET expired_before = max(expired_before, excluded.expired_before)`).run(cutoff);
+  const rows = db.query(`SELECT id, agent_id, input_tokens + output_tokens + cache_read_tokens + cache_write_tokens
+    + COALESCE(reasoning_output_tokens, 0) AS tokens, usage_status IS NOT NULL AS unknown_reports
+    FROM session_token_usage WHERE created_at < ? ORDER BY created_at, id LIMIT ?`)
+    .all(cutoff, limit) as { id: number; agent_id: string; tokens: number; unknown_reports: number }[];
+  for (const row of rows) {
+    db.query(`INSERT INTO retired_token_usage_totals VALUES (?, ?, ?)
+      ON CONFLICT(agent_id) DO UPDATE SET tokens = tokens + excluded.tokens,
+        unknown_reports = unknown_reports + excluded.unknown_reports`).run(row.agent_id, row.tokens, row.unknown_reports);
+    db.query("DELETE FROM session_token_usage WHERE id = ?").run(row.id);
+  }
+  return { pruned: rows.length, moreWork: rows.length === limit };
+}

package/src/db/token-usage-write-ops.ts

@@ -13,7 +13,7 @@ export const TOKEN_USAGE_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Obj
     idempotent: true,
     fn: (args: { input: SessionTokenUsageInput }, ctx: WriterOpContext) => {
       const result = recordSessionTokenUsage(args.input);
-      const budgetNotices = recordExhaustedSpawnBudgets(result.record.agentId, ctx.now);
+      const budgetNotices = result.record ? recordExhaustedSpawnBudgets(result.record.agentId, ctx.now) : [];
       return { ...result, budgetNotices };
     },
   },