@humwick/server 0.130.2 → 0.130.3

package/src/db/terminal-events.ts

@@ -111,3 +111,18 @@ export function clearAgentNotice(child: Pick<AgentCard, "id" | "epoch" | "meta">
   getDb().query("DELETE FROM agent_exit_notices WHERE agent_id = ? AND epoch = ? AND spawn_request_id = ? AND kind = ?")
     .run(child.id, child.epoch, stringValue(child.meta?.spawnRequestId) ?? "", kind);
 }
+
+export interface AgentNoticeSlot {
+  child: Pick<AgentCard, "id" | "epoch" | "meta">;
+  kind: string;
+}
+
+/** Called inside notifications.agentNotice, so failure rolls back the slot and all sends. */
+export function insertAgentNoticeSlot({ child, kind }: AgentNoticeSlot, now: number): boolean {
+  if (kind === "agent.ready") {
+    getDb().query("DELETE FROM agent_exit_notices WHERE agent_id = ? AND kind = 'agent.ready' AND NOT (epoch = ? AND spawn_request_id = ?)")
+      .run(child.id, child.epoch, stringValue(child.meta?.spawnRequestId) ?? "");
+  }
+  return getDb().query("INSERT OR IGNORE INTO agent_exit_notices (agent_id, epoch, spawn_request_id, kind, created_at) VALUES (?, ?, ?, ?, ?)")
+    .run(child.id, child.epoch, stringValue(child.meta?.spawnRequestId) ?? "", kind, now).changes > 0;
+}

package/src/db/timer-cutover-bodies.ts

@@ -0,0 +1,292 @@
+import { getDb, currentWriteTransactionContext } from "./connection";
+import { clearExpiredLegacyMessageClaimsInTx, timerMessageDeadlineSnapshot } from "./timer-message-cutover";
+import { normalizePhase, readCutover, appendJournal } from "./timer-cutover-state";
+import type { KernelTimersDesiredMode, TimerMessageBackend, TimerMessageCutoverRow } from "./timer-message-flip";
+import type { TimerMessageCutoverPhase, TimerMessageCutoverJournalEntry, TimerMessageDeadlineGuard } from "./timer-cutover-core";
+import type { WriterOpSpec } from "./write-ops";
+
+export interface CutoverWriterState {
+  ownerId: string; epoch?: number; leaseMs: number; quiesceTimeoutMs: number; worstCaseMs: number;
+  authoritativeActivation: boolean; operationLeaseFence?: { requestId: string; ownerId: string };
+}
+export interface CutoverStepArgs {
+  abort: { now: number; reason: string };
+  activate: { now: number; epoch: number };
+  completeLegacy: { now: number; epoch: number };
+  completeTransfer: { now: number; epoch: number; mode: KernelTimersDesiredMode; expectedBackend: TimerMessageBackend };
+  reverse: { now: number; epoch: number; targetMode: KernelTimersDesiredMode; sourceObservedMode: KernelTimersDesiredMode };
+  clearClaims: { now: number };
+  acquire: { now: number; create: boolean };
+  renew: { now: number; epoch: number };
+  closeGate: { epoch: number; now: number };
+  transition: { epoch: number; phase: TimerMessageCutoverPhase; reason: string; now: number };
+  finishAbort: { epoch: number; reason: string; now: number };
+}
+export interface CutoverStepResults {
+  abort: boolean;
+  activate: void;
+  completeLegacy: void;
+  completeTransfer: void;
+  reverse: void;
+  clearClaims: number;
+  acquire: { acquiredEpoch?: number };
+  renew: boolean;
+  closeGate: boolean;
+  transition: void;
+  finishAbort: void;
+}
+function owned(state: CutoverWriterState, epoch: number, now: number): TimerMessageCutoverRow {
+    const row = readCutover();
+    if (!row || row.epoch !== epoch || row.leaseOwner !== state.ownerId || (row.leaseExpiresAt ?? 0) <= now) {
+      throw new Error(`cutover epoch ${epoch} is not owned by ${state.ownerId}`);
+    }
+    return row;
+  }
+function deadlineGuard(state: CutoverWriterState, at: number): TimerMessageDeadlineGuard {
+  const deadlines = timerMessageDeadlineSnapshot();
+  const slackMs = deadlines.earliestDeadline === null ? null : deadlines.earliestDeadline - at;
+  return { safe: slackMs === null || slackMs >= state.worstCaseMs, slackMs, requiredMs: state.worstCaseMs, deadlines };
+}
+const steps = {
+  abort: (state: CutoverWriterState, { now, reason }: CutoverStepArgs["abort"]) => {
+      const row = readCutover();
+      if (!row || normalizePhase(row.phase) === "idle") return false;
+      return getDb().query(`UPDATE timer_message_cutover
+        SET abort_requested = 1, phase_journal = ?, updated_at = ? WHERE id = 1`)
+        .run(appendJournal(row, { epoch: row.epoch, phase: "aborting", at: now, reason }), now).changes === 1;
+
+  },
+  activate: (state: CutoverWriterState, { now, epoch }: CutoverStepArgs["activate"]) => {
+      const row = owned(state, epoch, now);
+      if (normalizePhase(row.phase) !== "quiesced" || row.messageMutationGate !== "closed") {
+        throw new Error("ingress routing activation requires the live quiesced gate owner");
+      }
+      const changed = getDb().query(`UPDATE timer_message_cutover SET ingress_policy = 'ingress',
+        phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?
+          AND phase = 'quiesced' AND message_mutation_gate = 'closed'`)
+        .run(appendJournal(row, { epoch, phase: "quiesced", at: now, reason: "authoritative ingress routing activated under fence" }),
+          now, epoch, state.ownerId, now);
+      if (changed.changes !== 1) throw new Error("cutover lease lost before ingress routing activation");
+
+  },
+  completeLegacy: (state: CutoverWriterState, { now, epoch }: CutoverStepArgs["completeLegacy"]) => {
+      const row = owned(state, epoch, now);
+      if (normalizePhase(row.phase) !== "quiesced" || row.messageMutationGate !== "closed" || row.ingressPolicy !== "ingress") {
+        throw new Error("G1 completion requires fenced authoritative ingress routing");
+      }
+      const changed = getDb().query(`UPDATE timer_message_cutover SET
+        protocol_version = MAX(COALESCE(protocol_version, 1), 2),
+        phase = 'idle', lease_owner = NULL, lease_expires_at = NULL,
+        quiesce_started_at = NULL, quiesce_deadline = NULL,
+        message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
+        phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?
+          AND phase = 'quiesced' AND message_mutation_gate = 'closed' AND ingress_policy = 'ingress'`)
+        .run(appendJournal(row, { epoch, phase: "idle", at: now, reason: "G1 proven; gate reopened under legacy authority" }),
+          now, epoch, state.ownerId, now);
+      if (changed.changes !== 1) throw new Error("cutover lease lost before G1 gate reopen");
+
+  },
+  completeTransfer: (state: CutoverWriterState, { now, epoch, mode, expectedBackend }: CutoverStepArgs["completeTransfer"]) => {
+      const row = owned(state, epoch, now);
+      if (normalizePhase(row.phase) !== "quiesced" || row.messageMutationGate !== "closed"
+        || row.ingressPolicy !== "ingress") {
+        throw new Error("authority-transfer completion requires the live quiesced ingress gate owner");
+      }
+      // Only ACTIVE (scheduled) rows must converge on the target backend — terminal rows never
+      // dispatch, so their backend is irrelevant and must not block completion (an ordinary kernel-mode
+      // cancel terminalizes a kernel-owned row that a reverse would otherwise be unable to move).
+      const wrongOwners = (getDb().query(`SELECT count(*) AS n FROM timer_message_authority
+        WHERE lifecycle='scheduled' AND backend != ?`).get(expectedBackend) as { n: number }).n;
+      if (wrongOwners !== 0) throw new Error(`${wrongOwners} scheduled MESSAGE owners are not ${expectedBackend}`);
+      const expectedFrozen = expectedBackend === "kernel" ? 1 : 0;
+      const projectionMismatch = (getDb().query(`SELECT count(*) AS n FROM timer_message_authority a
+          JOIN scheduled_timers s ON s.id=a.timer_id
+          WHERE a.lifecycle='scheduled' AND (s.command IS NOT NULL OR s.message_authority_frozen != ?)`)
+          .get(expectedFrozen) as { n: number }).n;
+      if (projectionMismatch !== 0) throw new Error(`${projectionMismatch} MESSAGE compatibility projections disagree with ${mode}`);
+      const changed = getDb().query(`UPDATE timer_message_cutover SET
+        desired_mode = ?, observed_mode = ?, phase = 'idle', lease_owner = NULL,
+        lease_expires_at = NULL, quiesce_started_at = NULL, quiesce_deadline = NULL,
+        message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
+        phase_journal = ?, updated_at = ?
+        WHERE id=1 AND epoch=? AND lease_owner=? AND lease_expires_at>?
+          AND phase='quiesced' AND message_mutation_gate='closed' AND ingress_policy='ingress'`)
+        .run(mode, mode, appendJournal(row, {
+          epoch, phase: "idle", at: now,
+          reason: `${mode} per-row ownership proven; MESSAGE mutation gate reopened`,
+        }), now, epoch, state.ownerId, now);
+      if (changed.changes !== 1) throw new Error("cutover lease lost before authority-transfer completion");
+
+  },
+  reverse: (state: CutoverWriterState, { now, epoch, targetMode, sourceObservedMode }: CutoverStepArgs["reverse"]) => {
+      const row = owned(state, epoch, now);
+      const changed = getDb().query(`UPDATE timer_message_cutover SET desired_mode=?,
+        phase_journal=?, updated_at=?
+        WHERE id=1 AND epoch=? AND lease_owner=? AND lease_expires_at>?
+          AND phase='quiesced' AND message_mutation_gate='closed' AND observed_mode=?`)
+        .run(targetMode, appendJournal(row, {
+          epoch, phase: "quiesced", at: now,
+          reason: `${targetMode} rollback desired; kernel decision commits quiesced for reverse projection`,
+        }), now, epoch, state.ownerId, now, sourceObservedMode);
+      if (changed.changes !== 1) throw new Error(`reverse transfer requires observed ${sourceObservedMode} under the live fence`);
+
+  },
+  clearClaims: (state: CutoverWriterState, { now }: CutoverStepArgs["clearClaims"]) => {return clearExpiredLegacyMessageClaimsInTx(currentWriteTransactionContext()!, now);
+  },
+  acquire: (state: CutoverWriterState, { now, create }: CutoverStepArgs["acquire"]) => {let acquiredEpoch: number | undefined; let takeover = false;
+    (() => {
+      // Stage-10 pins the resolved leg before it asks this independent coordinator to close the
+      // global gate. Check that pin's operation lease under THIS acquisition transaction: if an
+      // abort won after pin but before requestQuiesce, it cannot be followed by a stale gate close.
+      const fence = state.operationLeaseFence;
+      if (fence) {
+        const live = getDb().query(`SELECT 1 FROM timer_message_cutover_operation
+          WHERE id=1 AND request_id=? AND state='running' AND abort_requested=0
+            AND lease_owner=? AND lease_expires_at > ?`)
+          .get(fence.requestId, fence.ownerId, now);
+        if (!live) return;
+      }
+      const row = readCutover();
+      if (!row) {
+        if (!create) return;
+        acquiredEpoch = 1;
+        const journal: TimerMessageCutoverJournalEntry[] = [{ epoch: 1, phase: "leased", at: now, reason: "initial lease acquired" }];
+        getDb().query(`INSERT INTO timer_message_cutover
+          (id, protocol_version, epoch, desired_mode, observed_mode, phase, lease_owner, lease_expires_at,
+           quiesce_started_at, quiesce_deadline, last_completed_timer, ingress_policy,
+           message_mutation_gate, gate_epoch, abort_requested, phase_journal, updated_at)
+          VALUES (1, 1, 1, 'shadow', 'shadow', 'leased', ?, ?, NULL, NULL, NULL, 'legacy', 'open', NULL, 0, ?, ?)`)
+          .run(state.ownerId, now + state.leaseMs, JSON.stringify(journal), now);
+        return;
+      }
+      const rowPhase = normalizePhase(row.phase);
+      const liveForeign = row.leaseOwner !== null && row.leaseOwner !== state.ownerId && (row.leaseExpiresAt ?? 0) > now;
+      if (liveForeign) return;
+      if (row.leaseOwner === state.ownerId && (row.leaseExpiresAt ?? 0) > now && state.epoch === row.epoch) {
+        acquiredEpoch = row.epoch;
+        return;
+      }
+      const nextEpoch = row.epoch + 1;
+      takeover = row.leaseOwner !== null || rowPhase !== "idle" || row.messageMutationGate === "closed";
+      const nextPhase: TimerMessageCutoverPhase = rowPhase === "idle" ? "leased" : rowPhase;
+      const entry: TimerMessageCutoverJournalEntry = {
+        epoch: nextEpoch,
+        phase: takeover ? "takeover" : "leased",
+        at: now,
+        reason: takeover ? `expired epoch ${row.epoch} taken over in phase ${rowPhase}` : "lease acquired",
+      };
+      const result = getDb().query(`UPDATE timer_message_cutover SET
+        protocol_version = COALESCE(protocol_version, 1), epoch = ?, phase = ?, lease_owner = ?, lease_expires_at = ?,
+        gate_epoch = CASE WHEN message_mutation_gate = 'closed' THEN ? ELSE NULL END,
+        phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND (lease_owner IS NULL OR lease_expires_at <= ? OR lease_owner = ?)`)
+        .run(nextEpoch, nextPhase, state.ownerId, now + state.leaseMs, nextEpoch,
+          appendJournal(row, entry), now, now, state.ownerId);
+      if (result.changes === 1) acquiredEpoch = nextEpoch;
+    })();
+    return { acquiredEpoch };
+  },
+  renew: (state: CutoverWriterState, { now, epoch }: CutoverStepArgs["renew"]) => {
+      const row = readCutover();
+      if (!row || row.epoch !== epoch || row.leaseOwner !== state.ownerId || (row.leaseExpiresAt ?? 0) <= now) return false;
+      const deadlineCap = row.messageMutationGate === "closed" && row.quiesceDeadline !== null
+        ? row.quiesceDeadline
+        : Number.MAX_SAFE_INTEGER;
+      const nextExpiry = Math.min(now + state.leaseMs, deadlineCap);
+      return getDb().query(`UPDATE timer_message_cutover SET lease_expires_at = ?, phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?`)
+        .run(nextExpiry, appendJournal(row, { epoch, phase: "renewed", at: now }), now,
+          epoch, state.ownerId, now).changes === 1;
+
+  },
+  closeGate: (state: CutoverWriterState, { epoch, now }: CutoverStepArgs["closeGate"]) => {
+      const row = owned(state, epoch, now);
+      // `requestQuiesce` acquires the coordinator lease before the following tick closes the gate.
+      // Recheck the Stage-10 operation lease in this close transaction: an abort (or expiry) that
+      // won in that interval must leave this independent coordinator open and unleased, never
+      // permit stale authority to close the MESSAGE gate after the abort's release observed it open.
+      const fence = state.operationLeaseFence;
+      if (fence) {
+        const live = getDb().query(`SELECT 1 FROM timer_message_cutover_operation
+          WHERE id=1 AND request_id=? AND state='running' AND abort_requested=0
+            AND lease_owner=? AND lease_expires_at > ?`)
+          .get(fence.requestId, fence.ownerId, now);
+        if (!live) {
+          const released = getDb().query(`UPDATE timer_message_cutover SET
+            phase = 'idle', lease_owner = NULL, lease_expires_at = NULL,
+            quiesce_started_at = NULL, quiesce_deadline = NULL,
+            message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
+            desired_mode = observed_mode, phase_journal = ?, updated_at = ?
+            WHERE id=1 AND epoch=? AND lease_owner=? AND lease_expires_at>? AND phase='leased'`)
+            .run(appendJournal(row, {
+              epoch, phase: "idle", at: now,
+              reason: "Stage-10 operation lease no longer authorizes MESSAGE gate close",
+            }), now, epoch, state.ownerId, now);
+          if (released.changes !== 1) throw new Error("cutover lease lost while releasing stale Stage-10 gate request");
+          state.epoch = undefined;
+          return false;
+        }
+      }
+      // Re-read all three deadline sources under the SAME writer lock that closes the gate. A
+      // deadline mutation that won the race is therefore observed here and refuses entry; one that
+      // loses cannot commit until after the closed marker and is caught by the next continuous tick.
+      const guard = deadlineGuard(state, now);
+      if (!guard.safe) {
+        const changed = getDb().query(`UPDATE timer_message_cutover SET phase = 'aborting', abort_requested = 1,
+          phase_journal = ?, updated_at = ? WHERE id = 1 AND epoch = ? AND lease_owner = ? AND phase = 'leased'`)
+          .run(appendJournal(row, {
+            epoch, phase: "aborting", at: now,
+            reason: `deadline guard refused atomic gate close (slack=${guard.slackMs}, required=${guard.requiredMs})`,
+          }), now, epoch, state.ownerId);
+        if (changed.changes !== 1) throw new Error("cutover lease lost while refusing gate close");
+        return false;
+      }
+      const deadline = Math.min(
+        now + state.quiesceTimeoutMs,
+        guard.deadlines.earliestDeadline === null ? Number.MAX_SAFE_INTEGER : guard.deadlines.earliestDeadline - 1,
+      );
+      const changed = getDb().query(`UPDATE timer_message_cutover SET phase = 'quiescing',
+        message_mutation_gate = 'closed', gate_epoch = ?, quiesce_started_at = ?, quiesce_deadline = ?,
+        ingress_policy = CASE WHEN ? THEN 'activating' ELSE ingress_policy END,
+        phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND phase = 'leased'`)
+        .run(epoch, now, deadline, state.authoritativeActivation ? 1 : 0,
+          appendJournal(row, { epoch, phase: "quiescing", at: now, reason: "MESSAGE mutation gate closed" }),
+          now, epoch, state.ownerId);
+      if (changed.changes !== 1) throw new Error("cutover lease lost before gate close");
+      return true;
+
+  },
+  transition: (state: CutoverWriterState, { epoch, phase, reason, now }: CutoverStepArgs["transition"]) => {
+      const row = owned(state, epoch, now);
+      const changed = getDb().query(`UPDATE timer_message_cutover SET phase = ?, phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?`)
+        .run(phase, appendJournal(row, { epoch, phase, at: now, reason }), now, epoch, state.ownerId, now);
+      if (changed.changes !== 1) throw new Error(`cutover lease lost before phase ${phase}`);
+
+  },
+  finishAbort: (state: CutoverWriterState, { epoch, reason, now }: CutoverStepArgs["finishAbort"]) => {
+      const row = owned(state, epoch, now);
+      const changed = getDb().query(`UPDATE timer_message_cutover SET
+        phase = 'idle', lease_owner = NULL, lease_expires_at = NULL,
+        quiesce_started_at = NULL, quiesce_deadline = NULL,
+        message_mutation_gate = 'open', gate_epoch = NULL, abort_requested = 0,
+        phase_journal = ?, updated_at = ?
+        WHERE id = 1 AND epoch = ? AND lease_owner = ? AND lease_expires_at > ?`)
+        .run(appendJournal(row, { epoch, phase: "idle", at: now, reason: `gate reopened: ${reason}` }),
+          now, epoch, state.ownerId, now);
+      if (changed.changes !== 1) throw new Error("cutover lease lost before abort release");
+
+  },
+};
+export const TIMER_CUTOVER_COORDINATOR_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object.freeze({
+  "timerCutover.coordinatorStep": { offLoop: true, idempotent: false,
+    fn: (input: { step: keyof CutoverStepArgs; state: CutoverWriterState; args: CutoverStepArgs[keyof CutoverStepArgs] }) => {
+      const epoch = input.state.epoch;
+      const body = steps[input.step] as (state: CutoverWriterState, args: typeof input.args) => unknown;
+      const value = body(input.state, input.args);
+      return { value: value ?? null, ...(input.state.epoch !== epoch ? { clearEpoch: true } : {}) };
+    } },
+});