@humwick/server 0.0.0-stage → 0.130.1

package/src/maintenance/scheduler.ts

@@ -0,0 +1,778 @@
+import { timePostBindPhase } from "../post-bind-startup";
+import { randomUUID } from "node:crypto";
+import { errMessage, isRecord } from "@humwick/sdk";
+import { classifyPidLiveness } from "@humwick/sdk/process-utils";
+import { CRITICAL_MAINTENANCE_PROGRESS_STALE_MS, createActivityEventAsync, getDb, isSqliteBusyError, MAINTENANCE_RUN_AUTORESET_GUARD_SQL, recordDrainConvoy, writeTransaction } from "../db";
+import { claimMaintenanceRunAsync, finishMaintenanceRunAsync, heartbeatMaintenanceRunAsync } from "../db/maintenance-job-runs-async";
+import type { MaintenanceRunOutcome } from "../db/maintenance-job-runs";
+import { withAbortDeadline } from "../async-deadline";
+import { MAINTENANCE_BREAKER_LAG_MS } from "../config";
+import { findPauseByJobId, getMaintenanceJobPause, notPausedSql } from "../db/maintenance-pauses";
+import { setLagBreakerObserver } from "../event-loop-lag";
+import { resetRateLimitResumeTrackerForTests } from "../services/rate-limit-resume";
+import { resetTransientTrackerForTests } from "../services/transient-agent-reaper";
+import { isEphemeralMode } from "../execution-mode";
+import { DEFAULT_DEADLINE_MS, SCHEDULER_TICK_MS } from "./constants";
+import { LEGACY_REAPER_JOB_IDS, definitions } from "./jobs";
+import { resetOrphanSessionTrackerForTests } from "./orphaned-sessions";
+import {
+  MaintenanceBreakerError,
+  MaintenanceCancelledError,
+  MaintenanceTimeoutError,
+  abortRun,
+  beginRun,
+  cancelMaintenanceRun,
+  endRun,
+  findMaintenanceRunsOverlapping,
+  gapBlockOverlapMs,
+  isRunActive,
+  listRecentMaintenanceRuns,
+  listRunningMaintenanceRuns,
+  markRunOffThread,
+  resetMaintenanceRunRegistryForTests,
+} from "./run-registry";
+import { clearBreakerStrikes, isRunStruck, recordBreakerStrike } from "./breaker";
+import type { MaintenanceJob, MaintenanceJobRun, MaintenanceRunRecord } from "../types";
+import type { MaintenanceJobDefinition, MaintenanceJobRow } from "./types";
+
+// #1728 stage 0 / #1724 stage 1 — the run registry (what is running, what just closed, attribution,
+// cancel/abort) lives in ./run-registry now; the scheduler re-exports its public surface so the API
+// routes and tests that import these from "./scheduler" (or the ../maintenance barrel) are unaffected
+// by the split. `cancelMaintenanceRun` is the new #1724 cancel primitive the route calls.
+export {
+  cancelMaintenanceRun,
+  findMaintenanceRunsOverlapping,
+  listRecentMaintenanceRuns,
+  listRunningMaintenanceRuns,
+  resetMaintenanceRunRegistryForTests,
+};
+
+let timer: Timer | null = null;
+const MAINTENANCE_PROGRESS_HEARTBEAT_MS = 60_000;
+
+// #1724 stage 1 / #1725 P1 — the circuit-breaker ACTUATOR seam, registered once at module load (like
+// the attributor the run-registry installs). Fired by the event-loop-lag sampler for any lag sample
+// at/above the breaker threshold: a genuine freeze. It (a) aborts the blamed run if it is STILL in
+// flight — the "kill/abandon the tick" of the brief, which preempts a hanging ASYNC handler — and
+// (b) takes a breaker strike against its job, which durably auto-pauses (non-critical) or escalates
+// (critical) once strikes accumulate. A synchronous freeze cannot be aborted mid-run (single thread;
+// that is Stage 3's worker terminate), and by the time this fires the frozen run has usually already
+// closed — so `abortRun` no-ops and only the strike lands. That is still the win.
+//
+// #1725 P1 — ATTRIBUTION REPAIR. The pre-P1 actuator struck EVERY overlapping run, so one convoy
+// freeze (many small jobs draining as one un-paced block, §1 of the plan) durably auto-paused 21-25
+// jobs from a single sample — the 2026-07-28 incident. Interval overlap cannot tell "this job blocked
+// the loop" from "this job merely ran while something else did"; the 4 P0 review passes proved no
+// overlap heuristic can. P1 attributes on MEASURED per-run block time, LOCALIZED IN TIME to the gap:
+//   1. Strike a run only when its block INTERVAL [startedAt, startedAt+blockMs] overlaps the gap window
+//      by a material share of it. Raw blockMs MAGNITUDE is not enough — an async job whose big sync
+//      prefix ran seconds before a later freeze (workspace-conflict-scan / ci-monitor / the whole
+//      Class-C set: all block ≥500ms before their first await) would otherwise be struck for a freeze
+//      it could not have caused. Overlap-in-time is what exonerates it (P1 review F1/E).
+//   2. If NO run's block overlaps the gap enough → scheduler-shape convoy: strike NOTHING; record a
+//      `maintenance-drain-convoy` health warn (src/db/maintenance-health.ts) ONLY when >=2 runs
+//      participated AND their summed in-gap block is at least half the freeze — otherwise it is an
+//      unattributed non-maintenance freeze, not a convoy (P1 review F3).
+//   3. HARD BLAST-RADIUS CAP: at most ONE new auto-pause per lag sample (operator Q4). If several
+//      qualify, pause the single largest in-gap block and warn about the rest. One freeze may never
+//      pause a fleet.
+setLagBreakerObserver((_gapStartAt, gapEndAt, lagMs) => {
+  const reason = `blocked the event loop ~${Math.round(lagMs)}ms (measured block)`;
+  // #1725 P1 F-A — the ACTUATOR uses the TIGHTENED window `[gapEndAt - lagMs, gapEndAt]`, the actual
+  // freeze interval, NOT the sampler's diagnostic window `[_gapStartAt, gapEndAt]`. The sampler builds
+  // its window from `elapsedMs` = lagMs + EVENT_LOOP_LAG_SAMPLE_MS, so the diagnostic window is one
+  // sample interval (~1s) WIDER than the freeze — its leading slice is the sampler's own expected tick
+  // spacing, during which a block contributed PROVABLY ZERO to lagMs. Localizing a strike to that too-
+  // wide window let a >=500ms sync prefix that ended BEFORE the freeze collect up to a full interval of
+  // spurious overlap and get struck (the Class-C prefix inversion). The wide window stays ONLY on the
+  // diagnostic attributor (event-loop-lag.ts), which deliberately wants to be inclusive.
+  const freezeStartAt = gapEndAt - lagMs;
+  // #1757 — an off-thread run (heavy work in a subprocess/worker; it only awaits on the main thread)
+  // CANNOT cause a main-thread freeze and its blockMs is ~0, so it is excluded up front. Already-struck
+  // runs are KEPT here (not pre-filtered) so they still count toward "does any single run account for
+  // this gap" and toward the convoy participant list (P1 review F4) — they are excluded only at the
+  // point of taking a NEW strike, below.
+  const overlapping = findMaintenanceRunsOverlapping(freezeStartAt, gapEndAt).filter((run) => !run.offThread);
+  if (overlapping.length === 0) return;
+  // A run accounts for the freeze only when its BLOCK INTERVAL overlaps the freeze window by a material
+  // share: half the freeze. The floor `MAINTENANCE_BREAKER_LAG_MS / 2` is INERT at the shipped defaults
+  // (the observer only fires at lagMs >= MAINTENANCE_BREAKER_LAG_MS, so 0.5*lagMs already dominates); it
+  // exists solely to keep a sane bound if someone lowers the breaker threshold below the sampler
+  // interval (P1 review F-D). gapBlockOverlapMs is exact-in-time for the sync jobs that caused the
+  // incident and ~0 for an async job whose block fell outside this freeze (#1757 / the F1/F-A class).
+  const shareThresholdMs = Math.max(MAINTENANCE_BREAKER_LAG_MS / 2, 0.5 * lagMs);
+  const ranked = overlapping.map((run) => ({ run, overlapMs: gapBlockOverlapMs(run, freezeStartAt, gapEndAt) }));
+  const accounted = ranked.filter((entry) => entry.overlapMs >= shareThresholdMs);
+  if (accounted.length === 0) {
+    // Convoy candidate: no single run's block accounts for the gap. Pausing a job is the wrong actuator
+    // for a scheduler-shape freeze (2026-07-28) — strike NOTHING. Only record it as a convoy when the
+    // in-freeze blocks of >=2 distinct JOBS sum to a material share of the freeze; a lone or tiny
+    // overlap is an unattributed non-maintenance freeze (something outside maintenance blocked the
+    // loop), not a convoy, and warning on it would sit at warn forever (P1 review F3).
+    //
+    // Participants are deduped by jobId (P1 review F5): the recent-run ring can hold two runs of the
+    // SAME job that both overlap one freeze, and the warn must name a job once (with its summed
+    // in-freeze block) rather than listing it twice — and the >=2 gate must count distinct jobs, not
+    // runs, or two runs of one job would masquerade as a convoy. Every overlapping (non-off-thread)
+    // run counts, INCLUDING any already struck: the operator needs the full picture of what drained
+    // together, not a set that silently drops the run struck earlier (P1 review F4). Each participant
+    // carries its IN-FREEZE block overlap (the number that actually qualified the convoy), NOT raw
+    // blockMs — raw blockMs summed to nonsense figures like "5900ms" for a 1000ms freeze (P1 review F-C).
+    const overlapByJob = new Map<string, number>();
+    for (const entry of ranked) overlapByJob.set(entry.run.jobId, (overlapByJob.get(entry.run.jobId) ?? 0) + entry.overlapMs);
+    const summedOverlapMs = [...overlapByJob.values()].reduce((sum, ms) => sum + ms, 0);
+    if (overlapByJob.size >= 2 && summedOverlapMs >= 0.5 * lagMs) {
+      recordDrainConvoy({
+        at: gapEndAt,
+        gapMs: Math.round(lagMs),
+        participants: [...overlapByJob].map(([jobId, overlapMs]) => ({ jobId, inGapBlockMs: Math.round(overlapMs) })),
+      });
+    }
+    return;
+  }
+  // A single run DOES account for the gap. Take at most ONE new strike — the largest in-gap block among
+  // the NOT-yet-struck accounted runs (the cap). If the accounting run(s) are ALL already struck (e.g.
+  // a second lag sample of the same freeze), strike nothing new: the freeze is already attributed, and
+  // re-blaming the next-largest would widen the blast radius past the 1/sample cap onto an innocent
+  // (P1 review E). Not a convoy either — a run genuinely accounts for it.
+  const strikable = accounted.filter((entry) => !isRunStruck(entry.run.runId));
+  if (strikable.length === 0) return;
+  const blamed = strikable.reduce((worst, entry) => (entry.overlapMs > worst.overlapMs ? entry : worst));
+  if (isRunActive(blamed.run.runId)) abortRun(blamed.run.runId, new MaintenanceBreakerError(`aborted by circuit breaker: ${reason}`));
+  const critical = definitions.find((job) => job.id === blamed.run.jobId)?.critical ?? false;
+  recordBreakerStrike({ runId: blamed.run.runId, jobId: blamed.run.jobId, critical, kind: "lag", reason });
+  const others = strikable.filter((entry) => entry !== blamed);
+  if (others.length > 0) {
+    console.warn(`[relay] maintenance circuit breaker: ${strikable.length} runs each blocked >=${Math.round(shareThresholdMs)}ms of this ${Math.round(lagMs)}ms freeze — struck ${blamed.run.jobId} (in-freeze block ${Math.round(blamed.overlapMs)}ms), one-per-sample cap; not struck: ${others.map((entry) => `${entry.run.jobId} (${Math.round(entry.overlapMs)}ms)`).join(", ")}`);
+  }
+});
+
+/** #3061 — time one on-loop step of a paced handler into the run's `stepBlockMs`. */
+async function measureRunBlock<T>(run: MaintenanceRunRecord, step: () => T | Promise<T>): Promise<T> {
+  const startedAt = performance.now();
+  try {
+    return await step();
+  } finally {
+    run.stepBlockMs = (run.stepBlockMs ?? 0) + performance.now() - startedAt;
+  }
+}
+
+/** The persisted `last_block_ms`: the sync prefix plus every measured step. */
+const runBlockTotalMs = (run: MaintenanceRunRecord): number => Math.round((run.blockMs ?? 0) + (run.stepBlockMs ?? 0));
+
+/** Exposes the internal `runJob` directly, bypassing the `definitions` lookup `runMaintenanceJobNow`
+ *  does, so tests can drive the run-registry/timeout/crash paths with a synthetic handler.
+ *  Production code never calls this name. */
+export const runMaintenanceJobForTests = (definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}) =>
+  runJob(definition, options);
+
+export function resetOrphanTrackerForTests(): void {
+  resetOrphanSessionTrackerForTests();
+  resetTransientTrackerForTests();
+  resetRateLimitResumeTrackerForTests();
+}
+
+export function startMaintenanceScheduler(): void {
+  // Reconcile the job table once at boot. A transient SQLITE_BUSY here must not
+  // abort startup — the first scheduler tick re-runs the reconcile with retry.
+  try {
+    ensureMaintenanceJobs();
+    scheduleRunOnStartJobs();
+  } catch (error) {
+    console.warn(`maintenance job reconcile at startup failed (will retry on next tick): ${errMessage(error)}`);
+    void ensureMaintenanceJobsResilient()
+      .then(() => {
+        scheduleRunOnStartJobs();
+        return runDueMaintenanceJobs();
+      })
+      .catch((retryError) => console.warn(`maintenance startup retry failed (will retry on next tick): ${errMessage(retryError)}`));
+  }
+  // #1493 — a relay restart orphans the maintenance-job lease its own dying
+  // process held moments earlier: lease_owner still names the dead PID and
+  // lease_until is still in the future, so the claim guard in runJob/
+  // runDueMaintenanceJobs (lease_until IS NULL OR lease_until <= now) blocks
+  // the job — e.g. the CD watcher — until the lease's TTL self-expires (up to
+  // deadlineMs + 5s). Reclaim any lease whose owner PID is no longer alive so
+  // maintenance resumes immediately after a restart instead of stalling.
+  try {
+    reclaimDeadPidLeases();
+  } catch (error) {
+    console.warn(`dead-PID maintenance lease reclaim at startup failed (will retry on next tick): ${errMessage(error)}`);
+  }
+  // Floated ticks are wrapped defensively: runDueMaintenanceJobs already catches
+  // its own errors (#1481), but a stray rejection from a floated promise must
+  // never bubble to an unhandledRejection that exits the relay process.
+  void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`));
+  if (timer) return;
+  timer = setInterval(
+    () => void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`)),
+    SCHEDULER_TICK_MS,
+  );
+}
+
+/**
+ * `runOnStart` means every Relay process start, not only the first insertion of
+ * a durable maintenance_jobs row. Historically ensureMaintenanceJobs used
+ * COALESCE on next_run_at, so an existing row retained its old interval and a
+ * restart did not run the job. Keep this as an explicit boot-only operation:
+ * doing it from the periodic reconcile would make run-on-start jobs run every
+ * scheduler tick.
+ */
+export function scheduleRunOnStartJobs(now: number = Date.now()): void {
+  getDb().query(`
+    UPDATE maintenance_jobs
+    SET next_run_at = ?, updated_at = ?
+    WHERE enabled = 1 AND run_on_start = 1
+  `).run(now, now);
+}
+
+export function listMaintenanceJobs(): MaintenanceJob[] {
+  try {
+    ensureMaintenanceJobs();
+  } catch (error) {
+    // A transient lock while reconciling must not fail a read-only listing; fall
+    // through and return the current rows (a later reconcile catches up). #1481.
+    console.warn(`maintenance job reconcile during list failed: ${errMessage(error)}`);
+  }
+  const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
+  const now = Date.now();
+  const currentIds = new Set(definitions.map((definition) => definition.id));
+  return rows.filter((row) => currentIds.has(row.id)).map((row) => rowToJob(row, now));
+}
+
+export async function runMaintenanceJobNow(id: string, options: { overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
+  await ensureMaintenanceJobsResilient();
+  const definition = definitions.find((job) => job.id === id);
+  if (!definition) throw new Error("maintenance job not found");
+  return runJob(definition, { force: true, overridePause: options.overridePause });
+}
+
+export async function runLegacyMaintenanceReaper(): Promise<Record<string, unknown>> {
+  const runs = await runMaintenanceJobsNow(LEGACY_REAPER_JOB_IDS);
+  const result = aggregateRunResults(runs);
+  await auditMaintenanceReaperRun(result);
+  return result;
+}
+
+async function runMaintenanceJobsNow(ids: string[]): Promise<MaintenanceJobRun[]> {
+  const runs: MaintenanceJobRun[] = [];
+  for (const id of ids) runs.push(await runMaintenanceJobNow(id));
+  return runs;
+}
+
+// #1724 stage 1 — re-entrancy guard for the serialized tick. The `setInterval` keeps firing every
+// SCHEDULER_TICK_MS while a prior tick is still draining; a concurrent second tick body would
+// re-introduce the parallel fan-out this stage removes, so a tick already in progress makes the next
+// no-op (the still-open tick, or the one after, picks up whatever came due). Combined with the
+// serialized drain below, this is the "one job at a time from a queue" the design (§5.4) calls for.
+let tickInFlight = false;
+
+export async function runDueMaintenanceJobs(): Promise<void> {
+  if (tickInFlight) return;
+  tickInFlight = true;
+  // A maintenance tick must DEGRADE, never crash the relay (#1481). The whole body
+  // is guarded: a transient SQLITE_BUSY (or any other failure) reconciling or
+  // querying the job table is logged and the tick is skipped — the next tick retries.
+  try {
+    await ensureMaintenanceJobsResilient();
+    const now = Date.now();
+    const progressStaleAt = now - CRITICAL_MAINTENANCE_PROGRESS_STALE_MS;
+    const rows = getDb().query(`
+      SELECT id FROM maintenance_jobs
+      WHERE enabled = 1
+        AND next_run_at IS NOT NULL
+        AND next_run_at <= ?
+        AND (lease_until IS NULL OR (lease_until <= ? AND (critical = 0 OR (${MAINTENANCE_RUN_AUTORESET_GUARD_SQL} AND last_progress_at <= ?))))
+        AND ${notPausedSql("maintenance_jobs.id")}
+      ORDER BY next_run_at ASC
+    `).all(now, now, progressStaleAt, now) as Array<{ id: string }>;
+    const due = rows
+      .map((row) => definitions.find((job) => job.id === row.id))
+      .filter((definition): definition is MaintenanceJobDefinition => definition !== undefined);
+    await drainDueJobs(due);
+  } catch (error) {
+    console.warn(`maintenance tick skipped after error: ${errMessage(error)}`);
+  } finally {
+    tickInFlight = false;
+  }
+}
+
+/**
+ * SERIALIZED (#1724 stage 1): `await` each job so due jobs run ONE AT A TIME, not the uncapped
+ * parallel `void runJob(...)` fan-out this replaced. That fan-out is what let a relay restart launch
+ * ~15 run-on-start jobs in a single tick, whose writes then contended on the one WAL writer lock and
+ * amplified the 2026-07-26 freeze (each victim write burns busy_timeout on the main thread). Draining
+ * them sequentially removes that contention. The tradeoff: a legitimately-long ASYNC job now delays
+ * the ones queued behind it — but it never blocks the event loop (that is what keeps request handling
+ * responsive, the property that actually matters), and a wedged job is bounded by its deadline, the
+ * cancel API, and the breaker. Each job is individually guarded so one failure can't abort the rest.
+ *
+ * #1725 P0 — and PACED: a macrotask yield between jobs. Serialization alone was not enough, because
+ * `await` on a SYNCHRONOUS handler resolves through microtasks only, and the microtask queue drains
+ * FULLY before any timer/immediate macrotask gets a turn. So N sync jobs ran as one contiguous block
+ * of Σ(jobs) — 23 due jobs froze the loop for ~1610ms as a single unit, during which no request, no
+ * SSE flush and not even the event-loop-lag sampler could run. That last part is why the freeze was
+ * then attributed to every maintenance run overlapping it at once and durably auto-paused 21-25 jobs:
+ * the sampler could not observe INDIVIDUAL jobs, only one giant gap covering the whole drain. The
+ * yield turns the worst gap into max(job) instead of Σ(jobs) — measured 1615ms → 143ms for ~0.3% of
+ * drain wall-clock — and gives the sampler its per-job resolution back. Attribution itself is NOT
+ * fixed here: telling "this job blocked the loop" from "this job merely ran while something else did"
+ * needs measured per-run block time, which is #1725 P1's `blockMs`. Guarded by
+ * scheduler.drain-lag.test.ts.
+ *
+ * Strictly serial still: exactly one job in flight, order preserved. This ADDS a yield between jobs;
+ * it does not reintroduce the pre-#1724 `void runJob(...)` fan-out.
+ */
+async function drainDueJobs(dueDefinitions: MaintenanceJobDefinition[]): Promise<void> {
+  for (const definition of dueDefinitions) {
+    try {
+      await runJob(definition);
+    } catch (error) {
+      console.warn(`maintenance job ${definition.id} failed: ${errMessage(error)}`);
+    }
+    // Outside the try on purpose: a failing job must not skip the pace the job behind it depends on.
+    await yieldToEventLoop();
+  }
+}
+
+/** #1725 P0 — one macrotask hop, the whole point of which is that it is NOT a microtask: it lets
+ *  pending timers and I/O callbacks (request handling, the event-loop-lag sampler) run before the
+ *  next job's synchronous body starts.
+ *
+ *  `setImmediate` per design §1.2, which measured it and `setTimeout(0)` as equivalent on total drain
+ *  wall-clock. They are NOT equivalent on the metric that matters, though — measured at 20ms probe
+ *  resolution by scheduler.drain-lag.test.ts, `setImmediate` lets the probe run once per TWO jobs
+ *  (worst gap ~142ms) where `setTimeout(0)` lands the continuation in the same timer phase as the
+ *  probe and yields once per job (worst gap ~74ms), for ~2% total wall-clock instead of ~0.3%.
+ *  §1.2's 100ms sampler was too coarse to separate them. Both are far inside the 250ms budget; if the
+ *  budget is ever tightened (or the drain-lag test proves flaky under CI load), swapping this one
+ *  expression for `setTimeout(resolve, 0)` roughly halves the worst gap. */
+function yieldToEventLoop(): Promise<void> {
+  return new Promise((resolve) => setImmediate(resolve));
+}
+
+/** #1724 stage 1 — drives the EXACT serialized drain the scheduler tick uses, over an injected set of
+ *  (synthetic) definitions, so serialization + crash-isolation are testable with handlers a test
+ *  controls — the real tick can only resolve ids against the code-owned `definitions`. Shares
+ *  `drainDueJobs` with the production tick, so it is not a parallel reimplementation. */
+export function drainDueMaintenanceJobsForTests(dueDefinitions: MaintenanceJobDefinition[]): Promise<void> {
+  return drainDueJobs(dueDefinitions);
+}
+
+/** #1724 stage 1 — reset the tick re-entrancy guard between tests (a test that aborts mid-tick could
+ *  otherwise leave it latched, wedging every later test's tick). Production never calls this. */
+export function resetSchedulerTickGuardForTests(): void {
+  tickInFlight = false;
+}
+
+// Bounded, ASYNC, loop-yielding retry around the (synchronous) job-table reconcile.
+// Mirrors runWriteWithBusyRetry (connection.ts) but wraps ensureMaintenanceJobs,
+// which runs its own BEGIN IMMEDIATE transaction and so cannot be nested inside
+// runWriteWithBusyRetry. busy_timeout already makes SQLite WAIT on a held lock; the
+// rare BUSY that survives it is retried here with jittered backoff that yields the
+// event loop between attempts. Anything else — or exhausted attempts — rethrows to
+// the caller, which degrades the tick rather than crashing the process (#1481).
+const ENSURE_BUSY_RETRY_ATTEMPTS = 4;
+const ENSURE_BUSY_RETRY_BASE_MS = 15;
+const ENSURE_BUSY_RETRY_MAX_MS = 120;
+
+async function ensureMaintenanceJobsResilient(): Promise<void> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      ensureMaintenanceJobs();
+      return;
+    } catch (error) {
+      if (attempt < ENSURE_BUSY_RETRY_ATTEMPTS && isSqliteBusyError(error)) {
+        const cap = Math.min(ENSURE_BUSY_RETRY_MAX_MS, ENSURE_BUSY_RETRY_BASE_MS * 2 ** (attempt - 1));
+        const delay = cap / 2 + Math.random() * (cap / 2);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+      throw error;
+    }
+  }
+}
+
+function ensureMaintenanceJobs(): void {
+  const db = getDb();
+  const now = Date.now();
+  // #1645 — `enabled` is re-asserted from `definitions` on every reconcile, which is what makes it
+  // CODE-owned and therefore self-healing. Before this it was set only on INSERT and deliberately
+  // omitted from the conflict clause, because the one column doubled as the operator's "stop" lever
+  // and a redeploy would have clobbered their intent. Operator intent now lives in
+  // `maintenance_job_pauses` (holder, reason, expiry), so re-asserting `enabled` overrides nobody —
+  // and a hand-written `UPDATE maintenance_jobs SET enabled=0` recovers at the next boot instead of
+  // persisting until someone writes more raw SQL (nine hours of deploy blackout, #1645).
+  //
+  // PRECONDITION: convertManualJobDisablesToPauses() (src/db/migrations.ts) must have run first, or
+  // this silently switches back on a job an operator deliberately stopped.
+  const stmt = db.query(`
+    INSERT INTO maintenance_jobs (id, title, description, interval_ms, timeout_ms, enabled, critical, run_on_start, next_run_at, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      title = excluded.title,
+      description = excluded.description,
+      interval_ms = excluded.interval_ms,
+      timeout_ms = excluded.timeout_ms,
+      run_on_start = excluded.run_on_start,
+      enabled = excluded.enabled, -- #1645: code-owned, see the note above this query
+      critical = excluded.critical,
+      next_run_at = COALESCE(maintenance_jobs.next_run_at, excluded.next_run_at),
+      updated_at = excluded.updated_at
+  `);
+  // Batch the upserts into a single BEGIN IMMEDIATE write so busy_timeout governs
+  // ONE lock acquisition (SQLite waits) instead of one per definition, cutting the
+  // lock churn that made this loop the crash site under contention (#1481).
+  writeTransaction(() => {
+    for (const definition of definitions) {
+      stmt.run(
+        definition.id,
+        definition.title,
+        definition.description,
+        definition.intervalMs,
+        // #1728: the DB column stays `timeout_ms` — renaming it is a migration, not asked for here.
+        // Code-side this is `deadlineMs`: see MaintenanceJobDefinition for why.
+        definition.deadlineMs ?? DEFAULT_DEADLINE_MS,
+        !isEphemeralMode() || definition.instanceSafe === true ? 1 : 0,
+        definition.critical ? 1 : 0,
+        definition.runOnStart ? 1 : 0,
+        definition.runOnStart ? now : now + definition.intervalMs,
+        now,
+      );
+    }
+  });
+}
+
+// #1493 — reclaim current-schema maintenance_jobs leases whose owner PID is proven dead.
+// `lease_owner` is `${pid}-${randomUUID()}` (see runJob below); a lease is only
+// ever held by this relay process's own PID (job handlers run in-process, not
+// in a spawned child), so a dead owner PID means a prior generation of this
+// same relay died mid-job and never reached the lease-clearing UPDATE in
+// runJob's success/failure paths. Waiting for lease_until to pass would stall
+// the job (and anything gated behind it, e.g. CD) for up to deadlineMs + 5s. This startup-only
+// recovery deliberately also covers pre-#2317 rows: a same-host owner PID that is positively dead
+// is proof that no old handler can overlap a replacement.
+// #3645: the crash is not a job failure — it neither bumps consecutive_failures nor replaces an existing
+// last_error (a non-null one belongs to the live failure streak; success clears it). The page-only guard remains on every
+// deadline-based reclaim/claim path, where that positive evidence is unavailable.
+export function reclaimDeadPidLeases(): void {
+  const db = getDb();
+  const now = Date.now();
+  const leased = db.query(`
+    SELECT id, lease_owner FROM maintenance_jobs
+    WHERE lease_owner IS NOT NULL
+  `).all() as Array<{ id: string; lease_owner: string }>;
+  for (const row of leased) {
+    const pid = parseLeaseOwnerPid(row.lease_owner);
+    if (pid === null || classifyPidLiveness(pid) !== "dead") continue;
+    db.query(`
+      UPDATE maintenance_jobs
+      SET lease_owner = NULL, lease_until = NULL, last_status = 'failed',
+          last_error = COALESCE(last_error, ?), run_token = NULL, last_progress_at = ?, updated_at = ?
+      WHERE id = ? AND lease_owner = ?
+    `).run(`reclaimed stale lease at startup: owner pid ${pid} is confirmed dead`, now, now, row.id, row.lease_owner);
+  }
+}
+
+function parseLeaseOwnerPid(owner: string): number | null {
+  const match = owner.match(/^(\d+)-/);
+  if (!match) return null;
+  const pid = Number(match[1]);
+  return Number.isFinite(pid) ? pid : null;
+}
+
+// #1847/#1870 S3 — the terminal outcome (status, result, lease release) is a critical writer op: a held
+// WAL lock delays it on the writer thread, never the event loop. Persistence sits OUTSIDE the breaker's
+// blockMs window. Never throws: an outcome that cannot land is logged and the lease self-heals at
+// lease_until.
+async function persistRunOutcome(outcome: MaintenanceRunOutcome): Promise<boolean> {
+  try {
+    return await finishMaintenanceRunAsync(outcome);
+  } catch (error) {
+    console.warn(`[relay] maintenance result persist failed for ${outcome.id}: ${errMessage(error)}`);
+    return false;
+  }
+}
+
+async function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
+  const startedAt = Date.now();
+  // One execution boundary covers ticks, forced runs and already-queued definitions.
+  if (isEphemeralMode() && definition.instanceSafe !== true) {
+    return { id: definition.id, status: "skipped", startedAt, finishedAt: startedAt, durationMs: 0,
+      result: { reason: "not-instance-safe" } };
+  }
+  // #1645 — an active pause blocks even a forced run, so "paused" is an actuator and not merely
+  // advisory (otherwise the first Run-now click defeats it silently). The audited break-glass is
+  // `overridePause`, which runs once and echoes the holder and reason it overrode.
+  const activePause = getMaintenanceJobPause(definition.id, startedAt);
+  if (activePause && !options.overridePause) {
+    return {
+      id: definition.id,
+      status: "skipped",
+      startedAt,
+      finishedAt: startedAt,
+      durationMs: 0,
+      result: { reason: `paused by ${activePause.holder}: ${activePause.reason}`, pause: activePause },
+    };
+  }
+  if (activePause && options.overridePause) {
+    await auditPauseOverride(definition, activePause);
+  }
+  const owner = `${process.pid}-${randomUUID()}`;
+  // Captured ONCE at claim. A run that lost ownership must not later clear the replacement's
+  // lease/status when its abandoned promise eventually resolves.
+  const runToken = randomUUID();
+  const deadlineMs = definition.deadlineMs ?? DEFAULT_DEADLINE_MS;
+  const leaseUntil = startedAt + deadlineMs + 5_000;
+  const progressStaleAt = startedAt - CRITICAL_MAINTENANCE_PROGRESS_STALE_MS;
+  const claimed = await claimMaintenanceRunAsync({ id: definition.id, owner, runToken, startedAt, leaseUntil, progressStaleAt, force: options.force === true });
+  if (!claimed) {
+    const state = listMaintenanceJobs().find((job) => job.id === definition.id);
+    return { id: definition.id, status: "skipped", startedAt, finishedAt: Date.now(), durationMs: 0, result: { reason: "not due or already running", state } };
+  }
+
+  if (definition.logRuns) console.info(`[relay] maintenance job ${definition.id} started`);
+
+  const abortController = new AbortController();
+  // A missed heartbeat fails closed: the watchdog waits for the full stale window before recovery.
+  const progressHeartbeat = setInterval(() => {
+    heartbeatMaintenanceRunAsync({ id: definition.id, runToken })
+      .catch((error) => console.warn(`[relay] maintenance progress heartbeat failed for ${definition.id}: ${errMessage(error)}`));
+  }, MAINTENANCE_PROGRESS_HEARTBEAT_MS);
+  // #1728 stage 0 — statement order is load-bearing: the registry entry is opened HERE,
+  // immediately before the handler is invoked (the next line), and closed in `finally` below —
+  // which runs on the success, failure, AND timeout-race paths alike (never left open). No tick
+  // deferral (queueMicrotask/.then/setImmediate) is introduced; this only reorders synchronous
+  // statements around a call that was already here.
+  // #1724 stage 1 — the AbortController is now registered in the run registry (via `beginRun`) so the
+  // cancel API and the breaker can reach this run in flight; `endRun` unregisters it in `finally`.
+  const run = beginRun(definition.id, startedAt, abortController);
+  let finishedAt = startedAt;
+  let timedOut = false;
+  let result: Record<string, unknown> | undefined;
+  try {
+    // #1757 — `markOffThread` lets a handler declare it has handed its heavy work to a subprocess so
+    // the lag breaker won't scapegoat its long await window; keyed to THIS run's id via the registry.
+    // #1725 P1 — measure the handler's SYNCHRONOUS block: the body runs to completion at the call
+    // below (a sync handler's whole loop occupancy), so `performance.now()` on both sides of the call,
+    // BEFORE the first await, is its exact block time. Stamped straight onto the live registry record
+    // (same object `activeRuns`/`recentRuns` hold, like `offThread`) so the breaker observer — which
+    // can only fire once the loop frees, i.e. after this line — reads it whether the run is still in
+    // flight or has just closed. For an async handler this is the prefix before its first real await
+    // (~0 for one that awaits immediately), which is still the right number to attribute a freeze on.
+    //
+    // This captures only the FIRST synchronous chunk — the prefix before the handler's first await. A
+    // handler shaped `block(); await x; block()` records only the first block (cumulative measurement
+    // is T4 scope). After the time-localized attribution below that is pure under-strike: an
+    // await-interleaving hog escapes to a warn, it can never mis-strike an innocent (P1 review F-B).
+    //
+    // The measurement is in a `finally`: a handler that blocks the loop and THEN throws SYNCHRONOUSLY
+    // (e.g. a 60s DB scan that ends in an exception) still occupied the loop for that whole block, and
+    // must record it — otherwise `blockMs` stays undefined, persists 0, and the freeze it caused is
+    // silently downgraded to a convoy warn instead of striking it (P1 review F2). The throw then
+    // propagates to runJob's own catch, which records the failure.
+    //
+    // #1725 P1 F-E — anchor the block interval HERE (Date.now, the gap window's clock), at the handler
+    // call, NOT at `startedAt` (stamped at the top of runJob, before the lease-claim UPDATE, which can
+    // burn busy_timeout for ~1s under WAL contention — exactly the load the freeze scenario runs under).
+    // Anchoring here keeps that pre-handler write from shifting the block interval early.
+    run.blockStartedAt = Date.now();
+    const blockStart = performance.now();
+    let pending: Promise<Record<string, unknown>> | Record<string, unknown>;
+    try {
+      pending = timePostBindPhase(`maintenance.${definition.id}`, () => withRelayEventDrain(() => definition.handler({ signal: abortController.signal, markOffThread: () => markRunOffThread(run.runId), measureBlock: (step) => measureRunBlock(run, step) })));
+    } finally {
+      run.blockMs = performance.now() - blockStart;
+    }
+    result = await withTimeout(Promise.resolve(pending), deadlineMs, abortController);
+    if (result.outcome === "failed") throw new Error(typeof result.reason === "string" ? result.reason : "maintenance handler reported failure");
+    const skipped = result.outcome === "skipped";
+    const status = skipped ? "skipped" : "succeeded";
+    finishedAt = Date.now();
+    const durationMs = finishedAt - startedAt;
+    // #1758 — a bounded-progress job (e.g. old-message-prune draining a retention backlog) resolves
+    // NORMALLY each pass having pruned only its slice, and signals `moreWork: true` when rows remain.
+    // That is success, not failure: reschedule the next pass at the shorter `backlogIntervalMs` (when
+    // the job declares one) so the backlog drains across successive passes instead of a daily trickle.
+    // A drained run (no `moreWork`) falls back to the normal `intervalMs`.
+    const moreWork = isRecord(result) && result.moreWork === true;
+    const nextInterval = moreWork && definition.backlogIntervalMs !== undefined ? definition.backlogIntervalMs : definition.intervalMs;
+    const persisted = await persistRunOutcome({ id: definition.id, runToken, status, finishedAt, nextRunAt: finishedAt + nextInterval,
+      durationMs, blockMs: runBlockTotalMs(run), error: null, result: JSON.stringify(result) });
+    // A clean run clears any accumulated breaker strikes: the breaker only trips on CONSECUTIVE bad
+    // runs, so one healthy run resets the count.
+    if (persisted && !skipped) clearBreakerStrikes(definition.id);
+    if (definition.logRuns) console.info(`[relay] maintenance job ${definition.id} ${status} in ${durationMs}ms`);
+    return { id: definition.id, status, startedAt, finishedAt, durationMs, result };
+  } catch (error) {
+    finishedAt = Date.now();
+    const isTimeout = error instanceof MaintenanceTimeoutError;
+    const cancelled = error instanceof MaintenanceCancelledError;
+    const breakerAborted = error instanceof MaintenanceBreakerError;
+    // "May still be running past when we gave up" for attribution: both a timeout and a breaker abort
+    // reject the race without the handler necessarily having stopped (#1729's shadow). A cooperative
+    // cancel, by contrast, is a clean stop.
+    timedOut = isTimeout || breakerAborted;
+    const durationMs = finishedAt - startedAt;
+    const message = errMessage(error);
+    // A cancel is deliberate operator intent, not the job breaking: record it as `cancelled` (not
+    // `failed`) and do NOT increment consecutive_failures (which drives the #1689 "failing" arm).
+    const status: "failed" | "cancelled" = cancelled ? "cancelled" : "failed";
+    const persisted = await persistRunOutcome({ id: definition.id, runToken, status, finishedAt, nextRunAt: finishedAt + definition.intervalMs,
+      durationMs, blockMs: runBlockTotalMs(run), error: message, result: result ? JSON.stringify(result) : null });
+    // #1724 stage 1 — feed the breaker. A timeout (deadline overrun) or a breaker abort is a strike;
+    // enough CONSECUTIVE strikes durably auto-pause the job (non-critical) or escalate it (critical).
+    // recordBreakerStrike dedups on runId, so a breaker abort already struck by the lag observer is a
+    // safe no-op here. A plain handler error is NOT a strike (that is the #1689 "failing" arm's job),
+    // and a cancel is never a strike.
+    if (persisted && (isTimeout || breakerAborted)) {
+      // A breaker abort is a lag event that the lag observer has usually already struck (deduped by
+      // runId here); a timeout is its own consecutive-overrun signal.
+      recordBreakerStrike({ runId: run.runId, jobId: definition.id, critical: definition.critical, kind: isTimeout ? "timeout" : "lag", reason: isTimeout ? `timed out after ${deadlineMs}ms` : message });
+    }
+    if (definition.logRuns) console.warn(`[relay] maintenance job ${definition.id} ${status} in ${durationMs}ms: ${message}`);
+    return { id: definition.id, status, startedAt, finishedAt, durationMs, error: message, ...(result ? { result } : {}) };
+  } finally {
+    clearInterval(progressHeartbeat);
+    // #1729 (out of scope here) — on the timeout-race path the handler promise is left pending
+    // and unawaited; `finishedAt` above is "when we gave up on it", not necessarily when the
+    // handler itself actually returned. This registry entry closes at the same moment, so a
+    // handler that keeps running (or blocking) past its own timeout is retrospectively
+    // UNDER-attributed: a later stall it causes falls outside this closed interval. `timedOut`
+    // (set in the catch block above) records exactly this case so `findMaintenanceRunsOverlapping`
+    // can treat the run as still-open rather than silently letting it age out of attribution.
+    endRun(run, finishedAt, timedOut);
+  }
+}
+
+// #1645 — the break-glass leaves a trail. Overriding a pause is legitimate under pressure (the
+// alternative friction is what sends people back to raw SQL), but it must never be quiet: the
+// override records who held the pause and why, so a run that contradicts an operator's stated
+// intent is attributable afterwards.
+async function auditPauseOverride(definition: MaintenanceJobDefinition, pause: { id: string; holder: string; reason: string; expiresAt?: number }): Promise<void> {
+  try {
+    await createActivityEventAsync({
+      clientId: `server-maintenance-pause-override-${definition.id}-${Date.now()}`,
+      kind: "operator",
+      title: "Maintenance pause overridden",
+      body: `${definition.title} was force-run while paused by ${pause.holder} (${pause.reason})`,
+      meta: definition.id,
+      icon: "ti-alert-octagon",
+      view: "activity",
+      metadata: {
+        source: "server",
+        maintenanceJobId: definition.id,
+        pauseId: pause.id,
+        pauseHolder: pause.holder,
+        pauseReason: pause.reason,
+        pauseExpiresAt: pause.expiresAt,
+        overridePause: true,
+      },
+    });
+  } catch {}
+}
+
+/**
+ * #1728 stage 0 / #1724 stage 1 — races the handler against the abort signal. The deadline fires by
+ * ABORTING the controller (not rejecting directly), so the signal is the single rejection channel and
+ * its reason is what the run is recorded against. That is what makes cancel and the breaker WORK
+ * against a non-cooperative async handler: an operator cancel or a breaker abort rejects this race
+ * even if the handler never observes `signal`, so a hanging async job (`new Promise(() => {})`) stops
+ * within a tick. The three abort reasons (Timeout/Cancelled/Breaker, from ./run-registry) are what
+ * `runJob`'s catch keys its recorded status off. A synchronous busy-loop is still un-preemptible here
+ * (single thread; that is Stage 3's worker `terminate()`); this bounds ASYNC hangs, Stage 1's promise.
+ *
+ * #2232 — the race itself now lives in `src/async-deadline.ts` so the workspace auto-merge scheduler
+ * bounds its per-repo cycle with the SAME primitive; only the maintenance-specific error types stay here.
+ */
+async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, abortController: AbortController): Promise<T> {
+  return await withAbortDeadline(promise, timeoutMs, abortController, {
+    timeoutError: (ms) => new MaintenanceTimeoutError(`maintenance job timed out after ${ms}ms`),
+    cancelledError: (reason) => new MaintenanceCancelledError(reason),
+  });
+}
+
+function rowToJob(row: MaintenanceJobRow, now: number): MaintenanceJob {
+  const leaseUntil = row.lease_until ?? undefined;
+  const pause = findPauseByJobId(row.id);
+  return {
+    id: row.id,
+    title: row.title,
+    description: row.description ?? undefined,
+    intervalMs: row.interval_ms,
+    // #1728: DB column is still `timeout_ms` (renaming it is a migration); the public field is
+    // `deadlineMs`, matching MaintenanceJobDefinition.
+    deadlineMs: row.timeout_ms,
+    enabled: row.enabled === 1,
+    critical: row.critical === 1,
+    // Includes a LAPSED timed pause: "expired and nobody resumed" is its own reportable state.
+    ...(pause ? { pause } : {}),
+    runOnStart: row.run_on_start === 1,
+    lastRunAt: row.last_run_at ?? undefined,
+    nextRunAt: row.next_run_at ?? undefined,
+    lastDurationMs: row.last_duration_ms ?? undefined,
+    lastBlockMs: row.last_block_ms ?? undefined,
+    lastStatus: leaseUntil && leaseUntil > now ? "running" : row.last_status as MaintenanceJob["lastStatus"],
+    lastError: row.last_error ?? undefined,
+    lastResult: parseJsonObject(row.last_result),
+    consecutiveFailures: row.consecutive_failures,
+    running: Boolean(leaseUntil && leaseUntil > now),
+    leaseUntil,
+    updatedAt: row.updated_at,
+  };
+}
+
+function aggregateRunResults(runs: MaintenanceJobRun[]): Record<string, unknown> {
+  const aggregate: Record<string, unknown> = { ok: true, runs };
+  for (const run of runs) {
+    if (!run.result) continue;
+    for (const [key, value] of Object.entries(run.result)) {
+      if (Array.isArray(value)) {
+        const current = Array.isArray(aggregate[key]) ? aggregate[key] as unknown[] : [];
+        aggregate[key] = [...current, ...value];
+      } else if (typeof value === "number") {
+        aggregate[key] = Number(aggregate[key] ?? 0) + value;
+      } else if (value !== undefined) {
+        aggregate[key] = value;
+      }
+    }
+  }
+  return aggregate;
+}
+
+async function auditMaintenanceReaperRun(result: Record<string, unknown>): Promise<void> {
+  const reapedAgentIds = Array.isArray(result.reapedAgentIds) ? result.reapedAgentIds : [];
+  const reapedOrchestratorIds = Array.isArray(result.reapedOrchestratorIds) ? result.reapedOrchestratorIds : [];
+  const releasedMessageIds = Array.isArray(result.releasedMessageIds) ? result.releasedMessageIds : [];
+  const releasedTaskIds = Array.isArray(result.releasedTaskIds) ? result.releasedTaskIds : [];
+  const releasedOrphanedTaskIds = Array.isArray(result.releasedOrphanedTaskIds) ? result.releasedOrphanedTaskIds : [];
+  const expiredQueuedMessageIds = Array.isArray(result.expiredQueuedMessageIds) ? result.expiredQueuedMessageIds : [];
+  const expiredCommandIds = Array.isArray(result.expiredCommandIds) ? result.expiredCommandIds : [];
+  const expiredArtifactIds = Array.isArray(result.expiredArtifactIds) ? result.expiredArtifactIds : [];
+  // #1912 S3.2 — an idle reaper sweep is observational only. It must not acquire the writer just
+  // to append a zero-count activity row; that lock convoy is exactly the #1725 failure mode.
+  if (reapedAgentIds.length === 0 && reapedOrchestratorIds.length === 0 && releasedMessageIds.length === 0 && releasedTaskIds.length === 0
+    && releasedOrphanedTaskIds.length === 0 && expiredQueuedMessageIds.length === 0 && expiredCommandIds.length === 0 && expiredArtifactIds.length === 0) return;
+  try {
+    await createActivityEventAsync({
+      clientId: "server-maintenance-reaper-" + Date.now(),
+      kind: "state",
+      title: "Maintenance reaper run",
+      body: `${reapedAgentIds.length} stale agent(s), ${reapedOrchestratorIds.length} stale orchestrator(s), ${releasedMessageIds.length} message claim(s), ${releasedTaskIds.length} task claim(s), ${releasedOrphanedTaskIds.length} orphaned task(s), ${expiredQueuedMessageIds.length} queued message(s), ${expiredCommandIds.length} command(s), ${expiredArtifactIds.length} artifact(s)`,
+      icon: "ti-broom",
+      view: "activity",
+      metadata: { source: "server", result },
+    });
+  } catch {}
+}
+
+function parseJsonObject(raw: string | null): Record<string, unknown> | undefined {
+  if (!raw) return undefined;
+  try {
+    const value = JSON.parse(raw);
+    return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+  } catch {
+    return undefined;
+  }
+}
+import { withRelayEventDrain } from "../relay-event-drain";