@humwick/server 0.130.1 → 0.130.3

package/src/maintenance/offline-agent-prune-job.ts

@@ -0,0 +1,28 @@
+import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS } from "../config";
+import { pruneOfflineAgentsAsync } from "../db/agents-async";
+import { emitAgentRemoved } from "../sse";
+import { emitWorkFinalityResult } from "../services/work-finality";
+import type { MaintenanceJobDefinition } from "./types";
+
+export const OFFLINE_AGENT_PRUNE_JOB: MaintenanceJobDefinition = {
+  id: "offline-agent-prune",
+  critical: false,
+  title: "Offline agent prune",
+  description: "Delete long-offline provider agents and release their claims.",
+  intervalMs: REAP_INTERVAL_MS,
+  runOnStart: false,
+  async handler(context) {
+    if (context.signal?.aborted) return { prunedAgentIds: [], aborted: true };
+    // #1084 crash-final-loss fix-forward — defense-in-depth: `notifyAgentOffline` is the primary
+    // flush point when an agent first goes offline, so this is normally a no-op by the time an
+    // agent reaches prune horizon; the async facade snapshots its held queue and performs the DB
+    // finalization in the SAME worker transaction before deleting the agent row.
+    let archivePurgeFailures = 0;
+    const prunedAgentIds = await pruneOfflineAgentsAsync(OFFLINE_PRUNE_MS, {
+      onFinality: emitWorkFinalityResult,
+      onArchivePurgeFailure: () => { archivePurgeFailures++; },
+    });
+    for (const id of prunedAgentIds) emitAgentRemoved(id);
+    return { prunedAgentIds, archivePurgeFailures };
+  },
+};

package/src/maintenance/orphaned-sessions.ts

@@ -1,4 +1,5 @@
-import { createCommand, listCommands } from "../commands-db";
+import { createCommandAsync } from "../commands-write";
+import { listCommands } from "../commands-db";
 import { createActivityEventAsync, getAgent, listOrchestrators } from "../db";
 import type { Command, ManagedAgent } from "../types";
 import { orphanGraceMs, orphanReapCooldownMs, orphanReapEnabled } from "./constants";
@@ -44,7 +45,7 @@ export async function reapOrphanedSessions(context: { signal?: AbortSignal } = {
   let orphaned = 0;
 
   for (const orch of listOrchestrators()) {
-    // #1819 — checkpoint per orchestrator and per managed session: a reap is `createCommand` +
+    // #1819 — checkpoint per orchestrator and per managed session: a reap is `createCommandAsync` +
     // `emitCommandEventIfAbsent` + activity event + `lastReapAt` stamp, indivisible once started.
     if (signal?.aborted) break;
     if (orch.status !== "online") continue;
@@ -70,7 +71,7 @@ export async function reapOrphanedSessions(context: { signal?: AbortSignal } = {
       // logical orphan reap has one command id and one command.requested event.
       const pending = listCommands({ target: orch.agentId, type: "agent.shutdown", limit: 500 })
         .find((command) => isPendingReapForSession(command, session));
-      const command = pending ?? createCommand({
+      const command = pending ?? (await createCommandAsync({
         type: "agent.shutdown",
         source: "system",
         target: orch.agentId,
@@ -89,7 +90,7 @@ export async function reapOrphanedSessions(context: { signal?: AbortSignal } = {
           requestedAt: now,
           orchestratorId: orch.id,
         },
-      });
+      }));
       // The command is durable but not externally observable until its audit row commits. A failed
       // audit must never let an orchestrator execute an unaudited reap (#1912 S3.2 class B).
       await createActivityEventAsync({

package/src/maintenance/pauses.ts

@@ -1,3 +1,4 @@
+import { resumeMaintenanceJobAsync, pauseMaintenanceJobAsync } from "../db/runtime-tail-async";
 // #1645 — behavior layer for maintenance-job pause/resume: validation, audit, and the drift report.
 //
 // Persistence is src/db/maintenance-pauses.ts; the scheduler gate reads from there. This module is
@@ -6,14 +7,7 @@
 // #1638 pause's justification existed only in one agent's session context and one memory file.
 
 import { errMessage } from "@humwick/sdk";
-import {
-  createActivityEventAsync,
-  latestSuccessfulDeployAsync,
-  listMaintenanceJobPauses,
-  pauseMaintenanceJob,
-  resumeMaintenanceJob,
-  type MaintenanceJobPauseRecord,
-} from "../db";
+import { createActivityEventAsync, latestSuccessfulDeployAsync, listMaintenanceJobPauses, type MaintenanceJobPauseRecord } from "../db";
 import { openRestartEpochsAsync } from "../db/restart-epoch-async.ts";
 import { ValidationError } from "../db/connection.ts";
 import { definitions } from "./jobs";
@@ -61,7 +55,7 @@ export async function pauseJob(input: PauseJobRequest): Promise<PauseJobResult>
     throw new ValidationError("pass either ttlMs or indefinite, not both");
   }
 
-  const result = pauseMaintenanceJob({ ...input, reason });
+  const result = (await pauseMaintenanceJobAsync({ ...input, reason }));
   if (!result.ok) return { ok: false, conflict: true, pause: result.pause };
 
   const ownedRestartEpoch = await ownedOpenRestartEpoch(definition.id);
@@ -126,7 +120,7 @@ export interface ResumeJobResult {
 export async function resumeJobAsync(opts: { jobId?: string; id?: string; now?: number }): Promise<ResumeJobResult> {
   if (opts.jobId) requireKnownJob(opts.jobId);
   const now = opts.now ?? Date.now();
-  const released = resumeMaintenanceJob({ jobId: opts.jobId, id: opts.id });
+  const released = (await resumeMaintenanceJobAsync({ jobId: opts.jobId, id: opts.id }));
   if (!released) return { resumed: false };
   const heldForMs = Math.max(0, now - released.createdAt);
   const lapsed = released.expiresAt !== undefined && released.expiresAt <= now;

package/src/maintenance/plan-ready-notice-job.ts

@@ -0,0 +1,71 @@
+import { createHash } from "node:crypto";
+import { relayEnv } from "@humwick/sdk/env";
+import { getGateway } from "../db/gateway";
+import { getPlan, listPlansByTeam } from "../db/plans";
+import type { DueReadyNodes, ReadyPlanScan } from "../db/plan-ready-notices";
+import { listActiveTeams } from "../db/teams";
+import { routeNotification } from "../notification-router";
+import { compactText, liveFrontier } from "../services/plan-wake-digest";
+
+const DEFAULT_READY_NOTICE_MINUTES = 30;
+
+/** #3660 — minutes a ready node may sit unstarted before the coordinator hears about it. */
+export function planReadyNoticeMs(): number {
+  const minutes = Number(relayEnv(process.env, "AGENT_RELAY_PLAN_READY_NOTICE_MINUTES"));
+  return (Number.isFinite(minutes) && minutes > 0 ? minutes : DEFAULT_READY_NOTICE_MINUTES) * 60_000;
+}
+
+/** Ready (all prerequisites landed), still pending after the Task projection, and nobody owns it. */
+function scanTeamPlans(teamId: string, signal: AbortSignal): Array<ReadyPlanScan & { title: string }> {
+  const scans: Array<ReadyPlanScan & { title: string }> = [];
+  let cursor: string | undefined;
+  do {
+    const page = listPlansByTeam(teamId, { limit: 100, cursor });
+    for (const summary of page.plans) {
+      if (signal.aborted) return scans;
+      if (!summary.nodeCount || summary.completion === "done") continue;
+      const stored = getPlan(summary.id);
+      if (!stored) continue;
+      const { frontier } = liveFrontier(stored);
+      scans.push({ planId: stored.id, title: stored.title ?? "Untitled", nodeIds: frontier.ready.filter(node => !node.ownerRef).map(node => node.id) });
+    }
+    cursor = page.nextCursor;
+  } while (cursor);
+  return scans;
+}
+
+export async function noticeUnstartedReadyNodes(signal: AbortSignal, now = Date.now()): Promise<Record<string, unknown>> {
+  const scans: Array<ReadyPlanScan & { title: string; teamId: string }> = [];
+  for (const team of listActiveTeams()) {
+    if (signal.aborted) return { notices: 0 };
+    if (team.coordinatorId) scans.push(...scanTeamPlans(team.id, signal).map(scan => ({ ...scan, teamId: team.id })));
+  }
+  if (signal.aborted) return { notices: 0 };
+  const due = await getGateway().write("plans.readyNotices.sync", { scans: scans.map(({ planId, nodeIds }) => ({ planId, nodeIds })), now, dueMs: planReadyNoticeMs() }) as unknown as DueReadyNodes[];
+  const byPlan = new Map(scans.map(scan => [scan.planId, scan]));
+  let notices = 0;
+  for (const item of due) {
+    const scan = byPlan.get(item.planId)!;
+    if (signal.aborted) break;
+    const nodeIds = item.nodes.map(node => node.nodeId);
+    const minutes = Math.round(planReadyNoticeMs() / 60_000);
+    const key = createHash("sha1").update(item.nodes.map(node => `${node.nodeId}@${node.readySince}`).join(",")).digest("hex").slice(0, 12);
+    await routeNotification({
+      type: "team.idle", scope: { teamId: scan.teamId }, recipients: [`coordinator:${scan.teamId}`],
+      message: { kind: "system", from: "system", subject: "Plan nodes ready but not started", replyExpected: false,
+        body: `${compactText(scan.title, 56)} (${scan.planId}): ready for ${minutes}+ min, not started: ${nodeIds.slice(0, 8).map(id => compactText(id, 32)).join(",")}${nodeIds.length > 8 ? `,+${nodeIds.length - 8}` : ""}. Start them or mark them blocked (relay_plan_get mode=resume).`,
+        payload: { kind: "plan.ready-unstarted", teamId: scan.teamId, planId: scan.planId, nodeIds },
+        idempotencyKey: `plan-ready:${scan.planId}:${key}` },
+    });
+    await getGateway().write("plans.readyNotices.mark", { planId: item.planId, nodeIds, now });
+    notices++;
+  }
+  return { notices };
+}
+
+export const PLAN_READY_NOTICE_JOB = {
+  id: "plan-ready-notice", title: "Ready Plan nodes not started", critical: false,
+  description: "Tell each coordinator once when a Plan node has been ready with nothing started for the notice window.",
+  intervalMs: 5 * 60_000, deadlineMs: 60_000, runOnStart: false,
+  handler: (context: { signal: AbortSignal }) => noticeUnstartedReadyNodes(context.signal),
+} satisfies import("./types").MaintenanceJobDefinition;

package/src/maintenance/scheduler.ts

@@ -1,14 +1,17 @@
+import { rescheduleRunOnStartJobsAsync, claimMaintenanceRunAsync, finishMaintenanceRunAsync, heartbeatMaintenanceRunAsync, ensureMaintenanceJobsAsync, reclaimDeadLeasesAsync } from "../db/maintenance-job-runs-async";
 import { timePostBindPhase } from "../post-bind-startup";
 import { randomUUID } from "node:crypto";
 import { errMessage, isRecord } from "@humwick/sdk";
 import { classifyPidLiveness } from "@humwick/sdk/process-utils";
-import { CRITICAL_MAINTENANCE_PROGRESS_STALE_MS, createActivityEventAsync, getDb, isSqliteBusyError, MAINTENANCE_RUN_AUTORESET_GUARD_SQL, recordDrainConvoy, writeTransaction } from "../db";
-import { claimMaintenanceRunAsync, finishMaintenanceRunAsync, heartbeatMaintenanceRunAsync } from "../db/maintenance-job-runs-async";
+import { CRITICAL_MAINTENANCE_PROGRESS_STALE_MS, createActivityEventAsync, getDb, MAINTENANCE_RUN_AUTORESET_GUARD_SQL, recordDrainConvoy, writeTransaction } from "../db";
+import { reclaimDeadLeases, upsertMaintenanceJobs } from "../db/maintenance-job-reconcile";
+import type { DeadLease, MaintenanceJobSpec } from "../db/maintenance-job-reconcile";
 import type { MaintenanceRunOutcome } from "../db/maintenance-job-runs";
 import { withAbortDeadline } from "../async-deadline";
 import { MAINTENANCE_BREAKER_LAG_MS } from "../config";
 import { findPauseByJobId, getMaintenanceJobPause, notPausedSql } from "../db/maintenance-pauses";
 import { setLagBreakerObserver } from "../event-loop-lag";
+import { framed, runInFrame } from "../hub-metrics";
 import { resetRateLimitResumeTrackerForTests } from "../services/rate-limit-resume";
 import { resetTransientTrackerForTests } from "../services/transient-agent-reaper";
 import { isEphemeralMode } from "../execution-mode";
@@ -76,7 +79,7 @@ const MAINTENANCE_PROGRESS_HEARTBEAT_MS = 60_000;
 //   3. HARD BLAST-RADIUS CAP: at most ONE new auto-pause per lag sample (operator Q4). If several
 //      qualify, pause the single largest in-gap block and warn about the rest. One freeze may never
 //      pause a fleet.
-setLagBreakerObserver((_gapStartAt, gapEndAt, lagMs) => {
+setLagBreakerObserver(async (_gapStartAt, gapEndAt, lagMs) => {
   const reason = `blocked the event loop ~${Math.round(lagMs)}ms (measured block)`;
   // #1725 P1 F-A — the ACTUATOR uses the TIGHTENED window `[gapEndAt - lagMs, gapEndAt]`, the actual
   // freeze interval, NOT the sampler's diagnostic window `[_gapStartAt, gapEndAt]`. The sampler builds
@@ -140,7 +143,7 @@ setLagBreakerObserver((_gapStartAt, gapEndAt, lagMs) => {
   const blamed = strikable.reduce((worst, entry) => (entry.overlapMs > worst.overlapMs ? entry : worst));
   if (isRunActive(blamed.run.runId)) abortRun(blamed.run.runId, new MaintenanceBreakerError(`aborted by circuit breaker: ${reason}`));
   const critical = definitions.find((job) => job.id === blamed.run.jobId)?.critical ?? false;
-  recordBreakerStrike({ runId: blamed.run.runId, jobId: blamed.run.jobId, critical, kind: "lag", reason });
+  (await recordBreakerStrike({ runId: blamed.run.runId, jobId: blamed.run.jobId, critical, kind: "lag", reason }));
   const others = strikable.filter((entry) => entry !== blamed);
   if (others.length > 0) {
     console.warn(`[relay] maintenance circuit breaker: ${strikable.length} runs each blocked >=${Math.round(shareThresholdMs)}ms of this ${Math.round(lagMs)}ms freeze — struck ${blamed.run.jobId} (in-freeze block ${Math.round(blamed.overlapMs)}ms), one-per-sample cap; not struck: ${others.map((entry) => `${entry.run.jobId} (${Math.round(entry.overlapMs)}ms)`).join(", ")}`);
@@ -163,8 +166,8 @@ const runBlockTotalMs = (run: MaintenanceRunRecord): number => Math.round((run.b
 /** Exposes the internal `runJob` directly, bypassing the `definitions` lookup `runMaintenanceJobNow`
  *  does, so tests can drive the run-registry/timeout/crash paths with a synthetic handler.
  *  Production code never calls this name. */
-export const runMaintenanceJobForTests = (definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}) =>
-  runJob(definition, options);
+export const runMaintenanceJobForTests = async (definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}) =>
+  (await runJob(definition, options));
 
 export function resetOrphanTrackerForTests(): void {
   resetOrphanSessionTrackerForTests();
@@ -173,41 +176,41 @@ export function resetOrphanTrackerForTests(): void {
 }
 
 export function startMaintenanceScheduler(): void {
-  // Reconcile the job table once at boot. A transient SQLITE_BUSY here must not
-  // abort startup — the first scheduler tick re-runs the reconcile with retry.
+  // #1870 S7: the boot reconcile, run-on-start reschedule and dead-lease reclaim are writer ops. The first
+  // tick waits for them, so a due run-on-start job is never claimed before its reschedule lands.
+  void bootMaintenanceJobs().then(() => runDueMaintenanceJobs())
+    .catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`));
+  // Floated ticks are wrapped defensively: runDueMaintenanceJobs already catches
+  // its own errors (#1481), but a stray rejection from a floated promise must
+  // never bubble to an unhandledRejection that exits the relay process.
+  if (timer) return;
+  timer = setInterval(
+    framed("timer maintenance", () => runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`))),
+    SCHEDULER_TICK_MS,
+  );
+}
+
+async function bootMaintenanceJobs(): Promise<void> {
+  // A transient SQLITE_BUSY here must not abort startup: every scheduler tick runs the reconcile again.
   try {
-    ensureMaintenanceJobs();
-    scheduleRunOnStartJobs();
+    await ensureMaintenanceJobsAsync(jobSpecs());
   } catch (error) {
     console.warn(`maintenance job reconcile at startup failed (will retry on next tick): ${errMessage(error)}`);
-    void ensureMaintenanceJobsResilient()
-      .then(() => {
-        scheduleRunOnStartJobs();
-        return runDueMaintenanceJobs();
-      })
-      .catch((retryError) => console.warn(`maintenance startup retry failed (will retry on next tick): ${errMessage(retryError)}`));
   }
-  // #1493 — a relay restart orphans the maintenance-job lease its own dying
-  // process held moments earlier: lease_owner still names the dead PID and
-  // lease_until is still in the future, so the claim guard in runJob/
-  // runDueMaintenanceJobs (lease_until IS NULL OR lease_until <= now) blocks
-  // the job — e.g. the CD watcher — until the lease's TTL self-expires (up to
-  // deadlineMs + 5s). Reclaim any lease whose owner PID is no longer alive so
-  // maintenance resumes immediately after a restart instead of stalling.
   try {
-    reclaimDeadPidLeases();
+    await rescheduleRunOnStartJobsAsync();
+  } catch (error) {
+    // The next tick retries it, so run-on-start jobs do not wait a full interval (#1870 S7).
+    runOnStartPending = true;
+    console.warn(`maintenance run-on-start reschedule failed (will retry on next tick): ${errMessage(error)}`);
+  }
+  // #1493 — a restart orphans the lease its dying process held: lease_owner still names the dead PID and
+  // lease_until is still in the future, which blocks the job (e.g. the CD watcher) until the TTL expires.
+  try {
+    await reclaimDeadLeasesAsync(findDeadLeases());
   } catch (error) {
     console.warn(`dead-PID maintenance lease reclaim at startup failed (will retry on next tick): ${errMessage(error)}`);
   }
-  // Floated ticks are wrapped defensively: runDueMaintenanceJobs already catches
-  // its own errors (#1481), but a stray rejection from a floated promise must
-  // never bubble to an unhandledRejection that exits the relay process.
-  void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`));
-  if (timer) return;
-  timer = setInterval(
-    () => void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`)),
-    SCHEDULER_TICK_MS,
-  );
 }
 
 /**
@@ -218,33 +221,41 @@ export function startMaintenanceScheduler(): void {
  * doing it from the periodic reconcile would make run-on-start jobs run every
  * scheduler tick.
  */
-export function scheduleRunOnStartJobs(now: number = Date.now()): void {
-  getDb().query(`
-    UPDATE maintenance_jobs
-    SET next_run_at = ?, updated_at = ?
-    WHERE enabled = 1 AND run_on_start = 1
-  `).run(now, now);
+export async function scheduleRunOnStartJobs(now: number = Date.now()): Promise<void> {
+  (await rescheduleRunOnStartJobsAsync(now));
+}
+
+function readMaintenanceJobs(): MaintenanceJob[] {
+  const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
+  const now = Date.now();
+  const currentIds = new Set(definitions.map((definition) => definition.id));
+  return rows.filter((row) => currentIds.has(row.id)).map((row) => rowToJob(row, now));
 }
 
+// A transient lock while reconciling must not fail a listing; return the current rows (a later reconcile catches up). #1481.
 export function listMaintenanceJobs(): MaintenanceJob[] {
   try {
     ensureMaintenanceJobs();
   } catch (error) {
-    // A transient lock while reconciling must not fail a read-only listing; fall
-    // through and return the current rows (a later reconcile catches up). #1481.
     console.warn(`maintenance job reconcile during list failed: ${errMessage(error)}`);
   }
-  const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
-  const now = Date.now();
-  const currentIds = new Set(definitions.map((definition) => definition.id));
-  return rows.filter((row) => currentIds.has(row.id)).map((row) => rowToJob(row, now));
+  return readMaintenanceJobs();
+}
+
+export async function listMaintenanceJobsAsync(): Promise<MaintenanceJob[]> {
+  try {
+    await ensureMaintenanceJobsAsync(jobSpecs());
+  } catch (error) {
+    console.warn(`maintenance job reconcile during list failed: ${errMessage(error)}`);
+  }
+  return readMaintenanceJobs();
 }
 
 export async function runMaintenanceJobNow(id: string, options: { overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
-  await ensureMaintenanceJobsResilient();
+  await ensureMaintenanceJobsAsync(jobSpecs());
   const definition = definitions.find((job) => job.id === id);
   if (!definition) throw new Error("maintenance job not found");
-  return runJob(definition, { force: true, overridePause: options.overridePause });
+  return (await runJob(definition, { force: true, overridePause: options.overridePause }));
 }
 
 export async function runLegacyMaintenanceReaper(): Promise<Record<string, unknown>> {
@@ -266,6 +277,7 @@ async function runMaintenanceJobsNow(ids: string[]): Promise<MaintenanceJobRun[]
 // no-op (the still-open tick, or the one after, picks up whatever came due). Combined with the
 // serialized drain below, this is the "one job at a time from a queue" the design (§5.4) calls for.
 let tickInFlight = false;
+let runOnStartPending = false;
 
 export async function runDueMaintenanceJobs(): Promise<void> {
   if (tickInFlight) return;
@@ -274,7 +286,11 @@ export async function runDueMaintenanceJobs(): Promise<void> {
   // is guarded: a transient SQLITE_BUSY (or any other failure) reconciling or
   // querying the job table is logged and the tick is skipped — the next tick retries.
   try {
-    await ensureMaintenanceJobsResilient();
+    await ensureMaintenanceJobsAsync(jobSpecs());
+    if (runOnStartPending) {
+      await rescheduleRunOnStartJobsAsync();
+      runOnStartPending = false;
+    }
     const now = Date.now();
     const progressStaleAt = now - CRITICAL_MAINTENANCE_PROGRESS_STALE_MS;
     const rows = getDb().query(`
@@ -363,84 +379,28 @@ export function drainDueMaintenanceJobsForTests(dueDefinitions: MaintenanceJobDe
  *  otherwise leave it latched, wedging every later test's tick). Production never calls this. */
 export function resetSchedulerTickGuardForTests(): void {
   tickInFlight = false;
+  runOnStartPending = false;
 }
 
-// Bounded, ASYNC, loop-yielding retry around the (synchronous) job-table reconcile.
-// Mirrors runWriteWithBusyRetry (connection.ts) but wraps ensureMaintenanceJobs,
-// which runs its own BEGIN IMMEDIATE transaction and so cannot be nested inside
-// runWriteWithBusyRetry. busy_timeout already makes SQLite WAIT on a held lock; the
-// rare BUSY that survives it is retried here with jittered backoff that yields the
-// event loop between attempts. Anything else — or exhausted attempts — rethrows to
-// the caller, which degrades the tick rather than crashing the process (#1481).
-const ENSURE_BUSY_RETRY_ATTEMPTS = 4;
-const ENSURE_BUSY_RETRY_BASE_MS = 15;
-const ENSURE_BUSY_RETRY_MAX_MS = 120;
-
-async function ensureMaintenanceJobsResilient(): Promise<void> {
-  for (let attempt = 1; ; attempt++) {
-    try {
-      ensureMaintenanceJobs();
-      return;
-    } catch (error) {
-      if (attempt < ENSURE_BUSY_RETRY_ATTEMPTS && isSqliteBusyError(error)) {
-        const cap = Math.min(ENSURE_BUSY_RETRY_MAX_MS, ENSURE_BUSY_RETRY_BASE_MS * 2 ** (attempt - 1));
-        const delay = cap / 2 + Math.random() * (cap / 2);
-        await new Promise((resolve) => setTimeout(resolve, delay));
-        continue;
-      }
-      throw error;
-    }
-  }
+function jobSpecs(): MaintenanceJobSpec[] {
+  return definitions.map((definition) => ({
+    id: definition.id,
+    title: definition.title,
+    description: definition.description,
+    intervalMs: definition.intervalMs,
+    deadlineMs: definition.deadlineMs ?? DEFAULT_DEADLINE_MS,
+    enabled: !isEphemeralMode() || definition.instanceSafe === true,
+    critical: definition.critical === true,
+    runOnStart: definition.runOnStart === true,
+  }));
 }
 
+// Boot/test twin of `ensureMaintenanceJobsAsync`: one BEGIN IMMEDIATE so busy_timeout governs a single lock
+// acquisition (#1481). The listing read path and tests use it; the running scheduler uses the writer op.
 function ensureMaintenanceJobs(): void {
-  const db = getDb();
+  const specs = jobSpecs();
   const now = Date.now();
-  // #1645 — `enabled` is re-asserted from `definitions` on every reconcile, which is what makes it
-  // CODE-owned and therefore self-healing. Before this it was set only on INSERT and deliberately
-  // omitted from the conflict clause, because the one column doubled as the operator's "stop" lever
-  // and a redeploy would have clobbered their intent. Operator intent now lives in
-  // `maintenance_job_pauses` (holder, reason, expiry), so re-asserting `enabled` overrides nobody —
-  // and a hand-written `UPDATE maintenance_jobs SET enabled=0` recovers at the next boot instead of
-  // persisting until someone writes more raw SQL (nine hours of deploy blackout, #1645).
-  //
-  // PRECONDITION: convertManualJobDisablesToPauses() (src/db/migrations.ts) must have run first, or
-  // this silently switches back on a job an operator deliberately stopped.
-  const stmt = db.query(`
-    INSERT INTO maintenance_jobs (id, title, description, interval_ms, timeout_ms, enabled, critical, run_on_start, next_run_at, updated_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(id) DO UPDATE SET
-      title = excluded.title,
-      description = excluded.description,
-      interval_ms = excluded.interval_ms,
-      timeout_ms = excluded.timeout_ms,
-      run_on_start = excluded.run_on_start,
-      enabled = excluded.enabled, -- #1645: code-owned, see the note above this query
-      critical = excluded.critical,
-      next_run_at = COALESCE(maintenance_jobs.next_run_at, excluded.next_run_at),
-      updated_at = excluded.updated_at
-  `);
-  // Batch the upserts into a single BEGIN IMMEDIATE write so busy_timeout governs
-  // ONE lock acquisition (SQLite waits) instead of one per definition, cutting the
-  // lock churn that made this loop the crash site under contention (#1481).
-  writeTransaction(() => {
-    for (const definition of definitions) {
-      stmt.run(
-        definition.id,
-        definition.title,
-        definition.description,
-        definition.intervalMs,
-        // #1728: the DB column stays `timeout_ms` — renaming it is a migration, not asked for here.
-        // Code-side this is `deadlineMs`: see MaintenanceJobDefinition for why.
-        definition.deadlineMs ?? DEFAULT_DEADLINE_MS,
-        !isEphemeralMode() || definition.instanceSafe === true ? 1 : 0,
-        definition.critical ? 1 : 0,
-        definition.runOnStart ? 1 : 0,
-        definition.runOnStart ? now : now + definition.intervalMs,
-        now,
-      );
-    }
-  });
+  writeTransaction(() => upsertMaintenanceJobs(specs, now));
 }
 
 // #1493 — reclaim current-schema maintenance_jobs leases whose owner PID is proven dead.
@@ -455,23 +415,19 @@ function ensureMaintenanceJobs(): void {
 // #3645: the crash is not a job failure — it neither bumps consecutive_failures nor replaces an existing
 // last_error (a non-null one belongs to the live failure streak; success clears it). The page-only guard remains on every
 // deadline-based reclaim/claim path, where that positive evidence is unavailable.
-export function reclaimDeadPidLeases(): void {
-  const db = getDb();
-  const now = Date.now();
-  const leased = db.query(`
-    SELECT id, lease_owner FROM maintenance_jobs
-    WHERE lease_owner IS NOT NULL
-  `).all() as Array<{ id: string; lease_owner: string }>;
+function findDeadLeases(): DeadLease[] {
+  const leased = getDb().query("SELECT id, lease_owner FROM maintenance_jobs WHERE lease_owner IS NOT NULL").all() as Array<{ id: string; lease_owner: string }>;
+  const dead: DeadLease[] = [];
   for (const row of leased) {
     const pid = parseLeaseOwnerPid(row.lease_owner);
-    if (pid === null || classifyPidLiveness(pid) !== "dead") continue;
-    db.query(`
-      UPDATE maintenance_jobs
-      SET lease_owner = NULL, lease_until = NULL, last_status = 'failed',
-          last_error = COALESCE(last_error, ?), run_token = NULL, last_progress_at = ?, updated_at = ?
-      WHERE id = ? AND lease_owner = ?
-    `).run(`reclaimed stale lease at startup: owner pid ${pid} is confirmed dead`, now, now, row.id, row.lease_owner);
+    if (pid !== null && classifyPidLiveness(pid) === "dead") dead.push({ id: row.id, owner: row.lease_owner, pid });
   }
+  return dead;
+}
+
+export function reclaimDeadPidLeases(): void {
+  const dead = findDeadLeases();
+  if (dead.length) writeTransaction(() => reclaimDeadLeases(dead, Date.now()));
 }
 
 function parseLeaseOwnerPid(owner: string): number | null {
@@ -494,7 +450,12 @@ async function persistRunOutcome(outcome: MaintenanceRunOutcome): Promise<boolea
   }
 }
 
-async function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
+/** #3798 — each run is its own hub-metrics frame, so a job's main-thread CPU is named per job. */
+function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
+  return runInFrame({ callsite: `job ${definition.id}` }, () => executeJob(definition, options));
+}
+
+async function executeJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean }): Promise<MaintenanceJobRun> {
   const startedAt = Date.now();
   // One execution boundary covers ticks, forced runs and already-queued definitions.
   if (isEphemeralMode() && definition.instanceSafe !== true) {
@@ -527,7 +488,7 @@ async function runJob(definition: MaintenanceJobDefinition, options: { force?: b
   const progressStaleAt = startedAt - CRITICAL_MAINTENANCE_PROGRESS_STALE_MS;
   const claimed = await claimMaintenanceRunAsync({ id: definition.id, owner, runToken, startedAt, leaseUntil, progressStaleAt, force: options.force === true });
   if (!claimed) {
-    const state = listMaintenanceJobs().find((job) => job.id === definition.id);
+    const state = readMaintenanceJobs().find((job) => job.id === definition.id);
     return { id: definition.id, status: "skipped", startedAt, finishedAt: Date.now(), durationMs: 0, result: { reason: "not due or already running", state } };
   }
 
@@ -628,7 +589,7 @@ async function runJob(definition: MaintenanceJobDefinition, options: { force?: b
     if (persisted && (isTimeout || breakerAborted)) {
       // A breaker abort is a lag event that the lag observer has usually already struck (deduped by
       // runId here); a timeout is its own consecutive-overrun signal.
-      recordBreakerStrike({ runId: run.runId, jobId: definition.id, critical: definition.critical, kind: isTimeout ? "timeout" : "lag", reason: isTimeout ? `timed out after ${deadlineMs}ms` : message });
+      (await recordBreakerStrike({ runId: run.runId, jobId: definition.id, critical: definition.critical, kind: isTimeout ? "timeout" : "lag", reason: isTimeout ? `timed out after ${deadlineMs}ms` : message }));
     }
     if (definition.logRuns) console.warn(`[relay] maintenance job ${definition.id} ${status} in ${durationMs}ms: ${message}`);
     return { id: definition.id, status, startedAt, finishedAt, durationMs, error: message, ...(result ? { result } : {}) };

package/src/maintenance/stalled-delivery-pager.ts

@@ -1,5 +1,6 @@
 import { errMessage } from "@humwick/sdk";
-import { getConfig, setConfig } from "../config-store";
+import { getConfig } from "../config-store";
+import { setConfigAsync } from "../config-store-write";
 import { getStalledDeliverySignalAsync } from "../db/stats-async.ts";
 import { notifyWorkspaceHumanEscalation } from "../workspace-human-escalation";
 import type { MaintenanceJobDefinition } from "./types";
@@ -56,8 +57,8 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
   }
   let alerted = false;
   let recovered = false;
-  const persist = () => {
-    if (JSON.stringify(state) !== JSON.stringify(prior)) setConfig(STATE_NAMESPACE, STATE_KEY, state, "stalled-delivery-pager");
+  const persist = async () => {
+    if (JSON.stringify(state) !== JSON.stringify(prior)) await setConfigAsync(STATE_NAMESPACE, STATE_KEY, state, "stalled-delivery-pager");
   };
 
   const { check } = signal;
@@ -68,7 +69,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
     state = { ...state, failStreak: 0, okStreak: state.stalled ? state.okStreak + 1 : 0 };
     if (state.stalled && state.okStreak >= STALL_PAGE_CONFIRM_CHECKS) {
       if (state.alerted) {
-        const result = notifyWorkspaceHumanEscalation({
+        const result = (await notifyWorkspaceHumanEscalation({
           subject: "Message delivery recovered",
           body: `Messages are being delivered to live agents again. The stall began ${new Date(state.since).toISOString()}.`,
           priority: "default",
@@ -76,7 +77,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
           source: "stalled-delivery-pager",
           payload: { kind: "message-delivery.recovered", since: state.since, recoveredAt: now },
           idempotencyKey: `stalled-delivery:${state.since}:recovered`,
-        });
+        }));
         recovered = result.accepted;
         if (!result.accepted) console.warn(`[relay] #3014 message delivery recovered — operator NOT paged: ${result.reason}`);
       }
@@ -87,8 +88,8 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
     if (!state.stalled && state.failStreak >= STALL_PAGE_CONFIRM_CHECKS) state = { ...state, stalled: true, since: now, alerted: false };
     if (state.stalled && !state.alerted) {
       // Persist the episode first so a crash after the send re-sends under the same key.
-      persist();
-      const result = notifyWorkspaceHumanEscalation({
+      await persist();
+      const result = (await notifyWorkspaceHumanEscalation({
         subject: "Message delivery stalled",
         body: [
           `${check.subjects?.length ?? 0} live, idle agents have had mail unattempted for over 2 minutes and no turn was delivered to any agent in 5 minutes.`,
@@ -101,13 +102,13 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
         source: "stalled-delivery-pager",
         payload: { kind: "message-delivery.stalled", since: state.since, count: check.count, subjects: check.subjects },
         idempotencyKey: `stalled-delivery:${state.since}:alert`,
-      });
+      }));
       alerted = result.accepted;
       if (!result.accepted) console.warn(`[relay] #3014 message delivery stalled — operator NOT paged: ${result.reason}`);
       state = { ...state, alerted: true };
     }
   }
-  persist();
+  await persist();
   return { state: state.stalled ? "stalled" : "ok", alerted, recovered };
 }