@humwick/server 0.130.0 → 0.130.2

package/src/host/disk-alert.ts

@@ -0,0 +1,121 @@
+// #3762 — warn before a host disk fills. Pure: the single-writer utilization op settles each beat
+// against the stored one (same seam as gate-capacity-alert.ts), so each level change alerts once.
+// Two triggers, one level ladder: used% crossing 85/92/97, or the free-space history projecting
+// full within 24 h (6 h = high). An episode alerts once per worse level and clears with a notice.
+import type { HostDiskWatch, HostMachineSample } from "@humwick/sdk";
+
+export type DiskLevel = 1 | 2 | 3;
+
+/** Used-share rungs: warn / high / critical. */
+export const DISK_USED_PCT_LEVELS: readonly [number, number, number] = [85, 92, 97];
+/** An episode only clears below this, so a disk hovering on a rung does not flap. */
+export const DISK_CLEAR_USED_PCT = 82;
+export const DISK_ETA_WARN_H = 24;
+export const DISK_ETA_HIGH_H = 6;
+/** Growth needs this much history before it is a rate rather than a blip. */
+export const DISK_MIN_TREND_SPAN_MS = 3 * 3_600_000;
+const HISTORY_SPACING_MS = 30 * 60_000;
+const HISTORY_MAX_SPAN_MS = 25 * 3_600_000;
+const TREND_WINDOW_MS = 24 * 3_600_000;
+
+export type DiskAlert =
+  | {
+    kind: "low"; level: DiskLevel; since: number; freeMb: number; totalMb: number; path?: string;
+    /** Free-space decline over the trend window, MB/h; present only while declining. */
+    rateMbPerHour?: number; etaHours?: number; spanHours?: number;
+  }
+  | { kind: "recovered"; since: number; freeMb: number; totalMb: number; path?: string };
+
+export interface DiskNotice {
+  subject: string;
+  body: string;
+  priority: "low" | "default" | "high" | "urgent";
+  idempotencyKey: string;
+}
+
+function usageLevel(usedPct: number): number {
+  return DISK_USED_PCT_LEVELS.filter((rung) => usedPct >= rung).length;
+}
+
+function growthLevel(etaHours: number | undefined): number {
+  if (etaHours === undefined) return 0;
+  return etaHours < DISK_ETA_HIGH_H ? 2 : etaHours < DISK_ETA_WARN_H ? 1 : 0;
+}
+
+function nextHistory(prev: HostDiskWatch["history"], at: number, freeMb: number): Array<[number, number]> {
+  const kept = (prev ?? []).filter(([t]) => t >= at - HISTORY_MAX_SPAN_MS && t < at);
+  const last = kept[kept.length - 1];
+  if (!last || at - last[0] >= HISTORY_SPACING_MS) kept.push([at, freeMb]);
+  return kept;
+}
+
+/** Decline rate over the oldest point inside the trend window, or undefined (too little history / not declining). */
+function trend(history: Array<[number, number]>, at: number, freeMb: number): { rateMbPerHour: number; etaHours: number; spanHours: number } | undefined {
+  const oldest = history.find(([t]) => t >= at - TREND_WINDOW_MS);
+  if (!oldest || at - oldest[0] < DISK_MIN_TREND_SPAN_MS) return undefined;
+  const declineMb = oldest[1] - freeMb;
+  if (declineMb <= 0) return undefined;
+  const spanHours = (at - oldest[0]) / 3_600_000;
+  const rateMbPerHour = declineMb / spanHours;
+  return { rateMbPerHour, etaHours: freeMb / rateMbPerHour, spanHours };
+}
+
+/**
+ * Stamps the relay-owned disk watch onto the stored sample and names the one alert owed, if any.
+ * A beat with no disk pair keeps the previous watch (missing telemetry is not a state change).
+ */
+export function settleDiskWatch(prev: HostMachineSample | undefined, next: HostMachineSample): { machine: HostMachineSample; alert?: DiskAlert } {
+  const before = prev?.diskWatch;
+  const { diskFreeMb: freeMb, diskTotalMb: totalMb } = next;
+  if (freeMb === undefined || totalMb === undefined || totalMb <= 0) {
+    return { machine: before ? { ...next, diskWatch: before } : next };
+  }
+  const history = nextHistory(before?.history, next.at, freeMb);
+  const projected = trend(history, next.at, freeMb);
+  const usedPct = ((totalMb - freeMb) / totalMb) * 100;
+  const level = Math.max(usageLevel(usedPct), growthLevel(projected?.etaHours));
+  const prevLevel = before?.level ?? 0;
+  const where = next.diskPath ? { path: next.diskPath } : {};
+
+  if (level > prevLevel) {
+    const since = before?.since ?? next.at;
+    const watch: HostDiskWatch = { level: level as DiskLevel, since, history };
+    return {
+      machine: { ...next, diskWatch: watch },
+      alert: {
+        kind: "low", level: level as DiskLevel, since, freeMb, totalMb, ...where,
+        ...(projected ? { rateMbPerHour: projected.rateMbPerHour, etaHours: projected.etaHours, spanHours: projected.spanHours } : {}),
+      },
+    };
+  }
+  if (prevLevel > 0 && level === 0 && usedPct < DISK_CLEAR_USED_PCT) {
+    return { machine: { ...next, diskWatch: { history } }, alert: { kind: "recovered", since: before!.since ?? next.at, freeMb, totalMb, ...where } };
+  }
+  return { machine: { ...next, diskWatch: { ...(before?.level ? { level: before.level, since: before.since } : {}), history } } };
+}
+
+const gb = (mb: number) => `${(mb / 1024).toFixed(mb >= 10_240 ? 0 : 1)} GB`;
+
+export function composeDiskNotice(hostId: string, label: string, alert: DiskAlert): DiskNotice {
+  const volume = alert.path ? ` (${alert.path})` : "";
+  const usedPct = Math.round(((alert.totalMb - alert.freeMb) / alert.totalMb) * 100);
+  if (alert.kind === "recovered") {
+    return {
+      subject: `Disk recovered on ${label}`,
+      body: `${label}${volume} has ${gb(alert.freeMb)} free of ${gb(alert.totalMb)} (${usedPct}% used); the low-disk episode is over. (#3762)`,
+      priority: "low",
+      idempotencyKey: `host-disk:${hostId}:${alert.since}:recovered`,
+    };
+  }
+  const word = alert.level === 3 ? "CRITICAL" : alert.level === 2 ? "very low" : "low";
+  const growth = alert.rateMbPerHour !== undefined && alert.etaHours !== undefined
+    ? ` Free space fell ~${gb(alert.rateMbPerHour * 24)}/day over the last ${Math.round(alert.spanHours ?? 0)} h; at that rate it is full in ~${Math.max(1, Math.round(alert.etaHours))} h.`
+    : "";
+  return {
+    subject: `Disk ${word} on ${label}: ${gb(alert.freeMb)} free (${usedPct}% used)`,
+    body: `${label}${volume} has ${gb(alert.freeMb)} free of ${gb(alert.totalMb)} (${usedPct}% used).${growth} `
+      + "Check workspaces, runtime logs and caches on that host before agents start failing with ENOSPC. (#3762)",
+    priority: alert.level === 3 ? "urgent" : alert.level === 2 ? "high" : "default",
+    idempotencyKey: `host-disk:${hostId}:${alert.since}:${alert.level}`,
+  };
+}

package/src/lifecycle-manager.ts

@@ -53,7 +53,8 @@ const DEFAULT_COMPACT_TARGET = 0.3;
 const DEFAULT_MAX_TASKS_BEFORE_COMPACT = 10;
 const DEFAULT_COMPACT_COOLDOWN_MS = 60_000;
 
-interface LifecycleManagerOptions { tickMs?: number; now?: () => number; buildManagedSpawnParams?: typeof buildManagedSpawnParamsAsync; }
+type ManagedSpawnParamsBuilder = (...args: Parameters<typeof buildManagedSpawnParamsAsync>) => Record<string, unknown> | Promise<Record<string, unknown>>;
+interface LifecycleManagerOptions { tickMs?: number; now?: () => number; buildManagedSpawnParams?: ManagedSpawnParamsBuilder; }
 
 // #1021 A1 — the outcome of trying to wake an on-demand policy via a queued message. A
 // caller (e.g. wakeRepoSteward) must be able to tell "a live agent will handle this" from
@@ -66,7 +67,7 @@ export class LifecycleManager {
   private timer?: Timer;
   private readonly tickMs: number;
   private readonly now: () => number;
-  private readonly buildManagedSpawnParams: typeof buildManagedSpawnParamsAsync; private readonly pendingSpawnPreparations = new Map<string, Promise<Command | null>>();
+  private readonly buildManagedSpawnParams: ManagedSpawnParamsBuilder; private readonly pendingSpawnPreparations = new Map<string, Promise<Command | null>>();
   private readonly updateRestartPending = new Set<string>();
   // #1761 — how many times a policy's stop has been forcefully re-issued while the process refused to
   // confirm dead. Bounds the escalation so a wedged steward is retried, then given up to `failed`,

package/src/maintenance/cd-watcher.ts

@@ -140,7 +140,7 @@ export async function runCdWatcherJob(context: MaintenanceJobContext = { signal:
   // `offThread: true` mirrors the #1725 off-thread jobs (old-message-prune, db-maintenance, …): it
   // records in the run's persisted result that the heavy work ran in a subprocess, so an operator
   // reading the last result sees the same proof the run registry acted on for breaker attribution.
-  return { ...result, offThread: true, transition: notifyMainCiTransition(result.observation) };
+  return { ...result, offThread: true, transition: await notifyMainCiTransition(result.observation) };
 }
 
 /** Best-effort Cosmos deploy hero-shot: read the ledger row this deploy just
@@ -164,9 +164,9 @@ async function emitDeployHeroShot(sha: string | null): Promise<void> {
 }
 
 /** Best-effort transition alert; never throws into the deploy job. */
-function notifyMainCiTransition(observation: CiObservation): Record<string, unknown> {
+async function notifyMainCiTransition(observation: CiObservation): Promise<Record<string, unknown>> {
   try {
-    const { transition, incidentId, notified, messages } = notifyCiTransition(observation);
+    const { transition, incidentId, notified, messages } = await notifyCiTransition(observation);
     return { flip: transition, notified, messages, ...(incidentId ? { incidentId } : {}) };
   } catch (error) {
     const message = errMessage(error);

package/src/maintenance/critical-job-watchdog.ts

@@ -17,12 +17,13 @@
 //   - the configured fleet-ops agent target, so something that can actually act is woken.
 
 import { errMessage } from "@humwick/sdk";
-import { getConfig, setConfig } from "../config-store";
+import { getConfig } from "../config-store";
+import { setConfigAsync } from "../config-store-write";
+import { resetExpiredCriticalRunsAsync } from "../db/maintenance-job-runs-async";
 import { stewardFallbackTarget } from "../config";
 import {
   evaluateCriticalMaintenanceJobs,
   legacyHungCriticalMaintenanceRuns,
-  resetExpiredCriticalMaintenanceRuns,
   type CriticalJobCondition,
   type LegacyHungCriticalMaintenanceRun,
   type ResetCriticalMaintenanceRun,
@@ -69,11 +70,11 @@ export interface CriticalJobWatchdogResult {
   suppressed?: "unchanged-within-reminder-window" | "awaiting-second-failure";
 }
 
-export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWatchdogResult {
+export async function runCriticalJobWatchdog(now: number = Date.now()): Promise<CriticalJobWatchdogResult> {
   // Do this before evaluating health so the durable scheduler latch is recovered even if the
   // notification path is unavailable.  A normal timed-out run clears itself; this is only the
   // safety net for a row stranded in `running` after its own deadline lease elapsed.
-  const resetRuns = resetExpiredCriticalMaintenanceRuns(now);
+  const resetRuns = await resetExpiredCriticalRunsAsync(now);
   const legacyHungRuns = legacyHungCriticalMaintenanceRuns(now);
   const conditions = evaluateCriticalMaintenanceJobs(now);
   const errors = conditions.filter((condition) => condition.status === "error");
@@ -86,9 +87,9 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     state.missedChecks[key] = checks;
     return checks >= 2;
   });
-  const persist = () => {
+  const persist = async () => {
     if (JSON.stringify(state) !== JSON.stringify(prior)) {
-      setConfig(STATE_NAMESPACE, STATE_KEY, state, "critical-job-watchdog");
+      await setConfigAsync(STATE_NAMESPACE, STATE_KEY, state, "critical-job-watchdog");
     }
   };
   const heldRestartEpoch = currentHeldRestartEpoch(now);
@@ -97,7 +98,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     // Recovered (or never degraded): clear the marker so the NEXT degradation notifies immediately
     // rather than being swallowed by the reminder window opened by the previous one.
     state.lastNotified = null;
-    persist();
+    await persist();
     return errors.length > 0
       ? { status: "error", degradedJobIds: errors.map((condition) => condition.jobId), notified: false, recipients: [], suppressed: "awaiting-second-failure" }
       : { status: conditions.length > 0 ? "warn" : "ok", degradedJobIds: [], notified: false, recipients: [] };
@@ -115,7 +116,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     ...(heldRestartEpoch ? [`epoch:${heldRestartEpoch.id}:${heldRestartEpoch.blockerCode}`] : []),
   ].sort().join("|");
   if (state.lastNotified?.fingerprint === fingerprint && now - state.lastNotified.notifiedAt < DEGRADED_REMINDER_MS) {
-    persist();
+    await persist();
     return {
       status: "error", degradedJobIds,
       ...(resetRuns.length > 0 ? { resetJobIds: resetRuns.map((run) => run.jobId) } : {}),
@@ -154,7 +155,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
   }
 
   state.lastNotified = { fingerprint, notifiedAt: now };
-  persist();
+  await persist();
   return {
     status: "error", degradedJobIds,
     ...(resetRuns.length > 0 ? { resetJobIds: resetRuns.map((run) => run.jobId) } : {}),
@@ -236,16 +237,14 @@ function composeNotice(
 
 /** Handler wrapper: a watchdog that throws must not mark itself failed-and-silent, so failures are
  *  reported as a result the job record carries. */
-export function criticalJobWatchdogJob(context: { signal?: AbortSignal } = {}): Record<string, unknown> {
-  // #1819 — SYNCHRONOUS and single-unit: one `evaluateCriticalMaintenanceJobs` read then one notify
-  // fan-out that must not be split (the `lastNotified` fingerprint is stamped only after the sends,
-  // so a half-run would re-page on the next tick). The only safe checkpoint is before it starts, and
-  // on one thread that is also the only one that could ever fire. Its work is bounded by the number
-  // of critical jobs (one today), so overrun is not a live risk; if that ever changes this is an
-  // #1809 offload candidate, not a place for more checks.
+export async function criticalJobWatchdogJob(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  // #1819 — single-unit: one `evaluateCriticalMaintenanceJobs` read then one notify fan-out that must not
+  // be split (the `lastNotified` fingerprint is stamped only after the sends, so a half-run would re-page
+  // on the next tick). The only safe checkpoint is before it starts. Its two writes (the lease reset and
+  // the state persist) are writer ops (#1870 S7).
   if (context.signal?.aborted) return { status: "ok", aborted: true, degradedJobIds: [], notified: false, recipients: [] };
   try {
-    return { ...runCriticalJobWatchdog() };
+    return { ...(await runCriticalJobWatchdog()) };
   } catch (error) {
     return { status: "error", error: errMessage(error) };
   }

package/src/maintenance/index.ts

@@ -1,6 +1,7 @@
 export {
   cancelMaintenanceRun,
   listMaintenanceJobs,
+  listMaintenanceJobsAsync,
   listRecentMaintenanceRuns,
   listRunningMaintenanceRuns,
   resetOrphanTrackerForTests,

package/src/maintenance/jobs.ts

@@ -498,8 +498,9 @@ export const definitions: MaintenanceJobDefinition[] = [
       // event loop was stalled, a live replacement's heartbeats may still be unprocessed, so the
       // grace widens by the observed lag instead of misreading the backlog as post-mortem silence.
       // #1895 (S3.2 L10, review finding 3) — routed through the async gateway facade.
-      const revokedOrphanTokens = await revokeOrphanedRuntimeTokensAsync(STALE_TTL_MS, eventLoopLagGraceMs());
-      return { revokedOrphanTokens };
+      // #3776 — paged short writes, bounded by the abort signal and a time budget.
+      const { revoked, complete } = await revokeOrphanedRuntimeTokensAsync(STALE_TTL_MS, eventLoopLagGraceMs(), { signal: context.signal });
+      return { revokedOrphanTokens: revoked, moreWork: !complete };
     },
   },
   {

package/src/maintenance/scheduler.ts

@@ -2,8 +2,11 @@ import { timePostBindPhase } from "../post-bind-startup";
 import { randomUUID } from "node:crypto";
 import { errMessage, isRecord } from "@humwick/sdk";
 import { classifyPidLiveness } from "@humwick/sdk/process-utils";
-import { CRITICAL_MAINTENANCE_PROGRESS_STALE_MS, createActivityEventAsync, getDb, isSqliteBusyError, MAINTENANCE_RUN_AUTORESET_GUARD_SQL, recordDrainConvoy, writeTransaction } from "../db";
+import { CRITICAL_MAINTENANCE_PROGRESS_STALE_MS, createActivityEventAsync, getDb, MAINTENANCE_RUN_AUTORESET_GUARD_SQL, recordDrainConvoy, writeTransaction } from "../db";
 import { claimMaintenanceRunAsync, finishMaintenanceRunAsync, heartbeatMaintenanceRunAsync } from "../db/maintenance-job-runs-async";
+import { ensureMaintenanceJobsAsync, reclaimDeadLeasesAsync, rescheduleRunOnStartJobsAsync } from "../db/maintenance-job-runs-async";
+import { reclaimDeadLeases, rescheduleRunOnStartJobs, upsertMaintenanceJobs } from "../db/maintenance-job-reconcile";
+import type { DeadLease, MaintenanceJobSpec } from "../db/maintenance-job-reconcile";
 import type { MaintenanceRunOutcome } from "../db/maintenance-job-runs";
 import { withAbortDeadline } from "../async-deadline";
 import { MAINTENANCE_BREAKER_LAG_MS } from "../config";
@@ -173,36 +176,13 @@ export function resetOrphanTrackerForTests(): void {
 }
 
 export function startMaintenanceScheduler(): void {
-  // Reconcile the job table once at boot. A transient SQLITE_BUSY here must not
-  // abort startup — the first scheduler tick re-runs the reconcile with retry.
-  try {
-    ensureMaintenanceJobs();
-    scheduleRunOnStartJobs();
-  } catch (error) {
-    console.warn(`maintenance job reconcile at startup failed (will retry on next tick): ${errMessage(error)}`);
-    void ensureMaintenanceJobsResilient()
-      .then(() => {
-        scheduleRunOnStartJobs();
-        return runDueMaintenanceJobs();
-      })
-      .catch((retryError) => console.warn(`maintenance startup retry failed (will retry on next tick): ${errMessage(retryError)}`));
-  }
-  // #1493 — a relay restart orphans the maintenance-job lease its own dying
-  // process held moments earlier: lease_owner still names the dead PID and
-  // lease_until is still in the future, so the claim guard in runJob/
-  // runDueMaintenanceJobs (lease_until IS NULL OR lease_until <= now) blocks
-  // the job — e.g. the CD watcher — until the lease's TTL self-expires (up to
-  // deadlineMs + 5s). Reclaim any lease whose owner PID is no longer alive so
-  // maintenance resumes immediately after a restart instead of stalling.
-  try {
-    reclaimDeadPidLeases();
-  } catch (error) {
-    console.warn(`dead-PID maintenance lease reclaim at startup failed (will retry on next tick): ${errMessage(error)}`);
-  }
+  // #1870 S7: the boot reconcile, run-on-start reschedule and dead-lease reclaim are writer ops. The first
+  // tick waits for them, so a due run-on-start job is never claimed before its reschedule lands.
+  void bootMaintenanceJobs().then(() => runDueMaintenanceJobs())
+    .catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`));
   // Floated ticks are wrapped defensively: runDueMaintenanceJobs already catches
   // its own errors (#1481), but a stray rejection from a floated promise must
   // never bubble to an unhandledRejection that exits the relay process.
-  void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`));
   if (timer) return;
   timer = setInterval(
     () => void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`)),
@@ -210,6 +190,29 @@ export function startMaintenanceScheduler(): void {
   );
 }
 
+async function bootMaintenanceJobs(): Promise<void> {
+  // A transient SQLITE_BUSY here must not abort startup: every scheduler tick runs the reconcile again.
+  try {
+    await ensureMaintenanceJobsAsync(jobSpecs());
+  } catch (error) {
+    console.warn(`maintenance job reconcile at startup failed (will retry on next tick): ${errMessage(error)}`);
+  }
+  try {
+    await rescheduleRunOnStartJobsAsync();
+  } catch (error) {
+    // The next tick retries it, so run-on-start jobs do not wait a full interval (#1870 S7).
+    runOnStartPending = true;
+    console.warn(`maintenance run-on-start reschedule failed (will retry on next tick): ${errMessage(error)}`);
+  }
+  // #1493 — a restart orphans the lease its dying process held: lease_owner still names the dead PID and
+  // lease_until is still in the future, which blocks the job (e.g. the CD watcher) until the TTL expires.
+  try {
+    await reclaimDeadLeasesAsync(findDeadLeases());
+  } catch (error) {
+    console.warn(`dead-PID maintenance lease reclaim at startup failed (will retry on next tick): ${errMessage(error)}`);
+  }
+}
+
 /**
  * `runOnStart` means every Relay process start, not only the first insertion of
  * a durable maintenance_jobs row. Historically ensureMaintenanceJobs used
@@ -219,29 +222,37 @@ export function startMaintenanceScheduler(): void {
  * scheduler tick.
  */
 export function scheduleRunOnStartJobs(now: number = Date.now()): void {
-  getDb().query(`
-    UPDATE maintenance_jobs
-    SET next_run_at = ?, updated_at = ?
-    WHERE enabled = 1 AND run_on_start = 1
-  `).run(now, now);
+  rescheduleRunOnStartJobs(now);
 }
 
+function readMaintenanceJobs(): MaintenanceJob[] {
+  const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
+  const now = Date.now();
+  const currentIds = new Set(definitions.map((definition) => definition.id));
+  return rows.filter((row) => currentIds.has(row.id)).map((row) => rowToJob(row, now));
+}
+
+// A transient lock while reconciling must not fail a listing; return the current rows (a later reconcile catches up). #1481.
 export function listMaintenanceJobs(): MaintenanceJob[] {
   try {
     ensureMaintenanceJobs();
   } catch (error) {
-    // A transient lock while reconciling must not fail a read-only listing; fall
-    // through and return the current rows (a later reconcile catches up). #1481.
     console.warn(`maintenance job reconcile during list failed: ${errMessage(error)}`);
   }
-  const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
-  const now = Date.now();
-  const currentIds = new Set(definitions.map((definition) => definition.id));
-  return rows.filter((row) => currentIds.has(row.id)).map((row) => rowToJob(row, now));
+  return readMaintenanceJobs();
+}
+
+export async function listMaintenanceJobsAsync(): Promise<MaintenanceJob[]> {
+  try {
+    await ensureMaintenanceJobsAsync(jobSpecs());
+  } catch (error) {
+    console.warn(`maintenance job reconcile during list failed: ${errMessage(error)}`);
+  }
+  return readMaintenanceJobs();
 }
 
 export async function runMaintenanceJobNow(id: string, options: { overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
-  await ensureMaintenanceJobsResilient();
+  await ensureMaintenanceJobsAsync(jobSpecs());
   const definition = definitions.find((job) => job.id === id);
   if (!definition) throw new Error("maintenance job not found");
   return runJob(definition, { force: true, overridePause: options.overridePause });
@@ -266,6 +277,7 @@ async function runMaintenanceJobsNow(ids: string[]): Promise<MaintenanceJobRun[]
 // no-op (the still-open tick, or the one after, picks up whatever came due). Combined with the
 // serialized drain below, this is the "one job at a time from a queue" the design (§5.4) calls for.
 let tickInFlight = false;
+let runOnStartPending = false;
 
 export async function runDueMaintenanceJobs(): Promise<void> {
   if (tickInFlight) return;
@@ -274,7 +286,11 @@ export async function runDueMaintenanceJobs(): Promise<void> {
   // is guarded: a transient SQLITE_BUSY (or any other failure) reconciling or
   // querying the job table is logged and the tick is skipped — the next tick retries.
   try {
-    await ensureMaintenanceJobsResilient();
+    await ensureMaintenanceJobsAsync(jobSpecs());
+    if (runOnStartPending) {
+      await rescheduleRunOnStartJobsAsync();
+      runOnStartPending = false;
+    }
     const now = Date.now();
     const progressStaleAt = now - CRITICAL_MAINTENANCE_PROGRESS_STALE_MS;
     const rows = getDb().query(`
@@ -363,84 +379,28 @@ export function drainDueMaintenanceJobsForTests(dueDefinitions: MaintenanceJobDe
  *  otherwise leave it latched, wedging every later test's tick). Production never calls this. */
 export function resetSchedulerTickGuardForTests(): void {
   tickInFlight = false;
+  runOnStartPending = false;
 }
 
-// Bounded, ASYNC, loop-yielding retry around the (synchronous) job-table reconcile.
-// Mirrors runWriteWithBusyRetry (connection.ts) but wraps ensureMaintenanceJobs,
-// which runs its own BEGIN IMMEDIATE transaction and so cannot be nested inside
-// runWriteWithBusyRetry. busy_timeout already makes SQLite WAIT on a held lock; the
-// rare BUSY that survives it is retried here with jittered backoff that yields the
-// event loop between attempts. Anything else — or exhausted attempts — rethrows to
-// the caller, which degrades the tick rather than crashing the process (#1481).
-const ENSURE_BUSY_RETRY_ATTEMPTS = 4;
-const ENSURE_BUSY_RETRY_BASE_MS = 15;
-const ENSURE_BUSY_RETRY_MAX_MS = 120;
-
-async function ensureMaintenanceJobsResilient(): Promise<void> {
-  for (let attempt = 1; ; attempt++) {
-    try {
-      ensureMaintenanceJobs();
-      return;
-    } catch (error) {
-      if (attempt < ENSURE_BUSY_RETRY_ATTEMPTS && isSqliteBusyError(error)) {
-        const cap = Math.min(ENSURE_BUSY_RETRY_MAX_MS, ENSURE_BUSY_RETRY_BASE_MS * 2 ** (attempt - 1));
-        const delay = cap / 2 + Math.random() * (cap / 2);
-        await new Promise((resolve) => setTimeout(resolve, delay));
-        continue;
-      }
-      throw error;
-    }
-  }
+function jobSpecs(): MaintenanceJobSpec[] {
+  return definitions.map((definition) => ({
+    id: definition.id,
+    title: definition.title,
+    description: definition.description,
+    intervalMs: definition.intervalMs,
+    deadlineMs: definition.deadlineMs ?? DEFAULT_DEADLINE_MS,
+    enabled: !isEphemeralMode() || definition.instanceSafe === true,
+    critical: definition.critical === true,
+    runOnStart: definition.runOnStart === true,
+  }));
 }
 
+// Boot/test twin of `ensureMaintenanceJobsAsync`: one BEGIN IMMEDIATE so busy_timeout governs a single lock
+// acquisition (#1481). The listing read path and tests use it; the running scheduler uses the writer op.
 function ensureMaintenanceJobs(): void {
-  const db = getDb();
+  const specs = jobSpecs();
   const now = Date.now();
-  // #1645 — `enabled` is re-asserted from `definitions` on every reconcile, which is what makes it
-  // CODE-owned and therefore self-healing. Before this it was set only on INSERT and deliberately
-  // omitted from the conflict clause, because the one column doubled as the operator's "stop" lever
-  // and a redeploy would have clobbered their intent. Operator intent now lives in
-  // `maintenance_job_pauses` (holder, reason, expiry), so re-asserting `enabled` overrides nobody —
-  // and a hand-written `UPDATE maintenance_jobs SET enabled=0` recovers at the next boot instead of
-  // persisting until someone writes more raw SQL (nine hours of deploy blackout, #1645).
-  //
-  // PRECONDITION: convertManualJobDisablesToPauses() (src/db/migrations.ts) must have run first, or
-  // this silently switches back on a job an operator deliberately stopped.
-  const stmt = db.query(`
-    INSERT INTO maintenance_jobs (id, title, description, interval_ms, timeout_ms, enabled, critical, run_on_start, next_run_at, updated_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(id) DO UPDATE SET
-      title = excluded.title,
-      description = excluded.description,
-      interval_ms = excluded.interval_ms,
-      timeout_ms = excluded.timeout_ms,
-      run_on_start = excluded.run_on_start,
-      enabled = excluded.enabled, -- #1645: code-owned, see the note above this query
-      critical = excluded.critical,
-      next_run_at = COALESCE(maintenance_jobs.next_run_at, excluded.next_run_at),
-      updated_at = excluded.updated_at
-  `);
-  // Batch the upserts into a single BEGIN IMMEDIATE write so busy_timeout governs
-  // ONE lock acquisition (SQLite waits) instead of one per definition, cutting the
-  // lock churn that made this loop the crash site under contention (#1481).
-  writeTransaction(() => {
-    for (const definition of definitions) {
-      stmt.run(
-        definition.id,
-        definition.title,
-        definition.description,
-        definition.intervalMs,
-        // #1728: the DB column stays `timeout_ms` — renaming it is a migration, not asked for here.
-        // Code-side this is `deadlineMs`: see MaintenanceJobDefinition for why.
-        definition.deadlineMs ?? DEFAULT_DEADLINE_MS,
-        !isEphemeralMode() || definition.instanceSafe === true ? 1 : 0,
-        definition.critical ? 1 : 0,
-        definition.runOnStart ? 1 : 0,
-        definition.runOnStart ? now : now + definition.intervalMs,
-        now,
-      );
-    }
-  });
+  writeTransaction(() => upsertMaintenanceJobs(specs, now));
 }
 
 // #1493 — reclaim current-schema maintenance_jobs leases whose owner PID is proven dead.
@@ -455,23 +415,19 @@ function ensureMaintenanceJobs(): void {
 // #3645: the crash is not a job failure — it neither bumps consecutive_failures nor replaces an existing
 // last_error (a non-null one belongs to the live failure streak; success clears it). The page-only guard remains on every
 // deadline-based reclaim/claim path, where that positive evidence is unavailable.
-export function reclaimDeadPidLeases(): void {
-  const db = getDb();
-  const now = Date.now();
-  const leased = db.query(`
-    SELECT id, lease_owner FROM maintenance_jobs
-    WHERE lease_owner IS NOT NULL
-  `).all() as Array<{ id: string; lease_owner: string }>;
+function findDeadLeases(): DeadLease[] {
+  const leased = getDb().query("SELECT id, lease_owner FROM maintenance_jobs WHERE lease_owner IS NOT NULL").all() as Array<{ id: string; lease_owner: string }>;
+  const dead: DeadLease[] = [];
   for (const row of leased) {
     const pid = parseLeaseOwnerPid(row.lease_owner);
-    if (pid === null || classifyPidLiveness(pid) !== "dead") continue;
-    db.query(`
-      UPDATE maintenance_jobs
-      SET lease_owner = NULL, lease_until = NULL, last_status = 'failed',
-          last_error = COALESCE(last_error, ?), run_token = NULL, last_progress_at = ?, updated_at = ?
-      WHERE id = ? AND lease_owner = ?
-    `).run(`reclaimed stale lease at startup: owner pid ${pid} is confirmed dead`, now, now, row.id, row.lease_owner);
+    if (pid !== null && classifyPidLiveness(pid) === "dead") dead.push({ id: row.id, owner: row.lease_owner, pid });
   }
+  return dead;
+}
+
+export function reclaimDeadPidLeases(): void {
+  const dead = findDeadLeases();
+  if (dead.length) writeTransaction(() => reclaimDeadLeases(dead, Date.now()));
 }
 
 function parseLeaseOwnerPid(owner: string): number | null {
@@ -527,7 +483,7 @@ async function runJob(definition: MaintenanceJobDefinition, options: { force?: b
   const progressStaleAt = startedAt - CRITICAL_MAINTENANCE_PROGRESS_STALE_MS;
   const claimed = await claimMaintenanceRunAsync({ id: definition.id, owner, runToken, startedAt, leaseUntil, progressStaleAt, force: options.force === true });
   if (!claimed) {
-    const state = listMaintenanceJobs().find((job) => job.id === definition.id);
+    const state = readMaintenanceJobs().find((job) => job.id === definition.id);
     return { id: definition.id, status: "skipped", startedAt, finishedAt: Date.now(), durationMs: 0, result: { reason: "not due or already running", state } };
   }
 

package/src/maintenance/stalled-delivery-pager.ts

@@ -1,5 +1,6 @@
 import { errMessage } from "@humwick/sdk";
-import { getConfig, setConfig } from "../config-store";
+import { getConfig } from "../config-store";
+import { setConfigAsync } from "../config-store-write";
 import { getStalledDeliverySignalAsync } from "../db/stats-async.ts";
 import { notifyWorkspaceHumanEscalation } from "../workspace-human-escalation";
 import type { MaintenanceJobDefinition } from "./types";
@@ -56,8 +57,8 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
   }
   let alerted = false;
   let recovered = false;
-  const persist = () => {
-    if (JSON.stringify(state) !== JSON.stringify(prior)) setConfig(STATE_NAMESPACE, STATE_KEY, state, "stalled-delivery-pager");
+  const persist = async () => {
+    if (JSON.stringify(state) !== JSON.stringify(prior)) await setConfigAsync(STATE_NAMESPACE, STATE_KEY, state, "stalled-delivery-pager");
   };
 
   const { check } = signal;
@@ -87,7 +88,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
     if (!state.stalled && state.failStreak >= STALL_PAGE_CONFIRM_CHECKS) state = { ...state, stalled: true, since: now, alerted: false };
     if (state.stalled && !state.alerted) {
       // Persist the episode first so a crash after the send re-sends under the same key.
-      persist();
+      await persist();
       const result = notifyWorkspaceHumanEscalation({
         subject: "Message delivery stalled",
         body: [
@@ -107,7 +108,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
       state = { ...state, alerted: true };
     }
   }
-  persist();
+  await persist();
   return { state: state.stalled ? "stalled" : "ok", alerted, recovered };
 }
 

package/src/maintenance/workspaces.ts

@@ -487,7 +487,7 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
       const proof = landedElsewhere === "confirmed" ? "ancestor" : landedElsewhere;
       await reconcileLandedWorkspace(fresh, zeroCommitCloseOut
         ? { strategy: preview && !("available" in preview) ? preview.strategy : "rebase-ff", baseRef, ahead: 0, unmergedAhead: 0, dirtyCount: 0 }
-        : { strategy: preview && !("available" in preview) ? preview.strategy : "rebase-ff", baseRef, headSha: currentTip, landed: true, landedSha: currentTip });
+        : { strategy: preview && !("available" in preview) ? preview.strategy : "rebase-ff", baseRef, headSha: currentTip, landed: true, landedSha: currentTip }, landedElsewhere === "confirmed" ? currentTip : undefined);
       await patchWorkspaceMetadataAsync(fresh.id, {
         landedElsewhere: true,
         landedElsewhereAt: now,