@humwick/server 0.130.1 → 0.130.3

package/src/maintenance/breaker.ts

@@ -1,3 +1,5 @@
+import { pauseMaintenanceJobAsync } from "../db/runtime-tail-async";
+import { createActivityEventAsync } from "../db/activity-async";
 // #1724 stage 1 — the maintenance circuit breaker's bookkeeping and trip actions.
 //
 // The design (§5.4, §8.5) asks for a breaker that, when a job repeatedly starves the event loop (or
@@ -15,7 +17,7 @@
 
 import { errMessage } from "@humwick/sdk";
 import { MAINTENANCE_BREAKER_MAX_STRIKES } from "../config";
-import { AUTO_BREAKER_PAUSE_HOLDER, createActivityEvent, pauseMaintenanceJob } from "../db";
+import { AUTO_BREAKER_PAUSE_HOLDER } from "../db";
 
 // Two strike counters per job, because the two failure signals want different reset semantics — and
 // getting this wrong silently defeats the breaker:
@@ -81,7 +83,7 @@ export interface BreakerStrikeResult {
  * and, on reaching MAINTENANCE_BREAKER_MAX_STRIKES, trips: a non-critical job is durably auto-paused;
  * a critical one is escalated LOUDLY but never paused. Idempotent per runId.
  */
-export function recordBreakerStrike(input: BreakerStrikeInput): BreakerStrikeResult {
+export async function recordBreakerStrike(input: BreakerStrikeInput): Promise<BreakerStrikeResult> {
   if (!markStruck(input.runId)) {
     return { struck: false, strikes: breakerStrikeCount(input.jobId), tripped: false };
   }
@@ -92,7 +94,7 @@ export function recordBreakerStrike(input: BreakerStrikeInput): BreakerStrikeRes
   if (count < MAINTENANCE_BREAKER_MAX_STRIKES) {
     return { struck: true, strikes: breakerStrikeCount(input.jobId), tripped: false };
   }
-  const action = trip(input.jobId, input.critical, input.reason);
+  const action = (await trip(input.jobId, input.critical, input.reason));
   return { struck: true, strikes: 0, tripped: true, action };
 }
 
@@ -119,34 +121,34 @@ export function isRunStruck(runId: string): boolean {
   return struckRunIds.has(runId);
 }
 
-function trip(jobId: string, critical: boolean, reason: string): "paused" | "escalated" | "pause-conflict" {
+async function trip(jobId: string, critical: boolean, reason: string): Promise<"paused" | "escalated" | "pause-conflict"> {
   // Reset BOTH counters first: after a trip the job is either paused (won't run) or escalated; either
   // way the next strike should start a fresh count rather than instantly re-trip on every later run.
   timeoutStrikes.delete(jobId);
   lagStrikes.delete(jobId);
   if (critical) {
-    escalateCritical(jobId, reason);
+    (await escalateCritical(jobId, reason));
     return "escalated";
   }
-  return autoPause(jobId, reason);
+  return (await autoPause(jobId, reason));
 }
 
-function autoPause(jobId: string, reason: string): "paused" | "pause-conflict" {
+async function autoPause(jobId: string, reason: string): Promise<"paused" | "pause-conflict"> {
   const pauseReason = `auto-paused by circuit breaker: ${reason}`;
   console.warn(`[relay] maintenance circuit breaker TRIPPED — durably auto-pausing ${jobId}: ${reason}`);
   try {
     // Indefinite on purpose: a job that was starving the loop must NOT quietly resume itself on a
     // timer. A human resumes it once it is fixed — and the `maintenance-auto-paused` health arm is
     // what makes sure a human sees it (this pause is otherwise invisible to the critical-only arm).
-    const result = pauseMaintenanceJob({ jobId, holder: AUTO_BREAKER_PAUSE_HOLDER, reason: pauseReason, indefinite: true });
-    auditBreakerEvent({
+    const result = (await pauseMaintenanceJobAsync({ jobId, holder: AUTO_BREAKER_PAUSE_HOLDER, reason: pauseReason, indefinite: true }));
+    (await auditBreakerEvent({
       jobId,
       clientId: `server-maintenance-breaker-pause-${jobId}-${result.pause.createdAt}`,
       title: `Maintenance circuit breaker auto-paused ${jobId}`,
       body: `${jobId} was auto-paused by the event-loop-lag circuit breaker after ${MAINTENANCE_BREAKER_MAX_STRIKES} strikes — ${reason}. It will stay paused until a human resumes it.`,
       icon: "ti-alert-octagon",
       metadata: { breaker: true, pauseId: result.pause.id, alreadyHeld: !result.ok },
-    });
+    }));
     return result.ok ? "paused" : "pause-conflict";
   } catch (error) {
     console.warn(`[relay] maintenance circuit breaker could not pause ${jobId}: ${errMessage(error)}`);
@@ -154,27 +156,27 @@ function autoPause(jobId: string, reason: string): "paused" | "pause-conflict" {
   }
 }
 
-function escalateCritical(jobId: string, reason: string): void {
+async function escalateCritical(jobId: string, reason: string): Promise<void> {
   // §8.5 — the one thing the breaker must never do is silence a critical job the way a bare
   // `enabled=0` silenced cd-watcher for nine hours. So a critical job is escalated, never paused: it
   // KEEPS running (and keeps being reported by criticalMaintenanceJobsCheck if it is genuinely
   // failing), and a loud, attributed event tells a human to intervene deliberately.
   console.warn(`[relay] maintenance circuit breaker: CRITICAL job ${jobId} repeatedly misbehaving (${reason}) — NOT auto-paused (see #1645/§8.5), escalating for a human`);
-  auditBreakerEvent({
+  (await auditBreakerEvent({
     jobId,
     clientId: `server-maintenance-breaker-escalate-${jobId}-${Date.now()}`,
     title: `Maintenance circuit breaker: critical job ${jobId} needs a human`,
     body: `${jobId} repeatedly starved the event loop or overran its deadline (${reason}). Because it is CRITICAL, the breaker did NOT auto-pause it — a machine silencing a critical job is the #1645 blackout. A human must decide what to do.`,
     icon: "ti-alert-triangle",
     metadata: { breaker: true, critical: true, escalated: true },
-  });
+  }));
 }
 
-function auditBreakerEvent(input: { jobId: string; clientId: string; title: string; body: string; icon: string; metadata: Record<string, unknown> }): void {
+async function auditBreakerEvent(input: { jobId: string; clientId: string; title: string; body: string; icon: string; metadata: Record<string, unknown> }): Promise<void> {
   try {
     // #1912 S3.2: sync-retained — the lag sampler invokes recordBreakerStrike synchronously and
     // cannot await without redesigning the event-loop attribution callback contract.
-    createActivityEvent({
+    (await createActivityEventAsync({
       clientId: input.clientId,
       kind: "operator",
       title: input.title,
@@ -183,7 +185,7 @@ function auditBreakerEvent(input: { jobId: string; clientId: string; title: stri
       icon: input.icon,
       view: "activity",
       metadata: { source: "server", maintenanceJobId: input.jobId, ...input.metadata },
-    });
+    }));
   } catch (error) {
     // The pause/escalation itself already happened; losing the timeline entry must not throw back
     // into the scheduler's tick.

package/src/maintenance/cd-watcher.ts

@@ -140,7 +140,7 @@ export async function runCdWatcherJob(context: MaintenanceJobContext = { signal:
   // `offThread: true` mirrors the #1725 off-thread jobs (old-message-prune, db-maintenance, …): it
   // records in the run's persisted result that the heavy work ran in a subprocess, so an operator
   // reading the last result sees the same proof the run registry acted on for breaker attribution.
-  return { ...result, offThread: true, transition: notifyMainCiTransition(result.observation) };
+  return { ...result, offThread: true, transition: await notifyMainCiTransition(result.observation) };
 }
 
 /** Best-effort Cosmos deploy hero-shot: read the ledger row this deploy just
@@ -164,9 +164,9 @@ async function emitDeployHeroShot(sha: string | null): Promise<void> {
 }
 
 /** Best-effort transition alert; never throws into the deploy job. */
-function notifyMainCiTransition(observation: CiObservation): Record<string, unknown> {
+async function notifyMainCiTransition(observation: CiObservation): Promise<Record<string, unknown>> {
   try {
-    const { transition, incidentId, notified, messages } = notifyCiTransition(observation);
+    const { transition, incidentId, notified, messages } = await notifyCiTransition(observation);
     return { flip: transition, notified, messages, ...(incidentId ? { incidentId } : {}) };
   } catch (error) {
     const message = errMessage(error);

package/src/maintenance/checkout-refresh.ts

@@ -9,6 +9,7 @@ import { isRecord } from "@humwick/sdk";
 import { createHash } from "node:crypto";
 import { sendMessageServiceAsync } from "../services/send-message";
 import { checkoutRefreshEnabled } from "../config";
+import { runGit } from "../services/git-process";
 
 async function notice(projectId: string, projectSlug: string, host: string, rootPath: string, reason: string): Promise<void> {
   const key = createHash("sha256").update(JSON.stringify([projectId, host, rootPath, reason])).digest("hex");
@@ -28,6 +29,7 @@ export function reconcileCheckoutRefresh(signal?: AbortSignal): Promise<Record<s
 async function reconcile(signal?: AbortSignal): Promise<Record<string, unknown>> {
   const dispatched: string[] = [];
   const skipped: string[] = [];
+  const remoteTips = new Map<string, Promise<string | undefined>>();
   for (const project of await listProjectsAsync()) {
     if (signal?.aborted) break;
     const roots = await rootsForProjectAsync(project.id);
@@ -55,8 +57,23 @@ async function reconcile(signal?: AbortSignal): Promise<Record<string, unknown>>
       if (prior && ["pending", "accepted", "running"].includes(prior.status)) continue;
       // The host reported once that it cannot refresh; re-asking every tick only repeats that (#3629).
       if (health === "unsupported") { skipped.push(`${project.slug}: unsupported host ${hostId} for ${root.rootPath}`); continue; }
+      // Completed receipts remain valid until the remote default branch moves (#3813).
+      if (prior?.status === "succeeded" && prior.result?.targetSha && ["current", "refreshed", "disabled"].includes(prior.result.status)) {
+        let tip = remoteTips.get(root.remoteUrl!);
+        if (!tip) {
+          tip = runGit(["ls-remote", "--symref", "--", root.remoteUrl!, "HEAD"], 10_000, 4096).then(result => {
+            if (result.exitCode !== 0 || result.unavailable || !/^ref: refs\/heads\/.+\tHEAD$/m.test(result.stdout)) return undefined;
+            return result.stdout.match(/^([0-9a-f]{40,64})\tHEAD$/m)?.[1];
+          });
+          remoteTips.set(root.remoteUrl!, tip);
+        }
+        const sha = await tip;
+        if (!sha) { skipped.push(`${project.slug}: remote default branch unavailable`); continue; }
+        if (sha === prior.result.targetSha) continue;
+      }
       // Leave one tick of backoff after terminal failure, including restart/lease recovery (#3691).
       if (prior && (["failed", "timed_out"].includes(prior.status) || prior.result?.status === "error") && Date.now() < (prior.updatedAt ?? 0) + 60_000) continue;
+      if (signal?.aborted) break;
       const command = await createCommandAsync({ type: "project.checkout.refresh", source: "system", target: `orchestrator-${hostId}`,
         params: { projectId: project.id, rootPath: root.rootPath, remoteUrl: root.remoteUrl, orchestratorId: hostId, enabled: true, ...(prior?.result ? { previousResult: prior.result } : {}) } });
       emitCommandEvent(command, "command.requested");
@@ -66,7 +83,7 @@ async function reconcile(signal?: AbortSignal): Promise<Record<string, unknown>>
   return { enabled: true, dispatched, skipped };
 }
 
-// A confirmed publish accelerates the same periodic reconciliation; external pushes converge on ticks.
+// Publish hints and external pushes use the same remote-tip comparison.
 subscribeRelayEvents((event) => {
   if (!event.type.startsWith("command.") || !isRecord(event.data.command)) return;
   const command = event.data.command;

package/src/maintenance/critical-job-watchdog.ts

@@ -17,12 +17,13 @@
 //   - the configured fleet-ops agent target, so something that can actually act is woken.
 
 import { errMessage } from "@humwick/sdk";
-import { getConfig, setConfig } from "../config-store";
+import { getConfig } from "../config-store";
+import { setConfigAsync } from "../config-store-write";
+import { resetExpiredCriticalRunsAsync } from "../db/maintenance-job-runs-async";
 import { stewardFallbackTarget } from "../config";
 import {
   evaluateCriticalMaintenanceJobs,
   legacyHungCriticalMaintenanceRuns,
-  resetExpiredCriticalMaintenanceRuns,
   type CriticalJobCondition,
   type LegacyHungCriticalMaintenanceRun,
   type ResetCriticalMaintenanceRun,
@@ -69,11 +70,11 @@ export interface CriticalJobWatchdogResult {
   suppressed?: "unchanged-within-reminder-window" | "awaiting-second-failure";
 }
 
-export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWatchdogResult {
+export async function runCriticalJobWatchdog(now: number = Date.now()): Promise<CriticalJobWatchdogResult> {
   // Do this before evaluating health so the durable scheduler latch is recovered even if the
   // notification path is unavailable.  A normal timed-out run clears itself; this is only the
   // safety net for a row stranded in `running` after its own deadline lease elapsed.
-  const resetRuns = resetExpiredCriticalMaintenanceRuns(now);
+  const resetRuns = await resetExpiredCriticalRunsAsync(now);
   const legacyHungRuns = legacyHungCriticalMaintenanceRuns(now);
   const conditions = evaluateCriticalMaintenanceJobs(now);
   const errors = conditions.filter((condition) => condition.status === "error");
@@ -86,9 +87,9 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     state.missedChecks[key] = checks;
     return checks >= 2;
   });
-  const persist = () => {
+  const persist = async () => {
     if (JSON.stringify(state) !== JSON.stringify(prior)) {
-      setConfig(STATE_NAMESPACE, STATE_KEY, state, "critical-job-watchdog");
+      await setConfigAsync(STATE_NAMESPACE, STATE_KEY, state, "critical-job-watchdog");
     }
   };
   const heldRestartEpoch = currentHeldRestartEpoch(now);
@@ -97,7 +98,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     // Recovered (or never degraded): clear the marker so the NEXT degradation notifies immediately
     // rather than being swallowed by the reminder window opened by the previous one.
     state.lastNotified = null;
-    persist();
+    await persist();
     return errors.length > 0
       ? { status: "error", degradedJobIds: errors.map((condition) => condition.jobId), notified: false, recipients: [], suppressed: "awaiting-second-failure" }
       : { status: conditions.length > 0 ? "warn" : "ok", degradedJobIds: [], notified: false, recipients: [] };
@@ -115,7 +116,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
     ...(heldRestartEpoch ? [`epoch:${heldRestartEpoch.id}:${heldRestartEpoch.blockerCode}`] : []),
   ].sort().join("|");
   if (state.lastNotified?.fingerprint === fingerprint && now - state.lastNotified.notifiedAt < DEGRADED_REMINDER_MS) {
-    persist();
+    await persist();
     return {
       status: "error", degradedJobIds,
       ...(resetRuns.length > 0 ? { resetJobIds: resetRuns.map((run) => run.jobId) } : {}),
@@ -127,18 +128,18 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
 
   const { subject, body, payload } = composeNotice(degraded, resetRuns, legacyHungRuns, heldRestartEpoch, now);
   const recipients: string[] = [];
-  const human = notifyWorkspaceHumanEscalation({
+  const human = (await notifyWorkspaceHumanEscalation({
     subject,
     body,
     payload,
     source: "maintenance-critical-job-watchdog",
     tags: ["agent-relay", "maintenance"],
-  });
+  }));
   if (human.accepted && human.channelId) recipients.push(human.channelId);
 
   const agentTarget = stewardFallbackTarget();
   if (agentTarget) {
-    const routed = notifyTarget("maintenance.degraded", {}, agentTarget, subject, body, payload);
+    const routed = await notifyTarget("maintenance.degraded", {}, agentTarget, subject, body, payload);
     if (routed) recipients.push(routed);
   }
 
@@ -154,7 +155,7 @@ export function runCriticalJobWatchdog(now: number = Date.now()): CriticalJobWat
   }
 
   state.lastNotified = { fingerprint, notifiedAt: now };
-  persist();
+  await persist();
   return {
     status: "error", degradedJobIds,
     ...(resetRuns.length > 0 ? { resetJobIds: resetRuns.map((run) => run.jobId) } : {}),
@@ -236,16 +237,14 @@ function composeNotice(
 
 /** Handler wrapper: a watchdog that throws must not mark itself failed-and-silent, so failures are
  *  reported as a result the job record carries. */
-export function criticalJobWatchdogJob(context: { signal?: AbortSignal } = {}): Record<string, unknown> {
-  // #1819 — SYNCHRONOUS and single-unit: one `evaluateCriticalMaintenanceJobs` read then one notify
-  // fan-out that must not be split (the `lastNotified` fingerprint is stamped only after the sends,
-  // so a half-run would re-page on the next tick). The only safe checkpoint is before it starts, and
-  // on one thread that is also the only one that could ever fire. Its work is bounded by the number
-  // of critical jobs (one today), so overrun is not a live risk; if that ever changes this is an
-  // #1809 offload candidate, not a place for more checks.
+export async function criticalJobWatchdogJob(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  // #1819 — single-unit: one `evaluateCriticalMaintenanceJobs` read then one notify fan-out that must not
+  // be split (the `lastNotified` fingerprint is stamped only after the sends, so a half-run would re-page
+  // on the next tick). The only safe checkpoint is before it starts. Its two writes (the lease reset and
+  // the state persist) are writer ops (#1870 S7).
   if (context.signal?.aborted) return { status: "ok", aborted: true, degradedJobIds: [], notified: false, recipients: [] };
   try {
-    return { ...runCriticalJobWatchdog() };
+    return { ...(await runCriticalJobWatchdog()) };
   } catch (error) {
     return { status: "error", error: errMessage(error) };
   }

package/src/maintenance/events.ts

@@ -9,7 +9,7 @@ export function emitCommand(command: Command): void {
 
 // Route a maintenance notification, swallowing failures (a stale/missing/misconfigured
 // target must never break the sweep). Returns the target on success, null otherwise.
-export function notifyTarget(
+export async function notifyTarget(
   type: NotificationType,
   scope: NotificationScope,
   target: string,
@@ -17,10 +17,10 @@ export function notifyTarget(
   body: string,
   payload: Record<string, unknown>,
   idempotencyKey?: string,
-): string | null {
+): Promise<string | null> {
   if (!target) return null;
   try {
-    routeNotification({ type, scope, recipients: [target], message: { subject, body, payload, idempotencyKey } });
+    await routeNotification({ type, scope, recipients: [target], message: { subject, body, payload, idempotencyKey } });
     return target;
   } catch {
     return null;

package/src/maintenance/history-retention-job.ts

@@ -0,0 +1,30 @@
+import { getGateway } from "../db/gateway";
+import type { MaintenanceJobDefinition } from "./types";
+
+export const HISTORY_RETENTION_JOB: MaintenanceJobDefinition = {
+  id: "history-retention", critical: false,
+  title: "Command, kernel replay and token history retention",
+  description: "Expire superseded checkout refresh commands after one hour, other non-spawn terminal commands and kernel replay receipts after 30 days, and raw usage after 90 days. Preserve references, expired replay keys and permanent budget spend (#3619, #3797).",
+  intervalMs: 60 * 60_000, backlogIntervalMs: 1_000, deadlineMs: 30_000, runOnStart: true,
+  async handler(context) {
+    const gateway = getGateway();
+    const result = { prunedCommands: 0, prunedKernelReceipts: 0, prunedTokenReports: 0, moreWork: false };
+    for (const [op, key] of [
+      ["historyRetention.commands", "prunedCommands"],
+      ["historyRetention.kernel", "prunedKernelReceipts"],
+      ["historyRetention.tokens", "prunedTokenReports"],
+    ] as const) {
+      if (context.signal.aborted) return { ...result, aborted: true, moreWork: true };
+      const start = performance.now();
+      let batch: { pruned: number; moreWork: boolean };
+      do {
+        if (context.signal.aborted) return { ...result, aborted: true, moreWork: true };
+        batch = await gateway.write(op, {}) as typeof batch;
+        result[key] += batch.pruned;
+        // Each await releases the writer for foreground work; bound the whole catch-up run (#3807).
+      } while (op === "historyRetention.commands" && batch.moreWork && performance.now() - start < 250);
+      result.moreWork ||= batch.moreWork;
+    }
+    return result;
+  },
+};

package/src/maintenance/index.ts

@@ -1,6 +1,7 @@
 export {
   cancelMaintenanceRun,
   listMaintenanceJobs,
+  listMaintenanceJobsAsync,
   listRecentMaintenanceRuns,
   listRunningMaintenanceRuns,
   resetOrphanTrackerForTests,

package/src/maintenance/jobs.ts

@@ -1,5 +1,9 @@
+import { OFFLINE_AGENT_PRUNE_JOB } from "./offline-agent-prune-job";
+import { pruneOrphanedSharedWorkspacesAsync, releaseExpiredDriveLeasesAsync } from "../db/runtime-tail-async";
+import { HISTORY_RETENTION_JOB } from "./history-retention-job";
 import { isHumanAgentId } from "@humwick/sdk";
-import { IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB } from "./long-running-work";
+import { REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
+import { IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB, PLAN_READY_NOTICE_JOB } from "./long-running-work";
 import { CHECKOUT_REFRESH_JOB } from "./checkout-refresh";
 import { CLAUDE_API_BUDGET_RECONCILE_JOB } from "./claude-api-budget-job";
 import { SERVICE_ACCESS_JOBS } from "./service-access-job";
@@ -15,27 +19,9 @@ import { deriveAgentBusyHealth } from "../agent-busy-health";
 import { expireStaleBusAgents } from "../bus";
 import { eventLoopLagGraceMs } from "../event-loop-lag";
 import { pruneOutboxAsync } from "../bus-outbox-async";
-import { expireCommands } from "../commands-db";
+import { expireCommandsAsync } from "../db/runtime-admin-async";
 import { getRetentionConfig } from "../retention-config-store";
-import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
-import {
-  createActivityEventAsync,
-  evaluatePoolBindingsAsync,
-  getAgent,
-  listAgents,
-  listStaleQueuedDirectTargetsAsync,
-  pruneOrphanedSharedWorkspaces,
-  pruneOfflineAgentsAsync,
-  pruneRetentionBatchAsync,
-  pruneProviderQuotaHistoryAsync,
-  reapStaleOrchestratorsAsync,
-  releaseExpiredClaimsAsync,
-  releaseExpiredDriveLeases,
-  releaseOrphanedTasksAsync,
-  reapStaleAgentsAsync,
-  revokeOrphanedRuntimeTokensAsync,
-  runDbMaintenanceAsync,
-} from "../db";
+import { createActivityEventAsync, evaluatePoolBindingsAsync, getAgent, listAgents, listStaleQueuedDirectTargetsAsync, pruneRetentionBatchAsync, pruneProviderQuotaHistoryAsync, reapStaleOrchestratorsAsync, releaseExpiredClaimsAsync, releaseOrphanedTasksAsync, reapStaleAgentsAsync, revokeOrphanedRuntimeTokensAsync, runDbMaintenanceAsync } from "../db";
 import { isAgentUnavailable } from "../db/agent-predicates";
 import { resolveTeamCoordinatorAgentId } from "../coordinator-authority";
 import { flushQueuedDirectMessages } from "../services/managed-running";
@@ -49,7 +35,6 @@ import { getLifecycleManager } from "../lifecycle-manager";
 import { applyCommandToRecipe } from "../recipe-runner";
 import { settleWorkspaceCommand } from "../workspace-command-settle";
 import {
-  emitAgentRemoved,
   emitAgentStatus,
   emitMessageClaimReleased,
   emitOrchestratorStatus,
@@ -64,7 +49,6 @@ import { runIdleAgentDigest } from "../services/idle-agent-digest";
 import { runShutdownWatchdog } from "../services/shutdown-watchdog";
 import { AGENT_HANG_WATCHDOG_MAINTENANCE_JOB } from "../services/agent-hang-watchdog-job";
 import { reapIdleStewards } from "../services/steward-reaper";
-import { emitWorkFinalityResult } from "../services/work-finality";
 import { escalateExpiredSpawnCommands } from "../services/spawn-loss-escalation";
 import { reapUnregisteredSpawnCommands } from "../services/spawn-registration-watchdog";
 import { resumeRateLimitedAgents } from "../services/rate-limit-resume";
@@ -182,7 +166,7 @@ export async function runStuckBusyWatchdog(input: { now?: number; signal?: Abort
       },
     });
 
-    const messages = routeNotification({
+    const messages = await routeNotification({
       type: "agent.stuck_busy",
       scope: { agentId: agent.id, parent: agent.spawnedBy, teamId: agent.teamId },
       message: {
@@ -300,9 +284,9 @@ export const definitions: MaintenanceJobDefinition[] = [
     runOnStart: true,
     // #1819 — SYNCHRONOUS, one atomic statement, no fan-out: nothing to split, so the pre-flight
     // check is the whole of its cooperation.
-    handler(context) {
+    async handler(context) {
       if (context.signal?.aborted) return { releasedDriveLeaseAgentIds: [], aborted: true };
-      return { releasedDriveLeaseAgentIds: releaseExpiredDriveLeases() };
+      return { releasedDriveLeaseAgentIds: (await releaseExpiredDriveLeasesAsync()) };
     },
   },
   {
@@ -339,7 +323,7 @@ export const definitions: MaintenanceJobDefinition[] = [
         // message belongs in the queue (this is hooked to agent availability,
         // not pool-slot reclaim).
         if (!agent || isAgentUnavailable(agent)) continue;
-        const flushed = flushQueuedDirectMessages(agent!.id);
+        const flushed = await flushQueuedDirectMessages(agent!.id);
         if (flushed.length > 0) reswept[target] = flushed.length;
       }
       return { resweptTargets: reswept };
@@ -359,19 +343,14 @@ export const definitions: MaintenanceJobDefinition[] = [
       // by heartbeat and deleted on removal, so a missing/offline agent record for the
       // target means it will never poll this command again. A merely-busy (online) host is
       // left alone, preserving the #930/#953 "don't sweep before the host can claim it".
-      const commands = expireCommands(Date.now(), {
-        isTargetGone: (command) => {
-          if (!command.target || command.target === "system") return false;
-          const agent = getAgent(command.target);
-          return !agent || agent.status === "offline";
-        },
-      });
+      const commands = await expireCommandsAsync();
+
       // #1819 — deliberately NOT checkpointed: `expireCommands` already moved these rows terminal
       // and nothing re-runs this settle, so bailing would strand their workspace and recipe forever.
       // Bounded by one expiry sweep.
       for (const command of commands) {
-        notifySelfResumeNoOp(command);
-        settleContextResetCarry(command);
+        await notifySelfResumeNoOp(command);
+        (await settleContextResetCarry(command));
         noteCompactionResult(command);
         await settleWorkspaceCommand(command, { onMergeSettled: scheduleRepoMerge });
         await applyCommandToRecipe(command);
@@ -412,7 +391,7 @@ export const definitions: MaintenanceJobDefinition[] = [
       // #1819 — deliberately NOT checkpointed, as in command-expiration: the attachment is already
       // expired, and this is the one place a parent is told its agent died.
       for (const id of expired.agentIds) {
-        getLifecycleManager().onAgentDisappeared(id);
+        (await getLifecycleManager().onAgentDisappeared(id));
         // #636/#659 — only after the stale reconnect grace expires do we turn a bare
         // heartbeat gap into a terminal parent notification. A live runner that
         // reconnects before this point heals from `stale` without a false exit.
@@ -420,7 +399,7 @@ export const definitions: MaintenanceJobDefinition[] = [
         // when a succeeded agent.shutdown command explains the silence, instead of the
         // heartbeat-lost/transport-loss verdict a graceful reap used to receive.
         const { reason, diagnosis } = classifyBusExpirationDeath(id);
-        notifyAgentOffline(id, reason, diagnosis);
+        (await notifyAgentOffline(id, reason, diagnosis));
       }
       for (const task of expired.orphanedTasks) emitTaskChanged(task, "task.updated");
       return {
@@ -464,26 +443,7 @@ export const definitions: MaintenanceJobDefinition[] = [
       return { reapedAgentIds };
     },
   },
-  {
-    id: "offline-agent-prune",
-    critical: false,
-    title: "Offline agent prune",
-    description: "Delete long-offline provider agents and release their claims.",
-    intervalMs: REAP_INTERVAL_MS,
-    runOnStart: false,
-    async handler(context) {
-      if (context.signal?.aborted) return { prunedAgentIds: [], aborted: true };
-      // #1084 crash-final-loss fix-forward — defense-in-depth: `notifyAgentOffline` is the primary
-      // flush point when an agent first goes offline, so this is normally a no-op by the time an
-      // agent reaches prune horizon; the async facade snapshots its held queue and performs the DB
-      // finalization in the SAME worker transaction before deleting the agent row.
-      const prunedAgentIds = await pruneOfflineAgentsAsync(OFFLINE_PRUNE_MS, {
-        onFinality: emitWorkFinalityResult,
-      });
-      for (const id of prunedAgentIds) emitAgentRemoved(id);
-      return { prunedAgentIds };
-    },
-  },
+  OFFLINE_AGENT_PRUNE_JOB,
   AGENT_IDENTITY_ARCHIVE_PRUNE_JOB,
   {
     id: "orphan-token-revoker",
@@ -498,8 +458,9 @@ export const definitions: MaintenanceJobDefinition[] = [
       // event loop was stalled, a live replacement's heartbeats may still be unprocessed, so the
       // grace widens by the observed lag instead of misreading the backlog as post-mortem silence.
       // #1895 (S3.2 L10, review finding 3) — routed through the async gateway facade.
-      const revokedOrphanTokens = await revokeOrphanedRuntimeTokensAsync(STALE_TTL_MS, eventLoopLagGraceMs());
-      return { revokedOrphanTokens };
+      // #3776 — paged short writes, bounded by the abort signal and a time budget.
+      const { revoked, complete } = await revokeOrphanedRuntimeTokensAsync(STALE_TTL_MS, eventLoopLagGraceMs(), { signal: context.signal });
+      return { revokedOrphanTokens: revoked, moreWork: !complete };
     },
   },
   {
@@ -509,12 +470,12 @@ export const definitions: MaintenanceJobDefinition[] = [
     description: "Stop spawned sessions whose relay agent is offline/gone but whose process the orchestrator still reports running, after a grace period for self-heal.",
     intervalMs: ORPHAN_REAPER_INTERVAL_MS,
     runOnStart: false,
-    handler: (context) => reapOrphanedSessions({ signal: context.signal }),
+    handler: async (context) => (await reapOrphanedSessions({ signal: context.signal })),
   },
-  { id: "transient-agent-reaper", critical: false, title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed; also flushes a persistent agent's held work-final report once it has settled idle (#1084 HIGH fix-forward).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => reapTransientAgents({ signal: context.signal }) },
+  { id: "transient-agent-reaper", critical: false, title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed; also flushes a persistent agent's held work-final report once it has settled idle (#1084 HIGH fix-forward).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: async (context) => (await reapTransientAgents({ signal: context.signal })) },
   { id: "idle-agent-digest", critical: false, title: "Idle-agent digest", description: "One digest per coordinator window listing idle children; stops provably safe ones when the team allows it (#3428). Off until the idle-digest setting enables it.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => runIdleAgentDigest({ signal: context.signal }) },
   { id: "shutdown-watchdog", critical: false, title: "Shutdown kill-switch watchdog", description: "#1746 — recover a NON-policy agent stuck `shutting-down` past its grace whose shutdown command never executed (owning host offline / command expired): re-issue exactly one forceful shutdown, then surface a terminal `kill-failed` if it is still alive a further grace window later. Policy/steward agents are excluded (the lifecycle-manager owns their retry ladder, #1761).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => runShutdownWatchdog({ signal: context.signal }) },
-  { id: "steward-reaper", critical: false, title: "Orphan steward reaper", description: "Independent backstop (#1761) for the lifecycle reconcile: reap ORPHANED stewards ONLY — a live steward agent no managed policy state references (left when a respawn overwrote the state's agentId, or handed off after the lifecycle exhausted its forceful stop budget). Reaps an orphan once idle past the dashboard keepalive grace, or immediately when steward spawning is disabled. Never reaps a busy/provider-blocked/mid-land (claim-holding) steward, reaps each orphan at most once (idempotent), and never touches a still-tracked steward — that one is the lifecycle reconcile's (its keepalive idle-timeout, policy-disabled stop, and forceful stopping-wedge escalation), so the two can never race.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => reapIdleStewards({ signal: context.signal }) },
+  { id: "steward-reaper", critical: false, title: "Orphan steward reaper", description: "Independent backstop (#1761) for the lifecycle reconcile: reap ORPHANED stewards ONLY — a live steward agent no managed policy state references (left when a respawn overwrote the state's agentId, or handed off after the lifecycle exhausted its forceful stop budget). Reaps an orphan once idle past the dashboard keepalive grace, or immediately when steward spawning is disabled. Never reaps a busy/provider-blocked/mid-land (claim-holding) steward, reaps each orphan at most once (idempotent), and never touches a still-tracked steward — that one is the lifecycle reconcile's (its keepalive idle-timeout, policy-disabled stop, and forceful stopping-wedge escalation), so the two can never race.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: async (context) => (await reapIdleStewards({ signal: context.signal })) },
   {
     id: "restart-epoch-reconcile",
     // CRITICAL: if this stops running, an epoch opened by a deploy is never released — every
@@ -555,7 +516,7 @@ export const definitions: MaintenanceJobDefinition[] = [
     description: "Auto-resume agents held on a provider usage/rate-limit stall once their reset window has passed (#286).",
     intervalMs: REAP_INTERVAL_MS,
     runOnStart: false,
-    handler: (context) => resumeRateLimitedAgents({ signal: context.signal }),
+    handler: async (context) => (await resumeRateLimitedAgents({ signal: context.signal })),
   },
   { id: "provider-still-held-nudge", critical: false, title: "Provider still-held nudge", description: "Notify a child's parent while it remains provider-blocked, and page the operator once a top-level provider block passes its threshold.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: async (context) => await sendStillHeldNudges(Date.now(), context.signal) },
   {
@@ -647,6 +608,7 @@ export const definitions: MaintenanceJobDefinition[] = [
         prunedMessages: result.prunedMessages,
         prunedActivityEvents: result.prunedActivityEvents,
         prunedSpawnCommands: result.prunedSpawnCommands,
+        slimmedSpawnProfiles: result.slimmedSpawnProfiles,
         offThread: result.offThread,
         moreWork: result.moreWork, // #1758 — bounded-progress signal; drives backlogIntervalMs reschedule.
       };
@@ -661,6 +623,7 @@ export const definitions: MaintenanceJobDefinition[] = [
   SNAPSHOT_LEDGER_PRUNE_JOB,
   PROVIDER_SECRET_SCRUB_JOB,
   WEB_TOOL_CALL_PRUNE_JOB,
+  HISTORY_RETENTION_JOB,
   CONTINUATION_ARCHIVE_SEARCH_SYNC_JOB,
   SHIPPING_STATS_ROLLUP_JOB,
   REPLY_OBLIGATION_PARITY_JOB,
@@ -761,9 +724,9 @@ export const definitions: MaintenanceJobDefinition[] = [
     intervalMs: REAP_INTERVAL_MS,
     runOnStart: true,
     // #1819 — SYNCHRONOUS, one atomic sweep: pre-flight check only, nothing to split.
-    handler(context) {
+    async handler(context) {
       if (context.signal?.aborted) return { prunedSharedWorkspaceIds: [], aborted: true };
-      const prunedSharedWorkspaceIds = pruneOrphanedSharedWorkspaces();
+      const prunedSharedWorkspaceIds = (await pruneOrphanedSharedWorkspacesAsync());
       return { prunedSharedWorkspaceIds };
     },
   },
@@ -829,7 +792,7 @@ export const definitions: MaintenanceJobDefinition[] = [
     intervalMs: WORKSPACE_IDLE_REFRESH_INTERVAL_MS,
     runOnStart: false,
     deadlineMs: 60 * 1000,
-    handler: (context) => refreshIdleWorkspaces({ signal: context.signal }),
+    handler: async (context) => (await refreshIdleWorkspaces({ signal: context.signal })),
   },
   {
     id: "workspace-gc",
@@ -843,14 +806,14 @@ export const definitions: MaintenanceJobDefinition[] = [
     // otherwise never run it at all — the backstop effectively off exactly when it's needed.
     runOnStart: true,
     deadlineMs: 60 * 1000,
-    handler: (context) => workspaceGC({ signal: context.signal }),
+    handler: async (context) => (await workspaceGC({ signal: context.signal })),
   },
   { id: "agent-exit-cleanup", critical: false, title: "Agent exit cleanup",
     description: "Save exited owners on origin, then remove their workspace, branch, outbox and provider home on each host (#2749).",
     intervalMs: 15 * 60_000, runOnStart: true, logRuns: true, deadlineMs: 60_000,
     handler: (context) => requestAgentExitCleanupSweep(context.signal),
   },
-  IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB,
+  IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB, PLAN_READY_NOTICE_JOB,
   REVIEW_REF_RECONCILE_JOB,
   CHECKOUT_REFRESH_JOB,
   ARTIFACT_CHECKOUT_GC_JOB,

package/src/maintenance/long-running-work.ts

@@ -5,6 +5,8 @@ import { routeNotification } from "../notification-router";
 import { DAY_MS } from "./constants";
 import type { MaintenanceJobDefinition } from "./types";
 
+export { PLAN_READY_NOTICE_JOB } from "./plan-ready-notice-job";
+
 export const PENDING_ISSUE_CLOSE_JOB: MaintenanceJobDefinition = {
   id: "pending-issue-closes", title: "Issues awaiting Task completion", critical: false,
   description: "Close landed delivery issues when their Tasks reach doneWhen and pass validation.",
@@ -23,7 +25,7 @@ export async function remindIdleEpics(signal: AbortSignal, now = Date.now()): Pr
     if (signal.aborted) break;
     const lines = [...renderTeamIdleEpics(team.id), ...teamFailures.map(failure => `- ${failure}`)];
     if (!lines.length) continue;
-    routeNotification({
+    await routeNotification({
       type: "team.idle", scope: { teamId: team.id }, recipients: [`coordinator:${team.id}`],
       message: { kind: "system", from: "system", subject: "Open idle epics and Plans", body: lines.join("\n"), replyExpected: false,
         payload: { kind: "team.idle-epics", teamId: team.id }, idempotencyKey: `idle-epics:${team.id}:${Math.floor(now / DAY_MS)}` },