@humwick/server 0.130.2 → 0.130.3

package/src/maintenance/breaker.ts

@@ -1,3 +1,5 @@
+import { pauseMaintenanceJobAsync } from "../db/runtime-tail-async";
+import { createActivityEventAsync } from "../db/activity-async";
 // #1724 stage 1 — the maintenance circuit breaker's bookkeeping and trip actions.
 //
 // The design (§5.4, §8.5) asks for a breaker that, when a job repeatedly starves the event loop (or
@@ -15,7 +17,7 @@
 
 import { errMessage } from "@humwick/sdk";
 import { MAINTENANCE_BREAKER_MAX_STRIKES } from "../config";
-import { AUTO_BREAKER_PAUSE_HOLDER, createActivityEvent, pauseMaintenanceJob } from "../db";
+import { AUTO_BREAKER_PAUSE_HOLDER } from "../db";
 
 // Two strike counters per job, because the two failure signals want different reset semantics — and
 // getting this wrong silently defeats the breaker:
@@ -81,7 +83,7 @@ export interface BreakerStrikeResult {
  * and, on reaching MAINTENANCE_BREAKER_MAX_STRIKES, trips: a non-critical job is durably auto-paused;
  * a critical one is escalated LOUDLY but never paused. Idempotent per runId.
  */
-export function recordBreakerStrike(input: BreakerStrikeInput): BreakerStrikeResult {
+export async function recordBreakerStrike(input: BreakerStrikeInput): Promise<BreakerStrikeResult> {
   if (!markStruck(input.runId)) {
     return { struck: false, strikes: breakerStrikeCount(input.jobId), tripped: false };
   }
@@ -92,7 +94,7 @@ export function recordBreakerStrike(input: BreakerStrikeInput): BreakerStrikeRes
   if (count < MAINTENANCE_BREAKER_MAX_STRIKES) {
     return { struck: true, strikes: breakerStrikeCount(input.jobId), tripped: false };
   }
-  const action = trip(input.jobId, input.critical, input.reason);
+  const action = (await trip(input.jobId, input.critical, input.reason));
   return { struck: true, strikes: 0, tripped: true, action };
 }
 
@@ -119,34 +121,34 @@ export function isRunStruck(runId: string): boolean {
   return struckRunIds.has(runId);
 }
 
-function trip(jobId: string, critical: boolean, reason: string): "paused" | "escalated" | "pause-conflict" {
+async function trip(jobId: string, critical: boolean, reason: string): Promise<"paused" | "escalated" | "pause-conflict"> {
   // Reset BOTH counters first: after a trip the job is either paused (won't run) or escalated; either
   // way the next strike should start a fresh count rather than instantly re-trip on every later run.
   timeoutStrikes.delete(jobId);
   lagStrikes.delete(jobId);
   if (critical) {
-    escalateCritical(jobId, reason);
+    (await escalateCritical(jobId, reason));
     return "escalated";
   }
-  return autoPause(jobId, reason);
+  return (await autoPause(jobId, reason));
 }
 
-function autoPause(jobId: string, reason: string): "paused" | "pause-conflict" {
+async function autoPause(jobId: string, reason: string): Promise<"paused" | "pause-conflict"> {
   const pauseReason = `auto-paused by circuit breaker: ${reason}`;
   console.warn(`[relay] maintenance circuit breaker TRIPPED — durably auto-pausing ${jobId}: ${reason}`);
   try {
     // Indefinite on purpose: a job that was starving the loop must NOT quietly resume itself on a
     // timer. A human resumes it once it is fixed — and the `maintenance-auto-paused` health arm is
     // what makes sure a human sees it (this pause is otherwise invisible to the critical-only arm).
-    const result = pauseMaintenanceJob({ jobId, holder: AUTO_BREAKER_PAUSE_HOLDER, reason: pauseReason, indefinite: true });
-    auditBreakerEvent({
+    const result = (await pauseMaintenanceJobAsync({ jobId, holder: AUTO_BREAKER_PAUSE_HOLDER, reason: pauseReason, indefinite: true }));
+    (await auditBreakerEvent({
       jobId,
       clientId: `server-maintenance-breaker-pause-${jobId}-${result.pause.createdAt}`,
       title: `Maintenance circuit breaker auto-paused ${jobId}`,
       body: `${jobId} was auto-paused by the event-loop-lag circuit breaker after ${MAINTENANCE_BREAKER_MAX_STRIKES} strikes — ${reason}. It will stay paused until a human resumes it.`,
       icon: "ti-alert-octagon",
       metadata: { breaker: true, pauseId: result.pause.id, alreadyHeld: !result.ok },
-    });
+    }));
     return result.ok ? "paused" : "pause-conflict";
   } catch (error) {
     console.warn(`[relay] maintenance circuit breaker could not pause ${jobId}: ${errMessage(error)}`);
@@ -154,27 +156,27 @@ function autoPause(jobId: string, reason: string): "paused" | "pause-conflict" {
   }
 }
 
-function escalateCritical(jobId: string, reason: string): void {
+async function escalateCritical(jobId: string, reason: string): Promise<void> {
   // §8.5 — the one thing the breaker must never do is silence a critical job the way a bare
   // `enabled=0` silenced cd-watcher for nine hours. So a critical job is escalated, never paused: it
   // KEEPS running (and keeps being reported by criticalMaintenanceJobsCheck if it is genuinely
   // failing), and a loud, attributed event tells a human to intervene deliberately.
   console.warn(`[relay] maintenance circuit breaker: CRITICAL job ${jobId} repeatedly misbehaving (${reason}) — NOT auto-paused (see #1645/§8.5), escalating for a human`);
-  auditBreakerEvent({
+  (await auditBreakerEvent({
     jobId,
     clientId: `server-maintenance-breaker-escalate-${jobId}-${Date.now()}`,
     title: `Maintenance circuit breaker: critical job ${jobId} needs a human`,
     body: `${jobId} repeatedly starved the event loop or overran its deadline (${reason}). Because it is CRITICAL, the breaker did NOT auto-pause it — a machine silencing a critical job is the #1645 blackout. A human must decide what to do.`,
     icon: "ti-alert-triangle",
     metadata: { breaker: true, critical: true, escalated: true },
-  });
+  }));
 }
 
-function auditBreakerEvent(input: { jobId: string; clientId: string; title: string; body: string; icon: string; metadata: Record<string, unknown> }): void {
+async function auditBreakerEvent(input: { jobId: string; clientId: string; title: string; body: string; icon: string; metadata: Record<string, unknown> }): Promise<void> {
   try {
     // #1912 S3.2: sync-retained — the lag sampler invokes recordBreakerStrike synchronously and
     // cannot await without redesigning the event-loop attribution callback contract.
-    createActivityEvent({
+    (await createActivityEventAsync({
       clientId: input.clientId,
       kind: "operator",
       title: input.title,
@@ -183,7 +185,7 @@ function auditBreakerEvent(input: { jobId: string; clientId: string; title: stri
       icon: input.icon,
       view: "activity",
       metadata: { source: "server", maintenanceJobId: input.jobId, ...input.metadata },
-    });
+    }));
   } catch (error) {
     // The pause/escalation itself already happened; losing the timeline entry must not throw back
     // into the scheduler's tick.

package/src/maintenance/checkout-refresh.ts

@@ -9,6 +9,7 @@ import { isRecord } from "@humwick/sdk";
 import { createHash } from "node:crypto";
 import { sendMessageServiceAsync } from "../services/send-message";
 import { checkoutRefreshEnabled } from "../config";
+import { runGit } from "../services/git-process";
 
 async function notice(projectId: string, projectSlug: string, host: string, rootPath: string, reason: string): Promise<void> {
   const key = createHash("sha256").update(JSON.stringify([projectId, host, rootPath, reason])).digest("hex");
@@ -28,6 +29,7 @@ export function reconcileCheckoutRefresh(signal?: AbortSignal): Promise<Record<s
 async function reconcile(signal?: AbortSignal): Promise<Record<string, unknown>> {
   const dispatched: string[] = [];
   const skipped: string[] = [];
+  const remoteTips = new Map<string, Promise<string | undefined>>();
   for (const project of await listProjectsAsync()) {
     if (signal?.aborted) break;
     const roots = await rootsForProjectAsync(project.id);
@@ -55,8 +57,23 @@ async function reconcile(signal?: AbortSignal): Promise<Record<string, unknown>>
       if (prior && ["pending", "accepted", "running"].includes(prior.status)) continue;
       // The host reported once that it cannot refresh; re-asking every tick only repeats that (#3629).
       if (health === "unsupported") { skipped.push(`${project.slug}: unsupported host ${hostId} for ${root.rootPath}`); continue; }
+      // Completed receipts remain valid until the remote default branch moves (#3813).
+      if (prior?.status === "succeeded" && prior.result?.targetSha && ["current", "refreshed", "disabled"].includes(prior.result.status)) {
+        let tip = remoteTips.get(root.remoteUrl!);
+        if (!tip) {
+          tip = runGit(["ls-remote", "--symref", "--", root.remoteUrl!, "HEAD"], 10_000, 4096).then(result => {
+            if (result.exitCode !== 0 || result.unavailable || !/^ref: refs\/heads\/.+\tHEAD$/m.test(result.stdout)) return undefined;
+            return result.stdout.match(/^([0-9a-f]{40,64})\tHEAD$/m)?.[1];
+          });
+          remoteTips.set(root.remoteUrl!, tip);
+        }
+        const sha = await tip;
+        if (!sha) { skipped.push(`${project.slug}: remote default branch unavailable`); continue; }
+        if (sha === prior.result.targetSha) continue;
+      }
       // Leave one tick of backoff after terminal failure, including restart/lease recovery (#3691).
       if (prior && (["failed", "timed_out"].includes(prior.status) || prior.result?.status === "error") && Date.now() < (prior.updatedAt ?? 0) + 60_000) continue;
+      if (signal?.aborted) break;
       const command = await createCommandAsync({ type: "project.checkout.refresh", source: "system", target: `orchestrator-${hostId}`,
         params: { projectId: project.id, rootPath: root.rootPath, remoteUrl: root.remoteUrl, orchestratorId: hostId, enabled: true, ...(prior?.result ? { previousResult: prior.result } : {}) } });
       emitCommandEvent(command, "command.requested");
@@ -66,7 +83,7 @@ async function reconcile(signal?: AbortSignal): Promise<Record<string, unknown>>
   return { enabled: true, dispatched, skipped };
 }
 
-// A confirmed publish accelerates the same periodic reconciliation; external pushes converge on ticks.
+// Publish hints and external pushes use the same remote-tip comparison.
 subscribeRelayEvents((event) => {
   if (!event.type.startsWith("command.") || !isRecord(event.data.command)) return;
   const command = event.data.command;

package/src/maintenance/critical-job-watchdog.ts

@@ -128,18 +128,18 @@ export async function runCriticalJobWatchdog(now: number = Date.now()): Promise<
 
   const { subject, body, payload } = composeNotice(degraded, resetRuns, legacyHungRuns, heldRestartEpoch, now);
   const recipients: string[] = [];
-  const human = notifyWorkspaceHumanEscalation({
+  const human = (await notifyWorkspaceHumanEscalation({
     subject,
     body,
     payload,
     source: "maintenance-critical-job-watchdog",
     tags: ["agent-relay", "maintenance"],
-  });
+  }));
   if (human.accepted && human.channelId) recipients.push(human.channelId);
 
   const agentTarget = stewardFallbackTarget();
   if (agentTarget) {
-    const routed = notifyTarget("maintenance.degraded", {}, agentTarget, subject, body, payload);
+    const routed = await notifyTarget("maintenance.degraded", {}, agentTarget, subject, body, payload);
     if (routed) recipients.push(routed);
   }
 

package/src/maintenance/events.ts

@@ -9,7 +9,7 @@ export function emitCommand(command: Command): void {
 
 // Route a maintenance notification, swallowing failures (a stale/missing/misconfigured
 // target must never break the sweep). Returns the target on success, null otherwise.
-export function notifyTarget(
+export async function notifyTarget(
   type: NotificationType,
   scope: NotificationScope,
   target: string,
@@ -17,10 +17,10 @@ export function notifyTarget(
   body: string,
   payload: Record<string, unknown>,
   idempotencyKey?: string,
-): string | null {
+): Promise<string | null> {
   if (!target) return null;
   try {
-    routeNotification({ type, scope, recipients: [target], message: { subject, body, payload, idempotencyKey } });
+    await routeNotification({ type, scope, recipients: [target], message: { subject, body, payload, idempotencyKey } });
     return target;
   } catch {
     return null;

package/src/maintenance/history-retention-job.ts

@@ -0,0 +1,30 @@
+import { getGateway } from "../db/gateway";
+import type { MaintenanceJobDefinition } from "./types";
+
+export const HISTORY_RETENTION_JOB: MaintenanceJobDefinition = {
+  id: "history-retention", critical: false,
+  title: "Command, kernel replay and token history retention",
+  description: "Expire superseded checkout refresh commands after one hour, other non-spawn terminal commands and kernel replay receipts after 30 days, and raw usage after 90 days. Preserve references, expired replay keys and permanent budget spend (#3619, #3797).",
+  intervalMs: 60 * 60_000, backlogIntervalMs: 1_000, deadlineMs: 30_000, runOnStart: true,
+  async handler(context) {
+    const gateway = getGateway();
+    const result = { prunedCommands: 0, prunedKernelReceipts: 0, prunedTokenReports: 0, moreWork: false };
+    for (const [op, key] of [
+      ["historyRetention.commands", "prunedCommands"],
+      ["historyRetention.kernel", "prunedKernelReceipts"],
+      ["historyRetention.tokens", "prunedTokenReports"],
+    ] as const) {
+      if (context.signal.aborted) return { ...result, aborted: true, moreWork: true };
+      const start = performance.now();
+      let batch: { pruned: number; moreWork: boolean };
+      do {
+        if (context.signal.aborted) return { ...result, aborted: true, moreWork: true };
+        batch = await gateway.write(op, {}) as typeof batch;
+        result[key] += batch.pruned;
+        // Each await releases the writer for foreground work; bound the whole catch-up run (#3807).
+      } while (op === "historyRetention.commands" && batch.moreWork && performance.now() - start < 250);
+      result.moreWork ||= batch.moreWork;
+    }
+    return result;
+  },
+};

package/src/maintenance/jobs.ts

@@ -1,5 +1,9 @@
+import { OFFLINE_AGENT_PRUNE_JOB } from "./offline-agent-prune-job";
+import { pruneOrphanedSharedWorkspacesAsync, releaseExpiredDriveLeasesAsync } from "../db/runtime-tail-async";
+import { HISTORY_RETENTION_JOB } from "./history-retention-job";
 import { isHumanAgentId } from "@humwick/sdk";
-import { IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB } from "./long-running-work";
+import { REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
+import { IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB, PLAN_READY_NOTICE_JOB } from "./long-running-work";
 import { CHECKOUT_REFRESH_JOB } from "./checkout-refresh";
 import { CLAUDE_API_BUDGET_RECONCILE_JOB } from "./claude-api-budget-job";
 import { SERVICE_ACCESS_JOBS } from "./service-access-job";
@@ -15,27 +19,9 @@ import { deriveAgentBusyHealth } from "../agent-busy-health";
 import { expireStaleBusAgents } from "../bus";
 import { eventLoopLagGraceMs } from "../event-loop-lag";
 import { pruneOutboxAsync } from "../bus-outbox-async";
-import { expireCommands } from "../commands-db";
+import { expireCommandsAsync } from "../db/runtime-admin-async";
 import { getRetentionConfig } from "../retention-config-store";
-import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
-import {
-  createActivityEventAsync,
-  evaluatePoolBindingsAsync,
-  getAgent,
-  listAgents,
-  listStaleQueuedDirectTargetsAsync,
-  pruneOrphanedSharedWorkspaces,
-  pruneOfflineAgentsAsync,
-  pruneRetentionBatchAsync,
-  pruneProviderQuotaHistoryAsync,
-  reapStaleOrchestratorsAsync,
-  releaseExpiredClaimsAsync,
-  releaseExpiredDriveLeases,
-  releaseOrphanedTasksAsync,
-  reapStaleAgentsAsync,
-  revokeOrphanedRuntimeTokensAsync,
-  runDbMaintenanceAsync,
-} from "../db";
+import { createActivityEventAsync, evaluatePoolBindingsAsync, getAgent, listAgents, listStaleQueuedDirectTargetsAsync, pruneRetentionBatchAsync, pruneProviderQuotaHistoryAsync, reapStaleOrchestratorsAsync, releaseExpiredClaimsAsync, releaseOrphanedTasksAsync, reapStaleAgentsAsync, revokeOrphanedRuntimeTokensAsync, runDbMaintenanceAsync } from "../db";
 import { isAgentUnavailable } from "../db/agent-predicates";
 import { resolveTeamCoordinatorAgentId } from "../coordinator-authority";
 import { flushQueuedDirectMessages } from "../services/managed-running";
@@ -49,7 +35,6 @@ import { getLifecycleManager } from "../lifecycle-manager";
 import { applyCommandToRecipe } from "../recipe-runner";
 import { settleWorkspaceCommand } from "../workspace-command-settle";
 import {
-  emitAgentRemoved,
   emitAgentStatus,
   emitMessageClaimReleased,
   emitOrchestratorStatus,
@@ -64,7 +49,6 @@ import { runIdleAgentDigest } from "../services/idle-agent-digest";
 import { runShutdownWatchdog } from "../services/shutdown-watchdog";
 import { AGENT_HANG_WATCHDOG_MAINTENANCE_JOB } from "../services/agent-hang-watchdog-job";
 import { reapIdleStewards } from "../services/steward-reaper";
-import { emitWorkFinalityResult } from "../services/work-finality";
 import { escalateExpiredSpawnCommands } from "../services/spawn-loss-escalation";
 import { reapUnregisteredSpawnCommands } from "../services/spawn-registration-watchdog";
 import { resumeRateLimitedAgents } from "../services/rate-limit-resume";
@@ -182,7 +166,7 @@ export async function runStuckBusyWatchdog(input: { now?: number; signal?: Abort
       },
     });
 
-    const messages = routeNotification({
+    const messages = await routeNotification({
       type: "agent.stuck_busy",
       scope: { agentId: agent.id, parent: agent.spawnedBy, teamId: agent.teamId },
       message: {
@@ -300,9 +284,9 @@ export const definitions: MaintenanceJobDefinition[] = [
     runOnStart: true,
     // #1819 — SYNCHRONOUS, one atomic statement, no fan-out: nothing to split, so the pre-flight
     // check is the whole of its cooperation.
-    handler(context) {
+    async handler(context) {
       if (context.signal?.aborted) return { releasedDriveLeaseAgentIds: [], aborted: true };
-      return { releasedDriveLeaseAgentIds: releaseExpiredDriveLeases() };
+      return { releasedDriveLeaseAgentIds: (await releaseExpiredDriveLeasesAsync()) };
     },
   },
   {
@@ -339,7 +323,7 @@ export const definitions: MaintenanceJobDefinition[] = [
         // message belongs in the queue (this is hooked to agent availability,
         // not pool-slot reclaim).
         if (!agent || isAgentUnavailable(agent)) continue;
-        const flushed = flushQueuedDirectMessages(agent!.id);
+        const flushed = await flushQueuedDirectMessages(agent!.id);
         if (flushed.length > 0) reswept[target] = flushed.length;
       }
       return { resweptTargets: reswept };
@@ -359,19 +343,14 @@ export const definitions: MaintenanceJobDefinition[] = [
       // by heartbeat and deleted on removal, so a missing/offline agent record for the
       // target means it will never poll this command again. A merely-busy (online) host is
       // left alone, preserving the #930/#953 "don't sweep before the host can claim it".
-      const commands = expireCommands(Date.now(), {
-        isTargetGone: (command) => {
-          if (!command.target || command.target === "system") return false;
-          const agent = getAgent(command.target);
-          return !agent || agent.status === "offline";
-        },
-      });
+      const commands = await expireCommandsAsync();
+
       // #1819 — deliberately NOT checkpointed: `expireCommands` already moved these rows terminal
       // and nothing re-runs this settle, so bailing would strand their workspace and recipe forever.
       // Bounded by one expiry sweep.
       for (const command of commands) {
-        notifySelfResumeNoOp(command);
-        settleContextResetCarry(command);
+        await notifySelfResumeNoOp(command);
+        (await settleContextResetCarry(command));
         noteCompactionResult(command);
         await settleWorkspaceCommand(command, { onMergeSettled: scheduleRepoMerge });
         await applyCommandToRecipe(command);
@@ -412,7 +391,7 @@ export const definitions: MaintenanceJobDefinition[] = [
       // #1819 — deliberately NOT checkpointed, as in command-expiration: the attachment is already
       // expired, and this is the one place a parent is told its agent died.
       for (const id of expired.agentIds) {
-        getLifecycleManager().onAgentDisappeared(id);
+        (await getLifecycleManager().onAgentDisappeared(id));
         // #636/#659 — only after the stale reconnect grace expires do we turn a bare
         // heartbeat gap into a terminal parent notification. A live runner that
         // reconnects before this point heals from `stale` without a false exit.
@@ -420,7 +399,7 @@ export const definitions: MaintenanceJobDefinition[] = [
         // when a succeeded agent.shutdown command explains the silence, instead of the
         // heartbeat-lost/transport-loss verdict a graceful reap used to receive.
         const { reason, diagnosis } = classifyBusExpirationDeath(id);
-        notifyAgentOffline(id, reason, diagnosis);
+        (await notifyAgentOffline(id, reason, diagnosis));
       }
       for (const task of expired.orphanedTasks) emitTaskChanged(task, "task.updated");
       return {
@@ -464,26 +443,7 @@ export const definitions: MaintenanceJobDefinition[] = [
       return { reapedAgentIds };
     },
   },
-  {
-    id: "offline-agent-prune",
-    critical: false,
-    title: "Offline agent prune",
-    description: "Delete long-offline provider agents and release their claims.",
-    intervalMs: REAP_INTERVAL_MS,
-    runOnStart: false,
-    async handler(context) {
-      if (context.signal?.aborted) return { prunedAgentIds: [], aborted: true };
-      // #1084 crash-final-loss fix-forward — defense-in-depth: `notifyAgentOffline` is the primary
-      // flush point when an agent first goes offline, so this is normally a no-op by the time an
-      // agent reaches prune horizon; the async facade snapshots its held queue and performs the DB
-      // finalization in the SAME worker transaction before deleting the agent row.
-      const prunedAgentIds = await pruneOfflineAgentsAsync(OFFLINE_PRUNE_MS, {
-        onFinality: emitWorkFinalityResult,
-      });
-      for (const id of prunedAgentIds) emitAgentRemoved(id);
-      return { prunedAgentIds };
-    },
-  },
+  OFFLINE_AGENT_PRUNE_JOB,
   AGENT_IDENTITY_ARCHIVE_PRUNE_JOB,
   {
     id: "orphan-token-revoker",
@@ -510,12 +470,12 @@ export const definitions: MaintenanceJobDefinition[] = [
     description: "Stop spawned sessions whose relay agent is offline/gone but whose process the orchestrator still reports running, after a grace period for self-heal.",
     intervalMs: ORPHAN_REAPER_INTERVAL_MS,
     runOnStart: false,
-    handler: (context) => reapOrphanedSessions({ signal: context.signal }),
+    handler: async (context) => (await reapOrphanedSessions({ signal: context.signal })),
   },
-  { id: "transient-agent-reaper", critical: false, title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed; also flushes a persistent agent's held work-final report once it has settled idle (#1084 HIGH fix-forward).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => reapTransientAgents({ signal: context.signal }) },
+  { id: "transient-agent-reaper", critical: false, title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed; also flushes a persistent agent's held work-final report once it has settled idle (#1084 HIGH fix-forward).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: async (context) => (await reapTransientAgents({ signal: context.signal })) },
   { id: "idle-agent-digest", critical: false, title: "Idle-agent digest", description: "One digest per coordinator window listing idle children; stops provably safe ones when the team allows it (#3428). Off until the idle-digest setting enables it.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => runIdleAgentDigest({ signal: context.signal }) },
   { id: "shutdown-watchdog", critical: false, title: "Shutdown kill-switch watchdog", description: "#1746 — recover a NON-policy agent stuck `shutting-down` past its grace whose shutdown command never executed (owning host offline / command expired): re-issue exactly one forceful shutdown, then surface a terminal `kill-failed` if it is still alive a further grace window later. Policy/steward agents are excluded (the lifecycle-manager owns their retry ladder, #1761).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => runShutdownWatchdog({ signal: context.signal }) },
-  { id: "steward-reaper", critical: false, title: "Orphan steward reaper", description: "Independent backstop (#1761) for the lifecycle reconcile: reap ORPHANED stewards ONLY — a live steward agent no managed policy state references (left when a respawn overwrote the state's agentId, or handed off after the lifecycle exhausted its forceful stop budget). Reaps an orphan once idle past the dashboard keepalive grace, or immediately when steward spawning is disabled. Never reaps a busy/provider-blocked/mid-land (claim-holding) steward, reaps each orphan at most once (idempotent), and never touches a still-tracked steward — that one is the lifecycle reconcile's (its keepalive idle-timeout, policy-disabled stop, and forceful stopping-wedge escalation), so the two can never race.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: (context) => reapIdleStewards({ signal: context.signal }) },
+  { id: "steward-reaper", critical: false, title: "Orphan steward reaper", description: "Independent backstop (#1761) for the lifecycle reconcile: reap ORPHANED stewards ONLY — a live steward agent no managed policy state references (left when a respawn overwrote the state's agentId, or handed off after the lifecycle exhausted its forceful stop budget). Reaps an orphan once idle past the dashboard keepalive grace, or immediately when steward spawning is disabled. Never reaps a busy/provider-blocked/mid-land (claim-holding) steward, reaps each orphan at most once (idempotent), and never touches a still-tracked steward — that one is the lifecycle reconcile's (its keepalive idle-timeout, policy-disabled stop, and forceful stopping-wedge escalation), so the two can never race.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: async (context) => (await reapIdleStewards({ signal: context.signal })) },
   {
     id: "restart-epoch-reconcile",
     // CRITICAL: if this stops running, an epoch opened by a deploy is never released — every
@@ -556,7 +516,7 @@ export const definitions: MaintenanceJobDefinition[] = [
     description: "Auto-resume agents held on a provider usage/rate-limit stall once their reset window has passed (#286).",
     intervalMs: REAP_INTERVAL_MS,
     runOnStart: false,
-    handler: (context) => resumeRateLimitedAgents({ signal: context.signal }),
+    handler: async (context) => (await resumeRateLimitedAgents({ signal: context.signal })),
   },
   { id: "provider-still-held-nudge", critical: false, title: "Provider still-held nudge", description: "Notify a child's parent while it remains provider-blocked, and page the operator once a top-level provider block passes its threshold.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: async (context) => await sendStillHeldNudges(Date.now(), context.signal) },
   {
@@ -648,6 +608,7 @@ export const definitions: MaintenanceJobDefinition[] = [
         prunedMessages: result.prunedMessages,
         prunedActivityEvents: result.prunedActivityEvents,
         prunedSpawnCommands: result.prunedSpawnCommands,
+        slimmedSpawnProfiles: result.slimmedSpawnProfiles,
         offThread: result.offThread,
         moreWork: result.moreWork, // #1758 — bounded-progress signal; drives backlogIntervalMs reschedule.
       };
@@ -662,6 +623,7 @@ export const definitions: MaintenanceJobDefinition[] = [
   SNAPSHOT_LEDGER_PRUNE_JOB,
   PROVIDER_SECRET_SCRUB_JOB,
   WEB_TOOL_CALL_PRUNE_JOB,
+  HISTORY_RETENTION_JOB,
   CONTINUATION_ARCHIVE_SEARCH_SYNC_JOB,
   SHIPPING_STATS_ROLLUP_JOB,
   REPLY_OBLIGATION_PARITY_JOB,
@@ -762,9 +724,9 @@ export const definitions: MaintenanceJobDefinition[] = [
     intervalMs: REAP_INTERVAL_MS,
     runOnStart: true,
     // #1819 — SYNCHRONOUS, one atomic sweep: pre-flight check only, nothing to split.
-    handler(context) {
+    async handler(context) {
       if (context.signal?.aborted) return { prunedSharedWorkspaceIds: [], aborted: true };
-      const prunedSharedWorkspaceIds = pruneOrphanedSharedWorkspaces();
+      const prunedSharedWorkspaceIds = (await pruneOrphanedSharedWorkspacesAsync());
       return { prunedSharedWorkspaceIds };
     },
   },
@@ -830,7 +792,7 @@ export const definitions: MaintenanceJobDefinition[] = [
     intervalMs: WORKSPACE_IDLE_REFRESH_INTERVAL_MS,
     runOnStart: false,
     deadlineMs: 60 * 1000,
-    handler: (context) => refreshIdleWorkspaces({ signal: context.signal }),
+    handler: async (context) => (await refreshIdleWorkspaces({ signal: context.signal })),
   },
   {
     id: "workspace-gc",
@@ -844,14 +806,14 @@ export const definitions: MaintenanceJobDefinition[] = [
     // otherwise never run it at all — the backstop effectively off exactly when it's needed.
     runOnStart: true,
     deadlineMs: 60 * 1000,
-    handler: (context) => workspaceGC({ signal: context.signal }),
+    handler: async (context) => (await workspaceGC({ signal: context.signal })),
   },
   { id: "agent-exit-cleanup", critical: false, title: "Agent exit cleanup",
     description: "Save exited owners on origin, then remove their workspace, branch, outbox and provider home on each host (#2749).",
     intervalMs: 15 * 60_000, runOnStart: true, logRuns: true, deadlineMs: 60_000,
     handler: (context) => requestAgentExitCleanupSweep(context.signal),
   },
-  IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB,
+  IDLE_EPICS_JOB, PENDING_ISSUE_CLOSE_JOB, PLAN_READY_NOTICE_JOB,
   REVIEW_REF_RECONCILE_JOB,
   CHECKOUT_REFRESH_JOB,
   ARTIFACT_CHECKOUT_GC_JOB,

package/src/maintenance/long-running-work.ts

@@ -5,6 +5,8 @@ import { routeNotification } from "../notification-router";
 import { DAY_MS } from "./constants";
 import type { MaintenanceJobDefinition } from "./types";
 
+export { PLAN_READY_NOTICE_JOB } from "./plan-ready-notice-job";
+
 export const PENDING_ISSUE_CLOSE_JOB: MaintenanceJobDefinition = {
   id: "pending-issue-closes", title: "Issues awaiting Task completion", critical: false,
   description: "Close landed delivery issues when their Tasks reach doneWhen and pass validation.",
@@ -23,7 +25,7 @@ export async function remindIdleEpics(signal: AbortSignal, now = Date.now()): Pr
     if (signal.aborted) break;
     const lines = [...renderTeamIdleEpics(team.id), ...teamFailures.map(failure => `- ${failure}`)];
     if (!lines.length) continue;
-    routeNotification({
+    await routeNotification({
       type: "team.idle", scope: { teamId: team.id }, recipients: [`coordinator:${team.id}`],
       message: { kind: "system", from: "system", subject: "Open idle epics and Plans", body: lines.join("\n"), replyExpected: false,
         payload: { kind: "team.idle-epics", teamId: team.id }, idempotencyKey: `idle-epics:${team.id}:${Math.floor(now / DAY_MS)}` },

package/src/maintenance/offline-agent-prune-job.ts

@@ -0,0 +1,28 @@
+import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS } from "../config";
+import { pruneOfflineAgentsAsync } from "../db/agents-async";
+import { emitAgentRemoved } from "../sse";
+import { emitWorkFinalityResult } from "../services/work-finality";
+import type { MaintenanceJobDefinition } from "./types";
+
+export const OFFLINE_AGENT_PRUNE_JOB: MaintenanceJobDefinition = {
+  id: "offline-agent-prune",
+  critical: false,
+  title: "Offline agent prune",
+  description: "Delete long-offline provider agents and release their claims.",
+  intervalMs: REAP_INTERVAL_MS,
+  runOnStart: false,
+  async handler(context) {
+    if (context.signal?.aborted) return { prunedAgentIds: [], aborted: true };
+    // #1084 crash-final-loss fix-forward — defense-in-depth: `notifyAgentOffline` is the primary
+    // flush point when an agent first goes offline, so this is normally a no-op by the time an
+    // agent reaches prune horizon; the async facade snapshots its held queue and performs the DB
+    // finalization in the SAME worker transaction before deleting the agent row.
+    let archivePurgeFailures = 0;
+    const prunedAgentIds = await pruneOfflineAgentsAsync(OFFLINE_PRUNE_MS, {
+      onFinality: emitWorkFinalityResult,
+      onArchivePurgeFailure: () => { archivePurgeFailures++; },
+    });
+    for (const id of prunedAgentIds) emitAgentRemoved(id);
+    return { prunedAgentIds, archivePurgeFailures };
+  },
+};

package/src/maintenance/orphaned-sessions.ts

@@ -1,4 +1,5 @@
-import { createCommand, listCommands } from "../commands-db";
+import { createCommandAsync } from "../commands-write";
+import { listCommands } from "../commands-db";
 import { createActivityEventAsync, getAgent, listOrchestrators } from "../db";
 import type { Command, ManagedAgent } from "../types";
 import { orphanGraceMs, orphanReapCooldownMs, orphanReapEnabled } from "./constants";
@@ -44,7 +45,7 @@ export async function reapOrphanedSessions(context: { signal?: AbortSignal } = {
   let orphaned = 0;
 
   for (const orch of listOrchestrators()) {
-    // #1819 — checkpoint per orchestrator and per managed session: a reap is `createCommand` +
+    // #1819 — checkpoint per orchestrator and per managed session: a reap is `createCommandAsync` +
     // `emitCommandEventIfAbsent` + activity event + `lastReapAt` stamp, indivisible once started.
     if (signal?.aborted) break;
     if (orch.status !== "online") continue;
@@ -70,7 +71,7 @@ export async function reapOrphanedSessions(context: { signal?: AbortSignal } = {
       // logical orphan reap has one command id and one command.requested event.
       const pending = listCommands({ target: orch.agentId, type: "agent.shutdown", limit: 500 })
         .find((command) => isPendingReapForSession(command, session));
-      const command = pending ?? createCommand({
+      const command = pending ?? (await createCommandAsync({
         type: "agent.shutdown",
         source: "system",
         target: orch.agentId,
@@ -89,7 +90,7 @@ export async function reapOrphanedSessions(context: { signal?: AbortSignal } = {
           requestedAt: now,
           orchestratorId: orch.id,
         },
-      });
+      }));
       // The command is durable but not externally observable until its audit row commits. A failed
       // audit must never let an orchestrator execute an unaudited reap (#1912 S3.2 class B).
       await createActivityEventAsync({

package/src/maintenance/pauses.ts

@@ -1,3 +1,4 @@
+import { resumeMaintenanceJobAsync, pauseMaintenanceJobAsync } from "../db/runtime-tail-async";
 // #1645 — behavior layer for maintenance-job pause/resume: validation, audit, and the drift report.
 //
 // Persistence is src/db/maintenance-pauses.ts; the scheduler gate reads from there. This module is
@@ -6,14 +7,7 @@
 // #1638 pause's justification existed only in one agent's session context and one memory file.
 
 import { errMessage } from "@humwick/sdk";
-import {
-  createActivityEventAsync,
-  latestSuccessfulDeployAsync,
-  listMaintenanceJobPauses,
-  pauseMaintenanceJob,
-  resumeMaintenanceJob,
-  type MaintenanceJobPauseRecord,
-} from "../db";
+import { createActivityEventAsync, latestSuccessfulDeployAsync, listMaintenanceJobPauses, type MaintenanceJobPauseRecord } from "../db";
 import { openRestartEpochsAsync } from "../db/restart-epoch-async.ts";
 import { ValidationError } from "../db/connection.ts";
 import { definitions } from "./jobs";
@@ -61,7 +55,7 @@ export async function pauseJob(input: PauseJobRequest): Promise<PauseJobResult>
     throw new ValidationError("pass either ttlMs or indefinite, not both");
   }
 
-  const result = pauseMaintenanceJob({ ...input, reason });
+  const result = (await pauseMaintenanceJobAsync({ ...input, reason }));
   if (!result.ok) return { ok: false, conflict: true, pause: result.pause };
 
   const ownedRestartEpoch = await ownedOpenRestartEpoch(definition.id);
@@ -126,7 +120,7 @@ export interface ResumeJobResult {
 export async function resumeJobAsync(opts: { jobId?: string; id?: string; now?: number }): Promise<ResumeJobResult> {
   if (opts.jobId) requireKnownJob(opts.jobId);
   const now = opts.now ?? Date.now();
-  const released = resumeMaintenanceJob({ jobId: opts.jobId, id: opts.id });
+  const released = (await resumeMaintenanceJobAsync({ jobId: opts.jobId, id: opts.id }));
   if (!released) return { resumed: false };
   const heldForMs = Math.max(0, now - released.createdAt);
   const lapsed = released.expiresAt !== undefined && released.expiresAt <= now;