@humwick/server 0.130.2 → 0.130.3

package/src/maintenance/plan-ready-notice-job.ts

@@ -0,0 +1,71 @@
+import { createHash } from "node:crypto";
+import { relayEnv } from "@humwick/sdk/env";
+import { getGateway } from "../db/gateway";
+import { getPlan, listPlansByTeam } from "../db/plans";
+import type { DueReadyNodes, ReadyPlanScan } from "../db/plan-ready-notices";
+import { listActiveTeams } from "../db/teams";
+import { routeNotification } from "../notification-router";
+import { compactText, liveFrontier } from "../services/plan-wake-digest";
+
+const DEFAULT_READY_NOTICE_MINUTES = 30;
+
+/** #3660 — minutes a ready node may sit unstarted before the coordinator hears about it. */
+export function planReadyNoticeMs(): number {
+  const minutes = Number(relayEnv(process.env, "AGENT_RELAY_PLAN_READY_NOTICE_MINUTES"));
+  return (Number.isFinite(minutes) && minutes > 0 ? minutes : DEFAULT_READY_NOTICE_MINUTES) * 60_000;
+}
+
+/** Ready (all prerequisites landed), still pending after the Task projection, and nobody owns it. */
+function scanTeamPlans(teamId: string, signal: AbortSignal): Array<ReadyPlanScan & { title: string }> {
+  const scans: Array<ReadyPlanScan & { title: string }> = [];
+  let cursor: string | undefined;
+  do {
+    const page = listPlansByTeam(teamId, { limit: 100, cursor });
+    for (const summary of page.plans) {
+      if (signal.aborted) return scans;
+      if (!summary.nodeCount || summary.completion === "done") continue;
+      const stored = getPlan(summary.id);
+      if (!stored) continue;
+      const { frontier } = liveFrontier(stored);
+      scans.push({ planId: stored.id, title: stored.title ?? "Untitled", nodeIds: frontier.ready.filter(node => !node.ownerRef).map(node => node.id) });
+    }
+    cursor = page.nextCursor;
+  } while (cursor);
+  return scans;
+}
+
+export async function noticeUnstartedReadyNodes(signal: AbortSignal, now = Date.now()): Promise<Record<string, unknown>> {
+  const scans: Array<ReadyPlanScan & { title: string; teamId: string }> = [];
+  for (const team of listActiveTeams()) {
+    if (signal.aborted) return { notices: 0 };
+    if (team.coordinatorId) scans.push(...scanTeamPlans(team.id, signal).map(scan => ({ ...scan, teamId: team.id })));
+  }
+  if (signal.aborted) return { notices: 0 };
+  const due = await getGateway().write("plans.readyNotices.sync", { scans: scans.map(({ planId, nodeIds }) => ({ planId, nodeIds })), now, dueMs: planReadyNoticeMs() }) as unknown as DueReadyNodes[];
+  const byPlan = new Map(scans.map(scan => [scan.planId, scan]));
+  let notices = 0;
+  for (const item of due) {
+    const scan = byPlan.get(item.planId)!;
+    if (signal.aborted) break;
+    const nodeIds = item.nodes.map(node => node.nodeId);
+    const minutes = Math.round(planReadyNoticeMs() / 60_000);
+    const key = createHash("sha1").update(item.nodes.map(node => `${node.nodeId}@${node.readySince}`).join(",")).digest("hex").slice(0, 12);
+    await routeNotification({
+      type: "team.idle", scope: { teamId: scan.teamId }, recipients: [`coordinator:${scan.teamId}`],
+      message: { kind: "system", from: "system", subject: "Plan nodes ready but not started", replyExpected: false,
+        body: `${compactText(scan.title, 56)} (${scan.planId}): ready for ${minutes}+ min, not started: ${nodeIds.slice(0, 8).map(id => compactText(id, 32)).join(",")}${nodeIds.length > 8 ? `,+${nodeIds.length - 8}` : ""}. Start them or mark them blocked (relay_plan_get mode=resume).`,
+        payload: { kind: "plan.ready-unstarted", teamId: scan.teamId, planId: scan.planId, nodeIds },
+        idempotencyKey: `plan-ready:${scan.planId}:${key}` },
+    });
+    await getGateway().write("plans.readyNotices.mark", { planId: item.planId, nodeIds, now });
+    notices++;
+  }
+  return { notices };
+}
+
+export const PLAN_READY_NOTICE_JOB = {
+  id: "plan-ready-notice", title: "Ready Plan nodes not started", critical: false,
+  description: "Tell each coordinator once when a Plan node has been ready with nothing started for the notice window.",
+  intervalMs: 5 * 60_000, deadlineMs: 60_000, runOnStart: false,
+  handler: (context: { signal: AbortSignal }) => noticeUnstartedReadyNodes(context.signal),
+} satisfies import("./types").MaintenanceJobDefinition;

package/src/maintenance/scheduler.ts

@@ -1,17 +1,17 @@
+import { rescheduleRunOnStartJobsAsync, claimMaintenanceRunAsync, finishMaintenanceRunAsync, heartbeatMaintenanceRunAsync, ensureMaintenanceJobsAsync, reclaimDeadLeasesAsync } from "../db/maintenance-job-runs-async";
 import { timePostBindPhase } from "../post-bind-startup";
 import { randomUUID } from "node:crypto";
 import { errMessage, isRecord } from "@humwick/sdk";
 import { classifyPidLiveness } from "@humwick/sdk/process-utils";
 import { CRITICAL_MAINTENANCE_PROGRESS_STALE_MS, createActivityEventAsync, getDb, MAINTENANCE_RUN_AUTORESET_GUARD_SQL, recordDrainConvoy, writeTransaction } from "../db";
-import { claimMaintenanceRunAsync, finishMaintenanceRunAsync, heartbeatMaintenanceRunAsync } from "../db/maintenance-job-runs-async";
-import { ensureMaintenanceJobsAsync, reclaimDeadLeasesAsync, rescheduleRunOnStartJobsAsync } from "../db/maintenance-job-runs-async";
-import { reclaimDeadLeases, rescheduleRunOnStartJobs, upsertMaintenanceJobs } from "../db/maintenance-job-reconcile";
+import { reclaimDeadLeases, upsertMaintenanceJobs } from "../db/maintenance-job-reconcile";
 import type { DeadLease, MaintenanceJobSpec } from "../db/maintenance-job-reconcile";
 import type { MaintenanceRunOutcome } from "../db/maintenance-job-runs";
 import { withAbortDeadline } from "../async-deadline";
 import { MAINTENANCE_BREAKER_LAG_MS } from "../config";
 import { findPauseByJobId, getMaintenanceJobPause, notPausedSql } from "../db/maintenance-pauses";
 import { setLagBreakerObserver } from "../event-loop-lag";
+import { framed, runInFrame } from "../hub-metrics";
 import { resetRateLimitResumeTrackerForTests } from "../services/rate-limit-resume";
 import { resetTransientTrackerForTests } from "../services/transient-agent-reaper";
 import { isEphemeralMode } from "../execution-mode";
@@ -79,7 +79,7 @@ const MAINTENANCE_PROGRESS_HEARTBEAT_MS = 60_000;
 //   3. HARD BLAST-RADIUS CAP: at most ONE new auto-pause per lag sample (operator Q4). If several
 //      qualify, pause the single largest in-gap block and warn about the rest. One freeze may never
 //      pause a fleet.
-setLagBreakerObserver((_gapStartAt, gapEndAt, lagMs) => {
+setLagBreakerObserver(async (_gapStartAt, gapEndAt, lagMs) => {
   const reason = `blocked the event loop ~${Math.round(lagMs)}ms (measured block)`;
   // #1725 P1 F-A — the ACTUATOR uses the TIGHTENED window `[gapEndAt - lagMs, gapEndAt]`, the actual
   // freeze interval, NOT the sampler's diagnostic window `[_gapStartAt, gapEndAt]`. The sampler builds
@@ -143,7 +143,7 @@ setLagBreakerObserver((_gapStartAt, gapEndAt, lagMs) => {
   const blamed = strikable.reduce((worst, entry) => (entry.overlapMs > worst.overlapMs ? entry : worst));
   if (isRunActive(blamed.run.runId)) abortRun(blamed.run.runId, new MaintenanceBreakerError(`aborted by circuit breaker: ${reason}`));
   const critical = definitions.find((job) => job.id === blamed.run.jobId)?.critical ?? false;
-  recordBreakerStrike({ runId: blamed.run.runId, jobId: blamed.run.jobId, critical, kind: "lag", reason });
+  (await recordBreakerStrike({ runId: blamed.run.runId, jobId: blamed.run.jobId, critical, kind: "lag", reason }));
   const others = strikable.filter((entry) => entry !== blamed);
   if (others.length > 0) {
     console.warn(`[relay] maintenance circuit breaker: ${strikable.length} runs each blocked >=${Math.round(shareThresholdMs)}ms of this ${Math.round(lagMs)}ms freeze — struck ${blamed.run.jobId} (in-freeze block ${Math.round(blamed.overlapMs)}ms), one-per-sample cap; not struck: ${others.map((entry) => `${entry.run.jobId} (${Math.round(entry.overlapMs)}ms)`).join(", ")}`);
@@ -166,8 +166,8 @@ const runBlockTotalMs = (run: MaintenanceRunRecord): number => Math.round((run.b
 /** Exposes the internal `runJob` directly, bypassing the `definitions` lookup `runMaintenanceJobNow`
  *  does, so tests can drive the run-registry/timeout/crash paths with a synthetic handler.
  *  Production code never calls this name. */
-export const runMaintenanceJobForTests = (definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}) =>
-  runJob(definition, options);
+export const runMaintenanceJobForTests = async (definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}) =>
+  (await runJob(definition, options));
 
 export function resetOrphanTrackerForTests(): void {
   resetOrphanSessionTrackerForTests();
@@ -185,7 +185,7 @@ export function startMaintenanceScheduler(): void {
   // never bubble to an unhandledRejection that exits the relay process.
   if (timer) return;
   timer = setInterval(
-    () => void runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`)),
+    framed("timer maintenance", () => runDueMaintenanceJobs().catch((error) => console.warn(`maintenance tick failed: ${errMessage(error)}`))),
     SCHEDULER_TICK_MS,
   );
 }
@@ -221,8 +221,8 @@ async function bootMaintenanceJobs(): Promise<void> {
  * doing it from the periodic reconcile would make run-on-start jobs run every
  * scheduler tick.
  */
-export function scheduleRunOnStartJobs(now: number = Date.now()): void {
-  rescheduleRunOnStartJobs(now);
+export async function scheduleRunOnStartJobs(now: number = Date.now()): Promise<void> {
+  (await rescheduleRunOnStartJobsAsync(now));
 }
 
 function readMaintenanceJobs(): MaintenanceJob[] {
@@ -255,7 +255,7 @@ export async function runMaintenanceJobNow(id: string, options: { overridePause?
   await ensureMaintenanceJobsAsync(jobSpecs());
   const definition = definitions.find((job) => job.id === id);
   if (!definition) throw new Error("maintenance job not found");
-  return runJob(definition, { force: true, overridePause: options.overridePause });
+  return (await runJob(definition, { force: true, overridePause: options.overridePause }));
 }
 
 export async function runLegacyMaintenanceReaper(): Promise<Record<string, unknown>> {
@@ -450,7 +450,12 @@ async function persistRunOutcome(outcome: MaintenanceRunOutcome): Promise<boolea
   }
 }
 
-async function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
+/** #3798 — each run is its own hub-metrics frame, so a job's main-thread CPU is named per job. */
+function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean } = {}): Promise<MaintenanceJobRun> {
+  return runInFrame({ callsite: `job ${definition.id}` }, () => executeJob(definition, options));
+}
+
+async function executeJob(definition: MaintenanceJobDefinition, options: { force?: boolean; overridePause?: boolean }): Promise<MaintenanceJobRun> {
   const startedAt = Date.now();
   // One execution boundary covers ticks, forced runs and already-queued definitions.
   if (isEphemeralMode() && definition.instanceSafe !== true) {
@@ -584,7 +589,7 @@ async function runJob(definition: MaintenanceJobDefinition, options: { force?: b
     if (persisted && (isTimeout || breakerAborted)) {
       // A breaker abort is a lag event that the lag observer has usually already struck (deduped by
       // runId here); a timeout is its own consecutive-overrun signal.
-      recordBreakerStrike({ runId: run.runId, jobId: definition.id, critical: definition.critical, kind: isTimeout ? "timeout" : "lag", reason: isTimeout ? `timed out after ${deadlineMs}ms` : message });
+      (await recordBreakerStrike({ runId: run.runId, jobId: definition.id, critical: definition.critical, kind: isTimeout ? "timeout" : "lag", reason: isTimeout ? `timed out after ${deadlineMs}ms` : message }));
     }
     if (definition.logRuns) console.warn(`[relay] maintenance job ${definition.id} ${status} in ${durationMs}ms: ${message}`);
     return { id: definition.id, status, startedAt, finishedAt, durationMs, error: message, ...(result ? { result } : {}) };

package/src/maintenance/stalled-delivery-pager.ts

@@ -69,7 +69,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
     state = { ...state, failStreak: 0, okStreak: state.stalled ? state.okStreak + 1 : 0 };
     if (state.stalled && state.okStreak >= STALL_PAGE_CONFIRM_CHECKS) {
       if (state.alerted) {
-        const result = notifyWorkspaceHumanEscalation({
+        const result = (await notifyWorkspaceHumanEscalation({
           subject: "Message delivery recovered",
           body: `Messages are being delivered to live agents again. The stall began ${new Date(state.since).toISOString()}.`,
           priority: "default",
@@ -77,7 +77,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
           source: "stalled-delivery-pager",
           payload: { kind: "message-delivery.recovered", since: state.since, recoveredAt: now },
           idempotencyKey: `stalled-delivery:${state.since}:recovered`,
-        });
+        }));
         recovered = result.accepted;
         if (!result.accepted) console.warn(`[relay] #3014 message delivery recovered — operator NOT paged: ${result.reason}`);
       }
@@ -89,7 +89,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
     if (state.stalled && !state.alerted) {
       // Persist the episode first so a crash after the send re-sends under the same key.
       await persist();
-      const result = notifyWorkspaceHumanEscalation({
+      const result = (await notifyWorkspaceHumanEscalation({
         subject: "Message delivery stalled",
         body: [
           `${check.subjects?.length ?? 0} live, idle agents have had mail unattempted for over 2 minutes and no turn was delivered to any agent in 5 minutes.`,
@@ -102,7 +102,7 @@ export async function runStalledDeliveryPager(now: number = Date.now()): Promise
         source: "stalled-delivery-pager",
         payload: { kind: "message-delivery.stalled", since: state.since, count: check.count, subjects: check.subjects },
         idempotencyKey: `stalled-delivery:${state.since}:alert`,
-      });
+      }));
       alerted = result.accepted;
       if (!result.accepted) console.warn(`[relay] #3014 message delivery stalled — operator NOT paged: ${result.reason}`);
       state = { ...state, alerted: true };

package/src/maintenance/teams.ts

@@ -1,18 +1,7 @@
-import {
-  clearTeamZeroMembersSince,
-  countLiveTeamMembers,
-  createActivityEventAsync,
-  agentsWithPendingOwnRelayRun,
-  getAgent,
-  getCrashReport,
-  hasTerminalEvent,
-  getTeamBusyState,
-  getTeamRoster,
-  listActiveTeams,
-  setTeamZeroMembersSince,
-  TEAM_AUTO_REAP_SUMMARY_PREFIX,
-} from "../db";
-import { clearTeamIdleEntry, coordinatorsWithPendingTimers, getTeamIdleEntries, setTeamIdleEntry } from "../db/team-idle";
+import { setTeamIdleEntryAsync, clearTeamIdleEntryAsync } from "../db/runtime-tail-async";
+import { setTeamZeroMembersSinceAsync, clearTeamZeroMembersSinceAsync } from "../db/teams-async";
+import { countLiveTeamMembers, createActivityEventAsync, agentsWithPendingOwnRelayRun, getAgent, getCrashReport, hasTerminalEvent, getTeamBusyState, getTeamRoster, listActiveTeams, TEAM_AUTO_REAP_SUMMARY_PREFIX } from "../db";
+import { coordinatorsWithPendingTimers, getTeamIdleEntries } from "../db/team-idle";
 import { openWorkflowRunHolders } from "../db/workflow-task-read-ops";
 import { STALE_TTL_MS, stewardFallbackTarget } from "../config";
 import { agentProviderState } from "../provider-state";
@@ -48,13 +37,13 @@ function memberIsAlive(member: AgentCard, freshCutoff: number): boolean {
  * after its land, a graceful shutdown). This watchdog must surface ONLY a genuine silent death:
  * an agent that went quiet with NO terminal event of any kind.
  */
-function detectSilentMembers(
+async function detectSilentMembers(
   teamId: string,
   coordinatorId: string | undefined,
   now: number,
   thresholdMs: number,
   out: { pingedMembers: string[]; watchedMembers: number },
-): void {
+): Promise<void> {
   const freshCutoff = now - thresholdMs;
   const watch = teamMemberWatchTracker.get(teamId) ?? new Map<string, TeamMemberWatch>();
   teamMemberWatchTracker.set(teamId, watch);
@@ -85,7 +74,7 @@ function detectSilentMembers(
     if (getCrashReport(memberId) || hasTerminalEvent(memberId, entry.epoch)) continue;
     if (!coordinatorId) continue;
     const minutes = Math.floor(gapMs / 60_000);
-    notifyTarget(
+    await notifyTarget(
       "team.member_silent",
       { teamId, agentId: memberId },
       coordinatorId,
@@ -179,7 +168,7 @@ async function detectGoneCoordinator(
     watch.escalated = true; // the committed no-target audit is the complete no-silent-dissolve action
     return;
   }
-  const sent = notifyTarget(
+  const sent = await notifyTarget(
     "team.coordinator_gone",
     { teamId, agentId: coordinatorId, parent: resolveCoordinatorAgentId(coordinatorId) ?? undefined },
     escalationTarget,
@@ -222,10 +211,10 @@ function teamIdlePayload(teamId: string, coordinatorId: string, thresholdMinutes
   };
 }
 
-function notifyTeamIdle(teamId: string, coordinatorId: string, thresholdMinutes: number, idleForMs: number, firstNoBusyAt: number): string | null {
+async function notifyTeamIdle(teamId: string, coordinatorId: string, thresholdMinutes: number, idleForMs: number, firstNoBusyAt: number): Promise<string | null> {
   const roundedIdleMinutes = Math.max(thresholdMinutes, Math.floor(idleForMs / 60_000));
   const payload = teamIdlePayload(teamId, coordinatorId, thresholdMinutes, idleForMs, firstNoBusyAt);
-  return notifyTarget(
+  return await notifyTarget(
     "team.idle",
     { teamId, agentId: coordinatorId },
     coordinatorId,
@@ -270,7 +259,7 @@ export async function runTeamIdleWatchdog(input: { now?: number; signal?: AbortS
     seen.add(team.id);
     // #634 — runs every cycle, independent of the team-level idle gate below: a single silent
     // member must surface even while OTHER members keep the team "busy".
-    detectSilentMembers(team.id, team.coordinatorId, now, thresholdMs, memberSilence);
+    await detectSilentMembers(team.id, team.coordinatorId, now, thresholdMs, memberSilence);
     // #412/#468 — coordinator-gone escalation, also every cycle and independent of the idle gate: a
     // dead coordinator must surface even if the team isn't (yet) idle, and the idle-ping path can't
     // reach it (it would just ping the void).
@@ -299,7 +288,7 @@ export async function runTeamIdleWatchdog(input: { now?: number; signal?: AbortS
       ? busy.busyMemberIds.filter((id) => id !== team.coordinatorId)
       : busy.busyMemberIds;
     if (busyMemberIds.length > 0) {
-      if (existingEntry) clearTeamIdleEntry(team.id);
+      if (existingEntry) (await clearTeamIdleEntryAsync(team.id));
       teamIdleTracker.delete(team.id);
       busyTeamIds.push(team.id);
       continue;
@@ -311,7 +300,7 @@ export async function runTeamIdleWatchdog(input: { now?: number; signal?: AbortS
     }
 
     const entry = existingEntry ?? { firstNoBusyAt: now, pinged: false };
-    if (!existingEntry) setTeamIdleEntry(team.id, entry);
+    if (!existingEntry) (await setTeamIdleEntryAsync(team.id, entry));
     teamIdleTracker.set(team.id, entry);
     const idleForMs = now - entry.firstNoBusyAt;
     if (idleForMs < thresholdMs) {
@@ -328,10 +317,10 @@ export async function runTeamIdleWatchdog(input: { now?: number; signal?: AbortS
       continue;
     }
 
-    const sent = notifyTeamIdle(team.id, team.coordinatorId, config.thresholdMinutes, idleForMs, entry.firstNoBusyAt);
+    const sent = await notifyTeamIdle(team.id, team.coordinatorId, config.thresholdMinutes, idleForMs, entry.firstNoBusyAt);
     if (!sent) continue;
     entry.pinged = true;
-    setTeamIdleEntry(team.id, entry);
+    (await setTeamIdleEntryAsync(team.id, entry));
     pingedTeamIds.push(team.id);
   }
 
@@ -341,7 +330,7 @@ export async function runTeamIdleWatchdog(input: { now?: number; signal?: AbortS
   // coordinator that was already told. Skip the sweep instead; the next full pass reclaims them.
   if (!signal?.aborted) {
     for (const key of teamIdleTracker.keys()) if (!seen.has(key)) {
-      clearTeamIdleEntry(key);
+      (await clearTeamIdleEntryAsync(key));
       teamIdleTracker.delete(key);
     }
     for (const key of teamMemberWatchTracker.keys()) if (!seen.has(key)) teamMemberWatchTracker.delete(key);
@@ -385,7 +374,7 @@ export async function reapZeroMemberTeams(context: { signal?: AbortSignal } = {}
     const live = countLiveTeamMembers(team.id, cutoff);
     if (live > 0) {
       if (team.zeroMembersSince !== undefined) {
-        clearTeamZeroMembersSince(team.id);
+        (await clearTeamZeroMembersSinceAsync(team.id));
         clearedTeamIds.push(team.id);
       }
       continue;
@@ -393,7 +382,7 @@ export async function reapZeroMemberTeams(context: { signal?: AbortSignal } = {}
 
     zeroMemberTeams++;
     if (team.zeroMembersSince === undefined) {
-      setTeamZeroMembersSince(team.id, now);
+      (await setTeamZeroMembersSinceAsync(team.id, now));
       trackedTeamIds.push(team.id);
       continue;
     }
@@ -401,7 +390,7 @@ export async function reapZeroMemberTeams(context: { signal?: AbortSignal } = {}
 
     const outcomeSummary = `${TEAM_AUTO_REAP_SUMMARY_PREFIX} zero live members for ${grace}ms`;
     // Leave outcomeStatus null (unknown): an abandoned team is NOT a failed one.
-    const dissolved = dissolveTeamById(team.id, { outcomeSummary });
+    const dissolved = await dissolveTeamById(team.id, { outcomeSummary });
     await createActivityEventAsync({
       clientId: `team-reap-${team.id}-${now}`,
       kind: "state",

package/src/maintenance/workspaces.ts

@@ -1,19 +1,9 @@
+import { releaseExpiredMergeLeasesAsync, reelectRepoStewardAsync } from "../db/runtime-tail-async";
+import { deleteWorkspaceAsync } from "../db/lifecycle-tail-async";
+import { createCommandAsync } from "../commands-write";
 import { releaseExpiredStewardHandbacks } from "./steward-handback";
-import { createCommand } from "../commands-db";
-import {
-  agentsWithRunningCommandTimers,
-  createActivityEventAsync,
-  deleteWorkspace,
-  getAgent,
-  getRepoSteward,
-  getWorkspace,
-  listOrchestrators,
-  listWorkspaces,
-  patchWorkspaceMetadata,
-  reelectRepoSteward,
-  releaseExpiredMergeLeases,
-  updateWorkspaceStatusAsync,
-  setWorkspaceBranchAsync, patchWorkspaceMetadataAsync} from "../db";
+
+import { agentsWithRunningCommandTimers, createActivityEventAsync, getAgent, getRepoSteward, getWorkspace, listOrchestrators, listWorkspaces, patchWorkspaceMetadata, updateWorkspaceStatusAsync, setWorkspaceBranchAsync, patchWorkspaceMetadataAsync } from "../db";
 import type { WorkspaceMergePreview, WorkspaceRecord, WorkspaceStatus } from "../types";
 import { reconcileLandedWorkspace } from "../branch-landed";
 import { workspaceStewardEligible } from "../steward-queue";
@@ -187,7 +177,7 @@ export async function scanWorkspaceConflicts(context: { signal?: AbortSignal } =
       // a landing steward would spawn, find nothing to resolve, and re-strand the row. Route it to
       // the branch agent's COORDINATOR (owner/fallback) via the custody path instead — they own the
       // decision to rebase or abandon. (The event-driven auto-merge hook escalates the same way.)
-      const targets = routeWorkspaceCustodyEscalation({
+      const targets = await routeWorkspaceCustodyEscalation({
         workspace: ws,
         subject: "Workspace land keeps reverting (moved base)",
         body: `Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} keeps getting its tip reverted as ${p.baseRef ?? "base"} moves under it (#1242); it can't auto-land. Please coordinate a rebase or clean-up.`,
@@ -360,7 +350,7 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
   const now = Date.now();
   const cutoff = now - WORKSPACE_RETENTION_MS;
   const reviewCutoff = now - WORKSPACE_REVIEW_TTL_MS;
-  const releasedLeaseRepos = releaseExpiredMergeLeases(now);
+  const releasedLeaseRepos = (await releaseExpiredMergeLeasesAsync(now));
 
   const all = listWorkspaces();
   // #1726 — reclaim expired-terminal rows in a BOUNDED batch. GC can be paused for long stretches
@@ -376,7 +366,7 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
   const terminalIds: string[] = [];
   for (const ws of terminalBatch) {
     if (signal?.aborted) break;
-    deleteWorkspace(ws.id);
+    (await deleteWorkspaceAsync(ws.id));
     terminalIds.push(ws.id);
   }
   // Rows the abort left undeleted are still expired and terminal, so they stay in the backlog the
@@ -394,7 +384,7 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
   for (const ws of all) {
     if (signal?.aborted) break;
     if (!STRANDABLE_STATUSES.has(ws.status) || ws.mode !== "isolated" || !ws.worktreePath) continue;
-    reelectRepoSteward(ws.repoRoot);
+    (await reelectRepoStewardAsync(ws.repoRoot));
     const fresh = getWorkspace(ws.id);
     if (!fresh || !STRANDABLE_STATUSES.has(fresh.status)) continue;
     const ownerAlive = isOwnerAlive(fresh.ownerAgentId);
@@ -518,7 +508,7 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
     // resolve. (A stranded row that is genuinely in `conflict` status is already handed to a steward
     // by the conflict-scan path; the GC backstop's job here is custody, not conflict resolution.)
     const strandedHours = Math.round((now - strandedAt) / (60 * 60 * 1000));
-    const targets = routeWorkspaceCustodyEscalation({
+    const targets = await routeWorkspaceCustodyEscalation({
       workspace: fresh,
       subject: "Stranded workspace needs an owner",
       body: `Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} is ${fresh.status} and has been stranded for ${strandedHours}h. Please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"} or clean up the worktree.`,
@@ -530,13 +520,13 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
       // fallback configured). Marking this "escalated" would silently swallow the strand and
       // repeat every sweep forever. Instead leave it UNescalated (don't stamp escalatedAt, so
       // the next sweep retries once recipients exist) and raise a louder operator-visible alarm.
-      raiseOperatorAlarm({
+      (await raiseOperatorAlarm({
         clientId: `workspace-gc-unescalatable-${fresh.id}:${strandedAt}:${Math.floor(now / escalationMs)}`,
         source: "workspace-gc",
         subject: "Stranded workspace has no one to escalate to",
         detail: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} has been ${fresh.status} for ${strandedHours}h but has no owner, coordinator, or fallback configured to escalate to. Configure AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK or reclaim the worktree manually.`,
         metadata: { maintenanceJobId: "workspace-gc", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt },
-      });
+      }));
       continue;
     }
     escalationTargets.push(...targets);
@@ -589,7 +579,7 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
     // operator alarm when it resolves to nobody) instead of `stewardAgentId ?? fallback`.
     // Most strandable workspaces have no steward, so the old notice silently dropped while
     // the state change still happened.
-    const targets = routeWorkspaceCustodyEscalationOrAlarm({
+    const targets = await routeWorkspaceCustodyEscalationOrAlarm({
       source: "workspace-gc",
       workspace: fresh,
       type: "workspace.auto_abandoned",
@@ -620,12 +610,12 @@ export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promi
     if (signal?.aborted) break;
     const orch = orchestrators.find((candidate) => workspacePathWithinBase(repoRoot, candidate.baseDir));
     if (!orch?.agentId) continue;
-    const command = createCommand({
+    const command = (await createCommandAsync({
       type: "workspace.prune",
       source: "system",
       target: orch.agentId,
       params: { repoRoot, requestedBy: "workspace-gc", requestedAt: now },
-    });
+    }));
     emitCommand(command);
     pruneCommands.push(command.id);
   }
@@ -696,7 +686,7 @@ export async function refreshIdleWorkspaces(context: { signal?: AbortSignal } =
     if (gitAhead !== undefined && gitAhead > 0) { skipped.push(`${ws.id}:ahead`); continue; }
     if (gitDirtyCount !== undefined && gitDirtyCount > 0) { skipped.push(`${ws.id}:dirty`); continue; }
 
-    const command = createCommand({
+    const command = (await createCommandAsync({
       type: "workspace.idle-refresh",
       source: "system",
       target: orch.agentId,
@@ -711,7 +701,7 @@ export async function refreshIdleWorkspaces(context: { signal?: AbortSignal } =
         requestedBy: "workspace-idle-refresh",
         requestedAt: now,
       },
-    });
+    }));
     emitCommand(command);
     dispatched.push(ws.id);
   }

package/src/managed-policy-escalation.ts

@@ -28,11 +28,11 @@ function managedPolicyEscalationRecipients(policy: SpawnPolicy, state: ManagedAg
 // agent — often the repo steward that lands branches — is down. Route those to the same
 // recipients as the offline branch, and when there is no reachable recipient raise an
 // operator-visible alarm instead of silently dropping the signal.
-export function escalateManagedSpawnFailure(
+export async function escalateManagedSpawnFailure(
   policy: SpawnPolicy,
   state: ManagedAgentState,
   opts: { phase: "failed" | "register-timeout" | "spawn-backoff"; error: string },
-): void {
+): Promise<void> {
   const recipients = managedPolicyEscalationRecipients(policy, state);
   const queuedMessages = queuedPolicyMessageCount(policy.name);
   const subject = opts.phase === "failed"
@@ -55,17 +55,17 @@ export function escalateManagedSpawnFailure(
     error: opts.error,
   };
   if (!recipients.length) {
-    raiseOperatorAlarm({
+    (await raiseOperatorAlarm({
       clientId: `managed-policy-alarm-${opts.phase}-${policy.name}-${policy.orchestratorId}`,
       source: "lifecycle-manager",
       subject,
       detail: `${body}\nNo owner, coordinator, or fallback is reachable to page.`,
       view: "managed-agents",
       metadata: payload,
-    });
+    }));
     return;
   }
-  routeNotification({
+  await routeNotification({
     type: "agent.spawn_failed",
     scope: { target: `policy:${policy.name}` },
     recipients,
@@ -85,14 +85,14 @@ export function escalateManagedSpawnFailure(
   });
 }
 
-export function escalateOfflineOrchestrator(policy: SpawnPolicy, state: ManagedAgentState, orch: Orchestrator | null | undefined): void {
+export async function escalateOfflineOrchestrator(policy: SpawnPolicy, state: ManagedAgentState, orch: Orchestrator | null | undefined): Promise<void> {
   if (state.consecutiveFailures < ORCHESTRATOR_OFFLINE_ESCALATION_FAILURES) return;
   const recipients = managedPolicyEscalationRecipients(policy, state);
   if (!recipients.length) return;
   const host = orch?.hostname ? `${orch.hostname} (${policy.orchestratorId})` : policy.orchestratorId;
   const queuedMessages = queuedPolicyMessageCount(policy.name);
   const nextRetry = state.backoffUntil ? new Date(state.backoffUntil).toISOString() : "not scheduled";
-  routeNotification({
+  await routeNotification({
     type: "agent.spawn_failed",
     scope: { target: `policy:${policy.name}` },
     recipients,

package/src/managed-policy.ts

@@ -1,3 +1,4 @@
+import { runnerRuntimeTokenEnvAsync, type RunnerLaunchAuthority, type RunnerRuntimeTokenInput } from "./runtime-tokens";
 import { getManifest } from "@humwick/providers";
 import { getAgentProfile, resolveAgentProfileKey } from "./config-store";
 import { effectiveProviderCatalogListAsync, ValidationError, withResolvedProvisioning, withResolvedProvisioningAsync } from "./db";
@@ -7,7 +8,6 @@ import { getAgent } from "./db/agents";
 import { getOrchestratorAsync } from "./db/orchestrators-async";
 import { composeProfileSpawnInstructions, composeProfileSpawnInstructionsAsync } from "./project-instructions";
 import { getSelfEffortCeiling } from "./db/self-effort-permissions";
-import { runnerRuntimeTokenEnv, runnerRuntimeTokenEnvAsync, type RunnerLaunchAuthority, type RunnerRuntimeTokenInput } from "./runtime-tokens";
 import { recordedLaunchMode, resolveHostLaunchMode } from "./launch-mode";
 import { buildSpawnCommand, generateSpawnAgentId, resolveSpawnModelParams, resolveSpawnModelParamsAsync } from "./spawn-command";
 import { applyDefaultSharedMcp } from "./shared-mcp";
@@ -65,7 +65,7 @@ function systemTagsForManagedPolicy(policy: SpawnPolicy): string[] {
   return tags;
 }
 
-export function buildManagedSpawnParams(policy: SpawnPolicy, requestId: string, ctx: ManagedSpawnContext): Record<string, unknown> {
+export async function buildManagedSpawnParams(policy: SpawnPolicy, requestId: string, ctx: ManagedSpawnContext): Promise<Record<string, unknown>> {
   if (policy.profile) policy = { ...policy, profile: resolveAgentProfileKey(policy.profile) ?? policy.profile };
   assertManagedProviderEnabled(managedPolicyProviderDisabledMessage(policy));
   const agentId = ctx.identityAgentId ?? generateSpawnAgentId();
@@ -97,7 +97,7 @@ export function buildManagedSpawnParams(policy: SpawnPolicy, requestId: string,
 
   return buildManagedCommand({ policy, requestId, ctx, agentId, prompt, agentProfile, orchestrator, extraCwdPrefixes, composed,
     modelParams: resolveSpawnModelParams(policy.provider, policy.model, policy.effort),
-    env: runnerRuntimeTokenEnv(managedTokenInput(policy, requestId, ctx, agentId, extraCwdPrefixes)),
+    env: (await runnerRuntimeTokenEnvAsync(managedTokenInput(policy, requestId, ctx, agentId, extraCwdPrefixes))),
   });
 }
 

package/src/managed-provider-gate.ts

@@ -1,9 +1,10 @@
+import { upsertManagedAgentStateAsync, updateManagedAgentStateAsync } from "./db/lifecycle-tail-async";
 // #2656 — "off means nothing new starts on it": a managed policy whose provider is disabled (operator
 // switch or all models disabled) must not launch. The lifecycle manager parks the policy with a
 // visible reason instead of spawning — no backoff churn, no crash loop — and its normal reconcile
 // tick resumes the policy on its own once the provider is re-enabled.
 import { findProviderCatalog, providerDisabledMessage, type ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
-import { getManagedAgentState, updateManagedAgentState, upsertManagedAgentState } from "./config-store";
+import { getManagedAgentState } from "./config-store";
 import { effectiveProviderCatalogList } from "./provider-catalog-store";
 import type { ManagedAgentState, SpawnPolicy } from "./types";
 
@@ -17,11 +18,11 @@ export function managedPolicyProviderDisabledMessage(policy: Pick<SpawnPolicy, "
  * Record the skipped launch on the policy's state (`stopped` + lastError) so the dashboard shows why.
  * Returns the new state only when it changed, so a caller emits once rather than every tick.
  */
-export function parkManagedPolicyForDisabledProvider(policy: SpawnPolicy, message: string): ManagedAgentState | undefined {
+export async function parkManagedPolicyForDisabledProvider(policy: SpawnPolicy, message: string): Promise<ManagedAgentState | undefined> {
   const state = getManagedAgentState(policy.name);
   if (state?.status === "stopped" && state.lastError === message) return undefined;
   const next = state
-    ? updateManagedAgentState(policy.name, { status: "stopped", spawnRequestId: undefined, lastError: message })
-    : upsertManagedAgentState({ policyName: policy.name, status: "stopped", orchestratorId: policy.orchestratorId, provider: policy.provider, restartCount: 0, consecutiveFailures: 0, lastError: message });
+    ? (await updateManagedAgentStateAsync(policy.name, { status: "stopped", spawnRequestId: undefined, lastError: message }))
+    : (await upsertManagedAgentStateAsync({ policyName: policy.name, status: "stopped", orchestratorId: policy.orchestratorId, provider: policy.provider, restartCount: 0, consecutiveFailures: 0, lastError: message }));
   return next ?? undefined;
 }

package/src/managed-spawn-reservation.ts

@@ -1,4 +1,5 @@
-import { getManagedAgentState, getSpawnPolicy, upsertManagedAgentState } from "./config-store";
+import { upsertManagedAgentStateAsync } from "./db/lifecycle-tail-async";
+import { getManagedAgentState, getSpawnPolicy } from "./config-store";
 import type { ManagedAgentState, SpawnPolicy } from "./types";
 
 /** True only while an async preparation still owns the policy's starting reservation. */
@@ -15,9 +16,9 @@ export function managedSpawnReservationCurrent(policy: SpawnPolicy, policySnapsh
 }
 
 /** Return a preparation's own reservation to its pre-spawn state when the host drains. */
-export function restoreDrainedSpawnReservation(policy: SpawnPolicy, requestId: string, prior: ManagedAgentState | null): void {
+export async function restoreDrainedSpawnReservation(policy: SpawnPolicy, requestId: string, prior: ManagedAgentState | null): Promise<void> {
   const current = getManagedAgentState(policy.name);
   if (current?.status !== "starting" || current.spawnRequestId !== requestId) return;
-  upsertManagedAgentState(prior ?? { policyName: policy.name, status: "stopped", orchestratorId: policy.orchestratorId,
-    provider: policy.provider, spawnRequestId: undefined, restartCount: 0, consecutiveFailures: 0 });
+  (await upsertManagedAgentStateAsync(prior ?? { policyName: policy.name, status: "stopped", orchestratorId: policy.orchestratorId,
+    provider: policy.provider, spawnRequestId: undefined, restartCount: 0, consecutiveFailures: 0 }));
 }