@humwick/server 0.130.1 → 0.130.3

package/src/state-adoption/cd-cutover.ts

@@ -2,15 +2,17 @@
 import { isRecord, withFileLock } from "@humwick/sdk";
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
-import { currentPendingDeployEvent, finishDeployEvent, forceFailDeployEvent, getDeployEvent, recordDeployEventExecutor, tryBeginDeployEvent } from "../db";
+import { currentPendingDeployEvent, finishDeployEvent, forceFailDeployEvent, getDeployEvent, getMaintenanceJobPause, recordDeployEventExecutor, recordDeployEventHosts, renewDeployEventLease, tryBeginDeployEvent } from "../db";
 import { localDeployExecutor, serializeDeployExecutor } from "../deploy-executor";
 import { atomicWrite, exists } from "./files";
 import { loadJournal, type AdoptionManifest } from "./index";
 import { parseLinuxPlan } from "./linux-plan";
 import { parseMacosPlan } from "./macos-plan";
+import { parseFleetPlan, type FleetPlan } from "./fleet-plan";
 
 type Direction = "adopt" | "rollback";
-export interface CutoverManifest { version: 1; migrationId: string; target: string; platform: "linux" | "macos"; handoff: unknown }
+export interface CutoverManifest { version: 1; migrationId: string; target: string; platform: "linux" | "macos"; handoff: unknown; fleet?: FleetPlan }
+export interface CutoverProgress { eventId: string; has(step: string): boolean; complete(step: string): void; steps(): string[] }
 export interface CdPause { id: string; holder: string }
 /** The existing maintenance pause API for the cd-watcher job. `pause` returns the active pause, ours or not. */
 export interface CdPauseControl { pause(holder: string, reason: string): Promise<CdPause>; current(): Promise<CdPause | null>; resume(): Promise<void> }
@@ -21,6 +23,8 @@ export interface CutoverContext {
   /** Runs a short deploy-ledger burst against the canonical DB. */
   ledger: <T>(fn: () => T) => T;
   interrupt?: (step: string) => void;
+  /** Complete an interrupted local hub handoff before using the API it supervises. */
+  recoverHandoff?: (rollback: boolean) => Promise<void>;
 }
 interface CutoverJournal { version: 1; manifest: CutoverManifest; events: Record<Direction, string>; pauseId?: string; completed: string[] }
 
@@ -28,7 +32,12 @@ export function parseCutoverManifest(value: unknown): { manifest: CutoverManifes
   if (!isRecord(value) || value.version !== 1 || typeof value.migrationId !== "string" || !/^[A-Za-z0-9._-]{1,64}$/.test(value.migrationId)
     || typeof value.target !== "string" || !/^[0-9a-f]{40}$/.test(value.target) || (value.platform !== "linux" && value.platform !== "macos")) throw new Error("Invalid CD cutover manifest");
   const adoption = (value.platform === "linux" ? parseLinuxPlan(value.handoff) : parseMacosPlan(value.handoff)).adoption;
-  return { manifest: { version: 1, migrationId: value.migrationId, target: value.target, platform: value.platform, handoff: value.handoff }, adoption };
+  const fleet = value.fleet === undefined ? undefined : parseFleetPlan(value.fleet);
+  if (fleet) {
+    const hub = fleet.hosts.find(host => host.id === fleet.hub)!;
+    if (hub.platform !== value.platform || JSON.stringify(hub.handoff) !== JSON.stringify(value.handoff)) throw new Error("Hub handoff must match the control host manifest");
+  }
+  return { manifest: { version: 1, migrationId: value.migrationId, target: value.target, platform: value.platform, handoff: value.handoff, ...(fleet ? { fleet } : {}) }, adoption };
 }
 
 async function withJournal<T>(input: unknown, create: boolean, fn: (journal: CutoverJournal, save: () => void, adoption: AdoptionManifest) => Promise<T>): Promise<T> {
@@ -55,7 +64,7 @@ const directionOf = (journal: CutoverJournal): Direction => journal.completed.so
  * Every step reconciles on retry: the pause is re-ensured, the journaled lease row is re-attached
  * and re-stamped with this process as executor, and the supervisor handoff resumes from its journal.
  */
-export async function runCutover(input: unknown, context: CutoverContext & { rollback?: boolean; handoff: (rollback: boolean) => Promise<unknown> }) {
+export async function runCutover(input: unknown, context: CutoverContext & { rollback?: boolean; handoff: (rollback: boolean, progress: CutoverProgress) => Promise<unknown> }) {
   return withJournal(input, !context.rollback, async (journal, save, adoption) => {
     const direction: Direction = context.rollback ? "rollback" : "adopt";
     const { manifest } = journal;
@@ -69,6 +78,20 @@ export async function runCutover(input: unknown, context: CutoverContext & { rol
     if (journal.completed.includes(`${direction}:complete`)) return result();
     acknowledge("prepared");
 
+    const hubPrefix = manifest.fleet && `host:${manifest.fleet.hub}:handoff`;
+    const interruptedDirection = (["rollback", "adopt"] as const).find(prior => hubPrefix
+      && journal.completed.includes(`${prior}:${hubPrefix}-started`) && !journal.completed.includes(`${prior}:${hubPrefix}`));
+    if (interruptedDirection && context.recoverHandoff) {
+      context.ledger(() => {
+        const lease = getDeployEvent(journal.events[interruptedDirection]);
+        const pause = getMaintenanceJobPause("cd-watcher");
+        if (lease?.status !== "pending" || lease.initiatedBy !== holder || !pause || pause.id !== journal.pauseId || pause.holder !== holder) throw new Error("Interrupted hub handoff no longer owns its pause and deploy lease");
+        recordDeployEventExecutor({ id: lease.id, executor: serializeDeployExecutor(localDeployExecutor(context.executorHost)) });
+        renewDeployEventLease({ id: lease.id });
+      });
+      await context.recoverHandoff(direction === "rollback");
+    }
+
     // A pause does not cancel an in-flight deploy; the lease below proves the previous executor settled.
     const pause = await context.pauses.pause(holder, `Humwick cutover ${manifest.migrationId} (${direction}) to ${manifest.target}; CD stays paused until explicit resume`);
     if (pause.holder !== holder) throw new Error(`cd-watcher is paused by ${pause.holder}; refusing the cutover`);
@@ -81,7 +104,7 @@ export async function runCutover(input: unknown, context: CutoverContext & { rol
       if (direction === "rollback" && getDeployEvent(journal.events.adopt)?.status === "pending") {
         forceFailDeployEvent({ id: journal.events.adopt, reason: `superseded by rollback of ${manifest.migrationId}` });
       }
-      const event = getDeployEvent(id) ?? tryBeginDeployEvent({ id, commitSha: manifest.target, hosts: [{ id: context.executorHost, status: "pending" }], initiatedBy: holder });
+      const event = getDeployEvent(id) ?? tryBeginDeployEvent({ id, commitSha: manifest.target, hosts: (manifest.fleet?.hosts ?? [{ id: context.executorHost }]).map(host => ({ id: host.id, status: "pending" })), initiatedBy: holder });
       if (!event) {
         const held = currentPendingDeployEvent();
         throw new Error(held ? `deploy lease ${held.id} is held by ${held.initiatedBy}; retry once it settles` : "an unreleased restart epoch blocks the deploy lease; retry once it settles");
@@ -94,14 +117,37 @@ export async function runCutover(input: unknown, context: CutoverContext & { rol
     acknowledge("lease");
 
     if (status === "pending") {
-      if (direction === "adopt" || handoffBegun(adoption, manifest.platform)) await context.handoff(direction === "rollback");
+      const progress: CutoverProgress = {
+        eventId: id,
+        has: (step) => journal.completed.includes(`${direction}:${step}`),
+        steps: () => [...journal.completed],
+        complete(step) {
+          acknowledge(step);
+          if (manifest.fleet && !context.ledger(() => recordDeployEventHosts(id, fleetStatuses()))) throw new Error(`Lost deploy lease ${id}`);
+        },
+      };
+      const heartbeat = setInterval(() => {
+        try { context.ledger(() => renewDeployEventLease({ id })); }
+        catch (error) { console.error(`Cutover lease heartbeat failed: ${error}`); }
+      }, 60_000);
+      heartbeat.unref();
+      try {
+        if (direction === "adopt" || manifest.fleet || handoffBegun(adoption, manifest.platform)) await context.handoff(direction === "rollback", progress);
+      } finally { clearInterval(heartbeat); }
       acknowledge("handoff");
       context.ledger(() => {
-        if (getDeployEvent(id)?.status === "pending") finishDeployEvent({ id, status: terminal, hosts: [{ id: context.executorHost, status: terminal }] });
+        const event = getDeployEvent(id);
+        if (event?.status === "pending") finishDeployEvent({ id, status: terminal, hosts: manifest.fleet ? fleetStatuses() : [{ id: context.executorHost, status: terminal }] });
+        else if (event?.status !== terminal) throw new Error(`Lost deploy lease ${id}; CD remains paused`);
       });
     }
     acknowledge("complete");
     return result();
+    function fleetStatuses() {
+      return manifest.fleet!.hosts.map(host => ({ id: host.id, status: direction === "rollback" && !journal.completed.includes(`adopt:host:${host.id}:deploy-started`) ? "skipped"
+        : journal.completed.includes(`${direction}:host:${host.id}:verified`) ? terminal : "pending",
+        reason: journal.completed.filter(step => step.startsWith(`${direction}:host:${host.id}:`)).map(step => step.split(":").at(-1)).join(", ") }));
+    }
   });
 }
 

package/src/state-adoption/fleet-cutover.ts

@@ -0,0 +1,35 @@
+// #3470: the fleet is one P4d cutover, with one pause/lease and durable progress per host.
+import { parseCutoverManifest, runCutover, type CutoverContext, type CutoverProgress } from "./cd-cutover";
+import { orderedFleetHosts, type FleetHost } from "./fleet-plan";
+
+export interface FleetCutoverContext extends CutoverContext {
+  rollback?: boolean;
+  deploy(host: FleetHost, sha: string, version: string | undefined, eventId: string): Promise<void>;
+  handoff(host: FleetHost, rollback: boolean, progress: CutoverProgress): Promise<void>;
+  attest(host: FleetHost, sha: string): Promise<void>;
+}
+export async function runFleetCutover(input: unknown, context: FleetCutoverContext) {
+  const { manifest } = parseCutoverManifest(input);
+  if (!manifest.fleet || manifest.fleet.hub !== context.executorHost) throw new Error("Fleet cutover must run on its pinned hub");
+  return runCutover(manifest, { ...context, handoff: async (rollback, progress) => {
+    const ordered = orderedFleetHosts(manifest.fleet!);
+    for (const host of rollback ? ordered.reverse() : ordered) {
+      const prefix = `host:${host.id}:`;
+      const touched = (phase: string) => progress.steps().includes(`adopt:${prefix}${phase}-started`);
+      if (rollback && !touched("deploy") && !touched("handoff")) continue;
+      const phase = async (name: string, action: () => Promise<void>) => {
+        if (progress.has(`${prefix}${name}`)) return;
+        progress.complete(`${prefix}${name}-started`);
+        await action();
+        progress.complete(`${prefix}${name}`);
+      };
+      const sha = rollback ? host.previous.sha : manifest.target;
+      // Restore supervisor names before the old executor resolves its legacy units on rollback.
+      if (rollback && touched("handoff")) await phase("handoff", () => context.handoff(host, true, progress));
+      if (!rollback || touched("deploy")) await phase("deploy", () => context.deploy(host, sha, rollback ? host.previous.version : undefined, progress.eventId));
+      if (!rollback) await phase("handoff", () => context.handoff(host, false, progress));
+      await context.attest(host, sha);
+      progress.complete(`${prefix}verified`);
+    }
+  } });
+}

package/src/state-adoption/fleet-plan.ts

@@ -0,0 +1,36 @@
+// #3470 P7b: host identities and rollback artifacts are pinned before acquiring the deployment lease.
+import { isRecord } from "@humwick/sdk";
+import { absolute } from "./files";
+import { parseLinuxPlan } from "./linux-plan";
+import { parseMacosPlan } from "./macos-plan";
+
+export interface FleetHost {
+  id: string;
+  hostname: string;
+  platform: "linux" | "macos";
+  handoff: unknown;
+  cli: string;
+  manifestPath: string;
+  cwd: string;
+  previous: { sha: string; version: string };
+}
+export interface FleetPlan { canary: string; hub: string; hosts: FleetHost[] }
+export function parseFleetPlan(value: unknown): FleetPlan {
+  if (!isRecord(value) || typeof value.canary !== "string" || typeof value.hub !== "string" || value.canary === value.hub
+    || !Array.isArray(value.hosts) || value.hosts.length < 2) throw new Error("Invalid fleet cutover plan");
+  const hosts = value.hosts.map((host): FleetHost => {
+    if (!isRecord(host) || typeof host.id !== "string" || !/^[A-Za-z0-9._-]+$/.test(host.id) || typeof host.hostname !== "string" || !host.hostname || !["linux", "macos"].includes(String(host.platform))
+      || !isRecord(host.previous) || typeof host.previous.sha !== "string" || !/^[0-9a-f]{40}$/.test(host.previous.sha)
+      || typeof host.previous.version !== "string" || !/^\d+\.\d+\.\d+$/.test(host.previous.version)) throw new Error("Invalid fleet host or rollback artifact");
+    (host.platform === "linux" ? parseLinuxPlan : parseMacosPlan)(host.handoff, host.hostname);
+    return { id: host.id, hostname: host.hostname, platform: host.platform as FleetHost["platform"], handoff: host.handoff,
+      cli: absolute(host.cli), manifestPath: absolute(host.manifestPath), cwd: absolute(host.cwd),
+      previous: { sha: host.previous.sha, version: host.previous.version } };
+  });
+  if (new Set(hosts.map(host => host.id)).size !== hosts.length || !hosts.some(host => host.id === value.hub)
+    || !hosts.some(host => host.id === value.canary && host.platform === "linux")) throw new Error("Fleet needs a unique hub and Linux canary");
+  return { canary: value.canary, hub: value.hub, hosts };
+}
+export function orderedFleetHosts(plan: FleetPlan): FleetHost[] {
+  return [plan.hosts.find(host => host.id === plan.canary)!, ...plan.hosts.filter(host => host.id !== plan.canary && host.id !== plan.hub), plan.hosts.find(host => host.id === plan.hub)!];
+}

package/src/state-adoption/index.ts

@@ -18,8 +18,8 @@ export interface AdoptionJournal {
   db?: Identity; env: EnvSnapshot[]; completed: string[];
 }
 
-export function parseAdoptionManifest(value: unknown): AdoptionManifest {
-  if (!isRecord(value) || value.version !== 1 || value.host !== hostname()
+export function parseAdoptionManifest(value: unknown, expectedHost = hostname()): AdoptionManifest {
+  if (!isRecord(value) || value.version !== 1 || value.host !== expectedHost
     || !Array.isArray(value.roots) || !value.roots.length || !Array.isArray(value.envFiles)) throw new Error("Invalid adoption manifest or wrong host");
   const roots = value.roots.map(pair => {
     if (!isRecord(pair)) throw new Error("Invalid directory pair");

package/src/state-adoption/linux-plan.ts

@@ -13,10 +13,10 @@ export interface LinuxPlan {
   adoption: AdoptionManifest; scope: SystemdUnitScope; unitDirectory: string;
   services: { legacy: string; canonical: string }[]; files: LinuxArtifact[]; preserve: { unit: string; mainPid: number }[];
 }
-export function parseLinuxPlan(value: unknown): LinuxPlan {
+export function parseLinuxPlan(value: unknown, expectedHost?: string): LinuxPlan {
   if (!isRecord(value) || !["user", "system"].includes(value.scope as string) || !Array.isArray(value.services) || !value.services.length
     || !Array.isArray(value.files) || !Array.isArray(value.preserve)) throw new Error("Invalid Linux handoff manifest");
-  const adoption = parseAdoptionManifest(value.adoption);
+  const adoption = parseAdoptionManifest(value.adoption, expectedHost);
   const unitDirectory = absolute(value.unitDirectory);
   const services = value.services.map(item => {
     if (!isRecord(item) || typeof item.legacy !== "string" || !/^agent-relay(?:-orchestrator)?(?:-dev)?\.service$/.test(item.legacy)

package/src/state-adoption/linux.ts

@@ -26,7 +26,7 @@ export async function adoptLinux(input: unknown, options: LinuxHandoffOptions =
   const inspect = async (unit: string): Promise<UnitState> => {
     const output = await manager("show", unit, "--property=ActiveState,UnitFileState,MainPID,InvocationID");
     const props = Object.fromEntries(output.trim().split("\n").map(line => { const i = line.indexOf("="); return [line.slice(0, i), line.slice(i + 1)]; }));
-    if (!["active", "inactive", "failed"].includes(props.ActiveState!) || !["enabled", "disabled", "static", "", "not-found"].includes(props.UnitFileState!)) throw new Error(`Unsupported service state: ${unit}`);
+    if (!["active", "inactive", "failed"].includes(props.ActiveState!) || !["enabled", "disabled", "static", "transient", "", "not-found"].includes(props.UnitFileState!)) throw new Error(`Unsupported service state: ${unit}`);
     return { active: props.ActiveState === "active", enabled: props.UnitFileState === "enabled", mainPid: Number(props.MainPID || 0), invocationId: props.InvocationID! };
   };
   return withFileLock(`${journalPath}.lock`, async () => {
@@ -34,7 +34,7 @@ export async function adoptLinux(input: unknown, options: LinuxHandoffOptions =
     const saved = prior?.linux as LinuxJournal | undefined;
     if (saved && (JSON.stringify(saved.plan) !== JSON.stringify(plan) || saved.artifacts?.length !== plan.files.length || saved.services?.length !== plan.services.length
       || saved.preserved?.length !== plan.preserve.length || !Array.isArray(saved.completed))) throw new Error("Linux journal does not match manifest");
-    if (options.rollback && !saved) throw new Error("No Linux handoff to roll back");
+    if (options.rollback && !saved) return { journal: journalPath, direction: "rollback", preserved: [], unchanged: true };
     const artifacts = checkArtifacts(plan, saved?.artifacts);
     const services = saved?.services ?? await Promise.all(plan.services.map(async service => {
       const canonical = await inspect(service.canonical);

package/src/state-adoption/macos-plan.ts

@@ -15,10 +15,10 @@ export function plistLabel(content: string, expected: string): void {
   const labels = [...content.matchAll(/<key>\s*Label\s*<\/key>\s*<string>([^<]*)<\/string>/g)];
   if (labels.length !== 1 || labels[0]![1] !== expected) throw new Error(`Plist Label must match ${expected}`);
 }
-export function parseMacosPlan(value: unknown): MacosPlan {
+export function parseMacosPlan(value: unknown, expectedHost?: string): MacosPlan {
   if (!isRecord(value) || typeof value.domain !== "string" || !/^gui\/\d+$/.test(value.domain)
     || !Array.isArray(value.services) || !value.services.length || !Array.isArray(value.files) || !Array.isArray(value.preserve)) throw new Error("Invalid macOS handoff manifest");
-  const adoption = parseAdoptionManifest(value.adoption);
+  const adoption = parseAdoptionManifest(value.adoption, expectedHost);
   const plistDirectory = absolute(value.plistDirectory);
   const services = value.services.map(item => {
     if (!isRecord(item) || typeof item.legacy !== "string"

package/src/state-adoption/macos.ts

@@ -47,7 +47,7 @@ export async function adoptMacos(input: unknown, options: MacosHandoffOptions =
     if (saved && (JSON.stringify(saved.plan) !== JSON.stringify(plan) || saved.artifacts?.length !== plan.files.length || saved.services?.length !== plan.services.length
       || !saved.services.every(state => typeof state.loaded === "boolean" && typeof state.disabled === "boolean" && typeof state.canonicalDisabled === "boolean")
       || !Array.isArray(saved.completed))) throw new Error("macOS journal does not match manifest");
-    if (options.rollback && !saved) throw new Error("No macOS handoff to roll back");
+    if (options.rollback && !saved) return { journal: journalPath, direction: "rollback", preserved: [], unchanged: true };
     const artifacts = checkMacosArtifacts(plan, saved?.artifacts);
     const flags = await disabled();
     const services = saved?.services ?? await Promise.all(plan.services.map(async service => {

package/src/steward-handoff.ts

@@ -1,6 +1,7 @@
+import { mergeAgentMetaAsync } from "./db/agents-async";
 import { stringValue } from "@humwick/sdk";
 import { getManagedAgentState, getSpawnPolicy } from "./config-store";
-import { getAgent, getWorkspace, listWorkspaces, mergeAgentMeta, patchWorkspaceMetadataAsync } from "./db";
+import { getAgent, getWorkspace, listWorkspaces, patchWorkspaceMetadataAsync } from "./db";
 import { agentGenerationKey } from "./db/agent-shutdown-in-flight";
 import { getLifecycleManager } from "./lifecycle-manager";
 import { isProvenStewardAgent, stewardHandoffReleased } from "./steward-provenance";
@@ -23,6 +24,6 @@ export async function finishWorkspaceStewardHandoff(workspace: WorkspaceRecord):
   if (!boundAgent && !boundSpawn && claimant !== agent.id) return;
   await patchWorkspaceMetadataAsync(workspace.id, { stewardClaim: null, stewardHandoffEndedAt: Date.now() });
   if (listWorkspaces().some(ws => ws.id !== workspace.id && workspaceActiveClaim(ws)?.by === agent.id)) return;
-  const command = getLifecycleManager().stopAgent(policy, true, "steward-handoff-ended");
-  if (command) mergeAgentMeta(agent.id, { stewardReleasedGeneration: agentGenerationKey(agent) }, { instanceId: agent.instanceId, epoch: agent.epoch });
+  const command = await getLifecycleManager().stopAgent(policy, true, "steward-handoff-ended");
+  if (command) (await mergeAgentMetaAsync(agent.id, { stewardReleasedGeneration: agentGenerationKey(agent) }, { instanceId: agent.instanceId, epoch: agent.epoch }));
 }

package/src/steward-policy-cleanup.ts

@@ -1,3 +1,4 @@
+import { setConfigAsync, deleteConfigAsync } from "./config-store-write";
 // #1955 — ONE-SHOT, IDEMPOTENT cleanup of the steward spawn-policy rows a live relay accumulated
 // before #1623/#1953/#1955. Pure planning here; `scripts/steward-policy-cleanup.ts` is the operator
 // CLI (DRY RUN by default). Nothing in the running relay calls this — it is deliberately manual,
@@ -21,7 +22,7 @@
 // so a second run over the applied state plans nothing. Safe to re-run.
 
 import { canonicalRepoRoot, legacyStewardPolicyNames, repoStewardPolicyName } from "./steward-identity";
-import { deleteConfig, getSpawnPolicy, listSpawnPolicies, setConfig } from "./config-store";
+import { getSpawnPolicy, listSpawnPolicies } from "./config-store";
 import { listWorkspaces } from "./db";
 import { basename } from "node:path";
 import type { SpawnPolicy } from "./types";
@@ -151,10 +152,10 @@ export interface StewardPolicyCleanupResult {
  * precondition, so a plan built against a state the live relay has since changed degrades to a skip
  * rather than clobbering the newer row.
  */
-export function applyStewardPolicyCleanup(
+export async function applyStewardPolicyCleanup(
   plan: StewardPolicyCleanupPlan,
   opts: { dryRun: boolean; updatedBy?: string },
-): StewardPolicyCleanupResult {
+): Promise<StewardPolicyCleanupResult> {
   const updatedBy = opts.updatedBy ?? "steward-policy-cleanup-1955";
   const log: string[] = [];
   let applied = 0;
@@ -184,9 +185,9 @@ export function applyStewardPolicyCleanup(
         // A stop recorded on EITHER half survives the merge — never lift an operator's stop (same
         // rule as the provision-time collapse in steward.ts).
         if (action.carriesStop && into.enabled !== false) {
-          setConfig("spawn-policy", into.name, { ...into, enabled: false }, updatedBy);
+          (await setConfigAsync("spawn-policy", into.name, { ...into, enabled: false }, updatedBy));
         }
-        deleteConfig("spawn-policy", action.policy, updatedBy);
+        (await deleteConfigAsync("spawn-policy", action.policy, updatedBy));
         applied++;
       }
       continue;
@@ -199,8 +200,8 @@ export function applyStewardPolicyCleanup(
       }
       log.push(`RENAME  ${action.policy} -> ${action.into} (cwd ${action.cwdFrom} -> ${action.cwdTo})`);
       if (!opts.dryRun) {
-        setConfig("spawn-policy", action.into, { ...current, name: action.into, ...repoDerivedFields(action.repoRoot) }, updatedBy);
-        deleteConfig("spawn-policy", action.policy, updatedBy);
+        (await setConfigAsync("spawn-policy", action.into, { ...current, name: action.into, ...repoDerivedFields(action.repoRoot) }, updatedBy));
+        (await deleteConfigAsync("spawn-policy", action.policy, updatedBy));
         applied++;
       }
       continue;
@@ -213,7 +214,7 @@ export function applyStewardPolicyCleanup(
     }
     log.push(`CWD     ${action.policy}: ${action.cwdFrom} -> ${action.cwdTo}`);
     if (!opts.dryRun) {
-      setConfig("spawn-policy", action.policy, { ...current, ...repoDerivedFields(action.repoRoot) }, updatedBy);
+      (await setConfigAsync("spawn-policy", action.policy, { ...current, ...repoDerivedFields(action.repoRoot) }, updatedBy));
       applied++;
     }
   }

package/src/steward.ts

@@ -1,6 +1,7 @@
 import { basename } from "node:path";
 import { stewardFallbackTarget } from "./config";
-import { deleteConfig, getManagedAgentState, getSpawnPolicy, getStewardConfig, setConfig } from "./config-store";
+import { getManagedAgentState, getSpawnPolicy, getStewardConfig } from "./config-store";
+import { deleteConfigAsync, setConfigAsync } from "./config-store-write";
 import { listOrchestrators } from "./db";
 import { getLifecycleManager } from "./lifecycle-manager";
 import { renderTemplate } from "./prompt-resolver";
@@ -137,7 +138,7 @@ function stewardPolicyDiffers(existing: SpawnPolicy, desired: SpawnPolicy): bool
  *   - it still has a live/transitioning managed process. Deleting the policy under a running process
  *     would orphan it: `onConfigChanged` finds no policy for a deleted key, so nothing would stop it.
  */
-function collapseLegacyStewardPolicies(canonicalName: string, repoRoot: string, owner: Orchestrator): boolean | undefined {
+async function collapseLegacyStewardPolicies(canonicalName: string, repoRoot: string, owner: Orchestrator): Promise<boolean | undefined> {
   let inheritedEnabled: boolean | undefined;
   for (const legacyName of legacyStewardPolicyNames(repoRoot, owner.id)) {
     if (legacyName === canonicalName) continue;
@@ -151,8 +152,8 @@ function collapseLegacyStewardPolicies(canonicalName: string, repoRoot: string,
       continue;
     }
     if (legacy.enabled === false) inheritedEnabled = false;
-    deleteConfig("spawn-policy", legacyName, "steward-legacy-collapse");
-    getLifecycleManager().onConfigChanged("spawn-policy", legacyName);
+    await deleteConfigAsync("spawn-policy", legacyName, "steward-legacy-collapse");
+    (await getLifecycleManager().onConfigChanged("spawn-policy", legacyName));
     console.warn(`[steward] collapsed legacy steward policy '${legacyName}' into '${canonicalName}' (#1955)`);
   }
   return inheritedEnabled;
@@ -165,7 +166,7 @@ function collapseLegacyStewardPolicies(canonicalName: string, repoRoot: string,
  * owns the repo. Idempotent — refreshes the policy only when config-derived fields
  * change, so it can be called on every sweep without churn.
  */
-export function ensureRepoSteward(repoRoot: string, owningOrchestrator?: Orchestrator): string | null {
+export async function ensureRepoStewardAsync(repoRoot: string, owningOrchestrator?: Orchestrator): Promise<string | null> {
   const config = getStewardConfig();
   if (!config.enabled) return null;
   const owner = owningOrchestrator ?? listOrchestrators().find((orch) =>
@@ -177,7 +178,7 @@ export function ensureRepoSteward(repoRoot: string, owningOrchestrator?: Orchest
   const name = repoStewardPolicyName(repoRoot, owner.id);
   // #1955 mechanism 1 — merge a legacy suffix-less twin for this (repo, host) BEFORE reading the
   // canonical row, so the pair converges on one policy instead of coexisting as two.
-  const inheritedEnabled = collapseLegacyStewardPolicies(name, repoRoot, owner);
+  const inheritedEnabled = await collapseLegacyStewardPolicies(name, repoRoot, owner);
   const baseDesired = desiredStewardPolicy(repoRoot, owner, config);
   const existing = getSpawnPolicy(name);
   // #1761 (9) — respect an operator's per-policy stop. desiredStewardPolicy hardcodes `enabled:true`;
@@ -193,7 +194,7 @@ export function ensureRepoSteward(repoRoot: string, owningOrchestrator?: Orchest
     : { ...baseDesired, ...(inheritedEnabled === undefined ? {} : { enabled: inheritedEnabled }) };
   if (existing && !stewardPolicyDiffers(existing.value, desired)) return name;
 
-  setConfig("spawn-policy", name, desired, "steward-autoprovision");
-  getLifecycleManager().onConfigChanged("spawn-policy", name);
+  await setConfigAsync("spawn-policy", name, desired, "steward-autoprovision");
+  (await getLifecycleManager().onConfigChanged("spawn-policy", name));
   return name;
 }

package/src/task-dispatch-live.ts

@@ -5,18 +5,20 @@ import { listOrchestratorsAsync } from "./db/orchestrators-async";
 import { listHostViewsAsync } from "./db/hosts-async";
 import { providerQuotaBandsFor, advertisedProvidersFrom } from "./spawn-targets";
 import { dispatchPolicyFor } from "./routing-policy-store";
-import { providerPlacementReady } from "./spawn-placement";
+import { providerPlacementReady, spawnHostCapAdmission } from "./spawn-placement";
 import { apiOverflowContextAsync, CLAUDE_API_PROVIDER } from "./claude-api-overflow";
 import { apiExpiryPaceAsync } from "./claude-api-expiry";
 import { liveTargetAdmission } from "./dispatch-availability";
 
 /** Reader-gateway shell for production callers. The ranking core remains the pure sync function. */
-export type DispatchSnapshot = DispatchContext & Required<Pick<DispatchContext, "now" | "catalog" | "bands" | "policy" | "preferences" | "advertisedProviders" | "targetAvailable">> & { probeUncertainty: Set<string> };
+export type DispatchSnapshot = DispatchContext & Required<Pick<DispatchContext, "now" | "catalog" | "bands" | "policy" | "preferences" | "advertisedProviders" | "targetAvailable">> & { probeUncertainty: Set<string>; pinnedHostAtAgentCap?: boolean };
 
 export async function captureDispatchContext(ctx: Omit<DispatchContext, "policy" | "bands" | "catalog"> = {}): Promise<DispatchSnapshot> {
   const now = ctx.now ?? Date.now();
   const [snapshot, orchestrators, hostViews] = await Promise.all([providerQuotaRoutingSnapshotAsync(now), listOrchestratorsAsync(), listHostViewsAsync()]);
-  const { targetAdmission: assess, targetUnavailableReason } = liveTargetAdmission(orchestrators, hostViews, ctx.orchestratorId, ctx.spawnCap);
+  const { targetAdmission: assess, targetUnavailableReason } = liveTargetAdmission(orchestrators, hostViews, snapshot.catalog, ctx.orchestratorId, ctx.spawnCap);
+  const pinnedHost = orchestrators.find(host => host.id === ctx.orchestratorId);
+  const pinnedView = pinnedHost && hostViews.find(view => view.id === pinnedHost.hostId);
   const probeUncertainty = new Set<string>();
   const targetAdmission = (target: Parameters<typeof assess>[0]) => {
     const result = assess(target);
@@ -42,6 +44,7 @@ export async function captureDispatchContext(ctx: Omit<DispatchContext, "policy"
     targetUnavailableReason,
     targetAdmission,
     probeUncertainty,
+    pinnedHostAtAgentCap: Boolean(pinnedView && spawnHostCapAdmission(ctx.spawnCap)(pinnedView)),
     bands,
     policy: dispatchPolicyFor(snapshot.policy),
     catalog: snapshot.catalog,

package/src/task-dispatcher.ts

@@ -1,4 +1,4 @@
-import { isOpenAiApiModel } from "@humwick/sdk";
+import { isOpenAiApiModel, ValidationError } from "@humwick/sdk";
 // #839 — the declarative task dispatcher.
 //
 // A pure function from (Task work-shape + live system state) → a concrete spawn
@@ -310,8 +310,8 @@ export function dispatchTask(shape: TaskDispatch | undefined, pin: DispatchPin =
     if (eligibility(target.provider)) throw routingResolutionError(preferences, catalog, eligibility);
     if (ctx.mustResolveAvailable && bandRank(bandFor(target.provider, bands)) > bandRank("normal")) warnings.push(`[CRITICAL] mustResolveAvailable: degraded to ${target.provider} (${bandFor(target.provider, bands)} band). Contention likely.`);
   }
-  if (!modelPinned && ctx.targetAvailable && !ctx.targetAvailable(target)) {
-    throw new Error(ctx.targetUnavailableReason?.(target) ?? `No eligible online host runs ${target.provider}/${target.model ?? "default"}${target.effort ? ` at effort ${target.effort}` : ""}.`);
+  if (ctx.targetAvailable && !ctx.targetAvailable(target)) {
+    throw new ValidationError(ctx.targetUnavailableReason?.(target) ?? `No eligible online host runs ${target.provider}/${target.model ?? "default"}${target.effort ? ` at effort ${target.effort}` : ""}.`);
   }
   if (factors.preferences) factors.preferences = { ...factors.preferences, excluded: preferenceExclusions(preferences, factors.ranking, (p) => eligibility(p) ?? (!advertised.has(p) ? "not-advertised" : undefined)) };
   // Record the FINAL provider's band + the chosen model's window for transparency.

package/src/teams-config-store.ts

@@ -1,4 +1,5 @@
-import { getConfig, setConfig } from "./config-store";
+import { getConfig } from "./config-store";
+import { setConfigAsync } from "./config-store-write";
 import { emitConfigChanged } from "./sse";
 import {
   DEFAULT_TEAMS_CONFIG,
@@ -14,9 +15,9 @@ export function getTeamsConfig(): TeamsConfig {
   return entry ? normalizeTeamsConfig(entry.value) : { ...DEFAULT_TEAMS_CONFIG };
 }
 
-export function setTeamsConfig(value: unknown, opts: { updatedBy?: string } = {}): TeamsConfig {
+export async function setTeamsConfigAsync(value: unknown, opts: { updatedBy?: string } = {}): Promise<TeamsConfig> {
   const normalized = normalizeTeamsConfig(value);
-  const entry = setConfig(TEAMS_CONFIG_NAMESPACE, TEAMS_CONFIG_KEY, normalized, opts.updatedBy);
+  const entry = await setConfigAsync(TEAMS_CONFIG_NAMESPACE, TEAMS_CONFIG_KEY, normalized, opts.updatedBy);
   emitConfigChanged(entry.namespace, entry.key, entry.version);
   return normalized;
 }

package/src/token-db-write.ts

@@ -13,6 +13,7 @@ import { getGateway } from "./db/gateway.ts";
 import type { CreateTokenInput, HandoffTokenInput, PruneExpiredTokenRecordsInput, TokenHandoff } from "./token-db.ts";
 import type { TokenRevokeReason } from "./security.ts";
 import type { TokenRecord } from "./types.ts";
+import type { LaunchFact } from "./db/launch-facts.ts";
 
 /** Mint a component token off-loop. The `jti` is generated here so it is BOTH the token's id and the
  * gateway idempotency key that satisfies the non-idempotent-op retry-safety gate (H5 §3.7.1). */
@@ -24,6 +25,15 @@ export function createTokenAsync(input: CreateTokenInput): Promise<{ token: stri
   }>;
 }
 
+/** Mint a runner token and, for a fresh launch, record its launch fact, in one writer op (#1870 S7). */
+export function createRunnerTokenAsync(input: CreateTokenInput, launch?: { agentId: string; fact: LaunchFact }): Promise<{ token: string; record: TokenRecord }> {
+  const jti = randomUUID();
+  return getGateway().write("tokens.createRunner", { input, jti, ...(launch ? { launch } : {}) }, { idempotencyKey: jti }) as unknown as Promise<{
+    token: string;
+    record: TokenRecord;
+  }>;
+}
+
 /** Hand a token off to its successor off-loop (#1245): fold the eligibility read, then mint and link the
  * successor or replay the pending one — atomic in one op (ADR §3.4). Resolves `null` when ineligible. */
 export function handoffTokenAsync(input: HandoffTokenInput): Promise<TokenHandoff | null> {

package/src/token-db.ts

@@ -324,8 +324,8 @@ export function getTokenProfile(id: string): TokenProfile | null {
   return row ? rowToProfile(row) : null;
 }
 
-export function upsertTokenProfile(input: CreateTokenProfileInput): TokenProfile {
-  const now = Date.now();
+export function upsertTokenProfile(input: CreateTokenProfileInput, submittedAt: number = Date.now()): TokenProfile {
+  const now = submittedAt;
   const id = input.id ?? slugify(input.name);
   const existing = getTokenProfile(id);
   if (existing?.builtIn) throw new Error("built-in token profiles cannot be modified");

package/src/token-usage-async.ts

@@ -5,12 +5,10 @@ import type { Message } from "./types";
 // of src/db directly (the transport-thinness ratchet, #348).
 import { getGateway } from "./db/gateway.ts";
 import type { SpawnBudgetReadback, WorkflowRunBudgetReadback } from "./db/spawn-budget.ts";
-import type { aggregateSessionTokenUsage, recordSessionTokenUsage, SessionTokenUsageAggregate, SessionTokenUsageRecord } from "./db/token-usage.ts";
+import type { aggregateSessionTokenUsage, recordSessionTokenUsage, SessionTokenUsageAggregate } from "./db/token-usage.ts";
 
 export const recordSessionTokenUsageAsync = async (input: Parameters<typeof recordSessionTokenUsage>[0]) => {
-  const { budgetNotices, ...result } = await getGateway().write("tokenUsage.record", { input }, { critical: false }) as unknown as {
-    record: SessionTokenUsageRecord;
-    inserted: boolean;
+  const { budgetNotices, ...result } = await getGateway().write("tokenUsage.record", { input }, { critical: false }) as unknown as ReturnType<typeof recordSessionTokenUsage> & {
     budgetNotices: Message[];
   };
   publishBudgetNotices(budgetNotices);

package/src/user-awaiting-reply.ts

@@ -1,7 +1,7 @@
+import { framed } from "./hub-metrics";
 import { userSinkId, canonicalHumanAgentId } from "@humwick/sdk";
 import { listUsers } from "./db/users";
-import { listPendingReplyObligationsAsync } from "./db/message-reads-async";
-import { getLatestMessageIdAsync } from "./db/message-reads-async";
+import { listPendingReplyObligationsAsync, getLatestMessageIdAsync } from "./db/message-reads-async";
 import { optedInNotificationRecipients } from "./db/notification-subscriptions";
 import { routeNotification } from "./notification-router";
 import { subscribeRelayEvents } from "./events";
@@ -12,7 +12,7 @@ export async function reconcileUserAwaitingReply(previous?: string): Promise<str
   const [pending, latestId] = await Promise.all([Promise.all(listUsers().map(user => listPendingReplyObligationsAsync(canonicalHumanAgentId(userSinkId(user.id)), 1, true))).then(rows => rows.flat()), getLatestMessageIdAsync()]);
   const signature = JSON.stringify([recipients, pending.length > 0]);
   if (signature === previous) return signature;
-  routeNotification({
+  await routeNotification({
     type: "user.awaiting_reply", scope: {}, recipients: [],
     message: {
       body: pending.length ? "The operator has an unanswered message." : "The operator has no unanswered messages.",
@@ -30,13 +30,13 @@ export function startUserAwaitingReply(): () => void {
   const unsubscribe = subscribeRelayEvents(event => {
     if (event.type.startsWith("message.") && event.data.to !== "hue:office") dirty = true;
   });
-  const timer = setInterval(() => {
+  const timer = setInterval(framed("timer user-awaiting-reply", () => {
     ticks++;
     if ((!dirty && ticks % 30 !== 0) || running) return;
     dirty = false; running = true;
     void reconcileUserAwaitingReply(signature).then(value => { signature = value; }).catch(error => {
       dirty = true; console.error("user.awaiting_reply reconciliation failed", error);
     }).finally(() => { running = false; });
-  }, 1000);
+  }), 1000);
   return () => { clearInterval(timer); unsubscribe(); };
 }