@humwick/server 0.130.1 → 0.130.2

package/src/branch-land-notice.ts

@@ -0,0 +1,93 @@
+import { isHumanAgentId } from "@humwick/sdk";
+import type { BranchLandedInput, resolveLandedTask } from "./branch-landed";
+import { getAgent, getTaskDetail } from "./db";
+import { renderTemplate } from "./prompt-resolver";
+import { routeNotification, INFORMATIONAL_TTL_MS } from "./notification-router";
+import { stampLifecycleSignal, taskSignalKey } from "./lifecycle-signal-registry";
+import { resolveCoordinatorAgentId } from "./steward-identity";
+import { resolveTaskOwnerAgentId } from "./task-owner";
+import { audienceWithTaskOwnerRole } from "./task-owner-notices";
+import { notifyLandUnconfirmed, verifyPublishedLand } from "./services/landed-ancestry";
+
+export async function announceVerifiedLand(input: BranchLandedInput, landedTask: ReturnType<typeof resolveLandedTask>, idempotencyKey?: string): Promise<void> {
+  const { workspace } = input;
+  const base = workspace.baseRef ?? "main";
+  const verification = input.mergedSha && input.landVerifiedSha === input.mergedSha
+    ? "confirmed" : await verifyPublishedLand(workspace.repoRoot, input.mergedSha, workspace.baseRef);
+  if (verification !== "confirmed") {
+    notifyLandUnconfirmed({ workspace, sha: input.mergedSha, targetRef: base.startsWith("origin/") ? base : `origin/${base}`, kind: verification });
+    return;
+  }
+  const landedBranch = workspace.branch;
+  const shortSha = input.mergedSha?.slice(0, 12);
+  const landedTaskPayload = landedTask ? { taskId: landedTask.taskId, taskLabel: landedTask.taskLabel } : {};
+  const author = workspace.ownerAgentId;
+  const shaLabel = shortSha ? ` as \`${shortSha}\`` : "";
+  const subjectLabel = input.subject ? ` — "${input.subject}"` : "";
+  const payload = {
+    verified: true,
+    kind: "branch.landed",
+    workspaceId: workspace.id,
+    repoRoot: workspace.repoRoot,
+    branch: landedBranch,
+    base,
+    sha: input.mergedSha,
+    author,
+    newBranch: input.newBranch,
+    pushed: input.pushed,
+    ...landedTaskPayload,
+  };
+
+  // The branch author cares most — push regardless of online (store-ahead delivers it on
+  // next poll if they've moved on, #234). A live owner continues in its worktree.
+  if (author) {
+    const branchLabel = landedBranch ? `\`${landedBranch}\`` : "Your branch";
+    const continueLabel = input.newBranch
+      ? ` You're now on \`${input.newBranch}\` — keep working there.`
+      : " Worktree reclaimed.";
+    // Durable (ttl null): the author's own branch landing must reach them even after they go
+    // offline (store-ahead #234) — not a bystander notice that goes stale.
+    const notices = routeNotification({
+      type: "branch.landed",
+      scope: { author, repoRoot: workspace.repoRoot, workspaceId: workspace.id },
+      recipients: [author],
+      ttlMs: null,
+      message: {
+        subject: "Your branch landed",
+        body: renderTemplate("notify.land.success", {
+          branch: branchLabel,
+          base,
+          sha: `${shaLabel}${subjectLabel}`,
+          publishLabel: "",
+          continueLabel,
+        }) + " verified: true.",
+        payload: { ...payload, notification: "landed-success" },
+        idempotencyKey,
+        replyExpected: false,
+      },
+    });
+    if (landedTask && input.mergedSha) {
+      for (const notice of notices) stampLifecycleSignal(notice.to, taskSignalKey(landedTask.taskId, "landed-on-main", input.mergedSha));
+    }
+  }
+
+  // Coordinator copy (#2837), informational. No explicit recipients: the router records the rule decision (#446).
+  const mainScope = { author, repoRoot: workspace.repoRoot, workspaceId: workspace.id, teamId: landedTask?.teamId ?? (author ? getAgent(author)?.teamId : undefined), coordinatorAgentIds: [...new Set([resolveCoordinatorAgentId(author), landedTask && resolveTaskOwnerAgentId(getTaskDetail(landedTask.taskId))].filter((id): id is string => !!id && !isHumanAgentId(id)))] };
+  const branchLabel = landedBranch ? `\`${landedBranch}\`` : "A branch";
+  const authorLabel = author ? ` by \`${author}\`` : "";
+  const notices = routeNotification({
+    type: "branch.landed",
+    scope: mainScope, ...audienceWithTaskOwnerRole("branch.landed", mainScope, landedTask?.taskId), // #2862
+    ttlMs: INFORMATIONAL_TTL_MS,
+    message: {
+      subject: `Merged to ${base}`,
+      body: `🔀 ${branchLabel}${authorLabel} merged to \`${base}\`${shaLabel}${subjectLabel}. verified: true.`,
+      payload: { ...payload, notification: "landed-success" },
+      idempotencyKey,
+      replyExpected: false,
+    },
+  });
+  if (landedTask && input.mergedSha) {
+    for (const notice of notices) stampLifecycleSignal(notice.to, taskSignalKey(landedTask.taskId, "landed-on-main", input.mergedSha));
+  }
+}

package/src/branch-landed.ts

@@ -1,3 +1,4 @@
+import { announceVerifiedLand } from "./branch-land-notice";
 import { errMessage, isHumanAgentId, type BaseWorktreeSyncResult } from "@humwick/sdk";
 import { emitRelayEvent } from "./events";
 import { routeNotification, INFORMATIONAL_TTL_MS } from "./notification-router";
@@ -32,6 +33,8 @@ export interface BranchLandedInput extends LandedTaskAttributionInput {
   workspace: Pick<WorkspaceRecord, "id" | "repoRoot" | "branch" | "baseRef" | "baseSha" | "ownerAgentId" | "orchestratorId">;
   /** SHA the base now points at after the land. */
   mergedSha?: string;
+  /** Landing host proof from its fresh origin ancestry check. Must match mergedSha. */
+  landVerifiedSha?: string;
   /** Original command settlement clock when recovering an older land. */
   landedAt?: number;
   /** Subject line of the landed commit, when the orchestrator reported it. */
@@ -168,7 +171,7 @@ async function landedCommitSubjects(input: BranchLandedInput, log: BranchLandedL
   }, log);
 }
 
-function resolveLandedTask(input: LandedTaskAttributionInput): {
+export function resolveLandedTask(input: LandedTaskAttributionInput): {
   taskId: number;
   taskLabel: string;
   projectId?: string;
@@ -348,79 +351,7 @@ export function announceBranchLanded(input: BranchLandedInput): void {
     }
   })());
 
-  const author = workspace.ownerAgentId;
-  const shaLabel = shortSha ? ` as \`${shortSha}\`` : "";
-  const subjectLabel = input.subject ? ` — "${input.subject}"` : "";
-  const payload = {
-    kind: "branch.landed",
-    workspaceId: workspace.id,
-    repoRoot: workspace.repoRoot,
-    branch: landedBranch,
-    base,
-    sha: input.mergedSha,
-    author,
-    newBranch: input.newBranch,
-    pushed: input.pushed,
-    ...landedTaskPayload,
-  };
-
-  // A local-only land (push disabled, or base has no upstream) has NOT reached the
-  // remote — say so rather than implying a published merge an agent would report as
-  // shipped (#285). `undefined` (older orchestrators) keeps the message generic.
-  const publishLabel = input.pushed === false ? ` Not yet pushed to \`origin/${base}\` — local only.` : "";
-
-  // The branch author cares most — push regardless of online (store-ahead delivers it on
-  // next poll if they've moved on, #234). A live owner continues in its worktree.
-  if (author) {
-    const branchLabel = landedBranch ? `\`${landedBranch}\`` : "Your branch";
-    const continueLabel = input.newBranch
-      ? ` You're now on \`${input.newBranch}\` — keep working there.`
-      : " Worktree reclaimed.";
-    // Durable (ttl null): the author's own branch landing must reach them even after they go
-    // offline (store-ahead #234) — not a bystander notice that goes stale.
-    const notices = routeNotification({
-      type: "branch.landed",
-      scope: { author, repoRoot: workspace.repoRoot, workspaceId: workspace.id },
-      recipients: [author],
-      ttlMs: null,
-      message: {
-        subject: "Your branch landed",
-        body: renderTemplate("notify.land.success", {
-          branch: branchLabel,
-          base,
-          sha: `${shaLabel}${subjectLabel}`,
-          publishLabel,
-          continueLabel,
-        }),
-        payload: { ...payload, notification: "landed-success" },
-        idempotencyKey,
-        replyExpected: false,
-      },
-    });
-    if (landedTask && input.mergedSha) {
-      for (const notice of notices) stampLifecycleSignal(notice.to, taskSignalKey(landedTask.taskId, "landed-on-main", input.mergedSha));
-    }
-  }
-
-  // Coordinator copy (#2837), informational. No explicit recipients: the router records the rule decision (#446).
-  const mainScope = { author, repoRoot: workspace.repoRoot, workspaceId: workspace.id, teamId: landedTask?.teamId ?? (author ? getAgent(author)?.teamId : undefined), coordinatorAgentIds: [...new Set([resolveCoordinatorAgentId(author), landedTask && resolveTaskOwnerAgentId(getTaskDetail(landedTask.taskId))].filter((id): id is string => !!id && !isHumanAgentId(id)))] };
-  const branchLabel = landedBranch ? `\`${landedBranch}\`` : "A branch";
-  const authorLabel = author ? ` by \`${author}\`` : "";
-  const notices = routeNotification({
-    type: "branch.landed",
-    scope: mainScope, ...audienceWithTaskOwnerRole("branch.landed", mainScope, landedTask?.taskId), // #2862
-    ttlMs: INFORMATIONAL_TTL_MS,
-    message: {
-      subject: `Merged to ${base}`,
-      body: `🔀 ${branchLabel}${authorLabel} merged to \`${base}\`${shaLabel}${subjectLabel}.${publishLabel}`,
-      payload: { ...payload, notification: "landed-success" },
-      idempotencyKey,
-      replyExpected: false,
-    },
-  });
-  if (landedTask && input.mergedSha) {
-    for (const notice of notices) stampLifecycleSignal(notice.to, taskSignalKey(landedTask.taskId, "landed-on-main", input.mergedSha));
-  }
+  trackLandLifecycleDrive(announceVerifiedLand(input, landedTask, idempotencyKey));
 }
 
 /**
@@ -682,7 +613,7 @@ const DRAINED_LAND_STRAND_BOOKKEEPING = {
  * (record the receipt, announce a real land, drain the steward/strand bookkeeping so no
  * watchdog re-wakes or re-pages a branch with nothing left to land).
  */
-export async function reconcileLandedWorkspace(ws: WorkspaceRecord, preview: WorkspaceMergePreview): Promise<LandedWorkspaceReconciliation> {
+export async function reconcileLandedWorkspace(ws: WorkspaceRecord, preview: WorkspaceMergePreview, landVerifiedSha?: string): Promise<LandedWorkspaceReconciliation> {
   const observedSha = typeof preview.prMergeSha === "string"
     ? preview.prMergeSha
     : typeof preview.landedSha === "string"
@@ -753,7 +684,7 @@ export async function reconcileLandedWorkspace(ws: WorkspaceRecord, preview: Wor
     metadata: { source: "server", maintenanceJobId: "workspace-conflict-scan", workspaceId: ws.id, fromStatus: ws.status, toStatus: nextStatus, ...(ownerAlive ? { ownerAlive: true, keptLive: true } : {}), ...(sha ? { landedSha: sha } : {}), ...(landedCommitCount === 0 ? { landedCommitCount: 0 } : {}) },
   });
   try {
-    notifyBranchLanded({ workspace: ws, mergedSha: sha, landedTipSha: sha, commitShas: sha ? [sha] : [], pushed: true, prLand: preview.prMerged === true, ...(landedCommitCount === 0 ? { landedCommitCount } : {}) });
+    notifyBranchLanded({ workspace: ws, mergedSha: sha, landVerifiedSha, landedTipSha: sha, commitShas: sha ? [sha] : [], pushed: true, prLand: preview.prMerged === true, ...(landedCommitCount === 0 ? { landedCommitCount } : {}) });
   } catch {
     // Notification is best-effort; the status change + activity event still stand.
   }

package/src/bus.ts

@@ -1,5 +1,5 @@
 import type { Server, ServerWebSocket } from "bun";
-import { ValidationError, applyAgentLifecycleUpdate, getAgent, getDb, orphanTasksForAgentAsync, recordHeartbeat, setStatus, validateAgentSession } from "./db";
+import { ValidationError, applyAgentLifecycleUpdateAsync, getAgent, getDb, orphanTasksForAgentAsync, recordHeartbeatAsync, validateAgentSession } from "./db";
 import { getOutboxCursor, hasReplayGap, replayEventsSlice, type BusEvent } from "./bus-outbox";
 import { emitAgentStatus } from "./agent-status-events";
 import { AgentStatusSubscribers, matchesSubscription, wantsAgentStatus } from "./transport-subscriptions";
@@ -34,7 +34,7 @@ import {
   type BusFrame,
   type RegisterFrame,
 } from "@humwick/sdk/protocol";
-import { errMessage, isRecord } from "@humwick/sdk";
+import { errMessage, fireAndForget, isRecord } from "@humwick/sdk";
 import { getComponentAuth, isComponentAuthorizedFor, isAuthorized, isOriginAllowed, unauthorized } from "./security";
 import type { AgentCard, ComponentToken, ContextState, Message, ProviderCapabilities, QuotaState, RegisterAgentInput, Task } from "./types";
 
@@ -138,8 +138,11 @@ export function busHandleClose(ws: BusWebSocket): void {
     // on the establishing socket id, so a foreign socket's close cannot evict the owner's binding and a stale
     // close cannot evict a newer register's binding).
     if (!conn.transportReconnect) {
-      setStatus(conn.agentId, "stale", { instanceId: conn.instanceId, epoch: conn.epoch });
-      queueAgentStatusEvent(conn.agentId);
+      const agentId = conn.agentId;
+      void fireAndForget(`[bus] mark ${agentId} stale on close`, async () => {
+        await applyAgentLifecycleUpdateAsync(agentId, { status: "stale" }, { instanceId: conn.instanceId, epoch: conn.epoch });
+        queueAgentStatusEvent(agentId);
+      });
     }
   }
 }
@@ -162,7 +165,7 @@ export async function expireStaleBusAgents(graceMs = busStaleGraceMs(), lagGrace
     // but do NOT revoke the token — a reconnect accepts the EXISTING token and re-marks
     // live. The #484 restart-strand revoke now fires on instance takeover in registerAgent
     // (a fresh instanceId → epoch bump), never on a heartbeat lapse.
-    setStatus(row.id, "offline");
+    await applyAgentLifecycleUpdateAsync(row.id, { status: "offline" });
     await emitAgentStatus(row.id);
     const tasks = await orphanTasksForAgentAsync(row.id);
     for (const task of tasks) {
@@ -187,7 +190,7 @@ async function handleFrame(ws: BusWebSocket, frame: ReturnType<typeof validateCl
       await handleRegister(ws, frame);
       return;
     case "heartbeat":
-      await withRegistered(ws, frame, (conn) => {
+      await withRegistered(ws, frame, async (conn) => {
         if (conn.agentId) {
           // #633 — a runner whose provider terminally exited stays alive and keeps sending idle
           // keepalives (semanticStatus is never "offline"). Those must NOT resurrect an agent the
@@ -199,11 +202,10 @@ async function handleFrame(ws: BusWebSocket, frame: ReturnType<typeof validateCl
           if (!heldOffline) {
             const wasUnavailable = current ? isAgentUnavailable(current) : false;
             const runtime = runtimeFromMeta(frame.payload.meta);
-            // #1468 — coalesce the steady-state liveness write; recordHeartbeat writes through
-            // synchronously only on a status change / resurrection (and persists last_seen + runtime
-            // then). This replaces the old heartbeat()+setStatus() pair that rewrote the row twice
-            // per beat even when nothing changed.
-            recordHeartbeat(conn.agentId, { guard: { instanceId: conn.instanceId, epoch: conn.epoch }, status: frame.payload.status, runtime });
+            // #1468 — coalesce the steady-state liveness write; only a status change / resurrection
+            // writes (last_seen + runtime), on the writer thread (#1870 S7). This replaces the old
+            // heartbeat()+setStatus() pair that rewrote the row twice per beat even when nothing changed.
+            await recordHeartbeatAsync(conn.agentId, { guard: { instanceId: conn.instanceId, epoch: conn.epoch }, status: frame.payload.status, runtime });
             queueAgentStatusEvent(conn.agentId);
             // #416 — flush queued messages if the agent transitioned from unavailable to available.
             if (wasUnavailable) {
@@ -225,19 +227,16 @@ async function handleFrame(ws: BusWebSocket, frame: ReturnType<typeof validateCl
       await withRegistered(ws, frame, async (conn) => {
         if (!conn.agentId) return sendError(ws, frame.id, "AGENT_REQUIRED", "status frames require an agent registration");
         const guard = { instanceId: conn.instanceId, epoch: conn.epoch };
-        // #1895 (S3.2 L10 review, D2) — DELIBERATELY left on the sync write path: this is the
-        // per-status-frame hot WS seam (every heartbeat/status update from every live agent), not the
-        // cold/admin lifecycle surface that leaf migrates. `withRegistered`'s callback runs with no
-        // thread hop mid-frame, so routing these writes through their async facades would not move
-        // them off-loop — it would only add a round-trip.
-        // #2111 — but the three writes ARE now one. This used to be getAgent(before) + setStatus
+        // #1870 S7 — the per-status-frame write runs on the writer thread; this socket's FIFO frame
+        // queue keeps its frames ordered across the hop, and the lifecycle ordinal fences it against
+        // every other lifecycle writer. #2111 — the three writes are one. This used to be getAgent(before) + setStatus
         // (guard read + before read + UPDATE in its own BEGIN IMMEDIATE) + markReady (guard read +
         // UPDATE) + mergeAgentMeta (two reads + UPDATE) + getAgent(after): ~7 row reads and 3 UPDATEs
         // of one row per frame. `applyAgentLifecycleUpdate` does one guard read, one UPDATE and hands
         // back the before/after cards the audit calls below need. The flap ledger is different: it is
         // a cold durable history and must stay ordered with its status edge, so await it within this
         // socket's existing FIFO frame queue.
-        const applied = applyAgentLifecycleUpdate(
+        const applied = await applyAgentLifecycleUpdateAsync(
           conn.agentId,
           { status: frame.payload.agentStatus, ready: frame.payload.ready, meta: frame.payload.meta },
           guard,
@@ -378,7 +377,7 @@ async function handleRegister(ws: BusWebSocket, frame: RegisterFrame): Promise<v
       machine: payload.machine,
       capabilities: payload.capabilities,
       status: runnerManaged ? "idle" : "online",
-      ready: true,
+      ready: !runnerManaged,
       instanceId: payload.instanceId,
       ...(providerCapabilities ? { providerCapabilities } : {}),
       ...(context ? { context } : {}),
@@ -400,7 +399,7 @@ async function handleRegister(ws: BusWebSocket, frame: RegisterFrame): Promise<v
   // newly registered agent stale. Fence the install on liveness and clean up the provisional
   // registration with the same instance/epoch guard used by an established connection's close.
   if (!liveBusSocketIds.has(ws.data.id)) {
-    if (payload.agentId && setStatus(payload.agentId, "stale", { instanceId: payload.instanceId, epoch })) {
+    if (payload.agentId && (await applyAgentLifecycleUpdateAsync(payload.agentId, { status: "stale" }, { instanceId: payload.instanceId, epoch })).changed) {
       queueAgentStatusEvent(payload.agentId);
     }
     return;

package/src/cd-deploy-notify.ts

@@ -1,3 +1,7 @@
+import { resolve } from "node:path";
+import { getDeployEventForCommitAsync, deployChangesForCommitsAsync } from "./db/deploy-events-async";
+import type { DeployChanges } from "./db/deploy-events";
+import { runGit } from "./services/git-process";
 import { VERSION } from "./config";
 import { reportedGitSha } from "./git-sha";
 import { routeNotification } from "./notification-router";
@@ -17,6 +21,7 @@ registerDefaultAudienceRules("cd.deploy_live", [audiencePredicates.persistentCoo
 export interface CdDeployLiveInput {
   /** Defaults to this process's package version. */
   version?: string;
+  repoRoot?: string;
   /** Defaults to the running process's attested commit sha (undefined on an npm install). */
   gitSha?: string;
 }
@@ -32,7 +37,7 @@ export interface CdDeployLiveInput {
  * there is no stable identity to key the one-per-deploy dedup on, and this notice is specifically
  * about the CD-deploy restart glitch that only that deployment model produces.
  */
-export function notifyCdDeployLive(input: CdDeployLiveInput = {}): void {
+export async function notifyCdDeployLive(input: CdDeployLiveInput = {}): Promise<void> {
   const version = input.version ?? VERSION;
   const gitSha = input.gitSha ?? reportedGitSha();
   if (!gitSha) {
@@ -43,6 +48,7 @@ export function notifyCdDeployLive(input: CdDeployLiveInput = {}): void {
     return;
   }
   const shortSha = gitSha.slice(0, 12);
+  const contents = await deployNoticeContents(gitSha, input.repoRoot ?? resolve(import.meta.dir, ".."));
 
   routeNotification({
     type: "cd.deploy_live",
@@ -50,10 +56,32 @@ export function notifyCdDeployLive(input: CdDeployLiveInput = {}): void {
     ttlMs: null,
     message: {
       subject: `Humwick ${version} is now running`,
-      body: `Humwick v${version} (\`${shortSha}\`) is now running after a restart. Any tool-call failures in the last ~1-2 min were the deploy, not an incident — you're good now; retry anything that failed.`,
+      body: `Humwick v${version} (\`${shortSha}\`) is now running after a restart. Interrupted tool calls were the deploy, not an incident; retry them.${contents}`,
       payload: { kind: "cd.deploy-live", version, gitSha },
       idempotencyKey: `cd-deploy-live:${gitSha}`,
       replyExpected: false,
     },
   });
 }
+
+async function deployNoticeContents(gitSha: string, repoRoot: string): Promise<string> {
+  const event = await getDeployEventForCommitAsync(gitSha);
+  if (!event) return "";
+  let changes: DeployChanges | undefined = event.changes;
+  if (changes?.state !== "ready") {
+    if (!event.previousSha) return "\nContents unavailable: no previous deployed SHA.";
+    const result = await runGit(["-C", repoRoot, "log", "--format=%H%x00%s", `${event.previousSha}..${gitSha}`, "--"], 5000, 1024 * 1024);
+    if (result.unavailable || result.exitCode !== 0) return "\nContents unavailable: deploy range could not be read.";
+    const commits = result.stdout.trim().split("\n").filter(Boolean).map(line => {
+      const [sha, ...subject] = line.split("\x00");
+      return { sha: sha!, subject: subject.join("\x00") };
+    });
+    changes = { state: "ready", ...await deployChangesForCommitsAsync(commits) };
+  }
+  const lines = changes.tasks.flatMap(task => [
+    `T${task.id} ${task.label.replace(/\s+/g, " ")}${task.issues.length ? ` — ${task.issues.map(issue => `${issue.repo}#${issue.externalKey} (${issue.relation ?? "references"})`).join(", ")}` : ""}`,
+    ...(task.onDeploy ? [`  onDeploy: ${task.onDeploy.replace(/\s+/g, " ")}`] : []),
+  ]);
+  lines.push(`${changes.unmatchedCommits.length} non-Task commit(s).`);
+  return `\n${lines.join("\n")}`;
+}

package/src/cd-host-lag-notify.ts

@@ -1,9 +1,10 @@
 import { resolveCdFleetAllowlist } from "../scripts/cd-config.ts";
 import { isCdFleetHostAllowlisted } from "../scripts/cd-fleet-membership.ts";
-import { getConfig, setConfig } from "./config-store.ts";
-import { createActivityEvent } from "./db/activity.ts";
+import { getConfig } from "./config-store.ts";
+import { setConfigAsync } from "./config-store-write.ts";
+import { createActivityEventAsync } from "./db/activity-async.ts";
 import { computeFleetConvergenceAsync, type FleetConvergence } from "./fleet-convergence.ts";
-import { raiseOperatorAlarm } from "./operator-alarm.ts";
+import { raiseOperatorAlarmAsync } from "./operator-alarm.ts";
 import { listOrchestratorsAsync } from "./db/orchestrators-async.ts";
 
 /** Server-side: a durable alarm per lag episode, and a recovery record when it clears. */
@@ -17,7 +18,7 @@ export async function notifyCdHostLag(convergence?: FleetConvergence, allowlist
     // Keep the episode's identity while targets advance, until this Host catches up.
     const since = previous[host.id] ?? host.lag.since;
     current[host.id] = since;
-    raiseOperatorAlarm({ clientId: `cd-host-lag:${host.id}:${since}`, source: "cd-host-lag", view: "deploys",
+    await raiseOperatorAlarmAsync({ clientId: `cd-host-lag:${host.id}:${since}`, source: "cd-host-lag", view: "deploys",
       subject: `CD deploy lag on ${host.id}`, dedupeEvent: true,
       detail: `${host.id} is ${Math.floor(host.lag.ageMs / 60_000)} minutes behind ${status.targetSha.slice(0, 12)}: ${host.deferred?.reason ?? host.reason}`,
       metadata: { orchestratorId: host.id, since, targetSha: status.targetSha, deferred: host.deferred } });
@@ -29,8 +30,8 @@ export async function notifyCdHostLag(convergence?: FleetConvergence, allowlist
     if (!host || !isCdFleetHostAllowlisted(host, allowlist)) { delete current[id]; continue; }
     if (!status.targetSha || host.gitSha !== status.targetSha) { current[id] = since; continue; }
     delete current[id];
-    createActivityEvent({ clientId: `cd-host-lag-resolved:${id}:${since}`, kind: "state", view: "deploys",
+    await createActivityEventAsync({ clientId: `cd-host-lag-resolved:${id}:${since}`, kind: "state", view: "deploys",
       title: `CD deploy lag cleared on ${id}`, metadata: { source: "cd-host-lag", orchestratorId: id, since, resolved: true } });
   }
-  if (JSON.stringify(current) !== JSON.stringify(previous)) setConfig("cd-host-lag", "alarms", current);
+  if (JSON.stringify(current) !== JSON.stringify(previous)) await setConfigAsync("cd-host-lag", "alarms", current);
 }

package/src/ci/incidents.ts

@@ -169,7 +169,7 @@ export function ingestCiObservation(observation: CiObservation, now: number = Da
 
 /** Gateway-routed form for async-capable polling jobs. The complete observation transition is one
  * writer operation, so no gateway await exposes a stale watch state to a later incident decision. */
-export async function ingestCiObservationAsync(observation: CiObservation): Promise<CiIncidentIngestResult> {
+export async function ingestCiObservationAsync(observation: CiObservation, now?: number): Promise<CiIncidentIngestResult> {
   const transition = transitionInput(observation);
-  return formatTransitionResult(await ingestCiObservationTransitionAsync(transition.input), transition.watchKey, transition.fingerprint);
+  return formatTransitionResult(await ingestCiObservationTransitionAsync(transition.input, now), transition.watchKey, transition.fingerprint);
 }

package/src/ci/monitor.ts

@@ -12,8 +12,8 @@ import { isPathWithinBase } from "../utils";
 import { computeCiWatchKey, isFailingJob } from "./incidents.ts";
 import { ghApi, type GitHubApi } from "../issue-tracker/github.ts";
 import { resolveCoordinatorAgentId } from "../steward-identity.ts";
-import { notifyCiRootMissing, notifyCiRootMissingAsync, type CiRootMissing } from "./root-missing.ts";
-import { notifyCiTransition, notifyCiTransitionAsync, type CiTransitionNotifyResult } from "./transition-notify.ts";
+import { notifyCiRootMissing, type CiRootMissing } from "./root-missing.ts";
+import { notifyCiTransition, type CiTransitionNotifyResult } from "./transition-notify.ts";
 import { workflowNotTriggered } from "./not-triggered.ts";
 import type { CiJobRef, CiObservation, CiObservationStatus, CiRunRef } from "./provider.ts";
 
@@ -343,7 +343,7 @@ export interface CiMonitorDeps {
   rootsForProject: (projectId: string) => ProjectRoot[];
   loadConfig: (root: ProjectRoot) => CiMonitorConfig | null | Promise<CiMonitorConfig | null>;
   api: GitHubApi;
-  notify: typeof notifyCiTransition;
+  notify: (...args: Parameters<typeof notifyCiTransition>) => CiTransitionNotifyResult | Promise<CiTransitionNotifyResult>;
   resolveCoordinator: (rootPath: string) => string | undefined;
   /** Commits workflows wait on: each watch of their root reports its verdict on them too. */
   shaWaits: () => Promise<CiShaWait[]>;
@@ -419,7 +419,6 @@ export async function runCiMonitor(input: Partial<CiMonitorDeps> = {}): Promise<
     readinessMs: 0, historyMs: 0, historyApiCalls: 0,
     transitions: { went_red: 0, back_green: 0, never_matched: 0, config_mismatch: 0, none: 0 },
   };
-  const useAsync = deps.notify === notifyCiTransition && !deps.now;
   // A repository counts as missing only if no root of it has the config, so a duplicate root cannot flap the notice.
   const missingRoots = new Map<string, CiRootMissing>();
   const presentRepositories = new Set<string>();
@@ -464,9 +463,7 @@ export async function runCiMonitor(input: Partial<CiMonitorDeps> = {}): Promise<
           const observation = await observeGitHubCiMonitorWatch(repository, watch, deps.api, runsPayload);
           if (deps.signal?.aborted) break;
           const notificationDeps = { resolveCoordinator: () => deps.resolveCoordinator(root.rootPath), ...(deps.now ? { now: deps.now() } : {}) };
-          const transition = useAsync
-            ? await notifyCiTransitionAsync(observation, notificationDeps)
-            : deps.notify(observation, notificationDeps);
+          const transition = await deps.notify(observation, notificationDeps);
           result.transitions[transition.transition] += 1;
           // The branch-tip row keys the watch; the same key carries its verdict on each waited commit.
           const watchKey = computeCiWatchKey(observation);
@@ -495,7 +492,7 @@ export async function runCiMonitor(input: Partial<CiMonitorDeps> = {}): Promise<
     if (presentRepositories.has(key) || deps.signal?.aborted) continue;
     const notifyDeps = { resolveCoordinator: () => deps.resolveCoordinator(missing.rootPath), ...(deps.now ? { now: deps.now() } : {}) };
     try {
-      if (useAsync ? await notifyCiRootMissingAsync(missing, notifyDeps) : notifyCiRootMissing(missing, notifyDeps)) result.missingRoots += 1;
+      if (await notifyCiRootMissing(missing, notifyDeps)) result.missingRoots += 1;
     } catch (error) {
       result.errors += 1;
       console.warn(`ci monitor ${missing.repository} root-missing notice skipped: ${errMessage(error)}`);

package/src/ci/root-missing.ts

@@ -1,4 +1,3 @@
-import { markCiRootMissingNotified } from "../db/ci-incidents.ts";
 import { markCiRootMissingNotifiedAsync } from "../db/ci-incidents-async.ts";
 import { routeCiIncidentNotification } from "./notifications.ts";
 import { resolveCoordinator, type CiTransitionNotifyDeps } from "./transition-notify.ts";
@@ -26,10 +25,6 @@ function notify(input: CiRootMissing, deps: Deps, fired: boolean): boolean {
   return true;
 }
 
-export function notifyCiRootMissing(input: CiRootMissing, deps: Deps = {}): boolean {
-  return notify(input, deps, markCiRootMissingNotified(input.repository, deps.now));
-}
-
-export async function notifyCiRootMissingAsync(input: CiRootMissing, deps: Deps = {}): Promise<boolean> {
-  return notify(input, deps, await markCiRootMissingNotifiedAsync(input.repository));
+export async function notifyCiRootMissing(input: CiRootMissing, deps: Deps = {}): Promise<boolean> {
+  return notify(input, deps, await markCiRootMissingNotifiedAsync(input.repository, deps.now));
 }

package/src/ci/transition-notify.ts

@@ -1,7 +1,7 @@
 import { stewardFallbackTarget } from "../config.ts";
 import { emitRelayEvent } from "../events.ts";
 import type { WorkspaceHumanEscalationResult } from "../workspace-human-escalation.ts";
-import { ingestCiObservation, ingestCiObservationAsync, type CiIncidentIngestResult } from "./incidents.ts";
+import { ingestCiObservationAsync, type CiIncidentIngestResult } from "./incidents.ts";
 import {
   routeCiIncidentNotification,
   type CiFailedNotificationPayload,
@@ -41,7 +41,7 @@ import type { CiObservation, CiRunRef } from "./provider.ts";
 export const CI_MAIN_WATCH_COORDINATOR = "cd-watcher";
 
 export interface CiTransitionNotifyDeps {
-  ingest?: (observation: CiObservation, now?: number) => CiIncidentIngestResult;
+  ingest?: (observation: CiObservation, now?: number) => CiIncidentIngestResult | Promise<CiIncidentIngestResult>;
   route?: (input: Parameters<typeof routeCiIncidentNotification>[0]) => RouteCiIncidentNotificationResult;
   /** Resolve the coordinator/owner agent the relay message addresses. Defaults to the configured
    * steward fallback, then the system watch actor. */
@@ -65,16 +65,17 @@ export interface CiTransitionNotifyResult {
  * Detect and (if a flip occurred) alert on a CI red/green transition for a single observed watch.
  * Idempotent per transition — safe to call every poll tick.
  */
-export function notifyCiTransition(
+export async function notifyCiTransition(
   observation: CiObservation | null | undefined,
   deps: CiTransitionNotifyDeps = {},
-): CiTransitionNotifyResult {
+): Promise<CiTransitionNotifyResult> {
   if (!observation) return { transition: "none", notified: false, messages: 0 };
-  const ingest = deps.ingest ?? ingestCiObservation;
+  const ingest = deps.ingest ?? ingestCiObservationAsync;
   const route = deps.route ?? routeCiIncidentNotification;
   const now = deps.now ?? Date.now();
 
-  const result = ingest(observation, now);
+  // Notification emission follows the committed transition (one writer op, #1870), never a pre-write observation.
+  const result = await ingest(observation, deps.now);
   emitCiStatus(result);
 
   if (result.kind === "opened") {
@@ -155,39 +156,6 @@ export function notifyCiTransition(
   return { transition: "none", ...(result.kind === "deduped" ? { incidentId: result.incident.id } : {}), notified: false, messages: 0 };
 }
 
-/** Gateway-routed companion for async maintenance polling. Notification emission follows the
- * committed transition, never a pre-write observation. */
-export async function notifyCiTransitionAsync(
-  observation: CiObservation | null | undefined,
-  deps: Omit<CiTransitionNotifyDeps, "ingest"> = {},
-): Promise<CiTransitionNotifyResult> {
-  if (!observation) return { transition: "none", notified: false, messages: 0 };
-  const route = deps.route ?? routeCiIncidentNotification;
-  const now = deps.now ?? Date.now();
-  const result = await ingestCiObservationAsync(observation);
-  emitCiStatus(result);
-  if (result.kind === "opened") {
-    const coordinatorAgentId = resolveCoordinator(deps);
-    const routed = route({ type: "ci.failed", payload: { incidentId: result.incident.id, provider: observation.provider, repository: observation.repository, branch: observation.branch, workflow: observation.workflow, failingHeadSha: observation.headSha, coordinatorAgentId, observedAt: observation.observedAt, ...(observation.conclusion ? { conclusion: observation.conclusion } : {}), failingRunIds: observation.failingRuns.map((run) => run.id), failingJobNames: failingJobNames(observation) }, alertHuman: true, subject: redSubject(observation), body: redBody(observation, result.incident.id) });
-    return { transition: "went_red", incidentId: result.incident.id, notified: true, messages: routed.messages.length, ...(routed.humanFallback ? { humanFallback: routed.humanFallback } : {}) };
-  }
-  if (result.kind === "resolved") {
-    const coordinatorAgentId = resolveCoordinator(deps);
-    const routed = route({ type: "ci.recovered", payload: { incidentId: result.incident.id, provider: observation.provider, repository: observation.repository, branch: observation.branch, workflow: observation.workflow, failingHeadSha: result.incident.openedHeadSha, coordinatorAgentId, recoveredAt: now, greenHeadSha: observation.headSha, ...(result.incident.resolution ? { resolution: result.incident.resolution } : {}) }, alertHuman: true, subject: greenSubject(observation), body: greenBody(observation, result.incident.id, result.incident.openedHeadSha) });
-    return { transition: "back_green", incidentId: result.incident.id, notified: true, messages: routed.messages.length, ...(routed.humanFallback ? { humanFallback: routed.humanFallback } : {}) };
-  }
-  if (result.kind === "never_matched") {
-    const coordinatorAgentId = resolveCoordinator(deps);
-    const routed = route({ type: "ci.never_matched", payload: { provider: observation.provider, repository: observation.repository, branch: observation.branch, workflow: observation.workflow, coordinatorAgentId, firstSeenAt: result.firstSeenAt, observedAt: observation.observedAt }, alertHuman: true, subject: neverMatchedSubject(observation), body: neverMatchedBody(observation) });
-    return { transition: "never_matched", notified: true, messages: routed.messages.length, ...(routed.humanFallback ? { humanFallback: routed.humanFallback } : {}) };
-  }
-  if (result.kind === "config_mismatch") {
-    const routed = route(configMismatchRoute(observation, resolveCoordinator(deps), result.unmatchedJobs));
-    return { transition: "config_mismatch", notified: true, messages: routed.messages.length, ...(routed.humanFallback ? { humanFallback: routed.humanFallback } : {}) };
-  }
-  return { transition: "none", ...(result.kind === "deduped" ? { incidentId: result.incident.id } : {}), notified: false, messages: 0 };
-}
-
 /** Live CI view (#3011): one `ci.status` event per watch status transition, never per poll. */
 function emitCiStatus(result: CiIncidentIngestResult): void {
   if (result.statusChange) emitRelayEvent({ type: "ci.status", source: "server", subject: result.watchKey, data: { ...result.statusChange } });

package/src/cli/cd-pauses.ts

@@ -0,0 +1,19 @@
+import type { CdPause, CdPauseControl } from "../state-adoption/cd-cutover";
+import { apiRequest, ApiRequestError } from "./_shared";
+
+/** The audited maintenance pause API (#1645) shared by local and fleet cutovers (#3470). */
+export const cdWatcherPauses: CdPauseControl = {
+  async pause(holder, reason) {
+    try {
+      return ((await apiRequest("POST", "/api/maintenance/jobs/cd-watcher/pause", { holder, reason, indefinite: true })) as { pause: CdPause }).pause;
+    } catch (error) {
+      if (error instanceof ApiRequestError && error.status === 409) return (error.responseBody as { pause: CdPause }).pause;
+      throw error;
+    }
+  },
+  async current() {
+    const { pauses } = (await apiRequest("GET", "/api/maintenance/pauses")) as { pauses: Array<CdPause & { jobId: string }> };
+    return pauses.find((pause) => pause.jobId === "cd-watcher") ?? null;
+  },
+  async resume() { await apiRequest("POST", "/api/maintenance/jobs/cd-watcher/resume"); },
+};

package/src/cli/orchestrator.ts

@@ -118,7 +118,7 @@ export async function handleOrchestratorCommand(args: string[]): Promise<void> {
   // Install the server package too so a real `humwick` bin lands in the
   // runtime .bin dir (#1421): under bunx there is no global install, so without
   // this the darwin shim below would point at a bin that does not exist.
-  const packageCommand = skipInstall ? [] : ["npm", "install", "--prefix", runtimePrefix, `agent-relay-runner@${version}`, `agent-relay-orchestrator@${version}`, `agent-relay-server@${version}`, OPENCODE_HARNESS_PACKAGE];
+  const packageCommand = skipInstall ? [] : ["npm", "install", "--prefix", runtimePrefix, `@humwick/runner@${version}`, `@humwick/orchestrator@${version}`, `@humwick/server@${version}`, OPENCODE_HARNESS_PACKAGE];
   // W1: on macOS, put `humwick` on the interactive PATH (shim + profile
   // block). Gated to darwin — Linux global-npm installs already get the shim via
   // the postinstall, and we avoid editing Linux user profiles unnecessarily.