@humwick/server 0.0.0-stage → 0.130.1

package/scripts/release-upgrade.ts

@@ -0,0 +1,1296 @@
+import { relayEnv } from "@humwick/sdk/env";
+/**
+ * Pure decision helpers for the local-host upgrade step of `bun run release`.
+ *
+ * Extracted from release.ts so the flag-selection / failure-classification logic
+ * is unit-testable (#309). The release script runs from the *new* source but
+ * invokes the *installed* `humwick` CLI — i.e. the version being replaced.
+ * That CLI may predate `--restart-deferred` (added in v0.30.0), so we must decide
+ * up front whether to even try it, and — when an upgrade attempt fails — classify
+ * *why* without guessing. The old code pattern-matched `/Unknown upgrade option/`
+ * against arbitrary failure output, which misread ETARGET/transient failures as
+ * "predates the flag" and silently degraded to `--no-restart` (the source of the
+ * recurring false "restart manually" warnings).
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { homedir, hostname as osHostname } from "node:os";
+import { join } from "node:path";
+import { errMessage, RELAY_TOKEN_HEADER, deriveOrchestratorId } from "@humwick/sdk";
+import { shellQuote } from "@humwick/sdk/shell-utils";
+import { isBenignBootoutFailure } from "../src/daemon.ts";
+import { isCdFleetHostAllowlisted, resolveCdFleetAllowlistAmbiguities } from "./cd-fleet-membership.ts";
+import { assessDeploymentHealth, type DeploymentHealthAssessment, type HealthProbeSnapshot } from "./release-health.ts";
+import type { DurableReleaseCompletionInput, DurableReleaseCompletionResult } from "./release-report.ts";
+
+export { assessDeploymentHealth, formatHealthGateFailure, formatHealthProbeSnapshot } from "./release-health.ts";
+export type { DeploymentHealthAssessment, DeploymentHealthRole, HealthProbeSnapshot } from "./release-health.ts";
+
+/** First published version whose `humwick upgrade` understands `--restart-deferred`. */
+export const RESTART_DEFERRED_MIN_VERSION = "0.30.0";
+
+/**
+ * Supervisors the remote self-upgrade command bus can actually drive — anything
+ * else makes `orchestrator.upgrade` throw immediately (see `planSelfUpgrade` in
+ * orchestrator/src/self-upgrade.ts, which requires one of these two).
+ */
+export const SUPPORTED_SELF_UPGRADE_SUPERVISORS = new Set(["systemd", "launchd"]);
+
+export type DiscoverableOrchestrator = {
+  id: string;
+  hostname?: string;
+  label?: string;
+  version?: string;
+  status?: string;
+  supervisor?: string;
+};
+
+export type RemoteUpgradeDiscovery = {
+  /** Online orchestrators with a supported supervisor, excluding the local host — the upgrade AND verify set. */
+  targets: string[];
+  /** Registered orchestrators intentionally excluded from remote upgrade, with why. */
+  skipped: Array<{ id: string; reason: string }>;
+  /** Registry identities retained so later fleet gating can match operator-visible names. */
+  identities: ReadonlyMap<string, DiscoverableOrchestrator>;
+};
+
+/**
+ * Orchestrators `bun run release` remote-upgrades after publish (the local host
+ * upgrades via the humwick CLI directly; these go through the relay command
+ * bus — see `triggerRemoteUpgrade` in release.ts).
+ *
+ * Replaces the #1250 hardcoded `REMOTE_ORCHESTRATORS` fleet list (#1252): a
+ * shipped, deployment-specific hostname list is wrong for any other humwick
+ * deployment's fleet, either stranding their hosts or failing their releases.
+ * The target set is now DERIVED from the relay's live orchestrator registry
+ * (`GET /api/orchestrators`, the same source #1250 used to confirm each host's
+ * supervisor) instead of hardcoded — so it auto-adapts as hosts join/leave.
+ *
+ * The returned `targets` set is exactly what the deploy verification must check
+ * (the #1250 invariant, now derived not hardcoded): a host that's verified but
+ * never made an upgrade target would be permanently stranded and fail every
+ * release (verification treats an online, un-upgraded host as a genuine
+ * mismatch, not a skip — see `planOrchestratorVersionVerification`).
+ *
+ * A host qualifies as a target when it's `online` AND its registered
+ * `supervisor` is one the remote self-upgrade command bus can drive (systemd or
+ * launchd). Everything else — offline, or an unsupported/unknown supervisor —
+ * is skipped with a reason, never a hard failure: it can't be auto-upgraded by
+ * design, not a release blocker. The local host is always excluded; it upgrades
+ * directly via the installed CLI, not through the relay command bus.
+ */
+export function discoverRemoteUpgradeTargets(
+  orchestrators: DiscoverableOrchestrator[],
+  localOrchestratorId: string,
+): RemoteUpgradeDiscovery {
+  const targets: string[] = [];
+  const skipped: Array<{ id: string; reason: string }> = [];
+  for (const orch of orchestrators) {
+    if (orch.id === localOrchestratorId) continue;
+    if (orch.status !== "online") {
+      skipped.push({ id: orch.id, reason: `offline (status: ${orch.status ?? "unknown"})` });
+      continue;
+    }
+    if (!orch.supervisor || !SUPPORTED_SELF_UPGRADE_SUPERVISORS.has(orch.supervisor)) {
+      skipped.push({ id: orch.id, reason: `unsupported supervisor: ${orch.supervisor ?? "unknown"}` });
+      continue;
+    }
+    targets.push(orch.id);
+  }
+  return {
+    targets,
+    skipped,
+    identities: new Map(orchestrators.map((orchestrator) => [orchestrator.id, orchestrator])),
+  };
+}
+
+/**
+ * Fail-closed CD fleet gate for the autonomous watcher path (#1303). Applied
+ * BETWEEN discovery and reporting, never inside `discoverRemoteUpgradeTargets`
+ * itself — a human running `bun run release` still targets every online,
+ * supported-supervisor host (they see the host list before anything happens);
+ * only the watcher's unattended trigger needs this narrower, explicit set.
+ *
+ * `allowlist` is `undefined` (unconfigured) or empty → every discovered target
+ * is demoted to skipped with a reason; the watcher deploys to nothing rather
+ * than silently falling back to "everything supervised" (the wbox bug this
+ * closes: a Win11/WSL2 dev box registers as `supervisor=systemd` and would
+ * otherwise qualify). Otherwise, only discovered targets whose id, current
+ * hostname, or label is present in the allowlist survive — everything else
+ * joins the skip list with its own reason, distinguishable from an ordinary
+ * discovery skip (offline / unsupported supervisor) for operators reading the
+ * release log.
+ *
+ * Discovery retains the registry identity behind each id so allowlist entries
+ * can match the current hostname (or label) after a repin.
+ */
+export function gateFleetTargets(
+  discovery: RemoteUpgradeDiscovery,
+  allowlist: Set<string> | undefined,
+  targetIds?: Set<string>,
+): RemoteUpgradeDiscovery {
+  if (!allowlist || allowlist.size === 0) {
+    return {
+      identities: discovery.identities,
+      targets: [],
+      skipped: [
+        ...discovery.targets.map((id) => ({
+          id,
+          reason: "CD fleet allowlist not configured (AGENT_RELAY_CD_FLEET) — autonomous deploy refuses to target any host",
+        })),
+        ...discovery.skipped,
+      ],
+    };
+  }
+  const isAllowlisted = (id: string): boolean =>
+    isCdFleetHostAllowlisted(discovery.identities.get(id) ?? { id }, allowlist);
+  const allowlistedTargets = discovery.targets.filter(isAllowlisted);
+  const targets = targetIds ? allowlistedTargets.filter((id) => targetIds.has(id)) : allowlistedTargets;
+  const skipped = [
+    ...discovery.skipped,
+    ...discovery.targets
+      .filter((id) => !isAllowlisted(id))
+      .map((id) => ({ id, reason: "not in CD fleet allowlist (AGENT_RELAY_CD_FLEET)" })),
+    ...(targetIds ? allowlistedTargets
+      .filter((id) => !targetIds.has(id))
+      .map((id) => ({ id, reason: "already attests the CD target artifact" })) : []),
+  ];
+  return { targets, skipped, identities: discovery.identities };
+}
+
+/**
+ * Discovery + deploy-log reporting in one step (#1252): resolves the remote
+ * upgrade target set via `discoverRemoteUpgradeTargets` and narrates the
+ * outcome (skips with reasons, then the discovered set) through the release
+ * script's logger. Returns the targets with their pre-upgrade versions so the
+ * deploy status table can seed `beforeVersion` without re-scanning the registry.
+ *
+ * `fleetGate`, when passed, additionally intersects discovery against an
+ * explicit allowlist via `gateFleetTargets` (#1303) — the caller opts in only
+ * for the autonomous watcher-triggered deploy path; a manual `bun run release`
+ * omits it and keeps the pre-#1303 "every online supported host" behavior.
+ */
+export function reportRemoteUpgradeTargets(
+  orchestrators: DiscoverableOrchestrator[],
+  localOrchestratorId: string,
+  log: (line: string) => void,
+  opts: { fleetGate?: { allowlist: Set<string> | undefined; targetIds?: Set<string> } } = {},
+): Array<{ id: string; beforeVersion?: string }> {
+  const discovery = discoverRemoteUpgradeTargets(orchestrators, localOrchestratorId);
+  const { targets, skipped } = opts.fleetGate
+    ? gateFleetTargets(discovery, opts.fleetGate.allowlist, opts.fleetGate.targetIds)
+    : discovery;
+  for (const ambiguity of resolveCdFleetAllowlistAmbiguities(orchestrators, opts.fleetGate?.allowlist)) {
+    log(`  ⚠ WARNING: AGENT_RELAY_CD_FLEET entry "${ambiguity.entry}" matches multiple online orchestrators: ${ambiguity.hostIds.join(", ")}. Prefer immutable orchestrator ids to disambiguate.`);
+  }
+  for (const skip of skipped) {
+    log(`  ⚠ skipping ${skip.id} for remote upgrade: ${skip.reason}`);
+  }
+  log(`→ discovered remote upgrade targets: ${targets.length ? targets.join(", ") : "(none)"}`);
+  return targets.map((id) => ({ id, beforeVersion: orchestrators.find((orch) => orch.id === id)?.version }));
+}
+
+/**
+ * Max wall-clock ms a `triggerRemoteUpgrade` call is allowed to block before the
+ * release script kills the child and dies with a clear message (#801).
+ *
+ * Without this, `humwick upgrade --host <id>` can block forever when the remote
+ * host's npm install retries endlessly on E404 for an unpublished workspace dep.
+ * Five minutes is generous for a normal upgrade; anything beyond that is a stuck install.
+ */
+export const REMOTE_UPGRADE_TIMEOUT_MS = 5 * 60 * 1000;
+export const LOCAL_UPGRADE_TIMEOUT_MS = 5 * 60 * 1000;
+
+/**
+ * The plain-relay systemd unit — created by `humwick daemon install`
+ * (src/daemon.ts `DEFAULT_DAEMON_NAME = "agent-relay"` → `${name}.service`). This is
+ * only ONE of the two units that can supervise the local relay; the deploy must pick
+ * the ACTUAL installed one via {@link resolveLocalRelayUnit}, never assume this name
+ * (#1442 — the old hardcoded `LOCAL_RELAY_SERVICE = "agent-relay.service"` restart
+ * exited 5 on orchestrator-install hosts, whose relay runs inside the unit below).
+ */
+export const PLAIN_RELAY_SERVICE = "agent-relay.service";
+
+/**
+ * The orchestrator systemd unit — created by `humwick orchestrator install`
+ * (src/cli/orchestrator.ts default `serviceName = "agent-relay-orchestrator"`). On an
+ * orchestrator-install host the relay server runs INSIDE this unit; there is no
+ * separate agent-relay.service.
+ */
+export const LOCAL_ORCHESTRATOR_SERVICE = "agent-relay-orchestrator.service";
+
+/**
+ * The plain-relay launchd label — the macOS analogue of {@link PLAIN_RELAY_SERVICE}
+ * (#1446). Matches src/daemon.ts's label scheme (`kind === "launchd" ? \`dev.agent-relay.${name}\`
+ * : name`) with `DEFAULT_DAEMON_NAME = "agent-relay"`.
+ */
+export const PLAIN_RELAY_LAUNCHD_LABEL = "dev.agent-relay.agent-relay";
+
+/**
+ * The orchestrator launchd label — the macOS analogue of {@link LOCAL_ORCHESTRATOR_SERVICE}
+ * (#1446), created by `humwick orchestrator install` on darwin. Matches
+ * src/cli/orchestrator.ts's default `serviceName = "agent-relay-orchestrator"` through
+ * src/daemon.ts's `dev.agent-relay.${name}` label scheme.
+ */
+export const LOCAL_ORCHESTRATOR_LAUNCHD_LABEL = "dev.agent-relay.agent-relay-orchestrator";
+
+/**
+ * Per-unit systemd LoadState classification, derived from
+ * `systemctl [--user] show <unit> --property=LoadState`:
+ * - `loaded`  → the unit exists and is known to systemd (`LoadState=loaded`).
+ * - `absent`  → systemd DEFINITIVELY does not know the unit (`LoadState=not-found`).
+ * - `error`   → the probe could not determine the state (timeout, D-Bus error, masked,
+ *               non-zero exit, unexpected output). This is NOT "absent" — it must fail
+ *               loud, never silently downgrade to a topology guess (#1442 P1).
+ */
+export type UnitLoadProbe =
+  | { kind: "loaded" }
+  | { kind: "absent" }
+  | { kind: "error"; detail: string };
+
+/** The subset of a shell result the LoadState probe needs (matches release.ts `sh`). */
+export type SystemctlShowResult = { code: number | null; out: string; err?: string; signal?: string | null };
+
+/** Extract the `LoadState=` value from `systemctl show` key=value output (or undefined). */
+export function parseSystemdLoadState(showOutput: string): string | undefined {
+  for (const line of showOutput.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("LoadState=")) return trimmed.slice("LoadState=".length);
+  }
+  return undefined;
+}
+
+/**
+ * Classify a `systemctl show <unit> --property=LoadState` invocation into a
+ * {@link UnitLoadProbe}. A loaded unit prints `LoadState=loaded` and a genuinely-unknown
+ * one prints `LoadState=not-found` (both exit 0), so the LoadState line — not the exit
+ * code — distinguishes existence. But a probe that TIMED OUT, exited non-zero, or
+ * returned an unexpected/masked state is NOT evidence of absence: it is an `error` the
+ * caller must surface, so a transient systemctl failure can never masquerade as
+ * "orchestrator topology" and skip the relay restart (#1442 P1 fail-loud).
+ */
+export function classifyUnitLoadProbe(result: SystemctlShowResult): UnitLoadProbe {
+  if (isShellTimeoutSignal(result.signal)) {
+    return { kind: "error", detail: "systemctl show timed out" };
+  }
+  if (result.code !== 0) {
+    const stderr = result.err?.trim();
+    return { kind: "error", detail: `systemctl show exited ${result.code ?? "null"}${stderr ? `: ${stderr}` : ""}` };
+  }
+  const loadState = parseSystemdLoadState(result.out);
+  if (loadState === "loaded") return { kind: "loaded" };
+  if (loadState === "not-found") return { kind: "absent" };
+  return { kind: "error", detail: `unexpected LoadState=${loadState ?? "(missing)"}` };
+}
+
+/** The subset of a shell result the launchd probe needs (matches release.ts `sh`). */
+export type LaunchctlPrintResult = { code: number | null; out: string; err?: string; signal?: string | null };
+
+/**
+ * Classify a `launchctl print gui/<uid>/<label>` invocation into a {@link UnitLoadProbe}
+ * — the launchd (macOS) analogue of {@link classifyUnitLoadProbe} (#1446). Unlike
+ * `systemctl show`, `launchctl print` does not distinguish loaded/not-found by an exit-0
+ * property line: it exits 0 with job details when loaded, and NON-ZERO with a
+ * "Could not find service …" (or equivalent, macOS-version-dependent) message when the
+ * label is not loaded. Reuses {@link isBenignBootoutFailure}'s message matching (already
+ * hardened across macOS versions for the identical not-loaded signal on `launchctl
+ * bootout`) so this probe and that uninstall path never diverge on what "not loaded"
+ * looks like. A timeout or any OTHER non-zero exit (permission, I/O, unexpected message)
+ * is NOT evidence of absence: it is an `error` the caller must surface, so a transient
+ * launchctl failure can never masquerade as "unit not installed" and silently skip the
+ * restart (mirrors #1442 P1 fail-loud for the systemd probe).
+ */
+export function classifyLaunchdPrintProbe(result: LaunchctlPrintResult): UnitLoadProbe {
+  if (isShellTimeoutSignal(result.signal)) {
+    return { kind: "error", detail: "launchctl print timed out" };
+  }
+  if (result.code === 0) return { kind: "loaded" };
+  const detail = (result.err?.trim() || result.out?.trim() || "");
+  if (isBenignBootoutFailure(detail)) return { kind: "absent" };
+  return { kind: "error", detail: `launchctl print exited ${result.code ?? "null"}${detail ? `: ${detail}` : ""}` };
+}
+
+/**
+ * Tri-state resolution of which systemd unit supervises the LOCAL relay (#1442):
+ * - `restart`            → a real, loaded agent-relay.service (plain-relay host): restart
+ *                          it mid-deploy as before.
+ * - `skip-orchestrator`  → agent-relay.service is DEFINITIVELY not-found AND the
+ *                          orchestrator unit is loaded (orchestrator-install host, relay
+ *                          runs inside it): SKIP the mid-deploy restart; the orchestrator
+ *                          is restarted exactly once later (deferred end / CD repair) and
+ *                          the deploy driver has been re-exec'd out of its cgroup
+ *                          (release-decouple) so it survives.
+ * - `indeterminate`      → any probe error, or neither unit loaded: FAIL LOUD. Never
+ *                          silently skip — a probe timeout on a plain host would otherwise
+ *                          leave the live relay stale while the deploy reports success.
+ */
+export type LocalRelayUnitResolution =
+  | { kind: "restart"; unit: string }
+  | { kind: "skip-orchestrator"; unit: string }
+  | { kind: "indeterminate"; reason: string };
+
+/**
+ * Resolve the local relay unit by probing the live system, so install and upgrade-deploy
+ * naming cannot diverge (#1442). Probes BOTH units; fail-closed on a genuine probe error or
+ * a total absence, but a LOADED plain relay unit is always the daemon to restart. Truth table
+ * (#1484 — a loaded agent-relay.service is unambiguously the relay daemon, EVEN when the
+ * orchestrator unit is also loaded on the same host):
+ * - relay loaded (orchestrator absent OR loaded) → `restart` agent-relay.service. Co-location
+ *   is the relay-HOST topology (a standalone relay daemon unit AND an orchestrator unit on one
+ *   box, e.g. macmini2), NOT ambiguity: agent-relay.service unambiguously supervises the daemon,
+ *   and the orchestrator has its OWN restart lifecycle (restartLocalOrchestratorForCdRepair on
+ *   the CD path / the deferred end-of-run finalize on the human path), so it is not this
+ *   resolver's concern. The deploy driver is decoupled into a sibling systemd scope
+ *   (release-decouple), so bouncing the plain unit never self-terminates the running deploy.
+ * - relay absent + orchestrator loaded → `skip-orchestrator` (relay runs inside the orchestrator)
+ * - any probe error (timeout/D-Bus/masked/unexpected) on either → `indeterminate` (fail loud)
+ * - neither loaded                      → `indeterminate`
+ * `probe(unit)` classifies a single unit's LoadState (see {@link classifyUnitLoadProbe}).
+ *
+ * HISTORY (#1484): #1442 round 3 (commit 4ef15f11) made BOTH-loaded an `indeterminate` abort on
+ * the theory that two loaded units were ambiguous. That took the CD pipeline fully red on the
+ * relay host (macmini2), which legitimately runs the daemon and orchestrator as separate units:
+ * every self-deploy aborted here after the runtime symlink was already swapped, stranding the
+ * daemon ahead of the orchestrator. Co-location is a supported topology, not ambiguity.
+ *
+ * `orchestratorUnit` defaults to {@link LOCAL_ORCHESTRATOR_SERVICE} but MUST be overridden
+ * with the actual installed unit name when the orchestrator was installed with a custom
+ * `--service-name` (#1444): probing the hardcoded default against a custom-named unit finds
+ * it definitively absent, so the resolver falls through to `indeterminate` and fails the
+ * deploy loud — safe, but wrong, since the custom unit is in fact loaded and restartable.
+ *
+ * `plainUnit` defaults to {@link PLAIN_RELAY_SERVICE} but MUST be overridden with the
+ * launchd label ({@link PLAIN_RELAY_LAUNCHD_LABEL}) on macOS (#1446) — `probe`/`unit` are
+ * opaque identifiers to this function, so a launchd caller passes launchd labels for BOTH
+ * `plainUnit` and `orchestratorUnit` and a `probe` that shells to `launchctl print`
+ * instead of `systemctl show`; the truth table below is unchanged either way.
+ */
+export function resolveLocalRelayUnit(
+  probe: (unit: string) => UnitLoadProbe,
+  orchestratorUnit: string = LOCAL_ORCHESTRATOR_SERVICE,
+  plainUnit: string = PLAIN_RELAY_SERVICE,
+): LocalRelayUnitResolution {
+  // Probe BOTH units before deciding: a loaded plain relay unit means `restart` regardless of
+  // the orchestrator (#1484 — co-located daemon+orchestrator is the relay-HOST topology, not
+  // ambiguity), but we still probe the orchestrator so a probe ERROR on it fails closed.
+  const relay = probe(plainUnit);
+  const orchestrator = probe(orchestratorUnit);
+  if (relay.kind === "error") {
+    return { kind: "indeterminate", reason: `could not determine ${plainUnit} state: ${relay.detail}` };
+  }
+  if (orchestrator.kind === "error") {
+    return { kind: "indeterminate", reason: `could not determine ${orchestratorUnit} state: ${orchestrator.detail}` };
+  }
+  const relayLoaded = relay.kind === "loaded";
+  const orchestratorLoaded = orchestrator.kind === "loaded";
+  // #1484: a loaded agent-relay.service is the relay daemon to restart even when the orchestrator
+  // unit is ALSO loaded (the relay host runs both as separate units). The orchestrator's own
+  // restart is handled elsewhere (restartLocalOrchestratorForCdRepair / end-of-run finalize).
+  if (relayLoaded) return { kind: "restart", unit: plainUnit };
+  if (orchestratorLoaded) return { kind: "skip-orchestrator", unit: orchestratorUnit };
+  return { kind: "indeterminate", reason: `neither ${plainUnit} nor ${orchestratorUnit} is loaded — cannot identify the local relay unit to restart` };
+}
+
+/**
+ * What {@link performLocalRelayRestart} actually did, so the caller can report an HONEST
+ * local host status (#1448):
+ * - `restarted` → the real agent-relay.service (plain-relay host) was restarted in place;
+ *                 the running relay process IS the new version, so "upgraded" is truthful.
+ * - `deferred`  → the relay runs inside the orchestrator unit (orchestrator-install host);
+ *                 the mid-deploy restart was SKIPPED. The new runtime is only STAGED — the
+ *                 running process is still the old one until the caller performs the verified
+ *                 orchestrator restart. The caller MUST NOT report "upgraded" off staged files
+ *                 (the #1448 false-success bug): it upgrades only once the restart attests.
+ */
+export type LocalRelayRestartOutcome =
+  | { kind: "restarted"; unit: string }
+  | { kind: "deferred"; unit: string };
+
+/**
+ * Perform the mid-deploy local relay restart against the ACTUAL installed unit (#1442).
+ * Extracted from release.ts so the tri-state skip/restart/fail-loud decision is
+ * unit-testable without shelling out to systemd.
+ *
+ * Behaviour (driven by {@link resolveLocalRelayUnit}):
+ * - `indeterminate`     → FAIL the deploy loud. A probe error or both-units-absent must
+ *                         never be silently downgraded to a skip: on a plain host that
+ *                         would leave the live relay stale while the deploy reports
+ *                         success (#1442 P1).
+ * - `skip-orchestrator` → SKIP the restart and log why, returning `deferred`. That unit
+ *                         supervises this deploy driver; it is restarted exactly once by
+ *                         the caller (before the remote fan-out, #1448) once the driver has
+ *                         been decoupled from its cgroup so it survives (#1310 / #1442 P0).
+ * - `restart`           → restart the real unit, rolling back only after its job has settled.
+ *
+ * `failDeploy` must not return (it aborts the release); it also owns marking the local
+ * host status failed.
+ */
+export function performLocalRelayRestart(opts: {
+  probe: (unit: string) => UnitLoadProbe;
+  restart: (unit: string) => import("./release-restart.ts").LocalRestartResult;
+  rollbackAndRetry?: (unit: string) => void;
+  log: (msg: string) => void;
+  failDeploy: (reason: string) => never;
+  /** Actual installed orchestrator unit name (#1444) — defaults to {@link LOCAL_ORCHESTRATOR_SERVICE}. */
+  orchestratorUnit?: string;
+  /** Plain-relay unit/label (#1446) — defaults to {@link PLAIN_RELAY_SERVICE}; pass the launchd label on macOS. */
+  plainUnit?: string;
+}): LocalRelayRestartOutcome {
+  const resolution = resolveLocalRelayUnit(opts.probe, opts.orchestratorUnit, opts.plainUnit);
+  if (resolution.kind === "indeterminate") {
+    opts.failDeploy(`cannot resolve the local relay unit to restart — refusing to finalize the deploy (would risk leaving the running relay stale): ${resolution.reason}`);
+  }
+  if (resolution.kind === "skip-orchestrator") {
+    opts.log(`  ↻ local relay runs inside ${resolution.unit} (orchestrator-install host) — deferring its restart to the verified orchestrator restart before the remote fan-out, so it survives the deploy driver and is not hostage to remote outcomes (#1448)`);
+    return { kind: "deferred", unit: resolution.unit };
+  }
+  const restart = opts.restart(resolution.unit);
+  if (restart.code === 0) return { kind: "restarted", unit: resolution.unit };
+  // Restore the previously-running runtime before failing, so a bad swap does not leave
+  // the relay down (best-effort — the deploy fails regardless, reported below).
+  if (restart.rollbackSafe ?? !isShellTimeoutSignal(restart.signal)) opts.rollbackAndRetry?.(resolution.unit);
+  opts.failDeploy(
+    restart.reason ?? (isShellTimeoutSignal(restart.signal)
+      ? `${resolution.unit} restart timed out${restart.timeoutMs ? ` after ${restart.timeoutMs / 1000}s` : ""}`
+      : `${resolution.unit} restart exited ${restart.code}`)
+  );
+}
+
+export type ReleaseDeployPhase =
+  | "local-runtime-install"
+  | "local-relay-restart"
+  | "local-relay-health"
+  | "remote-upgrade"
+  | "remote-version-verify"
+  | "remote-health"
+  | "dashboard-deploy"
+  | "local-orchestrator-reconnect"
+  | "live-spawn-smoke"
+  | "success-report"
+  | "local-orchestrator-restart";
+
+export function planReleaseCompletionPhases(opts: { smoke: boolean }): ReleaseDeployPhase[] {
+  return [
+    "local-runtime-install",
+    "local-relay-restart",
+    "local-relay-health",
+    "remote-upgrade",
+    "remote-version-verify",
+    "remote-health",
+    "dashboard-deploy",
+    ...(opts.smoke ? ["local-orchestrator-reconnect" as const, "live-spawn-smoke" as const] : []),
+    "success-report",
+    "local-orchestrator-restart",
+  ];
+}
+
+export type DetachedRestartInvocation = {
+  cmd: string;
+  args: string[];
+  detached: true;
+  stdio: "ignore";
+};
+
+export function detachedSystemdRestartInvocation(
+  services: string[],
+  opts: { unitName: string; description?: string },
+): DetachedRestartInvocation {
+  return {
+    cmd: "systemd-run",
+    args: [
+      "--user",
+      "--collect",
+      `--unit=${opts.unitName}`,
+      ...(opts.description ? [`--description=${opts.description}`] : []),
+      "systemctl",
+      "--user",
+      "restart",
+      ...services,
+    ],
+    detached: true,
+    stdio: "ignore",
+  };
+}
+
+/**
+ * The launchd (macOS) analogue of {@link detachedSystemdRestartInvocation} (#1446).
+ * `systemd-run --collect` schedules the restart as an independent transient unit so it
+ * survives the caller's own teardown; launchd has no equivalent transient-unit primitive,
+ * so — mirroring orchestrator/src/self-upgrade.ts's already-proven launchd restart path —
+ * this backgrounds a plain shell command (`… &`) instead: `launchctl kickstart -kp` both
+ * kills the running job and restarts it, and the trailing `&` detaches it from this
+ * process's shell so it keeps running after the caller (and its own supervised job) exits.
+ */
+export function detachedLaunchdRestartInvocation(label: string, opts: { uid: number }): DetachedRestartInvocation {
+  return {
+    cmd: "/bin/sh",
+    args: ["-c", `launchctl kickstart -kp ${shellQuote(`gui/${opts.uid}/${label}`)} &`],
+    detached: true,
+    stdio: "ignore",
+  };
+}
+
+export type PublishCompleteSigtermReport = {
+  arm(): void;
+  disarm(): void;
+};
+
+export function installPublishCompleteSigtermReport(opts: {
+  writeStderr?: (text: string) => void;
+  exit?: (code: number) => never;
+} = {}): PublishCompleteSigtermReport {
+  let armed = false;
+  process.once("SIGTERM", () => {
+    if (armed) {
+      (opts.writeStderr ?? ((text) => console.error(text)))(
+        "\n⚠ publish complete, deploy interrupted by local restart — run `bun run release --resume` to finish remote deploy + smoke.",
+      );
+    }
+    (opts.exit ?? ((code) => process.exit(code)))(143);
+  });
+  return {
+    arm() { armed = true; },
+    disarm() { armed = false; },
+  };
+}
+
+type DetachedSpawner = (
+  cmd: string,
+  args: string[],
+  opts: { detached: true; stdio: "ignore" },
+) => { unref(): void };
+
+export function runLiveSpawnSmoke(opts: {
+  noSmoke: boolean;
+  verbose: boolean;
+  log: (msg: string) => void;
+  sh: (cmd: string, args: string[], opts?: { inherit?: boolean }) => { code: number; out?: string; err?: string };
+}): number {
+  if (opts.noSmoke) {
+    opts.log("\n(--no-smoke — skipping live-spawn smoke.)");
+    return 0;
+  }
+  const failMessage = (code: number) =>
+    `shipped successfully — live-spawn smoke FAILED, investigate/fix-forward (exit ${code}). Release publish/deploy already completed.`;
+  if (opts.verbose) {
+    opts.log("\n→ live-spawn smoke …");
+    const r = opts.sh("bun", ["run", "smoke:spawn"], { inherit: true });
+    if (r.code !== 0) {
+      console.error(`\n⚠ ${failMessage(r.code)}`);
+      return r.code || 1;
+    }
+    opts.log("✓ live-spawn smoke passed");
+    return 0;
+  }
+  process.stdout.write("\n→ live-spawn smoke … ");
+  const started = Date.now();
+  const r = opts.sh("bun", ["run", "smoke:spawn"]);
+  const secs = ((Date.now() - started) / 1000).toFixed(1);
+  if (r.code === 0) {
+    process.stdout.write(`✓ (${secs}s)\n`);
+    return 0;
+  }
+  process.stdout.write(`⚠ (${secs}s) ${failMessage(r.code)}\n`);
+  if (r.out) process.stdout.write(`${r.out}\n`);
+  if (r.err) process.stderr.write(`${r.err}\n`);
+  return r.code || 1;
+}
+
+export async function finishReleaseWithLocalOrchestratorRestart(opts: {
+  versionLabel: string;
+  successMessage: string;
+  noSmoke: boolean;
+  runSmoke: () => number;
+  durableCompletionReport?: (input: DurableReleaseCompletionInput) => Promise<DurableReleaseCompletionResult>;
+  log: (msg: string) => void;
+  die: (msg: string) => never;
+  spawn: DetachedSpawner;
+  sigtermReport?: PublishCompleteSigtermReport;
+  restartLocalOrchestrator?: boolean;
+  exit?: (code: number) => never;
+  /** Actual installed orchestrator unit name (#1444) — defaults to {@link LOCAL_ORCHESTRATOR_SERVICE}. */
+  orchestratorUnit?: string;
+  /** Local supervisor (#1446) — defaults to "systemd"; pass "launchd" on macOS. */
+  supervisor?: "systemd" | "launchd";
+  /** launchd only: the gui domain uid (`process.getuid()`); ignored under systemd. */
+  uid?: number;
+}): Promise<void> {
+  const phases = planReleaseCompletionPhases({ smoke: !opts.noSmoke });
+  const reportIndex = phases.indexOf("success-report");
+  const restartIndex = phases.indexOf("local-orchestrator-restart");
+  if (reportIndex === -1 || restartIndex === -1 || reportIndex >= restartIndex) {
+    opts.die("invalid release phase plan: local orchestrator restart must be final");
+  }
+  const smokeCode = opts.runSmoke();
+  const reportMessage = smokeCode === 0
+    ? opts.successMessage
+    : `\n⚠ ${opts.versionLabel} published/deployed, but live-spawn smoke failed (exit ${smokeCode}).`;
+  opts.log(reportMessage);
+  const durableReport = await opts.durableCompletionReport?.({
+    versionLabel: opts.versionLabel,
+    successMessage: opts.successMessage,
+    smokeCode,
+  });
+  if (durableReport?.status === "failed") {
+    opts.die(`release completed, but durable completion report failed before final orchestrator restart: ${durableReport.reason}`);
+  }
+  if (durableReport?.status === "sent") {
+    opts.log(`✓ durable release completion report queued to ${durableReport.to}${durableReport.messageId ? ` (#${durableReport.messageId})` : ""}`);
+  }
+  opts.sigtermReport?.disarm();
+  if (opts.restartLocalOrchestrator === false) {
+    // Either the local host isn't a deploy target, or it was already restarted + attested
+    // earlier in the run (the #1448 verified finalize before the remote fan-out).
+    opts.log("→ local orchestrator restart not needed here (already restarted, or not a deploy target)");
+    if (smokeCode !== 0) (opts.exit ?? ((code) => process.exit(code)))(smokeCode);
+    return;
+  }
+  const orchestratorUnit = opts.orchestratorUnit ?? LOCAL_ORCHESTRATOR_SERVICE;
+  const supervisor = opts.supervisor ?? "systemd";
+  const unitName = `agent-relay-release-orchestrator-restart-${process.pid}`;
+  const invocation = supervisor === "launchd"
+    ? detachedLaunchdRestartInvocation(orchestratorUnit, { uid: opts.uid ?? 0 })
+    : detachedSystemdRestartInvocation([orchestratorUnit], {
+        unitName,
+        description: `Agent Relay ${opts.versionLabel} final local orchestrator restart`,
+      });
+  try {
+    opts.spawn(invocation.cmd, invocation.args, { detached: invocation.detached, stdio: invocation.stdio }).unref();
+    opts.log(`→ scheduled detached local orchestrator restart (${unitName})`);
+  } catch (error) {
+    console.error(`\n⚠ could not schedule detached local orchestrator restart: ${errMessage(error)}`);
+    console.error(`  Run manually: ${supervisor === "launchd" ? `launchctl kickstart -kp gui/${opts.uid ?? 0}/${orchestratorUnit}` : `systemctl --user restart ${orchestratorUnit}`}`);
+  }
+  if (smokeCode !== 0) (opts.exit ?? ((code) => process.exit(code)))(smokeCode);
+}
+
+export type DeployHostStatusKind = "pending" | "upgraded" | "skipped" | "failed" | "timed-out";
+
+export type DeployHostStatus = {
+  id: string;
+  status: DeployHostStatusKind;
+  beforeVersion?: string;
+  afterVersion?: string;
+  reason?: string;
+};
+
+export function formatDeployHostStatus(host: DeployHostStatus, targetVersion: string): string {
+  const before = host.beforeVersion ?? "unknown";
+  const after = host.afterVersion ?? targetVersion;
+  const reason = host.reason ? ` (${host.reason})` : "";
+  if (host.status === "upgraded") return `${host.id}: upgraded ${before}→${after}${reason}`;
+  if (host.status === "skipped") return `${host.id}: skipped${reason}`;
+  if (host.status === "failed") return `${host.id}: failed(${host.reason ?? "unknown"})`;
+  if (host.status === "timed-out") return `${host.id}: timed-out${reason}`;
+  return `${host.id}: pending${reason}`;
+}
+
+/**
+ * Did a `spawnSync` call with a `timeout` set actually time out?
+ * Node.js kills the child with the `killSignal` (default SIGTERM) when the timeout
+ * fires, then returns with `status: null` and `signal: "SIGTERM"`.
+ */
+export function isShellTimeoutSignal(signal: string | null | undefined): boolean {
+  return signal === "SIGTERM";
+}
+
+/**
+ * Coerce a `GET /api/orchestrators` response into an orchestrator array WITHOUT ever throwing
+ * (#1448 review-B). A non-ok response, an unreachable relay (`null`), a malformed/truncated
+ * 200 body (`res.json()` rejects), or a non-array payload all degrade to `[]` — the same as
+ * "no fleet discovered". Before this, a bad 200 body threw out of the deploy and bypassed the
+ * local-orchestrator finalize, re-stranding the local host staged-but-old. The bounded verify
+ * loop re-polls, and the local finalize still runs.
+ */
+export async function parseOrchestratorListResponse<T>(
+  res: { ok: boolean; json: () => Promise<unknown> } | null,
+): Promise<T[]> {
+  if (!res?.ok) return [];
+  try {
+    const body = await res.json();
+    return Array.isArray(body) ? (body as T[]) : [];
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Is `value` a URL `new URL(value)` can parse (#1448 review-B)? An orchestrator-reported
+ * `apiUrl` is attacker/bug-influenced; feeding a malformed one to the health probe's
+ * `new URL("/api/health", apiUrl)` THROWS past the deploy's finalize handler. Validate first.
+ */
+export function isParseableUrl(value: string): boolean {
+  try { new URL(value); return true; } catch { return false; }
+}
+
+/**
+ * Deterministic version probe (#322). The restart-flag decision must reflect the
+ * binary that will *actually* run the upgrade — NOT whatever `humwick` PATH
+ * happens to surface. `bun run release` PREPENDS `~/.bun/bin` to PATH, so a stale
+ * `~/.bun/bin/agent-relay` symlink (a Jun-1 leftover sat at 0.10.25) would win over
+ * the canonical CLI and make the probe report an ancient version → degrade to
+ * `--no-restart` for no reason, every release.
+ *
+ * The canonical managed `humwick` shim (`~/.local/bin/agent-relay`) execs
+ * `bun ~/.agent-relay/runtime/node_modules/agent-relay-server/src/index.ts`, so the
+ * runtime install IS the version that runs the upgrade. Read its package.json
+ * directly (no subprocess, no PATH) for the probe, and pin deploy-critical
+ * invocations to the runtime-owned CLI/entry path.
+ */
+function runtimeServerDir(home: string): string {
+  const modules = join(home, ".agent-relay", "runtime", "node_modules");
+  // #3470 — a registry-installed npm runtime keeps the legacy package path until scoped publication.
+  const canonical = join(modules, "@humwick", "server");
+  return existsSync(canonical) ? canonical : join(modules, "agent-relay-server");
+}
+
+/** Installed runtime's server package.json — the deterministic probe source. */
+export function runtimeServerPackageJson(home: string): string {
+  return join(runtimeServerDir(home), "package.json");
+}
+
+/** Entry script the managed `humwick` shim execs (`bun <this> upgrade …`). */
+export function runtimeServerEntry(home: string): string {
+  return join(runtimeServerDir(home), "src", "index.ts");
+}
+
+/** Runtime-owned CLI shim; deploy-critical release invocations must prefer this over PATH. */
+export function runtimeHumwickBin(home: string): string {
+  return join(home, ".agent-relay", "runtime", "node_modules", ".bin", "agent-relay");
+}
+
+export function runtimeHumwickInvocation(
+  home: string,
+  args: string[],
+  runtime: { binExists: boolean; entryExists: boolean },
+): { cmd: string; args: string[] } {
+  if (runtime.binExists) return { cmd: runtimeHumwickBin(home), args };
+  if (runtime.entryExists) return { cmd: "bun", args: [runtimeServerEntry(home), ...args] };
+  return { cmd: "agent-relay", args };
+}
+
+type OrchestratorVersion = {
+  id: string;
+  hostname?: string;
+  label?: string;
+  version?: string;
+  status?: string;
+  gitSha?: string;
+  /** Reported local supervisor (#1413 fix #3) — a `launchd` host re-attests slowly (macmini). */
+  supervisor?: string;
+};
+type OrchestratorHealth = { status?: string; restartRequired?: boolean };
+type ReleaseOrchestrator = OrchestratorVersion & { health?: OrchestratorHealth };
+
+export type LocalOrchestratorReconnectSnapshot = {
+  url: string;
+  httpStatus?: number;
+  error?: string;
+  seenIds?: string[];
+  local?: ReleaseOrchestrator;
+  body?: unknown;
+};
+
+export type LocalOrchestratorReconnectResult = {
+  ok: boolean;
+  localOrchestratorId: string;
+  checks: number;
+  last: LocalOrchestratorReconnectSnapshot;
+};
+
+export type LocalOrchestratorReconnectOptions = {
+  timeoutMs?: number;
+  intervalMs?: number;
+  requestTimeoutMs?: number;
+  headers?: HeadersInit;
+  fetch?: FetchLike;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => number;
+  /** Require the restarted process's live git-sha attestation before success. */
+  expectedGitSha?: string;
+  /**
+   * Require the reconnected orchestrator to REPORT this version before success (#1448).
+   * The npm-reuse deploy path has no gitSha to attest, so "the restart actually loaded the
+   * new runtime" is proven by the registry version instead — otherwise a bounce that came
+   * back on the OLD version would falsely pass as an "upgraded" local host.
+   */
+  expectedVersion?: string;
+};
+
+export const DEFAULT_LOCAL_ORCHESTRATOR_RECONNECT_TIMEOUT_MS = 45_000;
+const DEFAULT_LOCAL_ORCHESTRATOR_RECONNECT_INTERVAL_MS = 2_000;
+
+/**
+ * #1413 fix #3 / #919 — re-attestation-driven host-version verification poll.
+ *
+ * A slow launchd re-attest (e.g. macmini) can take longer than the base poll budget. Cutting off
+ * at a fixed count spuriously fails the deploy and forces a churn retry that pointlessly re-bounces
+ * hosts. So keep polling PAST the base budget ONLY while a slow-supervisor (launchd) target is
+ * still un-attested AND the mismatch set is still SHRINKING (hosts are attesting, just slowly), up
+ * to a hard cap; give up once it stalls (`stallPolls` consecutive polls with no improvement).
+ *
+ * An all-systemd fleet — and any lag with NO launchd host pending — keeps the exact original
+ * behaviour (stop at `basePolls`), so the launchd widening is scoped strictly to the named slow case.
+ *
+ * #919 — the base budget (BASE_POLLS×5s = 150s) is also too short for a still-applying git-checkout
+ * SYSTEMD target. A git-checkout self-upgrade clones the artifact, installs deps, restarts the
+ * orchestrator and re-registers, which legitimately runs up to the relay's own in-flight window
+ * (`ORCH_UPGRADE_DEADLINE_MS` = 300s, src/routes/orchestrator.ts) — especially on the co-located
+ * control host, where the mid-deploy relay restart briefly drops the target's command long-poll so
+ * it picks the upgrade up late. That apply shows NO mismatch-shrink progress for minutes (and, when
+ * two commits share a release version, the relay's version-keyed upgrade-state settles uselessly),
+ * so the launchd shrink-stall heuristic can't observe it either. So a still-mismatched required
+ * remote target on a git-checkout deploy (`artifactPending`) earns the FULL extended budget up to
+ * `maxPolls` (MAX_POLLS×5s = 450s, sized to exceed the relay's 300s window plus restart/re-register
+ * slack) instead of a premature "reports gitSha <old>, expected <new>" false-failure while it is
+ * still legitimately applying. The `maxPolls` hard cap still bounds a genuinely-stuck host.
+ */
+export function shouldContinueAttestationPoll(state: {
+  /** Polls completed so far. */
+  polls: number;
+  /** Minimum polls before a stall can end the loop (the original fixed budget). */
+  basePolls: number;
+  /** Hard cap — never poll beyond this many, however much progress is being made. */
+  maxPolls: number;
+  /** A `launchd` (slow-re-attest) target is still mismatched this poll. */
+  launchdPending: boolean;
+  /**
+   * #919: a still-mismatched required remote target on a git-checkout (artifact) deploy — a
+   * self-upgrade that clones, installs, restarts and re-registers, legitimately taking up to the
+   * relay's ORCH_UPGRADE_DEADLINE_MS window. Earns the full extended budget up to `maxPolls`.
+   */
+  artifactPending: boolean;
+  /** Consecutive polls since the mismatch count last improved. */
+  pollsSinceImprovement: number;
+  /** Stop extending after this many no-improvement polls past `basePolls`. */
+  stallPolls: number;
+}): boolean {
+  if (state.polls >= state.maxPolls) return false;
+  if (state.polls < state.basePolls) return true;
+  // #919: a git-checkout target still applying earns the full extended budget up to maxPolls —
+  // its apply shows no shrink-progress for minutes, so the launchd stall heuristic can't gate it;
+  // the maxPolls cap (sized past the relay's own upgrade deadline) is the backstop.
+  if (state.artifactPending) return true;
+  // Past the base budget: only a still-pending, still-progressing launchd host earns more time.
+  if (!state.launchdPending) return false;
+  return state.pollsSinceImprovement < state.stallPolls;
+}
+
+export type OrchestratorVersionVerification = {
+  /** Online (or required) hosts still off the target version — a loud release failure. */
+  mismatches: OrchestratorVersion[];
+  /** Hosts deliberately not participating in this verification pass — skipped, not a failure. */
+  skipped: OrchestratorVersion[];
+};
+
+export type OrchestratorVersionVerificationOptions = {
+  /**
+   * Hosts whose restart is intentionally deferred until after smoke/report.
+   * Their pre-restart version is expected to lag during this pass.
+   */
+  deferredRestartIds?: string[];
+  /**
+   * The CD fleet allowlist (#1308): when set, only hosts whose id, current
+   * hostname, or label is in this set — plus the explicit `requiredIds` deploy
+   * targets and the deferred-local host — are asserted to reach
+   * `targetVersion`. An online host that is NOT in the allowlist is skipped, not
+   * counted as a mismatch: it was intentionally never a deploy target
+   * (fail-closed allowlist, #1303), so leaving it on an old version must not fail
+   * the deploy. Omitted on manual deploys, which keep verifying every online
+   * host as before.
+   */
+  fleetAllowlist?: string[];
+};
+
+/**
+ * Split the relay's orchestrator list into hosts that must reach `targetVersion`
+ * and hosts to skip, given the explicit deploy targets `requiredIds` (#933).
+ *
+ * Signal: the orchestrator registry has no enabled/disabled flag — the only
+ * reachability signal is `status` ("online"/"offline"), set online on heartbeat
+ * and flipped to offline by the relay's staleness sweep. A disabled host stops
+ * heartbeating, so it reads as offline. So:
+ *  - `status === "offline"` AND not an explicit deploy target → SKIP. A host that
+ *    deliberately isn't participating (e.g. maclinux) must not fail the release.
+ *  - everything else (online hosts, and required deploy targets regardless of
+ *    reported status) → still verified. An online host stuck on the old version,
+ *    or a required target that's missing/lagging, is a genuine failure and stays
+ *    loud — no regression in real-failure detection.
+ *
+ * #1308: when `opts.fleetAllowlist` is set (the autonomous CD path), a host
+ * outside that allowlist is also skipped even while online + off-version — it
+ * was intentionally never a deploy target (fail-closed allowlist, #1303), so it
+ * must not poison the deploy verdict. Required deploy targets stay verified.
+ */
+export function planOrchestratorVersionVerification(
+  orchestrators: OrchestratorVersion[],
+  targetVersion: string,
+  requiredIds: string[] = [],
+  opts: OrchestratorVersionVerificationOptions = {},
+): OrchestratorVersionVerification {
+  const required = new Set(requiredIds);
+  const deferredRestartIds = new Set(opts.deferredRestartIds ?? []);
+  const fleetAllowlist = opts.fleetAllowlist ? new Set(opts.fleetAllowlist) : undefined;
+  const byId = new Map(orchestrators.map((orch) => [orch.id, orch]));
+  const mismatches: OrchestratorVersion[] = [];
+  const skipped: OrchestratorVersion[] = [];
+  for (const orch of orchestrators) {
+    if (deferredRestartIds.has(orch.id) && !required.has(orch.id)) {
+      skipped.push(orch);
+      continue;
+    }
+    // #1308: a host outside the fleet allowlist was never a deploy target — skip
+    // it, even when online + off-version, so it can't poison the deploy verdict.
+    if (fleetAllowlist && !isCdFleetHostAllowlisted(orch, fleetAllowlist) && !required.has(orch.id)) {
+      skipped.push(orch);
+      continue;
+    }
+    if (orch.status === "offline" && !required.has(orch.id)) {
+      skipped.push(orch);
+      continue;
+    }
+    if (orch.version !== targetVersion) mismatches.push(orch);
+  }
+  // A required deploy target absent from the relay list entirely is still a
+  // mismatch — after a remote upgrade it should have re-registered.
+  for (const id of requiredIds) {
+    if (!byId.has(id) && !mismatches.some((orch) => orch.id === id)) {
+      mismatches.push({ id });
+    }
+  }
+  return { mismatches, skipped };
+}
+
+/** Back-compat thin wrapper: just the mismatch set (skips are dropped). */
+export function orchestratorVersionMismatches(
+  orchestrators: OrchestratorVersion[],
+  targetVersion: string,
+  requiredIds: string[] = [],
+): OrchestratorVersion[] {
+  return planOrchestratorVersionVerification(orchestrators, targetVersion, requiredIds).mismatches;
+}
+
+export function resolveReleaseLocalOrchestratorId(
+  homeDir: string = process.env.HOME ?? process.env.USERPROFILE ?? homedir(),
+  env: Record<string, string | undefined> = process.env,
+): string {
+  const configPath = join(homeDir, ".agent-relay", "orchestrator.json");
+  let configId: string | undefined;
+  if (existsSync(configPath)) {
+    try {
+      const parsed = JSON.parse(readFileSync(configPath, "utf8")) as { id?: string };
+      if (typeof parsed.id === "string") configId = parsed.id;
+    } catch {
+      // Unparsable config: fall through to env/hostname, matching the runtime.
+    }
+  }
+  // #1554 — the ONE canonical derivation (config id > env > hostname), same as the orchestrator
+  // runtime and resolveLocalOrchestratorId. This must not hand-roll its own variant: the CD
+  // reconnect-verification path here compares this id against the id the host actually registered
+  // under, and a trailing-whitespace env value (`AGENT_RELAY_ORCHESTRATOR_ID="wbox-1 "`) previously
+  // produced "wbox-1 " here but "wbox-1" at registration — a false post-deploy verification failure.
+  return deriveOrchestratorId({ configId, envId: relayEnv(env, "AGENT_RELAY_ORCHESTRATOR_ID"), hostname: osHostname() });
+}
+
+function orchestratorsUrl(baseUrl: string): string {
+  return new URL("/api/orchestrators", baseUrl).toString();
+}
+
+async function readJsonBody(res: Response): Promise<unknown> {
+  const text = await res.text().catch(() => "");
+  if (!text) return undefined;
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    return text;
+  }
+}
+
+function localOrchestratorReconnectOk(
+  orch: ReleaseOrchestrator | undefined,
+  attest: { expectedGitSha?: string; expectedVersion?: string } = {},
+): boolean {
+  return (
+    !!orch &&
+    orch.status !== "offline" &&
+    orch.health?.restartRequired === false &&
+    (!attest.expectedGitSha || orch.gitSha === attest.expectedGitSha) &&
+    (!attest.expectedVersion || orch.version === attest.expectedVersion)
+  );
+}
+
+export async function waitForLocalOrchestratorReconnect(
+  baseUrl: string,
+  localOrchestratorId: string,
+  opts: LocalOrchestratorReconnectOptions = {},
+): Promise<LocalOrchestratorReconnectResult> {
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_LOCAL_ORCHESTRATOR_RECONNECT_TIMEOUT_MS;
+  const intervalMs = opts.intervalMs ?? DEFAULT_LOCAL_ORCHESTRATOR_RECONNECT_INTERVAL_MS;
+  const requestTimeoutMs = opts.requestTimeoutMs ?? DEFAULT_HEALTH_REQUEST_TIMEOUT_MS;
+  const fetchImpl = opts.fetch ?? fetch;
+  const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const now = opts.now ?? (() => Date.now());
+  const url = orchestratorsUrl(baseUrl);
+  const started = now();
+  let checks = 0;
+  let last: LocalOrchestratorReconnectSnapshot = { url, error: "orchestrator reconnect check was not attempted" };
+
+  while (now() - started <= timeoutMs) {
+    try {
+      const res = await fetchImpl(url, { headers: opts.headers, signal: AbortSignal.timeout(requestTimeoutMs) });
+      const body = await readJsonBody(res);
+      const orchestrators = Array.isArray(body) ? body as ReleaseOrchestrator[] : [];
+      const local = orchestrators.find((orch) => orch.id === localOrchestratorId);
+      last = {
+        url,
+        httpStatus: res.status,
+        body,
+        seenIds: orchestrators.map((orch) => orch.id),
+        ...(local ? { local } : {}),
+      };
+      checks++;
+      if (res.ok && localOrchestratorReconnectOk(local, { expectedGitSha: opts.expectedGitSha, expectedVersion: opts.expectedVersion })) {
+        return { ok: true, localOrchestratorId, checks, last };
+      }
+    } catch (error) {
+      last = { url, error: errMessage(error) };
+      checks++;
+    }
+
+    const elapsed = now() - started;
+    if (elapsed >= timeoutMs) break;
+    await sleep(Math.min(intervalMs, timeoutMs - elapsed));
+  }
+
+  return { ok: false, localOrchestratorId, checks, last };
+}
+
+export function formatLocalOrchestratorReconnectSnapshot(snapshot: LocalOrchestratorReconnectSnapshot): string {
+  if (snapshot.error) return `${snapshot.url}: ${snapshot.error}`;
+  const local = snapshot.local;
+  if (!local) {
+    const seen = snapshot.seenIds?.length ? snapshot.seenIds.join(", ") : "none";
+    return `${snapshot.url}: HTTP ${snapshot.httpStatus ?? "unknown"} local orchestrator missing (seen: ${seen})`;
+  }
+  const health = local.health
+    ? `${local.health.status ?? "unknown"} restartRequired=${String(local.health.restartRequired)}`
+    : "missing";
+  return `${snapshot.url}: HTTP ${snapshot.httpStatus ?? "unknown"} ${local.id} status=${local.status ?? "unknown"} health=${health}`;
+}
+
+export async function waitForLocalOrchestratorReconnectBeforeSmoke(opts: {
+  noSmoke: boolean;
+  relayUrl: string;
+  token: string;
+  localOrchestratorId: string;
+  log: (msg: string) => void;
+}): Promise<void> {
+  if (opts.noSmoke) return;
+  opts.log(`\n→ waiting for local orchestrator ${opts.localOrchestratorId} to reconnect before smoke …`);
+  const reconnect = await waitForLocalOrchestratorReconnect(opts.relayUrl, opts.localOrchestratorId, {
+    headers: { [RELAY_TOKEN_HEADER]: opts.token },
+  });
+  if (reconnect.ok) {
+    opts.log(`✓ local orchestrator ${opts.localOrchestratorId} reconnected for smoke (${reconnect.checks} checks)`);
+    return;
+  }
+  opts.log(
+    `  ⚠ local orchestrator ${opts.localOrchestratorId} did not reconnect after ${reconnect.checks} checks; ` +
+    `running smoke anyway: ${formatLocalOrchestratorReconnectSnapshot(reconnect.last)}`,
+  );
+}
+
+/**
+ * Format the host-verification mismatch list. For a git-checkout deploy (#1311),
+ * pass `artifactSha` so the failure reads as the gitSha comparison that actually
+ * failed — `<id>@<gitSha> (expected <artifactSha>)` — instead of a nonsensical
+ * VERSION comparison ("reports 0.129.7 (expected 0.129.7)") where the version
+ * already matches and only the pinned SHA differs.
+ */
+export function formatOrchestratorVersionMismatches(
+  mismatches: OrchestratorVersion[],
+  targetVersion: string,
+  opts: { artifactSha?: string } = {},
+): string {
+  return mismatches.map((orch) => opts.artifactSha
+    ? `${orch.id}@${orch.gitSha ?? "missing"} (expected ${opts.artifactSha})`
+    : `${orch.id}@${orch.version ?? "missing"} (expected ${targetVersion})`).join(", ");
+}
+
+/** Compare dotted semver-ish strings. Returns true when `a >= b`. */
+export function semverGte(a: string, b: string): boolean {
+  const pa = a.split(".").map(Number);
+  const pb = b.split(".").map(Number);
+  for (let i = 0; i < 3; i++) {
+    const x = pa[i] ?? 0;
+    const y = pb[i] ?? 0;
+    if (x !== y) return x > y;
+  }
+  return true;
+}
+
+/** Extract the first `x.y.z` from `humwick --version` output, or undefined. */
+export function parseInstalledVersion(out: string): string | undefined {
+  return out.match(/\d+\.\d+\.\d+/)?.[0];
+}
+
+/**
+ * Does the *installed* CLI (at `installedVersion`) understand `--restart-deferred`?
+ * Capability probe by version, NOT by error-string guessing. When the version is
+ * unknown we conservatively assume "no" and use `--no-restart` (which every past
+ * release understands) — the services are restarted by release.ts regardless, so
+ * the only cost of being wrong is a cosmetic nag, never a missed restart.
+ */
+export function supportsRestartDeferred(installedVersion: string | undefined): boolean {
+  return installedVersion ? semverGte(installedVersion, RESTART_DEFERRED_MIN_VERSION) : false;
+}
+
+type UpgradeFailureKind = "unknown-option" | "etarget" | "other";
+
+/**
+ * Classify why `humwick upgrade <flag>` failed, given its combined out+err.
+ *
+ *  - `unknown-option`: the CLI *specifically* rejected the flag we passed
+ *    (genuine "predates the flag" — only then is degrading to `--no-restart` valid).
+ *    Matched against the exact `Unknown upgrade option "<flag>"` the CLI throws, so
+ *    an unrelated failure that merely contains those words elsewhere won't trip it.
+ *  - `etarget`: npm replica lag / no-matching-version — transient, retry same flag.
+ *  - `other`: anything else — surface it; do NOT degrade the restart flag.
+ */
+export function classifyUpgradeFailure(flag: string, combinedOutput: string): UpgradeFailureKind {
+  // Exact-flag match: the CLI throws `Unknown upgrade option "<arg>"` during arg
+  // parsing, before doing any work. Escape regex metachars in the flag.
+  const escaped = flag.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  if (new RegExp(`Unknown upgrade option "${escaped}"`).test(combinedOutput)) return "unknown-option";
+  if (/ETARGET|notarget|No matching version/i.test(combinedOutput)) return "etarget";
+  return "other";
+}
+
+export type HealthGateResult = {
+  host: string;
+  ok: boolean;
+  checks: number;
+  consecutiveOk: number;
+  /** #2406 — how many consecutive acceptable probes this gate was asking for, so a failure can say
+   *  "ran out of time at 1 of 2" instead of leaving the reader to infer a rejection that never happened. */
+  consecutiveTarget: number;
+  /**
+   * #2406 round 2 — how many probes in the WINDOW were assessed and REJECTED (had blockers). The
+   * whole point is that `lastAssessment` describes one probe, so it cannot answer "did this gate
+   * reject anything?": a window of genuine liveness failures followed by one recovered probe leaves
+   * `lastAssessment.acceptable === true`, and a message inferred from that alone would tell the next
+   * operator nothing was rejected while the database was failing. Counted over the window, so the
+   * claim is made from a window-level fact. Probes that never got an assessment (non-2xx, transport
+   * error) are NOT counted here — unreachable is not rejected, and they are described separately.
+   */
+  rejectedProbes: number;
+  /** Blockers from the FIRST rejected probe — the earliest real fault the window saw, which is the
+   *  one an operator needs; later probes are usually the same fault or its recovery. */
+  firstRejectionBlockers?: string[];
+  last: HealthProbeSnapshot;
+  lastAssessment?: DeploymentHealthAssessment;
+};
+
+type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+export type HealthGateOptions = {
+  timeoutMs?: number;
+  intervalMs?: number;
+  consecutiveOk?: number;
+  requestTimeoutMs?: number;
+  fetch?: FetchLike;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => number;
+  /** #2672 — accept a probe only from a process that attests this build in the same response. */
+  gitSha?: string;
+};
+
+const DEFAULT_HEALTH_GATE_TIMEOUT_MS = 60_000;
+/**
+ * #2406 — the budget for the ONE health gate that races a restart it just caused: the local relay,
+ * probed immediately after `performLocalRelayRestart`. Every OTHER host reaches its health gate only
+ * after `pollHostVersionMismatches` has already watched it re-attest the new version (150s base, up
+ * to 450s while a slow host is still making progress), so those gates start against a process that is
+ * demonstrably back. The local gate has no such warm-up — its clock starts while the relay is still
+ * booting — and the default 60s was not enough for a hub with a multi-GB database, which cost the
+ * 2026-08-25 f2624db5 deploy its entire first attempt (deploy 19e52959: the last probe in the window
+ * was ACCEPTABLE, just one short of the two-in-a-row target). Same tolerance as everywhere else, a
+ * budget that matches the phase's actual precondition.
+ */
+export const LOCAL_RELAY_RESTART_HEALTH_GATE_TIMEOUT_MS = 180_000;
+const DEFAULT_HEALTH_GATE_INTERVAL_MS = 2_000;
+const DEFAULT_HEALTH_GATE_CONSECUTIVE_OK = 2;
+const DEFAULT_HEALTH_REQUEST_TIMEOUT_MS = 5_000;
+
+function healthUrl(baseUrl: string): string {
+  return new URL("/api/health", baseUrl).toString();
+}
+
+async function readHealthBody(res: Response): Promise<unknown> {
+  const text = await res.text().catch(() => "");
+  if (!text) return undefined;
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    return text;
+  }
+}
+
+export function bodyHealthAcceptable(body: unknown): boolean {
+  return assessDeploymentHealth(body).acceptable;
+}
+
+async function probeHealth(url: string, fetchImpl: FetchLike, requestTimeoutMs: number): Promise<HealthProbeSnapshot> {
+  try {
+    const res = await fetchImpl(url, { signal: AbortSignal.timeout(requestTimeoutMs) });
+    const body = await readHealthBody(res);
+    return { url, httpStatus: res.status, body };
+  } catch (e) {
+    return { url, error: errMessage(e) };
+  }
+}
+
+export async function waitForConsecutiveAcceptableHealth(
+  host: string,
+  baseUrl: string,
+  opts: HealthGateOptions = {},
+): Promise<HealthGateResult> {
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_HEALTH_GATE_TIMEOUT_MS;
+  const intervalMs = opts.intervalMs ?? DEFAULT_HEALTH_GATE_INTERVAL_MS;
+  const consecutiveTarget = opts.consecutiveOk ?? DEFAULT_HEALTH_GATE_CONSECUTIVE_OK;
+  const requestTimeoutMs = opts.requestTimeoutMs ?? DEFAULT_HEALTH_REQUEST_TIMEOUT_MS;
+  const fetchImpl = opts.fetch ?? fetch;
+  const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const now = opts.now ?? (() => Date.now());
+  const url = healthUrl(baseUrl);
+  const started = now();
+  let checks = 0;
+  let consecutiveOk = 0;
+  let last: HealthProbeSnapshot = { url, error: "health check was not attempted" };
+  let lastAssessment: DeploymentHealthAssessment | undefined;
+  let rejectedProbes = 0, firstRejectionBlockers: string[] | undefined;
+
+  while (now() - started <= timeoutMs) {
+    last = await probeHealth(url, fetchImpl, requestTimeoutMs);
+    checks++;
+    lastAssessment = last.httpStatus && last.httpStatus >= 200 && last.httpStatus < 300
+      ? assessDeploymentHealth(last.body, { gitSha: opts.gitSha })
+      : undefined;
+    if (lastAssessment?.blockers.length) { rejectedProbes++; firstRejectionBlockers ??= lastAssessment.blockers; }
+    if (last.httpStatus && last.httpStatus >= 200 && last.httpStatus < 300 && lastAssessment?.acceptable) {
+      consecutiveOk++;
+      const rejected = firstRejectionBlockers ? { firstRejectionBlockers } : {};
+      if (consecutiveOk >= consecutiveTarget) return { host, ok: true, checks, consecutiveOk, consecutiveTarget, rejectedProbes, ...rejected, last, lastAssessment };
+    } else {
+      consecutiveOk = 0;
+    }
+
+    const elapsed = now() - started;
+    if (elapsed >= timeoutMs) break;
+    await sleep(Math.min(intervalMs, timeoutMs - elapsed));
+  }
+
+  return { host, ok: false, checks, consecutiveOk, consecutiveTarget, rejectedProbes, ...(firstRejectionBlockers ? { firstRejectionBlockers } : {}), last, lastAssessment };
+}
+
+export const waitForConsecutiveOkHealth = waitForConsecutiveAcceptableHealth;