@humwick/server 0.0.0-stage → 0.130.1

package/scripts/cd-watcher.ts

@@ -0,0 +1,985 @@
+#!/usr/bin/env bun
+import { relayEnv, writeChildEnv } from "@humwick/sdk/env";
+/**
+ * Host-side continuous-deploy trigger (#1275).
+ *
+ * This deliberately runs on the relay host: GitHub Actions cannot reach the
+ * Tailnet relay. It reuses release.ts's deploy-only path, so all existing
+ * interlocks, version verification and double health gates remain the deploy
+ * authority. A failed child leaves the ledger failed and never stamps Tasks.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+import { errMessage } from "@humwick/sdk";
+import { gitCheckoutArtifactPaths } from "@humwick/sdk/git-checkout-artifact";
+import type { LandGate } from "@humwick/sdk";
+import { cleanLandGatesConfig, landGateDefinitionsDigest } from "@humwick/sdk/land-gates";
+import { closeDb, currentPendingDeployEvent, DEPLOY_LEASE_MS, type DeployCommit, type DeployEvent, type DeployHost, deployChangesForCommits, deployEventLeaseAt, finishDeployEvent, getDbPath, getDeployEvent, type LandGateAttestationRecord, latestCompleteDeploy, latestSuccessfulDeploy, latestFailedDeployForCommit, listDeployEventsNeedingChanges, listOrchestrators, openExistingDb, persistDeployChangesSummary, reconcilePendingDeployEvent, stampVerifiedTasks, tryBeginDeployEvent } from "../src/db.ts";
+import { type BoundaryDeploymentLookup } from "./cd-land-boundary.ts";
+
+// #1902 gateway deferral: this detached CLI deliberately brackets direct ledger
+// access with openExistingDb/closeDb around long synchronous git/release work, so
+// it never pins the live relay WAL across a subprocess. `runCdWatcher` is itself
+// synchronous (its command seam is spawnSync-like), therefore an isolated await
+// would not be safe without redesigning that lifecycle. The live server's routes,
+// MCP, and maintenance transport callers use deploy-events-async.ts instead.
+import { GithubActionsCiProvider, type CiCommandRunner, type CiObservation, type CiWatchDescriptor, commandRunner } from "../src/ci/index.ts";
+import { deployExecutorConfirmedGone } from "../src/deploy-executor.ts";
+import { resolveReleaseLocalOrchestratorId } from "./release-upgrade.ts";
+import { resolveCdRestartEpochTickBudgetMs, CD_WATCHER_INITIATOR, resolveCanonicalRelayDbPath, withDeployLedgerDb, resolveCdFleetAllowlist, resolveCdRepoRoot, shouldRunCdWatcherOnThisHost } from "./cd-config.ts";
+import { ownedRestartEpochDrains, releaseOwnedRestartEpochDrains } from "../src/restart-epoch-deploy-hold.ts";
+import { restartEpochById } from "../src/db/restart-epoch.ts";
+import { holdAndBindCdHostDeploy, planCdHostScopes, relayHostsTurn, upgradeHeldByLand } from "./cd-host-epochs.ts";
+import { cdDeployStampsTasks, completeCdDeployTasks } from "./release-cd-stamp.ts";
+import { isCdFleetHostAllowlisted } from "./cd-fleet-membership.ts";
+import { resolveFleetShaCoverage, type FleetShaCoverage } from "./fleet-sha-coverage.ts";
+import { readServedDashboardSha, resolveDashboardRebuild } from "./dashboard-build-marker.ts";
+import { CD_DEPLOY_WORKTREE_ENV, createCdDeployWorktree, reclaimStaleDeployWorktreesAtTickStart, reclaimStaleDeployWorktreesBeforeDeploy, removeCdDeployWorktree } from "./cd-deploy-worktree-gc.ts";
+import { clearCdDeployDebounceState, decideCdDeployDebounce, readCdDeployDebounceState, resolveCdDeployDebounce, writeCdDeployDebounceState, type CdDeployDebounce } from "./cd-deploy-debounce.ts";
+
+export { shouldRunCdWatcherOnThisHost } from "./cd-config.ts";
+export { CD_WATCHER_DEBOUNCE_MS, CD_WATCHER_MAX_DEFER_MS, CD_WATCHER_MIN_INTERVAL_MS, decideCdDeployDebounce, resolveCdDeployDebounce } from "./cd-deploy-debounce.ts";
+
+type Command = { code: number; out: string; err: string };
+// The repo's one command seam (spawnSync-shaped, with an optional env overlay) — #2727 needs the
+// env to hand the deploy driver its worktree root. Shared with the CI helpers via this exact type.
+type Run = CiCommandRunner;
+const run: Run = commandRunner;
+export type CdWatcherResult = {
+  /** Converged proves the live eligible fleet is healthy; skipped defers without recovery evidence. */
+  outcome: "skipped" | "converged" | "deployed" | "failed";
+  sha: string | null;
+  previousSha: string | null;
+  reason: string;
+  observation: CiObservation;
+};
+export const CD_WATCHER_FAILURE_BACKOFF_MS = 15 * 60 * 1000;
+
+/** The fleet watcher remains a zero-config watch: its existing checkout, gh CLI
+ * authentication and inferred GitHub host are the observer defaults. */
+export function cdCiWatch(root: string): CiWatchDescriptor {
+  return { provider: "github-actions", repository: root, workingDirectory: root, branch: "main", workflow: "ci.yml", jobs: ["deploy-checks"], observer: {} };
+}
+
+export function observeMainCi(runCommand: Run, root: string, observedAt = Date.now(), _boundaryLookup?: BoundaryDeploymentLookup): CiObservation {
+  const watch = cdCiWatch(root);
+  // #1770 blocker 1 — fetch the release tags alongside main. cd-watcher's dedicated detection clone
+  // only ever fetched `main`; a branch refspec does NOT auto-follow tags (git 2.43), so
+  // `git tag --points-at` in isReleaseTagValidated always returned "" and the tag-on-green accept
+  // path was dead code. The explicit `refs/tags/*` refspec makes the vX.Y.Z release tag visible so
+  // the forward-deploy path can actually run instead of collapsing onto the no-downgrade guard.
+  //
+  // #1770 round 3, blocker F1 — the refspec MUST be forced (`+`). A rewritten/force-moved/
+  // deleted-and-recut `vX.Y.Z` tag makes an unforced `refs/tags/*:refs/tags/*` update print
+  // `! [rejected] ... (would clobber existing tag)` and the whole `git fetch` exits non-zero, even
+  // though the `main` branch update in the same fetch succeeded. That throws below, uncaught, out of
+  // runCdWatcher — wedging every future tick (no deploys, no CI red/green alerting) until someone
+  // hand-deletes the stale local tag. Forcing tag updates makes them always fast-forward locally, so
+  // a divergent origin-side tag can never block the fetch (empirically verified: unforced exits 1 on
+  // a clobbering tag, `+`-forced exits 0 and just overwrites the local tag object).
+  const fetch = runCommand("git", ["fetch", "origin", watch.branch, "+refs/tags/*:refs/tags/*"], root);
+  if (fetch.code !== 0) {
+    throw new Error(`git fetch origin ${watch.branch} failed: ${fetch.err || fetch.out || `exit ${fetch.code}`}`);
+  }
+  const sha = runCommand("git", ["rev-parse", `origin/${watch.branch}`], root).out;
+  if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error(`origin/${watch.branch} did not resolve to a SHA: ${sha || "empty"}`);
+  return new GithubActionsCiProvider(runCommand).observe({ watch, headSha: sha, observedAt });
+}
+
+const RELEASE_TAG_PATTERN = /^v\d+\.\d+\.\d+$/;
+
+/**
+ * #1770 — Accept a release's tag-on-green as a deployable signal.
+ *
+ * `bun run release` pushes the `vX.Y.Z` version-bump commit straight to origin/main; `ci.yml`
+ * never runs on that push (the ci-tag job pushes main with GITHUB_TOKEN, which fires no
+ * workflows), so observeMainCi sees the bump commit as missing/pending and the CI walk-back
+ * below would otherwise fall to the prior green ancestor and REVERT the release. But the tree
+ * that becomes the bump commit was validated by release.yml's tag-on-green job (`ci-tag`), which
+ * runs the full suite once on the dispatch SOURCE and only then bumps + tags + pushes.
+ *
+ * #1770 should-fix 3 — validate the tag against a green release.yml run in its ancestry rather than
+ * inferring "dispatch source == first parent" from topology: `pushReleaseWithRebaseRetry` rebases
+ * the bump commit when main moves between dispatch and push, so the first parent is no longer the
+ * validated source (false-negative), and a busy main could make the new tip a prior release's
+ * source (false-positive). Instead, read the latest SUCCESSFUL release.yml run's head_sha — the
+ * source it validated — and confirm that source is an ancestor of (the bump commit is built on top
+ * of it, directly or via the rebased-onto main tip). A random tag whose commit does not descend from
+ * a green release run is rejected.
+ *
+ * #1770 round 3, F3 — this is deliberately NOT "the tag's own run": `gh run list` above filters to
+ * COMPLETED runs, and during a live release the run that is about to produce and push THIS tag is
+ * still `in_progress` (ci-tag pushes the bump commit only after the suite passes, but the workflow
+ * run itself doesn't finish until later). So this reads the latest run that had ALREADY completed —
+ * ordinarily the PREVIOUS release's dispatch — and merely confirms the bump commit descends from
+ * that prior green source. That is exactly what the ancestor check needs (the tree was validated
+ * somewhere in this release's lineage), just not the tag's own in-flight run.
+ *
+ * Best-effort: any git/gh failure (missing tag, gh/auth error, no green release run) returns false,
+ * leaving the existing ci.yml path — and the no-downgrade guard — unchanged.
+ */
+export function isReleaseTagValidated(runCommand: Run, root: string, sha: string): boolean {
+  const tags = runCommand("git", ["tag", "--points-at", sha], root);
+  if (tags.code !== 0) return false;
+  if (!tags.out.split("\n").some((tag) => RELEASE_TAG_PATTERN.test(tag.trim()))) return false;
+  // The head_sha of the latest successful workflow_dispatch release run is the source that job ran
+  // the full suite against before committing + tagging this release on top of it.
+  const runSource = runCommand("gh", [
+    "api", "repos/{owner}/{repo}/actions/workflows/release.yml/runs",
+    "--jq", '[.workflow_runs[] | select(.event == "workflow_dispatch" and .status == "completed" and .conclusion == "success")][0].head_sha // ""',
+  ], root);
+  const source = runSource.out.trim();
+  if (runSource.code !== 0 || !/^[0-9a-f]{40}$/i.test(source)) return false;
+  // The validated source must be an ancestor of (or equal to) the commit this release tag points at.
+  return runCommand("git", ["merge-base", "--is-ancestor", source, sha], root).code === 0;
+}
+
+/** Resolve one attestation row for an exact tree oid. Injected so the DB lifecycle stays owned by
+ *  `runCdWatcher` (#1765: no connection may be held across the multi-second `git fetch`). */
+export type LandGateAttestationLookup = (treeSha: string) => LandGateAttestationRecord | null;
+
+export type LandGateAttestationVerdict =
+  | { attested: true; treeSha: string; gateNames: string[] }
+  | { attested: false; reason: string };
+
+/**
+ * #1744 / Phase −1 F1 — accept the LAND gate's green run as the deploy signal for a tree.
+ *
+ * `bun run ci` IS `bun run ci:land`: the workflow ci.yml runs on every main push is the same step
+ * list the land gate already ran (the land gate is in fact a strict superset — it forces
+ * AGENT_RELAY_CI_LAND_FULL=1, which disables ci:land's path-aware step skipping). Waiting ~17 min
+ * for that second run before deploying buys nothing when the tree is byte-identical; it was the
+ * dominant term in a ≈31-minute land→fleet.
+ *
+ * The safety argument is entirely in the KEY. `treeSha` is git's content address for the whole
+ * worktree, so a matching row means the bytes about to be deployed are the exact bytes the full
+ * gate executed against — no commit-graph, timestamp, author or message can differ into it, and a
+ * rebase/squash/amend that changes one byte simply has no row. Everything else here is fail-CLOSED:
+ * an unresolvable tree, an absent row, an unreadable or malformed gate config, a gate config that
+ * declares a required gate the attestation does not name (a branch that ADDS a gate is judged by the
+ * BASE's list, so the new one genuinely never ran on this tree) — each returns `attested: false`,
+ * which leaves the caller on the pre-existing "wait for a green ci.yml" path.
+ */
+export function isLandGateAttested(runCommand: Run, root: string, sha: string, lookup: LandGateAttestationLookup): LandGateAttestationVerdict {
+  const tree = runCommand("git", ["rev-parse", `${sha}^{tree}`], root);
+  const treeSha = tree.out.trim().toLowerCase();
+  if (tree.code !== 0 || !/^[0-9a-f]{40}$/.test(treeSha)) return { attested: false, reason: `could not resolve the tree of ${shortSha(sha)}` };
+
+  let record: LandGateAttestationRecord | null;
+  try {
+    record = lookup(treeSha);
+  } catch (error) {
+    return { attested: false, reason: `land-gate attestation lookup failed: ${errMessage(error)}` };
+  }
+  if (!record) return { attested: false, reason: `no land-gate attestation for tree ${treeSha.slice(0, 12)}` };
+  if (record.level !== "full" || record.gatesRan <= 0) return { attested: false, reason: `land-gate attestation for tree ${treeSha.slice(0, 12)} is not a full gate run` };
+  // #3362 — a partial (affected-only) land gate is not coverage evidence; main CI must judge the tree.
+  if ((record.suiteCoverage ?? "full") !== "full") return { attested: false, reason: `land-gate attestation for tree ${treeSha.slice(0, 12)} covers only part of the suite` };
+
+  // The requirement is read from the LANDED tree, not from the attestation: the attestation states
+  // what ran, the tree states what this repo requires, and only the tree's own requirement can
+  // decide whether "what ran" was enough. Reading it from the deployable tree makes an added gate
+  // fail closed instead of being silently inherited from an older config.
+  //
+  // And the comparison is over DEFINITIONS, not names. The gates that ran came from the BASE ref's
+  // config (a branch must not weaken the gate that judges it), so a branch can keep a required
+  // gate's NAME and swap its `command`: the base's strong command runs, the names still line up,
+  // and a name-only check would hand this tree a strong-suite attestation while the gate that
+  // judges every FUTURE land is now the weak one. `landGateDefinitionsDigest` covers name, command,
+  // cwd and timeout, so any such edit is a mismatch and cd-watcher waits for ci.yml instead.
+  const required = requiredLandGates(runCommand, root, sha);
+  if (!required) return { attested: false, reason: `could not read ${LAND_GATES_CONFIG_PATH} at ${shortSha(sha)}` };
+  if (required.length === 0) return { attested: false, reason: `${LAND_GATES_CONFIG_PATH} at ${shortSha(sha)} declares no required gates` };
+  const landedDigest = landGateDefinitionsDigest(required);
+  if (record.gatesDigest !== landedDigest) {
+    return {
+      attested: false,
+      reason: `land-gate attestation for tree ${treeSha.slice(0, 12)} covers gate set ${(record.gatesDigest || "(none)").slice(0, 12)} [${record.gateNames.join(", ")}] but ${shortSha(sha)} requires ${landedDigest.slice(0, 12)} [${required.map((g) => g.name).join(", ")}]`,
+    };
+  }
+  return { attested: true, treeSha, gateNames: record.gateNames };
+}
+
+const LAND_GATES_CONFIG_PATH = ".agent-relay/land-gates.json";
+
+/** The NON-optional gate DEFINITIONS the given tree declares, via the SAME validator the land-gate
+ *  runner resolves its list with (`cleanLandGatesConfig`) — a second, looser parser here could
+ *  disagree about what "required" means or normalize `cwd` differently, and either disagreement
+ *  breaks the digest comparison in one direction or the other. `undefined` (not `[]`) when the
+ *  config cannot be read or does not validate: the caller must treat an unknown requirement as
+ *  unattestable, never as no requirement. */
+function requiredLandGates(runCommand: Run, root: string, sha: string): LandGate[] | undefined {
+  const shown = runCommand("git", ["show", `${sha}:${LAND_GATES_CONFIG_PATH}`], root);
+  if (shown.code !== 0 || !shown.out.trim()) return undefined;
+  try {
+    return cleanLandGatesConfig(JSON.parse(shown.out), root).filter((gate) => !gate.optional);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * #1770 — No-downgrade guard. Returns the currently-deployed SHA that `candidate` is a strict
+ * ANCESTOR of (⇒ deploying `candidate` would roll that host BACKWARD), or undefined when the
+ * candidate moves every deployed SHA forward or sideways. Equal SHAs are a no-op, not a downgrade.
+ *
+ * This is the safety invariant behind #1770: `latestSuccessfulDeploy()` (the ledger) can lag the
+ * live fleet — `bun run release` deploys the bump commit directly without a cd-watcher ledger row —
+ * so the CI walk-back can pick a green ANCESTOR of what the fleet already runs and revert the
+ * release. Comparing the candidate against the live process-attested fleet SHAs makes that
+ * structurally impossible regardless of any CI-signal gap. A `merge-base` error (e.g. an object not
+ * yet fetched) is treated as "not an ancestor" — the guard never blocks on incomplete information.
+ *
+ * #1770 should-fix 5 — only a deployed SHA that is itself reachable from origin/main can represent
+ * a real backward move on the release line. A sideways/stale attestation (an object that is not an
+ * ancestor of origin/main) is ignored so it cannot wedge CD by making the guard refuse every
+ * forward deploy.
+ */
+export function findDowngradeTarget(runCommand: Run, root: string, candidate: string, deployedShas: Iterable<string>): string | undefined {
+  const checked = new Set<string>();
+  for (const deployed of deployedShas) {
+    if (!deployed || deployed === candidate || checked.has(deployed)) continue;
+    checked.add(deployed);
+    if (runCommand("git", ["merge-base", "--is-ancestor", deployed, "origin/main"], root).code !== 0) continue;
+    if (runCommand("git", ["merge-base", "--is-ancestor", candidate, deployed], root).code === 0) return deployed;
+  }
+  return undefined;
+}
+
+/**
+ * Select the newest green main SHA after the last successful deploy without
+ * letting a known-red later commit be silently skipped. Pending/cancelled
+ * successors are safe to leave for a later deploy; a completed failure is not.
+ */
+export function selectDeployableMainCi(runCommand: Run, root: string, previousSha: string | undefined, observedAt = Date.now(), _attestationLookup?: LandGateAttestationLookup,
+  _boundaryLookup: BoundaryDeploymentLookup = () => null): { sha?: string; observation: CiObservation; reason?: string } {
+  const observation = observeMainCi(runCommand, root, observedAt);
+  // #3362 §11: neither a land attestation nor a historical boundary verdict proves this job ran.
+  if (observation.status === "green") return { sha: observation.headSha, observation };
+  if ((observation.status === "missing" || observation.status === "pending")
+    && isReleaseTagValidated(runCommand, root, observation.headSha)) {
+    return { sha: observation.headSha, observation };
+  }
+  // The first deployment retains the existing tip-only rule: without a durable
+  // lower bound, walking the entire repository history is neither bounded nor useful.
+  if (!previousSha) return { observation };
+  if (runCommand("git", ["merge-base", "--is-ancestor", previousSha, "origin/main"], root).code !== 0) {
+    return { observation, reason: "Last successful deploy SHA is not an ancestor of origin/main; refusing historical CI fallback." };
+  }
+  const commits = runCommand("git", ["rev-list", "--first-parent", "origin/main", `^${previousSha}`], root);
+  if (commits.code !== 0) throw new Error(`could not list main commits since the last deploy: ${commits.err || commits.out}`);
+
+  let newerFailure: string | undefined;
+  for (const sha of commits.out.split("\n").filter((value) => /^[0-9a-f]{40}$/i.test(value))) {
+    const candidate = sha === observation.headSha
+      ? observation
+      : new GithubActionsCiProvider(runCommand).observe({ watch: cdCiWatch(root), headSha: sha, observedAt });
+    if (candidate.status === "green") {
+      if (newerFailure) {
+        return {
+          observation,
+          reason: `A newer origin/main SHA (${shortSha(newerFailure)}) failed CI; waiting for a newer green SHA.`,
+        };
+      }
+      return { sha, observation };
+    }
+    if (!newerFailure && candidate.status === "failing") newerFailure = sha;
+  }
+  return { observation };
+}
+
+export function greenMainSha(runCommand: Run, root: string): string | undefined {
+  const observation = observeMainCi(runCommand, root);
+  return observation.status === "green" ? observation.headSha : undefined;
+}
+
+export function commitsInDeployRange(runCommand: Run, root: string, previousSha: string | undefined, sha: string): string[] {
+  return changesInDeployRange(runCommand, root, previousSha, sha).map((commit) => commit.sha);
+}
+
+/**
+ * #1947 — Is this repo path guaranteed to be OUTSIDE the deployed runtime surface?
+ *
+ * The set is intentionally a small, explicit allowlist of things the running system never imports:
+ * documentation and design artifacts (anything under `docs/`, plus any Markdown file anywhere —
+ * README/CONTRIBUTING/design docs) and test-only sources (`*.test.*` / `*.spec.*`, which covers
+ * `*.integration.test.ts`). EVERYTHING ELSE — source, scripts, configs, lockfiles, unrecognized
+ * paths — is treated as deployed, so it can only ever FORCE a deploy, never suppress one. That is the
+ * fail-safe direction #1947 requires: a path we are not certain about is a deployed path.
+ */
+export function isNonDeployedPath(path: string): boolean {
+  const p = path.trim();
+  if (!p) return false; // an empty/unparseable entry counts as deployed → forces a deploy
+  if (p.startsWith("docs/")) return true; // documentation + design artifacts
+  if (/\.md$/i.test(p)) return true; // Markdown anywhere (README, CONTRIBUTING, design docs)
+  if (/(^|\/)[^/]+\.(test|spec)\.(ts|tsx|js|jsx|mjs|cjs)$/i.test(p)) return true; // tests-only
+  return false;
+}
+
+export type DeployRangePathScope =
+  | { deploy: true; reason: string }
+  | { deploy: false; nonDeployedPaths: string[] };
+
+/**
+ * #1947 — Decide whether the `previousSha..sha` range can SKIP the fleet redeploy because every file
+ * it changes is outside the deployed surface (see `isNonDeployedPath`).
+ *
+ * FAIL-SAFE = DEPLOY. This returns `{ deploy: true }` on every uncertain signal: a git failure or
+ * throw (change detection unavailable), an empty file list (an ambiguous range — never a proof that
+ * nothing deployable changed), or a single path outside the known non-deployed allowlist. Only a
+ * non-empty diff in which EVERY file is provably non-deployed returns `{ deploy: false }`. Uses a
+ * net `git diff A B` (a file changed then reverted within the range nets out — the deployed bytes are
+ * identical, so skipping is correct) and `--no-renames` so a rename surfaces both its old and new
+ * paths, making the classification strictly more likely to see a deployed path.
+ */
+export function classifyDeployRangePaths(runCommand: Run, root: string, previousSha: string, sha: string): DeployRangePathScope {
+  let result: Command;
+  try {
+    result = runCommand("git", ["diff", "--name-only", "-z", "--no-renames", previousSha, sha], root);
+  } catch (error) {
+    return { deploy: true, reason: `change detection unavailable (git diff threw: ${errMessage(error)})` };
+  }
+  if (result.code !== 0) return { deploy: true, reason: `change detection unavailable (git diff exit ${result.code}: ${result.err || result.out})` };
+  const files = result.out.split("\0").map((file) => file.trim()).filter(Boolean);
+  if (files.length === 0) return { deploy: true, reason: `no files reported for ${previousSha}..${sha}; deploying` };
+  const deployed = files.filter((file) => !isNonDeployedPath(file));
+  if (deployed.length > 0) {
+    return { deploy: true, reason: `range touches ${deployed.length} deployed path(s) (e.g. ${deployed.slice(0, 3).join(", ")})` };
+  }
+  return { deploy: false, nonDeployedPaths: files };
+}
+
+/**
+ * Ordered deploy range from one NUL-delimited `git log`. The first deploy keeps
+ * the historic tip-only behavior: it asks git for exactly the deploy SHA, not
+ * its entire ancestry.
+ */
+export function changesInDeployRange(runCommand: Run, root: string, previousSha: string | undefined, sha: string): DeployCommit[] {
+  // With no prior successful deploy, `git rev-list <sha>` means all history and
+  // corrupts lead-time metrics. The validated tip is the only safe first range.
+  const range = previousSha ? `${previousSha}..${sha}` : sha;
+  const args = previousSha
+    ? ["log", "--reverse", "-z", "--format=%H%x00%s", range]
+    : ["log", "-1", "-z", "--format=%H%x00%s", sha];
+  const result = runCommand("git", args, root);
+  if (result.code !== 0) throw new Error(`could not list commits in ${range}: ${result.err || result.out}`);
+  const parts = result.out.split("\0");
+  const changes: DeployCommit[] = [];
+  for (let i = 0; i + 1 < parts.length; i += 2) {
+    const commitSha = parts[i]!;
+    if (!/^[0-9a-f]{40}$/i.test(commitSha)) continue;
+    changes.push({ sha: commitSha, subject: parts[i + 1]! });
+  }
+  return changes;
+}
+
+/** Git artifacts are the CD default; npm remains the explicit emergency fallback. */
+export function cdDeployArgs(sha: string, mode = relayEnv(process.env, "AGENT_RELAY_CD_ARTIFACT_MODE") ?? "git-checkout", deployEventId?: string, targetHostIds?: string[]): string[] {
+  if (mode !== "git-checkout" && mode !== "npm-reuse") throw new Error(`invalid CD artifact mode "${mode}"`);
+  const args = mode === "git-checkout"
+    ? ["run", "scripts/release.ts", "--deploy-only", "--no-smoke", "--artifact", "git-checkout", "--commit", sha]
+    : ["run", "scripts/release.ts", "--deploy-only", "--no-smoke", "--artifact", "npm-reuse"];
+  return [
+    ...args,
+    ...(deployEventId ? ["--deploy-event", deployEventId] : []),
+    ...(targetHostIds?.length ? ["--cd-targets", targetHostIds.join(",")] : []),
+  ];
+}
+
+/** Best-effort `liveIn` label for the deployed SHA — the deployed worktree's own version,
+ * falling back to the SHA itself if the checkout is unreadable for any reason. */
+export function resolveDeployedVersion(worktree: string, sha: string): string {
+  try {
+    const pkg = JSON.parse(readFileSync(join(worktree, "package.json"), "utf8")) as { version?: string };
+    return typeof pkg.version === "string" && pkg.version ? pkg.version : sha;
+  } catch {
+    return sha;
+  }
+}
+
+/**
+ * #1770 round 3, F2 — best-effort package.json version AT `sha`, read directly from the
+ * already-fetched git objects (no worktree checkout needed — none exists yet for the candidate at
+ * this point in the tick). `resolveFleetShaCoverage`'s `targetVersion` option is the #1457 escape
+ * hatch that recognizes an npm-runtime host (which can only ever report a running `version`, never
+ * a `gitSha`) as covered once it matches the candidate's own version. Without this, that host sits
+ * in `coverage.unattested` forever and the fail-closed guard below can never clear for it — and
+ * npm-runtime hosts are the fleet MAJORITY (#1774).
+ *
+ * Best-effort like resolveDeployedVersion: any git/parse failure (unreadable object, malformed
+ * JSON, or a test double that doesn't stub `show`) returns undefined, reproducing the guard's prior
+ * (safe, if stricter) behavior — it never blocks or throws on a version lookup it couldn't make.
+ */
+export function resolveVersionAtSha(runCommand: Run, root: string, sha: string): string | undefined {
+  try {
+    const show = runCommand("git", ["show", `${sha}:package.json`], root);
+    if (show.code !== 0) return undefined;
+    const pkg = JSON.parse(show.out) as { version?: string };
+    return typeof pkg.version === "string" && pkg.version ? pkg.version : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function shortSha(sha: string | undefined): string | null {
+  return sha?.slice(0, 7) ?? null;
+}
+
+/**
+ * #1532 — self-heal a stranded `pending` fleet-lease row to its TERMINAL TRUTH, BEFORE the
+ * coverage short-circuit below returns early on an already-converged fleet (which is exactly
+ * how the orphaned-pending row `3dedd68a` stayed RED for 7h+: the >2h reclaim lived only inside
+ * tryBeginDeployEvent, past that return). Runs every tick — including the maintenance
+ * scheduler's `runOnStart` boot fire, which is the "survivor reconciles the killed driver's row
+ * on boot" half of the fix (#1532 fix #3): the co-located driver on the CD control host is
+ * SIGTERM-killed by the very relay restart it triggers, so the restarted daemon's first
+ * cd-watcher tick writes the terminal state the driver could not.
+ *
+ * Truth comes from the SAME live attestation resolveFleetShaCoverage uses, against the STUCK
+ * row's own target sha (never the current green tip):
+ *  - fleet genuinely attests it → `succeeded` (real attestation; no lease wait — a converged
+ *    fleet means the deploy is done however the driver died).
+ *  - #2105: else, if the DRIVER PROCESS that claimed the row is CONFIRMED gone → `failed` in
+ *    minutes, without waiting out the lease. This is the liveness half #1532 lacked: the 2h TTL
+ *    below is a timer, and a timer cannot tell a dead owner from a slow one, so an all-hosts deploy
+ *    whose driver died in the gap left by the relay restart IT triggered held the fleet lease — and
+ *    blocked every later green sha — for the full 2h. Strictly ordered AFTER the attestation check
+ *    so a deploy that actually shipped is recorded `succeeded`, never raced to `failed`.
+ *  - otherwise, only once the lease has expired (owner provably dead, not merely slow) → `failed`.
+ *
+ * Idempotent + race-safe via reconcilePendingDeployEvent's single gated UPDATE. Returns the
+ * finalized event, or null when there is nothing (yet) to reconcile.
+ */
+/**
+ * #2407 — project live fleet attestation onto the deploy row's own per-host bookkeeping.
+ *
+ * A deploy row is OPENED with every target host at `pending`; the deploy driver (release.ts) is what
+ * normally rewrites those entries as each host is verified. On the autonomous CD path that driver is
+ * SIGTERM-killed by the very relay restart it triggers, so the self-heal below is the only writer the
+ * row ever gets — and it used to append a single synthetic `fleet` entry while leaving every real
+ * host at `pending`. The result on the live ledger: `succeeded` deploys whose hosts all still read
+ * `pending`, which is precisely what #2407 reports as "the record is not finalized".
+ *
+ * The truth used here is the SAME coverage that decided `succeeded`, so nothing is inferred:
+ *  - a host in `covered` attests the target sha itself → `upgraded`;
+ *  - a host in `versionCovered` (#1457: no gitSha, but the target RELEASE version) → `upgraded` with
+ *    the weaker evidence spelled out in its reason, never silently conflated with a sha attestation;
+ *  - anything else is left EXACTLY as the row had it. Absence of attestation is not a status.
+ */
+export function attestedDeployHostStatuses(hosts: DeployHost[], coverage: FleetShaCoverage): DeployHost[] {
+  const covered = new Map(coverage.covered.map((host) => [host.id, host]));
+  const versionCovered = new Map(coverage.versionCovered.map((host) => [host.id, host]));
+  return hosts.map((host) => {
+    const shaAttested = covered.get(host.id);
+    if (shaAttested) {
+      return { ...host, status: "upgraded", ...(shaAttested.version ? { toVersion: shaAttested.version } : {}),
+        reason: "attests the target sha" };
+    }
+    const versionAttested = versionCovered.get(host.id);
+    if (versionAttested) {
+      return { ...host, status: "upgraded", ...(versionAttested.version ? { toVersion: versionAttested.version } : {}),
+        reason: "attests the target release version (no gitSha reported)" };
+    }
+    return host;
+  });
+}
+
+export function reconcileStalePendingDeploy(opts: { now?: number; leaseMs?: number; executorGraceMs?: number } = {}): DeployEvent | null {
+  const pending = currentPendingDeployEvent();
+  if (!pending) return null;
+  const leaseAt = deployEventLeaseAt(pending);
+  // #2189 P7 — a pending row bound to an epoch that is still QUIESCING is held, not stale. The epoch
+  // opens the ledger row up front (it is the fleet mutex now), and a tick that defers on a healthy
+  // gate leaves that row pending across ticks by design. Reconciling it to `failed` here would put
+  // the SHA into deploy backoff for work that is proceeding exactly as intended — and the next tick
+  // would refuse to resume its own hold.
+  if (pending.restartEpochId) {
+    const epoch = restartEpochById(pending.restartEpochId);
+    if (epoch && (epoch.state === "requested" || epoch.state === "quiesced")) return null;
+  }
+  const now = opts.now ?? Date.now();
+  const leaseMs = opts.leaseMs ?? DEPLOY_LEASE_MS;
+  const allowlist = resolveCdFleetAllowlist();
+  // #2988 — a CD row restarts only its own batch of Hosts, so converged means THOSE attest the sha.
+  const rowHosts = pending.initiatedBy === CD_WATCHER_INITIATOR ? new Set(pending.hosts.map((host) => host.id)) : undefined;
+  const coverage = resolveFleetShaCoverage(
+    listOrchestrators().filter((host) => !rowHosts || rowHosts.has(host.id)),
+    pending.commitSha,
+    allowlist,
+    { localOrchestratorId: resolveReleaseLocalOrchestratorId(), targetVersion: pending.version },
+  );
+  const converged = coverage.expected.length > 0 && coverage.behind.length === 0 && coverage.unattested.length === 0;
+  if (converged) {
+    return reconcilePendingDeployEvent({
+      id: pending.id, status: "succeeded", requireLeaseExpired: false, now,
+      // #2407 — record what the fleet ACTUALLY attests per host. The row was opened with every host
+      // at `pending` and the driver that would have replaced those entries is the process that died,
+      // so without this the ledger keeps a `succeeded` deploy whose hosts all read `pending` forever.
+      hosts: attestedDeployHostStatuses(pending.hosts, coverage),
+      reason: "self-healed: fleet attests the target sha but the deploy driver could not finalize the ledger (#1532)",
+    });
+  }
+  // #2105 — `requireExecutor` fences the write to the very process this verdict is about: the probe
+  // happens outside reconcilePendingDeployEvent's transaction, and `release.ts --resume` may legally
+  // re-stamp the row with a NEW, live driver in between.
+  if (deployExecutorConfirmedGone({
+    executor: pending.executor, startedAt: leaseAt, localHost: resolveReleaseLocalOrchestratorId(), now,
+    ...(opts.executorGraceMs !== undefined ? { graceMs: opts.executorGraceMs } : {}),
+  })) {
+    return reconcilePendingDeployEvent({
+      id: pending.id, status: "failed", requireLeaseExpired: false, requireExecutor: pending.executor, now,
+      reason: "self-healed: the deploy driver process is confirmed gone and the fleet has not converged on the target sha (#2105)",
+    });
+  }
+  if (now - leaseAt >= leaseMs) {
+    return reconcilePendingDeployEvent({
+      id: pending.id, status: "failed", requireLeaseExpired: true, now, leaseMs,
+      reason: "self-healed: deploy lease expired and the fleet has not converged on the target sha (#1532)",
+    });
+  }
+  return null;
+}
+
+/**
+ * #1551 part 2 — reconcile a stale SERVED dashboard bundle when the fleet's runtimes are already
+ * converged (the catch-22: once every host attests the target SHA, the host-SHA coverage below is
+ * satisfied so no deploy runs to rebuild the dashboard that an earlier FAILED deploy left stale).
+ *
+ * Only meaningful for a $PUBLIC_DIR override host (the served dir `humwick upgrade` never
+ * touches) — with PUBLIC_DIR unset the runtime upgrade itself refreshes the bundled dashboard, so
+ * there's nothing to reconcile. When the served bundle's build-SHA marker predates the target SHA,
+ * rebuild the dashboard DIRECTLY from the deps-ready converged artifact checkout (the one the fleet
+ * already runs) into $PUBLIC_DIR — no relay/orchestrator restart, no host mutation, just a local
+ * build + mirror. Best-effort: any missing prerequisite or a build failure is logged and left for a
+ * later full deploy, never a hard failure (the fleet's runtimes are healthy).
+ *
+ * Returns `{ rebuilt: true }` only when it actually rebuilt+mirrored a fresh bundle.
+ */
+export function reconcileServedDashboard(opts: { root: string; sha: string; command: Run; home?: string }): { rebuilt: boolean; reason: string } {
+  const publicDir = process.env.PUBLIC_DIR?.trim();
+  if (!publicDir) return { rebuilt: false, reason: "PUBLIC_DIR unset — the runtime upgrade refreshes the bundled dashboard" };
+  const servedDir = resolve(publicDir);
+  const servedSha = readServedDashboardSha(servedDir);
+  const decision = resolveDashboardRebuild({ servedSha, targetSha: opts.sha, hostRepairPending: false });
+  if (!decision.rebuild) return { rebuilt: false, reason: decision.reason };
+
+  const repoUrl = opts.command("git", ["config", "--get", "remote.origin.url"], opts.root).out;
+  if (!repoUrl) {
+    console.error(`cd-watcher: served dashboard is stale (${decision.reason}) but no origin remote URL to locate the converged checkout — a later full deploy will rebuild it`);
+    return { rebuilt: false, reason: decision.reason };
+  }
+  let checkoutDir: string;
+  try {
+    checkoutDir = gitCheckoutArtifactPaths(opts.home ?? homedir(), opts.sha, repoUrl).checkoutDir;
+  } catch (error) {
+    console.error(`cd-watcher: could not resolve the artifact checkout for a dashboard rebuild: ${errMessage(error)}`);
+    return { rebuilt: false, reason: decision.reason };
+  }
+  if (!existsSync(join(checkoutDir, "node_modules"))) {
+    console.error(`cd-watcher: served dashboard is stale (${decision.reason}) but the converged checkout ${checkoutDir} has no installed deps — a later full deploy will rebuild it`);
+    return { rebuilt: false, reason: decision.reason };
+  }
+  console.log(`cd-watcher: ${decision.reason} — rebuilding from ${checkoutDir} → ${servedDir}`);
+  // deploy-dashboard.ts reads this to stamp the served bundle's build-SHA marker (so a consistent
+  // served dir converges to the target SHA and this stops re-triggering). PUBLIC_DIR is already in
+  // this process's env (the control host serves from it); the build inherits both.
+  const priorBuildSha = relayEnv(process.env, "AGENT_RELAY_DASHBOARD_BUILD_SHA");
+  writeChildEnv(process.env, { AGENT_RELAY_DASHBOARD_BUILD_SHA: opts.sha });
+  try {
+    const build = opts.command("bun", ["run", "build:dashboard"], checkoutDir);
+    if (build.code !== 0) {
+      console.error(`cd-watcher: dashboard rebuild failed (exit ${build.code}): ${build.err || build.out}`);
+      return { rebuilt: false, reason: `dashboard rebuild attempted but failed (${decision.reason})` };
+    }
+    return { rebuilt: true, reason: `Rebuilt the served dashboard (${decision.reason}).` };
+  } finally {
+    if (priorBuildSha === undefined) writeChildEnv(process.env, {}, ["AGENT_RELAY_DASHBOARD_BUILD_SHA"]);
+    else writeChildEnv(process.env, { AGENT_RELAY_DASHBOARD_BUILD_SHA: priorBuildSha });
+  }
+}
+
+/** #3692 — this watcher is synchronous, so the post-stamp hub wake runs as the same helper release.ts awaits. */
+function wakeHubSync(command: Run, taskIds: number[]): void {
+  if (!taskIds.length) return;
+  try {
+    const wake = command("bun", ["run", join(import.meta.dir, "cd-hub-wake.ts"), ...taskIds.map(String)]);
+    if (wake.code !== 0) throw new Error(wake.err || wake.out);
+  } catch (error) {
+    // Best effort: the deploy already succeeded, and the next hub boot reconciles waiting plans.
+    console.error(`cd-watcher: plans waiting on the deployed Tasks were not woken: ${errMessage(error)}`);
+  }
+}
+
+export { CD_RESTART_EPOCH_TICK_BUDGET_MS, resolveCdRestartEpochTickBudgetMs } from "./cd-config.ts";
+
+export function runCdWatcher(opts: {
+  root: string;
+  dbPath: string;
+  runCommand?: Run;
+  dashboardArtifactHome?: string;
+  now?: number;
+  debounce?: CdDeployDebounce;
+  /** #2727 — the temp dir deploy worktrees are created in; defaults to the OS temp dir. */
+  deployWorktreeTempDir?: string;
+  /** #2189 — test seam for the restart-epoch hold (clock/sleep/budget). Production passes nothing
+   *  and takes a real wall clock, Bun.sleepSync, and the env-resolved tick budget. */
+  restartEpochHold?: { now?: () => number; sleep?: (ms: number) => void; tickBudgetMs?: number; pollIntervalMs?: number };
+} = { root: process.cwd(), dbPath: resolveCanonicalRelayDbPath() }): CdWatcherResult {
+  const command = opts.runCommand ?? run;
+  const now = opts.now ?? Date.now();
+  // #2727 — one seam for both reclaim sites; the module owns the policy (including the live-deploy
+  // fence the start site needs) and logs itself, so a failed recovery can never reach the deploy.
+  const reclaimSweep = () => ({
+    repoRoot: opts.root, run: command,
+    ...(opts.deployWorktreeTempDir ? { tempDir: opts.deployWorktreeTempDir } : {}),
+  });
+  // #1868 — ATTACH to the DB the live Relay owns; never bootstrap it. The prior initDb() here re-ran
+  // the FULL catalog DDL + applyMigrations() + an unconditional `PRAGMA optimize` (which re-ANALYZEs
+  // churned tables like `messages`) against the live ~1.5GB DB on EVERY ~80s tick — taking the SQLite
+  // writer/checkpoint lock while the main-loop hot-path writes spun in `busy_timeout`, freezing the
+  // server's event loop (freezes up to 96s). openExistingDb runs NONE of that DDL/optimize work: the
+  // live server already bootstrapped and owns the schema; this child only reads/writes the deploy
+  // ledger. Mirrors the reopen below (#1685) and the general #1724 off-loop rule.
+  openExistingDb(opts.dbPath);
+  // Best-effort self-heal: a transient registry/db throw here must never abort the tick (which
+  // would also skip the CI-transition alert downstream). The orphan just heals on a later tick.
+  try {
+    reconcileStalePendingDeploy({ now });
+  } catch (error) {
+    console.error(`cd-watcher: stale-pending reconcile failed (continuing tick): ${errMessage(error)}`);
+  }
+  const changesBackfill = listDeployEventsNeedingChanges(3);
+  // #2988 — a partial batch is a step of a rollout, not its end: ranges start at the last complete one.
+  const previous = latestCompleteDeploy();
+  const lastBatch = latestSuccessfulDeploy();
+  const previousSha = shortSha(previous?.commitSha);
+  // #1765 — RELEASE the DB connection across the multi-second network `git fetch` in
+  // selectDeployableMainCi below. This runs in cd-watcher's detached child, which attaches to the
+  // database the live Relay owns; holding even an idle connection open across the fetch pins a WAL
+  // read-mark that blocks the server's synchronous checkpoint, freezing its event loop for the whole
+  // fetch (the recurring ~2.3s event-loop-lag spike). The fetch/CI observation needs no DB, so close
+  // now and reopen only for the coverage/lease/deploy writes after the observation is in hand. This
+  // is the general #1724 rule: an off-thread job must not hold a DB connection across a long non-DB op.
+  closeDb();
+  // #2727 — recover the deploy worktrees a killed run left behind (~168 MB each), in the same closed
+  // window as the git work below: the sweep spawns git and recursively removes those trees, so it must
+  // not pin the live relay's WAL checkpoint. Its pending-row read OPENS the ledger for that one point
+  // read — this window holds no connection by design — and the whole fence lives in the module.
+  reclaimStaleDeployWorktreesAtTickStart({
+    ...reclaimSweep(),
+    readPendingLeaseAt: () => withDeployLedgerDb(opts.dbPath, () => { const p = currentPendingDeployEvent(); return p ? deployEventLeaseAt(p) : null; }),
+    leaseMs: DEPLOY_LEASE_MS,
+  });
+  // #1799 — legacy successful rows are reconciled in this detached child, never
+  // in a request handler. The SQLite handle is closed around every git log so the
+  // live relay's WAL checkpoint cannot be pinned by subprocess work.
+  for (const event of changesBackfill) {
+    try {
+      const changes = changesInDeployRange(command, opts.root, event.previousSha, event.commitSha);
+      withDeployLedgerDb(opts.dbPath, () => {
+        const matched = deployChangesForCommits(changes);
+        persistDeployChangesSummary({
+          id: event.id,
+          summary: { range: { ...(event.previousSha ? { previousSha: event.previousSha } : {}), commitSha: event.commitSha }, ...matched, computedAt: now },
+        });
+      });
+    } catch (error) {
+      console.error(`cd-watcher: deploy changes backfill failed for ${event.id} (continuing tick): ${errMessage(error)}`);
+    }
+  }
+  const selection = selectDeployableMainCi(command, opts.root, previous?.commitSha, now);
+  // Recover a killed driver's Task facts and backfill lands older than the attribution range.
+  try {
+    wakeHubSync(command, withDeployLedgerDb(opts.dbPath, () => completeCdDeployTasks({ root: opts.root, runCommand: command })).taskIds);
+  } catch (error) {
+    console.error(`cd-watcher: Task deployment recovery failed (retrying next tick): ${errMessage(error)}`);
+  }
+  // #1947 — Path-scoped CD: classify the last-deployed → candidate range with the DB STILL RELEASED,
+  // mirroring the #1765/#1790 git-subprocess discipline (the reopen below happens after this git diff).
+  // We only DECIDE here; the skip is actioned after the reopen so the "DB open on return" post-condition
+  // every other exit path relies on is preserved. Guarded on a real baseline: the first deploy has no
+  // last-deployed SHA to diff against, so it deploys (fail-safe). See classifyDeployRangePaths for the
+  // FAIL-SAFE = DEPLOY contract on every uncertain signal.
+  const pathScope = selection.sha && previous?.commitSha
+    ? classifyDeployRangePaths(command, opts.root, previous.commitSha, selection.sha)
+    : undefined;
+  // Reopen for the DB work below. openExistingDb (not initDb): the live Relay owns and has already
+  // migrated the schema, so re-running the catalog DDL/migrations every tick would needlessly
+  // re-contend with the live server (#1685/#1868).
+  openExistingDb(opts.dbPath);
+  const { observation, sha } = selection;
+  const skip = (result: CdWatcherResult): CdWatcherResult => {
+    releaseOwnedRestartEpochDrains(CD_WATCHER_INITIATOR, result.reason, now);
+    return result;
+  };
+  if (!sha) return skip({ outcome: "skipped", sha: null, previousSha, reason: selection.reason ?? "No green origin/main SHA is available.", observation });
+  const targetSha = shortSha(sha);
+  // #1947 — Path-scoped CD skip, decided above (DB was still released for the git diff). BEFORE any
+  // deploy decision: if the last-deployed → candidate range changes ONLY non-deployed paths (docs,
+  // design artifacts, tests-only) we skip the full fleet redeploy. The watcher cursor is NOT corrupted
+  // — latestSuccessfulDeploy is left untouched — so nothing stalls: the next real code land computes
+  // its range from the same baseline and deploys the whole span (the intervening docs commits fold in
+  // harmlessly), while repeated docs-only ticks just re-skip cheaply and idempotently. A pending failed
+  // deploy in backoff, a downgrade guard, etc. are all deploy-suppressing too, so skipping earlier here
+  // never causes a deploy those later guards would have refused.
+  if (pathScope && !pathScope.deploy) {
+    console.error(`cd-watcher: PATH-SCOPED SKIP (#1947): ${previousSha}..${targetSha} changes only non-deployed paths (${pathScope.nonDeployedPaths.length} file(s): ${pathScope.nonDeployedPaths.slice(0, 5).join(", ")}${pathScope.nonDeployedPaths.length > 5 ? ", …" : ""}); skipping the fleet redeploy.`);
+    return skip({
+      outcome: "skipped",
+      sha: targetSha,
+      previousSha,
+      reason: `Path-scoped CD (#1947): ${previousSha}..${targetSha} changes only non-deployed paths (docs/design/tests, ${pathScope.nonDeployedPaths.length} file(s)); skipping the fleet redeploy without moving the fleet.`,
+      observation,
+    });
+  }
+  const hosts = listOrchestrators();
+  const allowlist = resolveCdFleetAllowlist();
+  // #1770 round 3, F2 — pass targetVersion (mirroring reconcileStalePendingDeploy's call above) so
+  // an npm-runtime host attested only by VERSION, not gitSha, is recognized as covered instead of
+  // sitting in coverage.unattested forever and permanently keeping the fail-closed guard below active.
+  const targetVersion = resolveVersionAtSha(command, opts.root, sha);
+  const localOrchestratorId = resolveReleaseLocalOrchestratorId();
+  const coverage = resolveFleetShaCoverage(hosts, sha, allowlist, { localOrchestratorId, targetVersion });
+  // #1770 — No-downgrade guard. Never deploy a candidate that is an ancestor of a SHA the fleet
+  // already runs (or the last-deployed ledger row): that would revert a release. The live
+  // process-attested fleet SHAs are the authority here, because `bun run release` deploys the bump
+  // commit directly without writing a cd-watcher ledger row, so the ledger alone can lag reality.
+  // blocker 2 (widen) — also fold in the currently-PENDING deploy row's commitSha. A release
+  // driver's deploy is `pending` (not yet `succeeded`) while its hosts restart, so
+  // latestSuccessfulDeploy() reads an older ancestor as `previous`; without the pending row the
+  // guard's deployed set would miss the SHA the fleet is actually converging onto (#1769 shape).
+  // Best-effort: a corrupt pending row must not abort the whole tick (the guard still has the
+  // ledger + live fleet SHAs). This mirrors the reconcileStalePendingDeploy swallow above.
+  let pendingDeployCommitSha: string | undefined;
+  try {
+    // #2988 — a draining epoch has no ledger row; a pending row is a deploy that has launched.
+    pendingDeployCommitSha = currentPendingDeployEvent()?.commitSha;
+  } catch (error) {
+    console.error(`cd-watcher: could not read the pending deploy row for the downgrade guard (continuing): ${errMessage(error)}`);
+  }
+  const deployedShas = [
+    ...(previous?.commitSha ? [previous.commitSha] : []),
+    ...(pendingDeployCommitSha ? [pendingDeployCommitSha] : []),
+    ...coverage.expected.map((host) => host.gitSha).filter((value): value is string => !!value),
+  ];
+  const downgradeOf = findDowngradeTarget(command, opts.root, sha, deployedShas);
+  if (downgradeOf) {
+    // should-fix 6 — a guard block that recurs every tick is a wedged reconciliation (fleet ahead of
+    // the newest deployable main SHA). Log loudly so the repeated skip is visible, not silent.
+    console.error(`cd-watcher: NO-DOWNGRADE GUARD BLOCKED a deploy (#1770): candidate ${targetSha} is an ancestor of already-deployed ${shortSha(downgradeOf)}; refusing to move the fleet backward. If this recurs every tick, reconcile the fleet/main mismatch.`);
+    return skip({
+      outcome: "skipped",
+      sha: targetSha,
+      previousSha,
+      reason: `No-downgrade guard (#1770): ${targetSha} is an ancestor of already-deployed ${shortSha(downgradeOf)}; refusing to move the fleet backward.`,
+      observation,
+    });
+  }
+  // #1770 blocker 2 — FAIL CLOSED on an unattested deploy target when the candidate is not
+  // origin/main HEAD. The no-downgrade guard above can only protect against SHAs the fleet
+  // ATTESTS; an `unattested` host has unknown state, so if we are about to deploy a walk-back SHA
+  // (a strict ancestor of HEAD, e.g. the CI fell back off the un-CI'd release bump) we cannot prove
+  // that host isn't actually running something NEWER — deploying would silently downgrade it
+  // (#1769's exact false-negative). When the candidate IS HEAD there is nothing newer on main to
+  // downgrade to, so a permanently-unattested legacy host is still repaired forward as before and
+  // CD is not wedged. Only the ambiguous walk-back-onto-unknown case is refused.
+  if (coverage.unattested.length > 0 && sha !== observation.headSha) {
+    const unattestedIds = coverage.unattested.map((host) => host.id).join(", ");
+    console.error(`cd-watcher: FAIL-CLOSED (#1770): candidate ${targetSha} is not origin/main HEAD (${shortSha(observation.headSha)}) and deploy target(s) [${unattestedIds}] cannot attest a gitSha; refusing to deploy a possible downgrade to a host of unknown state.`);
+    return skip({
+      outcome: "skipped",
+      sha: targetSha,
+      previousSha,
+      reason: `Fail-closed (#1770): ${targetSha} is a non-HEAD walk-back and target host(s) [${unattestedIds}] have no attested gitSha; refusing to risk a downgrade to a host of unknown state.`,
+      observation,
+    });
+  }
+  // A successful fleet-level ledger row only records a past attempt. The live
+  // process-attested SHA is the authority. Missing attestation is separately
+  // tracked because a too-old/misdetecting host may be permanently unable to
+  // report one; failed retries are therefore throttled below.
+  const repairIds = [...coverage.behind, ...coverage.unattested].map((host) => host.id);
+  if (repairIds.length === 0) {
+    if (coverage.expected.length === 0) return skip({ outcome: "skipped", sha: targetSha, previousSha,
+      reason: "No online upgradeable hosts are in the CD fleet allowlist.", observation });
+    const debounce = opts.debounce ?? resolveCdDeployDebounce();
+    // A prior deploy may have restarted this child before it could clean up. Once the fleet
+    // attests that same target, discard the old burst clock so the next independent land gets its
+    // own full quiet window rather than inheriting a stale max-defer deadline.
+    if (readCdDeployDebounceState(debounce.statePath)?.sha === sha) clearCdDeployDebounceState(debounce.statePath);
+    // #1551 part 2: every host's runtime attests the target SHA, but the SERVED dashboard bundle can
+    // still predate it (an earlier failed deploy skipped release.ts's build step). With host-SHA
+    // coverage satisfied no deploy will ever rebuild it — so reconcile the served bundle directly
+    // (best-effort, no host restart) to close the converged-fleet catch-22.
+    closeDb();
+    let dashboardReconcile: ReturnType<typeof reconcileServedDashboard>;
+    try {
+      dashboardReconcile = reconcileServedDashboard({ root: opts.root, sha, command, home: opts.dashboardArtifactHome });
+    } finally {
+      openExistingDb(opts.dbPath);
+    }
+    if (dashboardReconcile.rebuilt) {
+      releaseOwnedRestartEpochDrains(CD_WATCHER_INITIATOR, dashboardReconcile.reason, now);
+      return { outcome: "deployed", sha: targetSha, previousSha, reason: dashboardReconcile.reason, observation };
+    }
+    const allEligible = resolveFleetShaCoverage(hosts, sha, new Set(hosts.map((host) => host.id)));
+    const excludedNeedsDeploy = [...allEligible.behind, ...allEligible.unattested]
+      .some((host) => !isCdFleetHostAllowlisted(host, allowlist));
+    return skip({
+      outcome: "converged",
+      sha: targetSha,
+      previousSha,
+      reason: excludedNeedsDeploy
+        ? "Hosts needing deployment are outside the CD fleet allowlist."
+        : "Target SHA is already deployed to every allowlisted host.",
+      observation,
+    });
+  }
+  const recentFailure = latestFailedDeployForCommit(sha);
+  if (recentFailure?.finishedAt !== undefined && now - recentFailure.finishedAt < CD_WATCHER_FAILURE_BACKOFF_MS) {
+    return skip({ outcome: "skipped", sha: targetSha, previousSha, reason: "A recent failed deploy is still in backoff.", observation });
+  }
+  const debounce = opts.debounce ?? resolveCdDeployDebounce();
+  let debounceState = readCdDeployDebounceState(debounce.statePath);
+  // The durable marker from a completed prior deploy can survive the control-host restart that
+  // deploy caused. A ledger-confirmed old target is not part of this new burst.
+  if (debounceState && debounceState.sha === lastBatch?.commitSha && debounceState.sha !== sha) debounceState = undefined;
+  // Partial batches restart Hosts too; complete-rollout attribution must not freeze the cooldown (#3139).
+  const cooldownApplies = lastBatch && lastBatch.commitSha !== sha && !recentFailure;
+  const debounceDecision = decideCdDeployDebounce({
+    sha, now, debounce, state: debounceState,
+    ...(cooldownApplies ? { lastDeployStartedAt: lastBatch.startedAt } : {}),
+  });
+  writeCdDeployDebounceState(debounce.statePath, debounceDecision.state);
+  const resumingDrain = ownedRestartEpochDrains(CD_WATCHER_INITIATOR).length > 0;
+  // The quiet window precedes admission closure; resetting it during a drain starves renewals (#2798).
+  if (!debounceDecision.deploy && cooldownApplies && now < lastBatch.startedAt + debounce.minIntervalMs) {
+    return skip({ outcome: "skipped", sha: targetSha, previousSha, reason: debounceDecision.reason!, observation });
+  }
+  if (!debounceDecision.deploy && !resumingDrain) {
+    return { outcome: "skipped", sha: targetSha, previousSha, reason: debounceDecision.reason!, observation };
+  }
+  // #2189 P7 / #2988 — durable restart epochs, one per Host, replace the old bounded merge-drain
+  // poller. An epoch that will not seal defers only its own Host: the SAME epoch keeps holding new
+  // work there until a later tick finishes it, while the Hosts that sealed deploy now in one batch.
+  // #2672 — decided before the plan, so neither the batch nor the relay Host's turn waits on such a Host.
+  const heldByLand = upgradeHeldByLand(hosts, localOrchestratorId);
+  const held = holdAndBindCdHostDeploy({
+    sha, previousSha: previous?.commitSha, ownerId: CD_WATCHER_INITIATOR, repairIds,
+    scopes: planCdHostScopes({ repairIds, orchestrators: hosts, localOrchestratorId, landHeld: (id) => heldByLand(id) !== null,
+      relayTurn: () => relayHostsTurn(hosts.find((host) => host.id === localOrchestratorId)?.gitSha,
+        coverage.expected.map((host) => host.gitSha),
+        // A merge-base git cannot answer (e.g. an unfetched object) counts as "lacks it": the hub's turn.
+        (ancestor, descendant) => command("git", ["merge-base", "--is-ancestor", ancestor, descendant], opts.root).code === 0) }),
+    tickBudgetMs: opts.restartEpochHold?.tickBudgetMs ?? resolveCdRestartEpochTickBudgetMs(),
+    // A LIVE clock, deliberately — NOT the tick's frozen `now`, against which the budget check
+    // `now() - startedAt >= budget` would never become true and the poll loop would spin forever.
+    now: opts.restartEpochHold?.now ?? Date.now,
+    sleep: opts.restartEpochHold?.sleep ?? ((ms: number) => { Bun.sleepSync(ms); }),
+    // #1765/#1790 — drop the prod DB connection across every poll sleep, the same rule every other
+    // long span in this file follows. The seal transactions between sleeps reopen it.
+    aroundSleep: (sleep) => { closeDb(); try { sleep(); } finally { openExistingDb(opts.dbPath); } },
+    ...(opts.restartEpochHold?.pollIntervalMs !== undefined ? { pollIntervalMs: opts.restartEpochHold.pollIntervalMs } : {}),
+  });
+  if (!held.bound) return { outcome: "skipped", sha: targetSha, previousSha, reason: held.reason, observation };
+  const { event, targetIds } = held;
+  // The fleet lease now owns this deploy. Clearing before long subprocess work prevents a later,
+  // unrelated green burst from inheriting this burst's max-defer deadline after a relay restart.
+  // A partial batch keeps it: the Hosts still waiting are this same burst, not a new one.
+  if (!event.rolloutPartial) clearCdDeployDebounceState(debounce.statePath);
+  // #1790 — RELEASE the prod-DB connection before the multi-second worktree-add + fleet deploy +
+  // smoke below, exactly as #1765 does across the git fetch above. The fleet-lease row is already
+  // committed by tryBeginDeployEvent; the only remaining DB work is the idempotent ledger-completion /
+  // task-stamp writes, which are reopened for in short bursts via withDeployLedgerDb and never held
+  // across a subprocess op that would pin a WAL read-mark and freeze the live Relay's checkpoint. The
+  // finally reopens the handle so callers after the run (the deploy-path unit tests read the ledger;
+  // prod just exits) see the same open connection they did before #1790.
+  closeDb();
+  // #2727 — reclaim what the PREVIOUS tick's killed deploy left behind (the start sweep above could
+  // not: that run was still live when it ran), in the same closed window as the worktree-add below.
+  reclaimStaleDeployWorktreesBeforeDeploy(reclaimSweep());
+  const worktree = createCdDeployWorktree(opts.deployWorktreeTempDir);
+  try {
+    const add = command("git", ["worktree", "add", "--detach", worktree.checkout, sha], opts.root);
+    if (add.code !== 0) throw new Error(`could not prepare deploy worktree: ${add.err || add.out || `exit ${add.code}`}`);
+    // deploy-only is intentional: it is the existing, proven command-bus fan-out.
+    // Its git-checkout mode consumes the green SHA directly; npm-reuse is opt-in.
+    // #2727 — hand the driver its worktree root so it records itself as a second owner: it re-execs
+    // into a transient scope that outlives this watcher, and only that ownership tells a later tick's
+    // sweep the tree is still in use. Env, not argv, because the re-exec inherits the environment.
+    const deploy = command("bun", cdDeployArgs(sha, undefined, event.id, targetIds), worktree.checkout,
+      { [CD_DEPLOY_WORKTREE_ENV]: worktree.root });
+    if (deploy.code !== 0) throw new Error(`fleet deploy failed: ${deploy.err || deploy.out || `exit ${deploy.code}`}`);
+    // release.ts normally owns this attached row and persists real host statuses.
+    // Keep a compatible fallback for a legacy/interrupted deploy child.
+    // The scoped release driver survives the relay restart; a legacy child may still need
+    // attribution repair here. Task completion always uses the shared ancestry check (#3168).
+    const fallbackNeeded = withDeployLedgerDb(opts.dbPath, () => {
+      const completed = getDeployEvent(event.id);
+      if (completed?.status === "pending") finishDeployEvent({ id: event.id, status: "succeeded", hosts: completed.hosts });
+      const finalized = getDeployEvent(event.id);
+      if (finalized?.status !== "succeeded") return null;
+      // release.ts owns the normal post-finalization cache + Task stamping. Only
+      // a legacy/interrupted child needs this watcher's fallback, so a healthy
+      // deploy runs exactly one git log in its surviving release process.
+      return finalized.changes?.state !== "ready" && cdDeployStampsTasks(finalized);
+    });
+    if (fallbackNeeded === null) throw new Error("fleet deploy did not complete successfully in the deploy ledger");
+    let fallbackCommits: string[] = [];
+    if (fallbackNeeded) {
+      const changes = changesInDeployRange(command, opts.root, previous?.commitSha, sha);
+      fallbackCommits = changes.map((commit) => commit.sha);
+      withDeployLedgerDb(opts.dbPath, () => {
+        const matched = deployChangesForCommits(changes);
+        persistDeployChangesSummary({
+          id: event.id,
+          summary: { range: { ...(event.previousSha ? { previousSha: event.previousSha } : {}), commitSha: event.commitSha }, ...matched, computedAt: now },
+        });
+      });
+    }
+    wakeHubSync(command, withDeployLedgerDb(opts.dbPath, () => completeCdDeployTasks({ root: opts.root, runCommand: command, deployId: event.id })).taskIds);
+    // #2168: Smoke spawning is disabled until its configuration is fixed. Deploy fan-out already
+    // runs with --no-smoke, so do not create a broken smoke agent from the deployed worktree here.
+    const smokeCode = 0;
+    if (fallbackNeeded && smokeCode === 0) {
+      // resolveDeployedVersion reads the worktree's package.json (no DB) — resolve it with the
+      // connection still closed, then reopen only for the stamp write.
+      const liveIn = resolveDeployedVersion(worktree.checkout, sha);
+      withDeployLedgerDb(opts.dbPath, () => {
+        stampVerifiedTasks({ deployId: event.id, commitShas: fallbackCommits, liveIn });
+      });
+    }
+    const reason = !event.rolloutPartial ? "Deployed target SHA to the allowlisted fleet."
+      : `Deployed target SHA to ${targetIds.join(", ")}; the other Hosts deploy as their restart epochs seal.`;
+    return { outcome: "deployed", sha: targetSha, previousSha, reason, observation };
+  } catch (error) {
+    const reason = errMessage(error);
+    withDeployLedgerDb(opts.dbPath, () => {
+      const current = getDeployEvent(event.id);
+      if (current?.status === "pending") finishDeployEvent({ id: event.id, status: "failed", hosts: [...current.hosts, { id: "fleet", status: "failed", reason }] });
+    });
+    return { outcome: "failed", sha: targetSha, previousSha, reason: `Deploy failed: ${reason}`, observation };
+  } finally {
+    removeCdDeployWorktree(worktree, { repoRoot: opts.root, run: command });
+    // #1790 — restore the open handle the run's callers expect afterwards (symmetric with #1765's
+    // reopen). Kept closed across the worktree-remove subprocess immediately above.
+    if (getDbPath() === undefined) openExistingDb(opts.dbPath);
+  }
+}
+
+if (import.meta.main) {
+  // The scheduler must make the same check before scheduling a tick. Keeping it
+  // here too makes direct/cron execution fail closed rather than creating a
+  // second host-local watcher.
+  if (!shouldRunCdWatcherOnThisHost()) {
+    console.log("cd-watcher: skipped (this host is not the designated CD control host)");
+    process.exit(0);
+  }
+  // `join(import.meta.dir, "..")` is this script's package root (…/scripts → package root),
+  // where package.json lives in every layout — the seed resolveCdRepoRoot needs to derive the
+  // origin URL and provision (or reuse) cd-watcher's dedicated detection clone.
+  const result = runCdWatcher({ root: resolveCdRepoRoot(join(import.meta.dir, "..")), dbPath: resolveCanonicalRelayDbPath() });
+  console.log(JSON.stringify(result));
+  process.exitCode = result.outcome === "failed" ? 1 : 0;
+}