@humwick/server 0.0.0-stage → 0.130.0

package/src/maintenance/workspaces.ts

@@ -0,0 +1,784 @@
+import { releaseExpiredStewardHandbacks } from "./steward-handback";
+import { createCommand } from "../commands-db";
+import {
+  agentsWithRunningCommandTimers,
+  createActivityEventAsync,
+  deleteWorkspace,
+  getAgent,
+  getRepoSteward,
+  getWorkspace,
+  listOrchestrators,
+  listWorkspaces,
+  patchWorkspaceMetadata,
+  reelectRepoSteward,
+  releaseExpiredMergeLeases,
+  updateWorkspaceStatusAsync,
+  setWorkspaceBranchAsync, patchWorkspaceMetadataAsync} from "../db";
+import type { WorkspaceMergePreview, WorkspaceRecord, WorkspaceStatus } from "../types";
+import { reconcileLandedWorkspace } from "../branch-landed";
+import { workspaceStewardEligible } from "../steward-queue";
+import { superviseLandConflict } from "../workspace-conflict-resolver";
+import { preparePrCompletionScan, reconcileLandedPr } from "../workspace-pr-completion";
+import { applyBranchFreshness, fetchHostMergePreview, fetchHostWorkspaceBaseTip, workspacePathWithinBase } from "../workspace-probe";
+import { workspaceAwaitingLandAuthorization } from "../land-authorization";
+import { workspaceLandBlocked } from "../land-block";
+import { isOwnerAlive } from "../workspace-merge";
+import { workspaceOwnerOrchestrator } from "../workspace-host";
+import { cleanupStaleWorkspaces } from "../workspace-cleanup-stale";
+import { IN_FLIGHT_WORKSPACE_STATUSES } from "../workspace-phase";
+import { routeWorkspaceCustodyEscalation, routeWorkspaceCustodyEscalationOrAlarm } from "../workspace-escalation";
+import { raiseOperatorAlarm } from "../operator-alarm";
+import { verifyLandedOrEquivalentContent } from "../services/landed-ancestry";
+import {
+  CONFLICT_SCAN_STATUSES,
+  DAY_MS,
+  LANDED_RECONCILE_STATUSES,
+  STRANDABLE_STATUSES,
+  TERMINAL_WORKSPACE_STATUSES,
+  WORKSPACE_GC_DELETE_BATCH,
+  WORKSPACE_CONFLICT_SCAN_BASE_CHECK_BATCH_SIZE,
+  WORKSPACE_CONFLICT_SCAN_BATCH_SIZE,
+  WORKSPACE_CONFLICT_SCAN_CONCURRENCY,
+  WORKSPACE_CONFLICT_SCAN_UNCHANGED_RECHECK_MS,
+  WORKSPACE_RETENTION_MS,
+  WORKSPACE_REVIEW_TTL_MS,
+  WORKSPACE_STALE_CLEANUP_PROBE_BATCH_SIZE,
+  stewardEscalationMs,
+} from "./constants";
+import { emitCommand } from "./events";
+
+export async function scanWorkspaceConflicts(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  const { signal } = context;
+  // #1819 — genuinely cancellable: every unit of work here is one remote probe of one workspace, so
+  // an abort landing during any of those awaits stops the scan at the NEXT workspace boundary instead
+  // of running the whole batch past the 60s deadline and into the following tick.
+  if (signal?.aborted) return { scanned: 0, aborted: true };
+  await releaseExpiredStewardHandbacks(signal);
+  const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.apiUrl);
+  if (!orchestrators.length) return { scanned: 0, skipped: "no online orchestrators" };
+
+  const now = Date.now();
+  // List order is newest-updated-first, which permanently starves an old backlog when the scan
+  // budget is smaller than the fleet. Prefer the row least recently successfully scanned instead.
+  const allCandidates = listWorkspaces()
+    .filter((ws) => ws.mode === "isolated" && Boolean(ws.worktreePath) && CONFLICT_SCAN_STATUSES.has(ws.status));
+  const candidates = allCandidates
+    .flatMap((ws) => {
+      const orch = workspaceOwnerOrchestrator(ws, orchestrators, { requireApiUrl: true });
+      return orch?.apiUrl ? [{ ws, orch, apiUrl: orch.apiUrl }] : [];
+    });
+  // A tip/status cache is only safe if compared with a CURRENT remote base. Check at most one
+  // workspace per repo before choosing the scan batch; an unreachable base check is fail-closed
+  // (that repo's rows stay eligible), never a reason to trust an old fingerprint.
+  const baseCheckRepoRoots = new Set<string>();
+  const baseCheckCandidates = [...candidates]
+    .sort(({ ws: a }, { ws: b }) => conflictScanAt(a) - conflictScanAt(b) || a.createdAt - b.createdAt || a.id.localeCompare(b.id))
+    .filter(({ ws }) => {
+      if (baseCheckRepoRoots.has(ws.repoRoot)) return false;
+      baseCheckRepoRoots.add(ws.repoRoot);
+      return true;
+    })
+    .slice(0, WORKSPACE_CONFLICT_SCAN_BASE_CHECK_BATCH_SIZE);
+  const currentBaseTips = new Map<string, string>();
+  await runWithConcurrency(baseCheckCandidates, WORKSPACE_CONFLICT_SCAN_CONCURRENCY, async ({ ws, apiUrl }) => {
+    const baseSha = await fetchHostWorkspaceBaseTip(apiUrl, ws);
+    if (!baseSha) return;
+    currentBaseTips.set(ws.repoRoot, baseSha);
+  }, signal);
+  // An abort during the base-tip probes must not fall through into the scan batch: without a current
+  // remote base every row would be (correctly) treated as changed, so the aborted run would then start
+  // the FULL probe batch it has no time budget left for.
+  if (signal?.aborted) return { scanned: 0, aborted: true };
+  const unchanged = candidates.filter(({ ws }) => workspaceConflictScanUnchanged(ws, now, currentBaseTips.get(ws.repoRoot)));
+  const batch = candidates
+    .filter(({ ws }) => !workspaceConflictScanUnchanged(ws, now, currentBaseTips.get(ws.repoRoot)))
+    .sort(({ ws: a }, { ws: b }) => conflictScanAt(a) - conflictScanAt(b) || a.createdAt - b.createdAt || a.id.localeCompare(b.id))
+    .slice(0, WORKSPACE_CONFLICT_SCAN_BATCH_SIZE);
+  const flagged: string[] = [], cleared: string[] = [], merged: string[] = [], notifiedStewards: string[] = [], prRecycled: string[] = [];
+  const reviewerArmed: string[] = [], reviewerChangesRequested: string[] = [], relayPrMerged: string[] = [], relayPrWaiting: string[] = [];
+  // #1473 — branches skipped because no delivery bound to THAT EXACT workspace authorized a land
+  // (held / never-delivered / taskless): they are excluded, fail-closed, from EVERY sweep path here —
+  // PR auto-merge, protected-tip revert, and conflict→steward.
+  const heldUndelivered: string[] = [];
+
+  const scan = async ({ ws, orch, apiUrl }: typeof batch[number]) => {
+    const preview = await fetchHostMergePreview(apiUrl, ws, {
+      checkPr: LANDED_RECONCILE_STATUSES.has(ws.status),
+      // The cache is sound only against an authoritative origin/main tip. A local base can be
+      // stale after another workspace lands and would recreate #1812's missed-conflict window.
+      requireRemoteBase: true,
+    });
+    if (!preview || (preview as { available?: false }).available === false) return;
+    const p = preview as WorkspaceMergePreview;
+    if (p.error || p.missing) return;
+
+    // Discard observations whose workspace changed during the host await. #3288
+    const currentWorkspace = getWorkspace(ws.id);
+    if (!currentWorkspace || currentWorkspace.status !== ws.status || currentWorkspace.worktreePath !== ws.worktreePath) return;
+    ws = currentWorkspace;
+
+    const meta = ws.metadata as Record<string, unknown>;
+    await applyBranchFreshness(ws, p);
+    // Persist only successful host observations. Failed/unavailable probes must be retried, not
+    // hidden behind the unchanged cache, and a future event-driven probe can invalidate this tip.
+    await patchWorkspaceMetadataAsync(ws.id, {
+      conflictScanAt: now,
+      conflictScanHeadSha: p.headSha,
+      conflictScanBaseSha: p.baseSha,
+      conflictScanStatus: ws.status,
+    });
+
+    const landed = p.landed === true || p.prMerged === true;
+    if (landed && LANDED_RECONCILE_STATUSES.has(ws.status)) {
+      const outcome = await reconcileLandedPr(ws, p);
+      if (outcome.kind === "recycle") { emitCommand(outcome.command); prRecycled.push(ws.id); return; }
+      if (outcome.kind === "wait") { relayPrWaiting.push(`${ws.id}:${outcome.reason}`); return; }
+      await reconcileLandedWorkspace(ws, p);
+      merged.push(ws.id);
+      return;
+    }
+
+    // Auto-conflict CLEAR-BACK (restore the pre-conflict status once the conflict resolves) is a
+    // cleanup, not a land — authorization-independent. Handle it BEFORE the land-authorization gate so
+    // a previously auto-flagged row can always recover, even an unauthorized one, and before
+    // PR-completion so a clearing row is never PR-processed on its way out.
+    if (p.conflict === false && ws.status === "conflict" && meta.autoConflict === true) {
+      const prior = typeof meta.preConflictStatus === "string" && meta.preConflictStatus !== "conflict"
+        ? meta.preConflictStatus as WorkspaceStatus
+        : "active";
+      await updateWorkspaceStatusAsync(ws.id, prior, { autoConflict: false, conflictClearedAt: Date.now() });
+      cleared.push(ws.id);
+      return;
+    }
+
+    // #1473: every automatic land path requires exact-head authorization, including active rows.
+    // Reconcile and conflict clear-back above remain authorization-independent.
+    if (workspaceAwaitingLandAuthorization(ws, { currentHead: p.headSha })) { heldUndelivered.push(ws.id); return; }
+
+    const prAction = await preparePrCompletionScan(ws, p, orch.agentId);
+    if (prAction.armCommand) { emitCommand(prAction.armCommand); reviewerArmed.push(ws.id); return; }
+    if (prAction.notifiedOwner) { reviewerChangesRequested.push(ws.id); return; }
+    if (prAction.refreshCommand) { emitCommand(prAction.refreshCommand); return; }
+    if (prAction.mergeCommand) { emitCommand(prAction.mergeCommand); relayPrMerged.push(ws.id); return; }
+    if (prAction.waitReason) {
+      relayPrWaiting.push(`${ws.id}:${prAction.waitReason}`);
+      // #2094 Fix A — DIAGNOSTIC ONLY. A merge_planned row whose PR-strategy land produced NO PR
+      // (`missing-pr`) is the strongest positive evidence of a restart-orphaned merge (the merge
+      // command never opened the PR). Durably stamp WHEN we first observed it — a patchWorkspaceMetadata,
+      // no `updatedAt` bump — so the auto-merge watchdog's executor-liveness reclaim (workspace-auto-merge.ts)
+      // has it available without re-probing. The reclaim DECISION stays in one home (the watchdog); the
+      // scan never reclaims, because a first-tick `missing-pr` can be a PR that is still genuinely opening —
+      // only executor-liveness disambiguates. Stamp once (guarded) so it records the FIRST observation.
+      if (prAction.waitReason === "missing-pr" && ws.status === "merge_planned" && typeof meta.mergeMissingPrObservedAt !== "number") {
+        await patchWorkspaceMetadataAsync(ws.id, { mergeMissingPrObservedAt: now });
+      }
+    }
+    if (prAction.stop) return;
+
+    // The landing lane owns an in-flight workspace; a preview must not hand it back. #3288
+    if (!workspaceStewardEligible(ws)) return;
+
+    // #1249/#1242 — a protected-tip restore (base moved, worker tip reverted) is NOT a content
+    // conflict, so `p.conflict` is false/undefined and every branch below misses it. The land
+    // silently reverts and re-dispatches forever. Escalate to a steward directly (the 10-min
+    // scan is the periodic safety net beside the event-driven auto-merge hook).
+    if (p.protectedTipRestored === true) {
+      // #1953 RC1 — a protected-tip revert (base moved, worker tip reverted) is NOT a git conflict:
+      // a landing steward would spawn, find nothing to resolve, and re-strand the row. Route it to
+      // the branch agent's COORDINATOR (owner/fallback) via the custody path instead — they own the
+      // decision to rebase or abandon. (The event-driven auto-merge hook escalates the same way.)
+      const targets = routeWorkspaceCustodyEscalation({
+        workspace: ws,
+        subject: "Workspace land keeps reverting (moved base)",
+        body: `Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} keeps getting its tip reverted as ${p.baseRef ?? "base"} moves under it (#1242); it can't auto-land. Please coordinate a rebase or clean-up.`,
+        payload: { kind: "workspace.protected-tip-revert", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: p.baseRef, protectedTip: p.protectedTip },
+        idempotencyKey: `workspace-protected-tip-revert:${ws.id}:${p.baseRef ?? ws.baseRef ?? "base"}:${p.protectedTip ?? "?"}`,
+      });
+      if (targets.length) notifiedStewards.push(...targets);
+      return;
+    }
+
+    if (p.conflict === undefined) return;
+
+    if (p.conflict === true && ws.status !== "conflict") {
+      await updateWorkspaceStatusAsync(ws.id, "conflict", {
+        autoConflict: true,
+        preConflictStatus: ws.status,
+        conflictAhead: p.ahead,
+        conflictBehind: p.behind,
+        conflictBaseRef: p.baseRef,
+        conflictDetectedAt: Date.now(),
+      });
+      flagged.push(ws.id);
+      await createActivityEventAsync({
+        clientId: "server-workspace-" + ws.id + "-conflict-" + Date.now(),
+        kind: "state",
+        title: "Merge conflict detected",
+        body: `${ws.branch ?? ws.id} can no longer merge cleanly into ${p.baseRef ?? "base"}`,
+        meta: ws.branch ?? ws.id,
+        icon: "ti-alert-triangle",
+        view: "orchestrators",
+        metadata: { source: "server", maintenanceJobId: "workspace-conflict-scan", workspaceId: ws.id, ahead: p.ahead, behind: p.behind },
+      });
+      // #1912 C3 — the activity write is now a gateway await. A concurrent stop/reconcile can
+      // change the workspace while it is pending, so never wake or notify from the pre-await row.
+      const current = getWorkspace(ws.id);
+      if (!current || current.status !== "conflict" || current.ownerAgentId !== ws.ownerAgentId) return;
+      // #2704 — the repo steward takes the conflict, or with none enabled its owner does.
+      const wake = await superviseLandConflict(current, "conflict", { tip: p.headSha });
+      if (wake.ok) notifiedStewards.push(wake.policy);
+    }
+  };
+
+  const scannedCount = await runWithConcurrency(batch, WORKSPACE_CONFLICT_SCAN_CONCURRENCY, scan, signal);
+
+  return {
+    scanned: scannedCount,
+    ...(signal?.aborted ? { aborted: true, unscannedAfterAbort: batch.length - scannedCount } : {}),
+    skippedUnchanged: unchanged.length,
+    unavailable: allCandidates.length - candidates.length,
+    flagged,
+    cleared,
+    merged,
+    prRecycled,
+    notifiedStewards,
+    reviewerArmed,
+    reviewerChangesRequested,
+    relayPrMerged,
+    relayPrWaiting,
+    heldUndelivered,
+  };
+}
+
+function conflictScanAt(ws: WorkspaceRecord): number {
+  const value = (ws.metadata as Record<string, unknown>).conflictScanAt;
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+function workspaceConflictScanUnchanged(ws: WorkspaceRecord, now: number, currentBaseSha: string | undefined): boolean {
+  const meta = ws.metadata as Record<string, unknown>;
+  return typeof meta.conflictScanHeadSha === "string"
+    && meta.conflictScanHeadSha === meta.gitHeadSha
+    && typeof meta.conflictScanBaseSha === "string"
+    && meta.conflictScanBaseSha === currentBaseSha
+    && meta.conflictScanStatus === ws.status
+    && now - conflictScanAt(ws) < WORKSPACE_CONFLICT_SCAN_UNCHANGED_RECHECK_MS;
+}
+
+/** #1819 — the abort checkpoint sits BETWEEN items, which is the only safe place: one `work(item)` is
+ *  one workspace's probe-then-decide unit, and stopping inside it could leave a status updated without
+ *  the metadata that explains it. Returns how many items actually ran so callers can report the
+ *  shortfall instead of over-reporting an aborted batch as fully scanned. */
+// #2408 — exported so workspace-orphans.ts's reaper can bound its own per-worktree/per-workspace
+// host probes at CONCURRENCY-wide fan-out, the exact same shape of problem this function already
+// solves for the conflict scan. One home for "run N items at bounded concurrency" (the no-duplication
+// rule) rather than a second hand-rolled copy.
+export async function runWithConcurrency<T>(items: T[], concurrency: number, work: (item: T) => Promise<void>, signal?: AbortSignal): Promise<number> {
+  let next = 0;
+  let completed = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      if (signal?.aborted) return;
+      await work(items[next++]!);
+      completed++;
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
+  return completed;
+}
+
+/** #2408 — least-recently-probed-first ordering, ascending by `lastCheckedAt.get(keyOf(item)) ?? 0`
+ *  (never-probed candidates read as 0 and sort first). Home for the bounded-probe-batch rotation
+ *  shared by every per-tick probe budget: the orphan reaper's orphan/missing-worktree/dead-owner
+ *  batches (#2408) and the stale-cleanup sweep (#2554). Lived private in workspace-orphans.ts until
+ *  the sweep became its second caller — moved here, whose module both already import from, rather
+ *  than duplicated. */
+export function sortLeastRecentlyProbed<T>(items: T[], keyOf: (item: T) => string, lastCheckedAt: Map<string, number>): T[] {
+  return [...items].sort((a, b) => {
+    const ka = keyOf(a), kb = keyOf(b);
+    return (lastCheckedAt.get(ka) ?? 0) - (lastCheckedAt.get(kb) ?? 0) || (ka < kb ? -1 : ka > kb ? 1 : 0);
+  });
+}
+
+type AutoAbandonDecision =
+  | { kind: "abandon" }
+  | { kind: "defer"; reason: string }
+  | { kind: "landed"; preview: WorkspaceMergePreview };
+
+// #1543 — exported so the steward-handoff watchdog (workspace-steward-watchdog.ts) can run the
+// SAME "already landed / empty-diff" detection on its own faster hand-off-scoped cadence, instead
+// of duplicating this predicate or waiting up to an hour for this module's own GC/auto-abandon
+// sweeps to close out a landed hand-off.
+//
+// #1631 — NAMED FOR WHAT IT ACTUALLY DECIDES. It was `previewProvesLanded`, which reads as proof
+// that work reached the base; only the first two disjuncts are that. The last two are satisfied
+// just as well by a branch that never carried work — the fresh empty `<branch>-N` a
+// land-and-continue recycle cuts is vacuously "contained in base" — and that vacuous truth
+// travelling under a name promising proof is what let two sweeps conclude "landed" about a
+// running agent's brand-new branch (#1620's phantom merge notice, #1631's retired workspace).
+//
+// The predicate is NOT tightened to exclude the vacuous case, deliberately: "nothing outstanding"
+// is a TRUE and useful statement about such a branch, and every caller's alternative branch is
+// worse for it, not better — the auto-abandon caller would mark it `abandoned` (equally terminal),
+// and the GC/watchdog callers would wake a steward and page a coordinator about a branch with
+// nothing to land, i.e. exactly the phantom escalations #1440/#1543 closed. The strength of the
+// proof was never the deciding input; what makes retiring a workspace safe is that its owner is
+// GONE. That check belongs at the transition, and lives in reconcileLandedWorkspace (#1631),
+// mirroring the host's own no-op land guard (#327).
+export function previewShowsNothingToLand(preview: WorkspaceMergePreview): boolean {
+  const cleanWorktree = (preview.dirtyCount ?? 0) === 0;
+  return preview.landed === true
+    || preview.prMerged === true
+    || preview.noop === true
+    || (cleanWorktree && (preview.unmergedAhead === 0 || preview.ahead === 0));
+}
+
+async function reviewAutoAbandonDecision(ws: WorkspaceRecord, orchestrators: ReturnType<typeof listOrchestrators>): Promise<AutoAbandonDecision> {
+  const owner = workspaceOwnerOrchestrator(ws, orchestrators, { requireApiUrl: true });
+  // #1025 C3 — no reachable owner means we cannot probe land-state. Deferring is the safe
+  // outcome for BOTH cases: a known-but-offline host (may reconnect) AND an unknown host (no
+  // orchestratorId, no path match). The old code abandoned the unknown-host case BLIND — the
+  // less-certain input got the more destructive outcome. Deferring instead keeps the row for
+  // the strand-escalation loop / orphan scan to handle with an actual probe.
+  if (!owner?.apiUrl) return { kind: "defer", reason: ws.orchestratorId ? "owner-unavailable" : "unknown-host" };
+  const preview = await fetchHostMergePreview(owner.apiUrl, ws, { checkPr: true, requireRemoteBase: true });
+  if (!preview || (preview as { available?: false }).available === false) return { kind: "defer", reason: "owner-unavailable" };
+  const p = preview as WorkspaceMergePreview;
+  if (p.error) return { kind: "defer", reason: "preview-error" };
+  if (previewShowsNothingToLand(p)) return { kind: "landed", preview: p };
+  if ((p.dirtyCount ?? 0) > 0) return { kind: "defer", reason: "dirty-worktree" };
+  return { kind: "abandon" };
+}
+
+export async function workspaceGC(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  const { signal } = context;
+  // #1819 — genuinely cancellable at three boundaries: the bounded terminal-delete batch, the strand
+  // escalation loop and the auto-abandon loop. The latter two await a host preview per row, so an
+  // abort lands mid-loop and stops at the next row. Every checkpoint is between rows — never between
+  // a status update and the metadata patch that explains it.
+  if (signal?.aborted) return { aborted: true };
+  const now = Date.now();
+  const cutoff = now - WORKSPACE_RETENTION_MS;
+  const reviewCutoff = now - WORKSPACE_REVIEW_TTL_MS;
+  const releasedLeaseRepos = releaseExpiredMergeLeases(now);
+
+  const all = listWorkspaces();
+  // #1726 — reclaim expired-terminal rows in a BOUNDED batch. GC can be paused for long stretches
+  // (it was, during the 2026-07-26 incident), so this backlog — cleaned/merged/abandoned rows past
+  // the retention cutoff — can be large; deleting the whole backlog in one synchronous pass is an
+  // unbounded write batch. Drain at most WORKSPACE_GC_DELETE_BATCH per run; a periodic (hourly) GC
+  // clears any backlog over a few cadences and the remainder is picked up next run. listWorkspaces
+  // is ORDER BY updated_at DESC, so the tail is the OLDEST expired rows — reclaim those first.
+  // (The per-row delete emits no SSE/event, so bounding the count also bounds any downstream work.)
+  // Unowned host artifacts are recovered independently of these retained registry rows (#2749).
+  const expiredTerminal = all.filter((ws) => TERMINAL_WORKSPACE_STATUSES.has(ws.status) && ws.updatedAt < cutoff);
+  const terminalBatch = expiredTerminal.slice(-WORKSPACE_GC_DELETE_BATCH);
+  const terminalIds: string[] = [];
+  for (const ws of terminalBatch) {
+    if (signal?.aborted) break;
+    deleteWorkspace(ws.id);
+    terminalIds.push(ws.id);
+  }
+  // Rows the abort left undeleted are still expired and terminal, so they stay in the backlog the
+  // next run reports and drains — the count must reflect what this run actually reclaimed.
+  const terminalBacklogRemaining = expiredTerminal.length - terminalIds.length;
+
+  const escalatedIds: string[] = [];
+  const reassignedIds: string[] = [];
+  const escalationTargets: string[] = [];
+  const reconciledLandedIds: string[] = [];
+  const settledLandedElsewhereIds: string[] = [];
+  const suppressedLandBlockedIds: string[] = [];
+  const restoredStrandedIds: string[] = [];
+  const escalationMs = stewardEscalationMs();
+  for (const ws of all) {
+    if (signal?.aborted) break;
+    if (!STRANDABLE_STATUSES.has(ws.status) || ws.mode !== "isolated" || !ws.worktreePath) continue;
+    reelectRepoSteward(ws.repoRoot);
+    const fresh = getWorkspace(ws.id);
+    if (!fresh || !STRANDABLE_STATUSES.has(fresh.status)) continue;
+    const ownerAlive = isOwnerAlive(fresh.ownerAgentId);
+    // A workspace explicitly barred from landing ends its useful life on
+    // its branch ref. A dead owner is not a custody emergency here: never page someone to
+    // merge a workspace that Relay itself has structurally forbidden from merging (#1688).
+    if (!ownerAlive && workspaceLandBlocked(fresh)) {
+      suppressedLandBlockedIds.push(fresh.id);
+      continue;
+    }
+    const meta = fresh.metadata as Record<string, unknown>;
+    const strandedAt = typeof meta.strandedAt === "number" ? meta.strandedAt : undefined;
+
+    const orch = workspaceOwnerOrchestrator(fresh, listOrchestrators().filter((o) => o.status === "online" && o.apiUrl), { requireApiUrl: true });
+    const preview = orch?.apiUrl ? await fetchHostMergePreview(orch.apiUrl, fresh, { checkPr: true, requireRemoteBase: true }) : null;
+    if (preview && !("available" in preview) && !preview.error && !preview.missing) {
+      const landed = previewShowsNothingToLand(preview);
+      if (landed) {
+        await reconcileLandedWorkspace(fresh, preview);
+        await patchWorkspaceMetadataAsync(fresh.id, { strandedAt: undefined, escalatedAt: undefined, strandedNotifiedSteward: undefined });
+        reconciledLandedIds.push(fresh.id);
+        continue;
+      }
+    }
+
+    // Only a fresh host observation may name the workspace tip. Metadata can be stale after
+    // a branch advances, so using it here could settle a newer, genuinely-unlanded commit.
+    // An unavailable host is therefore fail-closed: retain the ordinary warning path.
+    const currentTip = preview && !("available" in preview) && !preview.error && !preview.missing
+      ? (preview as WorkspaceMergePreview).headSha
+      : undefined;
+    const strandedBranch = typeof meta.strandedBranch === "string" ? meta.strandedBranch : undefined;
+    const strandedTip = typeof meta.strandedTip === "string" ? meta.strandedTip : undefined;
+    const branchChangedSinceStranding = strandedBranch !== undefined && strandedBranch !== fresh.branch;
+    const tipChangedSinceStranding = strandedTip !== undefined && currentTip !== undefined && strandedTip !== currentTip;
+    const legacyStrandWithoutBranch = strandedAt !== undefined && strandedBranch === undefined;
+
+    // #1698 — a stranded timestamp belongs to the current branch/tip, never the workspace
+    // lineage. A recycle can retain arbitrary metadata, so a branch transition, new commit, or
+    // pre-marker row starts a fresh clock instead of handing a live worker to a steward.
+    if (ownerAlive) {
+      // #2554 — active⟺live in BOTH directions: an owner back from the (possibly false-offline)
+      // dead restores its stranded row to `active`, so a heartbeat blip can never permanently
+      // demote a live workspace. Status first, then clear the strand markers (the #1698 clock
+      // belongs to the next strand observation, not to the recovered one).
+      if (fresh.status === "stranded") {
+        await updateWorkspaceStatusAsync(fresh.id, "active");
+        await patchWorkspaceMetadataAsync(fresh.id, {
+          strandedAt: undefined,
+          strandedBranch: undefined,
+          strandedTip: undefined,
+          escalatedAt: undefined,
+          strandedNotifiedSteward: undefined,
+          strandRestoredAt: now,
+        });
+        restoredStrandedIds.push(fresh.id);
+        continue;
+      }
+      await patchWorkspaceMetadataAsync(fresh.id, { strandedAt: undefined, strandedBranch: undefined, strandedTip: undefined, escalatedAt: undefined, strandedNotifiedSteward: undefined });
+      continue;
+    }
+    // A duplicate workspace can be stranded after its work was re-landed through a fresh
+    // relander branch. Host previews are advisory and may be unavailable after the original
+    // owner exits, so verify the recorded/observed tip directly against origin/<base>. Passing
+    // the remote-tracking ref explicitly is essential: a local main can be ahead of origin on this
+    // host. A positive proof is ancestry, an exactly equal tip/base tree (the same content
+    // independently landed through a different commit, #2002), or — the old-fork case (#2011) —
+    // every one of the strand's own commits already present upstream by patch-id even though its
+    // tree trails main on later unrelated work. A genuinely-unmerged commit, unresolvable tip, and
+    // every git error deliberately preserve the normal warning path.
+    const baseRef = fresh.baseRef ?? "main";
+    const originBaseRef = baseRef.startsWith("origin/") ? baseRef : `origin/${baseRef}`;
+    const landedElsewhere = currentTip
+      ? await verifyLandedOrEquivalentContent(fresh.repoRoot, currentTip, originBaseRef)
+      : "unresolvable";
+    if (landedElsewhere === "confirmed" || landedElsewhere === "content-equal" || landedElsewhere === "patch-equivalent") {
+      // A tree-equivalent (#2002) or patch-equivalent (#2011) tip is explicitly NOT an ancestor of
+      // origin/base. It can close the stranded row, but must never be attributed as a landed SHA:
+      // that would forge a landing receipt, branch.landed notification, and issue/task lifecycle
+      // update for a commit absent from main. Model both as the existing proven zero-commit
+      // reconciliation (no landedSha, no receipt, only a quiet land_noop). Only a true ancestor
+      // ("confirmed") carries the landed SHA.
+      //
+      // #2558 — and a "confirmed" tip EQUAL to the workspace's own base sha is vacuous too: the
+      // branch carries nothing ahead of base, so the base sitting on main proves nothing landed.
+      // Passing it as landedSha forged a landing receipt citing the base itself and drove the
+      // linked Task (even a canceled one) to landed-on-main. Fold it into the same quiet
+      // zero-commit close-out — the workspace settles, nothing is attributed, no Task is driven.
+      const zeroCommitCloseOut = landedElsewhere !== "confirmed" || currentTip === fresh.baseSha;
+      const proof = landedElsewhere === "confirmed" ? "ancestor" : landedElsewhere;
+      await reconcileLandedWorkspace(fresh, zeroCommitCloseOut
+        ? { strategy: preview && !("available" in preview) ? preview.strategy : "rebase-ff", baseRef, ahead: 0, unmergedAhead: 0, dirtyCount: 0 }
+        : { strategy: preview && !("available" in preview) ? preview.strategy : "rebase-ff", baseRef, headSha: currentTip, landed: true, landedSha: currentTip });
+      await patchWorkspaceMetadataAsync(fresh.id, {
+        landedElsewhere: true,
+        landedElsewhereAt: now,
+        landedElsewhereTip: currentTip,
+        landedElsewhereProof: proof,
+      });
+      reconciledLandedIds.push(fresh.id);
+      settledLandedElsewhereIds.push(fresh.id);
+      continue;
+    }
+    if (strandedAt === undefined || legacyStrandWithoutBranch || branchChangedSinceStranding || tipChangedSinceStranding) {
+      await patchWorkspaceMetadataAsync(fresh.id, {
+        strandedAt: now,
+        strandedBranch: fresh.branch,
+        strandedTip: currentTip,
+        escalatedAt: undefined,
+        strandedNotifiedSteward: undefined,
+      });
+      continue;
+    }
+    const escalatedAt = typeof meta.escalatedAt === "number" ? meta.escalatedAt : 0;
+    if (now - strandedAt < escalationMs) continue;
+    if (escalatedAt && now - escalatedAt < escalationMs) continue;
+
+    // #1953 RC1 — a stranded (owner-exited) row is a CUSTODY problem, not a git conflict: it needs a
+    // new owner/coordinator to adopt or clean it up, which the escalation below already does. It no
+    // longer wakes a landing steward — a stranded-but-clean row would spawn one that finds nothing to
+    // resolve. (A stranded row that is genuinely in `conflict` status is already handed to a steward
+    // by the conflict-scan path; the GC backstop's job here is custody, not conflict resolution.)
+    const strandedHours = Math.round((now - strandedAt) / (60 * 60 * 1000));
+    const targets = routeWorkspaceCustodyEscalation({
+      workspace: fresh,
+      subject: "Stranded workspace needs an owner",
+      body: `Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} is ${fresh.status} and has been stranded for ${strandedHours}h. Please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"} or clean up the worktree.`,
+      payload: { kind: "workspace.stranded-escalation", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt },
+      idempotencyKey: `workspace-stranded:${fresh.id}:${strandedAt}:${Math.floor(now / escalationMs)}`,
+    });
+    if (!targets.length) {
+      // #1021 A4 — the escalation reached ZERO recipients (owner exited, coordinator idle, no
+      // fallback configured). Marking this "escalated" would silently swallow the strand and
+      // repeat every sweep forever. Instead leave it UNescalated (don't stamp escalatedAt, so
+      // the next sweep retries once recipients exist) and raise a louder operator-visible alarm.
+      raiseOperatorAlarm({
+        clientId: `workspace-gc-unescalatable-${fresh.id}:${strandedAt}:${Math.floor(now / escalationMs)}`,
+        source: "workspace-gc",
+        subject: "Stranded workspace has no one to escalate to",
+        detail: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} has been ${fresh.status} for ${strandedHours}h but has no owner, coordinator, or fallback configured to escalate to. Configure AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK or reclaim the worktree manually.`,
+        metadata: { maintenanceJobId: "workspace-gc", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt },
+      });
+      continue;
+    }
+    escalationTargets.push(...targets);
+    await patchWorkspaceMetadataAsync(fresh.id, { escalatedAt: now });
+    escalatedIds.push(fresh.id);
+    await createActivityEventAsync({
+      clientId: `workspace-gc-escalate-${fresh.id}-${now}`,
+      kind: "state",
+      title: "Workspace escalated",
+      body: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} - stranded ${fresh.status} escalated to ${targets.join(", ")}`,
+      meta: fresh.branch ?? fresh.id,
+      icon: "ti-alert-octagon",
+      view: "orchestrators",
+      metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: fresh.id, escalationTargets: targets },
+    });
+  }
+
+  const abandonedIds: string[] = [];
+  const autoAbandonDeferred: string[] = [];
+  const notifiedStewards: string[] = [];
+  const reviewOrchestrators = listOrchestrators();
+  for (const ws of all) {
+    if (signal?.aborted) break;
+    if (ws.mode !== "isolated" || !ws.worktreePath) continue;
+    // #1025 C2 — auto-abandon covers the whole strandable set (ready, review_requested,
+    // conflict, merge_planned), not just `review_requested`; a stuck `ready` was exempt and
+    // leaked forever. Clock the TTL on `readyAt` (the stable hand-off time) rather than
+    // `updatedAt` — updatedAt bumps on every owner heartbeat, so a live-but-idle owner's
+    // workspace never aged past the TTL. Fall back to updatedAt when readyAt is absent
+    // (e.g. an active→conflict autoflag that never passed through `ready`).
+    if (!STRANDABLE_STATUSES.has(ws.status)) continue;
+    if ((ws.readyAt ?? ws.updatedAt) >= reviewCutoff) continue;
+    const fresh = getWorkspace(ws.id);
+    if (!fresh || !STRANDABLE_STATUSES.has(fresh.status)) continue;
+    if ((fresh.readyAt ?? fresh.updatedAt) >= reviewCutoff) continue;
+    const decision = await reviewAutoAbandonDecision(fresh, reviewOrchestrators);
+    if (decision.kind === "defer") {
+      autoAbandonDeferred.push(`${fresh.id}:${decision.reason}`);
+      continue;
+    }
+    if (decision.kind === "landed") {
+      await reconcileLandedWorkspace(fresh, decision.preview);
+      reconciledLandedIds.push(fresh.id);
+      continue;
+    }
+
+    await updateWorkspaceStatusAsync(fresh.id, "abandoned", { autoAbandoned: true, abandonedReason: `${fresh.status} TTL exceeded`, abandonedAt: now });
+    abandonedIds.push(fresh.id);
+    // #1025 C4 — route through the owner/coordinator/fallback custody chain (with a loud
+    // operator alarm when it resolves to nobody) instead of `stewardAgentId ?? fallback`.
+    // Most strandable workspaces have no steward, so the old notice silently dropped while
+    // the state change still happened.
+    const targets = routeWorkspaceCustodyEscalationOrAlarm({
+      source: "workspace-gc",
+      workspace: fresh,
+      type: "workspace.auto_abandoned",
+      subject: "Workspace auto-abandoned",
+      body: `Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} was auto-abandoned after ${Math.round(WORKSPACE_REVIEW_TTL_MS / DAY_MS)}d ${fresh.status} without action. Run workspace cleanup to reclaim the worktree.`,
+      payload: { kind: "workspace.auto-abandoned", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status },
+      idempotencyKey: `workspace-auto-abandoned:${fresh.id}:${now}`,
+    });
+    if (targets.length) notifiedStewards.push(...targets);
+    await createActivityEventAsync({
+      clientId: `workspace-gc-abandon-${fresh.id}-${now}`,
+      kind: "state",
+      title: "Workspace auto-abandoned",
+      body: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} - ${fresh.status} past the ${Math.round(WORKSPACE_REVIEW_TTL_MS / DAY_MS)}d TTL`,
+      meta: fresh.branch ?? fresh.id,
+      icon: "ti-clock-x",
+      view: "orchestrators",
+      metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: fresh.id, status: fresh.status },
+    });
+  }
+
+  const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.agentId);
+  const reposWithWorkspaces = new Set(
+    all.filter((ws) => ws.mode === "isolated" && Boolean(ws.worktreePath)).map((ws) => ws.repoRoot),
+  );
+  const pruneCommands: string[] = [];
+  for (const repoRoot of reposWithWorkspaces) {
+    if (signal?.aborted) break;
+    const orch = orchestrators.find((candidate) => workspacePathWithinBase(repoRoot, candidate.baseDir));
+    if (!orch?.agentId) continue;
+    const command = createCommand({
+      type: "workspace.prune",
+      source: "system",
+      target: orch.agentId,
+      params: { repoRoot, requestedBy: "workspace-gc", requestedAt: now },
+    });
+    emitCommand(command);
+    pruneCommands.push(command.id);
+  }
+
+  return {
+    ...(signal?.aborted ? { aborted: true } : {}),
+    prunedTerminal: terminalIds,
+    terminalBacklogRemaining,
+    autoAbandoned: abandonedIds,
+    autoAbandonDeferred,
+    notifiedStewards,
+    pruneCommands,
+    releasedLeaseRepos,
+    escalated: escalatedIds,
+    reassigned: reassignedIds,
+    escalationTargets,
+    reconciledLanded: reconciledLandedIds,
+    settledLandedElsewhere: settledLandedElsewhereIds,
+    suppressedLandBlocked: suppressedLandBlockedIds,
+    restoredStranded: restoredStrandedIds,
+  };
+}
+
+/**
+ * Proactively refresh idle branch-agent workspaces against origin/main (#478).
+ *
+ * For each `active` isolated workspace whose owner agent is NOT busy and whose
+ * stored probe data indicates no WIP (dirty=0, ahead=0), dispatch a
+ * `workspace.idle-refresh` command to the owning orchestrator. The orchestrator
+ * re-verifies liveness, fetches origin, and — when the worktree is cleanly
+ * behind — performs a `checkout -B <fresh> origin/main` so the agent is already
+ * current when it next wakes. Workspaces with WIP, commits ahead, or busy owners
+ * are skipped silently.
+ */
+export async function refreshIdleWorkspaces(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  const { signal } = context;
+  // #1819 — the body never awaits (it only enqueues commands), so on one thread these checks cannot
+  // fire mid-run today: this job is in the SYNCHRONOUS set and its deadline cannot interrupt it. The
+  // checkpoints are here so the guarantee arrives with the first await, not a later incident.
+  if (signal?.aborted) return { dispatched: [], skipped: [], aborted: true };
+  const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.agentId);
+  if (!orchestrators.length) return { dispatched: 0, skipped: 0, reason: "no online orchestrators" };
+
+  const candidates = listWorkspaces().filter(
+    (ws) => ws.mode === "isolated" && ws.status === "active" && Boolean(ws.worktreePath),
+  );
+
+  const dispatched: string[] = [];
+  const skipped: string[] = [];
+  const now = Date.now();
+  const runningCommandOwners = agentsWithRunningCommandTimers();
+
+  for (const ws of candidates) {
+    if (signal?.aborted) break;
+    const orch = workspaceOwnerOrchestrator(ws, orchestrators);
+    if (!orch?.agentId) { skipped.push(`${ws.id}:no-orchestrator`); continue; }
+
+    // Model-idle owners can still have detached commands executing in their worktree. #3448
+    const owner = ws.ownerAgentId ? getAgent(ws.ownerAgentId) : null;
+    if (!owner || owner.status === "busy") { skipped.push(`${ws.id}:owner-busy`); continue; }
+    if (runningCommandOwners.has(owner.id)) { skipped.push(`${ws.id}:owner-command-active`); continue; }
+
+    // Gate: skip if stored metadata shows WIP or commits ahead.
+    // The orchestrator re-checks live before mutating; stored data is a fast pre-filter.
+    const meta = ws.metadata as Record<string, unknown>;
+    const gitAhead = typeof meta.gitAhead === "number" ? meta.gitAhead : undefined;
+    const gitDirtyCount = typeof meta.gitDirtyCount === "number" ? meta.gitDirtyCount : undefined;
+    if (gitAhead !== undefined && gitAhead > 0) { skipped.push(`${ws.id}:ahead`); continue; }
+    if (gitDirtyCount !== undefined && gitDirtyCount > 0) { skipped.push(`${ws.id}:dirty`); continue; }
+
+    const command = createCommand({
+      type: "workspace.idle-refresh",
+      source: "system",
+      target: orch.agentId,
+      correlationId: ws.id,
+      params: {
+        workspaceId: ws.id,
+        repoRoot: ws.repoRoot,
+        worktreePath: ws.worktreePath,
+        branch: ws.branch,
+        baseRef: ws.baseRef,
+        baseSha: ws.baseSha,
+        requestedBy: "workspace-idle-refresh",
+        requestedAt: now,
+      },
+    });
+    emitCommand(command);
+    dispatched.push(ws.id);
+  }
+
+  return { dispatched, skipped, ...(signal?.aborted ? { aborted: true } : {}) };
+}
+
+/**
+ * Apply the result of a settled `workspace.idle-refresh` command.
+ * When the orchestrator refreshed the worktree, repoint the DB row to the
+ * new branch and update base_sha so the next merge targets the live branch.
+ */
+export async function applyIdleRefreshResult(workspaceId: string, result: Record<string, unknown>): Promise<boolean> {
+  const newBranch = typeof result.newBranch === "string" ? result.newBranch : undefined;
+  const baseSha = typeof result.baseSha === "string" ? result.baseSha : undefined;
+  if (!newBranch || result.refreshed !== true) return false;
+  await setWorkspaceBranchAsync(workspaceId, newBranch, baseSha);
+  return true;
+}
+
+// #2554 — least-recently-probed rotation tracker for the stale-cleanup sweep. In-memory, like the
+// orphan reaper's trackers: a relay restart re-arms the rotation from scratch — conservative.
+const staleCleanupLastProbedAt = new Map<string, number>();
+
+/** #2554 — test seam: the rotation tracker is module state; reset between tests. */
+export function resetStaleCleanupSweepStateForTests(): void {
+  staleCleanupLastProbedAt.clear();
+}
+
+/**
+ * #2554 — the periodic, fleet-wide run of the `cleanupStaleWorkspaces` primitive. The primitive
+ * (workspace-cleanup-stale.ts) already carried the correct safety — offline/dead owner, no steward
+ * claim, no live squatter, land-safe tree (missing/landed/empty, or a server-side proof for an
+ * owner process-confirmed gone whose host is unreachable), guarded dispatch — but nothing invoked
+ * it periodically, so dead-owner `active` rows (the dominant end state for reaped reviewers,
+ * research/scratch agents, and crashed workers) leaked their row and worktree forever.
+ *
+ * This wrapper owns ONLY candidate narrowing and the per-tick probe budget: every per-row decision
+ * (steward claims, live squatters, land-safety proofs, guarded dispatch) stays in the primitive, so
+ * there is exactly one reclaim predicate in the repo. Candidates are picked by the cheap probe-free
+ * liveness gate only; a live owner's worktree is never even listed. The batch bound mirrors the
+ * orphan reaper's #2408 shape: at most WORKSPACE_STALE_CLEANUP_PROBE_BATCH_SIZE sequential
+ * owning-host probes (10s worst case each) per pass, least-recently-probed first, with `moreWork`
+ * rescheduling the next pass at the short backlog interval instead of the full one.
+ */
+export async function staleWorkspaceCleanupSweep(context: { signal?: AbortSignal } = {}): Promise<Record<string, unknown>> {
+  const { signal } = context;
+  if (signal?.aborted) return { scanned: 0, probed: 0, moreWork: false, aborted: true };
+  const now = Date.now();
+  const candidates = listWorkspaces().filter((ws) =>
+    ws.mode === "isolated" && Boolean(ws.worktreePath)
+    && !TERMINAL_WORKSPACE_STATUSES.has(ws.status)
+    // a cleanup/land was already dispatched — re-sweeping would duplicate the command
+    && !IN_FLIGHT_WORKSPACE_STATUSES.has(ws.status)
+    // never list, never probe, a live owner's worktree
+    && !isOwnerAlive(ws.ownerAgentId),
+  );
+  // Rows that left the candidate set (terminalized, dispatched, owner back alive) must not pin
+  // stale rotation entries forever — same tracker hygiene as the orphan reaper's dead-owner map.
+  const candidateIds = new Set(candidates.map((ws) => ws.id));
+  for (const key of staleCleanupLastProbedAt.keys()) {
+    if (!candidateIds.has(key)) staleCleanupLastProbedAt.delete(key);
+  }
+  if (!candidates.length) return { scanned: 0, probed: 0, moreWork: false };
+  const batch = sortLeastRecentlyProbed(candidates, (ws) => ws.id, staleCleanupLastProbedAt)
+    .slice(0, WORKSPACE_STALE_CLEANUP_PROBE_BATCH_SIZE);
+  const result = await cleanupStaleWorkspaces({ dryRun: false, workspaceIds: batch.map((ws) => ws.id), signal });
+  for (const ws of batch) staleCleanupLastProbedAt.set(ws.id, now);
+  return { ...(result as Record<string, unknown>), probed: batch.length, moreWork: candidates.length > batch.length };
+}