agent-relay-server 0.27.1 → 0.27.2

package/src/workspace-orphans.ts

@@ -5,11 +5,15 @@
 // visible to the agent or the dashboard, so unlanded work can sit stranded for
 // weeks (one real casualty: a CI-guard test, recovered by hand).
 //
-// THE invariant (single home, see `worktreeReapable`): reaping is gated on
-// "nothing would be lost" — landed or empty — NEVER on session liveness or a
-// timer. A worktree holding un-landed commits is flagged for attention, never
-// force-removed. This module is the disk⇄DB reconciler the GC's `git worktree
-// prune` (a no-op while the directory exists) never was.
+// THE invariant (single home, see `worktreeReapable`): a worktree is reaped only
+// when BOTH hold — "nothing would be lost" (landed or empty; un-landed commits are
+// flagged, never force-removed) AND "nobody is using it" (the owner is dead, and no
+// live row claims the path). #278 proved the first half alone is not enough: a
+// post-land recycled worktree (ahead 0) owned by a LIVE session is reap-safe by
+// land-state, yet destroying it kills that session's toolchain mid-flight. Path-keyed
+// row matching (never repoRoot-scoped) plus an owner-liveness guard enforce the
+// second half. This module is the disk⇄DB reconciler the GC's `git worktree prune`
+// (a no-op while the directory exists) never was.
 
 import { resolve } from "node:path";
 import { RELAY_TOKEN_HEADER } from "agent-relay-sdk";
@@ -20,6 +24,7 @@ import { emitRelayEvent } from "./events";
 import { isPathWithinBase } from "./utils";
 import { TERMINAL_WORKSPACE_STATUSES, worktreeReapable, type WorktreeReapState } from "./workspace-phase";
 import { isOwnerAlive } from "./workspace-merge";
+import { applyWorkspaceAction } from "./workspace-actions";
 
 // Don't re-flag the same un-landed orphan every sweep — surface it once, then
 // stay quiet for this window. In-memory (keyed by worktree path) like the
@@ -29,10 +34,21 @@ const UNLANDED_FLAG_COOLDOWN_MS = Number(process.env.AGENT_RELAY_ORPHAN_FLAG_COO
 // remove them (parity with the session reaper's detect-only switch).
 const orphanWorktreeReapEnabled = () => process.env.AGENT_RELAY_ORPHAN_WORKTREE_REAP !== "0";
 const flaggedAt = new Map<string, number>();
-const IN_FLIGHT_MISSING_WORKTREE_STATUSES = new Set<WorkspaceStatus>(["merge_planned", "cleanup_requested"]);
+const IN_FLIGHT_WORKSPACE_STATUSES = new Set<WorkspaceStatus>(["merge_planned", "cleanup_requested"]);
+
+// #279 dead-owner grace window: a tracked worktree whose owner has died is only
+// reaped after the owner is observed dead continuously for this window, so a
+// reconnecting agent isn't raced. Keyed by resolved worktree path; in-memory like
+// the session reaper's tracker (a restart re-arms the clock — conservative).
+const orphanGraceMs = (): number => {
+  const v = Number(process.env.AGENT_RELAY_ORPHAN_GRACE_MS);
+  return Number.isFinite(v) && v >= 0 ? v : 30 * 60 * 1000;
+};
+const deadOwnerTracker = new Map<string, { firstSeenDeadAt: number }>();
 
 export function resetOrphanWorktreeStateForTests(): void {
   flaggedAt.clear();
+  deadOwnerTracker.clear();
 }
 
 interface OnlineOrchestrator {
@@ -138,6 +154,17 @@ export interface CollectOrphansResult {
     baseSha?: string;
     ownerAgentId?: string;
   }>;
+  /** Non-terminal isolated rows whose worktree IS present on disk (#279). Tracked,
+   * not orphaned — the reaper decides on owner-liveness + grace + land-state. */
+  deadOwnerWorkspaces: Array<{
+    workspaceId: string;
+    worktreePath: string;
+    repoRoot: string;
+    branch?: string;
+    baseRef?: string;
+    baseSha?: string;
+    ownerAgentId?: string;
+  }>;
   reason?: string;
 }
 
@@ -150,11 +177,27 @@ export interface CollectOrphansResult {
  */
 export async function collectWorkspaceOrphans(): Promise<CollectOrphansResult> {
   const orchestrators = onlineOrchestrators();
-  if (!orchestrators.length) return { orphans: [], missingWorktrees: [], reason: "no online orchestrators" };
+  if (!orchestrators.length) return { orphans: [], missingWorktrees: [], deadOwnerWorkspaces: [], reason: "no online orchestrators" };
 
   const all = listWorkspaces();
   const orphans: WorkspaceOrphan[] = [];
   const missingWorktrees: CollectOrphansResult["missingWorktrees"] = [];
+  const deadOwnerWorkspaces: CollectOrphansResult["deadOwnerWorkspaces"] = [];
+
+  // Worktree paths are globally unique, so a row from ANY repoRoot that records a
+  // path is the authoritative claim on it. Match disk→DB across ALL rows, NEVER
+  // scoped to one repoRoot: the scoped version false-orphaned chained workspaces
+  // (managed session → base checkout → isolated worktree, whose row records the base
+  // checkout as repoRoot, not the main repo the probe is seeded from), so the reaper
+  // destroyed a live session's worktree (#278).
+  const rowsByPath = new Map(all.filter((ws) => ws.worktreePath).map((ws) => [resolve(ws.worktreePath), ws]));
+
+  // Union of every probed repo's on-disk worktrees + the repoRoots we actually
+  // reached. The DB→disk (missing-worktree) pass runs ONCE, globally, after the loop
+  // against these — per-repo scoping there falsely flagged chained rows as missing
+  // (their worktree lives under a different repo's probe than the one iterating). #278
+  const onDiskAll = new Set<string>();
+  const probedRepoRoots = new Set<string>();
 
   for (const repoRoot of knownRepoRoots(all)) {
     const orch = orchestrators.find((candidate) => candidate.apiUrl && isPathWithinBase(repoRoot, candidate.baseDir));
@@ -162,40 +205,32 @@ export async function collectWorkspaceOrphans(): Promise<CollectOrphansResult> {
     const probe = await fetchHostProbe(orch.apiUrl, repoRoot);
     if (!probe?.worktrees) continue;
 
-    const liveRowsByPath = new Map(
-      all
-        .filter((ws) => ws.repoRoot === repoRoot && ws.worktreePath && !TERMINAL_WORKSPACE_STATUSES.has(ws.status))
-        .map((ws) => [resolve(ws.worktreePath), ws]),
-    );
-    const onDisk = new Set(probe.worktrees.map((wt) => (wt.path ? resolve(wt.path) : "")).filter(Boolean));
-
-    // DB→disk drift: a live isolated row whose worktree is no longer on disk.
-    for (const [path, ws] of liveRowsByPath) {
-      if (ws.mode === "isolated" && !onDisk.has(path)) {
-        missingWorktrees.push({
-          workspaceId: ws.id,
-          worktreePath: ws.worktreePath,
-          repoRoot,
-          status: ws.status,
-          branch: ws.branch,
-          baseRef: ws.baseRef,
-          baseSha: ws.baseSha,
-          ownerAgentId: ws.ownerAgentId,
-        });
-      }
-    }
+    probedRepoRoots.add(resolve(repoRoot));
+    for (const wt of probe.worktrees) if (wt.path) onDiskAll.add(resolve(wt.path));
 
-    // disk→DB drift: a worktree on disk with no live row.
-    const rowsByPath = new Map(
-      all.filter((ws) => ws.repoRoot === repoRoot && ws.worktreePath).map((ws) => [resolve(ws.worktreePath), ws]),
-    );
+    // disk→DB drift: a worktree on disk with no live row → orphan. A worktree with a
+    // live isolated row is NOT an orphan, but if its owner has died it's the #279
+    // dead-owner case — surface it for the reaper to evaluate (liveness + grace).
     for (const worktree of probe.worktrees) {
       if (!worktree.path || resolve(worktree.path) === resolve(repoRoot)) continue;
       // Only agent-relay-created worktrees (agent/* branches) are reclaimable —
       // never touch a user's own linked worktrees.
       if (!worktree.branch?.startsWith("agent/")) continue;
       const row = rowsByPath.get(resolve(worktree.path));
-      if (row && !TERMINAL_WORKSPACE_STATUSES.has(row.status)) continue; // tracked & live
+      if (row && !TERMINAL_WORKSPACE_STATUSES.has(row.status)) {
+        if (row.mode === "isolated") {
+          deadOwnerWorkspaces.push({
+            workspaceId: row.id,
+            worktreePath: worktree.path,
+            repoRoot: row.repoRoot,
+            branch: row.branch ?? worktree.branch,
+            baseRef: row.baseRef,
+            baseSha: row.baseSha,
+            ownerAgentId: row.ownerAgentId,
+          });
+        }
+        continue; // tracked & live — not an orphan
+      }
 
       const orphan: WorkspaceOrphan = {
         worktreePath: worktree.path,
@@ -223,7 +258,32 @@ export async function collectWorkspaceOrphans(): Promise<CollectOrphansResult> {
     }
   }
 
-  return { orphans, missingWorktrees };
+  // DB→disk drift: a non-terminal isolated row whose worktree is gone from disk. Run
+  // once, globally, against the union of probed repos. Flag "missing" only when the
+  // path is absent from EVERY probe AND the row was COVERABLE — its repoRoot was
+  // probed, OR its repoRoot is itself a worktree we saw on disk (the chained case: an
+  // isolated row's repoRoot is its session base checkout, a linked worktree of a
+  // probed repo). The coverability gate is what stops rows under an un-probeable host
+  // from being falsely flagged missing. #278
+  for (const ws of all) {
+    if (ws.mode !== "isolated" || !ws.worktreePath) continue;
+    if (TERMINAL_WORKSPACE_STATUSES.has(ws.status)) continue;
+    if (onDiskAll.has(resolve(ws.worktreePath))) continue;
+    const coverable = probedRepoRoots.has(resolve(ws.repoRoot)) || onDiskAll.has(resolve(ws.repoRoot));
+    if (!coverable) continue;
+    missingWorktrees.push({
+      workspaceId: ws.id,
+      worktreePath: ws.worktreePath,
+      repoRoot: ws.repoRoot,
+      status: ws.status,
+      branch: ws.branch,
+      baseRef: ws.baseRef,
+      baseSha: ws.baseSha,
+      ownerAgentId: ws.ownerAgentId,
+    });
+  }
+
+  return { orphans, missingWorktrees, deadOwnerWorkspaces };
 }
 
 function dispatchCleanup(orch: OnlineOrchestrator, orphan: WorkspaceOrphan): string {
@@ -254,7 +314,7 @@ function dispatchCleanup(orch: OnlineOrchestrator, orphan: WorkspaceOrphan): str
  * directions surface. Never removes on uncertainty.
  */
 export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>> {
-  const { orphans, missingWorktrees, reason } = await collectWorkspaceOrphans();
+  const { orphans, missingWorktrees, deadOwnerWorkspaces, reason } = await collectWorkspaceOrphans();
   if (reason) return { skipped: reason };
 
   const orchestrators = onlineOrchestrators();
@@ -263,14 +323,27 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
   const flagged: string[] = [];
   const autoAbandoned: string[] = [];
   const flaggedMissingWorktrees: string[] = [];
+  const deadOwnerReaped: string[] = [];
+  const deadOwnerFlagged: string[] = [];
   const now = Date.now();
 
+  // Defense in depth (#278): the matching fix in collectWorkspaceOrphans should keep
+  // any path with a live, non-terminal row out of `orphans`. This is the belt to that
+  // suspenders — re-read rows fresh and never reap a path one still claims, even if a
+  // future refactor reintroduces a lookup gap. False-reaping a live session is fatal.
+  const nonTerminalRowsByPath = new Map(
+    listWorkspaces()
+      .filter((ws) => ws.worktreePath && !TERMINAL_WORKSPACE_STATUSES.has(ws.status))
+      .map((ws) => [resolve(ws.worktreePath), ws]),
+  );
+
   for (const orphan of orphans) {
     const orch = orchestrators.find((candidate) => isPathWithinBase(orphan.repoRoot, candidate.baseDir));
     if (!orch) continue;
 
     if (orphan.safeToReap === true) {
       if (!reapEnabled) continue; // detect-only mode
+      if (nonTerminalRowsByPath.has(resolve(orphan.worktreePath))) continue; // still claimed — never reap (#278)
       const commandId = dispatchCleanup(orch, orphan);
       reaped.push(orphan.worktreePath);
       flaggedAt.delete(orphan.worktreePath);
@@ -315,7 +388,7 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
     if (!workspace || TERMINAL_WORKSPACE_STATUSES.has(workspace.status) || workspace.mode !== "isolated" || !workspace.worktreePath) continue;
     const key = `missing:${workspace.worktreePath}`;
     const ownerAlive = isOwnerAlive(workspace.ownerAgentId);
-    const inFlight = IN_FLIGHT_MISSING_WORKTREE_STATUSES.has(workspace.status);
+    const inFlight = IN_FLIGHT_WORKSPACE_STATUSES.has(workspace.status);
     const last = flaggedAt.get(key) ?? 0;
     if (ownerAlive || inFlight) {
       if (now - last < UNLANDED_FLAG_COOLDOWN_MS) continue;
@@ -410,9 +483,95 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
     });
   }
 
+  // #279: a tracked worktree whose owner died — the common stale case the reaper
+  // never handled (inverse of #278's false kill). Gate on owner-liveness + a grace
+  // window + "nothing would be lost", then dispatch through the GUARDED action (the
+  // #254 isOwnerAlive re-check at dispatch is the safety net against a stale read).
+  const graceMs = orphanGraceMs();
+  const liveDeadOwnerKeys = new Set<string>();
+  for (const cand of deadOwnerWorkspaces) {
+    const workspace = getWorkspace(cand.workspaceId);
+    if (!workspace || workspace.mode !== "isolated" || !workspace.worktreePath) continue;
+    if (TERMINAL_WORKSPACE_STATUSES.has(workspace.status)) continue;
+    if (IN_FLIGHT_WORKSPACE_STATUSES.has(workspace.status)) continue; // cleanup/merge already dispatched
+    const key = `dead-owner:${resolve(workspace.worktreePath)}`;
+    liveDeadOwnerKeys.add(key);
+
+    if (isOwnerAlive(workspace.ownerAgentId)) {
+      deadOwnerTracker.delete(key); // owner alive — reset the grace window (regression vs #278)
+      continue;
+    }
+
+    // Owner observed dead. Require continuous dead-ness for the grace window so a
+    // reconnecting agent isn't raced; the in-memory tracker re-arms on restart. A
+    // grace of 0 reaps on the first dead observation (the env's escape hatch).
+    const tracked = deadOwnerTracker.get(key);
+    const firstSeenDeadAt = tracked?.firstSeenDeadAt ?? now;
+    if (!tracked) deadOwnerTracker.set(key, { firstSeenDeadAt });
+    if (now - firstSeenDeadAt < graceMs) continue;
+
+    // Grace elapsed + owner dead. Gate on land-state — "nothing would be lost".
+    const orch = orchestrators.find((candidate) => candidate.apiUrl && isPathWithinBase(workspace.repoRoot, candidate.baseDir));
+    const preview = orch?.apiUrl
+      ? await fetchWorktreeReapState(orch.apiUrl, workspace.worktreePath, workspace.baseRef)
+      : null;
+    const safeToReap = preview && !preview.missing && !preview.error
+      ? worktreeReapable({ landed: preview.landed, ahead: preview.ahead, unmergedAhead: preview.unmergedAhead, dirtyCount: preview.dirtyCount })
+      : undefined;
+
+    if (safeToReap === true) {
+      if (!reapEnabled) continue; // detect-only mode
+      // Route through the guarded action (#279): no `force`, so a flipped-alive owner
+      // is rejected (409) — our safety net. Also does the cleanup_requested transition
+      // + audit. Attributed to a distinct requester so incidents are traceable.
+      const result = applyWorkspaceAction(workspace, {
+        action: "cleanup",
+        agentId: "workspace-dead-owner-reaper",
+        auditMetadata: { source: "server", maintenanceJobId: "workspace-orphan-reaper", requestedBy: "workspace-dead-owner-reaper", deadOwner: workspace.ownerAgentId },
+      });
+      if (!result.ok) continue; // guard tripped (owner came back) or no owning orchestrator — re-evaluate next sweep
+      if (result.command) {
+        emitRelayEvent({ type: `command.${result.command.status}`, source: result.command.source, subject: result.command.id, data: { command: result.command } });
+      }
+      deadOwnerReaped.push(workspace.id);
+      deadOwnerTracker.delete(key);
+      continue;
+    }
+
+    // Un-landed or un-probeable: flag as stranded (dead owner), never remove.
+    const flagKey = `dead-owner-stranded:${resolve(workspace.worktreePath)}`;
+    liveDeadOwnerKeys.add(flagKey);
+    const last = flaggedAt.get(flagKey) ?? 0;
+    if (now - last < UNLANDED_FLAG_COOLDOWN_MS) continue;
+    flaggedAt.set(flagKey, now);
+    deadOwnerFlagged.push(workspace.id);
+    const detail = safeToReap === undefined
+      ? "host could not be probed for land-state"
+      : (preview && (preview.dirtyCount ?? 0) > 0)
+        ? "uncommitted changes in the worktree"
+        : `${preview?.unmergedAhead ?? preview?.ahead ?? "?"} un-landed commit(s)`;
+    createActivityEvent({
+      clientId: `workspace-dead-owner-stranded-${workspace.id}-${now}`,
+      kind: "state",
+      title: "Dead-owner worktree needs attention",
+      body: `${workspace.branch ?? workspace.id} in ${workspace.repoRoot} — owning agent ${workspace.ownerAgentId ?? "?"} is gone and the worktree holds work that hasn't landed (${detail}). Recover the commits, then clean it up explicitly.`,
+      meta: workspace.branch ?? workspace.id,
+      icon: "ti-alert-triangle",
+      view: "orchestrators",
+      metadata: { source: "server", maintenanceJobId: "workspace-orphan-reaper", workspaceId: workspace.id, worktreePath: workspace.worktreePath, branch: workspace.branch, ownerAgentId: workspace.ownerAgentId, deadOwner: true },
+    });
+  }
+  // Drop grace-tracker entries for worktrees that are gone (reaped/landed) so a
+  // future re-orphaning re-arms the window from scratch.
+  for (const key of deadOwnerTracker.keys()) if (!liveDeadOwnerKeys.has(key)) deadOwnerTracker.delete(key);
+
   // Forget cooldown entries for orphans that are gone (reaped/recovered) so a
   // future re-orphaning of the same path re-announces immediately.
-  const liveKeys = new Set([...orphans.map((o) => o.worktreePath), ...missingWorktrees.map((m) => `missing:${m.worktreePath}`)]);
+  const liveKeys = new Set([
+    ...orphans.map((o) => o.worktreePath),
+    ...missingWorktrees.map((m) => `missing:${m.worktreePath}`),
+    ...liveDeadOwnerKeys,
+  ]);
   for (const key of flaggedAt.keys()) if (!liveKeys.has(key) && !reaped.includes(key)) flaggedAt.delete(key);
 
   return {
@@ -422,6 +581,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
     autoAbandoned,
     flaggedMissingWorktrees,
     missingWorktrees: missingWorktrees.map((m) => m.workspaceId),
+    deadOwnerReaped,
+    deadOwnerFlagged,
     reapEnabled,
   };
 }