@gethmy/harness 1.0.0

package/src/worktree.ts

@@ -0,0 +1,639 @@
+import { execFileSync, execSync } from "node:child_process";
+import { existsSync, rmSync } from "node:fs";
+import { resolve } from "node:path";
+import { log } from "./log.js";
+import { installCommand } from "./pm.js";
+
+const TAG = "worktree";
+
+/**
+ * Minimal client surface the push-rescue needs — just the card-comment write.
+ * Kept structural (not a `HarmonyApiClient` import) so the rescue stays decoupled
+ * and is trivial to mock in unit tests.
+ */
+export interface RescueCommentClient {
+  addComment(
+    cardId: string,
+    body: string,
+    opts?: { commentType?: string },
+  ): Promise<unknown>;
+}
+
+/**
+ * Thrown when the base branch cannot be fetched from origin, so the worktree
+ * cannot be provably based on *current* main (card #408). The worker treats
+ * this as an infrastructure failure — requeue WITHOUT burning the give-up
+ * budget — rather than building the branch on a stale ref and silently
+ * reverting already-merged work.
+ */
+export class WorktreeBaseError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "WorktreeBaseError";
+  }
+}
+
+/**
+ * Fetch the base branch from origin, retrying transient failures. A new branch
+ * MUST be based on the current remote tip — silently falling back to a stale
+ * local `origin/<baseBranch>` ref is what produced the #408 wholesale-revert
+ * branches. After exhausting retries this throws `WorktreeBaseError`.
+ *
+ * `fetchImpl` is injectable so the retry/backoff contract is unit-testable
+ * without touching the network.
+ */
+export function fetchBaseBranch(
+  repoRoot: string,
+  baseBranch: string,
+  attempts = 3,
+  fetchImpl: (root: string, branch: string) => void = (root, branch) =>
+    execFileSync("git", ["fetch", "origin", branch], {
+      cwd: root,
+      stdio: "pipe",
+    }),
+): void {
+  let lastErr: unknown;
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      fetchImpl(repoRoot, baseBranch);
+      return;
+    } catch (err) {
+      lastErr = err;
+      log.warn(
+        TAG,
+        `fetch origin ${baseBranch} failed (attempt ${attempt}/${attempts})`,
+      );
+    }
+  }
+  // Prefer git's stderr (captured by `stdio: "pipe"`) over the generic
+  // "Command failed: …" message so connectivity failures are legible (#444).
+  const e = lastErr as { stderr?: unknown } | null;
+  const detail =
+    e?.stderr?.toString?.().trim() ||
+    (lastErr instanceof Error ? lastErr.message : String(lastErr));
+  throw new WorktreeBaseError(
+    `Could not fetch origin/${baseBranch} after ${attempts} attempts — ` +
+      `refusing to build on a stale base. ${detail}`,
+  );
+}
+
+/**
+ * Decide which ref a new worktree's branch should be based on.
+ *
+ * The default is a fresh per-attempt branch from `origin/<baseBranch>` — a first
+ * implement attempt owns its branch, so resetting to current origin is correct.
+ * But two cases must instead continue the branch's OWN pushed tip: a Playbook
+ * *stage* run shares one per-card branch across stages (a later stage — or a
+ * gate-fail re-run — must build on the tip a prior stage pushed, or that commit
+ * is orphaned and the next stage sees an empty diff, #561); and a generic
+ * *rework* (attempt ≥ 2) must fix the implementation the review rejected rather
+ * than reimplement it from scratch (#689/#732). So when `continueExisting` is
+ * set and the branch is already on origin, base the worktree on
+ * `origin/<branchName>` (its real tip). A first attempt (or the first stage) has
+ * nothing pushed yet, so `branchExistsOnRemote` returns false and we fall back
+ * to a fresh branch — that path is unchanged.
+ *
+ * `branchExistsOnRemote` is a thunk so the decision is unit-testable without git,
+ * and is only consulted on the continuation path (generic runs never probe).
+ */
+export function resolveWorktreeStartRef(
+  baseBranch: string,
+  branchName: string,
+  continueExisting: boolean,
+  branchExistsOnRemote: () => boolean,
+): string {
+  if (continueExisting && branchExistsOnRemote()) {
+    return `origin/${branchName}`;
+  }
+  return `origin/${baseBranch}`;
+}
+
+/**
+ * Fetch a branch from origin so `origin/<branchName>` resolves locally, and
+ * report whether the branch exists on the remote. Connectivity has already been
+ * proven by the preceding `fetchBaseBranch()`, so a failure here means the ref
+ * simply isn't on origin yet (a fresh first-stage run) rather than a network
+ * fault — hence we return false instead of throwing.
+ */
+function fetchExistingBranch(repoRoot: string, branchName: string): boolean {
+  try {
+    execFileSync("git", ["fetch", "origin", branchName], {
+      cwd: repoRoot,
+      stdio: "pipe",
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export interface CreateWorktreeOptions {
+  /**
+   * Continue an existing pushed branch from its origin tip instead of force-
+   * resetting it to `origin/<baseBranch>`. Set for Playbook stage runs (#561)
+   * and generic reworks (attempt ≥ 2, #689/#732). Falls back to a fresh branch
+   * when the branch isn't on origin yet.
+   */
+  continueExisting?: boolean;
+}
+
+/**
+ * Create a git worktree for the agent to work in.
+ * Returns the absolute path to the new worktree.
+ */
+export function createWorktree(
+  basePath: string,
+  baseBranch: string,
+  branchName: string,
+  opts: CreateWorktreeOptions = {},
+): string {
+  const repoRoot = execFileSync("git", ["rev-parse", "--show-toplevel"], {
+    encoding: "utf-8",
+  }).trim();
+
+  const worktreeDir = resolve(repoRoot, basePath, branchName);
+
+  if (existsSync(worktreeDir)) {
+    log.warn(TAG, `Worktree already exists at ${worktreeDir}, cleaning up`);
+    cleanupWorktree(worktreeDir, branchName);
+  }
+
+  // Prune stale worktree metadata. If a previous daemon crashed or its
+  // worktree dir was deleted externally, git may still think the branch is
+  // checked out, which blocks `git branch -D` and `git worktree add`.
+  // `--expire=now` overrides `gc.worktreePruneExpire` (default 3 months) so
+  // freshly-orphaned entries are removed immediately.
+  try {
+    execFileSync("git", ["worktree", "prune", "--expire=now"], {
+      cwd: repoRoot,
+      stdio: "pipe",
+    });
+  } catch {
+    // non-fatal
+  }
+
+  // Fetch latest from remote so the new branch is provably based on CURRENT
+  // origin/<baseBranch>. A persistent fetch failure throws WorktreeBaseError
+  // rather than silently building on a stale ref — the latter is what produced
+  // the #408 branches that wholesale-reverted already-merged work.
+  fetchBaseBranch(repoRoot, baseBranch);
+
+  // Pick the base ref. Normally a fresh per-attempt branch from
+  // origin/<baseBranch>; for a stage continuation, the branch's own pushed tip
+  // so a prior stage's commits survive into this run (#561). `-B` resets the
+  // local branch to whichever ref we chose.
+  const startRef = resolveWorktreeStartRef(
+    baseBranch,
+    branchName,
+    opts.continueExisting ?? false,
+    () => fetchExistingBranch(repoRoot, branchName),
+  );
+
+  log.info(
+    TAG,
+    `Creating worktree: ${worktreeDir} (branch: ${branchName}, base: ${startRef})`,
+  );
+  try {
+    execFileSync(
+      "git",
+      ["worktree", "add", "-B", branchName, worktreeDir, startRef],
+      { cwd: repoRoot, stdio: "pipe" },
+    );
+  } catch (err) {
+    // Last-resort recovery: if `-B` still fails (e.g. branch checked out in
+    // another registered worktree), free the branch and retry.
+    const msg = err instanceof Error ? err.message : String(err);
+    log.warn(TAG, `worktree add failed, attempting forced recovery: ${msg}`);
+    // A DIFFERENT worktree may hold this branch — e.g. a `review-<branch>`
+    // worktree left behind by an interrupted review. git allows one worktree
+    // per branch, so both the retry AND the `git branch -D` below stay blocked
+    // until that holder is removed. Evict it first (#732). Removing our OWN
+    // target path is handled separately below (the holder is never our path
+    // here, but pass it as `exceptDir` to be safe).
+    removeWorktreeHoldingBranch(repoRoot, branchName, worktreeDir);
+    // Remove any registered worktree at this path (phantom or otherwise).
+    try {
+      execFileSync("git", ["worktree", "remove", worktreeDir, "--force"], {
+        cwd: repoRoot,
+        stdio: "pipe",
+      });
+    } catch {
+      // best-effort
+    }
+    // Force-prune any stale worktree admin entries referencing this branch.
+    try {
+      execFileSync("git", ["worktree", "prune", "--expire=now"], {
+        cwd: repoRoot,
+        stdio: "pipe",
+      });
+    } catch {
+      // best-effort
+    }
+    try {
+      execFileSync("git", ["branch", "-D", branchName], {
+        cwd: repoRoot,
+        stdio: "pipe",
+      });
+    } catch {
+      // ignore; retry will surface the real error
+    }
+    execFileSync(
+      "git",
+      ["worktree", "add", "-B", branchName, worktreeDir, startRef],
+      { cwd: repoRoot, stdio: "pipe" },
+    );
+  }
+
+  // Install dependencies in the worktree
+  log.info(TAG, "Installing dependencies in worktree...");
+  try {
+    execSync(installCommand(), {
+      cwd: worktreeDir,
+      stdio: "pipe",
+      timeout: 60_000,
+    });
+  } catch {
+    log.warn(TAG, "Install failed (may be fine if deps are hoisted)");
+  }
+
+  return worktreeDir;
+}
+
+/**
+ * Remove a git worktree and its branch.
+ */
+export function cleanupWorktree(
+  worktreePath: string,
+  branchName?: string,
+): void {
+  const repoRoot = execFileSync("git", ["rev-parse", "--show-toplevel"], {
+    encoding: "utf-8",
+  }).trim();
+
+  if (existsSync(worktreePath)) {
+    try {
+      execFileSync("git", ["worktree", "remove", worktreePath, "--force"], {
+        cwd: repoRoot,
+        stdio: "pipe",
+      });
+      log.info(TAG, `Removed worktree: ${worktreePath}`);
+    } catch (err) {
+      log.warn(
+        TAG,
+        `Failed to remove worktree cleanly: ${err instanceof Error ? err.message : err}`,
+      );
+      // Force-remove the directory if git worktree remove failed
+      if (existsSync(worktreePath)) {
+        rmSync(worktreePath, { recursive: true, force: true });
+      }
+      // Prune stale worktree entries
+      try {
+        execFileSync("git", ["worktree", "prune", "--expire=now"], {
+          cwd: repoRoot,
+          stdio: "pipe",
+        });
+      } catch {
+        // best-effort
+      }
+    }
+  } else {
+    // Directory already gone — an idempotent re-teardown (e.g. completion tore
+    // the worktree down, then a racing cancel re-entered cleanup, #671). Calling
+    // `git worktree remove` here would always throw and log a misleading
+    // "Failed to remove worktree cleanly" warning. Quietly prune any stale
+    // registration instead; the branch delete below still runs.
+    try {
+      execFileSync("git", ["worktree", "prune", "--expire=now"], {
+        cwd: repoRoot,
+        stdio: "pipe",
+      });
+    } catch {
+      // best-effort
+    }
+  }
+
+  // Delete the branch so it doesn't block future runs
+  if (branchName) {
+    try {
+      execFileSync("git", ["branch", "-D", branchName], {
+        cwd: repoRoot,
+        stdio: "pipe",
+      });
+    } catch {
+      // best-effort
+    }
+  }
+}
+
+/**
+ * Find the worktree (if any) currently holding `refs/heads/<branchName>` and,
+ * unless it is `exceptDir`, force-remove it so the branch is free to be
+ * (re)created elsewhere. Returns the evicted path, or `null` when no *other*
+ * worktree held the branch.
+ *
+ * Git permits only ONE worktree per branch. The implement branch
+ * `agent-attempts/<id>-<slug>` is checked out by the review worktree at a
+ * DIFFERENT path (`review-<branch>`), so when a review is interrupted
+ * (timeout / cancel / crash) its worktree can linger holding that branch. A
+ * later rework then fails at `git worktree add -B <branch> …` AND at the
+ * recovery's `git branch -D <branch>` with `fatal: '<branch>' is already used
+ * by worktree at …` — because both the create and the delete are blocked while
+ * a live worktree holds the ref. Evicting that holder first is the missing step
+ * (#732): the old recovery only ever removed the caller's OWN target path, never
+ * the foreign worktree actually pinning the branch.
+ *
+ * Best-effort and total: a `git worktree list` / remove failure returns `null`
+ * rather than throwing, leaving the caller's own error to surface.
+ */
+export function removeWorktreeHoldingBranch(
+  repoRoot: string,
+  branchName: string,
+  exceptDir?: string,
+): string | null {
+  let listing: string;
+  try {
+    listing = execFileSync("git", ["worktree", "list", "--porcelain"], {
+      cwd: repoRoot,
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+  } catch {
+    return null;
+  }
+
+  // Porcelain output is newline-separated records; within a record a
+  // `worktree <path>` line is followed by an optional `branch refs/heads/<ref>`.
+  const target = `refs/heads/${branchName}`;
+  let currentPath: string | null = null;
+  let holderPath: string | null = null;
+  for (const line of listing.split("\n")) {
+    if (line.startsWith("worktree ")) {
+      currentPath = line.slice("worktree ".length).trim();
+    } else if (line.startsWith("branch ")) {
+      const ref = line.slice("branch ".length).trim();
+      if (ref === target && currentPath) {
+        holderPath = currentPath;
+        break;
+      }
+    }
+  }
+
+  if (!holderPath) return null;
+  // Never evict the caller's own target worktree. Resolve both sides so a
+  // trailing-slash / symlink difference can't defeat the guard.
+  if (exceptDir && resolve(holderPath) === resolve(exceptDir)) return null;
+
+  try {
+    execFileSync("git", ["worktree", "remove", holderPath, "--force"], {
+      cwd: repoRoot,
+      stdio: "pipe",
+    });
+    log.warn(
+      TAG,
+      `Evicted worktree ${holderPath} holding branch ${branchName} so it can be reused (#732)`,
+    );
+  } catch (err) {
+    log.warn(
+      TAG,
+      `Failed to evict worktree ${holderPath} holding ${branchName}: ${
+        err instanceof Error ? err.message : err
+      }`,
+    );
+    return null;
+  }
+  // Clear the now-orphaned admin entry so `git branch -D` / `worktree add` see a
+  // free branch.
+  try {
+    execFileSync("git", ["worktree", "prune", "--expire=now"], {
+      cwd: repoRoot,
+      stdio: "pipe",
+    });
+  } catch {
+    // best-effort
+  }
+  return holderPath;
+}
+
+/** Resolve the main repo root, independent of any worktree cwd. */
+function resolveRepoRoot(): string {
+  return execFileSync("git", ["rev-parse", "--show-toplevel"], {
+    encoding: "utf-8",
+  }).trim();
+}
+
+/**
+ * Does a local branch ref `refs/heads/<branchName>` exist? Used to make teardown
+ * idempotent: a re-teardown (or a teardown of a never-created branch) must not
+ * probe a missing ref. `git show-ref --verify --quiet` exits non-zero (no
+ * stderr) when the ref is absent.
+ */
+function localBranchExists(branchName: string, repoRoot: string): boolean {
+  try {
+    execFileSync(
+      "git",
+      ["show-ref", "--verify", "--quiet", `refs/heads/${branchName}`],
+      { cwd: repoRoot, stdio: "ignore" },
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Does `branchName` hold commits that are NOT yet on origin? — the
+ * "committed-but-unpushed" detector the package otherwise lacks (card #587).
+ *
+ * Compares the branch to ITS OWN remote-tracking refs, NOT to base. A base diff
+ * (`origin/<base>..HEAD`, what `completion.checkHasCommits` does) is true on every
+ * feature branch even after a clean push, so reusing it here would fire a
+ * redundant rescue + spurious comment on the already-pushed failed-verify path.
+ * `git rev-list <branch> --not --remotes=origin` lists commits reachable from the
+ * branch but from no origin ref; non-empty ⇒ the branch ref is not fully durable
+ * on origin — only then is a rescue warranted.
+ *
+ * Runs with `cwd = main repo root` so it still works after the worktree dir was
+ * removed (e.g. GC ran before recovery). Returns false on any git error — fail
+ * safe toward the existing delete behaviour rather than block teardown on a flaky
+ * git call.
+ */
+export function branchAheadOfItsRemote(
+  branchName: string,
+  repoRoot = resolveRepoRoot(),
+): boolean {
+  // A deleted (or never-created) branch has nothing to rescue — and running
+  // `git rev-list` on a missing ref prints `fatal: ambiguous argument
+  // '<branch>': unknown revision or path...` to stderr. Gate on ref existence so
+  // an idempotent second teardown is a clean no-op instead of a spurious fatal
+  // in the daemon log (#671).
+  if (!localBranchExists(branchName, repoRoot)) return false;
+  try {
+    const out = execFileSync(
+      "git",
+      ["rev-list", branchName, "--not", "--remotes=origin"],
+      // stdio pipes stderr (rather than inheriting it) so any git diagnostic is
+      // captured with the throw, never leaked to the daemon's own stderr.
+      { cwd: repoRoot, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] },
+    ).trim();
+    return out.length > 0;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Push an about-to-be-deleted branch to origin under its OWN name so the
+ * committed work is durable before teardown destroys the local ref (card #587).
+ *
+ * Pushes the existing `agent-attempts/<shortId>-<slug>` name — no new ref, no
+ * timestamp — so it dedups with completion's failed-verify push and inherits the
+ * existing `failedAttemptRetentionDays` GC. Runs from the main repo root (decision
+ * #3: a branch push needs no worktree cwd, so the rescue still works when the
+ * worktree dir is already gone).
+ *
+ * On success, posts ONE board comment with the recovery ref + a `git fetch &&
+ * git checkout` hint. Once-ness is structural, not a dedup helper: a successful
+ * rescue lets the caller delete the branch, so the teardown path cannot re-enter
+ * for that branch; a failed rescue keeps the branch but (decision #4) does not
+ * comment.
+ *
+ * Returns `true` only when the push landed. A failed push returns `false` so the
+ * caller leaves the local branch ref intact — never `git branch -D` work that is
+ * not durable somewhere.
+ */
+export async function rescueUnpushedBranch(
+  client: RescueCommentClient,
+  cardId: string,
+  branchName: string,
+  repoRoot = resolveRepoRoot(),
+): Promise<boolean> {
+  // Imported lazily to keep git-pr.ts's ~600 lines of PR/provider logic out of
+  // worktree.ts's static dependency graph. git-pr.ts no longer does any work
+  // at module scope (execFileAsync there is lazily created on first call, not
+  // hoisted), so this is no longer about protecting a partial
+  // `node:child_process` mock from a top-level side effect — it is a plain
+  // load-on-demand: the rescue path is the only place worktree.ts needs
+  // git-pr, and every other worktree consumer (worktree-gc, review-worktree,
+  // …) shouldn't pay to load it.
+  const { getBranchWebUrl, pushBranch } = await import("./git-pr.js");
+  try {
+    pushBranch(branchName, repoRoot);
+  } catch (err) {
+    log.error(
+      TAG,
+      `push-rescue failed for ${branchName} — leaving local branch ref intact (recoverable via git reflog / the local branch): ${
+        err instanceof Error ? err.message : err
+      }`,
+    );
+    return false;
+  }
+
+  log.warn(
+    TAG,
+    `push-rescued unpushed branch ${branchName} to origin before teardown`,
+  );
+
+  // Comment is best-effort: a failed comment must NOT undo a successful rescue
+  // (the work is already durable on origin).
+  try {
+    const url = getBranchWebUrl(branchName, repoRoot);
+    const recover = url
+      ? `View it at ${url} or recover locally: \`git fetch && git checkout ${branchName}\``
+      : `Recover it locally: \`git fetch && git checkout ${branchName}\``;
+    const body =
+      `⚠ Run ended before completion. Committed work was push-rescued to ` +
+      `\`origin/${branchName}\` so it isn't lost. ${recover}`;
+    await client.addComment(cardId, body, { commentType: "message" });
+  } catch (err) {
+    log.warn(
+      TAG,
+      `push-rescue comment failed for ${branchName} (work is still safe on origin): ${
+        err instanceof Error ? err.message : err
+      }`,
+    );
+  }
+
+  return true;
+}
+
+/**
+ * Teardown chokepoint with a push-rescue guard (card #587). Every destructive,
+ * branch-passing teardown (completion, recovery, watchdog requeue, defensive
+ * cleanup) routes through here so it inherits the "push before you destroy"
+ * guarantee the completion path already had — closing the dangling-commit hole
+ * where a committed-but-never-pushed branch was `git branch -D`'d into an
+ * unreachable object.
+ *
+ * If the branch holds commits not on origin, push-rescue it first; if the rescue
+ * push fails, KEEP the local branch ref (skip the delete) so the commit stays
+ * recoverable. Then delegate to `cleanupWorktree` for the (unchanged) worktree
+ * removal + conditional branch delete.
+ *
+ * When the branch is already on origin (the happy + failed-verify completion
+ * paths, which push first) the predicate is false and this is a no-op rescue —
+ * behaviour-identical to the old raw `cleanupWorktree(path, branch)`. The GC
+ * sweep (no branch) never calls this; it stays on raw synchronous
+ * `cleanupWorktree`.
+ */
+export async function teardownWorktree(
+  client: RescueCommentClient,
+  cardId: string | null | undefined,
+  worktreePath: string,
+  branchName?: string,
+): Promise<void> {
+  let skipBranchDelete = false;
+
+  if (branchName && cardId) {
+    let repoRoot: string;
+    try {
+      repoRoot = resolveRepoRoot();
+    } catch {
+      // Can't resolve the repo — fall back to a plain cleanup rather than risk
+      // a bogus rescue. cleanupWorktree resolves the root itself.
+      cleanupWorktree(worktreePath, branchName);
+      return;
+    }
+
+    if (branchAheadOfItsRemote(branchName, repoRoot)) {
+      const ok = await rescueUnpushedBranch(
+        client,
+        cardId,
+        branchName,
+        repoRoot,
+      );
+      if (!ok) {
+        skipBranchDelete = true;
+        log.error(
+          TAG,
+          `Keeping local branch ${branchName} (push-rescue failed) to avoid orphaning its commit`,
+        );
+      }
+    }
+  }
+
+  cleanupWorktree(worktreePath, skipBranchDelete ? undefined : branchName);
+}
+
+/**
+ * Generate a branch name from a card's short ID and title.
+ *
+ * Agent branches start under the failedBranchPrefix (default `agent-attempts/`)
+ * and are renamed to the approvedBranchPrefix (default `agent/`) only after the
+ * Review pipeline approves them. Branches under `agent-attempts/` are pruned
+ * by the GC after `failedAttemptRetentionDays`.
+ */
+export function makeBranchName(
+  shortId: number,
+  title: string,
+  prefix = "agent-attempts/",
+): string {
+  const slug = title
+    .toLowerCase()
+    .trim()
+    .replace(/[^\w\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 40);
+  return `${prefix}${shortId}-${slug || "task"}`;
+}