@bevel-software/platform-core-backend 0.13.6 → 0.15.1

package/src/modules/workflow/workflow.service.ts

@@ -25,6 +25,7 @@ import type {
   AcquireLockResult,
   AuthUser,
   Branch,
+  BranchSyncOutcome,
   BranchWorkspaceStatus,
   CancelChangeRequestResult,
   Change,
@@ -40,9 +41,10 @@ import type {
   MergeChangeRequestOutcome,
   OpenChangeRequestInput,
   PostChangeRequestCommentInput,
+  RemoteSyncPullResult,
 } from '@bevel-software/platform-shared';
 import { isProtectedBranch, DEFAULT_BRANCH } from '@bevel-software/platform-shared';
-import { and, eq } from 'drizzle-orm';
+import { and, eq, or } from 'drizzle-orm';
 import type { Database } from '../database/connection.js';
 import { changeRequests } from '../database/schema.js';
 import type { GitService } from './git/git.service.js';
@@ -65,6 +67,7 @@ import {
   RolesYamlPreservationError,
   PullRebaseConflictError,
   PushNeedsAgentResolutionError,
+  RemoteBranchGoneError,
   WorkflowDomainError,
   WorkflowValidationError,
 } from '../../shared/domain-errors.js';
@@ -100,6 +103,33 @@ function isOpenPairViolation(err: unknown): boolean {
   return typeof message === 'string' && message.includes(OPEN_PAIR_CONSTRAINT);
 }
 
+/**
+ * The one sentence a remote-sync conflict is reported with — in the sync
+ * response, on the branch's banner and in the log — so whoever sees any of
+ * the three reads the same thing: which branch, which files, that automatic
+ * recovery is under way, and what to do if it does not clear.
+ */
+export function syncConflictMessage(branch: string, conflictedPaths: string[]): string {
+  // Bounded: the same sentence goes to the response, the banner and the log,
+  // and it must be one sentence in all three whatever the conflict's size.
+  // The full path list rides beside it as `conflictedPaths`.
+  const shown = conflictedPaths.slice(0, SYNC_CONFLICT_FILES_NAMED);
+  const more = conflictedPaths.length - shown.length;
+  const files =
+    conflictedPaths.length === 0
+      ? 'some files'
+      : more > 0
+        ? `${shown.join(', ')} and ${more} more file${more === 1 ? '' : 's'}`
+        : shown.join(', ');
+  return (
+    `${branch} is not in sync yet: ${files} changed both in Hexis and on the git host. ` +
+    `Recovery is queued; if this stays, open the files on ${branch} in Hexis, keep what you want, and save.`
+  );
+}
+
+/** How many conflicted files the sync-conflict sentence names before "and N more". */
+const SYNC_CONFLICT_FILES_NAMED = 3;
+
 /** Redact the shared GitHub token from any error string before surfacing it. */
 function redactTokens(msg: string): string {
   const tokens = [process.env.GITHUB_TOKEN, process.env.GH_TOKEN].filter(
@@ -246,7 +276,10 @@ export class WorkflowService implements IWorkflowService {
    */
   private orphanSweepStarted = false;
 
-  async listBranches(workspaceId: string, opts?: { freshFetch?: boolean }): Promise<Branch[]> {
+  async listBranches(
+    workspaceId: string,
+    opts?: { freshFetch?: boolean; strictFetch?: boolean },
+  ): Promise<Branch[]> {
     const branches = await this.git.listBranches(workspaceId, opts);
     if (!this.orphanSweepStarted) {
       this.orphanSweepStarted = true;
@@ -268,7 +301,14 @@ export class WorkflowService implements IWorkflowService {
   }
 
   createBranch(workspaceId: string, name: string, fromBase?: string): Promise<Branch> {
-    return this.git.createBranch(workspaceId, name, fromBase);
+    // Under the branch's lifecycle lock, like `deleteBranch` and the
+    // retirement of a clone the host deleted: creating a branch is the one
+    // operation that can bring a "gone" branch back, and the retirement's
+    // "still gone on origin?" check is only a guarantee if nothing can
+    // recreate the branch between that check and the delete.
+    return this.branchLifecycle.run(`branch:${name}`, () =>
+      this.git.createBranch(workspaceId, name, fromBase),
+    );
   }
 
   /**
@@ -282,6 +322,85 @@ export class WorkflowService implements IWorkflowService {
    */
   private readonly branchLifecycle = new WorkspaceMutex();
 
+  /**
+   * Hold the lifecycle lock of BOTH branches a change request names, so a
+   * check-then-insert spanning them excludes either one being deleted.
+   *
+   * ATOMICALLY, via `runAll` — not `run(a, () => run(b, …))`. Taking the keys
+   * one at a time leaves a gap between them in which a delete of the second
+   * branch can start, pass its "is any open request using this?" check
+   * (the row does not exist yet) and finish, so the request lands pointing at
+   * a branch that was removed while it was being opened. That is the same
+   * failure this PR is about, reintroduced one level down — and it would have
+   * been a REGRESSION on the source branch, which the previous single-key
+   * lock held for the whole sequence.
+   *
+   * Reserving the set in one step also removes the deadlock question that
+   * sequential acquisition raises: two requests may legally be open in
+   * opposite directions (`A -> B` and `B -> A`), and nothing is ever held
+   * while waiting for the other key.
+   *
+   * A PROTECTED target's key is NOT reserved. No path in the system can
+   * delete a protected branch (`git.deleteBranch` throws
+   * `ProtectedBranchError` before any lock, `retireMergedSourceBranch`
+   * returns early on one), so the key would exclude nothing. It would only
+   * cost: nearly every request targets the protected default branch, so one
+   * shared key there would serialize every open on the hottest path for the
+   * duration of a fetch + merge + push + diff. The source needs no such
+   * carve-out; `openChangeRequest` refuses protected sources before locking.
+   */
+  private runOnBranchPair<T>(
+    source: string,
+    target: string,
+    fn: () => Promise<T>,
+  ): Promise<T> {
+    const keys = [`branch:${source}`];
+    if (!isProtectedBranch(target)) keys.push(`branch:${target}`);
+    return this.branchLifecycle.runAll(keys, fn);
+  }
+
+  /**
+   * The open change requests that deleting `branch` would strand — every one
+   * that NAMES it, at EITHER end.
+   *
+   * Both ends, not just the source. A change request is a proposal to move
+   * commits FROM one branch INTO another and it needs both to exist: the
+   * detail read resolves the target BEFORE the source, so a request whose
+   * target has been deleted cannot be read, declined, or deleted either. It
+   * is also the worse half to lose — a stranded source is closed by the boot
+   * sweep, while a stranded target used to survive every restart, because
+   * every guard here asked only "is this branch anybody's SOURCE?".
+   *
+   * That question is what let a merge retire a branch a second, still-open
+   * request was pointing INTO: the pair `X -> B` (merged, B retired) and
+   * `Y -> X` (open, now unusable forever).
+   *
+   * Returns the first such request's number and which end of it names the
+   * branch, or null when the branch is free. The end matters to the caller's
+   * error message: "withdraw your request" is only actionable when the asker
+   * owns the request, and a request proposing INTO the branch is someone
+   * else's.
+   */
+  private async openChangeRequestOn(
+    branch: string,
+  ): Promise<{ number: number; end: 'source' | 'target' } | null> {
+    const [row] = await this.db
+      .select({ number: changeRequests.number, sourceBranch: changeRequests.sourceBranch })
+      .from(changeRequests)
+      .where(
+        and(
+          or(
+            eq(changeRequests.sourceBranch, branch),
+            eq(changeRequests.targetBranch, branch),
+          ),
+          eq(changeRequests.state, 'open'),
+        ),
+      )
+      .limit(1);
+    if (!row) return null;
+    return { number: row.number, end: row.sourceBranch === branch ? 'source' : 'target' };
+  }
+
   async deleteBranch(
     workspaceId: string,
     name: string,
@@ -296,16 +415,19 @@ export class WorkflowService implements IWorkflowService {
       // its own open-request check under this same lock, and the legacy
       // `onlyIfNoRemote` prune only fires when origin no longer has the ref.
       if (!opts?.systemCleanup && !opts?.onlyIfNoRemote) {
-        const open = await this.db
-          .select({ number: changeRequests.number })
-          .from(changeRequests)
-          .where(
-            and(eq(changeRequests.sourceBranch, name), eq(changeRequests.state, 'open')),
-          )
-          .limit(1);
-        if (open.length > 0) {
+        const open = await this.openChangeRequestOn(name);
+        if (open !== null) {
+          // Two different situations, two different actors. A request FROM
+          // this branch is the deleter's own to withdraw; a request INTO it
+          // belongs to someone else, and telling the branch owner to
+          // "withdraw" it points them at a verb they cannot use.
           throw new WorkflowValidationError(
-            `Branch "${name}" has an open change request (#${open[0].number}) — withdraw or decline it before deleting the branch.`,
+            open.end === 'source'
+              ? `Branch "${name}" has an open change request (#${open.number}). ` +
+                `Withdraw or decline it before deleting the branch.`
+              : `Open change request #${open.number} proposes changes into "${name}". ` +
+                `Its author can withdraw it, or an admin can decline it; ` +
+                `after that the branch can be deleted.`,
           );
         }
       }
@@ -363,9 +485,41 @@ export class WorkflowService implements IWorkflowService {
     return this.git.fetch(workspaceId);
   }
 
+  /**
+   * `git pull` on a workspace, plus the one announcement a pull owes the rest
+   * of the process.
+   *
+   * A pull rewrites a working tree without going through the write routes or
+   * the commit pipeline, so it emits neither of the signals those produce —
+   * and the catalogs that scan the DEFAULT branch's tree (skills, tool
+   * manuals, the plugin index) are keyed to exactly that tree. Every path that
+   * pulls the default workspace therefore has to say so, or those catalogs
+   * keep serving the pre-pull scan for the rest of their TTL: a merge landing
+   * an approved skill, the recovery ladder after a non-fast-forward push, and
+   * a user's update-from-remote alike. Routing every pull through here makes
+   * that ONE mechanism — "the default branch's tree changed" — instead of a
+   * separate special case per event. (The remote sync, `syncWorkspaceFromRemote`,
+   * pulls directly: it owes the same announcement to every branch and per
+   * file, and makes it from the same `treeChanged` verdict.)
+   *
+   * Announced only after the pull RESOLVES, and only when it CHANGED the
+   * tree's content: a pull that threw left the tree as it was, and so did an
+   * "already up to date" pull or one that landed only content-identical
+   * commits — dropping every catalog and making every attached browser
+   * refetch its file tree for a change that did not happen buys a re-scan
+   * and nothing else. The pull itself reports the change, since it is the
+   * only code that can observe it under the workspace mutex.
+   */
+  private async pullWorkspace(workspaceId: string): Promise<void> {
+    const { treeChanged } = await this.git.pull(workspaceId);
+    if (treeChanged && branchForWorkspaceId(workspaceId) === DEFAULT_BRANCH) {
+      this.events?.emit({ kind: 'fs-tree-changed', workspaceId, branch: DEFAULT_BRANCH });
+    }
+  }
+
   async updateFromRemote(workspaceId: string, user?: AuthUser): Promise<void> {
     try {
-      await this.git.pull(workspaceId);
+      await this.pullWorkspace(workspaceId);
     } catch (err) {
       if (err instanceof PullRebaseConflictError) {
         await this.queuePullConflictRecovery(workspaceId, err, user);
@@ -374,6 +528,157 @@ export class WorkflowService implements IWorkflowService {
     }
   }
 
+  async retireRemoteGoneClone(workspaceId: string): Promise<boolean> {
+    const branch = branchForWorkspaceId(workspaceId);
+    const id = workspaceIdForBranch(branch);
+    // The lock `createBranch` and `deleteBranch` take: no Hexis operation can
+    // bring this branch back into being while the clone is examined and
+    // removed. The sync's own hold on the clone ended when its outcome was
+    // returned, so the branch is asked about AGAIN here rather than trusted
+    // from then — a recreate that landed in between keeps its clone. A clone
+    // being bootstrapped right now is left alone too: it can only be cloning
+    // a branch that exists, so the retirement is already moot. What remains
+    // is a recreate pushed from OUTSIDE Hexis in the milliseconds between the
+    // origin check and the delete, with a bootstrap starting in that same
+    // window; the next sync of that branch clones it fresh.
+    return this.branchLifecycle.run(`branch:${branch}`, async () => {
+      if (!(await this.workspaceService.hasBootstrappedWorkspace(id))) return false;
+      if (this.workspaceService.isBootstrapInFlight(branch)) return false;
+      if (await this.git.remoteBranchExists(branch, branch)) return false;
+      await this.workspaceService.deleteWorkspace(id);
+      // Whatever a sync still owed that clone is owed to nothing now.
+      this.owedAnnouncements.delete(id);
+      return true;
+    });
+  }
+
+  /**
+   * Cap on per-path `file-changed` events one sync announces for a branch.
+   * Past it the tree event alone carries the news — a bulk import that
+   * rewrote hundreds of files would otherwise fan hundreds of events out to
+   * every open browser to say what one refresh says.
+   */
+  private static readonly SYNC_FILE_EVENT_CAP = 200;
+
+  /**
+   * Announcements a sync still OWES, per workspace: a pull landed but the
+   * events for it could not be sent. The pull is not redone on retry — HEAD
+   * has not moved since — so the announcement cannot be re-derived from sha
+   * movement; it is remembered here and delivered by the next sync of that
+   * workspace, merged with whatever that sync finds.
+   */
+  private readonly owedAnnouncements = new Map<string, { after: string; changedPaths: string[] }>();
+
+  async syncWorkspaceFromRemote(workspaceId: string): Promise<BranchSyncOutcome> {
+    const branch = branchForWorkspaceId(workspaceId);
+    const id = workspaceIdForBranch(branch);
+    // Two spellings of one workspace id are in circulation — the encoded
+    // directory name (`ali%2Fx`, what the clone listing yields) and the
+    // Express-decoded form (`ali/x`, what every HTTP route hands the git
+    // layer) — and the git layer's mutex keys on the raw string. Speaking the
+    // routes' spelling to git is what puts a webhook's rebase in the same
+    // queue as the author's concurrent save on a slashed branch; the event
+    // bus gets the encoded form, which every SSE consumer canonicalises.
+    const gitId = branch;
+    // Every failure below is an OUTCOME, never a throw: the caller runs this
+    // over many clones, and one broken clone must not abort the others.
+    let pulled: RemoteSyncPullResult;
+    try {
+      pulled = await this.git.syncFromRemote(gitId);
+    } catch (err) {
+      if (err instanceof RemoteBranchGoneError) {
+        // Not a failure: the host deleted the branch, so there is nothing to
+        // sync and the caller retires the stale clone. No banner — nobody is
+        // editing a branch that no longer exists. An announcement still owed
+        // for that clone dies with it: a branch recreated under the same name
+        // is a different tree, and replaying the old paths against it would
+        // announce changes that never happened there.
+        this.owedAnnouncements.delete(id);
+        console.log(`[sync] branch "${branch}" no longer exists on the host`);
+        return { branch, outcome: 'remote-gone' };
+      }
+      if (err instanceof PullRebaseConflictError) {
+        // Same path as `updateFromRemote`: the rebase was aborted inside
+        // the pull (the clone is exactly what it was) and the divergence goes
+        // to the retry → recovery-agent → escalate ladder, attributed to the
+        // recovery bot since no person triggered this. It is still reported
+        // as a conflict because it is not resolved at request time — the
+        // caller's retry, or the ladder, settles it; the banner clears on the
+        // next clean pull or push.
+        await this.queuePullConflictRecovery(gitId, err);
+        const message = syncConflictMessage(branch, err.conflictedPaths);
+        console.warn(`[sync] ${message}`);
+        this.noteGitSyncFailed(gitId, branch, err, { paths: err.conflictedPaths, message });
+        return { branch, outcome: 'conflict', conflictedPaths: err.conflictedPaths, error: message };
+      }
+      const message = sanitizeError(err);
+      console.warn(`[sync] pull failed for branch "${branch}": ${message}`);
+      this.noteGitSyncFailed(gitId, branch, err);
+      return { branch, outcome: 'error', error: message };
+    }
+    // A clean pull is proof origin is reachable and this clone rebases onto
+    // it — the condition a `git-sync-failed` banner on this branch was
+    // waiting for, whether a sync or a push raised it.
+    this.noteGitSyncOk(gitId, branch);
+    const { before, after, treeChanged, changedPaths } = pulled;
+    // What this branch reports about ITS clone is decided by the pull alone:
+    // `updated` when HEAD moved, `up-to-date` otherwise.
+    const outcome: BranchSyncOutcome =
+      after === null || after === before
+        ? { branch, outcome: 'up-to-date', to: after ?? '' }
+        : { branch, outcome: 'updated', from: before, to: after };
+    // What it ANNOUNCES is what has changed since the last announcement that
+    // was actually delivered: this pull's content change, if any, plus one an
+    // earlier sync landed but could not send. The announcement keys on the
+    // pull's verdict about CONTENT — commits that left the tree identical are
+    // nothing for a browser to refetch or a catalog to drop.
+    const owed = this.owedAnnouncements.get(id);
+    if (!treeChanged && !owed) return outcome;
+    const announce = {
+      after: after ?? owed?.after ?? '',
+      changedPaths: [...new Set([...(owed?.changedPaths ?? []), ...changedPaths])],
+    };
+    try {
+      // Announce the new tree. The tree event alone already refreshes every
+      // open file explorer and, on the default branch, drops the skill / tool /
+      // plugin catalogues (`catalog-cache-invalidation.ts`); the CR list
+      // caches the touched paths it resolved against the old tree, so drop
+      // that too.
+      this.prs.invalidateListCache();
+      this.events?.emit({ kind: 'fs-tree-changed', workspaceId: id, branch });
+      // Per-file events so an open tab on a file the host rewrote refetches
+      // instead of showing stale bytes until its next tree refresh. The paths
+      // were observed under the same hold as the pull, so they are exactly
+      // this sync's — never a save that landed alongside. Capped: a bulk
+      // import would otherwise fan hundreds of events out to say what one
+      // refresh says.
+      if (announce.changedPaths.length <= WorkflowService.SYNC_FILE_EVENT_CAP) {
+        for (const p of announce.changedPaths) {
+          this.events?.emit({
+            kind: 'file-changed',
+            workspaceId: id,
+            branch,
+            path: `${this.kbDirName}/${p}`,
+            newSha: announce.after,
+            byUserId: 'system',
+            byUserName: 'Git sync',
+          });
+        }
+      }
+    } catch (err) {
+      // The pull landed, so origin is fine and no banner is raised — but the
+      // caller must hear that this branch's announcement did not happen, and
+      // the announcement stays owed: the retry the error invites will not
+      // move HEAD again, so it must find the debt here rather than in the shas.
+      this.owedAnnouncements.set(id, announce);
+      const message = sanitizeError(err);
+      console.warn(`[sync] pulled "${branch}" but could not announce it: ${message}`);
+      return { branch, outcome: 'error', error: message };
+    }
+    this.owedAnnouncements.delete(id);
+    return outcome;
+  }
+
   /**
    * A pull hit a rebase conflict — the workspace carries local commits that
    * conflict with origin, a state that never resolves on its own (every
@@ -700,7 +1005,7 @@ export class WorkflowService implements IWorkflowService {
         let recoveryError: unknown = null;
         if (looksLikeNonFastForward) {
           try {
-            await this.git.pull(workspaceId);
+            await this.pullWorkspace(workspaceId);
             await this.git.push(workspaceId, user);
             recovered = true;
             this.noteGitSyncOk(workspaceId, branch);
@@ -951,7 +1256,34 @@ export class WorkflowService implements IWorkflowService {
    * machinery is unaffected — all this adds is that someone finds out, rather
    * than the deployment looking healthy while nothing reaches the remote.
    */
-  private noteGitSyncFailed(workspaceId: string, branch: string, err: unknown): void {
+  private noteGitSyncFailed(
+    workspaceId: string,
+    branch: string,
+    err: unknown,
+    /**
+     * The files a person must reconcile, and the sentence we composed about
+     * them. That sentence is ours (branch name + repo paths, no git stderr),
+     * so it goes out verbatim — the same string the sync response and the
+     * log carry — rather than through the sanitiser, whose 200-character
+     * clip would make the banner disagree with them. Callers that already
+     * hold the sentence pass it; otherwise it is derived below.
+     */
+    explicitConflict?: { paths: string[]; message: string },
+  ): void {
+    // A rebase conflict is a conflict whichever path ran into it. The remote
+    // sync names its files on purpose; the push-recovery paths (autosave, the
+    // lock release, the queued retries) raise the same banner with the raw
+    // error — and the banner keeps the LATEST failure per branch, so a retry
+    // landing after the sync would replace the conflict variant, with the
+    // files as links, by the generic "check the server logs" one. Seen on
+    // staging: the sync's 409 named the file, the worker's retry a moment
+    // later took the links away. Derive the paths from the error itself so
+    // every path agrees.
+    const conflict =
+      explicitConflict ??
+      (err instanceof PullRebaseConflictError
+        ? { paths: err.conflictedPaths, message: syncConflictMessage(branch, err.conflictedPaths) }
+        : undefined);
     // Same canonicalization as `noteGitSyncOk` — the pair must agree on keys.
     const id = branchForWorkspaceId(workspaceId);
     this.gitSyncFailing.add(id);
@@ -961,7 +1293,8 @@ export class WorkflowService implements IWorkflowService {
       branch,
       // Git stderr can quote a credentialed URL; the banner is user-facing and
       // the string also lands in client logs, so sanitize before it leaves.
-      reason: sanitizeError(err),
+      reason: conflict ? conflict.message : sanitizeError(err),
+      ...(conflict ? { conflictedPaths: conflict.paths } : {}),
     });
   }
 
@@ -1020,7 +1353,7 @@ export class WorkflowService implements IWorkflowService {
       let recoveryError: unknown = null;
       if (looksLikeNonFastForward) {
         try {
-          await this.git.pull(workspaceId);
+          await this.pullWorkspace(workspaceId);
           await this.git.push(workspaceId, user, opts);
           recovered = true;
           this.noteGitSyncOk(workspaceId, branch);
@@ -1284,12 +1617,17 @@ export class WorkflowService implements IWorkflowService {
     }
 
     // The whole open sequence — uniqueness check through row insert — runs
-    // under the source branch's lifecycle lock. `retireMergedSourceBranch`
-    // takes the same lock around its own check-then-delete, so a merge
-    // cleanup can never race into the multi-second gap between this
-    // method's "no open request" answer and its row landing, and delete the
-    // branch out from under a request being opened.
-    return this.branchLifecycle.run(`branch:${input.sourceBranch}`, async () => {
+    // under the lifecycle lock of BOTH branches. `retireMergedSourceBranch`
+    // and `deleteBranch` take the lock of the branch they are removing around
+    // their own check-then-delete, so a cleanup can never race into the
+    // multi-second gap between this method's "no open request" answer and its
+    // row landing, and delete a branch out from under a request being opened.
+    //
+    // BOTH, because that gap is symmetric: a request is unusable if EITHER of
+    // its branches is retired inside the window, and holding only the source
+    // left a `Y -> X` request open to X being deleted under it — the very
+    // failure the widened guards exist to prevent.
+    return this.runOnBranchPair(input.sourceBranch, input.targetBranch, async () => {
     // Uniqueness rule — A→B blocks A→B (the spec is explicit that B→A in
     // parallel is still allowed). `listOpenPrs` is cached for 30s so we
     // force-fresh to catch a CR that opened just now.
@@ -1623,7 +1961,7 @@ export class WorkflowService implements IWorkflowService {
     // refs, but the restore commits from the working tree — a stale tree
     // would push non-fast-forward and fail loudly anyway; this just makes
     // that rare.
-    await this.git.pull(ws.id).catch(() => undefined);
+    await this.pullWorkspace(ws.id).catch(() => undefined);
 
     // The verb acts on the request as it is NOW — never a cached file list.
     const paths = await this.git.changedPathsForPr(ws.id, baseBranch, headBranch);
@@ -1701,14 +2039,56 @@ export class WorkflowService implements IWorkflowService {
       throw new WorkflowDomainError('This change request has already been applied.', 422);
     }
 
-    const ws = await this.workspaceService.getOrCreateForBranch(summary.base);
-    await this.workspaceService.ensureRemotesFetched(ws.id).catch(() => undefined);
-    const isAdmin = await this.accessControl.canWriteAtRef(
-      ws.id,
-      `origin/${summary.base}`,
-      user.email,
-      'roles.yaml',
-    );
+    // Which `roles.yaml` authorizes this? The base branch's, exactly as every
+    // other admin check here. When the base branch itself is gone, this
+    // bootstrap (or the at-ref check below) fails; that failure is NOT proven
+    // absence, so it never re-keys authorization to some other branch. It
+    // kicks the deleted-branch sweep instead: the one mechanism that proves a
+    // branch is missing (fresh fetch, then a set lookup) and closes every
+    // request stranded by it, this one included. This call still fails; by
+    // the next attempt the sweep has either closed the row (nothing left to
+    // delete) or shown the base alive, meaning the failure was transient and
+    // a retry can succeed.
+    let ws: { id: string };
+    let isAdmin: boolean | null;
+    try {
+      ws = await this.workspaceService.getOrCreateForBranch(summary.base);
+      // STRICT: the admin check below reads roles.yaml at `origin/<base>`,
+      // and on a swallowed fetch failure that ref is whatever the clone last
+      // saw — a user whose rights were since revoked would still pass
+      // against the stale tree. Authorization never runs against refs of
+      // unknown age.
+      try {
+        await this.workspaceService.ensureRemotesFetched(ws.id, { strict: true });
+      } catch {
+        throw new WorkflowDomainError(
+          `Could not verify the base branch "${summary.base}" — origin was unreachable. Retry in a moment.`,
+          409,
+        );
+      }
+      isAdmin = await this.accessControl.canWriteAtRef(
+        ws.id,
+        `origin/${summary.base}`,
+        user.email,
+        'roles.yaml',
+      );
+    } catch (err) {
+      this.kickDeletedBranchSweep();
+      throw err;
+    }
+    // null is "the ref did not resolve", not "denied" — and saying "only an
+    // admin can do this" to an admin whose base branch is merely unfetchable
+    // sent people chasing their role instead of the branch. Kick the sweep
+    // (it proves absence itself, with a strict fetch, and closes the
+    // stranded rows) and fail with the honest condition; by the next
+    // attempt the sweep has either closed the row or shown the base alive.
+    if (isAdmin === null) {
+      this.kickDeletedBranchSweep();
+      throw new WorkflowDomainError(
+        `The base branch "${summary.base}" could not be resolved — it may have been deleted, or origin was unreachable. A background check is running; retry in a moment.`,
+        409,
+      );
+    }
     if (isAdmin !== true) {
       throw new WorkflowDomainError('Only an admin can delete a change request.', 403);
     }
@@ -1780,17 +2160,23 @@ export class WorkflowService implements IWorkflowService {
   }
 
   /**
-   * Close every open change request whose source branch no longer exists.
+   * Close every open change request either of whose branches no longer exists.
    *
-   * A change request is a proposal to merge a branch. When that branch is
-   * gone — deleted after a manual merge, pruned as abandoned, or left behind
-   * by a feature that has itself been removed — the request cannot be read,
-   * reviewed, applied or declined. It is not a pending decision, it is a
-   * tombstone, and it costs more than tidiness: `listOpenPrs` resolves each
-   * open request's changed paths, so every one of these throws
+   * A change request is a proposal to merge one branch into another, so it
+   * needs BOTH. When either is gone — deleted after a manual merge, pruned as
+   * abandoned, or left behind by a feature that has itself been removed — the
+   * request cannot be read, reviewed, applied or declined. It is not a pending
+   * decision, it is a tombstone, and it costs more than tidiness: `listOpenPrs`
+   * resolves each open request's changed paths, so every one of these throws
    * `unknown branch` on every poll, filling the log and slowing the list it
    * appears in.
    *
+   * The TARGET half is the one that used to be missed, and it was the worse
+   * half: a missing source could only ever arrive by a path that also closed
+   * the request, while a request whose target had been retired under it was
+   * unreadable, undeclinable, undeletable — and invisible here, so it survived
+   * every restart.
+   *
    * CLOSED, NOT DELETED. The row is evidence — who proposed what, when, and
    * that it was never applied — and it is the only remaining trace once the
    * branch is gone. Closing takes it out of every open list (which is the
@@ -1799,7 +2185,7 @@ export class WorkflowService implements IWorkflowService {
    * operation in the workflow that destroys history without a human asking.
    *
    * ONE fetch, then a set lookup. The freshness matters more than the cost:
-   * `changedPathsForPr` fails when the branch is missing from THAT
+   * `changedPathsForPr` fails when a branch is missing from THAT
    * workspace's refs, which is not the same thing as missing from origin —
    * a clone that never fetched a branch someone else created reports exactly
    * the same error. Closing on that signal would eat live requests. So the
@@ -1815,7 +2201,11 @@ export class WorkflowService implements IWorkflowService {
    */
   async closeChangeRequestsWithDeletedBranches(): Promise<number> {
     const open = await this.db
-      .select({ number: changeRequests.number, sourceBranch: changeRequests.sourceBranch })
+      .select({
+        number: changeRequests.number,
+        sourceBranch: changeRequests.sourceBranch,
+        targetBranch: changeRequests.targetBranch,
+      })
       .from(changeRequests)
       .where(eq(changeRequests.state, 'open'));
     if (open.length === 0) return 0;
@@ -1824,6 +2214,11 @@ export class WorkflowService implements IWorkflowService {
     try {
       const branches = await this.git.listBranches(workspaceIdForBranch(DEFAULT_BRANCH), {
         freshFetch: true,
+        // The list is used to PROVE absence, and the listing's normal
+        // degrade-to-stale behaviour would report every branch created since
+        // the last good fetch as missing — closing live requests. Strict:
+        // a failed fetch throws into the catch below, which closes nothing.
+        strictFetch: true,
       });
       live = new Set(branches.map((b) => b.name));
     } catch (err) {
@@ -1839,7 +2234,15 @@ export class WorkflowService implements IWorkflowService {
 
     let closed = 0;
     for (const cr of open) {
-      if (live.has(cr.sourceBranch)) continue;
+      // Either end being gone makes the request unusable — the detail read
+      // resolves the target first, so a missing target fails it just as hard
+      // as a missing source, and no other path ever closes those.
+      const missing = !live.has(cr.sourceBranch)
+        ? cr.sourceBranch
+        : !live.has(cr.targetBranch)
+          ? cr.targetBranch
+          : null;
+      if (!missing) continue;
       const now = new Date();
       // Guarded on `state = 'open'`, so a merge or withdraw racing this call
       // wins and this becomes a no-op.
@@ -1852,13 +2255,45 @@ export class WorkflowService implements IWorkflowService {
       this.prs.invalidateDetailCache(cr.number);
       this.events?.emit({ kind: 'change-request-rejected', number: cr.number });
       console.log(
-        `[cr] closed change request #${cr.number}: source branch "${cr.sourceBranch}" no longer exists`,
+        `[cr] closed change request #${cr.number}: branch "${missing}" no longer exists`,
       );
       closed++;
     }
     return closed;
   }
 
+  /**
+   * The sweep above, on demand. Boot runs it once (`create-core-server`),
+   * which clears requests stranded BEFORE the process started; a branch that
+   * goes missing mid-uptime would otherwise keep its requests broken until
+   * the next restart. Verbs that hit a failure which MIGHT mean a missing
+   * branch kick it here.
+   *
+   * Detached, because the kicking verb is already surfacing its own error
+   * and must not hold its response on a fetch. Coalesced, because the kick
+   * sites are user actions that cluster when something is stranded (retry,
+   * decline, delete) and one sweep answers all of them. The sweep proves
+   * absence itself and fails safe by closing nothing, so a spurious kick
+   * costs one fetch and never closes a live request.
+   */
+  private sweepKick: Promise<void> | null = null;
+  private kickDeletedBranchSweep(): void {
+    if (this.sweepKick) return;
+    this.sweepKick = this.closeChangeRequestsWithDeletedBranches()
+      .then((n) => {
+        if (n > 0) {
+          console.log(
+            `[cr] on-demand sweep closed ${n} stranded change request${n === 1 ? '' : 's'}`,
+          );
+        }
+      })
+      .catch((err) => console.warn('[cr] on-demand deleted-branch sweep failed:', err))
+      .finally(() => {
+        this.sweepKick = null;
+      });
+  }
+
+
   /**
    * Reject (close-without-merging) a change request. Per PLAN.md spec, the
    * authorized set is:
@@ -1892,15 +2327,26 @@ export class WorkflowService implements IWorkflowService {
       // The path list comes from the PR detail — fetch fresh so the rule
       // applies against the current head (a stale list would let a user
       // reject a CR whose files they no longer all own).
-      const detail = await this.prs.getPrDetail(number, {
-        fresh: true,
-        workspaceId,
-        viewerEmail: user.email,
-        // Only `files[].path` is read below; a patch per file would be one
-        // git subprocess each, generated for nothing, on every reject by a
-        // non-author.
-        patches: false,
-      });
+      let detail;
+      try {
+        detail = await this.prs.getPrDetail(number, {
+          fresh: true,
+          workspaceId,
+          viewerEmail: user.email,
+          // Only `files[].path` is read below; a patch per file would be one
+          // git subprocess each, generated for nothing, on every reject by a
+          // non-author.
+          patches: false,
+        });
+      } catch (err) {
+        // The detail read resolves both branch tips (`resolvePrShas`), so a
+        // request one of whose branches is gone fails HERE, on the verb that
+        // exists to get rid of it. Kick the sweep (which proves the absence
+        // and closes the stranded rows) and surface the failure; see
+        // `kickDeletedBranchSweep`.
+        this.kickDeletedBranchSweep();
+        throw err;
+      }
       const paths = detail?.files.map((f) => f.path) ?? [];
       if (paths.length > 0) {
         const writeMap = await this.accessControl.canWriteBatchAtRef(
@@ -2026,7 +2472,7 @@ export class WorkflowService implements IWorkflowService {
     try {
       const targetWorkspace = await this.workspaceService.getOrCreateForBranch(baseBranch);
       targetWorkspaceId = targetWorkspace.id;
-      await this.git.pull(targetWorkspace.id);
+      await this.pullWorkspace(targetWorkspace.id);
     } catch (err) {
       // Still best-effort for the merge response (the merge already landed
       // on origin) — but a rebase CONFLICT here means the target workspace
@@ -2086,17 +2532,12 @@ export class WorkflowService implements IWorkflowService {
       // until retirement finishes (and fails loudly on the missing branch,
       // retryable, instead of losing the branch mid-open).
       await this.branchLifecycle.run(`branch:${sourceBranch}`, async () => {
-        const stillOpen = await this.db
-          .select({ number: changeRequests.number })
-          .from(changeRequests)
-          .where(
-            and(
-              eq(changeRequests.sourceBranch, sourceBranch),
-              eq(changeRequests.state, 'open'),
-            ),
-          )
-          .limit(1);
-        if (stillOpen.length > 0) return;
+        // EITHER end (see `openChangeRequestOn`). The request being merged is
+        // already `merged` by now, so it never matches itself; anything left
+        // is a live request that still needs this branch — including one
+        // proposing INTO it, which the source-only question used to miss.
+        const stillOpen = await this.openChangeRequestOn(sourceBranch);
+        if (stillOpen !== null) return;
 
         const targetWs = await this.workspaceService.getOrCreateForBranch(baseBranch);
         // The remote-tracking ref must be current, or deleteBranch's