mandrel 2.36.0 → 2.37.0

package/.agents/scripts/lib/worktree/lifecycle/pending-cleanup.js

@@ -15,14 +15,39 @@
  *
  * The manifest itself lives under `.worktrees/`, which is already
  * git-ignored; no tracked state is mutated.
+ *
+ * **Story #5112 — the manifest is shared mutable state.** `recordPendingCleanup`
+ * runs from the close reap and `drainPendingCleanup` runs from the `/plan`
+ * boot, so two concurrent delivery sessions can read-modify-write the same
+ * file. Three properties close that:
+ *
+ *   1. **Atomic write.** Every write lands in a pid-scoped temp file and is
+ *      then `rename`d over the manifest, so a reader never observes a half-
+ *      written array (`readManifest` treats a torn or absent file as empty,
+ *      which used to silently *drop* the backlog).
+ *   2. **Mutual exclusion.** Read-modify-write runs under the shared
+ *      `acquireSweepLock` primitive on a manifest-adjacent lockfile.
+ *   3. **Merge before write.** The drain holds its snapshot across awaited
+ *      removals, so it re-reads and merges by `storyId` before its final
+ *      write — an entry recorded mid-drain survives instead of being erased
+ *      by a stale snapshot.
  */
 
 import fs from 'node:fs';
 import { rm as fsPromisesRm } from 'node:fs/promises';
 import path from 'node:path';
 import { NOOP_LOGGER } from '../../Logger.js';
+import { acquireSweepLock } from '../../single-story-sweep/sweep-lock.js';
 
 const MANIFEST_FILENAME = '.pending-cleanup.json';
+const MANIFEST_LOCK_FILENAME = '.pending-cleanup.lock';
+
+/**
+ * Staleness threshold for the manifest lock. Deliberately short: the guarded
+ * section is a read, an in-memory merge and a rename — milliseconds — so a
+ * lockfile older than this is a crashed holder, not a slow one.
+ */
+const MANIFEST_LOCK_TIMEOUT_MS = 5_000;
 /**
  * After a reap hands off to the manifest, `attempts` counts failed
  * `drainPendingCleanup` passes (initial hand-off uses `attempts: 0`). The entry
@@ -35,6 +60,49 @@ export function manifestPath(worktreeRoot) {
   return path.join(worktreeRoot, MANIFEST_FILENAME);
 }
 
+/**
+ * Path of the lockfile guarding manifest read-modify-write. Manifest-adjacent
+ * (same gitignored `.worktrees/` directory) so it needs no extra config and
+ * shares the manifest's lifetime. Module-private: the lock is internal to
+ * this manifest's read-modify-write, and callers only ever observe that a
+ * mutation completed and left nothing behind.
+ *
+ * @param {string} worktreeRoot
+ * @returns {string}
+ */
+function manifestLockPath(worktreeRoot) {
+  return path.join(worktreeRoot, MANIFEST_LOCK_FILENAME);
+}
+
+/**
+ * Run `fn` under the manifest lock. Best-effort in the same sense as every
+ * other use of this primitive: on contention the work still runs (the
+ * merge-before-write below is what makes a lost race non-destructive), so a
+ * stuck lockfile can never wedge a reap. Always releases.
+ *
+ * @template T
+ * @param {string} worktreeRoot
+ * @param {() => T} fn
+ * @returns {T}
+ */
+function withManifestLock(worktreeRoot, fn) {
+  const lock = acquireSweepLock({
+    lockPath: manifestLockPath(worktreeRoot),
+    timeoutMs: MANIFEST_LOCK_TIMEOUT_MS,
+  });
+  try {
+    return fn();
+  } finally {
+    if (lock.acquired) {
+      try {
+        lock.release();
+      } catch {
+        // Release is best-effort; a stale lockfile expires on its own.
+      }
+    }
+  }
+}
+
 export function readManifest(worktreeRoot) {
   const p = manifestPath(worktreeRoot);
   if (!fs.existsSync(p)) return [];
@@ -47,6 +115,13 @@ export function readManifest(worktreeRoot) {
   }
 }
 
+/**
+ * Write the manifest atomically: serialize into a pid-scoped temp file in the
+ * same directory (so `rename` stays within one filesystem and is therefore
+ * atomic), then rename it over the manifest. A concurrent reader sees either
+ * the whole previous manifest or the whole new one — never a truncated array
+ * that `readManifest`'s catch would silently turn into "no pending cleanups".
+ */
 function writeManifest(worktreeRoot, entries) {
   const p = manifestPath(worktreeRoot);
   if (!Array.isArray(entries) || entries.length === 0) {
@@ -58,7 +133,37 @@ function writeManifest(worktreeRoot, entries) {
     return;
   }
   fs.mkdirSync(worktreeRoot, { recursive: true });
-  fs.writeFileSync(p, `${JSON.stringify(entries, null, 2)}\n`, 'utf8');
+  writeFileAtomic(p, `${JSON.stringify(entries, null, 2)}\n`);
+}
+
+/**
+ * Write `contents` to `targetPath` atomically: serialize into a pid-scoped
+ * temp file beside it (same directory, so the rename stays within one
+ * filesystem and is therefore atomic), then rename it into place. On failure
+ * the temp file is reaped and the error propagates — a half-written file is
+ * never left where a reader could pick it up.
+ *
+ * @param {string} targetPath
+ * @param {string} contents
+ */
+function writeFileAtomic(targetPath, contents) {
+  const tmp = `${targetPath}.${process.pid}.tmp`;
+  try {
+    fs.writeFileSync(tmp, contents, 'utf8');
+    fs.renameSync(tmp, targetPath);
+  } catch (err) {
+    reapTempFile(tmp);
+    throw err;
+  }
+}
+
+/** Best-effort removal of an abandoned atomic-write temp file. */
+function reapTempFile(tmp) {
+  try {
+    fs.unlinkSync(tmp);
+  } catch {
+    // Temp file already gone (or never created) — nothing to reap.
+  }
 }
 
 /**
@@ -71,38 +176,45 @@ export function recordPendingCleanup(
   worktreeRoot,
   { storyId, branch, path: wtPath, push = false },
 ) {
-  const now = new Date().toISOString();
-  const entries = readManifest(worktreeRoot);
-  const idx = entries.findIndex((e) => e.storyId === storyId);
-  if (idx >= 0) {
-    entries[idx] = {
-      ...entries[idx],
-      branch,
-      path: wtPath,
-      push,
-      lastFailedAt: now,
-      attempts: (entries[idx].attempts ?? 0) + 1,
-    };
-  } else {
-    entries.push({
-      storyId,
-      branch,
-      path: wtPath,
-      push,
-      firstFailedAt: now,
-      lastFailedAt: now,
-      attempts: 0,
-    });
-  }
-  writeManifest(worktreeRoot, entries);
-  return entries.find((e) => e.storyId === storyId);
+  // The read, the merge and the write are one critical section: without the
+  // lock a concurrent drain's write can land between our read and our write
+  // and erase this hand-off entirely.
+  return withManifestLock(worktreeRoot, () => {
+    const now = new Date().toISOString();
+    const entries = readManifest(worktreeRoot);
+    const idx = entries.findIndex((e) => e.storyId === storyId);
+    if (idx >= 0) {
+      entries[idx] = {
+        ...entries[idx],
+        branch,
+        path: wtPath,
+        push,
+        lastFailedAt: now,
+        attempts: (entries[idx].attempts ?? 0) + 1,
+      };
+    } else {
+      entries.push({
+        storyId,
+        branch,
+        path: wtPath,
+        push,
+        firstFailedAt: now,
+        lastFailedAt: now,
+        attempts: 0,
+      });
+    }
+    writeManifest(worktreeRoot, entries);
+    return entries.find((e) => e.storyId === storyId);
+  });
 }
 
 export function removePendingCleanup(worktreeRoot, storyId) {
-  const entries = readManifest(worktreeRoot).filter(
-    (e) => e.storyId !== storyId,
-  );
-  writeManifest(worktreeRoot, entries);
+  withManifestLock(worktreeRoot, () => {
+    const entries = readManifest(worktreeRoot).filter(
+      (e) => e.storyId !== storyId,
+    );
+    writeManifest(worktreeRoot, entries);
+  });
 }
 
 async function removeStuckWorktreePath(wtPath, { git, repoRoot, fsRm }) {
@@ -176,6 +288,32 @@ async function retryStage1ForEntry(entry, ctx) {
   return { success: true, ...cleanup };
 }
 
+/**
+ * Write the drain's result, merged against whatever the manifest holds *now*.
+ *
+ * The drain awaits a removal per entry, so its `entries` snapshot can be
+ * minutes old by the time it writes. Writing that snapshot back wholesale
+ * erased any entry a concurrent `recordPendingCleanup` added mid-drain — the
+ * exact hand-off that reap had just decided it could not complete. Merging by
+ * `storyId` keeps it: rows this pass re-computed win, rows it successfully
+ * drained are dropped, and everything else the manifest has gained survives.
+ *
+ * @param {string} worktreeRoot
+ * @param {object[]} next          Rows this drain re-computed (still pending).
+ * @param {Set<number|string>} drainedIds Story ids this drain cleared.
+ */
+function commitDrainedManifest(worktreeRoot, next, drainedIds) {
+  withManifestLock(worktreeRoot, () => {
+    const byId = new Map(next.map((entry) => [entry.storyId, entry]));
+    for (const entry of readManifest(worktreeRoot)) {
+      if (byId.has(entry.storyId)) continue;
+      if (drainedIds.has(entry.storyId)) continue;
+      byId.set(entry.storyId, entry);
+    }
+    writeManifest(worktreeRoot, [...byId.values()]);
+  });
+}
+
 /**
  * Drain the pending-cleanup manifest: for each entry, retry Stage 1
  * cleanup. Successful entries are removed; failing entries have their
@@ -252,7 +390,7 @@ export async function drainPendingCleanup({
     next.push(updated);
   }
 
-  writeManifest(worktreeRoot, next);
+  commitDrainedManifest(worktreeRoot, next, new Set(drained));
   return {
     drained,
     drainedDetails,

package/.agents/scripts/lib/worktree/node-modules-strategy.js

@@ -401,10 +401,7 @@ export function describeAttemptFailure(result, timeoutMs) {
 }
 
 /** Relative path of the per-machine pnpm-store prime sentinel (under tempRoot). */
-export const PNPM_STORE_PRIME_SENTINEL = path.join(
-  'temp',
-  '.pnpm-store-primed',
-);
+const PNPM_STORE_PRIME_SENTINEL = path.join('temp', '.pnpm-store-primed');
 
 /**
  * Pure: one-time per-machine pnpm content-addressable-store prime.
@@ -424,7 +421,7 @@ export const PNPM_STORE_PRIME_SENTINEL = path.join(
  *
  * @returns {{ primed: 'primed' | 'cached' | 'failed' | 'skipped', reason?: string }}
  */
-export function primePnpmStore({
+function primePnpmStore({
   strategy,
   repoRoot,
   logger,