mandrel 2.53.0 → 2.55.0

package/.agents/scripts/lib/full-suite-lock.js

@@ -28,12 +28,14 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+import { Worker } from 'node:worker_threads';
 
 import { mainCheckoutRoot } from './config/temp-paths.js';
 import {
   acquireLockWithWait,
   acquireSweepLock,
   readLockHolderPid,
+  refreshLockSync,
 } from './single-story-sweep/sweep-lock.js';
 
 /** Environment escape hatch: set to `0`/`false`/`off`/`no` to disable. */
@@ -56,6 +58,13 @@ const DEFAULT_WAIT_MS = 20 * 60_000;
 /** Poll interval while waiting. */
 const DEFAULT_POLL_MS = 2_000;
 
+/**
+ * Mtime-refresh interval for the spawn heartbeat. A third of the staleness
+ * window, matching the sweep primitive's own divisor: two consecutive missed
+ * beats still leave a live holder looking live.
+ */
+const DEFAULT_SPAWN_HEARTBEAT_MS = DEFAULT_STALE_MS / 3;
+
 const FALSEY = /^(0|false|off|no)$/i;
 
 /**
@@ -144,6 +153,170 @@ function beginLock({
   return { lock: null, lockPath };
 }
 
+/**
+ * The heartbeat body, run on a worker thread (Story #5278).
+ *
+ * Sleeps in `Atomics.wait` slices and refreshes the lockfile's mtime between
+ * them, stopping the moment the main thread flips the shared stop flag (and
+ * `Atomics.notify`s it), the lockfile stops being ours, or any I/O fails.
+ * Ownership is re-read from the file's first line on every beat for the same
+ * reason the in-process heartbeat does it: after a steal the file belongs to
+ * someone else, and bumping its mtime would keep *their* lock alive on our
+ * behalf.
+ *
+ * Inline source rather than a module of its own: it is nine lines of loop
+ * whose whole meaning is the lock it refreshes, and a separate worker entry
+ * point would be a second file that no reader of this one can see.
+ */
+const HEARTBEAT_WORKER_SOURCE = `
+const fs = require('node:fs');
+const { workerData } = require('node:worker_threads');
+const { lockPath, ownerId, intervalMs, stopBuffer } = workerData;
+const stop = new Int32Array(stopBuffer);
+while (Atomics.load(stop, 0) === 0) {
+  Atomics.wait(stop, 0, 0, intervalMs);
+  if (Atomics.load(stop, 0) !== 0) break;
+  try {
+    if (String(fs.readFileSync(lockPath, 'utf8')).split('\\n', 1)[0] !== ownerId) break;
+    const now = new Date();
+    fs.utimesSync(lockPath, now, now);
+  } catch {
+    break;
+  }
+}
+`;
+
+/**
+ * Keep a held lock's mtime advancing while this thread is blocked inside a
+ * synchronous spawn (Story #5278).
+ *
+ * The primitive's own heartbeat is a `setInterval`, so it only fires when the
+ * holder's event loop gets a turn. `runCapture` spawns the suite with
+ * `spawnSync`: the loop stops turning for the entire run, the mtime freezes
+ * at acquisition time, and a sibling reading that mtime concludes — correctly,
+ * on the evidence available to it — that the holder died, breaks the lock, and
+ * starts a second full suite beside the first. That is the exact collision
+ * this lock exists to prevent, and it fires most reliably on the slowest
+ * suites, where it costs the most.
+ *
+ * A worker thread has its own event loop, unaffected by the main thread's
+ * blocking spawn, so it is the only place a refresh can happen at all here.
+ *
+ * **Best-effort, like everything else on this path.** A worker that cannot
+ * start (a runtime with threads disabled, a resource limit) leaves the
+ * pre-#5278 behaviour exactly as it was; it never throws and never delays the
+ * spawn it guards.
+ *
+ * @param {{ lockPath: string|null, ownerId?: string, heartbeatMs: number }} opts
+ * @returns {() => void} `stop()` — idempotent, safe when no worker started.
+ */
+function startSpawnHeartbeat({ lockPath, ownerId, heartbeatMs }) {
+  if (!(lockPath && typeof ownerId === 'string' && heartbeatMs > 0)) {
+    return () => {};
+  }
+  let worker = null;
+  let stop = null;
+  try {
+    const stopBuffer = new SharedArrayBuffer(4);
+    stop = new Int32Array(stopBuffer);
+    worker = new Worker(HEARTBEAT_WORKER_SOURCE, {
+      eval: true,
+      workerData: { lockPath, ownerId, intervalMs: heartbeatMs, stopBuffer },
+    });
+    // A heartbeat must never be the reason the process stays alive.
+    worker.unref();
+    worker.on('error', () => {});
+  } catch {
+    return () => {};
+  }
+  let stopped = false;
+  return () => {
+    if (stopped) return;
+    stopped = true;
+    try {
+      Atomics.store(stop, 0, 1);
+      Atomics.notify(stop, 0);
+      worker.terminate();
+    } catch {
+      // Already gone.
+    }
+  };
+}
+
+/**
+ * Start the spawn heartbeat without ever letting it become a reason the suite
+ * does not run.
+ *
+ * The whole module's posture is that a lock defect may slow a suite down and
+ * may never skip one, and a heartbeat is the least load-bearing thing on the
+ * path — it exists only so a *rival* reads the mtime correctly. So a starter
+ * that throws (an environment without worker threads, a resource limit, an
+ * injected seam) resolves to "no heartbeat", never to a failed close.
+ *
+ * @param {Function} startFn
+ * @param {{ lockPath: string|null, ownerId?: string, heartbeatMs: number }} opts
+ * @returns {() => void}
+ */
+function safeStartHeartbeat(startFn, opts) {
+  try {
+    return startFn(opts) ?? (() => {});
+  } catch {
+    return () => {};
+  }
+}
+
+/**
+ * Stamp a held lock as current immediately before its critical section
+ * begins (Story #5278).
+ *
+ * A lock acquired at the end of a twenty-minute wait was created — or last
+ * heartbeat-refreshed — long before the spawn it is about to guard, and the
+ * synchronous caller's event loop is about to stop turning for the whole
+ * duration of that spawn. Refreshing here is the last chance to put a current
+ * mtime on the file.
+ *
+ * Routed through the primitive's own {@link refreshLockSync} rather than the
+ * holder's `refresh()` so an injected `acquireOnceFn` seam that returns a
+ * bare `{ acquired, release }` is refreshed identically to the real one.
+ *
+ * @param {{ ownerId?: string }|null} held
+ * @param {string|null} lockPath
+ * @param {object} fsImpl
+ */
+function refreshBeforeSpawn(held, lockPath, fsImpl) {
+  if (!(held?.acquired && typeof held.ownerId === 'string' && lockPath)) return;
+  refreshLockSync({ lockPath, ownerId: held.ownerId, fsImpl });
+}
+
+/**
+ * The post-wait re-probe (Story #5278).
+ *
+ * Waiting for the lock is waiting for *someone else's* full suite against
+ * this same checkout. By the time it finishes, the thing this caller was
+ * about to spawn the suite to establish may already be true — the holder
+ * deposited the stamp, or a sibling recorded the evidence. Spawning anyway
+ * pays for a whole suite to re-derive a fact that is already on disk, which
+ * is exactly the cost the lock exists to avoid.
+ *
+ * Consulted **only after a real wait**: an uncontended caller's freshness
+ * probe ran moments ago and nothing has happened since, so re-running it
+ * would be pure overhead on the hot path.
+ *
+ * @template T
+ * @param {(() => T|undefined)|undefined} skipIfSatisfied
+ * @param {boolean} waited
+ * @returns {{ satisfied: boolean, value?: T }}
+ */
+function probeAlreadySatisfied(skipIfSatisfied, waited) {
+  if (!(waited && typeof skipIfSatisfied === 'function')) {
+    return { satisfied: false };
+  }
+  const value = skipIfSatisfied();
+  return value === undefined
+    ? { satisfied: false }
+    : { satisfied: true, value };
+}
+
 /**
  * Block a synchronous caller for `ms` without a timer. `runCapture` spawns the
  * suite with `spawnSync`, so its whole call stack is synchronous and there is
@@ -178,7 +351,13 @@ function sleepSync(ms) {
  *   sleepFn?: (ms: number) => void,
  *   acquireOnceFn?: typeof acquireSweepLock,
  *   lockPath?: string,
- * }} opts
+ *   skipIfSatisfied?: () => T|undefined,
+ *   spawnHeartbeatMs?: number,
+ *   startSpawnHeartbeatFn?: typeof startSpawnHeartbeat,
+ * }} opts `skipIfSatisfied` is the post-wait re-probe (Story #5278): after a
+ *   contended wait it decides whether the thing this spawn would establish is
+ *   already true, and a non-`undefined` return is returned in the spawn's
+ *   place. Never consulted on the uncontended path.
  * @param {() => T} spawn
  * @returns {T}
  */
@@ -195,6 +374,9 @@ export function withFullSuiteLockSync(
     sleepFn = sleepSync,
     acquireOnceFn = acquireSweepLock,
     lockPath: explicitLockPath,
+    skipIfSatisfied,
+    spawnHeartbeatMs = DEFAULT_SPAWN_HEARTBEAT_MS,
+    startSpawnHeartbeatFn = startSpawnHeartbeat,
   },
   spawn,
 ) {
@@ -208,10 +390,12 @@ export function withFullSuiteLockSync(
     lockPath: explicitLockPath,
   });
   let held = lock;
+  let waited = false;
   if (held === null && lockPath !== null) {
     const deadline = nowFn() + Math.max(0, waitMs);
     for (;;) {
       if (nowFn() >= deadline) break;
+      waited = true;
       sleepFn(Math.max(0, pollMs));
       const attempt = acquireOnceFn({ lockPath, timeoutMs: staleMs, fsImpl });
       if (attempt.acquired) {
@@ -222,7 +406,26 @@ export function withFullSuiteLockSync(
     }
   }
   try {
-    return spawn();
+    const probe = probeAlreadySatisfied(skipIfSatisfied, waited);
+    if (probe.satisfied) {
+      log(
+        '[full-suite-lock] ⏭ the run we waited for already covered this tree — skipping the spawn.',
+      );
+      return probe.value;
+    }
+    refreshBeforeSpawn(held, lockPath, fsImpl);
+    // The main thread is about to stop turning for the whole spawn, so the
+    // holder's own interval heartbeat cannot fire; this one runs off-thread.
+    const stopHeartbeat = safeStartHeartbeat(startSpawnHeartbeatFn, {
+      lockPath: held?.acquired ? lockPath : null,
+      ownerId: held?.ownerId,
+      heartbeatMs: spawnHeartbeatMs,
+    });
+    try {
+      return spawn();
+    } finally {
+      stopHeartbeat();
+    }
   } finally {
     if (held?.acquired) held.release();
   }
@@ -248,7 +451,20 @@ export function lockedCapture(runCaptureFn, config) {
   const enabled = isFullSuiteLockEnabled({ config });
   return (captureOpts = {}) =>
     withFullSuiteLockSync(
-      { cwd: captureOpts.cwd, log: captureOpts.log, enabled },
+      {
+        cwd: captureOpts.cwd,
+        log: captureOpts.log,
+        enabled,
+        // Story #5278 — the capture path supplies its own freshness re-probe
+        // per call, because only it knows which scope ('full' /
+        // 'incremental') the stamp has to satisfy. A `true` means the suite
+        // we queued behind already stamped this tree, so this caller reports
+        // success (exit 0) without spawning a second one.
+        skipIfSatisfied:
+          typeof captureOpts.recheckFresh === 'function'
+            ? () => (captureOpts.recheckFresh() ? 0 : undefined)
+            : undefined,
+      },
       () => runCaptureFn(captureOpts),
     );
 }
@@ -264,7 +480,7 @@ export function lockedCapture(runCaptureFn, config) {
  * @template T
  * @param {Parameters<typeof withFullSuiteLockSync>[0] & {
  *   acquireWithWaitFn?: typeof acquireLockWithWait,
- * }} opts
+ * }} opts `skipIfSatisfied` behaves exactly as in the sync wrapper.
  * @param {() => Promise<T>} spawn
  * @returns {Promise<T>}
  */
@@ -280,6 +496,7 @@ export async function withFullSuiteLockAsync(
     acquireOnceFn = acquireSweepLock,
     acquireWithWaitFn = acquireLockWithWait,
     lockPath: explicitLockPath,
+    skipIfSatisfied,
   },
   spawn,
 ) {
@@ -293,17 +510,26 @@ export async function withFullSuiteLockAsync(
     lockPath: explicitLockPath,
   });
   let held = lock;
+  let waited = false;
   if (held === null && lockPath !== null) {
-    const waited = await acquireWithWaitFn({
+    waited = true;
+    const attempt = await acquireWithWaitFn({
       lockPath,
       waitMs,
       pollMs,
       timeoutMs: staleMs,
       fsImpl,
     });
-    if (waited.acquired) held = waited;
+    if (attempt.acquired) held = attempt;
   }
   try {
+    const probe = probeAlreadySatisfied(skipIfSatisfied, waited);
+    if (probe.satisfied) {
+      log(
+        '[full-suite-lock] ⏭ the run we waited for already covered this tree — skipping the spawn.',
+      );
+      return probe.value;
+    }
     return await spawn();
   } finally {
     if (held?.acquired) held.release();