@yemi33/minions 0.1.2426 → 0.1.2428

package/docs/design-state-storage.md

@@ -48,7 +48,9 @@ release .lock file
 Key properties:
 - **Synchronous blocking** — `withFileLock` spins with `sleepMs(25)` until lock acquired or 5s timeout (source: `engine/shared.js` `withFileLock` ~L1329)
 - **Whole-file granularity** — updating one field in one work item rewrites all 180 items (370 KB)
-- **Stale lock recovery** — locks older than 5 min (`LOCK_STALE_MS = 300_000`) are force-removed; holders that recorded a `{pid, ts}` payload are kept alive past the threshold while `process.kill(pid, 0)` succeeds, with a hard last-resort cap at 5×LOCK_STALE_MS (source: `engine/shared.js`, P-b7d4e8f2)
+- **Stale lock recovery** — locks older than 5 min (`LOCK_STALE_MS = 300_000`) are force-removed; holders that recorded a `{pid, ts, token}` payload are kept alive past the threshold while `process.kill(pid, 0)` succeeds, with a hard last-resort cap at 5×LOCK_STALE_MS (source: `engine/shared.js`, P-b7d4e8f2)
+- **Identity-checked reap and release** — every acquisition stamps a unique `token` into the lock payload. Reaping never unlinks the lock path directly: the reaper re-observes the same token after a short grace delay, takes custody with an atomic `rename`, and verifies the quarantined file is the acquisition it judged stale (restoring it otherwise). Release likewise unlinks only while the on-disk token is still ours. Removing a lock by path alone is a TOCTOU that lets a second writer into the critical section and silently loses one update (source: `engine/shared.js` `withFileLock`, W-ms2nbtkj00016ffc)
+- **Compromised sections fail loudly** — if the lock is gone or re-owned when the critical section ends, `withFileLock` throws `ELOCKCOMPROMISED`. `mutateJsonFileLocked` / `mutateTextFileLocked` first re-apply the mutation against the current on-disk state (up to `LOCK_COMPROMISE_REDOS`) when their own write did not survive, so a raced update is recovered rather than dropped
 - **Read caching** — only `dispatch.json` has a 2s TTL cache (source: `engine/queries.js`)
 
 ### 1.3 Read vs Write Ratio

package/engine/shared.js

@@ -972,7 +972,16 @@ function mutateTextFileLocked(filePath, mutateFn, {
       safeWrite(filePath, finalText);
     }
     return finalText;
-  }, { retries, retryBackoffMs });
+  }, {
+    retries,
+    retryBackoffMs,
+    compromiseRedos: LOCK_COMPROMISE_REDOS,
+    // Our lock was stolen mid-section, so another writer shared it. Redo the
+    // read-modify-write against the current on-disk state only if our own write
+    // did not survive — otherwise the update is intact and a redo would just
+    // re-apply it.
+    onCompromised: (written) => (safeRead(filePath) === written ? 'accept' : 'redo'),
+  });
 }
 
 function safeUnlink(p) {
@@ -1406,14 +1415,73 @@ function pruneCrashDiagnosticsReports(config) {
   return removed;
 }
 
+/**
+ * Read a lock file's holder payload.
+ *
+ * Returns `{ missing, unreadable, payload }` so callers can tell "the lock is
+ * gone" (definitive — someone removed it) from "we couldn't read it right now"
+ * (transient Windows sharing violation) from "it's there but has no parseable
+ * payload" (legacy lock, or the sub-millisecond window between create and
+ * payload write).
+ */
+function _readLockPayload(lockPath) {
+  let raw;
+  try {
+    raw = fs.readFileSync(lockPath, 'utf8');
+  } catch (err) {
+    if (err && err.code === 'ENOENT') return { missing: true, unreadable: false, payload: null };
+    return { missing: false, unreadable: true, payload: null };
+  }
+  try {
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed === 'object') return { missing: false, unreadable: false, payload: parsed };
+  } catch { /* empty (create/write window) or legacy lock → no payload */ }
+  return { missing: false, unreadable: false, payload: null };
+}
+
+/**
+ * Does `payload` describe the same lock acquisition we observed and judged
+ * stale? `observed` is the payload read during the EEXIST inspection.
+ *
+ * Tokens are the authoritative identity. Locks written by an older engine (or
+ * caught in the create/write window) have no token, so fall back to pid — and
+ * treat "still no payload" as "still the same anonymous lock".
+ */
+function _isSameLockAcquisition(observed, payload) {
+  if (observed && observed.token) return !!(payload && payload.token === observed.token);
+  if (payload && payload.token) return false; // a fresh tokened lock replaced it
+  if (observed && Number.isFinite(observed.pid)) return !!(payload && payload.pid === observed.pid);
+  return payload === null;
+}
+
+// Bounded redo budget for a critical section whose lock was compromised (stolen
+// by a reaper or otherwise re-owned mid-section). Read-modify-write callers
+// re-run against the current on-disk state so a raced write is re-applied
+// instead of being silently lost; see mutateJsonFileLocked/mutateTextFileLocked.
+const LOCK_COMPROMISE_REDOS = 3;
+
+/**
+ * Run `fn` under an exclusive file lock at `lockPath`.
+ *
+ * `onCompromised(result)` is consulted when the lock was stolen mid-section
+ * (a reaper removed it, or it is now owned by a different token) — i.e. another
+ * holder shared the critical section. Return `'accept'` to keep `result`, or
+ * `'redo'` to re-run `fn` under a fresh lock (bounded by `compromiseRedos`).
+ * Without the hook a compromised section throws `ELOCKCOMPROMISED` rather than
+ * returning a value that may have been produced from, or clobbered by, a raced
+ * snapshot.
+ */
 function withFileLock(lockPath, fn, {
   timeoutMs = 5000,
   retryDelayMs = 25,
   retries = 0,
-  retryBackoffMs = 1000
+  retryBackoffMs = 1000,
+  compromiseRedos = 0,
+  onCompromised = null
 } = {}) {
   let lastErr = null;
   const maxAttempts = 1 + Math.max(0, retries);
+  let redosLeft = Math.max(0, compromiseRedos);
   for (let attempt = 0; attempt < maxAttempts; attempt++) {
     if (attempt > 0) {
       // Exponential backoff between retry attempts: retryBackoffMs * 2^(attempt-1)
@@ -1421,19 +1489,32 @@ function withFileLock(lockPath, fn, {
       sleepMs(backoff);
     }
     const start = Date.now();
-    let fd = null;
+    let acquired = false;
+    // Per-acquisition identity. The reaper and the release path both compare
+    // against this so neither ever destroys a lock it does not own — an unlink
+    // keyed only on the path is a TOCTOU that lets two holders into the
+    // critical section at once (W-ms2nbtkj00016ffc).
+    const token = `${process.pid}-${uid()}`;
+    let tokenPersisted = false;
     while (Date.now() - start < timeoutMs) {
       try {
         const dir = path.dirname(lockPath);
         if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-        fd = fs.openSync(lockPath, 'wx');
+        const fd = fs.openSync(lockPath, 'wx');
         // P-b7d4e8f2 — record holder identity so the stale-lock reaper can
         // distinguish a still-alive slow holder from a crashed one. Best-effort:
         // the lock's existence (not its contents) provides mutual exclusion, so
-        // a write failure here must NOT abort acquisition.
+        // a write failure here must NOT abort acquisition (it only downgrades us
+        // to the legacy pid-keyed release path).
         try {
-          fs.writeSync(fd, JSON.stringify({ pid: process.pid, ts: Date.now() }));
+          fs.writeSync(fd, JSON.stringify({ pid: process.pid, ts: Date.now(), token }));
+          tokenPersisted = true;
         } catch { /* payload is advisory; lock semantics unaffected */ }
+        // Close immediately: an open handle keeps the file in Windows'
+        // "pending delete" limbo after unlink, which makes waiters keep seeing
+        // (and repeatedly trying to reap) a lock that is already released.
+        try { fs.closeSync(fd); } catch { /* handle is not load-bearing */ }
+        acquired = true;
         break;
       } catch (err) {
         // EEXIST is the cross-platform exclusive-create collision.
@@ -1453,29 +1534,60 @@ function withFileLock(lockPath, fn, {
         try {
           const stat = fs.statSync(lockPath);
           const mtimeAge = Date.now() - stat.mtimeMs;
-          let holderPid = null;
-          try {
-            const raw = fs.readFileSync(lockPath, 'utf8');
-            const parsed = JSON.parse(raw);
-            if (parsed && Number.isFinite(parsed.pid) && parsed.pid > 0) {
-              holderPid = parsed.pid;
-            }
-          } catch { /* legacy/empty/corrupt lock → mtime-only fallback */ }
+          const observed = _readLockPayload(lockPath).payload;
+          const holderPid = observed && Number.isFinite(observed.pid) && observed.pid > 0
+            ? observed.pid
+            : null;
 
           const shouldReap = holderPid !== null
             ? !processUtils.isPidAlive(holderPid) || mtimeAge > LOCK_STALE_MS * 5
             : mtimeAge > LOCK_STALE_MS;
 
           if (shouldReap) {
+            // Do not race a lock that is simply being released. Require the
+            // SAME acquisition (identified by its token) to still be present
+            // after a short grace delay: a normally-releasing holder's lock
+            // disappears inside that window, and a fresh holder's lock carries
+            // a different token. Without this, the reaper routinely removed a
+            // brand-new live holder's lock in the microseconds after the
+            // previous holder exited (W-ms2nbtkj00016ffc).
+            sleepMs(retryDelayMs);
+            const recheck = _readLockPayload(lockPath);
+            if (recheck.missing || recheck.unreadable
+              || !_isSameLockAcquisition(observed, recheck.payload)) {
+              continue; // resolved itself — back to normal contention
+            }
+            // Reap by IDENTITY, never by path. Renaming is atomic, so exactly
+            // one racer takes custody and can then verify that what it took is
+            // the acquisition it judged stale.
+            const reapPath = `${lockPath}.reap-${process.pid}-${uid()}`;
+            let reaped = false;
             try {
-              fs.unlinkSync(lockPath);
-            } catch (unlinkErr) {
-              // ENOENT: another process deleted the lock between stat and unlink — safe to retry.
-              // EPERM on Windows: file is in 'pending delete' state from a concurrent
-              // reaper — the OS will finalize the delete shortly; retry will succeed.
-              const tolerable = unlinkErr.code === 'ENOENT' ||
-                (process.platform === 'win32' && unlinkErr.code === 'EPERM');
-              if (!tolerable) throw unlinkErr;
+              fs.renameSync(lockPath, reapPath);
+              reaped = true;
+            } catch (reapErr) {
+              // ENOENT: another reaper/holder already removed it — retry.
+              // EPERM on Windows: pending-delete state — retry.
+              const tolerable = reapErr.code === 'ENOENT' ||
+                (process.platform === 'win32' && reapErr.code === 'EPERM');
+              if (!tolerable) throw reapErr;
+            }
+            if (reaped) {
+              const quarantined = _readLockPayload(reapPath).payload;
+              if (_isSameLockAcquisition(observed, quarantined)) {
+                try { fs.unlinkSync(reapPath); } catch { /* already gone */ }
+              } else {
+                // We took custody of a lock we never judged stale — a live
+                // holder replaced the stale one in the last instant. Put it
+                // back and keep waiting. linkSync fails if a newer lock already
+                // exists, which is the correct outcome (that lock is the live
+                // one); the displaced holder then detects the compromise at
+                // release rather than losing its write silently.
+                try { fs.linkSync(reapPath, lockPath); } catch { /* superseded */ }
+                try { fs.unlinkSync(reapPath); } catch { /* already gone */ }
+                sleepMs(retryDelayMs);
+                continue;
+              }
             }
             continue; // lock just removed — retry immediately
           }
@@ -1490,31 +1602,64 @@ function withFileLock(lockPath, fn, {
         sleepMs(retryDelayMs);
       }
     }
-    if (fd === null) {
+    if (!acquired) {
       lastErr = new Error(`Lock timeout: ${lockPath}`);
       continue; // retry if attempts remain
     }
 
+    let result;
+    let thrown = null;
     try {
-      const result = fn();
+      result = fn();
       // P-a3f9b2c1 — Defensive: detect a thenable return and throw synchronously.
-      // The `finally` below releases the lock immediately after `fn()` returns;
-      // an async callback would let the lock be released before its body
-      // completes, silently breaking mutual exclusion. Clean up our own fd /
-      // lock first, then throw so the caller cannot ignore the failure.
+      // The lock is released immediately after `fn()` returns; an async callback
+      // would let the lock be released before its body completes, silently
+      // breaking mutual exclusion. Release below, then throw so the caller
+      // cannot ignore the failure.
       if (result && typeof result.then === 'function') {
-        try { fs.closeSync(fd); } catch { /* cleanup */ }
-        try { fs.unlinkSync(lockPath); } catch { /* cleanup */ }
-        fd = null; // suppress double-cleanup in `finally`
+        result = undefined;
         throw new Error('withFileLock: fn must be synchronous; got Promise. Use synchronous operations only.');
       }
-      return result;
-    } finally {
-      if (fd !== null) {
-        try { fs.closeSync(fd); } catch { /* cleanup */ }
-        try { fs.unlinkSync(lockPath); } catch { /* cleanup */ }
+    } catch (err) {
+      thrown = err;
+    }
+
+    // Release by IDENTITY. If the lock file is gone, or now carries a different
+    // token, a reaper stole it mid-section: someone else was in the critical
+    // section with us, so (a) do NOT unlink — that would destroy the successor's
+    // lock and cascade the breach — and (b) report the breach instead of
+    // silently returning a value that may have been produced from, or clobbered
+    // by, a raced snapshot.
+    let compromised = false;
+    const held = tokenPersisted ? _readLockPayload(lockPath) : null;
+    if (held && held.missing) {
+      compromised = true;
+    } else if (held && held.payload && held.payload.token && held.payload.token !== token) {
+      compromised = true;
+    } else {
+      try {
+        fs.unlinkSync(lockPath);
+      } catch (unlinkErr) {
+        // ENOENT means it was removed under us — same breach as `missing`.
+        if (unlinkErr && unlinkErr.code === 'ENOENT') compromised = true;
+        // Anything else is best-effort; the stale reaper will clear it.
       }
     }
+
+    if (thrown) throw thrown;
+    if (compromised) {
+      if (typeof onCompromised === 'function' && redosLeft > 0) {
+        redosLeft--;
+        if (onCompromised(result) !== 'redo') return result;
+        attempt--; // a compromised section is not a failed acquisition attempt
+        sleepMs(retryDelayMs);
+        continue;
+      }
+      const err = new Error(`Lock compromised (concurrent holder detected): ${lockPath}`);
+      err.code = 'ELOCKCOMPROMISED';
+      throw err;
+    }
+    return result;
   }
   throw lastErr;
 }
@@ -1640,7 +1785,21 @@ function mutateJsonFileLocked(filePath, mutateFn, {
       }
       throw error;
     }
-  }, { retries, retryBackoffMs });
+  }, {
+    retries,
+    retryBackoffMs,
+    compromiseRedos: LOCK_COMPROMISE_REDOS,
+    // Our lock was stolen mid-section, so another writer shared it and one of
+    // the two updates was clobbered. Redo the read-modify-write against the
+    // current on-disk state only if our own write did not survive; the other
+    // writer detects and redoes its own loss the same way, so the union is
+    // preserved without double-applying an intact mutation.
+    onCompromised: (written) => {
+      if (written === null) return fs.existsSync(filePath) ? 'redo' : 'accept';
+      const current = safeJsonNoRestore(filePath);
+      return JSON.stringify(current) === JSON.stringify(written) ? 'accept' : 'redo';
+    },
+  });
 }
 
 function mutateControl(mutator) {
@@ -9264,6 +9423,9 @@ module.exports = {
   resolveDispatchPrompt,
   deleteDispatchPromptSidecar,
   withFileLock,
+  _readLockPayload, // exported for testing
+  _isSameLockAcquisition, // exported for testing
+  LOCK_COMPROMISE_REDOS, // exported for testing
   mutateJsonFileLocked,
   mutateControl, recordEngineRespawn,
   mutateEngineState, // W-mp60tw0u000j3931

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2426",
+  "version": "0.1.2428",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"