@yemi33/minions 0.1.2147 → 0.1.2149

package/engine/worktree-gc.js

@@ -79,18 +79,60 @@ function _markStuckSuccess(resolvedPath, opts = {}) {
     try {
       const writer = typeof opts.writeToInbox === 'function' ? opts.writeToInbox : shared.writeToInbox;
       const basename = path.basename(resolvedPath);
-      const body = [
-        `# Worktree recovered: ${basename}`,
-        '',
-        `Previously stuck worktree dir **${resolvedPath}** has been successfully removed`,
-        `after escalation. Suppression cleared.`,
-      ].join('\n');
-      writer('engine', `worktree-recovered-${basename}`, body);
+      // W-mq6f2fe0000557fa — when the recovery is the direct result of an
+      // auto-reap kill, write a distinct slug + body so operators can see
+      // which orphan-sweep escalations were resolved by the engine itself
+      // vs. naturally by an external holder release.
+      const viaReap = opts.recoveryReason === 'holder-reap';
+      const reapedPids = Array.isArray(opts.reapedPids) ? opts.reapedPids : [];
+      const slug = viaReap
+        ? `worktree-recovered-${basename}-via-holder-reap`
+        : `worktree-recovered-${basename}`;
+      const bodyLines = viaReap
+        ? [
+          `# Worktree recovered via holder-reap: ${basename}`,
+          '',
+          `Previously stuck worktree dir **${resolvedPath}** was successfully removed`,
+          `after the engine auto-killed the orphan spawn-agent holder(s)`,
+          `${reapedPids.length > 0 ? `(PID${reapedPids.length > 1 ? 's' : ''} ${reapedPids.join(', ')})` : ''}`.trim(),
+          `under the \`engine.autoReapOrphanWorktreeHolders\` flag (W-mq6f2fe0000557fa).`,
+          `Suppression cleared.`,
+        ]
+        : [
+          `# Worktree recovered: ${basename}`,
+          '',
+          `Previously stuck worktree dir **${resolvedPath}** has been successfully removed`,
+          `after escalation. Suppression cleared.`,
+        ];
+      writer('engine', slug, bodyLines.join('\n'));
     } catch { /* note best-effort */ }
   }
   return wasEscalated;
 }
 
+// W-mq6f2fe0000557fa — gate: is this holder an unambiguous orphan spawn-agent
+// process we can safely kill? Requires ALL of:
+//   (a) cmdline contains 'spawn-agent.js' (only minions agent wrappers qualify)
+//   (b) cmdline references the worktree basename (extra confirmation it's
+//       OUR spawn-agent and not an unrelated process that happens to mention
+//       the path)
+//   (c) ageMs > agentTimeoutMs * 2 (the worktree was definitely abandoned by
+//       its dispatch; agentTimeout default is 5h, so 10h)
+//
+// The orphan-sweep code path already guarantees no live dispatch row claims
+// the path — we'd never be here otherwise — so the "no live dispatch" check
+// is implicit, not duplicated here.
+function _isSafeReapHolder(holder, basename, minAgeMs) {
+  if (!holder || !holder.pid || !holder.cmdline) return false;
+  const cmd = String(holder.cmdline);
+  if (!/spawn-agent\.js/i.test(cmd)) return false;
+  if (basename && !cmd.includes(basename)) return false;
+  const age = Number(holder.ageMs) || 0;
+  if (age <= 0) return false; // unknown age — refuse to kill
+  if (age < minAgeMs) return false;
+  return true;
+}
+
 function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
   const cfgEng = (opts.config && opts.config.engine) || {};
   const threshold = cfgEng.worktreeStuckThreshold || shared.ENGINE_DEFAULTS.worktreeStuckThreshold || 10;
@@ -100,7 +142,7 @@ function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
   rec.failures++;
   rec.lastAttempt = now;
   rec.lastError = errorMsg || rec.lastError || 'unknown';
-  if (rec.failures < threshold) return { escalated: false, alreadyEscalated: false };
+  if (rec.failures < threshold) return { escalated: false, alreadyEscalated: false, holders: [], reapedPids: [] };
 
   const alreadyEscalated = rec.escalatedAt > 0;
   if (!alreadyEscalated) {
@@ -112,15 +154,57 @@ function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
     rec.suppressedUntil = now + suppressMs;
   }
 
+  // W-mq6f2fe0000557fa — orphan-sweep enrichment + opt-in auto-reap. Scans
+  // and reap-attempts run on EVERY escalated tick (no per-day dedup) so that
+  // a fresh holder appearing after the first escalation still gets killed.
+  // The inbox NOTE is still dedup'd per UTC day below.
+  let holders = [];
+  let reapedPids = [];
+  const isOrphanSweep = opts.reason === 'orphan-sweep';
+  if (isOrphanSweep) {
+    try {
+      const findHoldersFn = typeof opts.findProcessesWithCwdInside === 'function'
+        ? opts.findProcessesWithCwdInside
+        : shared.findProcessesWithCwdInside;
+      const scanTimeoutMs = cfgEng.orphanHolderScanTimeoutMs
+        ?? shared.ENGINE_DEFAULTS.orphanHolderScanTimeoutMs
+        ?? 5000;
+      holders = findHoldersFn(resolvedPath, { timeoutMs: scanTimeoutMs }) || [];
+    } catch { holders = []; }
+
+    if (cfgEng.autoReapOrphanWorktreeHolders === true && holders.length > 0) {
+      const basename = path.basename(resolvedPath);
+      const agentTimeoutMs = Number(cfgEng.agentTimeout)
+        || Number(shared.ENGINE_DEFAULTS.agentTimeout)
+        || (5 * 60 * 60 * 1000);
+      const minAgeMs = agentTimeoutMs * 2;
+      const killFn = typeof opts.killImmediate === 'function'
+        ? opts.killImmediate
+        : shared.killImmediate;
+      for (const h of holders) {
+        if (!_isSafeReapHolder(h, basename, minAgeMs)) continue;
+        try {
+          killFn({ pid: h.pid });
+          reapedPids.push(h.pid);
+        } catch { /* fail-open — keep trying other holders */ }
+      }
+      if (reapedPids.length > 0) {
+        try { shared.bumpWorktreeGcMetric('orphanHoldersReaped'); } catch { /* optional */ }
+      }
+    }
+  }
+
   // Dedup inbox note per UTC day.
   const today = new Date(now).toISOString().slice(0, 10);
-  if (rec.lastNoteDate === today) return { escalated: !alreadyEscalated, alreadyEscalated };
+  if (rec.lastNoteDate === today) {
+    return { escalated: !alreadyEscalated, alreadyEscalated, holders, reapedPids };
+  }
   rec.lastNoteDate = today;
 
   try {
     const writer = typeof opts.writeToInbox === 'function' ? opts.writeToInbox : shared.writeToInbox;
     const basename = path.basename(resolvedPath);
-    const body = [
+    const bodyParts = [
       `# Worktree stuck: ${basename}`,
       '',
       `Worktree dir **${resolvedPath}** has failed removal ${rec.failures} consecutive times.`,
@@ -153,16 +237,116 @@ function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
       '',
       `Per-tick warn spam is suppressed for the next`,
       `\`ENGINE_DEFAULTS.worktreeStuckSuppressMs\` (default 60min).`,
-    ].join('\n');
-    writer('engine', `worktree-stuck-${basename}`, body);
+    ];
+
+    // W-mq6f2fe0000557fa — append the holder section for orphan-sweep callers.
+    if (isOrphanSweep) {
+      bodyParts.push('', '## Live holders');
+      bodyParts.push('');
+      if (holders.length === 0) {
+        bodyParts.push(
+          '_No holder process found via cwd scan._',
+          '',
+          'If the dir is still stuck, the holder may be a non-process resource (',
+          'antivirus scan-in-progress, OneDrive sync, indexer); try again after',
+          '~60s, or run `handle.exe` interactively for a deeper search.'
+        );
+      } else {
+        for (const h of holders) {
+          const tail = String(h.cmdline || '').replace(/\r?\n/g, ' ').slice(0, 300);
+          const ageStr = h.ageMs ? `${Math.floor(h.ageMs / 1000)}s` : 'unknown';
+          bodyParts.push(`- **PID ${h.pid}** (age: ${ageStr})`);
+          bodyParts.push('  ```');
+          bodyParts.push(`  ${tail}`);
+          bodyParts.push('  ```');
+        }
+        bodyParts.push('');
+        if (cfgEng.autoReapOrphanWorktreeHolders === true) {
+          if (reapedPids.length > 0) {
+            bodyParts.push(
+              `## Auto-reap`,
+              '',
+              `Engine auto-killed ${reapedPids.length} unambiguous orphan`,
+              `spawn-agent process${reapedPids.length === 1 ? '' : 'es'}: PID${reapedPids.length === 1 ? '' : 's'} ${reapedPids.join(', ')}.`,
+              `If the kill released the cwd lock, the next sweep tick will`,
+              `successfully remove the worktree and emit a`,
+              `\`worktree-recovered-${basename}-via-holder-reap\` recovery note.`
+            );
+          } else {
+            bodyParts.push(
+              `## Auto-reap`,
+              '',
+              `\`engine.autoReapOrphanWorktreeHolders\` is ON but no holder`,
+              `qualified for auto-kill (cmdline must match \`spawn-agent.js\`,`,
+              `reference the worktree basename, and process age must exceed`,
+              `\`engine.agentTimeout * 2\`). Manual intervention may be needed.`
+            );
+          }
+        } else {
+          bodyParts.push(
+            `_To auto-kill orphan spawn-agent processes holding this worktree,_`,
+            `_flip \`engine.autoReapOrphanWorktreeHolders\` ON in dashboard Settings._`
+          );
+        }
+      }
+    }
+
+    writer('engine', `worktree-stuck-${basename}`, bodyParts.join('\n'));
   } catch { /* note best-effort */ }
 
-  return { escalated: !alreadyEscalated, alreadyEscalated };
+  return { escalated: !alreadyEscalated, alreadyEscalated, holders, reapedPids };
 }
 
 // Test-only reset hook. Production code never calls this.
 function _resetStuckPathsForTesting() { _stuckPaths.clear(); }
 
+// W-mq6f2fe0000557fa — post-auto-reap retry. Called only when
+// `_maybeEscalateStuck` reports `reapedPids.length > 0`. Clears the per-path
+// failure cooldown (otherwise the post-3-strike suppression in
+// `shared.removeWorktree` silently skips the retry), waits briefly for the
+// OS to release the killed processes' file handles, then attempts ONE more
+// removal. On success, writes the recovered-via-holder-reap note via
+// `_markStuckSuccess` with the reaped-pid attribution.
+function _postReapRetry(wtPath, gitRoot, parentDir, resolvedPath, _removeWorktree, opts, projStats, result, log, reapedPids) {
+  try { shared.clearWorktreeFailureCache(resolvedPath); } catch { /* optional */ }
+  // Brief settle window for OS to release file handles from killed processes.
+  try {
+    const sleepFn = typeof opts.sleepSyncFn === 'function'
+      ? opts.sleepSyncFn
+      : (ms) => {
+        try {
+          require('child_process').execFileSync(process.execPath, ['-e', `setTimeout(()=>process.exit(0), ${Number(ms) || 0})`], { stdio: 'ignore' });
+        } catch { /* timeout / spawn fail — proceed anyway */ }
+      };
+    sleepFn(2000);
+  } catch { /* sleep failures should not block the retry */ }
+
+  try {
+    const removed = _removeWorktree(wtPath, gitRoot, parentDir);
+    if (removed) {
+      // Down-count the failure we recorded just before; this dispatch
+      // ultimately succeeded after the auto-reap.
+      if (projStats) { projStats.failed = Math.max(0, projStats.failed - 1); projStats.evicted++; }
+      if (result) { result.failed = Math.max(0, result.failed - 1); result.evicted++; }
+      _markStuckSuccess(resolvedPath, {
+        writeToInbox: opts.writeToInbox,
+        recoveryReason: 'holder-reap',
+        reapedPids,
+      });
+      try { shared.bumpWorktreeGcMetric('recoveredViaHolderReap'); } catch { /* optional */ }
+      if (typeof log === 'function') {
+        log('info', `worktree-gc: removed ${wtPath} after auto-reaping holder(s) ${reapedPids.join(', ')}`);
+      }
+      return true;
+    }
+  } catch (retryErr) {
+    if (typeof log === 'function') {
+      log('warn', `worktree-gc: post-reap retry threw for ${wtPath}: ${retryErr && retryErr.message}`);
+    }
+  }
+  return false;
+}
+
 /**
  * Decide whether a dispatch-end worktree should be GC'd.
  *
@@ -494,19 +678,39 @@ function pruneOrphanWorktrees(opts) {
           log('info', `worktree-gc: boot-evicted orphan ${name} for project ${project.name || 'default'}`);
         } else {
           projStats.failed++; result.failed++;
-          const { alreadyEscalated } = _maybeEscalateStuck(wtResolved, 'remove returned false', { config: opts.config, writeToInbox: opts.writeToInbox });
-          if (!alreadyEscalated && !suppressed) {
+          const escResult = _maybeEscalateStuck(wtResolved, 'remove returned false', {
+            config: opts.config, writeToInbox: opts.writeToInbox,
+            reason: 'orphan-sweep',
+            findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
+            killImmediate: opts.killImmediate,
+          });
+          if (!escResult.alreadyEscalated && !suppressed) {
             log('warn', `worktree-gc: boot-evict returned false for ${wtPath}`);
           }
+          // W-mq6f2fe0000557fa — if auto-reap killed any holders, retry the
+          // removal once after a brief settle window. Clear the failure
+          // cache entry so the post-reap retry isn't suppressed by the
+          // 3-strike cooldown.
+          if (escResult.reapedPids && escResult.reapedPids.length > 0) {
+            _postReapRetry(wtPath, rootDir, wtParent, wtResolved, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
+          }
         }
       } catch (rmErr) {
         projStats.failed++; result.failed++;
         const wtResolved = path.resolve(wtPath);
         const suppressed = _isStuckPathSuppressed(wtResolved);
-        const { alreadyEscalated } = _maybeEscalateStuck(wtResolved, rmErr && rmErr.message, { config: opts.config, writeToInbox: opts.writeToInbox });
-        if (!alreadyEscalated && !suppressed) {
+        const escResult = _maybeEscalateStuck(wtResolved, rmErr && rmErr.message, {
+          config: opts.config, writeToInbox: opts.writeToInbox,
+          reason: 'orphan-sweep',
+          findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
+          killImmediate: opts.killImmediate,
+        });
+        if (!escResult.alreadyEscalated && !suppressed) {
           log('warn', `worktree-gc: boot-evict threw for ${wtPath}: ${rmErr.message}`);
         }
+        if (escResult.reapedPids && escResult.reapedPids.length > 0) {
+          _postReapRetry(wtPath, rootDir, wtParent, wtResolved, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
+        }
       }
     }
     result.perProject[project.name || rootDir] = projStats;
@@ -693,17 +897,33 @@ function pruneOrphanWorktreesFromGitRegistry(opts) {
           log('info', `worktree-gc: out-of-root evicted ${wtAbs} for project ${project.name || 'default'}`);
         } else {
           projStats.failed++; result.failed++;
-          const { alreadyEscalated } = _maybeEscalateStuck(wtAbs, 'remove returned false', { config: opts.config, writeToInbox: opts.writeToInbox });
-          if (!alreadyEscalated && !suppressed) {
+          const escResult = _maybeEscalateStuck(wtAbs, 'remove returned false', {
+            config: opts.config, writeToInbox: opts.writeToInbox,
+            reason: 'orphan-sweep',
+            findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
+            killImmediate: opts.killImmediate,
+          });
+          if (!escResult.alreadyEscalated && !suppressed) {
             log('warn', `worktree-gc: out-of-root remove returned false for ${wtAbs}`);
           }
+          if (escResult.reapedPids && escResult.reapedPids.length > 0) {
+            _postReapRetry(wtAbs, rootDir, parentDir, wtAbs, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
+          }
         }
       } catch (rmErr) {
         projStats.failed++; result.failed++;
-        const { alreadyEscalated } = _maybeEscalateStuck(wtAbs, rmErr && rmErr.message, { config: opts.config, writeToInbox: opts.writeToInbox });
-        if (!alreadyEscalated && !suppressed) {
+        const escResult = _maybeEscalateStuck(wtAbs, rmErr && rmErr.message, {
+          config: opts.config, writeToInbox: opts.writeToInbox,
+          reason: 'orphan-sweep',
+          findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
+          killImmediate: opts.killImmediate,
+        });
+        if (!escResult.alreadyEscalated && !suppressed) {
           log('warn', `worktree-gc: out-of-root remove threw for ${wtAbs}: ${rmErr.message}`);
         }
+        if (escResult.reapedPids && escResult.reapedPids.length > 0) {
+          _postReapRetry(wtAbs, rootDir, parentDir, wtAbs, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
+        }
       }
     }