@yemi33/minions 0.1.2044 → 0.1.2046

package/engine.js

@@ -630,6 +630,26 @@ async function syncReusedWorktree(rootDir, worktreePath, branchName, gitOpts = {
 }
 
 // Find an existing worktree already checked out on a given branch
+// Parse the holder path out of a `git worktree add` "already used" error so
+// callers can give the operator something actionable instead of just re-
+// throwing the raw git message. The two surface forms emitted by current git
+// versions:
+//
+//   fatal: 'work/W-X' is already used by worktree at 'D:/squad'
+//   fatal: 'work/W-X' is already checked out at 'D:/squad'
+//
+// Returns null on unparseable input so callers fall through to the generic
+// re-throw path safely. Path is returned exactly as git printed it (forward
+// or back slashes), since callers compare it against rootDir which is
+// shaped the same way by upstream.
+function _parseAlreadyUsedHolderPath(errMsg) {
+  if (!errMsg) return null;
+  // Match the path inside single quotes after either "already used by worktree at"
+  // or "already checked out at". Non-greedy so we stop at the closing quote.
+  const m = String(errMsg).match(/already (?:used by worktree|checked out) at ['"]([^'"]+)['"]/);
+  return m ? m[1] : null;
+}
+
 async function findExistingWorktree(repoDir, branchName) {
   try {
     const out = await shared.shellSafeGit(['worktree', 'list', '--porcelain'], { cwd: repoDir, timeout: 10000 });
@@ -658,6 +678,65 @@ function isWorktreeRetryableError(err) {
     || msg.includes('already exists');
 }
 
+// Distinct from isWorktreeRetryableError above: that one covers worktree-add
+// LOCAL contention (index lock, file busy, already exists). This one covers
+// `git fetch` NETWORK-side transients where retrying once is cheap and often
+// recovers — network blips, transient TLS errors, github.com 5xx during a
+// fetch. We deliberately do NOT retry on auth failures, repo-not-found, or
+// "couldn't find remote ref" — those won't change on a 1-2s retry, and
+// re-running them just wastes the dispatch budget.
+function _isTransientGitNetworkError(err) {
+  const msg = String(err?.message || '');
+  return msg.includes('ETIMEDOUT')
+    || msg.includes('ECONNRESET')
+    || msg.includes('ECONNREFUSED')
+    || msg.includes('EAI_AGAIN')
+    || msg.includes('Could not resolve host')
+    || msg.includes('Connection reset')
+    || msg.includes('Connection timed out')
+    || msg.includes('Operation timed out')
+    || msg.includes('timed out')
+    || msg.includes('502 Bad Gateway')
+    || msg.includes('503 Service Unavailable')
+    || msg.includes('504 Gateway Timeout')
+    || msg.includes('RPC failed')
+    || msg.includes('HTTP/2 stream');
+}
+
+// Wraps shellSafeGit(['fetch', ...]) with a single retry on transient
+// network errors. Preserves the pre-fix convention that fetch failures
+// are NON-fatal here: we log + swallow the terminal error so the caller
+// falls back to the local ref. The retry just gives transient errors one
+// shot to recover before that fallback kicks in — empirically eliminates
+// most "couldn't find remote ref" cascades downstream caused by a 0-second
+// network blip during the initial fetch.
+//
+// `args` is the array passed to git AFTER 'fetch' (e.g. ['origin', branch]).
+// `label` is a short string used in the log line for triage.
+async function _fetchWithTransientRetry(args, opts, label) {
+  const _attempt = async () => shared.shellSafeGit(['fetch', ...args], opts);
+  try {
+    await _attempt();
+    return true;
+  } catch (e) {
+    if (!_isTransientGitNetworkError(e)) {
+      log('warn', `git fetch ${label}: ${String(e.message || e).split('\n')[0]}`);
+      return false; // swallow non-transient — caller falls back to local ref
+    }
+    const firstMsg = String(e.message || e).split('\n')[0];
+    log('warn', `git fetch ${label}: transient (${firstMsg}) — retrying once after 1.5s`);
+    await new Promise(r => setTimeout(r, 1500));
+    try {
+      await _attempt();
+      log('info', `git fetch ${label}: succeeded on retry`);
+      return true;
+    } catch (e2) {
+      log('warn', `git fetch ${label}: retry also failed (${String(e2.message || e2).split('\n')[0]}) — falling back to local ref`);
+      return false;
+    }
+  }
+}
+
 function removeStaleIndexLock(rootDir) {
   const lockFile = path.join(rootDir, '.git', 'index.lock');
   try {
@@ -1055,6 +1134,62 @@ async function spawnAgent(dispatchItem, config) {
     );
     cleanupTempAgent(agentId);
   };
+  // Atomic read of the dispatch state to detect when another active
+  // dispatch holds `branchName`. Returns the conflicting dispatch object
+  // (or null when none). Uses mutateDispatch for the file-lock read even
+  // though we don't mutate — the snapshot it gives back is consistent
+  // with what a concurrent writer would see. Caller uses the return value
+  // both as a boolean (truthy = conflict) and to extract the other
+  // dispatch's id for the error message.
+  const _isBranchActivelyUsedByOtherDispatch = (branch, currentId) => {
+    if (!branch || !currentId) return null;
+    const targetBranch = sanitizeBranch(branch);
+    let conflict = null;
+    mutateDispatch((dp) => {
+      conflict = (dp.active || []).find(d => {
+        const dBranch = d.meta?.branch ? sanitizeBranch(d.meta.branch) : '';
+        return dBranch === targetBranch && d.id !== currentId;
+      }) || null;
+      return dp;
+    });
+    return conflict;
+  };
+  // Fail-fast handler for the "branch is checked out elsewhere AND prune
+  // couldn't recover it" case. This is the bug class operators hit when the
+  // project root itself (or a sibling worktree we can't see via
+  // findExistingWorktree) is genuinely holding the branch — the existing
+  // "throw eRemote" path retried every tick forever with no surface signal,
+  // and findExistingWorktree deliberately filters out the project root so
+  // the reuse path never fires for that case. Mark the dispatch non-
+  // retryable so the storm stops, and put the holder path in the message
+  // so the operator can see what to do (`git -C <holder> checkout master`
+  // when the holder is the project root, or kill/finish the agent owning
+  // the sibling worktree). Returns truthy when caller should `return null`.
+  const _failBranchHeldByExternalWorktree = (branchName, holderPath, rawErr) => {
+    const isProjectRoot = holderPath && rootDir
+      && path.resolve(holderPath) === path.resolve(rootDir);
+    const summary = isProjectRoot
+      ? `Branch ${branchName} is held by the project root (${holderPath}). Spawning a worktree against it would create a nested checkout. Switch the root back to master to release the branch.`
+      : `Branch ${branchName} is held by an external worktree at ${holderPath} that this engine can't reach (findExistingWorktree filters out paths at-or-inside the project root, and the holder isn't visible to our worktree list). Resolve the lock and retry.`;
+    log('error', `spawnAgent: ${summary}`);
+    _cleanupPromptFiles();
+    completeDispatch(
+      id,
+      DISPATCH_RESULT.ERROR,
+      summary.slice(0, 800),
+      isProjectRoot
+        ? 'Branch held by the project root checkout. Recovery: `git -C <projectRoot> checkout master` (or whichever branch your engine root should be on). Until the root releases the branch, every retry will fail identically.'
+        : 'Branch held externally. Recovery: find the holding worktree (`git worktree list` from the project root), finish or remove it, then re-dispatch.',
+      { failureClass: FAILURE_CLASS.WORKTREE_PREFLIGHT, agentRetryable: false },
+    );
+    cleanupTempAgent(agentId);
+    // Preserve the raw git output in the dispatch log for debugging — kept
+    // as a side log line rather than the summary so the summary stays
+    // actionable.
+    if (rawErr && rawErr.message) {
+      log('debug', `spawnAgent: raw worktree-add failure for ${id}: ${String(rawErr.message).split('\n')[0]}`);
+    }
+  };
   _phaseT.afterPrompt = Date.now();
 
   if (branchName && READ_ONLY_ROOT_TASK_TYPES.has(type)) {
@@ -1173,6 +1308,31 @@ async function spawnAgent(dispatchItem, config) {
         }
       }
 
+      // Pre-add concurrency guard: refuse to attempt `git worktree add` for a
+      // branch that's already in flight under another dispatch. The
+      // post-failure activelyUsed check on the non-shared path further down
+      // still fires for the legitimate-reuse-of-existing-worktree case, but
+      // the pre-add check closes the race window where two concurrent
+      // dispatches both call `git worktree add` on the same branch before
+      // either notices the conflict — and it now also covers the
+      // shared-branch path, which previously had no activelyUsed guard at
+      // all (a human-triggered review on a shared-branch plan item could
+      // blindly reuse the in-flight implement's worktree).
+      const _activelyUsedByOther = _isBranchActivelyUsedByOtherDispatch(branchName, id);
+      if (_activelyUsedByOther) {
+        const summary = `branch ${branchName} is actively used by dispatch ${_activelyUsedByOther.id} — refusing to spawn a concurrent worktree against the same branch`;
+        log('warn', `spawnAgent: ${summary}`);
+        _cleanupPromptFiles();
+        completeDispatch(
+          id,
+          DISPATCH_RESULT.ERROR,
+          summary,
+          'Another dispatch holds this branch right now. This is retryable — the engine will try again on the next tick after the other dispatch completes.',
+        );
+        cleanupTempAgent(agentId);
+        return null;
+      }
+
       try {
         if (!fs.existsSync(worktreePath)) {
           const isSharedBranch = meta?.branchStrategy === 'shared-branch' || meta?.useExistingBranch;
@@ -1183,7 +1343,7 @@ async function spawnAgent(dispatchItem, config) {
 
           if (isSharedBranch) {
             log('info', `Creating worktree for shared branch: ${worktreePath} on ${branchName}`);
-            try { await shared.shellSafeGit(['fetch', 'origin', branchName], { ..._gitOpts, cwd: rootDir }); } catch (e) { log('warn', 'git: ' + e.message); }
+            await _fetchWithTransientRetry(['origin', branchName], { ..._gitOpts, cwd: rootDir }, `origin/${branchName} (shared-branch pre-create)`);
             try {
               await runWorktreeAdd(rootDir, worktreePath, [branchName], _worktreeGitOpts, worktreeCreateRetries);
             } catch (eShared) {
@@ -1205,7 +1365,16 @@ async function spawnAgent(dispatchItem, config) {
                   if (pruned > 0) {
                     log('info', `Pruned ${pruned} stale worktree entry(ies) for shared branch ${branchName}; retrying worktree add`);
                     await runWorktreeAdd(rootDir, worktreePath, [branchName], _worktreeGitOpts, 0);
-                  } else { throw eShared; }
+                  } else {
+                    // Prune returned 0 — the holder is real, not stale metadata.
+                    // findExistingWorktree returned null because the holder is
+                    // at-or-inside the project root (the deliberate filter), or
+                    // because git's view drifted from ours. Surface non-retryably
+                    // with the holder path so the operator can act.
+                    const holder = _parseAlreadyUsedHolderPath(eShared.message);
+                    if (holder) { _failBranchHeldByExternalWorktree(branchName, holder, eShared); return null; }
+                    throw eShared;
+                  }
                 }
               } else if (eShared.message?.includes('invalid reference') || eShared.message?.includes('not a valid ref')) {
                 // Branch doesn't exist yet (first item in plan) — create it from main
@@ -1235,7 +1404,7 @@ async function spawnAgent(dispatchItem, config) {
             if (_branchOnRemote) {
               // Mirror shared-branch fetch+add (~line 1157-1159).
               log('info', `origin/${branchName} exists — checking out remote branch instead of -b from ${mainRef}`);
-              try { await shared.shellSafeGit(['fetch', 'origin', branchName], { ..._gitOpts, cwd: rootDir, timeout: 30000 }); } catch (e) { log('warn', 'git: ' + e.message); }
+              await _fetchWithTransientRetry(['origin', branchName], { ..._gitOpts, cwd: rootDir, timeout: 30000 }, `origin/${branchName} (non-shared pre-create)`);
               try {
                 await runWorktreeAdd(rootDir, worktreePath, [branchName], _worktreeGitOpts, worktreeCreateRetries);
               } catch (eRemote) {
@@ -1267,7 +1436,14 @@ async function spawnAgent(dispatchItem, config) {
                     if (pruned > 0) {
                       log('info', `Pruned ${pruned} stale worktree entry(ies) for ${branchName}; retrying worktree add`);
                       await runWorktreeAdd(rootDir, worktreePath, [branchName], _worktreeGitOpts, 0);
-                    } else { throw eRemote; }
+                    } else {
+                      // Same recovery shape as the shared-branch path above:
+                      // prune was a no-op, so the holder is a real worktree.
+                      // Mark non-retryable with the holder path in the message.
+                      const holder = _parseAlreadyUsedHolderPath(eRemote.message);
+                      if (holder) { _failBranchHeldByExternalWorktree(branchName, holder, eRemote); return null; }
+                      throw eRemote;
+                    }
                   }
                 } else { throw eRemote; }
               }
@@ -1285,12 +1461,12 @@ async function spawnAgent(dispatchItem, config) {
               // the dep-merge phase's own fetch + the on-failure
               // `git reset --hard origin/<mainRef>` recovery remain as safety nets.
               let _freshCreateBase = mainRef;
-              try {
-                await shared.shellSafeGit(['fetch', 'origin', mainRef], { ..._gitOpts, cwd: rootDir, timeout: 30000 });
-                _freshCreateBase = `origin/${mainRef}`;
-              } catch (mainFetchErr) {
-                log('warn', `Failed to fetch origin/${mainRef} before fresh-create worktree for ${branchName}: ${mainFetchErr.message} — falling back to local ${mainRef}`);
-              }
+              const _baseFetchOk = await _fetchWithTransientRetry(
+                ['origin', mainRef],
+                { ..._gitOpts, cwd: rootDir, timeout: 30000 },
+                `origin/${mainRef} (fresh-create base for ${branchName})`
+              );
+              if (_baseFetchOk) _freshCreateBase = `origin/${mainRef}`;
               try {
                 await runWorktreeAdd(rootDir, worktreePath, ['-b', branchName, _freshCreateBase], _worktreeGitOpts, worktreeCreateRetries);
               } catch (e1) {
@@ -1311,7 +1487,7 @@ async function spawnAgent(dispatchItem, config) {
                   }
                 } else {
                   // Branch already exists — try checkout without -b
-                  try { await shared.shellSafeGit(['fetch', 'origin', branchName], { ..._gitOpts, cwd: rootDir }); } catch (e) { log('warn', 'git: ' + e.message); }
+                  await _fetchWithTransientRetry(['origin', branchName], { ..._gitOpts, cwd: rootDir }, `origin/${branchName} (fresh-create fallback)`);
                   try {
                     await runWorktreeAdd(rootDir, worktreePath, [branchName], _worktreeGitOpts, worktreeCreateRetries);
                     log('info', `Reusing existing branch: ${branchName}`);
@@ -5134,6 +5310,39 @@ function discoverFromWorkItems(config, project) {
     const hardPinnedAgent = routing.getHardPinnedAgent(item, config.agents || {});
     const hardPinRequested = !!hardPinnedAgent;
     let agentId = hardPinnedAgent || resolveAgent(workType, config, { agentHints });
+    // W-mpmwxn1j — Per-agent retry threshold. When the same agent has failed
+    // this WI maxRetriesPerAgent times, force-reassign to a different
+    // eligible agent. Hard-pinned agents bypass reassignment (operator
+    // intent wins). If no alternate is available we fall back to the same
+    // agent + write a dated inbox note so the operator can intervene.
+    if (agentId && !hardPinRequested) {
+      const maxPerAgent = shared.resolveMaxRetriesPerAgent(config);
+      const failsForAgent = shared.getAgentRetryCount(item, agentId);
+      if (failsForAgent >= maxPerAgent) {
+        const altAgent = resolveAgent(workType, config, { agentHints, excludeAgent: agentId });
+        if (altAgent && altAgent !== agentId) {
+          log('info', `Per-agent retry threshold reached: ${item.id} reassigning ${agentId} → ${altAgent} (${failsForAgent}/${maxPerAgent} failures by ${agentId})`);
+          agentId = altAgent;
+        } else {
+          // No alternate available — log + inbox note (writeToInbox dedupes
+          // per-day-per-slug, so re-runs in the same day stay quiet).
+          log('warn', `Per-agent retry threshold reached for ${item.id} (${agentId}) but no alternate agent available for work type "${workType}" — falling back to same agent`);
+          try {
+            shared.writeToInbox('engine', `per-agent-retry-no-alternate-${item.id}`,
+              `# Per-agent retry threshold — no alternate available\n\n` +
+              `Work item: \`${item.id}\` — ${item.title || ''}\n\n` +
+              `Agent **${agentId}** has failed this WI ${failsForAgent} times ` +
+              `(threshold: ${maxPerAgent}). The engine attempted to reassign to a ` +
+              `different eligible agent for work type **${workType}** but no alternate was found ` +
+              `(routing.md preferred/fallback both excluded, no idle named agent, ` +
+              `temp agents disabled or budget exhausted). Re-dispatching to **${agentId}** anyway ` +
+              `to avoid deadlock.\n\n` +
+              `Action: review routing.md, add another agent for this work type, or enable ` +
+              `\`allowTempAgents\` so the engine has a fallback target.\n`);
+          } catch (e) { log('warn', 'per-agent retry inbox write failed: ' + e.message); }
+        }
+      }
+    }
     let reservedAgentId = agentId;
     const cfgAgents = config.agents || {};
     const budgetBlocked = Object.keys(cfgAgents).some(id => {
@@ -6339,6 +6548,26 @@ async function tickInner() {
   // 2. Consolidate inbox
   safe('consolidateInbox', () => consolidateInbox(config));
 
+  // 2.1. Auto-consolidate memory — opt-in periodic KB sweep. Inbox→notes
+  // already runs above every tick (threshold-gated); this phase only adds
+  // the KB sweep that was previously dashboard-button-only. Gated by
+  // engine.autoConsolidateMemory; 4h cadence enforced inside shouldAutoSweep().
+  if (config.engine?.autoConsolidateMemory === true) {
+    safe('autoSweepKb', () => {
+      const { shouldAutoSweep, spawnSweepRunnerDetached } = require('./engine/kb-sweep');
+      const decision = shouldAutoSweep();
+      if (!decision.shouldSpawn) return;
+      const result = spawnSweepRunnerDetached({
+        log: (level, msg) => log(level === 'error' ? 'warn' : 'info', `auto-sweep: ${msg}`),
+      });
+      if (result.ok) {
+        log('info', `auto-sweep: spawned KB sweep (reason=${decision.reason}, pid=${result.pid})`);
+      } else {
+        log('warn', `auto-sweep: spawn failed: ${result.error}`);
+      }
+    });
+  }
+
   // 2.5. Periodic cleanup + MCP sync (every 10 ticks = ~5 minutes)
   if (tickCount % 10 === 0) {
     try { await runCleanup(config); } catch (e) { log('warn', `runCleanup: ${e.message}`); }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2044",
+  "version": "0.1.2046",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"