@yemi33/minions 0.1.576 → 0.1.578

package/CHANGELOG.md

@@ -1,8 +1,14 @@
 # Changelog
 
-## 0.1.576 (2026-04-08)
+## 0.1.578 (2026-04-08)
 
 ### Other
+- refactor: extract _spawnProcess helper, fix dep fetch error tracking, cleanup file leak
+
+## 0.1.577 (2026-04-08)
+
+### Other
+- perf: parallelize worktree/prompt, direct CLI spawn, parallel dep fetch
 - chore: remove orphaned comment, fix PID write test assertion
 
 ## 0.1.575 (2026-04-08)

package/dashboard.js

@@ -788,7 +788,7 @@ async function ccCall(message, { store = 'cc', sessionKey, extraContext, label =
   // Attempt 1: resume existing session — skip preamble (session already has context)
   if (sessionId && maxTurns > 1) {
     result = await llm.callLLM(buildPrompt({ includePreamble: false }), '', {
-      timeout, label, model, maxTurns, allowedTools, sessionId, effort: ccEffort,
+      timeout, label, model, maxTurns, allowedTools, sessionId, effort: ccEffort, direct: true,
     });
     llm.trackEngineUsage(label, result.usage);
 
@@ -823,7 +823,7 @@ async function ccCall(message, { store = 'cc', sessionKey, extraContext, label =
   // Attempt 2: fresh session (include preamble for full context)
   const freshPrompt = buildPrompt();
   result = await llm.callLLM(freshPrompt, CC_STATIC_SYSTEM_PROMPT, {
-    timeout, label, model, maxTurns, allowedTools, effort: ccEffort,
+    timeout, label, model, maxTurns, allowedTools, effort: ccEffort, direct: true,
   });
   llm.trackEngineUsage(label, result.usage);
 
@@ -837,7 +837,7 @@ async function ccCall(message, { store = 'cc', sessionKey, extraContext, label =
   console.log(`[${label}] Fresh call also failed (code=${result.code}, empty=${!result.text}), retrying once more...`);
   await new Promise(r => setTimeout(r, 2000));
   result = await llm.callLLM(freshPrompt, CC_STATIC_SYSTEM_PROMPT, {
-    timeout, label, model, maxTurns, allowedTools, effort: ccEffort,
+    timeout, label, model, maxTurns, allowedTools, effort: ccEffort, direct: true,
   });
   llm.trackEngineUsage(label, result.usage);
 
@@ -3350,7 +3350,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         const llmPromise = callLLMStreaming(prompt, CC_STATIC_SYSTEM_PROMPT, {
           timeout: 900000, label: 'command-center', model: streamModel, maxTurns: 50,
           allowedTools: 'Bash,Read,Write,Edit,Glob,Grep,WebFetch,WebSearch',
-          sessionId, effort: streamEffort,
+          sessionId, effort: streamEffort, direct: true,
           onChunk: (text) => {
             try { res.write('data: ' + JSON.stringify({ type: 'chunk', text }) + '\n\n'); } catch {}
           },

package/engine/llm.js

@@ -43,32 +43,87 @@ function trackEngineUsage(category, usage) {
   } catch (e) { console.error('metrics update:', e.message); }
 }
 
+// ── Claude Binary Resolution (cached by spawn-agent.js) ─────────────────────
+
+let _claudeBinCache = null;
+function _resolveClaudeBin() {
+  if (_claudeBinCache) return _claudeBinCache;
+  const caps = shared.safeJson(path.join(ENGINE_DIR, 'claude-caps.json'));
+  if (caps?.claudeBin && require('fs').existsSync(caps.claudeBin)) {
+    _claudeBinCache = { bin: caps.claudeBin, native: !!caps.claudeIsNative };
+    return _claudeBinCache;
+  }
+  return null;
+}
+
+// ── Spawn Helpers ───────────────────────────────────────────────────────────
+
+function _buildCliArgs({ model, maxTurns, allowedTools, effort, sessionId, sysPromptFile }) {
+  const args = ['-p', '--output-format', 'stream-json', '--max-turns', String(maxTurns), '--model', model, '--verbose'];
+  if (sysPromptFile) args.push('--system-prompt-file', sysPromptFile);
+  if (allowedTools) args.push('--allowedTools', allowedTools);
+  if (effort) args.push('--effort', effort);
+  args.push('--permission-mode', 'bypassPermissions');
+  if (sessionId) args.push('--resume', sessionId);
+  return args;
+}
+
+/**
+ * Spawn a claude CLI process. Returns { proc, cleanupFiles } or null if binary not cached.
+ * When direct=true, spawns claude CLI directly (fewer syscalls). Otherwise uses spawn-agent.js.
+ */
+function _spawnProcess(promptText, sysPromptText, { direct, label, model, maxTurns, allowedTools, effort, sessionId }) {
+  const fs = require('fs');
+  const id = uid();
+  const tmpDir = path.join(ENGINE_DIR, 'tmp');
+  if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
+
+  const cleanupFiles = [];
+  const resolved = direct ? _resolveClaudeBin() : null;
+
+  if (resolved) {
+    let sysTmpPath = null;
+    if (!sessionId && sysPromptText) {
+      sysTmpPath = path.join(tmpDir, `direct-sys-${id}.md`);
+      fs.writeFileSync(sysTmpPath, sysPromptText);
+      cleanupFiles.push(sysTmpPath);
+    }
+    const cliArgs = _buildCliArgs({ model, maxTurns, allowedTools, effort, sessionId, sysPromptFile: sysTmpPath });
+    const proc = resolved.native
+      ? runFile(resolved.bin, cliArgs, { cwd: MINIONS_DIR, stdio: ['pipe', 'pipe', 'pipe'], env: cleanChildEnv() })
+      : runFile(process.execPath, [resolved.bin, ...cliArgs], { cwd: MINIONS_DIR, stdio: ['pipe', 'pipe', 'pipe'], env: cleanChildEnv() });
+    try { proc.stdin.write(promptText); proc.stdin.end(); } catch { /* broken pipe */ }
+    return { proc, cleanupFiles };
+  }
+
+  // Indirect: use spawn-agent.js
+  const promptPath = path.join(tmpDir, `${label}-prompt-${id}.md`);
+  const sysPath = path.join(tmpDir, `${label}-sys-${id}.md`);
+  safeWrite(promptPath, promptText);
+  safeWrite(sysPath, sysPromptText || '');
+  cleanupFiles.push(promptPath, sysPath);
+
+  const spawnScript = path.join(ENGINE_DIR, 'spawn-agent.js');
+  const args = [
+    spawnScript, promptPath, sysPath,
+    '--output-format', 'stream-json', '--max-turns', String(maxTurns), '--model', model,
+    '--verbose',
+  ];
+  if (allowedTools) args.push('--allowedTools', allowedTools);
+  if (effort) args.push('--effort', effort);
+  args.push('--permission-mode', 'bypassPermissions');
+  if (sessionId) args.push('--resume', sessionId);
+
+  const proc = runFile(process.execPath, args, { cwd: MINIONS_DIR, stdio: ['pipe', 'pipe', 'pipe'], env: cleanChildEnv() });
+  return { proc, cleanupFiles };
+}
+
 // ── Core LLM Call ───────────────────────────────────────────────────────────
 
-function callLLM(promptText, sysPromptText, { timeout = 120000, label = 'llm', model = 'sonnet', maxTurns = 1, allowedTools = '', sessionId = null, effort = null } = {}) {
+function callLLM(promptText, sysPromptText, { timeout = 120000, label = 'llm', model = 'sonnet', maxTurns = 1, allowedTools = '', sessionId = null, effort = null, direct = false } = {}) {
   return new Promise((resolve) => {
-    const id = uid();
-    const tmpDir = path.join(ENGINE_DIR, 'tmp');
-    if (!require('fs').existsSync(tmpDir)) require('fs').mkdirSync(tmpDir, { recursive: true });
-    const promptPath = path.join(tmpDir, `${label}-prompt-${id}.md`);
-    const sysPath = path.join(tmpDir, `${label}-sys-${id}.md`);
-    safeWrite(promptPath, promptText);
-    safeWrite(sysPath, sysPromptText || '');
-
-    const spawnScript = path.join(ENGINE_DIR, 'spawn-agent.js');
-    const args = [
-      spawnScript, promptPath, sysPath,
-      '--output-format', 'stream-json', '--max-turns', String(maxTurns), '--model', model,
-      '--verbose',
-    ];
-    if (allowedTools) args.push('--allowedTools', allowedTools);
-    if (effort) args.push('--effort', effort);
-    args.push('--permission-mode', 'bypassPermissions');
-
-    if (sessionId) args.push('--resume', sessionId);
-
     const _startMs = Date.now();
-    const proc = runFile(process.execPath, args, { cwd: MINIONS_DIR, stdio: ['pipe', 'pipe', 'pipe'], env: cleanChildEnv() });
+    const { proc, cleanupFiles } = _spawnProcess(promptText, sysPromptText, { direct, label, model, maxTurns, allowedTools, effort, sessionId });
 
     let stdout = '';
     let stderr = '';
@@ -79,8 +134,7 @@ function callLLM(promptText, sysPromptText, { timeout = 120000, label = 'llm', m
 
     proc.on('close', (code) => {
       clearTimeout(timer);
-      safeUnlink(promptPath);
-      safeUnlink(sysPath);
+      for (const f of cleanupFiles) safeUnlink(f);
       const parsed = parseStreamJsonOutput(stdout);
       const durationMs = Date.now() - _startMs;
       const usage = parsed.usage ? { ...parsed.usage, durationMs } : { durationMs };
@@ -89,8 +143,7 @@ function callLLM(promptText, sysPromptText, { timeout = 120000, label = 'llm', m
 
     proc.on('error', (err) => {
       clearTimeout(timer);
-      safeUnlink(promptPath);
-      safeUnlink(sysPath);
+      for (const f of cleanupFiles) safeUnlink(f);
       resolve({ text: '', usage: null, sessionId: null, code: 1, stderr: err.message, raw: '' });
     });
   });
@@ -118,30 +171,11 @@ function isResumeSessionStillValid(result) {
  * Returns the same result object as callLLM when the process completes.
  * onChunk(text) is called for each assistant text block as it arrives.
  */
-function callLLMStreaming(promptText, sysPromptText, { timeout = 120000, label = 'llm', model = 'sonnet', maxTurns = 1, allowedTools = '', sessionId = null, onChunk = () => {}, onToolUse = null, effort = null } = {}) {
+function callLLMStreaming(promptText, sysPromptText, { timeout = 120000, label = 'llm', model = 'sonnet', maxTurns = 1, allowedTools = '', sessionId = null, onChunk = () => {}, onToolUse = null, effort = null, direct = false } = {}) {
   let _abort = null;
   const promise = new Promise((resolve) => {
-    const id = uid();
-    const tmpDir = path.join(ENGINE_DIR, 'tmp');
-    if (!require('fs').existsSync(tmpDir)) require('fs').mkdirSync(tmpDir, { recursive: true });
-    const promptPath = path.join(tmpDir, `${label}-prompt-${id}.md`);
-    const sysPath = path.join(tmpDir, `${label}-sys-${id}.md`);
-    safeWrite(promptPath, promptText);
-    safeWrite(sysPath, sysPromptText || '');
-
-    const spawnScript = path.join(ENGINE_DIR, 'spawn-agent.js');
-    const args = [
-      spawnScript, promptPath, sysPath,
-      '--output-format', 'stream-json', '--max-turns', String(maxTurns), '--model', model,
-      '--verbose',
-    ];
-    if (allowedTools) args.push('--allowedTools', allowedTools);
-    if (effort) args.push('--effort', effort);
-    args.push('--permission-mode', 'bypassPermissions');
-    if (sessionId) args.push('--resume', sessionId);
-
     const _startMs = Date.now();
-    const proc = runFile(process.execPath, args, { cwd: MINIONS_DIR, stdio: ['pipe', 'pipe', 'pipe'], env: cleanChildEnv() });
+    const { proc, cleanupFiles } = _spawnProcess(promptText, sysPromptText, { direct, label, model, maxTurns, allowedTools, effort, sessionId });
 
     _abort = () => { shared.killImmediate(proc); };
 
@@ -181,8 +215,7 @@ function callLLMStreaming(promptText, sysPromptText, { timeout = 120000, label =
 
     proc.on('close', (code) => {
       clearTimeout(timer);
-      safeUnlink(promptPath);
-      safeUnlink(sysPath);
+      for (const f of cleanupFiles) safeUnlink(f);
       const parsed = parseStreamJsonOutput(stdout);
       const durationMs = Date.now() - _startMs;
       const usage = parsed.usage ? { ...parsed.usage, durationMs } : { durationMs };
@@ -191,8 +224,7 @@ function callLLMStreaming(promptText, sysPromptText, { timeout = 120000, label =
 
     proc.on('error', (err) => {
       clearTimeout(timer);
-      safeUnlink(promptPath);
-      safeUnlink(sysPath);
+      for (const f of cleanupFiles) safeUnlink(f);
       resolve({ text: '', usage: null, sessionId: null, code: 1, stderr: err.message, raw: '' });
     });
   });

package/engine.js

@@ -278,6 +278,22 @@ async function spawnAgent(dispatchItem, config) {
   const _gitOpts = { stdio: 'pipe', timeout: 30000, windowsHide: true, env: shared.gitEnv() };
   const _worktreeGitOpts = { ..._gitOpts, timeout: worktreeCreateTimeout };
 
+  // Build prompt before worktree setup — prompt doesn't depend on worktree path
+  // and this avoids blocking 200ms of file reads behind 20-60s of git operations
+  const systemPrompt = buildSystemPrompt(agentId, config, project);
+  const agentContext = buildAgentContext(agentId, config, project);
+  const fullTaskPrompt = agentContext
+    ? `## Agent Context\n\n${agentContext}\n---\n\n## Your Task\n\n${taskPrompt}`
+    : taskPrompt;
+  const tmpDir = path.join(ENGINE_DIR, 'tmp');
+  if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
+  const safeId = id.replace(/[:\\/*?"<>|]/g, '-');
+  const promptPath = path.join(tmpDir, `prompt-${safeId}.md`);
+  safeWrite(promptPath, fullTaskPrompt);
+  const sysPromptPath = path.join(tmpDir, `sysprompt-${safeId}.md`);
+  safeWrite(sysPromptPath, systemPrompt);
+  const _cleanupPromptFiles = () => { safeUnlink(promptPath); safeUnlink(sysPromptPath); };
+
   if (branchName) {
     const wtSuffix = id ? id.split('-').pop() : shared.uid();
     const projectSlug = (project.name || 'default').replace(/[^a-zA-Z0-9_-]/g, '-');
@@ -413,24 +429,41 @@ async function spawnAgent(dispatchItem, config) {
         try {
           const depBranches = resolveDependencyBranches(depIds, meta?.item?.sourcePlan, project, config);
           let depMergeFailed = false;
-          for (const { branch: depBranch, prId } of depBranches) {
-            // Skip refs already known to be missing this tick (avoids repeated 30s ETIMEDOUT)
-            if (_failedRefCache.has(depBranch)) {
-              log('warn', `Skipping dependency ${depBranch} — already failed to fetch this tick`);
+          // Fetch all dependency branches in parallel (git fetches are independent)
+          const fetchable = depBranches.filter(d => !_failedRefCache.has(d.branch));
+          const unfetchable = depBranches.filter(d => _failedRefCache.has(d.branch));
+          for (const { branch: depBranch } of unfetchable) {
+            log('warn', `Skipping dependency ${depBranch} — already failed to fetch this tick`);
+            depMergeFailed = true;
+          }
+          const fetchResults = await Promise.allSettled(
+            fetchable.map(({ branch: depBranch }) =>
+              execAsync(`git fetch origin "${depBranch}"`, { ..._gitOpts, cwd: rootDir }).then(() => depBranch)
+            )
+          );
+          for (let i = 0; i < fetchResults.length; i++) {
+            if (fetchResults[i].status === 'rejected') {
+              const failedBranch = fetchable[i].branch;
+              _failedRefCache.add(failedBranch);
+              log('warn', `Failed to fetch dependency ${failedBranch}: ${fetchResults[i].reason?.message}`);
               depMergeFailed = true;
-              continue;
             }
-            try {
-              await execAsync(`git fetch origin "${depBranch}"`, { ..._gitOpts, cwd: rootDir });
-              await execAsync(`git merge "origin/${depBranch}" --no-edit`, { ..._gitOpts, cwd: worktreePath });
-              log('info', `Merged dependency branch ${depBranch} (${prId}) into worktree ${branchName}`);
-            } catch (mergeErr) {
-              _failedRefCache.add(depBranch);
-              log('warn', `Failed to merge dependency ${depBranch} into ${branchName}: ${mergeErr.message}`);
-              depMergeFailed = true;
+          }
+          // Merge fetched branches sequentially (merges modify the worktree)
+          if (!depMergeFailed) {
+            for (const { branch: depBranch, prId } of fetchable) {
+              try {
+                await execAsync(`git merge "origin/${depBranch}" --no-edit`, { ..._gitOpts, cwd: worktreePath });
+                log('info', `Merged dependency branch ${depBranch} (${prId}) into worktree ${branchName}`);
+              } catch (mergeErr) {
+                log('warn', `Failed to merge dependency ${depBranch} into ${branchName}: ${mergeErr.message}`);
+                depMergeFailed = true;
+                break;
+              }
             }
           }
           if (depMergeFailed) {
+            _cleanupPromptFiles();
             completeDispatch(id, DISPATCH_RESULT.ERROR, `Dependency merge failed — will retry next tick`);
             return;
           }
@@ -441,30 +474,11 @@ async function spawnAgent(dispatchItem, config) {
     }
   }
 
-  // Build lean system prompt (identity + rules, ~2-4KB) and bulk context (history, notes, skills)
-  const systemPrompt = buildSystemPrompt(agentId, config, project);
-  const agentContext = buildAgentContext(agentId, config, project);
-
   // Safety check: warn if a write-capable task is running in the main repo without a worktree
   if (cwd === rootDir && ['implement', 'implement:large', 'fix', 'test', 'verify', 'plan-to-prd'].includes(type)) {
     log('warn', `Agent ${agentId} running ${type} task in main repo (no worktree) for ${id} — changes may land on master directly`);
   }
 
-  // Prepend bulk context to task prompt — keeps system prompt small and stable
-  const fullTaskPrompt = agentContext
-    ? `## Agent Context\n\n${agentContext}\n---\n\n## Your Task\n\n${taskPrompt}`
-    : taskPrompt;
-
-  // Write prompt and system prompt to temp files (avoids shell escaping issues)
-  const tmpDir = path.join(ENGINE_DIR, 'tmp');
-  if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true });
-  const safeId = id.replace(/[:\\/*?"<>|]/g, '-');
-  const promptPath = path.join(tmpDir, `prompt-${safeId}.md`);
-  safeWrite(promptPath, fullTaskPrompt);
-
-  const sysPromptPath = path.join(tmpDir, `sysprompt-${safeId}.md`);
-  safeWrite(sysPromptPath, systemPrompt);
-
   // Build claude CLI args
   const args = [
     '--output-format', claudeConfig.outputFormat || 'stream-json',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.576",
+  "version": "0.1.578",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"