@yemi33/minions 0.1.1736 → 0.1.1737

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.1.1737 (2026-05-05)
+
+### Other
+- perf: buffered metrics, snapshot-diff sidecar, parallel worktree branch probes
+
 ## 0.1.1736 (2026-05-05)
 
 ### Other

package/engine/cleanup.js

@@ -8,7 +8,7 @@ const path = require('path');
 const shared = require('./shared');
 const queries = require('./queries');
 
-const { exec, execSilent, log, ts, ENGINE_DEFAULTS } = shared;
+const { exec, execAsync, execSilent, log, ts, ENGINE_DEFAULTS } = shared;
 const { safeJson, safeWrite, safeReadDir, mutateCooldowns, mutateWorkItems, mutateJsonFileLocked, getProjects, projectWorkItemsPath, projectPrPath,
   sanitizeBranch, KB_CATEGORIES } = shared;
 const { getDispatch, getAgentStatus } = queries;
@@ -56,6 +56,15 @@ function getWorktreeBranch(wtPath) {
   }
 }
 
+async function getWorktreeBranchAsync(wtPath) {
+  try {
+    const out = await execAsync(`git -C "${wtPath}" branch --show-current`, { encoding: 'utf8', timeout: 5000 });
+    return (out || '').toString().trim();
+  } catch {
+    return '';
+  }
+}
+
 let _orphanPidProcessNamesCache = null;
 function _orphanPidProcessNames() {
   if (_orphanPidProcessNamesCache) return _orphanPidProcessNamesCache;
@@ -134,7 +143,7 @@ function _killProcessInWorktree(dir, activeProcesses, activeIds) {
 
 // ─── Cleanup Orchestrator ────────────────────────────────────────────────────
 
-function runCleanup(config, verbose = false) {
+async function runCleanup(config, verbose = false) {
   const activeProcesses = engine().activeProcesses;
   const projects = getProjects(config);
   let cleaned = { tempFiles: 0, liveOutputs: 0, worktrees: 0, zombies: 0 };
@@ -248,11 +257,26 @@ function runCleanup(config, verbose = false) {
       const dispatch = getDispatch();
       const activeDispatchIds = new Set((dispatch.active || []).map(d => d.id));
 
+      // Probe `git branch --show-current` for every worktree in chunks of 5.
+      // Sequential probing was the dominant cost in the cleanup phase
+      // (5–15s tick stall every 10 ticks at 50+ worktrees), but unbounded
+      // Promise.all would spawn 50+ concurrent git children — bad on Windows
+      // where each fork pays AV-scan overhead. Mirrors engine/ado.js:611.
+      const BRANCH_PROBE_CONCURRENCY = 5;
+      const branchMap = new Map();
+      for (let i = 0; i < allDirs.length; i += BRANCH_PROBE_CONCURRENCY) {
+        const batch = allDirs.slice(i, i + BRANCH_PROBE_CONCURRENCY);
+        const pairs = await Promise.all(
+          batch.map(async ({ wtPath }) => [wtPath, await getWorktreeBranchAsync(wtPath)])
+        );
+        for (const [wtPath, branch] of pairs) branchMap.set(wtPath, branch);
+      }
+
       for (const { dir, wtPath } of allDirs) {
 
         let shouldClean = false;
         let isProtected = false;
-        const actualBranch = getWorktreeBranch(wtPath);
+        const actualBranch = branchMap.get(wtPath) || '';
 
         // Check if this worktree's branch is merged/abandoned
         // Prefer actual git branch metadata; compact Windows dirs intentionally omit branch names.

package/engine/cli.js

@@ -745,6 +745,16 @@ const commands = {
     }
     watchForWorkChanges();
 
+    // Drain in-memory buffers (metrics, logs) before any process.exit. Logs
+    // a stderr line if a flush fails so a refactor mistake (typo, missing
+    // export) doesn't go silent.
+    function drainBuffers() {
+      try { require('./llm').flushMetricsBuffer(); }
+      catch (err) { try { console.error(`[shutdown] flushMetricsBuffer failed: ${err.message}`); } catch {} }
+      try { shared.flushLogs(); }
+      catch (err) { try { console.error(`[shutdown] flushLogs failed: ${err.message}`); } catch {} }
+    }
+
     // Graceful shutdown — wait for active agents before exiting
     let shuttingDown = false;
     function gracefulShutdown(signal) {
@@ -768,7 +778,7 @@ const commands = {
           e.log('warn', 'Graceful shutdown skipped control.json stopped transition; control file is owned by a different engine process');
         }
         e.log('info', 'Graceful shutdown complete (no active agents)');
-        shared.flushLogs(); // drain buffered log entries before exit
+        drainBuffers();
         console.log('No active agents — stopped.');
         process.exit(0);
       }
@@ -785,7 +795,7 @@ const commands = {
             e.log('warn', 'Graceful shutdown skipped control.json stopped transition; control file is owned by a different engine process');
           }
           e.log('info', 'Graceful shutdown complete (all agents finished)');
-          shared.flushLogs(); // drain buffered log entries before exit
+          drainBuffers();
           console.log('All agents finished — stopped.');
           process.exit(0);
         }
@@ -796,7 +806,7 @@ const commands = {
             e.log('warn', 'Graceful shutdown skipped control.json stopped transition; control file is owned by a different engine process');
           }
           e.log('warn', `Graceful shutdown timed out after ${timeout / 1000}s with ${e.activeProcesses.size} agent(s) still active`);
-          shared.flushLogs(); // drain buffered log entries before exit
+          drainBuffers();
           console.log(`Shutdown timeout (${timeout / 1000}s) — force exiting with ${e.activeProcesses.size} agent(s) still running.`);
           process.exit(1);
         }
@@ -811,7 +821,7 @@ const commands = {
       const msg = reason instanceof Error ? reason.stack || reason.message : String(reason);
       console.error(`[FATAL] Unhandled promise rejection: ${msg}`);
       try { shared.log('fatal', `Unhandled promise rejection: ${msg}`); } catch { /* best effort */ }
-      try { shared.flushLogs(); } catch { /* best effort */ }
+      drainBuffers();
       process.exit(1);
     });
 
@@ -819,7 +829,7 @@ const commands = {
       const msg = err instanceof Error ? err.stack || err.message : String(err);
       console.error(`[FATAL] Uncaught exception: ${msg}`);
       try { shared.log('fatal', `Uncaught exception: ${msg}`); } catch { /* best effort */ }
-      try { shared.flushLogs(); } catch { /* best effort */ }
+      drainBuffers();
       process.exit(1);
     });
   },
@@ -1359,11 +1369,11 @@ const commands = {
     console.log(`\nDone: ${killed.length} dispatches killed, agents reset.`);
   },
 
-  cleanup() {
+  async cleanup() {
     const e = engine();
     const config = getConfig();
     console.log('\n=== Cleanup ===\n');
-    const result = e.runCleanup(config, true);
+    const result = await e.runCleanup(config, true);
     console.log(`\nDone: ${result.tempFiles} temp files, ${result.liveOutputs} live outputs, ${result.worktrees} worktrees, ${result.zombies} zombies cleaned.`);
   },
 

package/engine/copilot-models.json

@@ -1,5 +1,5 @@
 {
   "runtime": "copilot",
   "models": null,
-  "cachedAt": "2026-05-05T21:38:46.577Z"
+  "cachedAt": "2026-05-05T22:02:28.540Z"
 }

package/engine/dispatch.js

@@ -24,18 +24,35 @@ function lifecycle() { if (!_lifecycle) _lifecycle = require('./lifecycle'); ret
 // ─── Dispatch Mutation ───────────────────────────────────────────────────────
 
 /**
- * Sweep pending + active dispatch entries and move any oversized prompts to
- * sidecar files. Keeps dispatch.json from bloating to hundreds of MB when
- * fix-type prompts inline PR diffs / build logs / coalesced feedback (#1167).
- * Safe to call on every mutation: small prompts are untouched.
+ * Walk pending + active dispatch entries and snapshot {id → prompt} for items
+ * that have a string prompt. Used to diff before/after a mutation so we only
+ * re-check items the mutator actually touched (#1167).
  */
-function _sidecarOversizedPrompts(dispatch) {
+function _snapshotPrompts(dispatch) {
+  const snap = new Map();
+  for (const list of [dispatch.pending, dispatch.active]) {
+    if (!Array.isArray(list)) continue;
+    for (const item of list) {
+      if (item && item.id && typeof item.prompt === 'string') snap.set(item.id, item.prompt);
+    }
+  }
+  return snap;
+}
+
+/**
+ * Sidecar oversized prompts only for items the mutator added or modified.
+ * Keeps dispatch.json safe from bloat (#1167) without paying O(n) on every
+ * unrelated mutation (e.g. status flips, completion-marking).
+ */
+function _sidecarChangedPrompts(dispatch, prevSnap) {
   const threshold = ENGINE_DEFAULTS.maxDispatchPromptBytes;
-  const lists = [dispatch.pending, dispatch.active];
-  for (const list of lists) {
+  for (const list of [dispatch.pending, dispatch.active]) {
     if (!Array.isArray(list)) continue;
     for (const item of list) {
-      if (item && typeof item.prompt === 'string') sidecarDispatchPrompt(item, threshold);
+      if (!item || typeof item.prompt !== 'string') continue;
+      const prev = item.id ? prevSnap.get(item.id) : undefined;
+      if (prev === item.prompt) continue; // untouched — already validated on insert
+      sidecarDispatchPrompt(item, threshold);
     }
   }
 }
@@ -46,10 +63,11 @@ function mutateDispatch(mutator) {
     dispatch.pending = Array.isArray(dispatch.pending) ? dispatch.pending : [];
     dispatch.active = Array.isArray(dispatch.active) ? dispatch.active : [];
     dispatch.completed = Array.isArray(dispatch.completed) ? dispatch.completed : [];
+    const prevSnap = _snapshotPrompts(dispatch);
     const next = mutator(dispatch) ?? dispatch;
-    // Prompt-size guard: runs on every write so a single bad item cannot bloat
-    // dispatch.json. Sidecars live in engine/contexts/<id>.md.
-    _sidecarOversizedPrompts(next);
+    // Prompt-size guard: only scan items whose prompt changed (or new items),
+    // so a 100-item status-flip doesn't re-byte-count every prompt.
+    _sidecarChangedPrompts(next, prevSnap);
     return next;
   }, { defaultValue: defaultDispatch });
   // Invalidate the read cache so next getDispatch() sees fresh data

package/engine/llm.js

@@ -26,41 +26,119 @@ const COPILOT_TASK_COMPLETE_GRACE_MS = 3000;
 const MISSING_RUNTIME_EXIT_CODE = 78;
 
 // ─── Engine-Usage Metrics ────────────────────────────────────────────────────
+//
+// Updates accumulate in an in-memory buffer and flush every
+// metricsFlushIntervalMs (default 10s). Replaces the per-call mutateJsonFileLocked
+// that was both serializing the LLM hot path and bumping metrics.json mtime
+// on every call — defeating the dashboard fast-state mtime-based early-exit.
+
+let _pendingMetrics = { engine: Object.create(null), daily: Object.create(null) };
+let _flushTimer = null;
+
+function _emptyEngineDelta() {
+  return { calls: 0, costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, cacheCreation: 0, totalDurationMs: 0, timedCalls: 0 };
+}
+
+function _emptyDailyDelta() {
+  return { costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0 };
+}
+
+function _ensureFlushTimer() {
+  if (_flushTimer) return;
+  const interval = shared.ENGINE_DEFAULTS.metricsFlushIntervalMs || 10000;
+  _flushTimer = setInterval(flushMetricsBuffer, interval);
+  if (typeof _flushTimer.unref === 'function') _flushTimer.unref();
+}
 
 function trackEngineUsage(category, usage) {
   if (!usage) return;
   if (category && (category.startsWith('_test') || category.startsWith('test-'))) return;
+
+  if (!_pendingMetrics.engine[category]) _pendingMetrics.engine[category] = _emptyEngineDelta();
+  const cat = _pendingMetrics.engine[category];
+  cat.calls++;
+  cat.costUsd += usage.costUsd || 0;
+  cat.inputTokens += usage.inputTokens || 0;
+  cat.outputTokens += usage.outputTokens || 0;
+  cat.cacheRead += usage.cacheRead || 0;
+  cat.cacheCreation += usage.cacheCreation || 0;
+  if (usage.durationMs) {
+    cat.totalDurationMs += usage.durationMs;
+    cat.timedCalls += 1;
+  }
+
+  const today = ts().slice(0, 10);
+  if (!_pendingMetrics.daily[today]) _pendingMetrics.daily[today] = _emptyDailyDelta();
+  const daily = _pendingMetrics.daily[today];
+  daily.costUsd += usage.costUsd || 0;
+  daily.inputTokens += usage.inputTokens || 0;
+  daily.outputTokens += usage.outputTokens || 0;
+  daily.cacheRead += usage.cacheRead || 0;
+
+  _ensureFlushTimer();
+}
+
+function flushMetricsBuffer() {
+  const pending = _pendingMetrics;
+  if (!Object.keys(pending.engine).length && !Object.keys(pending.daily).length) return;
+  _pendingMetrics = { engine: Object.create(null), daily: Object.create(null) };
+
   try {
     const metricsPath = path.join(ENGINE_DIR, 'metrics.json');
     mutateJsonFileLocked(metricsPath, (metrics) => {
       if (!metrics._engine) metrics._engine = {};
-      if (!metrics._engine[category]) {
-        metrics._engine[category] = { calls: 0, costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, cacheCreation: 0 };
-      }
-      const cat = metrics._engine[category];
-      cat.calls++;
-      cat.costUsd += usage.costUsd || 0;
-      cat.inputTokens += usage.inputTokens || 0;
-      cat.outputTokens += usage.outputTokens || 0;
-      cat.cacheRead += usage.cacheRead || 0;
-      cat.cacheCreation = (cat.cacheCreation || 0) + (usage.cacheCreation || 0);
-      if (usage.durationMs) {
-        cat.totalDurationMs = (cat.totalDurationMs || 0) + usage.durationMs;
-        cat.timedCalls = (cat.timedCalls || 0) + 1;
+      for (const [category, delta] of Object.entries(pending.engine)) {
+        if (!metrics._engine[category]) {
+          metrics._engine[category] = { calls: 0, costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, cacheCreation: 0 };
+        }
+        const cat = metrics._engine[category];
+        cat.calls = (cat.calls || 0) + delta.calls;
+        cat.costUsd = (cat.costUsd || 0) + delta.costUsd;
+        cat.inputTokens = (cat.inputTokens || 0) + delta.inputTokens;
+        cat.outputTokens = (cat.outputTokens || 0) + delta.outputTokens;
+        cat.cacheRead = (cat.cacheRead || 0) + delta.cacheRead;
+        cat.cacheCreation = (cat.cacheCreation || 0) + delta.cacheCreation;
+        if (delta.timedCalls > 0) {
+          cat.totalDurationMs = (cat.totalDurationMs || 0) + delta.totalDurationMs;
+          cat.timedCalls = (cat.timedCalls || 0) + delta.timedCalls;
+        }
       }
-
-      const today = ts().slice(0, 10);
       if (!metrics._daily) metrics._daily = {};
-      if (!metrics._daily[today]) metrics._daily[today] = { costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, tasks: 0 };
-      const daily = metrics._daily[today];
-      daily.costUsd += usage.costUsd || 0;
-      daily.inputTokens += usage.inputTokens || 0;
-      daily.outputTokens += usage.outputTokens || 0;
-      daily.cacheRead += usage.cacheRead || 0;
-
+      for (const [day, delta] of Object.entries(pending.daily)) {
+        if (!metrics._daily[day]) {
+          metrics._daily[day] = { costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, tasks: 0 };
+        }
+        const d = metrics._daily[day];
+        d.costUsd += delta.costUsd;
+        d.inputTokens += delta.inputTokens;
+        d.outputTokens += delta.outputTokens;
+        d.cacheRead += delta.cacheRead;
+      }
       return metrics;
     });
-  } catch (e) { console.error('metrics update:', e.message); }
+  } catch (e) {
+    // Re-merge pending so the next flush retries — never drop counters silently.
+    for (const [category, delta] of Object.entries(pending.engine)) {
+      if (!_pendingMetrics.engine[category]) _pendingMetrics.engine[category] = _emptyEngineDelta();
+      const c = _pendingMetrics.engine[category];
+      c.calls += delta.calls; c.costUsd += delta.costUsd;
+      c.inputTokens += delta.inputTokens; c.outputTokens += delta.outputTokens;
+      c.cacheRead += delta.cacheRead; c.cacheCreation += delta.cacheCreation;
+      c.totalDurationMs += delta.totalDurationMs; c.timedCalls += delta.timedCalls;
+    }
+    for (const [day, delta] of Object.entries(pending.daily)) {
+      if (!_pendingMetrics.daily[day]) _pendingMetrics.daily[day] = _emptyDailyDelta();
+      const d = _pendingMetrics.daily[day];
+      d.costUsd += delta.costUsd; d.inputTokens += delta.inputTokens;
+      d.outputTokens += delta.outputTokens; d.cacheRead += delta.cacheRead;
+    }
+    console.error('metrics flush:', e.message);
+  }
+}
+
+function _resetMetricsBufferForTest() {
+  _pendingMetrics = { engine: Object.create(null), daily: Object.create(null) };
+  if (_flushTimer) { clearInterval(_flushTimer); _flushTimer = null; }
 }
 
 // ─── Runtime Binary Resolution (TTL-cached) ──────────────────────────────────
@@ -83,7 +161,10 @@ function _resolveBin(runtime) {
   if (!runtime) return null;
   const key = runtime.name;
   const cached = _binCache.get(key);
-  if (cached && Date.now() - cached.ts < _BIN_TTL && fs.existsSync(cached.bin)) {
+  // Trust the 30-min TTL — skip per-call existsSync (10-50ms on Windows w/ AV).
+  // If the binary disappears mid-window, spawn fails with ENOENT and the
+  // adapter's resolveBinary() reprobes on the next cache miss.
+  if (cached && Date.now() - cached.ts < _BIN_TTL) {
     return { bin: cached.bin, native: cached.native, leadingArgs: cached.leadingArgs };
   }
   let resolved = null;
@@ -700,10 +781,12 @@ module.exports = {
   callLLM,
   callLLMStreaming,
   trackEngineUsage,
+  flushMetricsBuffer,
   // Exposed for unit tests — engine code MUST use the runtime adapter contract.
   _buildSpawnAgentFlags,
   _resolveBin,
   _resetBinCache,
+  _resetMetricsBufferForTest,
   _resolveRuntimeFor,
   _resolveModelFor,
   _resolveModelForRuntime,

package/engine/shared.js

@@ -918,6 +918,7 @@ const ENGINE_DEFAULTS = {
   docSessionTtlMs: 7 * 24 * 60 * 60 * 1000, // 7d — longer-lived doc sessions, still bounded
   docSessionMaxEntries: 200, // cap doc-chat session map/disk store by least-recent activity
   ccLiveStreamMaxAgeMs: 30 * 60 * 1000, // hard cap reconnect buffers if abort/cleanup stalls
+  metricsFlushIntervalMs: 10000, // batch trackEngineUsage writes to metrics.json — flushed every 10s instead of per-call to cut lock contention and dashboard mtime churn
   maxLlmRawBytes: 256 * 1024, // keep only a bounded stdout tail from direct Claude calls
   maxLlmStderrBytes: 64 * 1024, // keep only a bounded stderr tail from direct Claude calls
   maxLlmLineBufferBytes: 128 * 1024, // cap the incremental JSON line buffer to avoid malformed-stream OOMs

package/engine.js

@@ -4071,7 +4071,7 @@ async function tickInner() {
 
   // 2.5. Periodic cleanup + MCP sync (every 10 ticks = ~5 minutes)
   if (tickCount % 10 === 0) {
-    safe('runCleanup', () => runCleanup(config));
+    try { await runCleanup(config); } catch (e) { log('warn', `runCleanup: ${e.message}`); }
   }
 
   // 2.55. Check persistent watches (3 tick-equivalents, default ~3 minutes)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.1736",
+  "version": "0.1.1737",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"