@yemi33/minions 0.1.2213 → 0.1.2215

package/bin/minions.js

@@ -58,6 +58,7 @@
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
+const net = require('net');
 const { spawn, spawnSync, execSync } = require('child_process');
 
 const PKG_ROOT = path.resolve(__dirname, '..');
@@ -136,6 +137,24 @@ function killByPort(port) {
 
 const isPortListening = (port) => getListeningPids(port).length > 0;
 
+/** Authoritative "is something accepting TCP connections on this port?" probe.
+ *  Resolves true iff a connection to 127.0.0.1:port completes within timeoutMs.
+ *  Unlike the netstat-based isPortListening, this can't false-negative under
+ *  CPU/event-loop load: the TCP handshake is satisfied by the OS listen backlog
+ *  even when the listener's JS event loop is momentarily blocked. Used by the
+ *  watchdog to overturn a transient `down` verdict before restarting a live
+ *  dashboard. Never throws; resolves false on any error/timeout. */
+function tcpPortAccepts(port, timeoutMs = 1500) {
+  return new Promise((resolve) => {
+    let settled = false;
+    const done = (val) => { if (!settled) { settled = true; try { socket.destroy(); } catch {} resolve(val); } };
+    const socket = net.connect({ host: '127.0.0.1', port: Number(port) });
+    socket.once('connect', () => done(true));
+    socket.once('error', () => done(false));
+    socket.setTimeout(timeoutMs, () => done(false));
+  });
+}
+
 /**
  * Wait until no process is listening on `port`, retrying a kill on each tick
  * for any stragglers that re-appeared (e.g. orphan child the original kill
@@ -1483,6 +1502,7 @@ ${fs.existsSync(path.join(PKG_ROOT, '.git')) ? `
       dashPort: DASHBOARD_PORT,
       readEnginePid,
       isPortListening,
+      confirmPortUp: tcpPortAccepts,
       isStopIntentSet: shared.isStopIntentSet || (() => false),
     }).then(result => {
       // ALWAYS exit 0 — the scheduler must never see a failure for the

package/dashboard.js

@@ -4672,6 +4672,7 @@ async function ccCallStreaming(message, { store = 'cc', sessionKey, extraContext
       engineConfig: CONFIG.engine,
       onChunk,
       onToolUse,
+      onToolUpdate,
     });
     if (onAbortReady) onAbortReady(p1.abort);
     result = await p1;
@@ -4710,6 +4711,7 @@ async function ccCallStreaming(message, { store = 'cc', sessionKey, extraContext
     engineConfig: CONFIG.engine,
     onChunk,
     onToolUse,
+    onToolUpdate,
   });
   if (onAbortReady) onAbortReady(p2.abort);
   result = await p2;
@@ -4730,6 +4732,7 @@ async function ccCallStreaming(message, { store = 'cc', sessionKey, extraContext
     engineConfig: CONFIG.engine,
     onChunk,
     onToolUse,
+    onToolUpdate,
   });
   if (onAbortReady) onAbortReady(p3.abort);
   result = await p3;
@@ -9255,12 +9258,22 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         liveState.text = text;
         if (liveState.writer) liveState.writer({ type: 'chunk', text, segmentId });
       },
-      onToolUse: (name, input) => {
+      onToolUse: (name, input, id) => {
         _touchCcLiveStream(liveState);
-        const entry = { name, input: input || {}, id: null, status: null };
+        // Claude (direct) now threads a tool_use id; with an id the chip starts
+        // 'pending' and can be flipped to completed/failed by onToolUpdate.
+        const entry = { name, input: input || {}, id: id || null, status: id ? 'pending' : null };
         toolUses.push(entry);
         liveState.tools.push(entry);
-        if (liveState.writer) liveState.writer({ type: 'tool', name, input: _lightToolInput(input), id: null });
+        if (liveState.writer) liveState.writer({ type: 'tool', name, input: _lightToolInput(input), id: id || null });
+      },
+      onToolUpdate: (id, status) => {
+        _touchCcLiveStream(liveState);
+        // toolUses and liveState.tools share entry references — patch once so a
+        // reconnect replays the resolved (green/red) state, not just pending.
+        const entry = toolUses.find((e) => e && e.id === id);
+        if (entry) entry.status = status;
+        if (liveState.writer) liveState.writer({ type: 'tool-update', id, status });
       },
     });
   }
@@ -9673,7 +9686,12 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           reconnectDone();
         });
         for (const tool of live.tools || []) {
-          writeCcEvent({ type: 'tool', name: tool.name, input: _lightToolInput(tool.input) });
+          writeCcEvent({ type: 'tool', name: tool.name, input: _lightToolInput(tool.input), id: tool.id || null });
+          // Replay the resolved chip state (green check / red X) for tools that
+          // already finished, so a reconnect doesn't stick on the pending dot.
+          if (tool.id && tool.status && tool.status !== 'pending') {
+            writeCcEvent({ type: 'tool-update', id: tool.id, status: tool.status });
+          }
         }
         if (live.text) writeCcEvent({ type: 'chunk', text: live.text });
         if (live.donePayload) {
@@ -10891,6 +10909,28 @@ What would you like to discuss or change? When you're happy, say "approve" and I
     } catch (e) { return jsonReply(res, e.statusCode || 500, { error: e.message }); }
   }
 
+  // W-mqidhwcc000m897e — read-only ADO token-acquisition health snapshot.
+  // Surfaces the graduated-backoff + transient/persistent classification state
+  // so an operator can diagnose "ADO polling paused" from one place:
+  //   { token: { lastSuccessAt, lastFailureReason, classification,
+  //              backoffUntil, consecutiveFailures } }.
+  async function handleDiagnosticsAdoToken(req, res) {
+    try {
+      let token = { lastSuccessAt: 0, lastFailureReason: null, classification: null, backoffUntil: 0, consecutiveFailures: 0 };
+      if (typeof ado.getAdoTokenHealth === 'function') {
+        const h = ado.getAdoTokenHealth() || {};
+        token = {
+          lastSuccessAt: Number(h.lastSuccessAt) || 0,
+          lastFailureReason: h.lastFailureReason || null,
+          classification: h.classification || null,
+          backoffUntil: Number(h.backoffUntil) || 0,
+          consecutiveFailures: Number(h.consecutiveFailures) || 0,
+        };
+      }
+      return jsonReply(res, 200, { token });
+    } catch (e) { return jsonReply(res, e.statusCode || 500, { error: e.message }); }
+  }
+
   // P-c3d4e5f6 — /api/diagnostics/memory.
   // Returns the latest in-process dashboard sample, the latest engine
   // sample (read fresh from engine/diagnostics-memory.json via safeJsonObj),
@@ -13633,6 +13673,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
     { method: 'POST', path: '/api/diagnostics/refresh', desc: 'Append a dashboard refresh-diagnostic ring buffer batch to engine/dashboard-diagnostics.log (rotated at 1 MB)', params: 'entries[]', handler: handleDiagnosticsRefresh },
     // Diagnostics — per-org ADO throttle state (W-mq03l6zh0006f0a1-d).
     { method: 'GET', path: '/api/diagnostics/ado-throttle', desc: 'Snapshot of per-org ADO throttle tracker state — { orgs: { [orgBase]: { throttled, retryAfter, consecutiveHits } } }. Falls back to a single `global` key when running against pre-per-org engines.', handler: handleDiagnosticsAdoThrottle },
+    { method: 'GET', path: '/api/diagnostics/ado-token', desc: 'Read-only ADO token-acquisition health — { token: { lastSuccessAt, lastFailureReason, classification, backoffUntil, consecutiveFailures } }. Surfaces the graduated-backoff + transient/persistent classification state (W-mqidhwcc000m897e) so an operator can diagnose paused ADO polling from one line.', handler: handleDiagnosticsAdoToken },
     // Diagnostics — engine + dashboard memory baseline (P-c3d4e5f6).
     { method: 'GET', path: '/api/diagnostics/memory', desc: 'Latest in-process dashboard memory sample plus the most-recent engine sample read from engine/diagnostics-memory.json. engineStale=true when the sidecar is missing or its capturedAt is > 5 min old.', handler: handleDiagnosticsMemory },
     { method: 'GET', path: '/api/diagnostics/memory/history', desc: 'In-memory ring buffer of memory samples. process=dashboard returns the dashboard\'s own collector (populated by startPeriodicSampling on boot). process=engine returns the dashboard\'s polled accumulation of engine/diagnostics-memory.json — engine.js only persists the latest sample to the sidecar, so engine-side history is rebuilt by the dashboard poller (dedup by capturedAt). Optional limit caps returned newest-N samples.', params: 'process (engine|dashboard), limit?', handler: handleDiagnosticsMemoryHistory },

package/engine/ado.js

@@ -6,7 +6,7 @@
 const path = require('path');
 const childProcess = require('child_process');
 const shared = require('./shared');
-const { exec, execAsync, getAdoOrgBase, log, ts, dateStamp, PR_STATUS, BUILD_STATUS, REVIEW_STATUS, FETCH_TIMEOUT_MS, ADO_TOKEN_REFRESH_MAX_RETRIES, createThrottleTracker } = shared;
+const { exec, execAsync, getAdoOrgBase, log, ts, dateStamp, PR_STATUS, BUILD_STATUS, REVIEW_STATUS, FETCH_TIMEOUT_MS, ADO_TOKEN_REFRESH_MAX_RETRIES, createThrottleTracker, createBackoffTracker, writeToInbox } = shared;
 const { getPrs } = require('./queries');
 const { mutateJsonFileLocked } = shared;
 const { acquireAdoToken } = require('./ado-token');
@@ -714,7 +714,118 @@ function _hasPendingReReviewWi(pr) {
 // ─── ADO Token Cache ─────────────────────────────────────────────────────────
 
 let _adoTokenCache = { token: null, expiresAt: 0 };
-let _adoTokenFailedUntil = 0; // backoff: skip token acquisition calls until this timestamp
+
+// ── ADO token acquisition backoff + health (W-mqidhwcc000m897e) ──────────────
+// Graduated backoff replaces the old flat 10-min blackout: a single transient
+// blip (broker hiccup, az cold-start, network) now backs off 30s and doubles
+// per consecutive failure, capped at 10m, resetting to zero on the next
+// success. Distinct from the per-org throttle tracker (server-driven 429/503).
+const ADO_TOKEN_BACKOFF_BASE_MS = 30000;
+const ADO_TOKEN_BACKOFF_MAX_MS = 10 * 60 * 1000;
+let _adoTokenBackoff = createBackoffTracker({ baseMs: ADO_TOKEN_BACKOFF_BASE_MS, maxMs: ADO_TOKEN_BACKOFF_MAX_MS });
+
+// Read-model for the token health diagnostic + reason-carrying skip warn.
+// Only the three fields the backoff tracker does NOT own live here; backoffUntil
+// and consecutiveFailures are read from `_adoTokenBackoff` on demand in
+// getAdoTokenHealth() so there is a single source of truth.
+let _adoTokenHealth = {
+  lastSuccessAt: 0,
+  lastFailureReason: null,
+  classification: null, // 'transient' | 'persistent' | null
+};
+
+// Dedup latch for the persistent-auth operator inbox note. Set when a note is
+// emitted; cleared on the next successful acquisition so a fresh episode
+// (after a human runs `az login`) can surface a new note.
+let _adoTokenAuthNoteSent = false;
+// Backoff-window the last no-token skip warn fired for. The skip warn logs on
+// the STATE TRANSITION into a new no-token window (each real acquisition
+// failure mints a new backoffUntil) and stays silent across the suppressed
+// poll/reconcile cycles within that same window.
+let _adoNoTokenWarnedBackoffUntil = -1;
+
+// Persistent (human-actionable) acquisition errors: az not logged in, no
+// subscription, azureauth interactive/broker consent required, missing tooling.
+// Everything else (timeout, ECONN*, broker busy, generic failure) is transient.
+const _ADO_TOKEN_PERSISTENT_RE = /\baz login\b|run ['"]?az login|az account set|no subscription(?:\s+found)?|not logged in|AADSTS\d+|interaction[_ ]required|consent[_ ]required|interactive authentication|reauthenticat|invalid_grant|is not recognized|command not found/i;
+
+/** Classify an acquisition error as 'persistent' (human-actionable, needs az
+ *  login / setup) or 'transient' (retry will likely self-heal). Scans the
+ *  combined error message plus every per-provider attempt error. Defaults to
+ *  'transient' so we never nag the operator on a recoverable blip. */
+function classifyAdoTokenError(err) {
+  const parts = [err?.message || ''];
+  if (Array.isArray(err?.attempts)) {
+    for (const a of err.attempts) parts.push(String(a?.error || ''));
+  }
+  const haystack = parts.join(' | ');
+  return _ADO_TOKEN_PERSISTENT_RE.test(haystack) ? 'persistent' : 'transient';
+}
+
+/** Short, single-line reason for the health read-model + skip-warn log. */
+function _summarizeAdoTokenError(err) {
+  return String(err?.message || 'unknown error').replace(/\s+/g, ' ').trim().slice(0, 240);
+}
+
+// Overridable note writer (test seam). Default routes through the shared
+// date-deduped inbox writer so at most one note lands per day even if the
+// in-memory latch is reset.
+let _adoTokenAuthNoteWriter = null;
+function _defaultAdoTokenAuthNoteWriter(reason) {
+  const body = [
+    '# ADO auth needs `az login` — PR polling/reconciliation paused',
+    '',
+    'ADO token acquisition is failing in a way that will NOT self-heal without a',
+    'human action (e.g. `az login`, selecting a subscription, or completing an',
+    'interactive azureauth consent). Engine PR status polling + reconciliation for',
+    'ADO repos are paused on a graduated backoff until a token can be minted again.',
+    '',
+    `- reason: ${reason || 'unknown'}`,
+    '- remedy: run `az login` (and `az account set --subscription <id>` if prompted)',
+    '  on the engine host, then the next successful acquisition auto-resumes polling.',
+    '',
+    'This note is deduped — it will not repeat every cycle. A fresh note is allowed',
+    'after the next successful token acquisition.',
+  ].join('\n');
+  return writeToInbox('engine', 'ado-token-auth-paused', body);
+}
+function _emitAdoTokenAuthNote(reason) {
+  try {
+    return (_adoTokenAuthNoteWriter || _defaultAdoTokenAuthNoteWriter)(reason);
+  } catch (e) {
+    try { log('warn', `Failed to write ADO token auth inbox note: ${e.message}`); } catch { /* best-effort */ }
+    return false;
+  }
+}
+
+/** Token-health snapshot for diagnostics (read-only). */
+function getAdoTokenHealth() {
+  const bs = _adoTokenBackoff.getState();
+  return {
+    lastSuccessAt: _adoTokenHealth.lastSuccessAt,
+    lastFailureReason: _adoTokenHealth.lastFailureReason,
+    classification: _adoTokenHealth.classification,
+    backoffUntil: bs.backoffUntil,
+    consecutiveFailures: bs.consecutiveFailures,
+  };
+}
+
+/** Consume a one-shot no-token warning. Returns shouldWarn=true only on the
+ *  transition into a new no-token backoff window (each real acquisition failure
+ *  mints a new backoffUntil), so the recurring poll/reconcile cycles within the
+ *  same window stay silent. Carries the underlying reason + backoffUntil so the
+ *  caller logs a single diagnosable line. */
+function consumeNoAdoTokenWarning() {
+  const backoffUntil = _adoTokenBackoff.getState().backoffUntil;
+  const shouldWarn = backoffUntil !== _adoNoTokenWarnedBackoffUntil;
+  if (shouldWarn) _adoNoTokenWarnedBackoffUntil = backoffUntil;
+  return {
+    shouldWarn,
+    reason: _adoTokenHealth.lastFailureReason,
+    classification: _adoTokenHealth.classification,
+    backoffUntil,
+  };
+}
 
 // ─── ADO Throttle State (per-org) ───────────────────────────────────────────
 // Tracks rate-limiting (HTTP 429/503) from ADO API responses, isolated per ADO
@@ -793,23 +904,36 @@ function isAdoAuthError(err) {
   return msg.includes('auth redirect') || msg.includes('HTML instead of JSON') || /ADO API (401|403)/.test(msg);
 }
 
-async function getAdoToken() {
+async function getAdoToken(opts = {}) {
+  // _acquire is a test seam — production callers pass no args (unchanged).
+  const acquire = opts._acquire || acquireAdoToken;
   if (_adoTokenCache.token && Date.now() < _adoTokenCache.expiresAt) {
     return _adoTokenCache.token;
   }
-  // If recent fetch failed, don't retry until backoff expires.
-  if (Date.now() < _adoTokenFailedUntil) return null;
+  // Graduated backoff — skip acquisition (return null) while backing off so a
+  // transient blip doesn't blackout polling/reconciliation for the full cap.
+  if (_adoTokenBackoff.isBackingOff()) return null;
   try {
-    const { token } = await acquireAdoToken({ execAsync, timeout: 15000 });
+    const { token } = await acquire({ execAsync, timeout: 15000 });
     _adoTokenCache = { token, expiresAt: Date.now() + 30 * 60 * 1000 };
-    _adoTokenFailedUntil = 0;
+    _adoTokenBackoff.recordSuccess();
+    _adoTokenHealth = { lastSuccessAt: Date.now(), lastFailureReason: null, classification: null };
+    _adoTokenAuthNoteSent = false; // allow a fresh persistent note next episode
+    _adoNoTokenWarnedBackoffUntil = -1; // allow a fresh skip warn next episode
     return token;
   } catch (e) {
-    log('warn', `Failed to get ADO token: ${e.message}`);
+    const classification = classifyAdoTokenError(e);
+    const reason = _summarizeAdoTokenError(e);
+    const { delayMs, consecutiveFailures } = _adoTokenBackoff.recordFailure();
+    _adoTokenHealth.lastFailureReason = reason;
+    _adoTokenHealth.classification = classification;
+    log('warn', `Failed to get ADO token (${classification}, attempt ${consecutiveFailures}): ${e.message} — backing off ${Math.round(delayMs / 1000)}s`);
+    if (classification === 'persistent' && !_adoTokenAuthNoteSent) {
+      _adoTokenAuthNoteSent = true;
+      _emitAdoTokenAuthNote(reason);
+    }
+    return null;
   }
-  // Back off for 10 minutes to avoid spamming auth commands.
-  _adoTokenFailedUntil = Date.now() + 10 * 60 * 1000;
-  return null;
 }
 
 async function adoFetch(url, token, opts = {}) {
@@ -1188,8 +1312,11 @@ async function pollPrStatus(config) {
 
   const token = await getAdoToken();
   if (!token) {
-    log('warn', 'Skipping PR status poll — no ADO token available');
-    // Don't set _adoPollHadAuthFailure — getAdoToken() has its own 10-min backoff,
+    const w = consumeNoAdoTokenWarning();
+    if (w.shouldWarn) {
+      log('warn', `Skipping PR status poll — no ADO token (${w.classification || 'unknown'}: ${w.reason || 'unknown'}); backing off until ${w.backoffUntil ? new Date(w.backoffUntil).toISOString() : 'unknown'}`);
+    }
+    // Don't set _adoPollHadAuthFailure — getAdoToken() has its own graduated backoff,
     // and setting the flag would hammer pollPrStatus() every tick with no useful work.
     return;
   }
@@ -1953,7 +2080,10 @@ async function pollPrHumanComments(config) {
 async function reconcilePrs(config) {
   const token = await getAdoToken();
   if (!token) {
-    log('warn', 'Skipping PR reconciliation — no ADO token available');
+    const w = consumeNoAdoTokenWarning();
+    if (w.shouldWarn) {
+      log('warn', `Skipping PR reconciliation — no ADO token (${w.classification || 'unknown'}: ${w.reason || 'unknown'}); backing off until ${w.backoffUntil ? new Date(w.backoffUntil).toISOString() : 'unknown'}`);
+    }
     return;
   }
 
@@ -2568,17 +2698,40 @@ function _setAdoThrottleForTest(state, orgBase = 'dev.azure.com/__test__') {
  *  Pass null to force getAdoToken() to return null synchronously (no exec). */
 function _setAdoTokenForTest(token) {
   if (token == null) {
-    // Clear cache AND set a future failure backoff so getAdoToken short-circuits
+    // Clear cache AND open a long backoff window so getAdoToken short-circuits
     // to null without spawning azureauth — otherwise tests would hang on the
     // 15s execAsync timeout or open a real auth popup.
     _adoTokenCache = { token: null, expiresAt: 0 };
-    _adoTokenFailedUntil = Date.now() + 60 * 60 * 1000;
+    _adoTokenBackoff._setForTest({ consecutiveFailures: 1, backoffUntil: Date.now() + 60 * 60 * 1000 });
   } else {
     _adoTokenCache = { token, expiresAt: Date.now() + 30 * 60 * 1000 };
-    _adoTokenFailedUntil = 0;
+    _adoTokenBackoff.recordSuccess();
+    _adoTokenHealth = { lastSuccessAt: Date.now(), lastFailureReason: null, classification: null };
+    _adoTokenAuthNoteSent = false;
+    _adoNoTokenWarnedBackoffUntil = -1;
   }
 }
 
+/** Reset all token backoff + health + dedup state — exported for testing only. */
+function _resetAdoTokenForTest() {
+  _adoTokenCache = { token: null, expiresAt: 0 };
+  _adoTokenBackoff._reset();
+  _adoTokenHealth = { lastSuccessAt: 0, lastFailureReason: null, classification: null };
+  _adoTokenAuthNoteSent = false;
+  _adoNoTokenWarnedBackoffUntil = -1;
+}
+
+/** Force the token backoff tracker's internal state — exported for testing only. */
+function _setAdoTokenBackoffForTest(overrides) {
+  _adoTokenBackoff._setForTest(overrides || {});
+}
+
+/** Override the persistent-auth inbox note writer — exported for testing only.
+ *  Pass null to restore the default (shared.writeToInbox-backed) writer. */
+function _setAdoTokenAuthNoteWriterForTest(fn) {
+  _adoTokenAuthNoteWriter = (typeof fn === 'function') ? fn : null;
+}
+
 // ─── One-Shot Startup Reconciliation for Abandoned PRs (W-mp60tw0u000j3931) ───
 //
 // ADO equivalent of engine/github.js reconcileAbandonedPrs. Same shape:
@@ -2748,6 +2901,13 @@ module.exports = {
   resetReviewerNegativeVote, // W-mp7b1g8q000fea45 — reset reviewer's prior negative vote on verdict flip
   needsAdoPollRetry,
   isAdoAuthError, // exported for testing
+  // W-mqidhwcc000m897e — token graduated backoff + classification + health
+  classifyAdoTokenError,
+  getAdoTokenHealth,
+  consumeNoAdoTokenWarning,
+  _resetAdoTokenForTest, // exported for testing
+  _setAdoTokenBackoffForTest, // exported for testing
+  _setAdoTokenAuthNoteWriterForTest, // exported for testing
   isAdoThrottled,
   getAdoThrottleState,
   getAdoThrottleStateAll,

package/engine/llm.js

@@ -507,6 +507,7 @@ function _createStreamAccumulator({
   maxTextLength = 0,
   onChunk = null,
   onToolUse = null,
+  onToolUpdate = null,
   onTaskComplete = null,
   onTerminalResult = null,
   onThinking = null,
@@ -574,12 +575,21 @@ function _createStreamAccumulator({
         onTerminalResult();
       }
     },
-    pushToolUse(name, input) {
+    pushToolUse(name, input, id = null) {
       if (!name) return;
-      const toolUse = { name, input: input || {} };
+      const toolUse = { name, input: input || {}, id: id || null };
       toolUses.push(toolUse);
       sawToolSinceText = true;
-      if (onToolUse) onToolUse(toolUse.name, toolUse.input);
+      if (onToolUse) onToolUse(toolUse.name, toolUse.input, toolUse.id);
+    },
+    updateToolUse(id, status) {
+      // Runtime-agnostic tool-completion signal. Adapters whose stream carries
+      // a per-tool result event (Claude's tool_result, Copilot's
+      // tool_call_update) call this to flip a previously-pushed chip from the
+      // neutral pending dot to a green check / red X. Adapters without such a
+      // signal simply never call it and the chip stays pending.
+      if (!id || !onToolUpdate) return;
+      onToolUpdate(id, status);
     },
     toolUseAlreadySeen(name, input) {
       if (!name) return false;
@@ -869,7 +879,7 @@ function callLLM(promptText, sysPromptText, opts = {}) {
 function callLLMStreaming(promptText, sysPromptText, opts = {}) {
   const {
     timeout = 120000, label = 'llm', maxTurns = 1, allowedTools = '',
-    sessionId = null, onChunk = () => {}, onToolUse = null,
+    sessionId = null, onChunk = () => {}, onToolUse = null, onToolUpdate = null,
     effort = null, direct = false,
     model: modelOverride, cli: cliOverride, engineConfig,
     maxBudget, bare, fallbackModel,
@@ -911,6 +921,7 @@ function callLLMStreaming(promptText, sysPromptText, opts = {}) {
       maxLineBufferBytes: ENGINE_DEFAULTS.maxLlmLineBufferBytes,
       onChunk,
       onToolUse,
+      onToolUpdate,
       // Terminal text from the runtime adapter signals the LLM has logically
       // completed — kick the drain timer so we don't block on a delayed
       // 'exit'/'close' when an inherited pipe keeps the parent's FDs open.

package/engine/runtimes/claude.js

@@ -636,8 +636,9 @@ function parseError(rawOutput) {
 // tracking) and translates Claude event shapes into ctx callbacks.
 //
 // `ctx` shape (provided by accumulator):
-//   maxTextLength, pushText(value), pushToolUse(name, input),
-//   notifyThinking(), notifyTaskComplete(summary, success),
+//   maxTextLength, pushText(value), pushToolUse(name, input, id),
+//   updateToolUse(id, status), notifyThinking(),
+//   notifyTaskComplete(summary, success),
 //   setUsage(usage), setSessionId(id), setText(value),
 //   toolUseAlreadySeen(name, input)
 
@@ -743,11 +744,27 @@ function createStreamConsumer(ctx) {
         } else if (THINKING_BLOCK_TYPES.has(block?.type)) {
           ctx.notifyThinking();
         } else if (block?.type === 'tool_use' && block.name) {
-          ctx.pushToolUse(block.name, block.input || {});
+          // Thread the tool_use id so the dashboard can flip this chip from
+          // pending → completed/failed when the matching tool_result lands.
+          ctx.pushToolUse(block.name, block.input || {}, block.id);
         }
       }
       if (assistantText) ctx.pushText(assistantText);
     }
+
+    if (obj.type === 'user' && Array.isArray(obj.message?.content)) {
+      // Claude reports tool execution results in a subsequent `user` turn:
+      // each block is {type:'tool_result', tool_use_id, is_error, content}.
+      // Flip the previously-pushed chip to completed (success) or failed
+      // (is_error). This is the Claude-direct analogue of Copilot's ACP
+      // tool_call_update; engine/llm.js stays runtime-agnostic — the chip
+      // turns green/red only because this adapter calls ctx.updateToolUse.
+      for (const block of obj.message.content) {
+        if (block?.type === 'tool_result' && block.tool_use_id) {
+          ctx.updateToolUse(block.tool_use_id, block.is_error ? 'failed' : 'completed');
+        }
+      }
+    }
   }
 
   function reset() {

package/engine/shared.js

@@ -8294,6 +8294,50 @@ function createThrottleTracker({ label, baseBackoffMs = 60000, maxBackoffMs = 32
   return { recordThrottle, recordSuccess, isThrottled, getState, _reset, _setForTest };
 }
 
+// ── Backoff Tracker Factory ─────────────────────────────────────────────────
+// Generic graduated-backoff tracker for retryable *acquisition* failures (e.g.
+// ADO token minting via engine/ado-token.js). Distinct from
+// createThrottleTracker on three points so callers don't reach for the wrong
+// one: (1) the FIRST failure backs off `baseMs`, not `baseMs*2` — a single
+// transient blip must not blackout for the full cap; (2) recordSuccess RESETS
+// consecutiveFailures to zero rather than the throttle tracker's gradual
+// per-success decay; (3) there is no server Retry-After / jitter concept. The
+// delay grows baseMs, 2·baseMs, 4·baseMs, … doubling, capped at maxMs.
+// now is injectable so tests don't depend on Date.now().
+function createBackoffTracker({ baseMs = 30000, maxMs = 10 * 60 * 1000 } = {}) {
+  let consecutiveFailures = 0;
+  let backoffUntil = 0;
+
+  function recordFailure(now = Date.now()) {
+    consecutiveFailures++;
+    const delayMs = Math.min(baseMs * Math.pow(2, consecutiveFailures - 1), maxMs);
+    backoffUntil = now + delayMs;
+    return { consecutiveFailures, delayMs, backoffUntil };
+  }
+
+  function recordSuccess() {
+    consecutiveFailures = 0;
+    backoffUntil = 0;
+  }
+
+  function isBackingOff(now = Date.now()) {
+    return now < backoffUntil;
+  }
+
+  function getState() {
+    return { consecutiveFailures, backoffUntil };
+  }
+
+  // Testing helpers
+  function _reset() { consecutiveFailures = 0; backoffUntil = 0; }
+  function _setForTest(overrides = {}) {
+    if (Number.isFinite(overrides.consecutiveFailures)) consecutiveFailures = overrides.consecutiveFailures;
+    if (Number.isFinite(overrides.backoffUntil)) backoffUntil = overrides.backoffUntil;
+  }
+
+  return { recordFailure, recordSuccess, isBackingOff, getState, _reset, _setForTest };
+}
+
 module.exports = {
   MINIONS_DIR,
   ENGINE_DIR,
@@ -8566,5 +8610,6 @@ module.exports = {
   getPinnedItems,
   _logBuffer, // exported for testing
   createThrottleTracker,
+  createBackoffTracker,
   backfillPrPrdItems,
 };

package/engine/watchdog.js

@@ -90,6 +90,16 @@ async function tick(opts) {
   const dashPort = opts.dashPort || DEFAULT_DASH_PORT;
   const readEnginePid = opts.readEnginePid;
   const isPortListening = opts.isPortListening;
+  // Optional authoritative "is the port actually accepting connections?" probe
+  // (a direct TCP connect). Used ONLY to overturn a `down` verdict from the
+  // primary `isPortListening` probe — never to manufacture a `down`. Rationale:
+  // the primary probe on Windows shells out to `netstat -ano | findstr` with a
+  // 5s timeout; on a loaded box (e.g. a self-hosted CI runner mid-test-suite)
+  // netstat blows past the timeout, throws, and is swallowed as "port down",
+  // which would restart a perfectly live dashboard. A TCP handshake is
+  // completed by the kernel's listen backlog even when the dashboard's JS event
+  // loop is briefly blocked, so it stays accurate exactly when netstat fails.
+  const confirmPortUp = opts.confirmPortUp;
   const isStopIntentSet = opts.isStopIntentSet;
   const spawner = opts.spawner || spawn;
   const now = opts.now || (() => new Date());
@@ -125,6 +135,19 @@ async function tick(opts) {
   let dashUp = false;
   try { dashUp = !!isPortListening(dashPort); } catch { dashUp = false; }
 
+  // Confirm a `down` verdict with a direct TCP connect before acting on it.
+  // This converts the common false-negative (netstat timed out under load) back
+  // to the truth without ever masking a real outage: confirmPortUp can only
+  // flip down→up, and only when the kernel actually accepts a connection.
+  if (!dashUp && typeof confirmPortUp === 'function') {
+    let tcpUp = false;
+    try { tcpUp = !!(await confirmPortUp(dashPort)); } catch { tcpUp = false; }
+    if (tcpUp) {
+      logLine(minionsHome, `port=${dashPort} reported down by primary probe but TCP connect succeeded — treating as up (false-negative suppressed)`);
+      dashUp = true;
+    }
+  }
+
   if (engineAlive && dashUp) {
     logLine(minionsHome, `ok engine=${enginePid} port=${dashPort} ts=${now().toISOString()}`);
     return { healthy: true, action: 'none' };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2213",
+  "version": "0.1.2215",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"