@yemi33/minions 0.1.953 → 0.1.955

package/CHANGELOG.md

@@ -1,8 +1,9 @@
 # Changelog
 
-## 0.1.953 (2026-04-14)
+## 0.1.955 (2026-04-14)
 
 ### Features
+-  ADO throttle detection, poll guards, and dashboard banner (#1051)
 - gate auto-fix conflict dispatch behind autoFixConflicts flag
 - add PR/build status CLI shim and agent guidance
 -  make ADO poll frequency configurable and ungate reconcilePrs
@@ -22,7 +23,6 @@
 - add missing branch_name to central dispatch vars (#967)
 - fix dispatch.js mutator fallback to use nullish coalescing (#966)
 - fix stall recovery nested lock violation (#965)
-- fix pending-rebases.json race condition with file locking (#964)
 
 ### Fixes
 - fix ENGINE_DEFAULTS ref and derive boolean settings from defaults
@@ -47,6 +47,7 @@
 -  429 retry on abort race applies to all sends, not just queued
 
 ### Other
+- test(engine): add regression tests for autoFixConflicts flag and DEFAULTS alias
 - refactor(ado): simplify ado-status and extract build/review status helpers
 - refactor(ado): extract stripRefsHeads helper, deduplicate refs/heads/ stripping
 - refactor(ado): simplify comments, fix declaration order, promote ado import

package/dashboard/js/refresh.js

@@ -75,6 +75,8 @@ function _processStatusUpdate(data) {
     }
   }
   if (_changed('version', data.version)) renderVersionBanner(data.version);
+  if (_changed('adoThrottle', data.adoThrottle)) renderAdoThrottleAlert(data.adoThrottle);
+  if (_changed('ghThrottle', data.ghThrottle)) renderGhThrottleAlert(data.ghThrottle);
   if (_changed('dispatch', data.dispatch)) renderDispatch(data.dispatch);
   window._lastDispatch = data.dispatch;
   window._lastWorkItems = data.workItems || [];

package/dashboard/js/render-dispatch.js

@@ -72,6 +72,34 @@ function renderEngineAlert(state, staleMs) {
   el.style.display = 'flex';
 }
 
+function renderAdoThrottleAlert(adoThrottle) {
+  const el = document.getElementById('ado-throttle-alert');
+  if (!el) return;
+  if (!adoThrottle || !adoThrottle.throttled) {
+    el.style.display = 'none';
+    el.innerHTML = '';
+    return;
+  }
+  const resumeTime = new Date(adoThrottle.retryAfter).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
+  el.innerHTML =
+    '<span class="engine-alert-msg">&#x26A0;&#xFE0F; ADO rate-limited — resume ~' + resumeTime + ' (' + adoThrottle.consecutiveHits + ' consecutive hit' + (adoThrottle.consecutiveHits !== 1 ? 's' : '') + ')</span>';
+  el.style.display = 'flex';
+}
+
+function renderGhThrottleAlert(ghThrottle) {
+  const el = document.getElementById('gh-throttle-alert');
+  if (!el) return;
+  if (!ghThrottle || !ghThrottle.throttled) {
+    el.style.display = 'none';
+    el.innerHTML = '';
+    return;
+  }
+  const resumeTime = new Date(ghThrottle.retryAfter).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
+  el.innerHTML =
+    '<span class="engine-alert-msg">&#x26A0;&#xFE0F; GitHub rate-limited — resume ~' + resumeTime + ' (' + ghThrottle.consecutiveHits + ' consecutive hit' + (ghThrottle.consecutiveHits !== 1 ? 's' : '') + ')</span>';
+  el.style.display = 'flex';
+}
+
 function renderDispatch(dispatch) {
   if (!dispatch) return;
 
@@ -247,4 +275,4 @@ function renderVersionBanner(version) {
   }
 }
 
-window.MinionsDispatch = { renderEngineStatus, renderEngineAlert, renderVersionBanner, renderDispatch, renderEngineLog, shortTime, showErrorDetails };
+window.MinionsDispatch = { renderEngineStatus, renderEngineAlert, renderAdoThrottleAlert, renderGhThrottleAlert, renderVersionBanner, renderDispatch, renderEngineLog, shortTime, showErrorDetails };

package/dashboard/pages/engine.html

@@ -1,3 +1,5 @@
+      <div class="engine-alert" id="ado-throttle-alert" style="display:none"></div>
+      <div class="engine-alert" id="gh-throttle-alert" style="display:none"></div>
       <section>
         <div id="engine-quick-stats" style="display:flex;gap:16px;margin-bottom:12px;font-size:11px;color:var(--muted)"></div>
       </section>

package/dashboard.js

@@ -22,6 +22,7 @@ const shared = require('./engine/shared');
 const queries = require('./engine/queries');
 const teams = require('./engine/teams');
 const ado = require('./engine/ado');
+const gh = require('./engine/github');
 const os = require('os');
 
 const { safeRead, safeReadDir, safeWrite, safeJson, safeJsonObj, safeJsonArr, safeUnlink, mutateJsonFileLocked, mutateWorkItems, getProjects: _getProjects, DONE_STATUSES, WI_STATUS, reopenWorkItem } = shared;
@@ -382,6 +383,8 @@ function getStatus() {
     verifyGuides: getVerifyGuides(),
     archivedPrds: getArchivedPrds(),
     engine: { ...getEngineState(), worktreeCount: _countWorktrees() },
+    adoThrottle: ado.getAdoThrottleState(),
+    ghThrottle: gh.getGhThrottleState(),
     dispatch: getDispatchQueue(),
     engineLog: getEngineLog(),
     metrics: getMetrics(),

package/engine/ado.js

@@ -5,7 +5,7 @@
 
 const path = require('path');
 const shared = require('./shared');
-const { exec, execAsync, getAdoOrgBase, addPrLink, log, ts, dateStamp, PR_STATUS } = shared;
+const { exec, execAsync, getAdoOrgBase, addPrLink, log, ts, dateStamp, PR_STATUS, createThrottleTracker } = shared;
 const { getPrs } = require('./queries');
 const { mutateJsonFileLocked } = shared;
 
@@ -47,6 +47,11 @@ function votesToReviewStatus(votes) {
 let _adoTokenCache = { token: null, expiresAt: 0 };
 let _adoTokenFailedUntil = 0; // backoff: skip azureauth calls until this timestamp
 
+// ─── ADO Throttle State ─────────────────────────────────────────────────────
+// Tracks rate-limiting (HTTP 429/503) from ADO API responses.
+// Uses shared createThrottleTracker factory: backoffMs starts at 60s, doubles, caps at 32 min.
+const _adoThrottle = createThrottleTracker({ label: 'ado', baseBackoffMs: 60000, maxBackoffMs: 32 * 60000 });
+
 // ─── Auth Failure Tracking ──────────────────────────────────────────────────
 // Set when pollPrStatus encounters auth errors mid-loop. The engine checks this
 // to bypass the normal 6-tick cadence and re-poll on the next tick.
@@ -97,6 +102,14 @@ async function adoFetch(url, token, opts = {}) {
     signal: AbortSignal.timeout(30000),
     body,
   });
+  // ── Throttle detection: intercept 429/503 BEFORE generic !res.ok ──
+  if (res.status === 429 || res.status === 503) {
+    const retryAfterSec = parseInt(res.headers.get('Retry-After'), 10);
+    const retryAfterMs = (retryAfterSec > 0) ? retryAfterSec * 1000 : 0;
+    _adoThrottle.recordThrottle(retryAfterMs);
+    const state = _adoThrottle.getState();
+    throw new Error(`ADO API throttled (${res.status}): retry after ${Math.round((state.retryAfter - Date.now()) / 1000)}s`);
+  }
   if (!res.ok) throw new Error(`ADO API ${method} ${res.status}: ${res.statusText}`);
   const text = await res.text();
   if (!text || text.trimStart().startsWith('<')) {
@@ -110,7 +123,10 @@ async function adoFetch(url, token, opts = {}) {
     }
     throw new Error(`ADO returned HTML instead of JSON (likely auth redirect) for ${url.split('?')[0]}`);
   }
-  return JSON.parse(text);
+  const json = JSON.parse(text);
+  // ── Success decay: decrement consecutiveHits, reset when fully recovered ──
+  _adoThrottle.recordSuccess();
+  return json;
 }
 
 /** Fetch raw text from ADO API (for build logs which aren't JSON). */
@@ -119,6 +135,14 @@ async function adoFetchText(url, token) {
     headers: { 'Authorization': `Bearer ${token}` },
     signal: AbortSignal.timeout(30000),
   });
+  // ── Throttle detection: intercept 429/503 BEFORE generic !res.ok ──
+  if (res.status === 429 || res.status === 503) {
+    const retryAfterSec = parseInt(res.headers.get('Retry-After'), 10);
+    const retryAfterMs = (retryAfterSec > 0) ? retryAfterSec * 1000 : 0;
+    _adoThrottle.recordThrottle(retryAfterMs);
+    const state = _adoThrottle.getState();
+    throw new Error(`ADO API throttled (${res.status}): retry after ${Math.round((state.retryAfter - Date.now()) / 1000)}s`);
+  }
   if (!res.ok) throw new Error(`ADO API ${res.status}: ${res.statusText}`);
   return res.text();
 }
@@ -858,6 +882,24 @@ async function fetchSinglePrBuildStatus(project, prNumber) {
   };
 }
 
+// ─── ADO Throttle Queries ────────────────────────────────────────────────────
+
+/** Returns true if ADO is throttled and retryAfter hasn't elapsed. Auto-clears when retryAfter passes. */
+const isAdoThrottled = () => _adoThrottle.isThrottled();
+
+/** Returns a snapshot of the current throttle state. Calls isAdoThrottled() for a fresh value. */
+const getAdoThrottleState = () => _adoThrottle.getState();
+
+/** Reset throttle state — exported for testing only. */
+function _resetAdoThrottle() {
+  _adoThrottle._reset();
+}
+
+/** Set throttle state directly — exported for testing only. */
+function _setAdoThrottleForTest(state) {
+  _adoThrottle._setForTest(state);
+}
+
 module.exports = {
   getAdoToken,
   adoFetch,
@@ -867,7 +909,11 @@ module.exports = {
   checkLiveReviewStatus,
   needsAdoPollRetry,
   isAdoAuthError, // exported for testing
+  isAdoThrottled,
+  getAdoThrottleState,
   fetchAdoPrMetadata,
   fetchSinglePrBuildStatus,
+  _resetAdoThrottle, // exported for testing
+  _setAdoThrottleForTest, // exported for testing
 };
 

package/engine/github.js

@@ -5,7 +5,7 @@
  */
 
 const shared = require('./shared');
-const { exec, execAsync, getProjects, projectPrPath, projectWorkItemsPath, safeJson, safeWrite, mutateJsonFileLocked, MINIONS_DIR, addPrLink, getPrLinks, log, ts, dateStamp, PR_STATUS, PR_POLLABLE_STATUSES } = shared;
+const { exec, execAsync, getProjects, projectPrPath, projectWorkItemsPath, safeJson, safeWrite, mutateJsonFileLocked, MINIONS_DIR, addPrLink, getPrLinks, log, ts, dateStamp, PR_STATUS, PR_POLLABLE_STATUSES, createThrottleTracker } = shared;
 const { getPrs } = require('./queries');
 const path = require('path');
 
@@ -68,13 +68,34 @@ function resetSlugBackoff(slug) {
   }
 }
 
+// ─── GitHub Rate-Limit Throttle ────────────────────────────────────────────
+// Tracks rate-limiting from GitHub API (gh CLI exits non-zero with rate-limit messages).
+// GitHub rate limits reset hourly, so cap at 60 min.
+const _ghThrottle = createThrottleTracker({ label: 'gh', baseBackoffMs: 60000, maxBackoffMs: 60 * 60000 });
+
+/** Returns true if GitHub is rate-limited and retryAfter hasn't elapsed. */
+const isGhThrottled = () => _ghThrottle.isThrottled();
+
+/** Returns a snapshot of the current GitHub throttle state. */
+const getGhThrottleState = () => _ghThrottle.getState();
+
 /** Run a `gh api` call and parse JSON result. Returns null on failure. */
 async function ghApi(endpoint, slug) {
   try {
     const cmd = `gh api "repos/${slug}${endpoint}"`;
     const result = await execAsync(cmd, { timeout: 30000, encoding: 'utf-8' });
-    return JSON.parse(result);
+    const parsed = JSON.parse(result);
+    _ghThrottle.recordSuccess();
+    return parsed;
   } catch (e) {
+    const msg = e?.message || e?.stderr || '';
+    // Detect GitHub rate-limit errors from gh CLI output
+    if (/rate.?limit|secondary rate|429/i.test(msg)) {
+      // Try to extract retry seconds from "N seconds" pattern in error message
+      const secMatch = msg.match(/(\d+)\s*seconds?/i);
+      const retryAfterMs = secMatch ? parseInt(secMatch[1], 10) * 1000 : 0;
+      _ghThrottle.recordThrottle(retryAfterMs);
+    }
     log('warn', `GitHub API error (${endpoint}): ${e.message}`);
     return null;
   }
@@ -731,10 +752,13 @@ module.exports = {
   pollPrHumanComments,
   reconcilePrs,
   checkLiveReviewStatus,
+  isGhThrottled,
+  getGhThrottleState,
   // Exported for testing
   isSlugInBackoff,
   recordSlugFailure,
   resetSlugBackoff,
   _ghPollBackoff,
+  _ghThrottle, // exported for testing
 };
 

package/engine/shared.js

@@ -1050,6 +1050,59 @@ function formatTranscriptEntry(t) {
   return '### ' + (t.agent || 'agent') + ' (' + (t.type || '') + ', Round ' + (t.round || '?') + ')\n\n' + (t.content || '');
 }
 
+// ── Throttle Tracker Factory ────────────────────────────────────────────────
+// Generic rate-limit tracker reusable by both ADO and GitHub integrations.
+// Returns an object with recordThrottle, recordSuccess, isThrottled, getState.
+
+function createThrottleTracker({ label, baseBackoffMs = 60000, maxBackoffMs = 32 * 60000 } = {}) {
+  let state = { throttled: false, retryAfter: 0, consecutiveHits: 0, backoffMs: baseBackoffMs };
+
+  function recordThrottle(retryAfterMs) {
+    state.consecutiveHits++;
+    state.backoffMs = Math.min(state.backoffMs * 2, maxBackoffMs);
+    const waitMs = (retryAfterMs > 0) ? retryAfterMs : state.backoffMs;
+    state.throttled = true;
+    state.retryAfter = Date.now() + waitMs;
+    log('warn', `[${label}] Throttled — retry after ${Math.round(waitMs / 1000)}s, consecutive hits: ${state.consecutiveHits}`);
+  }
+
+  function recordSuccess() {
+    if (state.consecutiveHits > 0) {
+      state.consecutiveHits--;
+      if (state.consecutiveHits === 0) {
+        state.throttled = false;
+        state.backoffMs = baseBackoffMs;
+      }
+    }
+  }
+
+  function isThrottled() {
+    if (!state.throttled) return false;
+    if (Date.now() >= state.retryAfter) {
+      // Auto-clear ALL state when retryAfter has elapsed
+      state.throttled = false;
+      state.backoffMs = baseBackoffMs;
+      state.consecutiveHits = 0;
+      return false;
+    }
+    return true;
+  }
+
+  function getState() {
+    return { throttled: isThrottled(), retryAfter: state.retryAfter, consecutiveHits: state.consecutiveHits };
+  }
+
+  // Testing helpers
+  function _reset() {
+    state = { throttled: false, retryAfter: 0, consecutiveHits: 0, backoffMs: baseBackoffMs };
+  }
+  function _setForTest(overrides) {
+    Object.assign(state, overrides);
+  }
+
+  return { recordThrottle, recordSuccess, isThrottled, getState, _reset, _setForTest };
+}
+
 module.exports = {
   MINIONS_DIR,
   PR_LINKS_PATH,
@@ -1113,5 +1166,6 @@ module.exports = {
   slugify,
   formatTranscriptEntry,
   _logBuffer, // exported for testing
+  createThrottleTracker,
 };
 

package/engine.js

@@ -1445,8 +1445,8 @@ function reconcileItemsWithPrs(items, allPrs, { onlyIds } = {}) {
 // ─── Inbox Consolidation (extracted to engine/consolidation.js) ──────────────
 
 const { consolidateInbox } = require('./engine/consolidation');
-const { pollPrStatus, pollPrHumanComments, reconcilePrs, checkLiveReviewStatus: adoCheckLiveReview, needsAdoPollRetry, getAdoToken } = require('./engine/ado');
-const { pollPrStatus: ghPollPrStatus, pollPrHumanComments: ghPollPrHumanComments, reconcilePrs: ghReconcilePrs, checkLiveReviewStatus: ghCheckLiveReview } = require('./engine/github');
+const { pollPrStatus, pollPrHumanComments, reconcilePrs, checkLiveReviewStatus: adoCheckLiveReview, needsAdoPollRetry, getAdoToken, isAdoThrottled } = require('./engine/ado');
+const { pollPrStatus: ghPollPrStatus, pollPrHumanComments: ghPollPrHumanComments, reconcilePrs: ghReconcilePrs, checkLiveReviewStatus: ghCheckLiveReview, isGhThrottled } = require('./engine/github');
 
 // ─── State Snapshot ─────────────────────────────────────────────────────────
 
@@ -3130,11 +3130,15 @@ async function tickInner() {
   // Awaited so PR state is consistent before discoverWork reads it
   // Also re-polls early if previous tick had ADO auth failures (stale build status recovery)
   if (tickCount % adoPollStatusEvery === 0 || needsAdoPollRetry()) {
-    if (adoPollEnabled) {
+    if (adoPollEnabled && !isAdoThrottled()) {
       try { await pollPrStatus(config); } catch (err) { log('warn', `ADO PR status poll error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }
+    } else if (adoPollEnabled && isAdoThrottled()) {
+      log('info', '[ado] PR status poll skipped — throttled');
     }
-    if (ghPollEnabled) {
+    if (ghPollEnabled && !isGhThrottled()) {
       try { await ghPollPrStatus(config); } catch (err) { log('warn', `GitHub PR status poll error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }
+    } else if (ghPollEnabled && isGhThrottled()) {
+      log('info', '[gh] PR status poll skipped — throttled');
     }
     try { await processPendingRebases(config); } catch (err) { log('warn', `Pending rebase processing error: ${err?.message || err}`); }
     // Sync PR status back to PRD items (missing → done when active PR exists)
@@ -3157,11 +3161,15 @@ async function tickInner() {
 
   // 2.7. Poll PR threads for human comments (every adoPollCommentsEvery ticks, default ~12 minutes)
   if (tickCount % adoPollCommentsEvery === 0) {
-    if (adoPollEnabled) {
+    if (adoPollEnabled && !isAdoThrottled()) {
       try { await pollPrHumanComments(config); } catch (err) { log('warn', `ADO PR comment poll error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }
+    } else if (adoPollEnabled && isAdoThrottled()) {
+      log('info', '[ado] PR comment poll skipped — throttled');
     }
-    if (ghPollEnabled) {
+    if (ghPollEnabled && !isGhThrottled()) {
       try { await ghPollPrHumanComments(config); } catch (err) { log('warn', `GitHub PR comment poll error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }
+    } else if (ghPollEnabled && isGhThrottled()) {
+      log('info', '[gh] PR comment poll skipped — throttled');
     }
     // Reconciliation runs regardless of poll flags — it's a recovery sweep, not a convenience poll
     try { await reconcilePrs(config); } catch (err) { log('warn', `ADO PR reconciliation error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.953",
+  "version": "0.1.955",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"