@yemi33/minions 0.1.493 → 0.1.495

package/CHANGELOG.md

@@ -1,8 +1,10 @@
 # Changelog
 
-## 0.1.493 (2026-04-07)
+## 0.1.495 (2026-04-07)
 
 ### Fixes
+- lock timeout retry with backoff to prevent tick cascade failures (#389)
+- GitHub PR poll backoff for inaccessible repos (closes #377) (#386)
 -  reconcile agent completions during engine downtime (closes #376) (#385)
 -  settings button shows modal immediately with loading state
 -  CC respects user scroll position — no auto-scroll when reading history

package/engine/github.js

@@ -30,6 +30,44 @@ function getRepoSlug(project) {
   return `${org}/${repo}`;
 }
 
+// ─── Per-Repo Poll Backoff ──────────────────────────────────────────────────
+// Tracks consecutive poll failures per repo slug to avoid spamming logs when
+// a repo is inaccessible. Backoff doubles each failure: 2min, 4min, 8min, 16min, max 30min.
+const _ghPollBackoff = new Map(); // slug → { failures, backoffUntil }
+const GH_POLL_BACKOFF_BASE_MS = 2 * 60 * 1000; // 2 minutes (one poll cycle)
+const GH_POLL_BACKOFF_MAX_MS = 30 * 60 * 1000;  // 30 minutes cap
+
+/** Check if a repo slug is currently in backoff. Returns true if should skip. */
+function isSlugInBackoff(slug) {
+  const entry = _ghPollBackoff.get(slug);
+  if (!entry) return false;
+  return Date.now() < entry.backoffUntil;
+}
+
+/** Record a poll failure for a repo slug, applying exponential backoff. */
+function recordSlugFailure(slug) {
+  const existing = _ghPollBackoff.get(slug);
+  const failures = (existing?.failures || 0) + 1;
+  const backoffMs = Math.min(GH_POLL_BACKOFF_BASE_MS * Math.pow(2, failures - 1), GH_POLL_BACKOFF_MAX_MS);
+  _ghPollBackoff.set(slug, { failures, backoffUntil: Date.now() + backoffMs });
+  if (failures === 1) {
+    log('warn', `GitHub poll: repo ${slug} failed — will retry in ${Math.round(backoffMs / 1000)}s`);
+  } else {
+    log('warn', `GitHub poll: repo ${slug} failed ${failures} times — backoff ${Math.round(backoffMs / 1000)}s`);
+  }
+}
+
+/** Reset backoff for a repo slug after a successful poll. */
+function resetSlugBackoff(slug) {
+  if (_ghPollBackoff.has(slug)) {
+    const entry = _ghPollBackoff.get(slug);
+    if (entry.failures > 0) {
+      log('info', `GitHub poll: repo ${slug} recovered after ${entry.failures} failure(s)`);
+    }
+    _ghPollBackoff.delete(slug);
+  }
+}
+
 /** Run a `gh api` call and parse JSON result. Returns null on failure. */
 function ghApi(endpoint, slug) {
   try {
@@ -42,6 +80,20 @@ function ghApi(endpoint, slug) {
   }
 }
 
+/**
+ * Run a `gh api` call with per-slug backoff tracking. Returns null on failure.
+ * On success, resets the slug's backoff. On failure, increments it.
+ */
+function ghApiWithBackoff(endpoint, slug) {
+  const result = ghApi(endpoint, slug);
+  if (result === null) {
+    recordSlugFailure(slug);
+  } else {
+    resetSlugBackoff(slug);
+  }
+  return result;
+}
+
 // ─── Shared PR Polling Loop ─────────────────────────────────────────────────
 
 async function forEachActiveGhPr(config, callback) {
@@ -52,10 +104,21 @@ async function forEachActiveGhPr(config, callback) {
     const slug = getRepoSlug(project);
     if (!slug) continue;
 
+    // Skip projects in backoff (inaccessible repo)
+    if (isSlugInBackoff(slug)) continue;
+
     const prs = getPrs(project);
     const activePrs = prs.filter(pr => pr.status === PR_STATUS.ACTIVE);
     if (activePrs.length === 0) continue;
 
+    // Probe repo accessibility before iterating PRs — avoids N warnings per inaccessible repo
+    const probe = ghApi('', slug);
+    if (probe === null) {
+      recordSlugFailure(slug);
+      continue;
+    }
+    resetSlugBackoff(slug);
+
     let projectUpdated = 0;
 
     for (const pr of activePrs) {
@@ -101,6 +164,7 @@ async function forEachActiveGhPr(config, callback) {
     const ghMatch = pr.url.match(/github\.com\/([^/]+\/[^/]+)\/pull\/(\d+)/);
     if (!ghMatch) continue;
     const slug = ghMatch[1];
+    if (isSlugInBackoff(slug)) continue;
     const prNum = ghMatch[2];
     try {
       const updated = await callback(null, pr, prNum, slug);
@@ -370,9 +434,16 @@ async function reconcilePrs(config) {
     const slug = getRepoSlug(project);
     if (!slug) continue;
 
+    // Skip projects in backoff (inaccessible repo)
+    if (isSlugInBackoff(slug)) continue;
+
     // Fetch open PRs
     const prsData = ghApi('/pulls?state=open&per_page=100', slug);
-    if (!prsData || !Array.isArray(prsData)) continue;
+    if (!prsData || !Array.isArray(prsData)) {
+      recordSlugFailure(slug);
+      continue;
+    }
+    resetSlugBackoff(slug);
 
     const ghPrs = prsData.filter(pr => {
       const branch = pr.head?.ref || '';
@@ -498,5 +569,10 @@ module.exports = {
   pollPrHumanComments,
   reconcilePrs,
   checkLiveReviewStatus,
+  // Exported for testing
+  isSlugInBackoff,
+  recordSlugFailure,
+  resetSlugBackoff,
+  _ghPollBackoff,
 };
 

package/engine/shared.js

@@ -174,51 +174,70 @@ const LOCK_STALE_MS = 60000; // 60 seconds — force-remove locks older than thi
 
 function withFileLock(lockPath, fn, {
   timeoutMs = 5000,
-  retryDelayMs = 25
+  retryDelayMs = 25,
+  retries = 0,
+  retryBackoffMs = 1000
 } = {}) {
-  const start = Date.now();
-  let fd = null;
-  while (Date.now() - start < timeoutMs) {
-    try {
-      const dir = path.dirname(lockPath);
-      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-      fd = fs.openSync(lockPath, 'wx');
-      break;
-    } catch (err) {
-      if (err.code !== 'EEXIST') throw err;
-      // Check for stale lock — if lock file is older than LOCK_STALE_MS, force-remove it
+  let lastErr = null;
+  const maxAttempts = 1 + Math.max(0, retries);
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    if (attempt > 0) {
+      // Exponential backoff between retry attempts: retryBackoffMs * 2^(attempt-1)
+      const backoff = retryBackoffMs * Math.pow(2, attempt - 1);
+      sleepMs(backoff);
+    }
+    const start = Date.now();
+    let fd = null;
+    while (Date.now() - start < timeoutMs) {
       try {
-        const stat = fs.statSync(lockPath);
-        if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
-          try {
-            fs.unlinkSync(lockPath);
-          } catch (unlinkErr) {
-            // ENOENT: another process deleted the lock between stat and unlink — safe to retry
-            if (unlinkErr.code !== 'ENOENT') throw unlinkErr;
+        const dir = path.dirname(lockPath);
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+        fd = fs.openSync(lockPath, 'wx');
+        break;
+      } catch (err) {
+        if (err.code !== 'EEXIST') throw err;
+        // Check for stale lock — if lock file is older than LOCK_STALE_MS, force-remove it
+        try {
+          const stat = fs.statSync(lockPath);
+          if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
+            try {
+              fs.unlinkSync(lockPath);
+            } catch (unlinkErr) {
+              // ENOENT: another process deleted the lock between stat and unlink — safe to retry
+              if (unlinkErr.code !== 'ENOENT') throw unlinkErr;
+            }
+            continue; // lock just removed — retry immediately
           }
-          continue; // lock just removed — retry immediately
+        } catch (staleErr) {
+          // ENOENT from statSync: lock file disappeared between EEXIST and stat — retry will succeed
+          if (staleErr.code !== 'ENOENT') throw staleErr;
         }
-      } catch (staleErr) {
-        // ENOENT from statSync: lock file disappeared between EEXIST and stat — retry will succeed
-        if (staleErr.code !== 'ENOENT') throw staleErr;
+        sleepMs(retryDelayMs);
       }
-      sleepMs(retryDelayMs);
     }
-  }
-  if (fd === null) throw new Error(`Lock timeout: ${lockPath}`);
+    if (fd === null) {
+      lastErr = new Error(`Lock timeout: ${lockPath}`);
+      continue; // retry if attempts remain
+    }
 
-  try {
-    return fn();
-  } finally {
-    try { fs.closeSync(fd); } catch { /* cleanup */ }
-    try { fs.unlinkSync(lockPath); } catch { /* cleanup */ }
+    try {
+      return fn();
+    } finally {
+      try { fs.closeSync(fd); } catch { /* cleanup */ }
+      try { fs.unlinkSync(lockPath); } catch { /* cleanup */ }
+    }
   }
+  throw lastErr;
 }
 
 function mutateJsonFileLocked(filePath, mutateFn, {
-  defaultValue = {}
+  defaultValue = {},
+  lockRetries,
+  lockRetryBackoffMs
 } = {}) {
   const lockPath = `${filePath}.lock`;
+  const retries = lockRetries ?? ENGINE_DEFAULTS.lockRetries;
+  const retryBackoffMs = lockRetryBackoffMs ?? ENGINE_DEFAULTS.lockRetryBackoffMs;
   return withFileLock(lockPath, () => {
     let data = safeJson(filePath);
     if (data === null || typeof data !== 'object') data = Array.isArray(defaultValue) ? [...defaultValue] : { ...defaultValue };
@@ -229,7 +248,7 @@ function mutateJsonFileLocked(filePath, mutateFn, {
     const finalData = next === undefined ? data : next;
     safeWrite(filePath, finalData);
     return finalData;
-  });
+  }, { retries, retryBackoffMs });
 }
 
 /**
@@ -479,6 +498,8 @@ const ENGINE_DEFAULTS = {
   versionCheckInterval: 3600000, // 1 hour — how often to check npm for updates (ms)
   logFlushInterval: 5000, // 5s — how often to flush buffered log entries to disk
   logBufferSize: 50, // flush immediately when buffer exceeds this many entries
+  lockRetries: 2, // retry lock acquisition this many times after initial timeout (total attempts = 1 + lockRetries)
+  lockRetryBackoffMs: 500, // base backoff between lock retries (doubles each attempt: 500ms, 1s, 2s, ...)
 };
 
 // ─── Status & Type Constants ─────────────────────────────────────────────────

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.493",
+  "version": "0.1.495",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"