@link-assistant/hive-mind 2.13.1 → 2.13.3

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/hive-mind
 
+## 2.13.3
+
+### Patch Changes
+
+- bf38253: Stop losing hours (and money) to retries of runs that already succeeded, and keep retrying a real provider outage for up to 12 hours. A gateway-error check matched any standalone `520`–`524` anywhere in the final message, so a solve run on issue #523 / PR #524 had each of its eleven _successful_ results re-classified as "Gateway error (502/504/52x)" and re-run — $7.65 and 3 h 55 min before `❌ Transient API error persisted after 10 retries`. HTTP status codes are now only recognised next to an error context (`API Error: 502`, `error code: 522`, `HTTP 504`) or a canonical gateway phrase, a run that Claude reported as successful is never retried (the near-miss is logged in verbose mode), and every tool's transient-retry loop is now driven by a wall-clock budget — 12 hours by default (`HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS`, `0` disables) with a 3-minute minimum delay (`HIVE_MIND_MIN_TRANSIENT_ERROR_DELAY_MS`) and a `3 → 30` min exponential backoff — instead of a fixed attempt count that gave up after ~3.5 hours. Full analysis in `docs/case-studies/issue-2169`.
+
+## 2.13.2
+
+### Patch Changes
+
+- 6b0435a: Make every Telegram message the bot sends visible. A single unpaired `_` in a repository name (`save_visiogetbb`) made `parse_mode: 'Markdown'` messages fail with `400: can't parse entities`, and because the plain-text fallback was installed on `bot.telegram` — while telegraf hands each handler a _different_ `Telegram` instance — the fallback never ran, so `/stop` cancelled the task but reported nothing. All sends now go through one funnel that validates the text against a port of TDLib's `parse_markdown()` before the call, logs every attempt/success/rejection, chunks at 4096 chars, and retries as plain text on any `400`; the funnel is re-installed on every per-update context, covers document captions, and a new `telegram-safety/no-unsafe-telegram-send` ESLint rule makes a raw `parse_mode` send a build error. Also: `/stop` and queue cards echo only the URL actually interpreted (no `#issuecomment-…` anchor), mentions no longer render a literal `\_`, and `/fix` rejects unsupported options (`--ci-de`) up front instead of silently spawning a wrong run.
+
 ## 2.13.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "2.13.1",
+  "version": "2.13.3",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/agent.lib.mjs

@@ -28,7 +28,7 @@ import { checkPlaywrightMcpPackageAvailability, getAgentPlaywrightMcpDisableEnv
 import { createAgentTokenUsage, accumulateAgentStepFinishUsage, parseAgentTokenUsage } from './agent-token-usage.lib.mjs';
 import { createJsonStreamScanner, parseJsonRecords } from './json-stream.lib.mjs';
 import { firstErrorText, stringifyErrorValue } from './error-text.lib.mjs';
-import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
+import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
 import { attachStreamingInput, finalizeBidirectionalHandler, setupBidirectionalHandler } from './bidirectional-interactive.lib.mjs';
 import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
 import { buildAgentArgs, detectFormalAiAgentRoutingMismatch, formatAgentArgsForDisplay, isAgentIdleEvent, isAgentStrongCompletionEvent } from './agent-command.lib.mjs';
@@ -571,13 +571,16 @@ export const executeAgentCommand = async params => {
 
   // Retry configuration
   let retryCount = 0;
+  // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
+  // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
+  const transientRetryBudget = createTransientRetryBudget();
 
   const executeWithRetry = async () => {
     // Execute agent command from the cloned repository directory
     if (retryCount === 0) {
       await log(`\n${formatAligned('🤖', 'Executing Agent:', argv.model.toUpperCase())}`);
     } else {
-      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
+      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
     }
 
     if (argv.verbose) {
@@ -986,7 +989,17 @@ export const executeAgentCommand = async params => {
         if (retryableError.isRetryable) {
           const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
           const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
-          if (retryCount < maxRetries) {
+          // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
+          // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
+          const retryDecision = transientRetryBudget.evaluate({
+            retryCount,
+            maxRetries,
+            initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
+            maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
+            minDelayMs: retryLimits.minTransientErrorDelayMs,
+          });
+          if (retryDecision.allowed) {
+            transientRetryBudget.grant();
             if (sessionId && !argv.resume) argv.resume = sessionId;
             // Issue #2037: retry the same model on capacity errors before falling back;
             // after a capacity-driven model switch, retry quickly instead of waiting the
@@ -999,17 +1012,18 @@ export const executeAgentCommand = async params => {
               retryCount,
               initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
               maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
+              minDelayMs: retryLimits.minTransientErrorDelayMs,
             });
             const delay = retryPlan.delay;
             const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
-            await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
+            await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
             await finalizeAgentBidirectionalHandler();
             await waitForRetryDelay(delay, log);
             await log('\n🔄 Retrying now...');
             retryCount++;
             return await executeWithRetry();
           }
-          await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
+          await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
         }
 
         // Build JSON error structure for consistent error reporting

package/src/buildUserMention.lib.mjs

@@ -1,3 +1,20 @@
+/**
+ * Make a display name safe to use as the label of a legacy-Markdown entity
+ * (`[label](url)`).
+ *
+ * Inside an entity TDLib copies bytes verbatim, so nothing needs escaping — but
+ * a literal `]` would terminate the entity early and turn the rest of the
+ * message into garbage. Those two delimiters are therefore dropped; `_` and `*`
+ * are deliberately left alone (issue #2166).
+ *
+ * @param {string} label - Raw display name.
+ * @returns {string} Label safe to embed between `[` and `]`.
+ */
+export function escapeMarkdownEntityLabel(label) {
+  if (!label || typeof label !== 'string') return label;
+  return label.replace(/[[\]]/g, '');
+}
+
 /**
  * Build a Telegram user mention link in various parse modes.
  *
@@ -42,9 +59,19 @@ export function buildUserMention({ user, id: idParam, username: usernameParam, f
   switch (parseMode) {
     case 'Markdown': {
       // Legacy Markdown: [text](url)
-      // Escape _ and * in display name to prevent "can't find end of entity" errors (issue #1460)
-      const escapedMarkdownName = displayName.replace(/_/g, '\\_').replace(/\*/g, '\\*');
-      return `[${escapedMarkdownName}](${link})`;
+      //
+      // Issue #2166: do NOT backslash-escape `_` / `*` here. TDLib's
+      // `parse_markdown()` only unescapes `\_ \* \` \[` at the *top level*; once it
+      // is inside an entity it copies bytes verbatim until the closing `]`:
+      //
+      //   while (i < size && text[i] != end_character) { … text[result_size++] = text[i++]; }
+      //
+      // So `[@my\_user](…)` renders the backslashes literally — that is the
+      // unpolished `\_` the issue reports. The label is already inside the entity,
+      // which is what actually prevents the "can't find end of entity" error from
+      // issue #1460; only the delimiters themselves are dangerous.
+      const labelName = escapeMarkdownEntityLabel(displayName);
+      return `[${labelName}](${link})`;
     }
     case 'MarkdownV2': {
       // MarkdownV2 requires escaping special characters

package/src/claude.connection.lib.mjs

@@ -5,7 +5,8 @@ if (typeof globalThis.use === 'undefined') {
 const { $ } = await use('command-stream');
 import { log } from './lib.mjs';
 import { reportError } from './sentry.lib.mjs';
-import { timeouts, getThinkingLevelToTokens, getTokensToThinkingLevel, supportsThinkingBudget, DEFAULT_MAX_THINKING_BUDGET } from './config.lib.mjs';
+import { timeouts, retryLimits, getThinkingLevelToTokens, getTokensToThinkingLevel, supportsThinkingBudget, DEFAULT_MAX_THINKING_BUDGET } from './config.lib.mjs';
+import { createTransientRetryBudget, waitWithCountdown } from './tool-retry.lib.mjs';
 import { buildAuthRemedyLines } from './formal-ai.lib.mjs';
 import { stringifyErrorValue } from './error-text.lib.mjs';
 import { mapModelToId } from './claude.model-utils.lib.mjs';
@@ -14,15 +15,39 @@ export const validateClaudeConnection = async (model = 'haiku') => {
   const mappedModel = mapModelToId(model);
   // Issue #2130: "run claude login" is wrong advice for a Formal-AI-served model.
   const authRemedyLines = buildAuthRemedyLines({ model, vendorRemedy: 'Please run: claude login' });
-  const maxRetries = 3;
-  const baseDelay = timeouts.retryBaseDelay;
+  // Issue #2169: a provider outage during validation used to abort the whole run after 3 quick
+  // retries (~seconds). Validation now shares the same wall-clock retry budget as execution —
+  // 12 h by default, 3-minute minimum wait, all configurable through HIVE_MIND_* env vars.
+  const maxRetries = retryLimits.maxTransientErrorRetries;
   let retryCount = 0;
+  const transientRetryBudget = createTransientRetryBudget();
+  // Returns true when a retry was performed (caller should recurse), false when the budget is spent.
+  const retryAfterOverload = async context => {
+    const retryDecision = transientRetryBudget.evaluate({
+      retryCount,
+      maxRetries,
+      initialDelayMs: retryLimits.initialTransientErrorDelayMs,
+      maxDelayMs: retryLimits.maxTransientErrorDelayMs,
+      minDelayMs: retryLimits.minTransientErrorDelayMs,
+    });
+    if (!retryDecision.allowed) {
+      await log(`❌ API overload error persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
+      await log('   The API appears to be heavily loaded. Please try again later.', { level: 'error' });
+      return false;
+    }
+    transientRetryBudget.grant();
+    const delayLabel = retryDecision.delayMs >= 60000 ? `${Math.round(retryDecision.delayMs / 60000)} min` : `${Math.round(retryDecision.delayMs / 1000)}s`;
+    await log(`⚠️ API overload error ${context}. Retrying in ${delayLabel} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
+    await waitWithCountdown(retryDecision.delayMs, log);
+    retryCount++;
+    return true;
+  };
   const attemptValidation = async () => {
     try {
       if (retryCount === 0) {
         await log('🔍 Validating Claude CLI connection...');
       } else {
-        await log(`🔄 Retry attempt ${retryCount}/${maxRetries} for Claude CLI validation...`);
+        await log(`🔄 Retry attempt ${retryCount} for Claude CLI validation (${transientRetryBudget.describeProgress()})...`);
       }
       try {
         const versionResult = await $`timeout ${Math.floor(timeouts.claudeCli / 6000)} claude --version`;
@@ -83,21 +108,8 @@ export const validateClaudeConnection = async (model = 'haiku') => {
       const jsonError = checkForJsonError(stdout) || checkForJsonError(stderr);
       const isOverloadError = (stdout.includes('API Error: 500') && stdout.includes('Overloaded')) || (stdout.includes('API Error: 529') && stdout.includes('Overloaded')) || (stderr.includes('API Error: 500') && stderr.includes('Overloaded')) || (stderr.includes('API Error: 529') && stderr.includes('Overloaded')) || (jsonError && (jsonError.type === 'api_error' || jsonError.type === 'overloaded_error') && jsonError.message === 'Overloaded');
       if (isOverloadError) {
-        if (retryCount < maxRetries) {
-          const delay = baseDelay * Math.pow(2, retryCount);
-          await log(`⚠️ API overload error during validation. Retrying in ${delay / 1000} seconds...`, {
-            level: 'warning',
-          });
-          await new Promise(resolve => setTimeout(resolve, delay));
-          retryCount++;
-          return await attemptValidation();
-        } else {
-          await log(`❌ API overload error persisted after ${maxRetries} retries during validation`, {
-            level: 'error',
-          });
-          await log('   The API appears to be heavily loaded. Please try again later.', { level: 'error' });
-          return false;
-        }
+        if (await retryAfterOverload('during validation')) return await attemptValidation();
+        return false;
       }
       const exitCode = result.code ?? result.exitCode ?? 0; // Bun shell compat
       if (exitCode !== 0) {
@@ -116,18 +128,8 @@ export const validateClaudeConnection = async (model = 'haiku') => {
       }
       if (jsonError) {
         if ((jsonError.type === 'api_error' || jsonError.type === 'overloaded_error') && jsonError.message === 'Overloaded') {
-          if (retryCount < maxRetries) {
-            const delay = baseDelay * Math.pow(2, retryCount);
-            await log(`⚠️ API overload error in response. Retrying in ${delay / 1000} seconds...`, {
-              level: 'warning',
-            });
-            await new Promise(resolve => setTimeout(resolve, delay));
-            retryCount++;
-            return await attemptValidation();
-          } else {
-            await log(`❌ API overload error persisted after ${maxRetries} retries`, { level: 'error' });
-            return false;
-          }
+          if (await retryAfterOverload('in response')) return await attemptValidation();
+          return false;
         }
         await log(`❌ Claude CLI returned error: ${jsonError.type} - ${jsonError.message}`, { level: 'error' });
         if (jsonError.type === 'forbidden') {
@@ -140,18 +142,8 @@ export const validateClaudeConnection = async (model = 'haiku') => {
     } catch (error) {
       const errorStr = error.message || error.toString();
       if ((errorStr.includes('API Error: 500') && errorStr.includes('Overloaded')) || (errorStr.includes('API Error: 529') && errorStr.includes('Overloaded')) || (errorStr.includes('api_error') && errorStr.includes('Overloaded')) || (errorStr.includes('overloaded_error') && errorStr.includes('Overloaded'))) {
-        if (retryCount < maxRetries) {
-          const delay = baseDelay * Math.pow(2, retryCount);
-          await log(`⚠️ API overload error during validation. Retrying in ${delay / 1000} seconds...`, {
-            level: 'warning',
-          });
-          await new Promise(resolve => setTimeout(resolve, delay));
-          retryCount++;
-          return await attemptValidation();
-        } else {
-          await log(`❌ API overload error persisted after ${maxRetries} retries`, { level: 'error' });
-          return false;
-        }
+        if (await retryAfterOverload('during validation')) return await attemptValidation();
+        return false;
       }
       await log(`❌ Failed to validate Claude CLI connection: ${error.message}`, { level: 'error' });
       await log('   💡 Make sure Claude CLI is installed and accessible', { level: 'error' });

package/src/claude.lib.mjs

@@ -28,7 +28,7 @@ import { buildMcpConfigWithoutPlaywright, ensureClaudePlaywrightMcpServer } from
 import { resolveClaudeSessionToolFlags } from './useless-tools.lib.mjs';
 import { ensureClaudeQuietConfig } from './claude-quiet-config.lib.mjs';
 import { fetchModelInfo } from './model-info.lib.mjs';
-import { classifyRetryableError, logExecutionContext, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
+import { classifyRetryableError, createTransientRetryBudget, describeClassificationEvidence, logExecutionContext, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
 import { resolveSubSessionSize } from './sub-session-size.lib.mjs'; // Issue #1706
 import { withAgentsMdAsClaudeMd } from './agents-md-claude-support.lib.mjs';
 import { deployHandoffSkill } from './handoff-skill.lib.mjs'; // Issue #1877
@@ -341,6 +341,9 @@ export const executeClaudeCommand = async params => {
   const escapePromptForShell = promptText => String(promptText).replace(/"/g, '\\"').replace(/\$/g, '\\$');
   await validateBidirectionalModeConfig(argv, log);
   let retryCount = 0;
+  // Issue #2169: total-time budget shared by every transient-error retry of this run (default
+  // 12 h). Created outside executeWithRetry so the elapsed clock survives the recursive calls.
+  const transientRetryBudget = createTransientRetryBudget();
   let baseBranchInterventionPrompt = null;
   let baseBranchInterventionResumeCount = 0;
   // Issue #1834 (PR #1835 feedback): corrupted-thinking-block recovery — resume the session first,
@@ -353,7 +356,8 @@ export const executeClaudeCommand = async params => {
     if (retryCount === 0) {
       await log(`\n${formatAligned('🤖', 'Executing Claude:', argv.model.toUpperCase())}`);
     } else {
-      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
+      // Issue #2169: the count cap is a backstop now, so report the retry budget instead.
+      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
     }
     if (argv.verbose) {
       // Issue #1949: logExecutionContext shows the requested alias with its resolved
@@ -1035,17 +1039,32 @@ export const executeClaudeCommand = async params => {
       }
       // Issues #1331, #1353, #1472/#1475: Unified transient error retry (exponential backoff, session preservation)
       const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || isRateLimitError || retryableLastError.isRetryable || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
+      // Issue #2169: a run that ended in a *successful* result event must never be retried.
+      // `lastMessage` holds the agent's own last text, so any prose that merely looks like an API
+      // error ("PR #524", "API Error: 503" quoted while working on an issue about it) used to flip
+      // `isTransientError` on and send a finished session into the retry loop. In the reported run
+      // all 11 attempts succeeded and were retried anyway, burning 3 h 54 min before the process
+      // exited 1 with the summary text presented as the error.
+      const runProducedSuccess = resultSuccessReceived && !commandFailed && !errorDuringExecution && exitCode === 0;
+      if (runProducedSuccess && isTransientError) {
+        await log(`🔍 Transient-error pattern seen in a successful run — not retrying (Issue #2169). Pattern: ${retryableLastError.label || 'flagged by stream detector'}; last message: ${JSON.stringify(lastMessage.substring(0, 200))}`, { verbose: true });
+        await log(`   Classification evidence: ${describeClassificationEvidence(lastMessage, retryableLastError.label)}`, { verbose: true });
+      }
       // Issue #2161: an account/subscription block short-circuits every retry
       // path. Stale transient flags from earlier in the run (an overload at hour
       // one, say) must not schedule a retry that is guaranteed to fail the same
       // way — and each retry would burn another full startup against a provider
       // that has already refused the credentials.
-      if ((commandFailed || isTransientError) && isTransientError && !subscriptionError) {
-        // Issue #1472/#1475: Startup/activity timeout → 30s–2min backoff; #1353: Request timeout → 5min–1hr; general → 2min–30min
+      if (!runProducedSuccess && isTransientError && !subscriptionError) {
+        // Issue #1472/#1475: Startup/activity timeout → 30s–2min backoff; #1353: Request timeout → 5min–1hr; general → 3min–30min
         const isTimeoutRetry = isStartupTimeout || isActivityTimeout;
-        const maxRetries = isTimeoutRetry ? retryLimits.maxTransientErrorRetries : isRequestTimeout ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
+        // Issue #2169: stream timeouts keep their own short count cap; API errors are governed by
+        // the 12-hour budget with a 3-minute floor on every wait.
+        const maxRetries = isTimeoutRetry ? retryLimits.maxStreamTimeoutRetries : isRequestTimeout ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
         const initialDelay = isTimeoutRetry ? 30000 : isRequestTimeout ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs;
         const maxDelay = isTimeoutRetry ? 120000 : isRequestTimeout ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs;
+        const minDelay = isTimeoutRetry ? 0 : retryLimits.minTransientErrorDelayMs;
+        const retryDecision = transientRetryBudget.evaluate({ retryCount, maxRetries, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
         // Issue #1437: Fail fast when API signals x-should-retry: false AND session made no progress
         const isStuckRetry = apiMarkedNotRetryable && retryCount >= retryLimits.maxNotRetryableAttempts && resultNumTurns <= 1;
         if (isStuckRetry) {
@@ -1072,18 +1091,22 @@ export const executeClaudeCommand = async params => {
             queuedFeedback, // Issue #817: Bidirectional mode feedback
           };
         }
-        if (retryCount < maxRetries) {
+        if (retryDecision.allowed) {
+          transientRetryBudget.grant();
           // Activity timeout preserves session (work was started), startup timeout does not (no session created)
           if (!isStartupTimeout && sessionId && !argv.resume) argv.resume = sessionId;
           // Issue #2037: retry same model on capacity errors before falling back; a switch retries fast.
-          const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: retryableLastError.message || lastMessage, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay });
+          const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: retryableLastError.message || lastMessage, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
           const delay = retryPlan.delay;
           const errorLabel = isStartupTimeout ? 'Stream startup timeout (Issue #1472/#1475)' : isActivityTimeout ? 'Stream activity timeout (Issue #1472)' : isRequestTimeout ? 'Request timeout' : retryableLastError.label || (isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('API Error: 529') && lastMessage.includes('Overloaded')) ? `API overload (${lastMessage.includes('529') ? '529' : '500'})` : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : isRateLimitError ? 'Server rate limited (429)' : '503 network error');
           const notRetryableHint = apiMarkedNotRetryable ? ' (API says not retryable — will stop early if no progress)' : '';
           const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
           const retryMode = isStartupTimeout ? ' (fresh start)' : ' (session preserved)';
-          await log(`\n⚠️ ${errorLabel} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${retryMode}${notRetryableHint}...`, { level: 'warning' });
+          await log(`\n⚠️ ${errorLabel} detected. Retry ${retryCount + 1} in ${delayLabel}${retryMode}${notRetryableHint} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
           await log(`   Error: ${isStartupTimeout ? `No output from Claude CLI within ${timeouts.streamStartupMs / 1000}s` : isActivityTimeout ? `No output for ${timeouts.streamActivityMs / 1000}s after previous activity` : lastMessage.substring(0, 200)}`, { verbose: true });
+          // Issue #2169: the 200-character excerpt above hid the token that actually triggered the
+          // classifier in the reported run, so also log where every status-looking token sits.
+          await log(`   Classification evidence: ${describeClassificationEvidence(retryableLastError.message || lastMessage, errorLabel)}`, { verbose: true });
           // Issue #1510: Post PR comment when force-killing and auto-resuming so reviewers can follow the session lifecycle
           if ((isActivityTimeout || isStartupTimeout) && owner && repo && prNumber && $) {
             try {
@@ -1102,7 +1125,10 @@ export const executeClaudeCommand = async params => {
           retryCount++;
           return await executeWithRetry();
         } else {
-          await log(`\n\n❌ Transient API error persisted after ${maxRetries} retries\n   Please try again later or check https://status.anthropic.com/`, { level: 'error' });
+          // Issue #2169: report *why* we stopped — count backstop vs 12-hour budget — and how long
+          // the run actually spent retrying, so an exhausted window is diagnosable from one line.
+          const exhaustionReason = transientRetryBudget.describeExhaustion(retryDecision);
+          await log(`\n\n❌ Transient API error persisted: ${exhaustionReason}\n   Please try again later or check https://status.anthropic.com/\n   Raise HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS to keep retrying for longer.`, { level: 'error' });
           // Issue #1886: fold captured cost so the carried-forward cost survives this retries-exhausted path.
           seedCumulativeAnthropicCost(argv.previousAnthropicCost);
           const cumulativeAnthropicCostUSDOnRetriesExhausted = addAnthropicRunCost(anthropicTotalCostUSD ?? anthropicCostFromAnyResult);
@@ -1118,7 +1144,7 @@ export const executeClaudeCommand = async params => {
             anthropicTotalCostUSD: cumulativeAnthropicCostUSDOnRetriesExhausted, // Issue #1104/#1886: Include cumulative cost even on failure
             resultSummary, // Issue #1263: Include result summary
             // Issue #1845/#1941: surface the actual error, rejecting meaningless fragments (e.g. a lone "}")
-            errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: `Transient API error persisted after ${maxRetries} retries`, toolLabel: 'Claude' }), exitCode },
+            errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: `Transient API error persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, toolLabel: 'Claude' }), exitCode },
             subscriptionError, // Issue #2161
             queuedFeedback, // Issue #817: Bidirectional mode feedback
           };
@@ -1253,19 +1279,24 @@ export const executeClaudeCommand = async params => {
         const maxRetries = isTimeoutException ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
         const initialDelay = isTimeoutException ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs;
         const maxDelay = isTimeoutException ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs;
-        if (retryCount < maxRetries) {
+        // Issue #2169: the same 12-hour budget with a 3-minute floor governs the exception path.
+        const minDelay = retryLimits.minTransientErrorDelayMs;
+        const retryDecision = transientRetryBudget.evaluate({ retryCount, maxRetries, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
+        if (retryDecision.allowed) {
+          transientRetryBudget.grant();
           if (sessionId && !argv.resume) argv.resume = sessionId;
           // Issue #2037: retry same model on capacity errors before falling back; a switch retries fast.
-          const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: errorStr, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay });
+          const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: errorStr, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
           const delay = retryPlan.delay;
           const errorLabel = isTimeoutException ? 'Request timeout' : retryableException.label || (errorStr.includes('Overloaded') ? `API overload (${errorStr.includes('529') ? '529' : '500'})` : errorStr.includes('Internal server error') ? 'Internal server error (500)' : '503 network error');
           const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
-          await log(`\n⚠️ ${errorLabel} in exception. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel} (session preserved)...`, { level: 'warning' });
+          await log(`\n⚠️ ${errorLabel} in exception. Retry ${retryCount + 1} in ${delayLabel} (session preserved, ${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
           await waitWithCountdown(delay, log);
           await log('\n🔄 Retrying now...');
           retryCount++;
           return await executeWithRetry();
         }
+        await log(`\n⏹️ Stopped retrying: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'warning' });
       }
       await log(`\n\n❌ Error executing Claude command: ${error.message}`, { level: 'error' });
       // Issue #1886: fold captured cost so the carried-forward cost survives this exception path too.

package/src/codex.lib.mjs

@@ -34,7 +34,7 @@ import { fetchModelInfo } from './model-info.lib.mjs';
 import { defaultModels, isFormalAiModel } from './models/index.mjs';
 import { buildAuthRemedyLines, buildFormalAiEnvExports, isPrepareOnly, logPreparedToolCommand, resolveFormalAiToolExecution } from './formal-ai.lib.mjs';
 import { buildFormalAiPricingInfo } from './formal-ai-pricing.lib.mjs'; // Issue #2119
-import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
+import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
 import { parseSubSessionSize, buildCodexSubSessionSizeConfigArgs, buildCodexDisable1mContextConfigArgs } from './sub-session-size.lib.mjs'; // Issue #1706
 import { getCumulativeContextInputTokens } from './context-fill.lib.mjs';
 import { deployHandoffSkill } from './handoff-skill.lib.mjs'; // Issue #1877
@@ -720,6 +720,9 @@ export const executeCodexCommand = async params => {
   const expectedBaseBranch = String(argv?.baseBranch || '').trim();
   // Retry configuration
   let retryCount = 0;
+  // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
+  // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
+  const transientRetryBudget = createTransientRetryBudget();
   let baseBranchInterventionPrompt = null;
   let baseBranchInterventionResumeCount = 0;
   const executeWithRetry = async () => {
@@ -727,7 +730,7 @@ export const executeCodexCommand = async params => {
     if (retryCount === 0) {
       await log(`\n${formatAligned('🤖', 'Executing Codex:', argv.model.toUpperCase())}`);
     } else {
-      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
+      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
     }
     if (argv.verbose) {
       await log(`   Model: ${argv.model}`, { verbose: true });
@@ -1103,20 +1106,30 @@ export const executeCodexCommand = async params => {
         } else if (retryableError.isRetryable) {
           const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
           const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
-          if (retryCount < maxRetries) {
+          // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
+          // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
+          const retryDecision = transientRetryBudget.evaluate({
+            retryCount,
+            maxRetries,
+            initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
+            maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
+            minDelayMs: retryLimits.minTransientErrorDelayMs,
+          });
+          if (retryDecision.allowed) {
+            transientRetryBudget.grant();
             if (sessionId && !argv.resume) argv.resume = sessionId;
             // Issue #2037: retry same model on capacity errors before falling back; a
             // capacity-driven switch retries fast, other transient errors use standard backoff.
-            const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs });
+            const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs, minDelayMs: retryLimits.minTransientErrorDelayMs });
             const delay = retryPlan.delay;
             const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
-            await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
+            await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
             await waitForRetryDelay(delay, log);
             await log('\n🔄 Retrying now...');
             retryCount++;
             return await executeWithRetry();
           }
-          await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
+          await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
         } else {
           await log(`\n\n❌ Codex emitted error event: ${codexErrorSummary.message}`, { level: 'error' });
           await log(`   Error events: item=${codexErrorSummary.counts.item}, turn=${codexErrorSummary.counts.turn}, stream=${codexErrorSummary.counts.stream}`, { level: 'error' });
@@ -1129,20 +1142,30 @@ export const executeCodexCommand = async params => {
         if (retryableError.isRetryable) {
           const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
           const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
-          if (retryCount < maxRetries) {
+          // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
+          // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
+          const retryDecision = transientRetryBudget.evaluate({
+            retryCount,
+            maxRetries,
+            initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
+            maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
+            minDelayMs: retryLimits.minTransientErrorDelayMs,
+          });
+          if (retryDecision.allowed) {
+            transientRetryBudget.grant();
             if (sessionId && !argv.resume) argv.resume = sessionId;
             // Issue #2037: retry same model on capacity errors before falling back; a
             // capacity-driven switch retries fast, other transient errors use standard backoff.
-            const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs });
+            const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs, minDelayMs: retryLimits.minTransientErrorDelayMs });
             const delay = retryPlan.delay;
             const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
-            await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
+            await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
             await waitForRetryDelay(delay, log);
             await log('\n🔄 Retrying now...');
             retryCount++;
             return await executeWithRetry();
           }
-          await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
+          await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
         }
         // Check for usage limit errors first (more specific)
         const limitInfo = detectUsageLimit(lastMessage);

package/src/config.lib.mjs

@@ -106,17 +106,34 @@ export const systemLimits = {
 };
 
 // Retry configurations
-// Issue #1331: All API error types use unified retry parameters:
-// 10 max retries, 2 minute initial delay, 30 minute max delay (exponential backoff), session preserved
+// Issue #1331: All API error types use unified retry parameters (exponential backoff, session preserved).
+// Issue #2169: the retry window is now governed by a *total time budget* (default 12 hours) rather
+// than by the retry count alone. A provider outage can last many hours; the previous 10-retry cap
+// with a 2-minute initial delay gave up after ~3.5 hours. The count caps below are kept as
+// backstops against runaway loops — the budget is the knob operators are expected to tune.
 export const retryLimits = {
   maxForkRetries: parseIntWithDefault('HIVE_MIND_MAX_FORK_RETRIES', 5),
   maxVerifyRetries: parseIntWithDefault('HIVE_MIND_MAX_VERIFY_RETRIES', 5),
   maxApiRetries: parseIntWithDefault('HIVE_MIND_MAX_API_RETRIES', 3),
   retryBackoffMultiplier: parseFloatWithDefault('HIVE_MIND_RETRY_BACKOFF_MULTIPLIER', 2),
   // Unified retry config for all transient API errors (Overloaded, 503, Internal Server Error)
-  maxTransientErrorRetries: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES', 10),
-  initialTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_TRANSIENT_ERROR_DELAY_MS', 2 * 60 * 1000), // 2 minutes
+  // Issue #2169: count backstop only. With the defaults below (3 min → 30 min backoff) the 12-hour
+  // budget is exhausted after ~26 retries, so this cap never fires unless an operator shortens the
+  // delays. Lower it (e.g. HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES=5) to fail faster than the budget.
+  maxTransientErrorRetries: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES', 100),
+  // Issue #2169: "minimum of 3 minutes" — the first (and therefore smallest) transient backoff.
+  initialTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_TRANSIENT_ERROR_DELAY_MS', 3 * 60 * 1000), // 3 minutes
   maxTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_DELAY_MS', 30 * 60 * 1000), // 30 minutes
+  // Issue #2169: total wall-clock budget for transient-API-error retries, measured from the first
+  // retry of a run. Retrying stops as soon as the *next* backoff would push the run past this
+  // window. Set to 0 to disable the budget and fall back to the count cap alone.
+  transientErrorRetryBudgetMs: parseIntWithDefault('HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS', 12 * 60 * 60 * 1000), // 12 hours
+  // Issue #2169: floor applied to every transient backoff, including operator-supplied initial
+  // delays. Keeps a misconfigured 5-second delay from hammering an API that is already struggling.
+  minTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_MIN_TRANSIENT_ERROR_DELAY_MS', 3 * 60 * 1000), // 3 minutes
+  // Issue #2169: stream startup/activity timeouts keep their own (short) count cap. They use 30s–2min
+  // backoffs and force-kill the CLI, so they must not inherit the 100-retry transient backstop.
+  maxStreamTimeoutRetries: parseIntWithDefault('HIVE_MIND_MAX_STREAM_TIMEOUT_RETRIES', 10),
   // Issue #2037: When a "model is at capacity" error triggers a switch to a *different*
   // fallback model, the long transient backoff is wasteful — the different model is
   // available now, so retry almost immediately instead of stalling for minutes.
@@ -131,7 +148,8 @@ export const retryLimits = {
   maxCapacityRetryDelayMs: parseIntWithDefault('HIVE_MIND_MAX_CAPACITY_RETRY_DELAY_MS', 4 * 60 * 1000), // 4 minutes
   // Request timeout retry configuration (Issue #1353)
   // Network timeouts need longer waits than API errors — Claude CLI already exhausted its own retries
-  maxRequestTimeoutRetries: parseIntWithDefault('HIVE_MIND_MAX_REQUEST_TIMEOUT_RETRIES', 10),
+  // Issue #2169: count backstop only — the shared 12-hour budget governs when to stop.
+  maxRequestTimeoutRetries: parseIntWithDefault('HIVE_MIND_MAX_REQUEST_TIMEOUT_RETRIES', 100),
   initialRequestTimeoutDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_REQUEST_TIMEOUT_DELAY_MS', 5 * 60 * 1000), // 5 minutes
   maxRequestTimeoutDelayMs: parseIntWithDefault('HIVE_MIND_MAX_REQUEST_TIMEOUT_DELAY_MS', 60 * 60 * 1000), // 1 hour
   // Not-retryable error fail-fast configuration (Issue #1437)