@link-assistant/hive-mind 2.13.2 → 2.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # @link-assistant/hive-mind
2
2
 
3
+ ## 2.13.4
4
+
5
+ ### Patch Changes
6
+
7
+ - 36b0ff9: Stop `solve` from crashing with `TypeError: A GitHub pull request URL requires owner, repo, and a positive integer number` when `--auto-continue` resumes an `issue-<n>-<hash>` branch that has no pull request yet. Continue mode has always had two shapes — resume an existing pull request, or reuse a leftover branch from an interrupted run (`prNumber: null`) — and since #2158 the "Your prepared Pull Request" URL was built from `prNumber` unconditionally, so the second shape aborted the run right after branch checkout, one step before `handleAutoPrCreation()` would have created the missing pull request. The URL is now built through a nullable `buildGitHubPullRequestUrlOrNull()` helper, the run logs that the pull request is still pending, and the strict builder reports the values it received so a bare stack trace is actionable. Full analysis in `docs/case-studies/issue-2170/`.
8
+
9
+ ## 2.13.3
10
+
11
+ ### Patch Changes
12
+
13
+ - bf38253: Stop losing hours (and money) to retries of runs that already succeeded, and keep retrying a real provider outage for up to 12 hours. A gateway-error check matched any standalone `520`–`524` anywhere in the final message, so a solve run on issue #523 / PR #524 had each of its eleven _successful_ results re-classified as "Gateway error (502/504/52x)" and re-run — $7.65 and 3 h 55 min before `❌ Transient API error persisted after 10 retries`. HTTP status codes are now only recognised next to an error context (`API Error: 502`, `error code: 522`, `HTTP 504`) or a canonical gateway phrase, a run that Claude reported as successful is never retried (the near-miss is logged in verbose mode), and every tool's transient-retry loop is now driven by a wall-clock budget — 12 hours by default (`HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS`, `0` disables) with a 3-minute minimum delay (`HIVE_MIND_MIN_TRANSIENT_ERROR_DELAY_MS`) and a `3 → 30` min exponential backoff — instead of a fixed attempt count that gave up after ~3.5 hours. Full analysis in `docs/case-studies/issue-2169`.
14
+
3
15
  ## 2.13.2
4
16
 
5
17
  ### Patch Changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-assistant/hive-mind",
3
- "version": "2.13.2",
3
+ "version": "2.13.4",
4
4
  "description": "AI-powered issue solver and hive mind for collaborative problem solving",
5
5
  "main": "src/hive.mjs",
6
6
  "type": "module",
package/src/agent.lib.mjs CHANGED
@@ -28,7 +28,7 @@ import { checkPlaywrightMcpPackageAvailability, getAgentPlaywrightMcpDisableEnv
28
28
  import { createAgentTokenUsage, accumulateAgentStepFinishUsage, parseAgentTokenUsage } from './agent-token-usage.lib.mjs';
29
29
  import { createJsonStreamScanner, parseJsonRecords } from './json-stream.lib.mjs';
30
30
  import { firstErrorText, stringifyErrorValue } from './error-text.lib.mjs';
31
- import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
31
+ import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
32
32
  import { attachStreamingInput, finalizeBidirectionalHandler, setupBidirectionalHandler } from './bidirectional-interactive.lib.mjs';
33
33
  import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
34
34
  import { buildAgentArgs, detectFormalAiAgentRoutingMismatch, formatAgentArgsForDisplay, isAgentIdleEvent, isAgentStrongCompletionEvent } from './agent-command.lib.mjs';
@@ -571,13 +571,16 @@ export const executeAgentCommand = async params => {
571
571
 
572
572
  // Retry configuration
573
573
  let retryCount = 0;
574
+ // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
575
+ // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
576
+ const transientRetryBudget = createTransientRetryBudget();
574
577
 
575
578
  const executeWithRetry = async () => {
576
579
  // Execute agent command from the cloned repository directory
577
580
  if (retryCount === 0) {
578
581
  await log(`\n${formatAligned('🤖', 'Executing Agent:', argv.model.toUpperCase())}`);
579
582
  } else {
580
- await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
583
+ await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
581
584
  }
582
585
 
583
586
  if (argv.verbose) {
@@ -986,7 +989,17 @@ export const executeAgentCommand = async params => {
986
989
  if (retryableError.isRetryable) {
987
990
  const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
988
991
  const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
989
- if (retryCount < maxRetries) {
992
+ // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
993
+ // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
994
+ const retryDecision = transientRetryBudget.evaluate({
995
+ retryCount,
996
+ maxRetries,
997
+ initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
998
+ maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
999
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
1000
+ });
1001
+ if (retryDecision.allowed) {
1002
+ transientRetryBudget.grant();
990
1003
  if (sessionId && !argv.resume) argv.resume = sessionId;
991
1004
  // Issue #2037: retry the same model on capacity errors before falling back;
992
1005
  // after a capacity-driven model switch, retry quickly instead of waiting the
@@ -999,17 +1012,18 @@ export const executeAgentCommand = async params => {
999
1012
  retryCount,
1000
1013
  initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
1001
1014
  maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
1015
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
1002
1016
  });
1003
1017
  const delay = retryPlan.delay;
1004
1018
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
1005
- await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
1019
+ await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
1006
1020
  await finalizeAgentBidirectionalHandler();
1007
1021
  await waitForRetryDelay(delay, log);
1008
1022
  await log('\n🔄 Retrying now...');
1009
1023
  retryCount++;
1010
1024
  return await executeWithRetry();
1011
1025
  }
1012
- await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
1026
+ await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
1013
1027
  }
1014
1028
 
1015
1029
  // Build JSON error structure for consistent error reporting
@@ -5,7 +5,8 @@ if (typeof globalThis.use === 'undefined') {
5
5
  const { $ } = await use('command-stream');
6
6
  import { log } from './lib.mjs';
7
7
  import { reportError } from './sentry.lib.mjs';
8
- import { timeouts, getThinkingLevelToTokens, getTokensToThinkingLevel, supportsThinkingBudget, DEFAULT_MAX_THINKING_BUDGET } from './config.lib.mjs';
8
+ import { timeouts, retryLimits, getThinkingLevelToTokens, getTokensToThinkingLevel, supportsThinkingBudget, DEFAULT_MAX_THINKING_BUDGET } from './config.lib.mjs';
9
+ import { createTransientRetryBudget, waitWithCountdown } from './tool-retry.lib.mjs';
9
10
  import { buildAuthRemedyLines } from './formal-ai.lib.mjs';
10
11
  import { stringifyErrorValue } from './error-text.lib.mjs';
11
12
  import { mapModelToId } from './claude.model-utils.lib.mjs';
@@ -14,15 +15,39 @@ export const validateClaudeConnection = async (model = 'haiku') => {
14
15
  const mappedModel = mapModelToId(model);
15
16
  // Issue #2130: "run claude login" is wrong advice for a Formal-AI-served model.
16
17
  const authRemedyLines = buildAuthRemedyLines({ model, vendorRemedy: 'Please run: claude login' });
17
- const maxRetries = 3;
18
- const baseDelay = timeouts.retryBaseDelay;
18
+ // Issue #2169: a provider outage during validation used to abort the whole run after 3 quick
19
+ // retries (~seconds). Validation now shares the same wall-clock retry budget as execution —
20
+ // 12 h by default, 3-minute minimum wait, all configurable through HIVE_MIND_* env vars.
21
+ const maxRetries = retryLimits.maxTransientErrorRetries;
19
22
  let retryCount = 0;
23
+ const transientRetryBudget = createTransientRetryBudget();
24
+ // Returns true when a retry was performed (caller should recurse), false when the budget is spent.
25
+ const retryAfterOverload = async context => {
26
+ const retryDecision = transientRetryBudget.evaluate({
27
+ retryCount,
28
+ maxRetries,
29
+ initialDelayMs: retryLimits.initialTransientErrorDelayMs,
30
+ maxDelayMs: retryLimits.maxTransientErrorDelayMs,
31
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
32
+ });
33
+ if (!retryDecision.allowed) {
34
+ await log(`❌ API overload error persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
35
+ await log(' The API appears to be heavily loaded. Please try again later.', { level: 'error' });
36
+ return false;
37
+ }
38
+ transientRetryBudget.grant();
39
+ const delayLabel = retryDecision.delayMs >= 60000 ? `${Math.round(retryDecision.delayMs / 60000)} min` : `${Math.round(retryDecision.delayMs / 1000)}s`;
40
+ await log(`⚠️ API overload error ${context}. Retrying in ${delayLabel} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
41
+ await waitWithCountdown(retryDecision.delayMs, log);
42
+ retryCount++;
43
+ return true;
44
+ };
20
45
  const attemptValidation = async () => {
21
46
  try {
22
47
  if (retryCount === 0) {
23
48
  await log('🔍 Validating Claude CLI connection...');
24
49
  } else {
25
- await log(`🔄 Retry attempt ${retryCount}/${maxRetries} for Claude CLI validation...`);
50
+ await log(`🔄 Retry attempt ${retryCount} for Claude CLI validation (${transientRetryBudget.describeProgress()})...`);
26
51
  }
27
52
  try {
28
53
  const versionResult = await $`timeout ${Math.floor(timeouts.claudeCli / 6000)} claude --version`;
@@ -83,21 +108,8 @@ export const validateClaudeConnection = async (model = 'haiku') => {
83
108
  const jsonError = checkForJsonError(stdout) || checkForJsonError(stderr);
84
109
  const isOverloadError = (stdout.includes('API Error: 500') && stdout.includes('Overloaded')) || (stdout.includes('API Error: 529') && stdout.includes('Overloaded')) || (stderr.includes('API Error: 500') && stderr.includes('Overloaded')) || (stderr.includes('API Error: 529') && stderr.includes('Overloaded')) || (jsonError && (jsonError.type === 'api_error' || jsonError.type === 'overloaded_error') && jsonError.message === 'Overloaded');
85
110
  if (isOverloadError) {
86
- if (retryCount < maxRetries) {
87
- const delay = baseDelay * Math.pow(2, retryCount);
88
- await log(`⚠️ API overload error during validation. Retrying in ${delay / 1000} seconds...`, {
89
- level: 'warning',
90
- });
91
- await new Promise(resolve => setTimeout(resolve, delay));
92
- retryCount++;
93
- return await attemptValidation();
94
- } else {
95
- await log(`❌ API overload error persisted after ${maxRetries} retries during validation`, {
96
- level: 'error',
97
- });
98
- await log(' The API appears to be heavily loaded. Please try again later.', { level: 'error' });
99
- return false;
100
- }
111
+ if (await retryAfterOverload('during validation')) return await attemptValidation();
112
+ return false;
101
113
  }
102
114
  const exitCode = result.code ?? result.exitCode ?? 0; // Bun shell compat
103
115
  if (exitCode !== 0) {
@@ -116,18 +128,8 @@ export const validateClaudeConnection = async (model = 'haiku') => {
116
128
  }
117
129
  if (jsonError) {
118
130
  if ((jsonError.type === 'api_error' || jsonError.type === 'overloaded_error') && jsonError.message === 'Overloaded') {
119
- if (retryCount < maxRetries) {
120
- const delay = baseDelay * Math.pow(2, retryCount);
121
- await log(`⚠️ API overload error in response. Retrying in ${delay / 1000} seconds...`, {
122
- level: 'warning',
123
- });
124
- await new Promise(resolve => setTimeout(resolve, delay));
125
- retryCount++;
126
- return await attemptValidation();
127
- } else {
128
- await log(`❌ API overload error persisted after ${maxRetries} retries`, { level: 'error' });
129
- return false;
130
- }
131
+ if (await retryAfterOverload('in response')) return await attemptValidation();
132
+ return false;
131
133
  }
132
134
  await log(`❌ Claude CLI returned error: ${jsonError.type} - ${jsonError.message}`, { level: 'error' });
133
135
  if (jsonError.type === 'forbidden') {
@@ -140,18 +142,8 @@ export const validateClaudeConnection = async (model = 'haiku') => {
140
142
  } catch (error) {
141
143
  const errorStr = error.message || error.toString();
142
144
  if ((errorStr.includes('API Error: 500') && errorStr.includes('Overloaded')) || (errorStr.includes('API Error: 529') && errorStr.includes('Overloaded')) || (errorStr.includes('api_error') && errorStr.includes('Overloaded')) || (errorStr.includes('overloaded_error') && errorStr.includes('Overloaded'))) {
143
- if (retryCount < maxRetries) {
144
- const delay = baseDelay * Math.pow(2, retryCount);
145
- await log(`⚠️ API overload error during validation. Retrying in ${delay / 1000} seconds...`, {
146
- level: 'warning',
147
- });
148
- await new Promise(resolve => setTimeout(resolve, delay));
149
- retryCount++;
150
- return await attemptValidation();
151
- } else {
152
- await log(`❌ API overload error persisted after ${maxRetries} retries`, { level: 'error' });
153
- return false;
154
- }
145
+ if (await retryAfterOverload('during validation')) return await attemptValidation();
146
+ return false;
155
147
  }
156
148
  await log(`❌ Failed to validate Claude CLI connection: ${error.message}`, { level: 'error' });
157
149
  await log(' 💡 Make sure Claude CLI is installed and accessible', { level: 'error' });
@@ -28,7 +28,7 @@ import { buildMcpConfigWithoutPlaywright, ensureClaudePlaywrightMcpServer } from
28
28
  import { resolveClaudeSessionToolFlags } from './useless-tools.lib.mjs';
29
29
  import { ensureClaudeQuietConfig } from './claude-quiet-config.lib.mjs';
30
30
  import { fetchModelInfo } from './model-info.lib.mjs';
31
- import { classifyRetryableError, logExecutionContext, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
31
+ import { classifyRetryableError, createTransientRetryBudget, describeClassificationEvidence, logExecutionContext, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
32
32
  import { resolveSubSessionSize } from './sub-session-size.lib.mjs'; // Issue #1706
33
33
  import { withAgentsMdAsClaudeMd } from './agents-md-claude-support.lib.mjs';
34
34
  import { deployHandoffSkill } from './handoff-skill.lib.mjs'; // Issue #1877
@@ -341,6 +341,9 @@ export const executeClaudeCommand = async params => {
341
341
  const escapePromptForShell = promptText => String(promptText).replace(/"/g, '\\"').replace(/\$/g, '\\$');
342
342
  await validateBidirectionalModeConfig(argv, log);
343
343
  let retryCount = 0;
344
+ // Issue #2169: total-time budget shared by every transient-error retry of this run (default
345
+ // 12 h). Created outside executeWithRetry so the elapsed clock survives the recursive calls.
346
+ const transientRetryBudget = createTransientRetryBudget();
344
347
  let baseBranchInterventionPrompt = null;
345
348
  let baseBranchInterventionResumeCount = 0;
346
349
  // Issue #1834 (PR #1835 feedback): corrupted-thinking-block recovery — resume the session first,
@@ -353,7 +356,8 @@ export const executeClaudeCommand = async params => {
353
356
  if (retryCount === 0) {
354
357
  await log(`\n${formatAligned('🤖', 'Executing Claude:', argv.model.toUpperCase())}`);
355
358
  } else {
356
- await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
359
+ // Issue #2169: the count cap is a backstop now, so report the retry budget instead.
360
+ await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
357
361
  }
358
362
  if (argv.verbose) {
359
363
  // Issue #1949: logExecutionContext shows the requested alias with its resolved
@@ -1035,17 +1039,32 @@ export const executeClaudeCommand = async params => {
1035
1039
  }
1036
1040
  // Issues #1331, #1353, #1472/#1475: Unified transient error retry (exponential backoff, session preservation)
1037
1041
  const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || isRateLimitError || retryableLastError.isRetryable || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
1042
+ // Issue #2169: a run that ended in a *successful* result event must never be retried.
1043
+ // `lastMessage` holds the agent's own last text, so any prose that merely looks like an API
1044
+ // error ("PR #524", "API Error: 503" quoted while working on an issue about it) used to flip
1045
+ // `isTransientError` on and send a finished session into the retry loop. In the reported run
1046
+ // all 11 attempts succeeded and were retried anyway, burning 3 h 54 min before the process
1047
+ // exited 1 with the summary text presented as the error.
1048
+ const runProducedSuccess = resultSuccessReceived && !commandFailed && !errorDuringExecution && exitCode === 0;
1049
+ if (runProducedSuccess && isTransientError) {
1050
+ await log(`🔍 Transient-error pattern seen in a successful run — not retrying (Issue #2169). Pattern: ${retryableLastError.label || 'flagged by stream detector'}; last message: ${JSON.stringify(lastMessage.substring(0, 200))}`, { verbose: true });
1051
+ await log(` Classification evidence: ${describeClassificationEvidence(lastMessage, retryableLastError.label)}`, { verbose: true });
1052
+ }
1038
1053
  // Issue #2161: an account/subscription block short-circuits every retry
1039
1054
  // path. Stale transient flags from earlier in the run (an overload at hour
1040
1055
  // one, say) must not schedule a retry that is guaranteed to fail the same
1041
1056
  // way — and each retry would burn another full startup against a provider
1042
1057
  // that has already refused the credentials.
1043
- if ((commandFailed || isTransientError) && isTransientError && !subscriptionError) {
1044
- // Issue #1472/#1475: Startup/activity timeout → 30s–2min backoff; #1353: Request timeout → 5min–1hr; general → 2min–30min
1058
+ if (!runProducedSuccess && isTransientError && !subscriptionError) {
1059
+ // Issue #1472/#1475: Startup/activity timeout → 30s–2min backoff; #1353: Request timeout → 5min–1hr; general → 3min–30min
1045
1060
  const isTimeoutRetry = isStartupTimeout || isActivityTimeout;
1046
- const maxRetries = isTimeoutRetry ? retryLimits.maxTransientErrorRetries : isRequestTimeout ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
1061
+ // Issue #2169: stream timeouts keep their own short count cap; API errors are governed by
1062
+ // the 12-hour budget with a 3-minute floor on every wait.
1063
+ const maxRetries = isTimeoutRetry ? retryLimits.maxStreamTimeoutRetries : isRequestTimeout ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
1047
1064
  const initialDelay = isTimeoutRetry ? 30000 : isRequestTimeout ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs;
1048
1065
  const maxDelay = isTimeoutRetry ? 120000 : isRequestTimeout ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs;
1066
+ const minDelay = isTimeoutRetry ? 0 : retryLimits.minTransientErrorDelayMs;
1067
+ const retryDecision = transientRetryBudget.evaluate({ retryCount, maxRetries, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
1049
1068
  // Issue #1437: Fail fast when API signals x-should-retry: false AND session made no progress
1050
1069
  const isStuckRetry = apiMarkedNotRetryable && retryCount >= retryLimits.maxNotRetryableAttempts && resultNumTurns <= 1;
1051
1070
  if (isStuckRetry) {
@@ -1072,18 +1091,22 @@ export const executeClaudeCommand = async params => {
1072
1091
  queuedFeedback, // Issue #817: Bidirectional mode feedback
1073
1092
  };
1074
1093
  }
1075
- if (retryCount < maxRetries) {
1094
+ if (retryDecision.allowed) {
1095
+ transientRetryBudget.grant();
1076
1096
  // Activity timeout preserves session (work was started), startup timeout does not (no session created)
1077
1097
  if (!isStartupTimeout && sessionId && !argv.resume) argv.resume = sessionId;
1078
1098
  // Issue #2037: retry same model on capacity errors before falling back; a switch retries fast.
1079
- const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: retryableLastError.message || lastMessage, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay });
1099
+ const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: retryableLastError.message || lastMessage, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
1080
1100
  const delay = retryPlan.delay;
1081
1101
  const errorLabel = isStartupTimeout ? 'Stream startup timeout (Issue #1472/#1475)' : isActivityTimeout ? 'Stream activity timeout (Issue #1472)' : isRequestTimeout ? 'Request timeout' : retryableLastError.label || (isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('API Error: 529') && lastMessage.includes('Overloaded')) ? `API overload (${lastMessage.includes('529') ? '529' : '500'})` : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : isRateLimitError ? 'Server rate limited (429)' : '503 network error');
1082
1102
  const notRetryableHint = apiMarkedNotRetryable ? ' (API says not retryable — will stop early if no progress)' : '';
1083
1103
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
1084
1104
  const retryMode = isStartupTimeout ? ' (fresh start)' : ' (session preserved)';
1085
- await log(`\n⚠️ ${errorLabel} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${retryMode}${notRetryableHint}...`, { level: 'warning' });
1105
+ await log(`\n⚠️ ${errorLabel} detected. Retry ${retryCount + 1} in ${delayLabel}${retryMode}${notRetryableHint} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
1086
1106
  await log(` Error: ${isStartupTimeout ? `No output from Claude CLI within ${timeouts.streamStartupMs / 1000}s` : isActivityTimeout ? `No output for ${timeouts.streamActivityMs / 1000}s after previous activity` : lastMessage.substring(0, 200)}`, { verbose: true });
1107
+ // Issue #2169: the 200-character excerpt above hid the token that actually triggered the
1108
+ // classifier in the reported run, so also log where every status-looking token sits.
1109
+ await log(` Classification evidence: ${describeClassificationEvidence(retryableLastError.message || lastMessage, errorLabel)}`, { verbose: true });
1087
1110
  // Issue #1510: Post PR comment when force-killing and auto-resuming so reviewers can follow the session lifecycle
1088
1111
  if ((isActivityTimeout || isStartupTimeout) && owner && repo && prNumber && $) {
1089
1112
  try {
@@ -1102,7 +1125,10 @@ export const executeClaudeCommand = async params => {
1102
1125
  retryCount++;
1103
1126
  return await executeWithRetry();
1104
1127
  } else {
1105
- await log(`\n\n❌ Transient API error persisted after ${maxRetries} retries\n Please try again later or check https://status.anthropic.com/`, { level: 'error' });
1128
+ // Issue #2169: report *why* we stopped count backstop vs 12-hour budget and how long
1129
+ // the run actually spent retrying, so an exhausted window is diagnosable from one line.
1130
+ const exhaustionReason = transientRetryBudget.describeExhaustion(retryDecision);
1131
+ await log(`\n\n❌ Transient API error persisted: ${exhaustionReason}\n Please try again later or check https://status.anthropic.com/\n Raise HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS to keep retrying for longer.`, { level: 'error' });
1106
1132
  // Issue #1886: fold captured cost so the carried-forward cost survives this retries-exhausted path.
1107
1133
  seedCumulativeAnthropicCost(argv.previousAnthropicCost);
1108
1134
  const cumulativeAnthropicCostUSDOnRetriesExhausted = addAnthropicRunCost(anthropicTotalCostUSD ?? anthropicCostFromAnyResult);
@@ -1118,7 +1144,7 @@ export const executeClaudeCommand = async params => {
1118
1144
  anthropicTotalCostUSD: cumulativeAnthropicCostUSDOnRetriesExhausted, // Issue #1104/#1886: Include cumulative cost even on failure
1119
1145
  resultSummary, // Issue #1263: Include result summary
1120
1146
  // Issue #1845/#1941: surface the actual error, rejecting meaningless fragments (e.g. a lone "}")
1121
- errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: `Transient API error persisted after ${maxRetries} retries`, toolLabel: 'Claude' }), exitCode },
1147
+ errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: `Transient API error persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, toolLabel: 'Claude' }), exitCode },
1122
1148
  subscriptionError, // Issue #2161
1123
1149
  queuedFeedback, // Issue #817: Bidirectional mode feedback
1124
1150
  };
@@ -1253,19 +1279,24 @@ export const executeClaudeCommand = async params => {
1253
1279
  const maxRetries = isTimeoutException ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
1254
1280
  const initialDelay = isTimeoutException ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs;
1255
1281
  const maxDelay = isTimeoutException ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs;
1256
- if (retryCount < maxRetries) {
1282
+ // Issue #2169: the same 12-hour budget with a 3-minute floor governs the exception path.
1283
+ const minDelay = retryLimits.minTransientErrorDelayMs;
1284
+ const retryDecision = transientRetryBudget.evaluate({ retryCount, maxRetries, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
1285
+ if (retryDecision.allowed) {
1286
+ transientRetryBudget.grant();
1257
1287
  if (sessionId && !argv.resume) argv.resume = sessionId;
1258
1288
  // Issue #2037: retry same model on capacity errors before falling back; a switch retries fast.
1259
- const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: errorStr, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay });
1289
+ const retryPlan = await prepareRetryAfterError({ tool: 'claude', argv, log, errorMessage: errorStr, retryCount, initialDelayMs: initialDelay, maxDelayMs: maxDelay, minDelayMs: minDelay });
1260
1290
  const delay = retryPlan.delay;
1261
1291
  const errorLabel = isTimeoutException ? 'Request timeout' : retryableException.label || (errorStr.includes('Overloaded') ? `API overload (${errorStr.includes('529') ? '529' : '500'})` : errorStr.includes('Internal server error') ? 'Internal server error (500)' : '503 network error');
1262
1292
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
1263
- await log(`\n⚠️ ${errorLabel} in exception. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel} (session preserved)...`, { level: 'warning' });
1293
+ await log(`\n⚠️ ${errorLabel} in exception. Retry ${retryCount + 1} in ${delayLabel} (session preserved, ${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
1264
1294
  await waitWithCountdown(delay, log);
1265
1295
  await log('\n🔄 Retrying now...');
1266
1296
  retryCount++;
1267
1297
  return await executeWithRetry();
1268
1298
  }
1299
+ await log(`\n⏹️ Stopped retrying: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'warning' });
1269
1300
  }
1270
1301
  await log(`\n\n❌ Error executing Claude command: ${error.message}`, { level: 'error' });
1271
1302
  // Issue #1886: fold captured cost so the carried-forward cost survives this exception path too.
package/src/codex.lib.mjs CHANGED
@@ -34,7 +34,7 @@ import { fetchModelInfo } from './model-info.lib.mjs';
34
34
  import { defaultModels, isFormalAiModel } from './models/index.mjs';
35
35
  import { buildAuthRemedyLines, buildFormalAiEnvExports, isPrepareOnly, logPreparedToolCommand, resolveFormalAiToolExecution } from './formal-ai.lib.mjs';
36
36
  import { buildFormalAiPricingInfo } from './formal-ai-pricing.lib.mjs'; // Issue #2119
37
- import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
37
+ import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
38
38
  import { parseSubSessionSize, buildCodexSubSessionSizeConfigArgs, buildCodexDisable1mContextConfigArgs } from './sub-session-size.lib.mjs'; // Issue #1706
39
39
  import { getCumulativeContextInputTokens } from './context-fill.lib.mjs';
40
40
  import { deployHandoffSkill } from './handoff-skill.lib.mjs'; // Issue #1877
@@ -720,6 +720,9 @@ export const executeCodexCommand = async params => {
720
720
  const expectedBaseBranch = String(argv?.baseBranch || '').trim();
721
721
  // Retry configuration
722
722
  let retryCount = 0;
723
+ // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
724
+ // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
725
+ const transientRetryBudget = createTransientRetryBudget();
723
726
  let baseBranchInterventionPrompt = null;
724
727
  let baseBranchInterventionResumeCount = 0;
725
728
  const executeWithRetry = async () => {
@@ -727,7 +730,7 @@ export const executeCodexCommand = async params => {
727
730
  if (retryCount === 0) {
728
731
  await log(`\n${formatAligned('🤖', 'Executing Codex:', argv.model.toUpperCase())}`);
729
732
  } else {
730
- await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
733
+ await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
731
734
  }
732
735
  if (argv.verbose) {
733
736
  await log(` Model: ${argv.model}`, { verbose: true });
@@ -1103,20 +1106,30 @@ export const executeCodexCommand = async params => {
1103
1106
  } else if (retryableError.isRetryable) {
1104
1107
  const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
1105
1108
  const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
1106
- if (retryCount < maxRetries) {
1109
+ // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
1110
+ // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
1111
+ const retryDecision = transientRetryBudget.evaluate({
1112
+ retryCount,
1113
+ maxRetries,
1114
+ initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
1115
+ maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
1116
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
1117
+ });
1118
+ if (retryDecision.allowed) {
1119
+ transientRetryBudget.grant();
1107
1120
  if (sessionId && !argv.resume) argv.resume = sessionId;
1108
1121
  // Issue #2037: retry same model on capacity errors before falling back; a
1109
1122
  // capacity-driven switch retries fast, other transient errors use standard backoff.
1110
- const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs });
1123
+ const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs, minDelayMs: retryLimits.minTransientErrorDelayMs });
1111
1124
  const delay = retryPlan.delay;
1112
1125
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
1113
- await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
1126
+ await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
1114
1127
  await waitForRetryDelay(delay, log);
1115
1128
  await log('\n🔄 Retrying now...');
1116
1129
  retryCount++;
1117
1130
  return await executeWithRetry();
1118
1131
  }
1119
- await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
1132
+ await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
1120
1133
  } else {
1121
1134
  await log(`\n\n❌ Codex emitted error event: ${codexErrorSummary.message}`, { level: 'error' });
1122
1135
  await log(` Error events: item=${codexErrorSummary.counts.item}, turn=${codexErrorSummary.counts.turn}, stream=${codexErrorSummary.counts.stream}`, { level: 'error' });
@@ -1129,20 +1142,30 @@ export const executeCodexCommand = async params => {
1129
1142
  if (retryableError.isRetryable) {
1130
1143
  const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
1131
1144
  const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
1132
- if (retryCount < maxRetries) {
1145
+ // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
1146
+ // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
1147
+ const retryDecision = transientRetryBudget.evaluate({
1148
+ retryCount,
1149
+ maxRetries,
1150
+ initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
1151
+ maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
1152
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
1153
+ });
1154
+ if (retryDecision.allowed) {
1155
+ transientRetryBudget.grant();
1133
1156
  if (sessionId && !argv.resume) argv.resume = sessionId;
1134
1157
  // Issue #2037: retry same model on capacity errors before falling back; a
1135
1158
  // capacity-driven switch retries fast, other transient errors use standard backoff.
1136
- const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs });
1159
+ const retryPlan = await prepareRetryAfterError({ tool: 'codex', argv, log, errorMessage: retryableError.message, retryCount, initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs, maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs, minDelayMs: retryLimits.minTransientErrorDelayMs });
1137
1160
  const delay = retryPlan.delay;
1138
1161
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
1139
- await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
1162
+ await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
1140
1163
  await waitForRetryDelay(delay, log);
1141
1164
  await log('\n🔄 Retrying now...');
1142
1165
  retryCount++;
1143
1166
  return await executeWithRetry();
1144
1167
  }
1145
- await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
1168
+ await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
1146
1169
  }
1147
1170
  // Check for usage limit errors first (more specific)
1148
1171
  const limitInfo = detectUsageLimit(lastMessage);
@@ -106,17 +106,34 @@ export const systemLimits = {
106
106
  };
107
107
 
108
108
  // Retry configurations
109
- // Issue #1331: All API error types use unified retry parameters:
110
- // 10 max retries, 2 minute initial delay, 30 minute max delay (exponential backoff), session preserved
109
+ // Issue #1331: All API error types use unified retry parameters (exponential backoff, session preserved).
110
+ // Issue #2169: the retry window is now governed by a *total time budget* (default 12 hours) rather
111
+ // than by the retry count alone. A provider outage can last many hours; the previous 10-retry cap
112
+ // with a 2-minute initial delay gave up after ~3.5 hours. The count caps below are kept as
113
+ // backstops against runaway loops — the budget is the knob operators are expected to tune.
111
114
  export const retryLimits = {
112
115
  maxForkRetries: parseIntWithDefault('HIVE_MIND_MAX_FORK_RETRIES', 5),
113
116
  maxVerifyRetries: parseIntWithDefault('HIVE_MIND_MAX_VERIFY_RETRIES', 5),
114
117
  maxApiRetries: parseIntWithDefault('HIVE_MIND_MAX_API_RETRIES', 3),
115
118
  retryBackoffMultiplier: parseFloatWithDefault('HIVE_MIND_RETRY_BACKOFF_MULTIPLIER', 2),
116
119
  // Unified retry config for all transient API errors (Overloaded, 503, Internal Server Error)
117
- maxTransientErrorRetries: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES', 10),
118
- initialTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_TRANSIENT_ERROR_DELAY_MS', 2 * 60 * 1000), // 2 minutes
120
+ // Issue #2169: count backstop only. With the defaults below (3 min → 30 min backoff) the 12-hour
121
+ // budget is exhausted after ~26 retries, so this cap never fires unless an operator shortens the
122
+ // delays. Lower it (e.g. HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES=5) to fail faster than the budget.
123
+ maxTransientErrorRetries: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES', 100),
124
+ // Issue #2169: "minimum of 3 minutes" — the first (and therefore smallest) transient backoff.
125
+ initialTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_TRANSIENT_ERROR_DELAY_MS', 3 * 60 * 1000), // 3 minutes
119
126
  maxTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_DELAY_MS', 30 * 60 * 1000), // 30 minutes
127
+ // Issue #2169: total wall-clock budget for transient-API-error retries, measured from the first
128
+ // retry of a run. Retrying stops as soon as the *next* backoff would push the run past this
129
+ // window. Set to 0 to disable the budget and fall back to the count cap alone.
130
+ transientErrorRetryBudgetMs: parseIntWithDefault('HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS', 12 * 60 * 60 * 1000), // 12 hours
131
+ // Issue #2169: floor applied to every transient backoff, including operator-supplied initial
132
+ // delays. Keeps a misconfigured 5-second delay from hammering an API that is already struggling.
133
+ minTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_MIN_TRANSIENT_ERROR_DELAY_MS', 3 * 60 * 1000), // 3 minutes
134
+ // Issue #2169: stream startup/activity timeouts keep their own (short) count cap. They use 30s–2min
135
+ // backoffs and force-kill the CLI, so they must not inherit the 100-retry transient backstop.
136
+ maxStreamTimeoutRetries: parseIntWithDefault('HIVE_MIND_MAX_STREAM_TIMEOUT_RETRIES', 10),
120
137
  // Issue #2037: When a "model is at capacity" error triggers a switch to a *different*
121
138
  // fallback model, the long transient backoff is wasteful — the different model is
122
139
  // available now, so retry almost immediately instead of stalling for minutes.
@@ -131,7 +148,8 @@ export const retryLimits = {
131
148
  maxCapacityRetryDelayMs: parseIntWithDefault('HIVE_MIND_MAX_CAPACITY_RETRY_DELAY_MS', 4 * 60 * 1000), // 4 minutes
132
149
  // Request timeout retry configuration (Issue #1353)
133
150
  // Network timeouts need longer waits than API errors — Claude CLI already exhausted its own retries
134
- maxRequestTimeoutRetries: parseIntWithDefault('HIVE_MIND_MAX_REQUEST_TIMEOUT_RETRIES', 10),
151
+ // Issue #2169: count backstop only — the shared 12-hour budget governs when to stop.
152
+ maxRequestTimeoutRetries: parseIntWithDefault('HIVE_MIND_MAX_REQUEST_TIMEOUT_RETRIES', 100),
135
153
  initialRequestTimeoutDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_REQUEST_TIMEOUT_DELAY_MS', 5 * 60 * 1000), // 5 minutes
136
154
  maxRequestTimeoutDelayMs: parseIntWithDefault('HIVE_MIND_MAX_REQUEST_TIMEOUT_DELAY_MS', 60 * 60 * 1000), // 1 hour
137
155
  // Not-retryable error fail-fast configuration (Issue #1437)
@@ -22,7 +22,7 @@ import { defaultModels, geminiModels, isFormalAiModel } from './models/index.mjs
22
22
  import { isPrepareOnly, logPreparedToolCommand, resolveFormalAiToolExecution } from './formal-ai.lib.mjs';
23
23
  import { buildFormalAiPricingInfo } from './formal-ai-pricing.lib.mjs'; // Issue #2119
24
24
  import { checkPlaywrightMcpPackageAvailability } from './playwright-mcp.lib.mjs';
25
- import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
25
+ import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
26
26
  import { getCumulativeContextInputTokens, toTokenCount } from './context-fill.lib.mjs';
27
27
  import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
28
28
  import { getTerminalEventCompletionHealth } from './tool-run-health.lib.mjs'; // Issue #1990
@@ -391,12 +391,15 @@ export const executeGeminiCommand = async params => {
391
391
  const { tempDir, workspaceTmpDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, geminiPath, $, waitForRetryDelay = waitWithCountdown } = params;
392
392
 
393
393
  let retryCount = 0;
394
+ // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
395
+ // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
396
+ const transientRetryBudget = createTransientRetryBudget();
394
397
 
395
398
  const executeWithRetry = async () => {
396
399
  if (retryCount === 0) {
397
400
  await log(`\n${formatAligned('🤖', 'Executing Gemini:', argv.model.toUpperCase())}`);
398
401
  } else {
399
- await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
402
+ await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
400
403
  }
401
404
 
402
405
  if (argv.verbose) {
@@ -515,7 +518,17 @@ export const executeGeminiCommand = async params => {
515
518
  if (retryableError.isRetryable) {
516
519
  const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
517
520
  const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
518
- if (retryCount < maxRetries) {
521
+ // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
522
+ // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
523
+ const retryDecision = transientRetryBudget.evaluate({
524
+ retryCount,
525
+ maxRetries,
526
+ initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
527
+ maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
528
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
529
+ });
530
+ if (retryDecision.allowed) {
531
+ transientRetryBudget.grant();
519
532
  // Issue #2037: retry the same model on capacity errors before falling back;
520
533
  // after a capacity-driven model switch, retry quickly instead of waiting the
521
534
  // full transient backoff — the new model may be available now.
@@ -527,16 +540,17 @@ export const executeGeminiCommand = async params => {
527
540
  retryCount,
528
541
  initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
529
542
  maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
543
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
530
544
  });
531
545
  const delay = retryPlan.delay;
532
546
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
533
- await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
547
+ await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
534
548
  await waitForRetryDelay(delay, log);
535
549
  await log('\n🔄 Retrying now...');
536
550
  retryCount++;
537
551
  return await executeWithRetry();
538
552
  }
539
- await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
553
+ await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
540
554
  }
541
555
 
542
556
  const limitInfo = detectUsageLimit(errorText);
@@ -270,10 +270,40 @@ export function canonicalizeGitHubUrl(url) {
270
270
  /** Build the canonical web URL for a pull request already identified by GitHub. */
271
271
  export function buildGitHubPullRequestUrl({ owner, repo, number } = {}) {
272
272
  if (!owner || !repo || !Number.isInteger(Number(number)) || Number(number) <= 0) {
273
- throw new TypeError('A GitHub pull request URL requires owner, repo, and a positive integer number');
273
+ // Issue #2170: the bare message left no way to tell which of the three
274
+ // parts was missing from a stack trace alone, so name the received values.
275
+ throw new TypeError(`A GitHub pull request URL requires owner, repo, and a positive integer number (received owner=${describeUrlPart(owner)}, repo=${describeUrlPart(repo)}, number=${describeUrlPart(number)})`);
274
276
  }
275
277
  return `https://github.com/${owner}/${repo}/pull/${Number(number)}`;
276
278
  }
279
+
280
+ /** Render a value for the diagnostic above without hiding null/undefined/''. */
281
+ function describeUrlPart(value) {
282
+ if (value === null) return 'null';
283
+ if (value === undefined) return 'undefined';
284
+ if (typeof value === 'string') return JSON.stringify(value);
285
+ return String(value);
286
+ }
287
+
288
+ /**
289
+ * Build a pull request URL when the pull request is already identified, and
290
+ * return `null` when it is not (for example, `--auto-continue` resumed a
291
+ * leftover branch whose pull request has not been created yet).
292
+ *
293
+ * Callers that legitimately run before a pull request exists must use this
294
+ * instead of `buildGitHubPullRequestUrl`, whose throw aborts the whole run.
295
+ *
296
+ * @see https://github.com/link-assistant/hive-mind/issues/2170
297
+ * @param {{owner?: string, repo?: string, number?: number|string|null}} [params]
298
+ * @returns {string|null}
299
+ */
300
+ export function buildGitHubPullRequestUrlOrNull(params = {}) {
301
+ try {
302
+ return buildGitHubPullRequestUrl(params);
303
+ } catch {
304
+ return null;
305
+ }
306
+ }
277
307
  /**
278
308
  * Check if a URL is a valid GitHub URL of a specific type
279
309
  * @param {string} url - The URL to check
@@ -20,8 +20,8 @@ export { buildCostInfoString };
20
20
  // #1756: route gh exec calls through transient + rate-limit retry wrapper
21
21
  import { execGhWithRetry } from './github-rate-limit.lib.mjs';
22
22
  import { QUIET_PROBE } from './quiet-probe.lib.mjs'; // issues #2130, #2135: keep read-only probe payloads out of the attached log
23
- import { buildGitHubPullRequestUrl, isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl } from './github-url-parser.lib.mjs';
24
- export { buildGitHubPullRequestUrl, isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl };
23
+ import { buildGitHubPullRequestUrl, buildGitHubPullRequestUrlOrNull, isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl } from './github-url-parser.lib.mjs';
24
+ export { buildGitHubPullRequestUrl, buildGitHubPullRequestUrlOrNull, isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl };
25
25
  // Issue #1625: Named marker constants (single source of truth) + in-memory tracking for tool-posted comments. See tool-comments.lib.mjs for design.
26
26
  import { SOLUTION_DRAFT_LOG_MARKER, SOLUTION_DRAFT_FAILED_MARKER, SOLUTION_DRAFT_FINISHED_WITH_ERRORS_MARKER, USAGE_LIMIT_REACHED_MARKER, NOW_WORKING_SESSION_IS_ENDED_MARKER, postTrackedComment, postTrackedCommentFromFile } from './tool-comments.lib.mjs';
27
27
  export const maskGitHubToken = maskToken; // Alias for backward compatibility
@@ -1171,6 +1171,7 @@ export default {
1171
1171
  isRateLimitError,
1172
1172
  batchCheckPullRequestsForIssues,
1173
1173
  buildGitHubPullRequestUrl,
1174
+ buildGitHubPullRequestUrlOrNull,
1174
1175
  parseGitHubUrl,
1175
1176
  normalizeGitHubUrl,
1176
1177
  isGitHubUrlType,
@@ -25,7 +25,7 @@ import { checkPlaywrightMcpPackageAvailability, getOpenCodePlaywrightMcpDisableE
25
25
  import { createAgentTokenUsage, accumulateAgentStepFinishUsage, parseAgentTokenUsage as parseOpenCodeTokenUsage } from './agent-token-usage.lib.mjs';
26
26
  import { createJsonStreamScanner } from './json-stream.lib.mjs';
27
27
  import { calculateAgentPricing } from './agent.lib.mjs';
28
- import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
28
+ import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
29
29
  import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
30
30
 
31
31
  export { parseOpenCodeTokenUsage };
@@ -193,13 +193,16 @@ export const executeOpenCodeCommand = async params => {
193
193
 
194
194
  // Retry configuration
195
195
  let retryCount = 0;
196
+ // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
197
+ // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
198
+ const transientRetryBudget = createTransientRetryBudget();
196
199
 
197
200
  const executeWithRetry = async () => {
198
201
  // Execute opencode command from the cloned repository directory
199
202
  if (retryCount === 0) {
200
203
  await log(`\n${formatAligned('🤖', 'Executing OpenCode:', argv.model.toUpperCase())}`);
201
204
  } else {
202
- await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
205
+ await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
203
206
  }
204
207
 
205
208
  if (argv.verbose) {
@@ -483,7 +486,17 @@ export const executeOpenCodeCommand = async params => {
483
486
  if (retryableError.isRetryable) {
484
487
  const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
485
488
  const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
486
- if (retryCount < maxRetries) {
489
+ // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
490
+ // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
491
+ const retryDecision = transientRetryBudget.evaluate({
492
+ retryCount,
493
+ maxRetries,
494
+ initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
495
+ maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
496
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
497
+ });
498
+ if (retryDecision.allowed) {
499
+ transientRetryBudget.grant();
487
500
  if (sessionId && !argv.resume) argv.resume = sessionId;
488
501
  // Issue #2037: retry the same model on capacity errors before falling back;
489
502
  // after a capacity-driven model switch, retry quickly instead of waiting the
@@ -496,16 +509,17 @@ export const executeOpenCodeCommand = async params => {
496
509
  retryCount,
497
510
  initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
498
511
  maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
512
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
499
513
  });
500
514
  const delay = retryPlan.delay;
501
515
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
502
- await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
516
+ await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
503
517
  await waitForRetryDelay(delay, log);
504
518
  await log('\n🔄 Retrying now...');
505
519
  retryCount++;
506
520
  return await executeWithRetry();
507
521
  }
508
- await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
522
+ await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
509
523
  }
510
524
 
511
525
  // Check for usage limit errors first (more specific)
package/src/qwen.lib.mjs CHANGED
@@ -22,7 +22,7 @@ import { qwenModels, defaultModels, isFormalAiModel } from './models/index.mjs';
22
22
  import { buildFormalAiEnvExports, isPrepareOnly, logPreparedToolCommand, resolveFormalAiToolExecution } from './formal-ai.lib.mjs';
23
23
  import { buildFormalAiPricingInfo } from './formal-ai-pricing.lib.mjs'; // Issue #2119
24
24
  import { checkPlaywrightMcpPackageAvailability } from './playwright-mcp.lib.mjs';
25
- import { classifyRetryableError, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
25
+ import { classifyRetryableError, createTransientRetryBudget, prepareRetryAfterError, waitWithCountdown } from './tool-retry.lib.mjs';
26
26
  import { getCumulativeContextInputTokens, getRestoredContextInputTokens, toTokenCount } from './context-fill.lib.mjs';
27
27
  import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
28
28
  import { getTerminalEventCompletionHealth } from './tool-run-health.lib.mjs'; // Issue #1990
@@ -489,6 +489,9 @@ export const executeQwenCommand = async params => {
489
489
  const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned = (_icon, label, value = '') => `${label} ${value}`.trim(), getResourceSnapshot = async () => ({ memory: '\nunknown', load: 'unknown' }), forkedRepo, feedbackLines, qwenPath = 'qwen', $: dollar = $, waitForRetryDelay = waitWithCountdown } = params;
490
490
 
491
491
  let retryCount = 0;
492
+ // Issue #2169: retries are bounded by a wall-clock budget (12 h by default, configurable via
493
+ // HIVE_MIND_TRANSIENT_ERROR_RETRY_BUDGET_MS) instead of a low attempt count.
494
+ const transientRetryBudget = createTransientRetryBudget();
492
495
  const promptFile = path.join(os.tmpdir(), `qwen_prompt_${Date.now()}_${process.pid}.txt`);
493
496
  const systemPromptFile = path.join(os.tmpdir(), `qwen_system_prompt_${Date.now()}_${process.pid}.txt`);
494
497
 
@@ -499,7 +502,7 @@ export const executeQwenCommand = async params => {
499
502
  if (retryCount === 0) {
500
503
  await log(`\n${formatAligned('🤖', 'Executing Qwen Code:', argv.model.toUpperCase())}`);
501
504
  } else {
502
- await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
505
+ await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount} (${transientRetryBudget.describeProgress()})`)}`);
503
506
  }
504
507
 
505
508
  if (argv.verbose) {
@@ -625,7 +628,17 @@ export const executeQwenCommand = async params => {
625
628
  if (retryableError.isRetryable) {
626
629
  const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
627
630
  const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
628
- if (retryCount < maxRetries) {
631
+ // Issue #2169: the attempt count is only a runaway backstop — the 12-hour wall-clock budget
632
+ // (configurable) decides when to stop, and every wait honours the 3-minute minimum.
633
+ const retryDecision = transientRetryBudget.evaluate({
634
+ retryCount,
635
+ maxRetries,
636
+ initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
637
+ maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
638
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
639
+ });
640
+ if (retryDecision.allowed) {
641
+ transientRetryBudget.grant();
629
642
  if (sessionId && !argv.resume) argv.resume = sessionId;
630
643
  // Issue #2037: retry the same model on capacity errors before falling back;
631
644
  // after a capacity-driven model switch, retry quickly instead of waiting the
@@ -638,16 +651,17 @@ export const executeQwenCommand = async params => {
638
651
  retryCount,
639
652
  initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
640
653
  maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
654
+ minDelayMs: retryLimits.minTransientErrorDelayMs,
641
655
  });
642
656
  const delay = retryPlan.delay;
643
657
  const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
644
- await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
658
+ await log(`\n⚠️ ${retryableError.label} detected. Retry ${retryCount + 1} in ${delayLabel}${sessionId ? ' (session preserved)' : ''} (${transientRetryBudget.describeProgress()})...`, { level: 'warning' });
645
659
  await waitForRetryDelay(delay, log);
646
660
  await log('\n🔄 Retrying now...');
647
661
  retryCount++;
648
662
  return await executeWithRetry();
649
663
  }
650
- await log(`\n\n❌ ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
664
+ await log(`\n\n❌ ${retryableError.label} persisted: ${transientRetryBudget.describeExhaustion(retryDecision)}`, { level: 'error' });
651
665
  } else if (exitCode === 130) {
652
666
  await log('\n\n⚠️ Qwen Code command interrupted (CTRL+C)');
653
667
  } else {
package/src/solve.mjs CHANGED
@@ -553,8 +553,18 @@ try {
553
553
  // Issue #2158: auto-continue can discover a PR while the input remains an
554
554
  // issue URL. Passing that issue URL as "Your prepared Pull Request" sent
555
555
  // the first Formal AI attempt to the wrong GitHub entity.
556
- prUrl = githubLib.buildGitHubPullRequestUrl({ owner, repo, number: prNumber });
557
- // prNumber is already set from earlier when we parsed the PR
556
+ //
557
+ // Issue #2170: continue mode does not imply a PR exists. --auto-continue
558
+ // also resumes a leftover `issue-<n>-<hash>` branch that never got a PR,
559
+ // and then prNumber is null. Building the URL unconditionally threw
560
+ // "A GitHub pull request URL requires owner, repo, and a positive integer
561
+ // number" and killed the run right after checkout. The PR is created a few
562
+ // lines below by handleAutoPrCreation, which sets both prUrl and prNumber.
563
+ prUrl = githubLib.buildGitHubPullRequestUrlOrNull({ owner, repo, number: prNumber });
564
+ if (!prUrl) {
565
+ await log(formatAligned('ℹ️', 'Continue mode:', 'Resuming an existing branch that has no pull request yet'));
566
+ await log(formatAligned('', 'Pull request:', 'Will be created before the tool session starts', 2));
567
+ }
558
568
  }
559
569
  // Handle auto PR creation using the new module
560
570
  const autoPrResult = await handleAutoPrCreation({
@@ -24,6 +24,22 @@ const normalizeModelKey = value => {
24
24
  .trim();
25
25
  };
26
26
 
27
+ // Issue #2169: HTTP status codes are only meaningful as errors when they appear in an
28
+ // HTTP-status context. A bare number in prose ("PR #524", "issue #523", "line 502") is
29
+ // not an error — matching it wrongly sent an entire successful session into a multi-hour
30
+ // retry loop (see docs/case-studies/issue-2169). `matchesHttpStatus` accepts:
31
+ // "API Error: 502", "error code: 522", "status 504", "HTTP/1.1 520", "code=524"
32
+ // "502 Bad Gateway", "524 A Timeout Occurred" (status followed by its canonical phrase)
33
+ const GATEWAY_STATUS_PHRASES = ['bad gateway', 'gateway timeout', 'gateway time-out', 'unknown error', 'web server is down', 'connection timed out', 'origin is unreachable', 'a timeout occurred'];
34
+ const HTTP_STATUS_PREFIX = String.raw`(?:http(?:s|/\d(?:\.\d)?)?\s*)?(?:api\s+)?(?:error|status(?:\s*code)?|code|response|returned|got)\s*(?:code\s*)?[:=#]?\s*`;
35
+ export const matchesHttpStatus = (lowerText, codePattern, phrases = []) => {
36
+ if (!lowerText) return false;
37
+ if (new RegExp(`${HTTP_STATUS_PREFIX}(?:${codePattern})\\b`).test(lowerText)) return true;
38
+ if (new RegExp(`\\bhttp\\s*(?:status\\s*)?(?:${codePattern})\\b`).test(lowerText)) return true;
39
+ if (phrases.length > 0 && new RegExp(`\\b(?:${codePattern})\\b[\\s,:;-]*(?:${phrases.join('|')})`).test(lowerText)) return true;
40
+ return false;
41
+ };
42
+
27
43
  export const classifyRetryableError = value => {
28
44
  const message = normalizeMessage(value);
29
45
  const lower = message.toLowerCase();
@@ -163,7 +179,12 @@ export const classifyRetryableError = value => {
163
179
  // These come from an intermediary (CDN/proxy/load balancer), not from a request the
164
180
  // client got wrong, and clear on their own — OpenAI/Anthropic/GitHub all front their
165
181
  // APIs with such proxies. Safe to retry the same request after a backoff.
166
- if (lower.includes('502 bad gateway') || lower.includes('bad gateway') || lower.includes('504 gateway timeout') || lower.includes('gateway time-out') || lower.includes('gateway timeout') || lower.includes('api error: 502') || lower.includes('api error: 504') || /\b52[0-4]\b/.test(lower)) {
182
+ // Issue #2169: the bare `/\b52[0-4]\b/` test used here before matched ANY standalone
183
+ // 520-524 in the text, so an agent's own success summary — "PR #524", "issue #523",
184
+ // "(`463c5ca`, PR #522)" — was classified as a gateway error and retried for hours.
185
+ // The number now has to appear in an HTTP-status context ("error code: 522",
186
+ // "API Error: 502", "HTTP 504") or next to the status' canonical phrase.
187
+ if (lower.includes('502 bad gateway') || lower.includes('bad gateway') || lower.includes('504 gateway timeout') || lower.includes('gateway time-out') || lower.includes('gateway timeout') || matchesHttpStatus(lower, '502|504|52[0-4]', GATEWAY_STATUS_PHRASES)) {
167
188
  return { message, isRetryable: true, isCapacity: false, label: 'Gateway error (502/504/52x)' };
168
189
  }
169
190
 
@@ -201,7 +222,9 @@ export const classifyRetryableError = value => {
201
222
  // Issue #1955: broadened to also catch the bare "503 Service Unavailable" that
202
223
  // GitHub/OpenAI/Anthropic return when a backend is briefly saturated — a
203
224
  // transient, self-clearing condition, safe to retry with the same request.
204
- if (lower.includes('api error: 503') || lower.includes('503 service unavailable') || lower.includes('service unavailable') || (lower.includes('503') && (lower.includes('upstream connect error') || lower.includes('remote connection failure')))) {
225
+ // Issue #2169: `matchesHttpStatus` accepts the status only next to an error-ish prefix
226
+ // ("api error: 503", "status code: 503"), never a bare number in prose.
227
+ if (lower.includes('api error: 503') || lower.includes('503 service unavailable') || lower.includes('service unavailable') || matchesHttpStatus(lower, '503') || (lower.includes('503') && (lower.includes('upstream connect error') || lower.includes('remote connection failure')))) {
205
228
  return { message, isRetryable: true, isCapacity: false, label: '503 network error' };
206
229
  }
207
230
 
@@ -212,8 +235,92 @@ export const classifyRetryableError = value => {
212
235
  return { message, isRetryable: false, isCapacity: false, label: null };
213
236
  };
214
237
 
215
- export const getRetryDelayMs = ({ retryCount, initialDelayMs = retryLimits.initialTransientErrorDelayMs, maxDelayMs = retryLimits.maxTransientErrorDelayMs } = {}) => {
216
- return Math.min(initialDelayMs * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), maxDelayMs);
238
+ // Issue #2169: `minDelayMs` is the floor for the transient-API-error paths ("with minimum of
239
+ // 3 minutes"). It defaults to 0 so the deliberately fast paths — capacity retries (15s), a
240
+ // model switch (5s), stream startup/activity timeouts (30s) — keep their short delays.
241
+ export const getRetryDelayMs = ({ retryCount, initialDelayMs = retryLimits.initialTransientErrorDelayMs, maxDelayMs = retryLimits.maxTransientErrorDelayMs, minDelayMs = 0 } = {}) => {
242
+ const backoff = Math.min(initialDelayMs * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), maxDelayMs);
243
+ return Math.max(backoff, Math.min(minDelayMs, maxDelayMs));
244
+ };
245
+
246
+ // Issue #2169: diagnosing a *mis*classification from the log used to be impossible — the retry
247
+ // line printed only the first 200 characters of the message, while the token that actually made
248
+ // the classifier fire ("PR #524", 1.6 KB later) was never shown. This renders the evidence for a
249
+ // classification: how long the message was and the ±40-character context around every HTTP-status
250
+ // -looking token in it, so the next false positive is diagnosable from a single verbose log line.
251
+ export const describeClassificationEvidence = (message, label = null, { maxMatches = 3, contextChars = 40 } = {}) => {
252
+ const text = String(message ?? '');
253
+ const parts = [`label=${JSON.stringify(label)}`, `messageChars=${text.length}`];
254
+ const matches = [];
255
+ for (const match of text.matchAll(/\b(?:4\d{2}|5\d{2})\b/g)) {
256
+ const start = Math.max(0, match.index - contextChars);
257
+ const end = Math.min(text.length, match.index + match[0].length + contextChars);
258
+ matches.push(`@${match.index} ${JSON.stringify(text.slice(start, end).replace(/\s+/g, ' '))}`);
259
+ if (matches.length >= maxMatches) break;
260
+ }
261
+ parts.push(matches.length > 0 ? `statusTokens=[${matches.join(', ')}]` : 'statusTokens=[]');
262
+ return parts.join(' ');
263
+ };
264
+
265
+ // Issue #2169: human-readable duration for retry logs — "45s", "12 min", "3h 15m", "12h".
266
+ export const formatRetryDuration = ms => {
267
+ if (!Number.isFinite(ms)) return 'unlimited';
268
+ if (ms < 60000) return `${Math.max(0, Math.round(ms / 1000))}s`;
269
+ const totalMinutes = Math.round(ms / 60000);
270
+ if (totalMinutes < 60) return `${totalMinutes} min`;
271
+ const hours = Math.floor(totalMinutes / 60);
272
+ const minutes = totalMinutes % 60;
273
+ return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`;
274
+ };
275
+
276
+ // Issue #2169: a provider outage can last many hours. The retry loops used to stop after a
277
+ // fixed number of attempts (10), which — with the old 2 min → 30 min backoff — gave up after
278
+ // ~3.5 hours. The budget below turns "how long do we keep trying" into the primary knob: retries
279
+ // continue while the *next* backoff still fits inside `budgetMs` (default 12 h), measured from
280
+ // the first retry of the run. The per-tool retry counts stay as runaway-loop backstops.
281
+ //
282
+ // Usage inside a tool's retry loop (the budget object lives outside `executeWithRetry` so it
283
+ // survives the recursive calls):
284
+ // const budget = createTransientRetryBudget();
285
+ // const decision = budget.evaluate({ retryCount, maxRetries, initialDelayMs, maxDelayMs, minDelayMs });
286
+ // if (decision.allowed) { budget.grant(); ...wait decision.delayMs... } else { fail(decision) }
287
+ export const createTransientRetryBudget = ({ budgetMs = retryLimits.transientErrorRetryBudgetMs, now = () => Date.now() } = {}) => {
288
+ let startedAt = null;
289
+ let retriesGranted = 0;
290
+ const elapsedMs = () => (startedAt === null ? 0 : Math.max(0, now() - startedAt));
291
+ const remainingMs = () => (budgetMs > 0 ? Math.max(0, budgetMs - elapsedMs()) : Infinity);
292
+ return {
293
+ budgetMs,
294
+ elapsedMs,
295
+ remainingMs,
296
+ get retriesGranted() {
297
+ return retriesGranted;
298
+ },
299
+ // Starts the clock on the first granted retry and counts it.
300
+ grant() {
301
+ if (startedAt === null) startedAt = now();
302
+ retriesGranted += 1;
303
+ },
304
+ evaluate({ retryCount = 0, maxRetries = retryLimits.maxTransientErrorRetries, initialDelayMs, maxDelayMs, minDelayMs = 0 } = {}) {
305
+ const delayMs = getRetryDelayMs({ retryCount, initialDelayMs, maxDelayMs, minDelayMs });
306
+ const base = { delayMs, elapsedMs: elapsedMs(), remainingMs: remainingMs(), budgetMs, retryCount, maxRetries };
307
+ if (retryCount >= maxRetries) return { ...base, allowed: false, reason: 'count' };
308
+ // Never start a wait that would run past the budget window.
309
+ if (budgetMs > 0 && delayMs > remainingMs()) return { ...base, allowed: false, reason: 'budget' };
310
+ return { ...base, allowed: true, reason: null };
311
+ },
312
+ // One-line explanation for the "giving up" log, e.g.
313
+ // "retry budget of 12h exhausted after 26 retries over 11h 45m".
314
+ describeExhaustion(decision) {
315
+ const spent = formatRetryDuration(decision?.elapsedMs ?? elapsedMs());
316
+ if (decision?.reason === 'count') return `retry limit of ${decision.maxRetries} attempts reached after ${spent} (budget ${formatRetryDuration(budgetMs)})`;
317
+ return `retry budget of ${formatRetryDuration(budgetMs)} exhausted after ${decision?.retryCount ?? retriesGranted} retries over ${spent}`;
318
+ },
319
+ // Progress suffix for each retry log line, e.g. "budget 25 min/12h used".
320
+ describeProgress() {
321
+ return budgetMs > 0 ? `budget ${formatRetryDuration(elapsedMs())}/${formatRetryDuration(budgetMs)} used` : 'budget disabled';
322
+ },
323
+ };
217
324
  };
218
325
 
219
326
  export const waitWithCountdown = async (delayMs, log) => {
@@ -352,7 +459,7 @@ export const maybeSwitchToFallbackModel = async ({ tool, argv, log, errorMessage
352
459
  // survives the recursive executeWithRetry calls without each tool tracking extra
353
460
  // state. It resets to 0 whenever we actually switch models, so every model in the
354
461
  // fallback chain gets its own batch of same-model retries before stepping down.
355
- export const prepareRetryAfterError = async ({ tool, argv, log, errorMessage, retryCount, initialDelayMs, maxDelayMs } = {}) => {
462
+ export const prepareRetryAfterError = async ({ tool, argv, log, errorMessage, retryCount, initialDelayMs, maxDelayMs, minDelayMs = 0 } = {}) => {
356
463
  const classification = classifyRetryableError(errorMessage);
357
464
  const isCapacity = classification.isCapacity === true && !!argv?.model;
358
465
  const capacityRetryCount = argv?._capacityRetryCount || 0;
@@ -373,13 +480,17 @@ export const prepareRetryAfterError = async ({ tool, argv, log, errorMessage, re
373
480
  const switchResult = await maybeSwitchToFallbackModel({ tool, argv, log, errorMessage });
374
481
  // A model switch starts a fresh batch of same-model retries for the new model.
375
482
  if (switchResult?.switched && argv) argv._capacityRetryCount = 0;
376
- const delay = switchResult?.switched ? retryLimits.modelSwitchRetryDelayMs : getRetryDelayMs({ retryCount, initialDelayMs, maxDelayMs });
483
+ const delay = switchResult?.switched ? retryLimits.modelSwitchRetryDelayMs : getRetryDelayMs({ retryCount, initialDelayMs, maxDelayMs, minDelayMs });
377
484
  return { delay, switched: switchResult?.switched === true };
378
485
  };
379
486
 
380
487
  export default {
381
488
  classifyRetryableError,
489
+ matchesHttpStatus,
382
490
  getRetryDelayMs,
491
+ describeClassificationEvidence,
492
+ formatRetryDuration,
493
+ createTransientRetryBudget,
383
494
  waitWithCountdown,
384
495
  resolveConfiguredFallbackModel,
385
496
  maybeSwitchToFallbackModel,