@link-assistant/hive-mind 1.78.10 → 1.78.11

package/CHANGELOG.md

@@ -1,5 +1,44 @@
 # @link-assistant/hive-mind
 
+## 1.78.11
+
+### Patch Changes
+
+- 24fb17e: fix(retry): auto-resume on server-side 429 "Server is temporarily limiting requests" rate-limit errors (#1924)
+
+  A long-running solve session (177 turns, ~72 min) was thrown away when the Claude
+  CLI surfaced a **server-side temporary rate limit**:
+
+  ```
+  API Error: Server is temporarily limiting requests (not your usage limit) · Rate limited
+  ```
+
+  The CLI reports this as a `result` event with `is_error: true` and
+  `api_error_status: 429`, and the HTTP response carries `x-should-retry: true`.
+  This is a transient throttle that clears on its own — distinct from an account
+  usage/quota limit (the message literally says "not your usage limit", and there
+  is no reset time to wait for).
+
+  Root cause: the error matched neither `classifyRetryableError` (no pattern for
+  the 429 throttle wording) nor `isUsageLimitError` (correctly, since it is not a
+  quota limit), so it fell through to a hard failure with exit code 1 and **no
+  auto-resume**, unlike every other transient class (overload 500/529, 503,
+  internal server error, request timeout, socket drops).
+
+  Fix: `classifyRetryableError` (in `src/tool-retry.lib.mjs`, the shared classifier
+  used by every tool wrapper — claude, codex, gemini, opencode, qwen, agent) now
+  recognises this throttle and marks it retryable (`isCapacity: false`, so no model
+  switch), so it retries with the session preserved (`--resume`) after a backoff.
+  `src/claude.lib.mjs` additionally detects the structured `api_error_status === 429`
+  directly (robust to wording changes) and logs a verbose diagnostic with the
+  `request_id`. The matcher is narrow so genuine account usage limits stay on the
+  usage-limit reset-time path.
+
+  Added `tests/test-issue-1924-rate-limit-retry.mjs` (18 assertions) and a full
+  case study with timeline, root-cause analysis, upstream references
+  (anthropics/claude-code#53915, #53922), and the captured logs under
+  `docs/case-studies/issue-1924`.
+
 ## 1.78.10
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "1.78.10",
+  "version": "1.78.11",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/claude.lib.mjs

@@ -648,6 +648,7 @@ export const executeClaudeCommand = async params => {
     let is503Error = false;
     let isInternalServerError = false;
     let isRequestTimeout = false;
+    let isRateLimitError = false; // Issue #1924: server-side 429 temporary rate limiting
     let apiMarkedNotRetryable = false;
     let resultNumTurns = 0;
     let stderrErrors = [];
@@ -977,6 +978,14 @@ export const executeClaudeCommand = async params => {
                     isRequestTimeout = true;
                     await log('⏱️ Detected request timeout from Claude CLI (will retry with --resume)', { verbose: true });
                   }
+                  // Issue #1924: Server-side temporary rate limiting (HTTP 429) — a transient
+                  // throttle, not an account usage limit ("...not your usage limit..."), so retry
+                  // with --resume. The message text is handled by classifyRetryableError; this also
+                  // catches the structured api_error_status if the wording ever changes.
+                  if (data.api_error_status === 429) {
+                    isRateLimitError = true;
+                    await log(`⚠️ Detected server-side rate limiting (429) from Claude CLI (will retry with --resume). request_id=${data.request_id || 'unknown'}`, { verbose: true });
+                  }
                   // Issue #1834: Detect corrupted extended-thinking-block 400 (un-resumable session).
                   // Capture diagnostics (request id, content path) to aid debugging and upstream reports.
                   if ((lastMessage.includes('thinking') || lastMessage.includes('redacted_thinking')) && lastMessage.includes('cannot be modified')) {
@@ -1174,7 +1183,7 @@ export const executeClaudeCommand = async params => {
         return await executeWithRetry();
       }
       // Issues #1331, #1353, #1472/#1475: Unified transient error retry (exponential backoff, session preservation)
-      const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || retryableLastError.isRetryable || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
+      const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || isRateLimitError || retryableLastError.isRetryable || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
       if ((commandFailed || isTransientError) && isTransientError) {
         // Issue #1472/#1475: Startup/activity timeout → 30s–2min backoff; #1353: Request timeout → 5min–1hr; general → 2min–30min
         const isTimeoutRetry = isStartupTimeout || isActivityTimeout;
@@ -1208,7 +1217,7 @@ export const executeClaudeCommand = async params => {
         }
         if (retryCount < maxRetries) {
           const delay = Math.min(initialDelay * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), maxDelay);
-          const errorLabel = isStartupTimeout ? 'Stream startup timeout (Issue #1472/#1475)' : isActivityTimeout ? 'Stream activity timeout (Issue #1472)' : isRequestTimeout ? 'Request timeout' : retryableLastError.label || (isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('API Error: 529') && lastMessage.includes('Overloaded')) ? `API overload (${lastMessage.includes('529') ? '529' : '500'})` : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : '503 network error');
+          const errorLabel = isStartupTimeout ? 'Stream startup timeout (Issue #1472/#1475)' : isActivityTimeout ? 'Stream activity timeout (Issue #1472)' : isRequestTimeout ? 'Request timeout' : retryableLastError.label || (isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('API Error: 529') && lastMessage.includes('Overloaded')) ? `API overload (${lastMessage.includes('529') ? '529' : '500'})` : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : isRateLimitError ? 'Server rate limited (429)' : '503 network error');
           const notRetryableHint = apiMarkedNotRetryable ? ' (API says not retryable — will stop early if no progress)' : '';
           const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
           const retryMode = isStartupTimeout ? ' (fresh start)' : ' (session preserved)';

package/src/tool-retry.lib.mjs

@@ -72,6 +72,22 @@ export const classifyRetryableError = value => {
     return { message, isRetryable: false, isCapacity: false, requiresFreshSession: true, label: 'Corrupted thinking blocks (un-resumable session)' };
   }
 
+  // Issue #1924: Server-side temporary rate limiting (HTTP 429), distinct from an
+  // account usage/quota limit. The Claude CLI surfaces this as a synthetic
+  // assistant/result message and an api_error_status of 429:
+  //   "API Error: Server is temporarily limiting requests (not your usage limit) · Rate limited"
+  // The response carries `x-should-retry: true` and the stream emits a
+  // `rate_limit_event` with `status: "rejected"`. Because the message explicitly
+  // says "not your usage limit", it is NOT a usage-limit reset-time situation and
+  // must NOT be routed through detectUsageLimit() (there is no reset time to wait
+  // for). It is a transient throttle that clears on its own, so it is safe to
+  // retry with the session preserved (--resume) after a backoff. Switching models
+  // does not help (the throttle is request-rate, not model capacity), so
+  // isCapacity is false.
+  if (lower.includes('temporarily limiting requests') || (lower.includes('rate limited') && lower.includes('not your usage limit')) || (lower.includes('rate_limit') && lower.includes('429'))) {
+    return { message, isRetryable: true, isCapacity: false, label: 'Server rate limited (429)' };
+  }
+
   if (lower.includes('api error: 503') || (lower.includes('503') && (lower.includes('upstream connect error') || lower.includes('remote connection failure')))) {
     return { message, isRetryable: true, isCapacity: false, label: '503 network error' };
   }