@link-assistant/hive-mind 1.24.1 → 1.24.3

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @link-assistant/hive-mind
 
+## 1.24.3
+
+### Patch Changes
+
+- 297e07c: Fix incorrect iteration counter and duplicate comments in auto-restart mode
+  - Fixed iteration counter to show actual AI restart count instead of check cycle number
+  - Added deduplication check to prevent duplicate "Ready to merge" status comments
+  - Added case study documentation for issue #1323
+
+## 1.24.2
+
+### Patch Changes
+
+- a74e10c: fix: add auto-resume with session preservation on Internal Server Error (Issue #1331)
+
+  When Claude tool returns `API Error: 500 Internal server error`, automatically retry with exponential backoff starting from 1 minute, capped at 30 minutes per retry, up to 10 retries. Session ID is preserved so Claude Code can resume from where it left off using `--resume <sessionId>`.
+
 ## 1.24.1
 
 ### Patch Changes

package/README.md

@@ -811,9 +811,21 @@ procinfo 62220
 
 ## Maintenance
 
-### Reboot server.
+### Enter latest screen
+
+```bash
+s=$(screen -ls | awk '/Detached/ {print $1; exit}'); echo "Entering $s"; screen -r "$s"; echo "Left $s";
+```
+
+### Enter oldest screen
 
+```bash
+s=$(screen -ls | awk '/Detached/ {last=$1} END{print last}'); echo "Entering $s"; screen -r "$s"; echo "Left $s";
 ```
+
+### Reboot server.
+
+```bash
 sudo reboot
 ```
 
@@ -821,7 +833,7 @@ That will remove all dangling unused proccesses and screens, which will in turn
 
 ### Cleanup disk space.
 
-```
+```bash
 df -h
 
 rm -rf /tmp

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "1.24.1",
+  "version": "1.24.3",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/claude.lib.mjs

@@ -747,17 +747,30 @@ export const executeClaudeCommand = async params => {
     repo,
     prNumber,
   } = params;
-  // Retry configuration for API overload errors
-  const maxRetries = 3;
-  const baseDelay = timeouts.retryBaseDelay;
+  // Issue #1331: Unified retry configuration for all transient API errors
+  // (Overloaded, 503 Network Error, Internal Server Error) - same params, all with session preservation
   let retryCount = 0;
+  // Helper: wait with per-minute countdown for delays >1 minute (Issue #1331)
+  const waitWithCountdown = async (delayMs, log) => {
+    if (delayMs <= 60000) {
+      await new Promise(resolve => setTimeout(resolve, delayMs));
+      return;
+    }
+    let remaining = delayMs;
+    const timer = setInterval(async () => {
+      remaining -= 60000;
+      if (remaining > 0) await log(`⏳ ${Math.round(remaining / 60000)} min remaining...`);
+    }, 60000);
+    await new Promise(resolve => setTimeout(resolve, delayMs));
+    clearInterval(timer);
+  };
   // Function to execute with retry logic
   const executeWithRetry = async () => {
     // Execute claude command from the cloned repository directory
     if (retryCount === 0) {
       await log(`\n${formatAligned('🤖', 'Executing Claude:', argv.model.toUpperCase())}`);
     } else {
-      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${maxRetries}`)}`);
+      await log(`\n${formatAligned('🔄', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
     }
     if (argv.verbose) {
       // Output the actual model being used
@@ -789,6 +802,7 @@ export const executeClaudeCommand = async params => {
     let lastMessage = '';
     let isOverloadError = false;
     let is503Error = false;
+    let isInternalServerError = false; // Issue #1331: Track 500 Internal server error
     let stderrErrors = [];
     let anthropicTotalCostUSD = null; // Capture Anthropic's official total_cost_usd from result
     let errorDuringExecution = false; // Issue #1088: Track if error_during_execution subtype occurred
@@ -979,6 +993,9 @@ export const executeClaudeCommand = async params => {
                     limitReached = true;
                     await log('⚠️ Detected session limit in result', { verbose: true });
                   }
+                  if (lastMessage.includes('Internal server error') && !lastMessage.includes('Overloaded')) {
+                    isInternalServerError = true;
+                  }
                 }
               }
               // Store last message for error detection
@@ -986,6 +1003,9 @@ export const executeClaudeCommand = async params => {
                 lastMessage = data.text;
               } else if (data.type === 'error') {
                 lastMessage = data.error || JSON.stringify(data);
+                if (lastMessage.includes('Internal server error')) {
+                  isInternalServerError = true;
+                }
               }
               // Check for API overload error and 503 errors
               if (data.type === 'assistant' && data.message && data.message.content) {
@@ -998,6 +1018,10 @@ export const executeClaudeCommand = async params => {
                       lastMessage = item.text;
                       await log('⚠️ Detected API overload error', { verbose: true });
                     }
+                    if (item.text.includes('API Error: 500') && item.text.includes('Internal server error') && !item.text.includes('Overloaded')) {
+                      isInternalServerError = true;
+                      lastMessage = item.text;
+                    }
                     // Check for 503 errors
                     if (item.text.includes('API Error: 503') || (item.text.includes('503') && item.text.includes('upstream connect error')) || (item.text.includes('503') && item.text.includes('remote connection failure'))) {
                       is503Error = true;
@@ -1110,64 +1134,22 @@ export const executeClaudeCommand = async params => {
         }
       }
 
-      if ((commandFailed || isOverloadError) && (isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')))) {
-        if (retryCount < maxRetries) {
-          // Calculate exponential backoff delay
-          const delay = baseDelay * Math.pow(2, retryCount);
-          await log(`\n⚠️ API overload error detected. Retrying in ${delay / 1000} seconds...`, { level: 'warning' });
-          await log(`   Error: ${lastMessage.substring(0, 200)}`, { verbose: true });
-          // Wait before retrying
-          await new Promise(resolve => setTimeout(resolve, delay));
-          // Increment retry count and retry
-          retryCount++;
-          return await executeWithRetry();
-        } else {
-          await log(`\n\n❌ API overload error persisted after ${maxRetries} retries\n   The API appears to be heavily loaded. Please try again later.`, { level: 'error' });
-          return {
-            success: false,
-            sessionId,
-            limitReached: false,
-            limitResetTime: null,
-            limitTimezone: null,
-            messageCount,
-            toolUseCount,
-            anthropicTotalCostUSD, // Issue #1104: Include cost even on failure
-            resultSummary, // Issue #1263: Include result summary
-          };
-        }
-      }
-      if ((commandFailed || is503Error) && argv.autoResumeOnErrors && (is503Error || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && lastMessage.includes('upstream connect error')) || (lastMessage.includes('503') && lastMessage.includes('remote connection failure')))) {
-        if (retryCount < retryLimits.max503Retries) {
-          // Calculate exponential backoff delay starting from 5 minutes
-          const delay = retryLimits.initial503RetryDelayMs * Math.pow(retryLimits.retryBackoffMultiplier, retryCount);
-          const delayMinutes = Math.round(delay / (1000 * 60));
-          await log(`\n⚠️ 503 network error detected. Retrying in ${delayMinutes} minutes...`, { level: 'warning' });
+      // Issue #1331: Unified handler for all transient API errors (Overloaded, 503, Internal Server Error)
+      // All use same params: 10 retries, 1min initial, 30min max, exponential backoff, session preserved
+      const isTransientError = isOverloadError || isInternalServerError || is503Error || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure')));
+      if ((commandFailed || isTransientError) && isTransientError) {
+        if (retryCount < retryLimits.maxTransientErrorRetries) {
+          const delay = Math.min(retryLimits.initialTransientErrorDelayMs * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), retryLimits.maxTransientErrorDelayMs);
+          const errorLabel = isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) ? 'API overload (500)' : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : '503 network error';
+          await log(`\n⚠️ ${errorLabel} detected. Retry ${retryCount + 1}/${retryLimits.maxTransientErrorRetries} in ${Math.round(delay / 60000)} min (session preserved)...`, { level: 'warning' });
           await log(`   Error: ${lastMessage.substring(0, 200)}`, { verbose: true });
-          await log(`   Retry ${retryCount + 1}/${retryLimits.max503Retries}`, { verbose: true });
-          // Show countdown for long waits
-          if (delay > 60000) {
-            const countdownInterval = 60000; // Every minute
-            let remainingMs = delay;
-            const countdownTimer = setInterval(async () => {
-              remainingMs -= countdownInterval;
-              if (remainingMs > 0) {
-                const remainingMinutes = Math.round(remainingMs / (1000 * 60));
-                await log(`⏳ ${remainingMinutes} minutes remaining until retry...`);
-              }
-            }, countdownInterval);
-            // Wait before retrying
-            await new Promise(resolve => setTimeout(resolve, delay));
-            clearInterval(countdownTimer);
-          } else {
-            // Wait before retrying
-            await new Promise(resolve => setTimeout(resolve, delay));
-          }
+          if (sessionId && !argv.resume) argv.resume = sessionId; // preserve session for resume
+          await waitWithCountdown(delay, log);
           await log('\n🔄 Retrying now...');
-          // Increment retry count and retry
           retryCount++;
           return await executeWithRetry();
         } else {
-          await log(`\n\n❌ 503 network error persisted after ${retryLimits.max503Retries} retries\n   The Anthropic API appears to be experiencing network issues.\n   Please try again later or check https://status.anthropic.com/`, { level: 'error' });
+          await log(`\n\n❌ Transient API error persisted after ${retryLimits.maxTransientErrorRetries} retries\n   Please try again later or check https://status.anthropic.com/`, { level: 'error' });
           return {
             success: false,
             sessionId,
@@ -1176,7 +1158,7 @@ export const executeClaudeCommand = async params => {
             limitTimezone: null,
             messageCount,
             toolUseCount,
-            is503Error: true,
+            is503Error, // preserve for callers that check this
             anthropicTotalCostUSD, // Issue #1104: Include cost even on failure
             resultSummary, // Issue #1263: Include result summary
           };
@@ -1338,31 +1320,17 @@ export const executeClaudeCommand = async params => {
         operation: 'run_claude_command',
       });
       const errorStr = error.message || error.toString();
-      if ((errorStr.includes('API Error: 500') && errorStr.includes('Overloaded')) || (errorStr.includes('api_error') && errorStr.includes('Overloaded'))) {
-        if (retryCount < maxRetries) {
-          // Calculate exponential backoff delay
-          const delay = baseDelay * Math.pow(2, retryCount);
-          await log(`\n⚠️ API overload error in exception. Retrying in ${delay / 1000} seconds...`, {
-            level: 'warning',
-          });
-          // Wait before retrying
-          await new Promise(resolve => setTimeout(resolve, delay));
-          // Increment retry count and retry
-          retryCount++;
-          return await executeWithRetry();
-        }
-      }
-      if (argv.autoResumeOnErrors && (errorStr.includes('API Error: 503') || (errorStr.includes('503') && errorStr.includes('upstream connect error')) || (errorStr.includes('503') && errorStr.includes('remote connection failure')))) {
-        if (retryCount < retryLimits.max503Retries) {
-          // Calculate exponential backoff delay starting from 5 minutes
-          const delay = retryLimits.initial503RetryDelayMs * Math.pow(retryLimits.retryBackoffMultiplier, retryCount);
-          const delayMinutes = Math.round(delay / (1000 * 60));
-          await log(`\n⚠️ 503 network error in exception. Retrying in ${delayMinutes} minutes...`, {
-            level: 'warning',
-          });
-          // Wait before retrying
-          await new Promise(resolve => setTimeout(resolve, delay));
-          // Increment retry count and retry
+      // Issue #1331: Unified handler for all transient API errors in exception block
+      // (Overloaded, 503, Internal Server Error) - same params, all with session preservation
+      const isTransientException = (errorStr.includes('API Error: 500') && (errorStr.includes('Overloaded') || errorStr.includes('Internal server error'))) || (errorStr.includes('api_error') && errorStr.includes('Overloaded')) || errorStr.includes('API Error: 503') || (errorStr.includes('503') && (errorStr.includes('upstream connect error') || errorStr.includes('remote connection failure')));
+      if (isTransientException) {
+        if (retryCount < retryLimits.maxTransientErrorRetries) {
+          const delay = Math.min(retryLimits.initialTransientErrorDelayMs * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), retryLimits.maxTransientErrorDelayMs);
+          const errorLabel = errorStr.includes('Overloaded') ? 'API overload (500)' : errorStr.includes('Internal server error') ? 'Internal server error (500)' : '503 network error';
+          await log(`\n⚠️ ${errorLabel} in exception. Retry ${retryCount + 1}/${retryLimits.maxTransientErrorRetries} in ${Math.round(delay / 60000)} min (session preserved)...`, { level: 'warning' });
+          if (sessionId && !argv.resume) argv.resume = sessionId;
+          await waitWithCountdown(delay, log);
+          await log('\n🔄 Retrying now...');
           retryCount++;
           return await executeWithRetry();
         }

package/src/config.lib.mjs

@@ -92,13 +92,17 @@ export const systemLimits = {
 };
 
 // Retry configurations
+// Issue #1331: All API error types use unified retry parameters:
+// 10 max retries, 1 minute initial delay, 30 minute max delay (exponential backoff), session preserved
 export const retryLimits = {
   maxForkRetries: parseIntWithDefault('HIVE_MIND_MAX_FORK_RETRIES', 5),
   maxVerifyRetries: parseIntWithDefault('HIVE_MIND_MAX_VERIFY_RETRIES', 5),
   maxApiRetries: parseIntWithDefault('HIVE_MIND_MAX_API_RETRIES', 3),
   retryBackoffMultiplier: parseFloatWithDefault('HIVE_MIND_RETRY_BACKOFF_MULTIPLIER', 2),
-  max503Retries: parseIntWithDefault('HIVE_MIND_MAX_503_RETRIES', 3),
-  initial503RetryDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_503_RETRY_DELAY_MS', 5 * 60 * 1000), // 5 minutes
+  // Unified retry config for all transient API errors (Overloaded, 503, Internal Server Error)
+  maxTransientErrorRetries: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_RETRIES', 10),
+  initialTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_TRANSIENT_ERROR_DELAY_MS', 60 * 1000), // 1 minute
+  maxTransientErrorDelayMs: parseIntWithDefault('HIVE_MIND_MAX_TRANSIENT_ERROR_DELAY_MS', 30 * 60 * 1000), // 30 minutes
 };
 
 // Claude Code CLI configurations

package/src/solve.auto-merge.lib.mjs

@@ -43,6 +43,37 @@ const { sanitizeLogContent, attachLogToGitHub } = githubLib;
 const restartShared = await import('./solve.restart-shared.lib.mjs');
 const { checkPRMerged, checkPRClosed, checkForUncommittedChanges, getUncommittedChangesDetails, executeToolIteration, buildAutoRestartInstructions, isApiError } = restartShared;
 
+/**
+ * Issue #1323: Check if a comment with specific content already exists on the PR
+ * This prevents duplicate status comments when multiple processes or restarts occur
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {number} prNumber - Pull request number
+ * @param {string} commentSignature - Unique signature to search for in comment body (e.g., "✅ Ready to merge")
+ * @param {boolean} verbose - Enable verbose logging
+ * @returns {Promise<boolean>} - True if a matching comment already exists
+ */
+const checkForExistingComment = async (owner, repo, prNumber, commentSignature, verbose = false) => {
+  try {
+    // Fetch recent PR comments (last 20 to avoid fetching entire history)
+    const result = await $`gh api repos/${owner}/${repo}/issues/${prNumber}/comments --jq '.[].body' 2>/dev/null`;
+    if (result.code === 0 && result.stdout) {
+      const bodies = result.stdout.toString();
+      const hasMatch = bodies.includes(commentSignature);
+      if (verbose && hasMatch) {
+        console.log(`[VERBOSE] Found existing comment with signature: "${commentSignature}"`);
+      }
+      return hasMatch;
+    }
+  } catch (error) {
+    // If check fails, allow posting to avoid silent failures
+    if (verbose) {
+      console.log(`[VERBOSE] Failed to check for existing comment: ${error.message}`);
+    }
+  }
+  return false;
+};
+
 /**
  * Check for new comments from non-bot users since last commit
  * @returns {Promise<{hasNewComments: boolean, comments: Array}>}
@@ -258,6 +289,11 @@ export const watchUntilMergeable = async params => {
   let latestSessionId = null;
   let latestAnthropicCost = null;
 
+  // Issue #1323: Track actual restart count separately from check cycle iteration
+  // `iteration` counts check cycles (how many times we check for blockers)
+  // `restartCount` counts actual AI tool executions (when we actually restart the AI)
+  let restartCount = 0;
+
   // Track consecutive API errors for retry limit
   const MAX_API_ERROR_RETRIES = 3;
   let consecutiveApiErrors = 0;
@@ -344,10 +380,17 @@ export const watchUntilMergeable = async params => {
           await log(formatAligned('', 'PR is ready to be merged manually', '', 2));
           await log(formatAligned('', 'Exiting auto-restart-until-mergeable mode', '', 2));
 
-          // Post success comment
+          // Issue #1323: Post success comment only if one doesn't already exist
+          // This prevents duplicate comments when multiple processes reach this point
           try {
-            const commentBody = `## ✅ Ready to merge\n\nThis pull request is now ready to be merged:\n- All CI checks have passed\n- No merge conflicts\n- No pending changes\n\n---\n*Monitored by hive-mind with --auto-restart-until-mergeable flag*`;
-            await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
+            const readyToMergeSignature = '## ✅ Ready to merge';
+            const hasExistingComment = await checkForExistingComment(owner, repo, prNumber, readyToMergeSignature, argv.verbose);
+            if (!hasExistingComment) {
+              const commentBody = `## ✅ Ready to merge\n\nThis pull request is now ready to be merged:\n- All CI checks have passed\n- No merge conflicts\n- No pending changes\n\n---\n*Monitored by hive-mind with --auto-restart-until-mergeable flag*`;
+              await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
+            } else {
+              await log(formatAligned('', 'Skipping duplicate "Ready to merge" comment', '', 2));
+            }
           } catch {
             // Don't fail if comment posting fails
           }
@@ -518,10 +561,14 @@ Once the billing issue is resolved, you can re-run the CI checks or push a new c
       }
 
       if (shouldRestart) {
+        // Issue #1323: Increment restart count (actual AI executions, not check cycles)
+        restartCount++;
+
         // Add standard instructions for auto-restart-until-mergeable mode using shared utility
         feedbackLines.push(...buildAutoRestartInstructions());
 
         await log(formatAligned('🔄', 'RESTART TRIGGERED:', restartReason));
+        await log(formatAligned('', 'Restart iteration:', `${restartCount}`, 2));
         await log('');
 
         // Post a comment to PR about the restart
@@ -601,7 +648,8 @@ Once the billing issue is resolved, you can re-run the CI checks or push a new c
             try {
               const logFile = getLogFile();
               if (logFile) {
-                const customTitle = `🔄 Auto-restart-until-mergeable Log (iteration ${iteration})`;
+                // Issue #1323: Use restartCount (actual AI executions) instead of iteration (check cycles)
+                const customTitle = `🔄 Auto-restart-until-mergeable Log (iteration ${restartCount})`;
                 await attachLogToGitHub({
                   logFile,
                   targetType: 'pr',
@@ -779,11 +827,17 @@ export const startAutoRestartUntilMergeable = async params => {
     await log(formatAligned('', 'Action:', 'PR is ready for manual merge by a repository maintainer', 2));
     await log('');
 
-    // Post a comment to the PR notifying the maintainer
+    // Issue #1323: Post a comment to the PR notifying the maintainer (with deduplication)
     try {
-      const commentBody = `## ✅ Ready to merge\n\nThis pull request is ready to be merged. Auto-merge was requested (\`--auto-merge\`) but cannot be performed because this PR was created from a fork (no write access to the target repository).\n\nPlease merge manually.\n\n---\n*hive-mind with --auto-merge flag (fork mode)*`;
-      await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
-      await log(formatAligned('', '💬 Posted merge readiness notification to PR', '', 2));
+      const readyToMergeSignature = '## ✅ Ready to merge';
+      const hasExistingComment = await checkForExistingComment(owner, repo, prNumber, readyToMergeSignature, argv.verbose);
+      if (!hasExistingComment) {
+        const commentBody = `## ✅ Ready to merge\n\nThis pull request is ready to be merged. Auto-merge was requested (\`--auto-merge\`) but cannot be performed because this PR was created from a fork (no write access to the target repository).\n\nPlease merge manually.\n\n---\n*hive-mind with --auto-merge flag (fork mode)*`;
+        await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
+        await log(formatAligned('', '💬 Posted merge readiness notification to PR', '', 2));
+      } else {
+        await log(formatAligned('', 'Skipping duplicate "Ready to merge" comment', '', 2));
+      }
     } catch {
       // Don't fail if comment posting fails
     }
@@ -802,11 +856,17 @@ export const startAutoRestartUntilMergeable = async params => {
       await log(formatAligned('', 'Action:', 'PR is ready for manual merge by a repository maintainer', 2));
       await log('');
 
-      // Post a comment to the PR notifying the maintainer
+      // Issue #1323: Post a comment to the PR notifying the maintainer (with deduplication)
       try {
-        const commentBody = `## ✅ Ready to merge\n\nThis pull request is ready to be merged. Auto-merge was requested (\`--auto-merge\`) but cannot be performed because the authenticated user lacks write access to \`${owner}/${repo}\` (current permission: \`${permission || 'unknown'}\`).\n\nPlease merge manually.\n\n---\n*hive-mind with --auto-merge flag*`;
-        await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
-        await log(formatAligned('', '💬 Posted merge readiness notification to PR', '', 2));
+        const readyToMergeSignature = '## ✅ Ready to merge';
+        const hasExistingComment = await checkForExistingComment(owner, repo, prNumber, readyToMergeSignature, argv.verbose);
+        if (!hasExistingComment) {
+          const commentBody = `## ✅ Ready to merge\n\nThis pull request is ready to be merged. Auto-merge was requested (\`--auto-merge\`) but cannot be performed because the authenticated user lacks write access to \`${owner}/${repo}\` (current permission: \`${permission || 'unknown'}\`).\n\nPlease merge manually.\n\n---\n*hive-mind with --auto-merge flag*`;
+          await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
+          await log(formatAligned('', '💬 Posted merge readiness notification to PR', '', 2));
+        } else {
+          await log(formatAligned('', 'Skipping duplicate "Ready to merge" comment', '', 2));
+        }
       } catch {
         // Don't fail if comment posting fails
       }