@link-assistant/hive-mind 1.23.9 → 1.23.11

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @link-assistant/hive-mind
 
+## 1.23.11
+
+### Patch Changes
+
+- f1ba29d: Comprehensive CI/CD status handling for --auto-restart-until-mergeable mode
+  - Detect when CI failures are caused by billing/spending limits via check run annotations
+  - For private repositories: Post an explanatory comment and stop (requires human intervention)
+  - For public repositories: Apply exponential backoff and wait (unusual case)
+  - Distinguish between CI failure, cancelled, pending, queued, and billing limit states
+  - Automatically re-trigger cancelled CI/CD workflow runs instead of restarting AI
+  - Only restart AI when genuine code failures occur (not for cancelled/pending/billing)
+  - Wait for all CI/CD checks to complete before deciding on AI restart
+  - New functions: getDetailedCIStatus(), rerunWorkflowRun(), rerunFailedJobs(), getWorkflowRunsForSha()
+  - Expanded test coverage: 45 tests covering all CI/CD status scenarios and decision logic
+
+## 1.23.10
+
+### Patch Changes
+
+- cc57624: Add retry logic for fork validation network errors (Issue #1311). The validateForkParent function now retries up to 3 times with exponential backoff for transient network errors like TCP timeouts. Network errors now show a distinct error message with helpful retry suggestions instead of incorrectly reporting a fork parent mismatch.
+
 ## 1.23.9
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "1.23.9",
+  "version": "1.23.11",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/github-merge.lib.mjs

@@ -790,6 +790,411 @@ export async function getDefaultBranch(owner, repo, verbose = false) {
   }
 }
 
+/**
+ * Get annotations for a check run
+ * Issue #1314: Used to detect billing limit errors
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {number} checkRunId - Check run ID
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<Array<Object>>} Array of annotation objects
+ */
+export async function getCheckRunAnnotations(owner, repo, checkRunId, verbose = false) {
+  try {
+    const { stdout } = await exec(`gh api repos/${owner}/${repo}/check-runs/${checkRunId}/annotations 2>/dev/null || echo "[]"`);
+    const annotations = JSON.parse(stdout.trim() || '[]');
+
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Check run ${checkRunId} has ${annotations.length} annotations`);
+    }
+
+    return annotations;
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Error fetching annotations for check run ${checkRunId}: ${error.message}`);
+    }
+    return [];
+  }
+}
+
+/**
+ * Check if repository is private
+ * Issue #1314: Used to determine behavior when billing limits are reached
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<{isPrivate: boolean, visibility: string|null}>}
+ */
+export async function getRepoVisibility(owner, repo, verbose = false) {
+  try {
+    const { stdout } = await exec(`gh api repos/${owner}/${repo} --jq '{isPrivate: .private, visibility: .visibility}'`);
+    const info = JSON.parse(stdout.trim());
+
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Repository ${owner}/${repo} visibility: ${info.visibility}, private: ${info.isPrivate}`);
+    }
+
+    return {
+      isPrivate: info.isPrivate === true,
+      visibility: info.visibility || null,
+    };
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Error checking repository visibility: ${error.message}`);
+    }
+    // Assume private if we can't determine (safer default)
+    return { isPrivate: true, visibility: null };
+  }
+}
+
+/**
+ * Known billing limit error message pattern
+ * Issue #1314: This is the exact message GitHub uses for billing/spending limit errors
+ */
+export const BILLING_LIMIT_ERROR_PATTERN = 'The job was not started because recent account payments have failed or your spending limit needs to be increased';
+
+/**
+ * Check if CI failure is due to billing/spending limits
+ * Issue #1314: Detects when GitHub Actions jobs fail due to billing issues rather than code problems
+ *
+ * Detection criteria:
+ * 1. Job has conclusion='failure'
+ * 2. Job has empty steps array (no steps were executed)
+ * 3. Job has runner_id=0 or null (no runner was assigned)
+ * 4. Annotation contains the billing limit error message
+ *
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {number} prNumber - Pull request number
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<{isBillingLimitError: boolean, message: string|null, affectedJobs: string[], allJobsAffected: boolean}>}
+ */
+export async function checkForBillingLimitError(owner, repo, prNumber, verbose = false) {
+  try {
+    // Get the PR's head SHA
+    const { stdout: prJson } = await exec(`gh pr view ${prNumber} --repo ${owner}/${repo} --json headRefOid`);
+    const prData = JSON.parse(prJson.trim());
+    const sha = prData.headRefOid;
+
+    // Get workflow runs for this SHA
+    const { stdout: runsJson } = await exec(`gh api "repos/${owner}/${repo}/actions/runs?head_sha=${sha}&per_page=10" --jq '.workflow_runs[].id'`);
+    const runIds = runsJson.trim().split('\n').filter(Boolean);
+
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Found ${runIds.length} workflow runs for PR #${prNumber} at SHA ${sha.substring(0, 7)}`);
+    }
+
+    const affectedJobs = [];
+    let totalJobs = 0;
+
+    // Check each workflow run's jobs
+    for (const runId of runIds) {
+      try {
+        const { stdout: jobsJson } = await exec(`gh api repos/${owner}/${repo}/actions/runs/${runId}/jobs --jq '.jobs'`);
+        const jobs = JSON.parse(jobsJson.trim() || '[]');
+
+        for (const job of jobs) {
+          totalJobs++;
+
+          // Check for billing limit indicators:
+          // 1. Conclusion is failure
+          // 2. Steps array is empty (no steps were executed)
+          // 3. Runner ID is 0 or null (no runner was assigned)
+          const hasNoSteps = !job.steps || job.steps.length === 0;
+          const hasNoRunner = job.runner_id === 0 || job.runner_id === null;
+
+          if (job.conclusion === 'failure' && hasNoSteps && hasNoRunner) {
+            // Fetch annotations to confirm billing limit error
+            const annotations = await getCheckRunAnnotations(owner, repo, job.id, verbose);
+
+            const billingAnnotation = annotations.find(a => a.message?.includes(BILLING_LIMIT_ERROR_PATTERN));
+
+            if (billingAnnotation) {
+              affectedJobs.push(job.name);
+
+              if (verbose) {
+                console.log(`[VERBOSE] /merge: Job "${job.name}" (ID: ${job.id}) failed due to billing limits`);
+              }
+            }
+          }
+        }
+      } catch (error) {
+        if (verbose) {
+          console.log(`[VERBOSE] /merge: Error checking jobs for run ${runId}: ${error.message}`);
+        }
+      }
+    }
+
+    const isBillingLimitError = affectedJobs.length > 0;
+    const allJobsAffected = totalJobs > 0 && affectedJobs.length === totalJobs;
+
+    if (verbose && isBillingLimitError) {
+      console.log(`[VERBOSE] /merge: Billing limit detected - ${affectedJobs.length}/${totalJobs} jobs affected`);
+    }
+
+    return {
+      isBillingLimitError,
+      message: isBillingLimitError ? BILLING_LIMIT_ERROR_PATTERN : null,
+      affectedJobs,
+      allJobsAffected,
+    };
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Error checking for billing limit: ${error.message}`);
+    }
+    return {
+      isBillingLimitError: false,
+      message: null,
+      affectedJobs: [],
+      allJobsAffected: false,
+    };
+  }
+}
+
+/**
+ * Re-run all jobs in a workflow run
+ * Issue #1314: Used to re-trigger CI jobs that were cancelled or not started
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {number} runId - Workflow run ID
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<{success: boolean, error: string|null}>}
+ */
+export async function rerunWorkflowRun(owner, repo, runId, verbose = false) {
+  try {
+    await exec(`gh api repos/${owner}/${repo}/actions/runs/${runId}/rerun -X POST`);
+    // GitHub returns 201 on success
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Successfully triggered re-run for workflow ${runId}`);
+    }
+    return { success: true, error: null };
+  } catch (error) {
+    // exec throws when command exits non-zero (e.g., 404 Not Found)
+    const errorMessage = error.stderr?.trim() || error.stdout?.trim() || error.message;
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Failed to re-run workflow ${runId}: ${errorMessage}`);
+    }
+    return { success: false, error: errorMessage };
+  }
+}
+
+/**
+ * Re-run only failed jobs in a workflow run
+ * Issue #1314: More targeted than full re-run, only retries failed jobs
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {number} runId - Workflow run ID
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<{success: boolean, error: string|null}>}
+ */
+export async function rerunFailedJobs(owner, repo, runId, verbose = false) {
+  try {
+    await exec(`gh api repos/${owner}/${repo}/actions/runs/${runId}/rerun-failed-jobs -X POST`);
+    // GitHub returns 201 on success
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Successfully triggered re-run of failed jobs for workflow ${runId}`);
+    }
+    return { success: true, error: null };
+  } catch (error) {
+    const errorMessage = error.stderr?.trim() || error.stdout?.trim() || error.message;
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Failed to re-run failed jobs for workflow ${runId}: ${errorMessage}`);
+    }
+    return { success: false, error: errorMessage };
+  }
+}
+
+/**
+ * Get detailed CI status for a PR, distinguishing between different non-success states
+ * Issue #1314: Enhanced version that separates cancelled, queued, and billing-limited states
+ *
+ * Possible returned statuses:
+ * - 'success': All checks passed
+ * - 'failure': Some checks failed (genuine code failures, timed_out, or action_required)
+ * - 'cancelled': Some checks were cancelled or stale (need re-triggering)
+ * - 'pending': Some checks are still running, queued, waiting, or requested
+ * - 'billing_limit': Failures are due to billing/spending limits (determined by caller)
+ * - 'no_checks': No CI checks found yet (race condition after push)
+ * - 'unknown': Unable to determine status
+ *
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {number} prNumber - Pull request number
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<Object>} Detailed CI status object
+ */
+export async function getDetailedCIStatus(owner, repo, prNumber, verbose = false) {
+  try {
+    // Get the PR's head SHA
+    const { stdout: prJson } = await exec(`gh pr view ${prNumber} --repo ${owner}/${repo} --json headRefOid`);
+    const prData = JSON.parse(prJson.trim());
+    const sha = prData.headRefOid;
+
+    // Get check runs for this SHA
+    const { stdout: checksJson } = await exec(`gh api repos/${owner}/${repo}/commits/${sha}/check-runs --paginate --jq '.check_runs'`);
+    const checkRuns = JSON.parse(checksJson.trim() || '[]');
+
+    // Get commit statuses
+    const { stdout: statusJson } = await exec(`gh api repos/${owner}/${repo}/commits/${sha}/status --jq '.statuses'`);
+    const statuses = JSON.parse(statusJson.trim() || '[]');
+
+    // Build detailed checks list
+    const allChecks = [
+      ...checkRuns.map(check => ({
+        name: check.name,
+        status: check.status, // queued, in_progress, completed
+        conclusion: check.conclusion, // success, failure, cancelled, timed_out, skipped, neutral, action_required, stale, null
+        type: 'check_run',
+        id: check.id,
+      })),
+      ...statuses.map(status => ({
+        name: status.context,
+        status: status.state === 'pending' ? 'in_progress' : 'completed',
+        conclusion: status.state === 'pending' ? null : status.state === 'success' ? 'success' : status.state === 'failure' ? 'failure' : status.state,
+        type: 'status',
+        id: null,
+      })),
+    ];
+
+    // No checks yet
+    if (allChecks.length === 0) {
+      if (verbose) {
+        console.log(`[VERBOSE] /merge: PR #${prNumber} has no CI checks yet - treating as no_checks`);
+      }
+      return {
+        status: 'no_checks',
+        checks: [],
+        sha,
+        hasFailures: false,
+        hasCancelled: false,
+        hasStale: false,
+        hasPending: false,
+        hasQueued: false,
+        allPassed: false,
+        failedChecks: [],
+        cancelledChecks: [],
+        staleChecks: [],
+        pendingChecks: [],
+        queuedChecks: [],
+        passedChecks: [],
+      };
+    }
+
+    // Categorize checks
+    // Note: GitHub check run conclusions include: success, failure, cancelled, timed_out, skipped,
+    // neutral, action_required, stale, null (not yet completed)
+    // GitHub check run statuses include: queued, in_progress, completed, waiting, requested, pending
+    const passedChecks = allChecks.filter(c => c.conclusion === 'success' || c.conclusion === 'skipped' || c.conclusion === 'neutral');
+    const failedChecks = allChecks.filter(c => c.conclusion === 'failure' || c.conclusion === 'timed_out' || c.conclusion === 'action_required');
+    const cancelledChecks = allChecks.filter(c => c.conclusion === 'cancelled');
+    const staleChecks = allChecks.filter(c => c.conclusion === 'stale');
+    const pendingChecks = allChecks.filter(c => (c.status === 'in_progress' || c.status === 'waiting' || c.status === 'requested' || c.status === 'pending') && c.conclusion === null);
+    const queuedChecks = allChecks.filter(c => c.status === 'queued' && c.conclusion === null);
+
+    const hasFailures = failedChecks.length > 0;
+    const hasCancelled = cancelledChecks.length > 0;
+    const hasStale = staleChecks.length > 0;
+    const hasPending = pendingChecks.length > 0;
+    const hasQueued = queuedChecks.length > 0;
+    const allPassed = !hasFailures && !hasCancelled && !hasStale && !hasPending && !hasQueued && passedChecks.length === allChecks.length;
+
+    // Determine overall status
+    let status;
+    if (allPassed) {
+      status = 'success';
+    } else if (hasPending || hasQueued) {
+      // Some checks are still running, queued, or waiting for a runner - wait for completion
+      status = 'pending';
+    } else if (hasStale && !hasFailures && !hasCancelled) {
+      // Stale checks need to be re-triggered (similar to cancelled)
+      status = 'cancelled';
+    } else if (hasFailures && !hasCancelled && !hasStale) {
+      status = 'failure';
+    } else if ((hasCancelled || hasStale) && !hasFailures) {
+      status = 'cancelled';
+    } else if (hasFailures && (hasCancelled || hasStale)) {
+      // Mixed: some failed, some cancelled/stale - report as failure (the failures need attention)
+      status = 'failure';
+    } else {
+      status = 'unknown';
+    }
+
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: PR #${prNumber} detailed CI status: ${status}`);
+      console.log(`[VERBOSE] /merge:   Total: ${allChecks.length}, Passed: ${passedChecks.length}, Failed: ${failedChecks.length}, Cancelled: ${cancelledChecks.length}, Stale: ${staleChecks.length}, Pending: ${pendingChecks.length}, Queued: ${queuedChecks.length}`);
+    }
+
+    return {
+      status,
+      checks: allChecks,
+      sha,
+      hasFailures,
+      hasCancelled,
+      hasStale,
+      hasPending,
+      hasQueued,
+      allPassed,
+      failedChecks,
+      cancelledChecks,
+      staleChecks,
+      pendingChecks,
+      queuedChecks,
+      passedChecks,
+    };
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Error getting detailed CI status: ${error.message}`);
+    }
+    return {
+      status: 'unknown',
+      checks: [],
+      sha: null,
+      hasFailures: false,
+      hasCancelled: false,
+      hasStale: false,
+      hasPending: false,
+      hasQueued: false,
+      allPassed: false,
+      failedChecks: [],
+      cancelledChecks: [],
+      staleChecks: [],
+      pendingChecks: [],
+      queuedChecks: [],
+      passedChecks: [],
+    };
+  }
+}
+
+/**
+ * Get workflow run IDs for a specific commit SHA
+ * Issue #1314: Helper to find workflow runs to re-trigger
+ * @param {string} owner - Repository owner
+ * @param {string} repo - Repository name
+ * @param {string} sha - Commit SHA
+ * @param {boolean} verbose - Whether to log verbose output
+ * @returns {Promise<Array<{id: number, status: string, conclusion: string|null, name: string}>>}
+ */
+export async function getWorkflowRunsForSha(owner, repo, sha, verbose = false) {
+  try {
+    const { stdout } = await exec(`gh api "repos/${owner}/${repo}/actions/runs?head_sha=${sha}&per_page=20" --jq '[.workflow_runs[] | {id: .id, status: .status, conclusion: .conclusion, name: .name}]'`);
+    const runs = JSON.parse(stdout.trim() || '[]');
+
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Found ${runs.length} workflow runs for SHA ${sha.substring(0, 7)}`);
+      for (const run of runs) {
+        console.log(`[VERBOSE] /merge:   - ${run.name} (${run.id}): status=${run.status}, conclusion=${run.conclusion}`);
+      }
+    }
+
+    return runs;
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] /merge: Error fetching workflow runs for SHA ${sha}: ${error.message}`);
+    }
+    return [];
+  }
+}
+
 export default {
   READY_LABEL,
   checkReadyLabelExists,
@@ -809,4 +1214,14 @@ export default {
   getActiveBranchRuns,
   waitForBranchCI,
   getDefaultBranch,
+  // Issue #1314: Billing limit detection
+  getCheckRunAnnotations,
+  getRepoVisibility,
+  checkForBillingLimitError,
+  BILLING_LIMIT_ERROR_PATTERN,
+  // Issue #1314: Enhanced CI status and re-run capabilities
+  getDetailedCIStatus,
+  rerunWorkflowRun,
+  rerunFailedJobs,
+  getWorkflowRunsForSha,
 };

package/src/lib.mjs

@@ -243,6 +243,22 @@ export const retry = async (fn, options = {}) => {
   }
 };
 
+/**
+ * Check if an error is a transient network error that can be retried.
+ * Used by validateForkParent to detect network timeouts (Issue #1311).
+ * @param {Error|string} error - The error to check
+ * @returns {boolean} True if the error is transient and retryable
+ */
+export const isTransientNetworkError = error => {
+  const msg = (error?.message || error?.toString() || '').toLowerCase();
+  const output = (error?.stderr?.toString() || error?.stdout?.toString() || '').toLowerCase();
+  const combined = msg + ' ' + output;
+
+  const transientPatterns = ['i/o timeout', 'dial tcp', 'connection refused', 'connection reset', 'econnreset', 'etimedout', 'enotfound', 'ehostunreach', 'enetunreach', 'network is unreachable', 'temporary failure', 'http 502', 'http 503', 'http 504', 'bad gateway', 'service unavailable', 'gateway timeout', 'tls handshake timeout', 'ssl_error', 'socket hang up'];
+
+  return transientPatterns.some(pattern => combined.includes(pattern));
+};
+
 /**
  * Format bytes to human readable string
  * @param {number} bytes - Number of bytes

package/src/solve.auto-merge.lib.mjs

@@ -33,7 +33,7 @@ const { reportError } = sentryLib;
 
 // Import GitHub merge functions
 const githubMergeLib = await import('./github-merge.lib.mjs');
-const { checkPRCIStatus, checkPRMergeable, checkMergePermissions, mergePullRequest, waitForCI } = githubMergeLib;
+const { checkPRMergeable, checkMergePermissions, mergePullRequest, waitForCI, checkForBillingLimitError, getRepoVisibility, BILLING_LIMIT_ERROR_PATTERN, getDetailedCIStatus, rerunWorkflowRun, getWorkflowRunsForSha } = githubMergeLib;
 
 // Import GitHub functions for log attachment
 const githubLib = await import('./github.lib.mjs');
@@ -141,23 +141,93 @@ const checkForNonBotComments = async (owner, repo, prNumber, issueNumber, lastCh
 
 /**
  * Get the reasons why PR is not mergeable
+ * Issue #1314: Comprehensive CI/CD status handling covering all possible states:
+ * - success: All CI passed → no blocker
+ * - failure: Genuine code failures → restart AI
+ * - cancelled: Manually cancelled or workflow cancelled → re-trigger, don't restart AI
+ * - pending/queued: Still running or waiting for runner → wait, don't restart AI
+ * - billing_limit: Billing/spending limit reached → stop (private) or wait (public)
+ * - no_checks: No CI checks yet (race condition) → wait
  */
 const getMergeBlockers = async (owner, repo, prNumber, verbose = false) => {
   const blockers = [];
 
-  // Check CI status
-  const ciStatus = await checkPRCIStatus(owner, repo, prNumber, verbose);
-  if (ciStatus.status === 'failure') {
+  // Use detailed CI status to distinguish between all possible states
+  const ciStatus = await getDetailedCIStatus(owner, repo, prNumber, verbose);
+
+  if (ciStatus.status === 'no_checks') {
+    // No CI checks exist yet - race condition after push, treat as pending
     blockers.push({
-      type: 'ci_failure',
-      message: 'CI/CD checks are failing',
-      details: ciStatus.checks.filter(c => c.conclusion === 'failure').map(c => c.name),
+      type: 'ci_pending',
+      message: 'CI/CD checks have not started yet (waiting for checks to appear)',
+      details: [],
     });
   } else if (ciStatus.status === 'pending') {
+    // CI is still running or queued - wait for completion
+    const pendingNames = [...ciStatus.pendingChecks, ...ciStatus.queuedChecks].map(c => c.name);
+    blockers.push({
+      type: 'ci_pending',
+      message: 'CI/CD checks are still running or queued',
+      details: pendingNames,
+    });
+  } else if (ciStatus.status === 'cancelled') {
+    // All non-passed checks are cancelled or stale (no genuine failures)
+    // First check if this is actually a billing limit issue (billing-limited jobs may appear as cancelled)
+    const billingCheck = await checkForBillingLimitError(owner, repo, prNumber, verbose);
+    if (billingCheck.isBillingLimitError) {
+      blockers.push({
+        type: 'billing_limit',
+        message: 'GitHub Actions billing/spending limit reached',
+        details: billingCheck.affectedJobs,
+        allJobsAffected: billingCheck.allJobsAffected,
+        billingMessage: billingCheck.message,
+      });
+    } else {
+      // These need to be re-triggered, NOT treated as AI-fixable failures
+      const cancelledOrStaleChecks = [...ciStatus.cancelledChecks, ...(ciStatus.staleChecks || [])];
+      blockers.push({
+        type: 'ci_cancelled',
+        message: 'CI/CD checks were cancelled or became stale',
+        details: cancelledOrStaleChecks.map(c => c.name),
+        sha: ciStatus.sha,
+      });
+    }
+  } else if (ciStatus.status === 'failure') {
+    // Some checks genuinely failed - check if it's billing limits first
+    const billingCheck = await checkForBillingLimitError(owner, repo, prNumber, verbose);
+
+    if (billingCheck.isBillingLimitError) {
+      blockers.push({
+        type: 'billing_limit',
+        message: 'GitHub Actions billing/spending limit reached',
+        details: billingCheck.affectedJobs,
+        allJobsAffected: billingCheck.allJobsAffected,
+        billingMessage: billingCheck.message,
+      });
+    } else {
+      // Check if there are also cancelled/stale checks alongside failures
+      const cancelledOrStaleChecks = [...(ciStatus.hasCancelled ? ciStatus.cancelledChecks : []), ...((ciStatus.hasStale && ciStatus.staleChecks) || [])];
+      if (cancelledOrStaleChecks.length > 0) {
+        blockers.push({
+          type: 'ci_cancelled',
+          message: 'Some CI/CD checks were cancelled or became stale (will be re-triggered)',
+          details: cancelledOrStaleChecks.map(c => c.name),
+          sha: ciStatus.sha,
+        });
+      }
+      blockers.push({
+        type: 'ci_failure',
+        message: 'CI/CD checks are failing',
+        details: ciStatus.failedChecks.map(c => c.name),
+      });
+    }
+  } else if (ciStatus.status === 'unknown') {
+    // Unable to determine CI status - treat as pending to be safe
+    // Do NOT treat as mergeable (which would be incorrect)
     blockers.push({
       type: 'ci_pending',
-      message: 'CI/CD checks are still running',
-      details: ciStatus.checks.filter(c => c.status !== 'completed').map(c => c.name),
+      message: 'CI/CD status could not be determined (will retry)',
+      details: [],
     });
   }
 
@@ -303,9 +373,112 @@ export const watchUntilMergeable = async params => {
         feedbackLines.push('Please review and address the feedback from these comments.');
       }
 
-      // Reason 2: CI failures
+      // Issue #1314: Check for billing limit errors BEFORE regular CI failures
+      // Billing limits require human intervention and should NOT trigger AI restarts
+      const billingBlocker = blockers.find(b => b.type === 'billing_limit');
+      if (billingBlocker) {
+        await log('');
+        await log(formatAligned('💳', 'GITHUB ACTIONS BILLING LIMIT DETECTED', ''));
+        await log(formatAligned('', 'Affected jobs:', billingBlocker.details.join(', '), 2));
+        await log(formatAligned('', 'All jobs affected:', billingBlocker.allJobsAffected ? 'Yes' : 'No', 2));
+        await log('');
+
+        // Check if this is a private repository
+        const repoInfo = await getRepoVisibility(owner, repo, argv.verbose);
+
+        if (repoInfo.isPrivate) {
+          // For private repos, human intervention is required - stop and post comment
+          await log(formatAligned('🛑', 'STOPPING', 'Private repository - billing limit requires human intervention'));
+          await log(formatAligned('', 'Action required:', "Check the 'Billing & plans' section in your GitHub settings", 2));
+
+          // Post comment explaining the billing limit issue
+          try {
+            const commentBody = `## 💳 GitHub Actions Billing Limit Reached
+
+The CI/CD jobs could not start due to billing/spending limits.
+
+**Affected jobs:**
+${billingBlocker.details.map(j => `- ${j}`).join('\n')}
+
+**Error message:**
+> ${billingBlocker.billingMessage || BILLING_LIMIT_ERROR_PATTERN}
+
+**Action Required:**
+Please check the 'Billing & plans' section in your GitHub settings and either:
+1. Add or update your payment method
+2. Increase your spending limit
+3. Wait for the free tier limits to reset (if applicable)
+
+Once the billing issue is resolved, you can re-run the CI checks or push a new commit to trigger a new run.
+
+---
+*Detected by hive-mind with --auto-restart-until-mergeable flag. This is NOT a code issue - human intervention is required.*`;
+            await $`gh pr comment ${prNumber} --repo ${owner}/${repo} --body ${commentBody}`;
+            await log(formatAligned('', '💬 Posted billing limit notification to PR', '', 2));
+          } catch (commentError) {
+            reportError(commentError, {
+              context: 'post_billing_limit_comment',
+              owner,
+              repo,
+              prNumber,
+              operation: 'comment_on_pr',
+            });
+            await log(formatAligned('', '⚠️  Could not post comment to PR', '', 2));
+          }
+
+          return { success: false, reason: 'billing_limit', latestSessionId, latestAnthropicCost };
+        } else {
+          // For public repos (unusual case), apply exponential backoff and wait
+          // Public repos typically have unlimited free CI, so this is unexpected
+          await log(formatAligned('⏳', 'Public repository with billing limit (unusual)', 'Applying exponential backoff'));
+          await log(formatAligned('', 'Next check in:', `${currentBackoffSeconds} seconds`, 2));
+
+          // Don't trigger AI restart - just wait and check again
+          // The backoff will be applied at the end of the loop
+          currentBackoffSeconds = Math.min(currentBackoffSeconds * 2, 3600); // Max 1 hour
+        }
+      }
+
+      // Issue #1314: Handle cancelled CI/CD checks - re-trigger them instead of restarting AI
+      // Cancelled checks (e.g., manually cancelled, cancelled by another workflow) should be
+      // re-triggered automatically. We should NOT restart the AI for these.
+      const cancelledBlocker = blockers.find(b => b.type === 'ci_cancelled');
+      if (cancelledBlocker && !billingBlocker) {
+        await log('');
+        await log(formatAligned('🔄', 'CANCELLED CI/CD CHECKS DETECTED', ''));
+        await log(formatAligned('', 'Cancelled checks:', cancelledBlocker.details.join(', '), 2));
+
+        // Attempt to re-trigger the cancelled/stale workflow runs
+        const sha = cancelledBlocker.sha;
+        if (sha) {
+          const runs = await getWorkflowRunsForSha(owner, repo, sha, argv.verbose);
+          const retriggerable = runs.filter(r => r.conclusion === 'cancelled' || r.conclusion === 'stale');
+          let rerunTriggered = false;
+
+          for (const run of retriggerable) {
+            await log(formatAligned('', `Re-triggering workflow "${run.name}" (${run.id})...`, '', 2));
+            const rerunResult = await rerunWorkflowRun(owner, repo, run.id, argv.verbose);
+            if (rerunResult.success) {
+              await log(formatAligned('', `✅ Re-triggered: ${run.name}`, '', 2));
+              rerunTriggered = true;
+            } else {
+              await log(formatAligned('', `⚠️  Could not re-trigger ${run.name}: ${rerunResult.error}`, '', 2));
+            }
+          }
+
+          if (rerunTriggered) {
+            await log(formatAligned('⏳', 'Waiting for re-triggered CI to complete...', '', 2));
+            // Don't restart AI - just wait for re-triggered jobs to complete
+            // The next iteration of the loop will check the new status
+          }
+        }
+        // Don't set shouldRestart for cancelled checks - wait for re-triggered jobs instead
+      }
+
+      // Reason 2: CI failures (only if NOT a billing limit issue and NOT just cancelled)
+      // Only restart AI when we have genuine code failures (real feedback to act on)
       const ciBlocker = blockers.find(b => b.type === 'ci_failure');
-      if (ciBlocker) {
+      if (ciBlocker && !billingBlocker) {
         shouldRestart = true;
         restartReason = restartReason ? `${restartReason}; CI failures` : 'CI failures detected';
         feedbackLines.push('❌ CI/CD checks are failing:');
@@ -468,8 +641,18 @@ export const watchUntilMergeable = async params => {
         // Update last check time after restart
         lastCheckTime = new Date();
       } else if (blockers.length > 0) {
-        // There are blockers but none that warrant a restart (e.g., CI pending)
-        await log(formatAligned('⏳', 'Waiting for:', blockers.map(b => b.message).join(', '), 2));
+        // There are blockers but none that warrant an AI restart
+        // Issue #1314: Distinguish between different waiting reasons
+        const pendingBlocker = blockers.find(b => b.type === 'ci_pending');
+        const cancelledOnly = blockers.every(b => b.type === 'ci_cancelled' || b.type === 'ci_pending');
+
+        if (cancelledOnly && cancelledBlocker) {
+          await log(formatAligned('🔄', 'Waiting for re-triggered CI:', cancelledBlocker.details.join(', '), 2));
+        } else if (pendingBlocker) {
+          await log(formatAligned('⏳', 'Waiting for CI:', pendingBlocker.details.length > 0 ? pendingBlocker.details.join(', ') : pendingBlocker.message, 2));
+        } else {
+          await log(formatAligned('⏳', 'Waiting for:', blockers.map(b => b.message).join(', '), 2));
+        }
       } else {
         await log(formatAligned('', 'No action needed', 'Continuing to monitor...', 2));
       }

package/src/solve.repository.lib.mjs

@@ -110,93 +110,82 @@ export const checkExistingForkOfRoot = async rootRepo => {
  * This prevents issues where a fork was created from an intermediate fork (fork of a fork)
  * instead of directly from the intended upstream repository.
  *
+ * Issue #1311: Added retry logic for transient network errors (TCP timeouts, etc.)
+ *
  * @param {string} forkRepo - The fork repository to validate (e.g., "user/repo")
  * @param {string} expectedUpstream - The expected upstream repository (e.g., "owner/repo")
- * @returns {Promise<{isValid: boolean, isFork: boolean, parent: string|null, source: string|null, error: string|null}>}
+ * @returns {Promise<{isValid: boolean, isFork: boolean, parent: string|null, source: string|null, error: string|null, isNetworkError?: boolean}>}
  */
 export const validateForkParent = async (forkRepo, expectedUpstream) => {
-  try {
-    const forkInfoResult = await $`gh api repos/${forkRepo} --jq '{fork: .fork, parent: .parent.full_name, source: .source.full_name}'`;
-
-    if (forkInfoResult.code !== 0) {
-      return {
-        isValid: false,
-        isFork: false,
-        parent: null,
-        source: null,
-        error: `Failed to get fork info for ${forkRepo}`,
-      };
-    }
+  // Issue #1311: Retry configuration for transient network errors
+  const maxAttempts = 3;
+  const baseDelay = 2000;
+  const networkErr = msg => ({ isValid: false, isFork: false, parent: null, source: null, error: msg, isNetworkError: true });
 
-    const forkInfo = JSON.parse(forkInfoResult.stdout.toString().trim());
-    const isFork = forkInfo.fork === true;
-    const parent = forkInfo.parent || null;
-    const source = forkInfo.source || null;
-
-    // If not a fork at all, it's invalid for our purposes
-    if (!isFork) {
-      return {
-        isValid: false,
-        isFork: false,
-        parent: null,
-        source: null,
-        error: `Repository ${forkRepo} is not a GitHub fork`,
-      };
-    }
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const forkInfoResult = await $`gh api repos/${forkRepo} --jq '{fork: .fork, parent: .parent.full_name, source: .source.full_name}'`;
+
+      // Check for network errors in non-zero exit code
+      if (forkInfoResult.code !== 0) {
+        const errorOutput = (forkInfoResult.stderr?.toString() || '') + (forkInfoResult.stdout?.toString() || '');
+        // Issue #1311: Retry on transient network errors
+        if (lib.isTransientNetworkError({ message: errorOutput })) {
+          if (attempt < maxAttempts) {
+            const delay = baseDelay * Math.pow(2, attempt - 1);
+            await log(`   ⚠️ Network error, retrying in ${delay / 1000}s... (${attempt}/${maxAttempts})`, { level: 'warning' });
+            await lib.sleep(delay);
+            continue;
+          }
+          return networkErr(`Network error after ${maxAttempts} attempts: ${errorOutput.substring(0, 200)}`);
+        }
+        return { isValid: false, isFork: false, parent: null, source: null, error: `Failed to get fork info for ${forkRepo}` };
+      }
 
-    // The fork's PARENT (immediate upstream) should match expectedUpstream
-    // The SOURCE (ultimate root) is also acceptable as it indicates the fork
-    // is part of the correct hierarchy, just at a different level
-    const parentMatches = parent === expectedUpstream;
-    const sourceMatches = source === expectedUpstream;
-
-    // Ideal case: parent matches directly (fork was made from expected upstream)
-    if (parentMatches) {
-      return {
-        isValid: true,
-        isFork: true,
-        parent,
-        source,
-        error: null,
-      };
-    }
+      const forkInfo = JSON.parse(forkInfoResult.stdout.toString().trim());
+      const isFork = forkInfo.fork === true;
+      const parent = forkInfo.parent || null;
+      const source = forkInfo.source || null;
 
-    // Special case: source matches but parent doesn't
-    // This means the fork was made from an intermediate fork
-    // For issue #967, this is the problematic case we want to catch
-    if (sourceMatches && !parentMatches) {
-      return {
-        isValid: false,
-        isFork: true,
-        parent,
-        source,
-        error: `Fork ${forkRepo} was created from ${parent} (intermediate fork), not directly from ${expectedUpstream}. ` + `This can cause pull requests to include unexpected commits from the intermediate fork.`,
-      };
-    }
+      // If not a fork at all, it's invalid for our purposes
+      if (!isFork) {
+        return { isValid: false, isFork: false, parent: null, source: null, error: `Repository ${forkRepo} is not a GitHub fork` };
+      }
 
-    // Neither parent nor source matches - completely different repository tree
-    return {
-      isValid: false,
-      isFork: true,
-      parent,
-      source,
-      error: `Fork ${forkRepo} is from a different repository tree (parent: ${parent}, source: ${source}) and cannot be used with ${expectedUpstream}`,
-    };
-  } catch (error) {
-    reportError(error, {
-      context: 'validate_fork_parent',
-      forkRepo,
-      expectedUpstream,
-      operation: 'check_fork_hierarchy',
-    });
-    return {
-      isValid: false,
-      isFork: false,
-      parent: null,
-      source: null,
-      error: `Error validating fork parent: ${error.message}`,
-    };
+      // The fork's PARENT (immediate upstream) should match expectedUpstream
+      // The SOURCE (ultimate root) is also acceptable as it indicates the fork is part of the correct hierarchy
+      const parentMatches = parent === expectedUpstream;
+      const sourceMatches = source === expectedUpstream;
+
+      if (parentMatches) {
+        return { isValid: true, isFork: true, parent, source, error: null };
+      }
+
+      // Special case: source matches but parent doesn't - fork was made from an intermediate fork
+      // For issue #967, this is the problematic case we want to catch
+      if (sourceMatches && !parentMatches) {
+        return { isValid: false, isFork: true, parent, source, error: `Fork ${forkRepo} was created from ${parent} (intermediate fork), not directly from ${expectedUpstream}. This can cause pull requests to include unexpected commits from the intermediate fork.` };
+      }
+
+      // Neither parent nor source matches - completely different repository tree
+      return { isValid: false, isFork: true, parent, source, error: `Fork ${forkRepo} is from a different repository tree (parent: ${parent}, source: ${source}) and cannot be used with ${expectedUpstream}` };
+    } catch (error) {
+      // Issue #1311: Retry on transient network errors
+      if (lib.isTransientNetworkError(error)) {
+        if (attempt < maxAttempts) {
+          const delay = baseDelay * Math.pow(2, attempt - 1);
+          await log(`   ⚠️ Network error, retrying in ${delay / 1000}s... (${attempt}/${maxAttempts})`, { level: 'warning' });
+          await lib.sleep(delay);
+          continue;
+        }
+        reportError(error, { context: 'validate_fork_parent', forkRepo, expectedUpstream, operation: 'check_fork_hierarchy', attempt, maxAttempts, isNetworkError: true });
+        return networkErr(`Network error after ${maxAttempts} attempts: ${error.message}`);
+      }
+      reportError(error, { context: 'validate_fork_parent', forkRepo, expectedUpstream, operation: 'check_fork_hierarchy' });
+      return { isValid: false, isFork: false, parent: null, source: null, error: `Error validating fork parent: ${error.message}` };
+    }
   }
+  return networkErr(`Failed to validate fork after ${maxAttempts} attempts`);
 };
 
 /**
@@ -513,6 +502,25 @@ export const setupRepository = async (argv, owner, repo, forkOwner = null, issue
       const forkValidation = await validateForkParent(existingForkName, `${owner}/${repo}`);
 
       if (!forkValidation.isValid) {
+        // Issue #1311: Handle network errors separately from fork mismatch errors
+        if (forkValidation.isNetworkError) {
+          await log('');
+          await log(`${formatAligned('❌', 'NETWORK ERROR DURING FORK VALIDATION', '')}`, { level: 'error' });
+          await log('');
+          await log('  🔍 What happened:');
+          await log(`     Failed to connect to GitHub API while validating fork.`);
+          await log(`     Error: ${forkValidation.error}`);
+          await log('');
+          await log('  💡 This is likely a temporary network issue. You can:');
+          await log('     1. Wait a moment and try again');
+          await log('     2. Check your internet connection');
+          await log('     3. Check GitHub status: https://www.githubstatus.com/');
+          await log('');
+          await log('     Or use --no-fork to skip fork validation if you have write access.');
+          await log('');
+          await safeExit(1, 'Network error during fork validation - please retry');
+        }
+
         // Fork parent mismatch detected - this prevents issue #967
         await log('');
         await log(`${formatAligned('❌', 'FORK PARENT MISMATCH DETECTED', '')}`, { level: 'error' });
@@ -842,29 +850,44 @@ Thank you!`;
       const forkValidation = await validateForkParent(actualForkName, `${owner}/${repo}`);
 
       if (!forkValidation.isValid) {
-        // Fork parent mismatch detected
-        await log('');
-        await log(`${formatAligned('⚠️', 'FORK PARENT MISMATCH WARNING', '')}`, { level: 'warning' });
-        await log('');
-        await log('  🔍 Issue detected:');
-        if (!forkValidation.isFork) {
-          await log(`     The repository ${actualForkName} is NOT a GitHub fork.`);
+        // Issue #1311: Handle network errors separately from fork mismatch errors
+        if (forkValidation.isNetworkError) {
+          await log('');
+          await log(`${formatAligned('⚠️', 'NETWORK ERROR DURING FORK VALIDATION', '')}`, { level: 'warning' });
+          await log('');
+          await log('  🔍 What happened:');
+          await log(`     Failed to connect to GitHub API while validating fork.`);
+          await log(`     Error: ${forkValidation.error}`);
+          await log('');
+          await log('  💡 This is likely a temporary network issue.');
+          await log('     Continuing with the fork, but validation was skipped.');
+          await log('');
+          // Note: We continue here since this is someone else's fork and we can't verify it
         } else {
-          await log(`     The fork ${actualForkName} was created from ${forkValidation.parent},`);
-          await log(`     not directly from the target repository ${owner}/${repo}.`);
+          // Fork parent mismatch detected
+          await log('');
+          await log(`${formatAligned('⚠️', 'FORK PARENT MISMATCH WARNING', '')}`, { level: 'warning' });
+          await log('');
+          await log('  🔍 Issue detected:');
+          if (!forkValidation.isFork) {
+            await log(`     The repository ${actualForkName} is NOT a GitHub fork.`);
+          } else {
+            await log(`     The fork ${actualForkName} was created from ${forkValidation.parent},`);
+            await log(`     not directly from the target repository ${owner}/${repo}.`);
+          }
+          await log('');
+          await log('  📦 Fork relationship:');
+          await log(`     • Fork: ${actualForkName}`);
+          await log(`     • Fork parent: ${forkValidation.parent || 'N/A'}`);
+          await log(`     • Fork source (root): ${forkValidation.source || 'N/A'}`);
+          await log(`     • Expected parent: ${owner}/${repo}`);
+          await log('');
+          await log('  ⚠️  This may cause pull requests to include unexpected commits.');
+          await log('     Consider using --fork to create your own fork instead.');
+          await log('');
+          // Note: We don't exit here since this is someone else's fork and we're just using it
+          // The user should be aware but can proceed (they didn't create this fork)
         }
-        await log('');
-        await log('  📦 Fork relationship:');
-        await log(`     • Fork: ${actualForkName}`);
-        await log(`     • Fork parent: ${forkValidation.parent || 'N/A'}`);
-        await log(`     • Fork source (root): ${forkValidation.source || 'N/A'}`);
-        await log(`     • Expected parent: ${owner}/${repo}`);
-        await log('');
-        await log('  ⚠️  This may cause pull requests to include unexpected commits.');
-        await log('     Consider using --fork to create your own fork instead.');
-        await log('');
-        // Note: We don't exit here since this is someone else's fork and we're just using it
-        // The user should be aware but can proceed (they didn't create this fork)
       } else {
         await log(`${formatAligned('✅', 'Fork parent validated:', `${forkValidation.parent}`)}`);
       }