@link-assistant/hive-mind 1.56.5 ā 1.56.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +2 -2
- package/src/agent.lib.mjs +31 -4
- package/src/auto-iteration-limits.lib.mjs +33 -0
- package/src/claude.lib.mjs +9 -4
- package/src/codex.lib.mjs +47 -5
- package/src/hive.config.lib.mjs +1 -1
- package/src/hive.mjs +3 -0
- package/src/models/index.mjs +17 -0
- package/src/opencode.lib.mjs +28 -6
- package/src/option-suggestions.lib.mjs +1 -0
- package/src/solve.auto-continue.lib.mjs +14 -0
- package/src/solve.auto-merge.lib.mjs +91 -24
- package/src/solve.config.lib.mjs +25 -3
- package/src/solve.error-handlers.lib.mjs +1 -1
- package/src/solve.execution.lib.mjs +1 -1
- package/src/solve.mjs +12 -15
- package/src/solve.pre-pr-failure-notifier.lib.mjs +1 -1
- package/src/solve.results.lib.mjs +14 -8
- package/src/solve.watch.lib.mjs +14 -9
- package/src/telegram-bot.mjs +32 -42
- package/src/telegram-solve-command.lib.mjs +58 -0
- package/src/tool-retry.lib.mjs +118 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 1.56.7
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 37c895c: Retry capacity-related tool failures with exponential backoff and support fallback models for Codex, Claude, OpenCode, and Agent resumes.
|
|
8
|
+
- 16f341d: Limit automatic restart/resume loops to five iterations by default and avoid pre-restart branch sync when local merge state must be resolved by the AI session.
|
|
9
|
+
|
|
10
|
+
## 1.56.6
|
|
11
|
+
|
|
12
|
+
### Patch Changes
|
|
13
|
+
|
|
14
|
+
- e4037e1: Support Telegram solve and hive commands when options are placed before the GitHub URL.
|
|
15
|
+
|
|
3
16
|
## 1.56.5
|
|
4
17
|
|
|
5
18
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-assistant/hive-mind",
|
|
3
|
-
"version": "1.56.
|
|
3
|
+
"version": "1.56.7",
|
|
4
4
|
"description": "AI-powered issue solver and hive mind for collaborative problem solving",
|
|
5
5
|
"main": "src/hive.mjs",
|
|
6
6
|
"type": "module",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"hive-telegram-bot": "./src/telegram-bot.mjs"
|
|
16
16
|
},
|
|
17
17
|
"scripts": {
|
|
18
|
-
"test": "node tests/solve-queue.test.mjs && node tests/limits-display.test.mjs && node tests/test-usage-limit.mjs && node tests/test-codex-support.mjs && node tests/test-build-cost-info-string.mjs && node tests/test-claude-code-install-method.mjs && node tests/test-claude-quiet-config.mjs && node tests/test-configure-claude-bin.mjs && node tests/test-docker-release-order.mjs && node tests/test-docker-box-migration.mjs && node tests/test-hive-screens.mjs && node tests/test-issue-1616-pr-issue-link-preservation.mjs && node tests/test-pre-pr-failure-notifier-1640.mjs && node tests/test-ready-to-merge-pagination-1645.mjs && node tests/test-require-gh-paginate-rule.mjs && node tests/test-telegram-message-filters.mjs && node tests/test-telegram-bot-command-aliases.mjs && node tests/test-telegram-bot-configuration-isolation-links-notation.mjs && node tests/test-extract-isolation-from-args.mjs && node tests/test-solve-queue-command.mjs && node tests/test-queue-display-1267.mjs && node tests/test-telegram-bot-launcher.mjs",
|
|
18
|
+
"test": "node tests/solve-queue.test.mjs && node tests/limits-display.test.mjs && node tests/test-usage-limit.mjs && node tests/test-codex-support.mjs && node tests/test-build-cost-info-string.mjs && node tests/test-claude-code-install-method.mjs && node tests/test-claude-quiet-config.mjs && node tests/test-configure-claude-bin.mjs && node tests/test-docker-release-order.mjs && node tests/test-docker-box-migration.mjs && node tests/test-hive-screens.mjs && node tests/test-issue-1616-pr-issue-link-preservation.mjs && node tests/test-pre-pr-failure-notifier-1640.mjs && node tests/test-ready-to-merge-pagination-1645.mjs && node tests/test-require-gh-paginate-rule.mjs && node tests/test-auto-restart-limits-1664.mjs && node tests/test-telegram-message-filters.mjs && node tests/test-telegram-bot-command-aliases.mjs && node tests/test-telegram-options-before-url.mjs && node tests/test-telegram-bot-configuration-isolation-links-notation.mjs && node tests/test-extract-isolation-from-args.mjs && node tests/test-solve-queue-command.mjs && node tests/test-queue-display-1267.mjs && node tests/test-telegram-bot-launcher.mjs",
|
|
19
19
|
"test:queue": "node tests/solve-queue.test.mjs",
|
|
20
20
|
"test:limits-display": "node tests/limits-display.test.mjs",
|
|
21
21
|
"test:usage-limit": "node tests/test-usage-limit.mjs",
|
package/src/agent.lib.mjs
CHANGED
|
@@ -15,13 +15,14 @@ const os = (await use('os')).default;
|
|
|
15
15
|
// Import log from general lib
|
|
16
16
|
import { log } from './lib.mjs';
|
|
17
17
|
import { reportError } from './sentry.lib.mjs';
|
|
18
|
-
import { timeouts } from './config.lib.mjs';
|
|
18
|
+
import { timeouts, retryLimits } from './config.lib.mjs';
|
|
19
19
|
import { detectUsageLimit, formatUsageLimitMessage } from './usage-limit.lib.mjs';
|
|
20
20
|
import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
|
|
21
21
|
import Decimal from 'decimal.js-light';
|
|
22
22
|
import { agentModels, defaultModels, freeToBaseModelMap } from './models/index.mjs';
|
|
23
23
|
import { checkPlaywrightMcpPackageAvailability, getAgentPlaywrightMcpDisableEnv } from './playwright-mcp.lib.mjs';
|
|
24
24
|
import { createAgentTokenUsage, accumulateAgentStepFinishUsage, parseAgentTokenUsage } from './agent-token-usage.lib.mjs';
|
|
25
|
+
import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
|
|
25
26
|
|
|
26
27
|
export { createAgentTokenUsage, accumulateAgentStepFinishUsage, parseAgentTokenUsage };
|
|
27
28
|
|
|
@@ -410,10 +411,9 @@ export const executeAgent = async params => {
|
|
|
410
411
|
};
|
|
411
412
|
|
|
412
413
|
export const executeAgentCommand = async params => {
|
|
413
|
-
const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, agentPath,
|
|
414
|
+
const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, agentPath, $, waitForRetryDelay = waitWithCountdown } = params;
|
|
414
415
|
|
|
415
416
|
// Retry configuration
|
|
416
|
-
const maxRetries = 3;
|
|
417
417
|
let retryCount = 0;
|
|
418
418
|
|
|
419
419
|
const executeWithRetry = async () => {
|
|
@@ -421,7 +421,7 @@ export const executeAgentCommand = async params => {
|
|
|
421
421
|
if (retryCount === 0) {
|
|
422
422
|
await log(`\n${formatAligned('š¤', 'Executing Agent:', argv.model.toUpperCase())}`);
|
|
423
423
|
} else {
|
|
424
|
-
await log(`\n${formatAligned('š', 'Retry attempt:', `${retryCount}/${
|
|
424
|
+
await log(`\n${formatAligned('š', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
|
|
425
425
|
}
|
|
426
426
|
|
|
427
427
|
if (argv.verbose) {
|
|
@@ -470,6 +470,11 @@ export const executeAgentCommand = async params => {
|
|
|
470
470
|
agentArgs += ' --verbose';
|
|
471
471
|
}
|
|
472
472
|
|
|
473
|
+
if (argv.resume) {
|
|
474
|
+
await log(`š Resuming from session: ${argv.resume}`);
|
|
475
|
+
agentArgs += ` --resume ${argv.resume} --no-fork`;
|
|
476
|
+
}
|
|
477
|
+
|
|
473
478
|
// Agent supports stdin in both plain text and JSON format
|
|
474
479
|
// We'll combine system and user prompts into a single message
|
|
475
480
|
const combinedPrompt = systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt;
|
|
@@ -783,6 +788,28 @@ export const executeAgentCommand = async params => {
|
|
|
783
788
|
}
|
|
784
789
|
|
|
785
790
|
if (exitCode !== 0 || outputError.detected) {
|
|
791
|
+
const retryableError = classifyRetryableError(outputError.match || streamingErrorMessage || lastMessage || fullOutput);
|
|
792
|
+
if (retryableError.isRetryable) {
|
|
793
|
+
const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
|
|
794
|
+
const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
|
|
795
|
+
if (retryCount < maxRetries) {
|
|
796
|
+
const delay = getRetryDelayMs({
|
|
797
|
+
retryCount,
|
|
798
|
+
initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
|
|
799
|
+
maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
|
|
800
|
+
});
|
|
801
|
+
const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
|
|
802
|
+
await log(`\nā ļø ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
|
|
803
|
+
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
804
|
+
await maybeSwitchToFallbackModel({ tool: 'agent', argv, log, errorMessage: retryableError.message });
|
|
805
|
+
await waitForRetryDelay(delay, log);
|
|
806
|
+
await log('\nš Retrying now...');
|
|
807
|
+
retryCount++;
|
|
808
|
+
return await executeWithRetry();
|
|
809
|
+
}
|
|
810
|
+
await log(`\n\nā ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
|
|
811
|
+
}
|
|
812
|
+
|
|
786
813
|
// Build JSON error structure for consistent error reporting
|
|
787
814
|
const errorInfo = {
|
|
788
815
|
type: 'error',
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_AUTO_ITERATION_LIMIT = 5;
|
|
4
|
+
|
|
5
|
+
export const normalizeAutoIterationLimit = (value, fallback = DEFAULT_AUTO_ITERATION_LIMIT) => {
|
|
6
|
+
if (value === 0 || value === '0') return 0;
|
|
7
|
+
|
|
8
|
+
const parsed = Number(value);
|
|
9
|
+
if (!Number.isFinite(parsed) || parsed < 1) return fallback;
|
|
10
|
+
|
|
11
|
+
return Math.floor(parsed);
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export const normalizeAutoIterationCounter = value => {
|
|
15
|
+
const parsed = Number(value);
|
|
16
|
+
if (!Number.isFinite(parsed) || parsed < 0) return 0;
|
|
17
|
+
|
|
18
|
+
return Math.floor(parsed);
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export const hasReachedAutoIterationLimit = (completedIterations, maxIterations) => {
|
|
22
|
+
const normalizedMax = normalizeAutoIterationLimit(maxIterations);
|
|
23
|
+
if (normalizedMax === 0) return false;
|
|
24
|
+
|
|
25
|
+
return normalizeAutoIterationCounter(completedIterations) >= normalizedMax;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export const formatAutoIterationLimit = maxIterations => {
|
|
29
|
+
const normalizedMax = normalizeAutoIterationLimit(maxIterations);
|
|
30
|
+
return normalizedMax === 0 ? 'unlimited' : `${normalizedMax}`;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export const shouldSyncBeforeRestart = ({ hasUncommittedChanges }) => !hasUncommittedChanges;
|
package/src/claude.lib.mjs
CHANGED
|
@@ -24,6 +24,7 @@ import { buildMcpConfigWithoutPlaywright } from './playwright-mcp.lib.mjs';
|
|
|
24
24
|
import { resolveClaudeSessionToolFlags } from './useless-tools.lib.mjs';
|
|
25
25
|
import { ensureClaudeQuietConfig } from './claude-quiet-config.lib.mjs';
|
|
26
26
|
import { fetchModelInfo } from './model-info.lib.mjs';
|
|
27
|
+
import { classifyRetryableError, maybeSwitchToFallbackModel } from './tool-retry.lib.mjs';
|
|
27
28
|
export { availableModels }; // Re-export for backward compatibility
|
|
28
29
|
export { fetchModelInfo };
|
|
29
30
|
const showResumeCommand = async (sessionId, tempDir, claudePath, model, log) => {
|
|
@@ -1148,8 +1149,9 @@ export const executeClaudeCommand = async params => {
|
|
|
1148
1149
|
|
|
1149
1150
|
// Issue #817: Stop bidirectional mode monitoring and collect queued feedback
|
|
1150
1151
|
queuedFeedback = await finalizeBidirectionalHandler(bidirectionalHandler, log);
|
|
1152
|
+
const retryableLastError = classifyRetryableError(lastMessage);
|
|
1151
1153
|
// Issues #1331, #1353, #1472/#1475: Unified transient error retry (exponential backoff, session preservation)
|
|
1152
|
-
const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
|
|
1154
|
+
const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || retryableLastError.isRetryable || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
|
|
1153
1155
|
if ((commandFailed || isTransientError) && isTransientError) {
|
|
1154
1156
|
// Issue #1472/#1475: Startup/activity timeout ā 30sā2min backoff; #1353: Request timeout ā 5minā1hr; general ā 2minā30min
|
|
1155
1157
|
const isTimeoutRetry = isStartupTimeout || isActivityTimeout;
|
|
@@ -1178,7 +1180,7 @@ export const executeClaudeCommand = async params => {
|
|
|
1178
1180
|
}
|
|
1179
1181
|
if (retryCount < maxRetries) {
|
|
1180
1182
|
const delay = Math.min(initialDelay * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), maxDelay);
|
|
1181
|
-
const errorLabel = isStartupTimeout ? 'Stream startup timeout (Issue #1472/#1475)' : isActivityTimeout ? 'Stream activity timeout (Issue #1472)' : isRequestTimeout ? 'Request timeout' : isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('API Error: 529') && lastMessage.includes('Overloaded')) ? `API overload (${lastMessage.includes('529') ? '529' : '500'})` : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : '503 network error';
|
|
1183
|
+
const errorLabel = isStartupTimeout ? 'Stream startup timeout (Issue #1472/#1475)' : isActivityTimeout ? 'Stream activity timeout (Issue #1472)' : isRequestTimeout ? 'Request timeout' : retryableLastError.label || (isOverloadError || (lastMessage.includes('API Error: 500') && lastMessage.includes('Overloaded')) || (lastMessage.includes('API Error: 529') && lastMessage.includes('Overloaded')) ? `API overload (${lastMessage.includes('529') ? '529' : '500'})` : isInternalServerError || lastMessage.includes('Internal server error') ? 'Internal server error (500)' : '503 network error');
|
|
1182
1184
|
const notRetryableHint = apiMarkedNotRetryable ? ' (API says not retryable ā will stop early if no progress)' : '';
|
|
1183
1185
|
const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
|
|
1184
1186
|
const retryMode = isStartupTimeout ? ' (fresh start)' : ' (session preserved)';
|
|
@@ -1199,6 +1201,7 @@ export const executeClaudeCommand = async params => {
|
|
|
1199
1201
|
}
|
|
1200
1202
|
// Activity timeout preserves session (work was started), startup timeout does not (no session created)
|
|
1201
1203
|
if (!isStartupTimeout && sessionId && !argv.resume) argv.resume = sessionId;
|
|
1204
|
+
await maybeSwitchToFallbackModel({ tool: 'claude', argv, log, errorMessage: retryableLastError.message || lastMessage });
|
|
1202
1205
|
await waitWithCountdown(delay, log);
|
|
1203
1206
|
await log('\nš Retrying now...');
|
|
1204
1207
|
retryCount++;
|
|
@@ -1375,11 +1378,12 @@ export const executeClaudeCommand = async params => {
|
|
|
1375
1378
|
operation: 'run_claude_command',
|
|
1376
1379
|
});
|
|
1377
1380
|
const errorStr = error.message || error.toString();
|
|
1381
|
+
const retryableException = classifyRetryableError(errorStr);
|
|
1378
1382
|
// Issue #1331: Unified handler for all transient API errors in exception block
|
|
1379
1383
|
// Issue #1353: Also handle "Request timed out" in exception block
|
|
1380
1384
|
// (Overloaded, 503, Internal Server Error, Request timed out) - all with session preservation
|
|
1381
1385
|
const isTimeoutException = errorStr === 'Request timed out' || errorStr.includes('Request timed out');
|
|
1382
|
-
const isTransientException = isTimeoutException ||
|
|
1386
|
+
const isTransientException = isTimeoutException || retryableException.isRetryable;
|
|
1383
1387
|
if (isTransientException) {
|
|
1384
1388
|
// Issue #1353: Use timeout-specific backoff for request timeouts
|
|
1385
1389
|
const maxRetries = isTimeoutException ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
|
|
@@ -1387,9 +1391,10 @@ export const executeClaudeCommand = async params => {
|
|
|
1387
1391
|
const maxDelay = isTimeoutException ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs;
|
|
1388
1392
|
if (retryCount < maxRetries) {
|
|
1389
1393
|
const delay = Math.min(initialDelay * Math.pow(retryLimits.retryBackoffMultiplier, retryCount), maxDelay);
|
|
1390
|
-
const errorLabel = isTimeoutException ? 'Request timeout' : errorStr.includes('Overloaded') ? `API overload (${errorStr.includes('529') ? '529' : '500'})` : errorStr.includes('Internal server error') ? 'Internal server error (500)' : '503 network error';
|
|
1394
|
+
const errorLabel = isTimeoutException ? 'Request timeout' : retryableException.label || (errorStr.includes('Overloaded') ? `API overload (${errorStr.includes('529') ? '529' : '500'})` : errorStr.includes('Internal server error') ? 'Internal server error (500)' : '503 network error');
|
|
1391
1395
|
await log(`\nā ļø ${errorLabel} in exception. Retry ${retryCount + 1}/${maxRetries} in ${Math.round(delay / 60000)} min (session preserved)...`, { level: 'warning' });
|
|
1392
1396
|
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
1397
|
+
await maybeSwitchToFallbackModel({ tool: 'claude', argv, log, errorMessage: errorStr });
|
|
1393
1398
|
await waitWithCountdown(delay, log);
|
|
1394
1399
|
await log('\nš Retrying now...');
|
|
1395
1400
|
retryCount++;
|
package/src/codex.lib.mjs
CHANGED
|
@@ -15,7 +15,7 @@ const os = (await use('os')).default;
|
|
|
15
15
|
// Import log from general lib
|
|
16
16
|
import { log } from './lib.mjs';
|
|
17
17
|
import { reportError } from './sentry.lib.mjs';
|
|
18
|
-
import { timeouts } from './config.lib.mjs';
|
|
18
|
+
import { timeouts, retryLimits } from './config.lib.mjs';
|
|
19
19
|
import { detectUsageLimit, formatUsageLimitMessage } from './usage-limit.lib.mjs';
|
|
20
20
|
import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
|
|
21
21
|
import { mapModelToId, resolveCodexReasoningEffort } from './codex.options.lib.mjs';
|
|
@@ -24,6 +24,7 @@ import { initProgressMonitoring } from './solve.progress-monitoring.lib.mjs';
|
|
|
24
24
|
import { getCodexPlaywrightMcpDisableConfigArgs } from './playwright-mcp.lib.mjs';
|
|
25
25
|
import { fetchModelInfo } from './model-info.lib.mjs';
|
|
26
26
|
import { defaultModels } from './models/index.mjs';
|
|
27
|
+
import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
|
|
27
28
|
import Decimal from 'decimal.js-light';
|
|
28
29
|
|
|
29
30
|
const CODEX_USAGE_FIELD_NAMES = ['input_tokens', 'cached_input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_creation_input_tokens', 'reasoning_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens', 'output_tokens_details.reasoning_tokens'];
|
|
@@ -648,12 +649,11 @@ export const executeCodex = async params => {
|
|
|
648
649
|
};
|
|
649
650
|
|
|
650
651
|
export const executeCodexCommand = async params => {
|
|
651
|
-
const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, codexPath, $, owner, repo, prNumber, calculatePricing = calculateCodexPricing } = params;
|
|
652
|
+
const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, codexPath, $, owner, repo, prNumber, calculatePricing = calculateCodexPricing, waitForRetryDelay = waitWithCountdown } = params;
|
|
652
653
|
|
|
653
654
|
const shellQuote = value => `"${String(value).replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`;
|
|
654
655
|
|
|
655
656
|
// Retry configuration
|
|
656
|
-
const maxRetries = 3;
|
|
657
657
|
let retryCount = 0;
|
|
658
658
|
|
|
659
659
|
const executeWithRetry = async () => {
|
|
@@ -661,7 +661,7 @@ export const executeCodexCommand = async params => {
|
|
|
661
661
|
if (retryCount === 0) {
|
|
662
662
|
await log(`\n${formatAligned('š¤', 'Executing Codex:', argv.model.toUpperCase())}`);
|
|
663
663
|
} else {
|
|
664
|
-
await log(`\n${formatAligned('š', 'Retry attempt:', `${retryCount}/${
|
|
664
|
+
await log(`\n${formatAligned('š', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
|
|
665
665
|
}
|
|
666
666
|
|
|
667
667
|
if (argv.verbose) {
|
|
@@ -711,7 +711,7 @@ export const executeCodexCommand = async params => {
|
|
|
711
711
|
let codexArgs = 'exec';
|
|
712
712
|
if (isResumeMode) {
|
|
713
713
|
await log(`š Resuming from session: ${argv.resume}`);
|
|
714
|
-
codexArgs += ` resume ${shellQuote(argv.resume)}`;
|
|
714
|
+
codexArgs += ` resume ${shellQuote(argv.resume)} --model ${shellQuote(mappedModel)}`;
|
|
715
715
|
} else {
|
|
716
716
|
codexArgs += ` --model ${shellQuote(mappedModel)}`;
|
|
717
717
|
}
|
|
@@ -930,6 +930,7 @@ export const executeCodexCommand = async params => {
|
|
|
930
930
|
const codexErrorSummary = getCodexErrorEventSummary(codexJsonState);
|
|
931
931
|
if (codexErrorSummary.hasError) {
|
|
932
932
|
const limitInfo = detectUsageLimit(codexErrorSummary.message || lastMessage);
|
|
933
|
+
const retryableError = classifyRetryableError(codexErrorSummary.message || lastMessage);
|
|
933
934
|
if (limitInfo.isUsageLimit) {
|
|
934
935
|
limitReached = true;
|
|
935
936
|
limitResetTime = limitInfo.resetTime;
|
|
@@ -944,6 +945,25 @@ export const executeCodexCommand = async params => {
|
|
|
944
945
|
for (const line of messageLines) {
|
|
945
946
|
await log(line, { level: 'warning' });
|
|
946
947
|
}
|
|
948
|
+
} else if (retryableError.isRetryable) {
|
|
949
|
+
const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
|
|
950
|
+
const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
|
|
951
|
+
if (retryCount < maxRetries) {
|
|
952
|
+
const delay = getRetryDelayMs({
|
|
953
|
+
retryCount,
|
|
954
|
+
initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
|
|
955
|
+
maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
|
|
956
|
+
});
|
|
957
|
+
const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
|
|
958
|
+
await log(`\nā ļø ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
|
|
959
|
+
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
960
|
+
await maybeSwitchToFallbackModel({ tool: 'codex', argv, log, errorMessage: retryableError.message });
|
|
961
|
+
await waitForRetryDelay(delay, log);
|
|
962
|
+
await log('\nš Retrying now...');
|
|
963
|
+
retryCount++;
|
|
964
|
+
return await executeWithRetry();
|
|
965
|
+
}
|
|
966
|
+
await log(`\n\nā ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
|
|
947
967
|
} else {
|
|
948
968
|
await log(`\n\nā Codex emitted error event: ${codexErrorSummary.message}`, { level: 'error' });
|
|
949
969
|
await log(` Error events: item=${codexErrorSummary.counts.item}, turn=${codexErrorSummary.counts.turn}, stream=${codexErrorSummary.counts.stream}`, { level: 'error' });
|
|
@@ -971,6 +991,28 @@ export const executeCodexCommand = async params => {
|
|
|
971
991
|
}
|
|
972
992
|
|
|
973
993
|
if (exitCode !== 0) {
|
|
994
|
+
const retryableError = classifyRetryableError(lastMessage);
|
|
995
|
+
if (retryableError.isRetryable) {
|
|
996
|
+
const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
|
|
997
|
+
const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
|
|
998
|
+
if (retryCount < maxRetries) {
|
|
999
|
+
const delay = getRetryDelayMs({
|
|
1000
|
+
retryCount,
|
|
1001
|
+
initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
|
|
1002
|
+
maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
|
|
1003
|
+
});
|
|
1004
|
+
const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
|
|
1005
|
+
await log(`\nā ļø ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
|
|
1006
|
+
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
1007
|
+
await maybeSwitchToFallbackModel({ tool: 'codex', argv, log, errorMessage: retryableError.message });
|
|
1008
|
+
await waitForRetryDelay(delay, log);
|
|
1009
|
+
await log('\nš Retrying now...');
|
|
1010
|
+
retryCount++;
|
|
1011
|
+
return await executeWithRetry();
|
|
1012
|
+
}
|
|
1013
|
+
await log(`\n\nā ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
|
|
1014
|
+
}
|
|
1015
|
+
|
|
974
1016
|
// Check for usage limit errors first (more specific)
|
|
975
1017
|
const limitInfo = detectUsageLimit(lastMessage);
|
|
976
1018
|
if (limitInfo.isUsageLimit) {
|
package/src/hive.config.lib.mjs
CHANGED
|
@@ -12,7 +12,7 @@ const HIVE_ONLY_OPTION_NAMES = new Set(['monitor-tag', 'all-issues', 'skip-issue
|
|
|
12
12
|
|
|
13
13
|
// Solve-only options that should NOT be registered in hive
|
|
14
14
|
// (they are internal to solve and not meaningful when passed from hive)
|
|
15
|
-
const SOLVE_ONLY_OPTION_NAMES = new Set(['resume', 'working-directory', 'only-prepare-command', 'session-type']);
|
|
15
|
+
const SOLVE_ONLY_OPTION_NAMES = new Set(['resume', 'working-directory', 'only-prepare-command', 'session-type', 'auto-resume-iteration']);
|
|
16
16
|
|
|
17
17
|
// Options that hive defines with different defaults/descriptions than solve.
|
|
18
18
|
// These are registered manually in hive config to preserve hive-specific behavior.
|
package/src/hive.mjs
CHANGED
|
@@ -464,6 +464,9 @@ if (isRunningDirectly) {
|
|
|
464
464
|
// Validate model names EARLY (simple string check, always runs)
|
|
465
465
|
const tool = argv.tool || 'claude';
|
|
466
466
|
await validateAndExitOnInvalidModel(argv.model, tool, safeExit);
|
|
467
|
+
if (argv.fallbackModel) {
|
|
468
|
+
await validateAndExitOnInvalidModel(argv.fallbackModel, tool, safeExit);
|
|
469
|
+
}
|
|
467
470
|
if (argv.planModel) {
|
|
468
471
|
if (tool !== 'claude') {
|
|
469
472
|
await log(`ā --plan-model is only supported with --tool claude (current tool: ${tool})`, { level: 'error' });
|
package/src/models/index.mjs
CHANGED
|
@@ -905,6 +905,23 @@ export const resolveModelId = (requestedModel, tool) => {
|
|
|
905
905
|
}
|
|
906
906
|
};
|
|
907
907
|
|
|
908
|
+
export const defaultFallbackModels = {
|
|
909
|
+
claude: {
|
|
910
|
+
'claude-opus-4-7': 'opus-4-6',
|
|
911
|
+
},
|
|
912
|
+
codex: {
|
|
913
|
+
'gpt-5.5': 'gpt-5.4',
|
|
914
|
+
},
|
|
915
|
+
};
|
|
916
|
+
|
|
917
|
+
export const resolveDefaultFallbackModel = (tool, model) => {
|
|
918
|
+
if (!model) return null;
|
|
919
|
+
|
|
920
|
+
const toolName = (tool || 'claude').toString().toLowerCase();
|
|
921
|
+
const resolvedModel = resolveModelId(model, toolName);
|
|
922
|
+
return defaultFallbackModels[toolName]?.[resolvedModel] || null;
|
|
923
|
+
};
|
|
924
|
+
|
|
908
925
|
/**
|
|
909
926
|
* Fetch model info and build the complete model information string for PR comments.
|
|
910
927
|
* Uses actual models from CLI JSON output when available.
|
package/src/opencode.lib.mjs
CHANGED
|
@@ -15,13 +15,14 @@ const os = (await use('os')).default;
|
|
|
15
15
|
// Import log from general lib
|
|
16
16
|
import { log } from './lib.mjs';
|
|
17
17
|
import { reportError } from './sentry.lib.mjs';
|
|
18
|
-
import { timeouts } from './config.lib.mjs';
|
|
18
|
+
import { timeouts, retryLimits } from './config.lib.mjs';
|
|
19
19
|
import { detectUsageLimit, formatUsageLimitMessage } from './usage-limit.lib.mjs';
|
|
20
20
|
import { sanitizeObjectStrings } from './unicode-sanitization.lib.mjs';
|
|
21
21
|
import { opencodeModels, defaultModels } from './models/index.mjs';
|
|
22
22
|
import { checkPlaywrightMcpPackageAvailability, getOpenCodePlaywrightMcpDisableEnv } from './playwright-mcp.lib.mjs';
|
|
23
23
|
import { createAgentTokenUsage, accumulateAgentStepFinishUsage, parseAgentTokenUsage as parseOpenCodeTokenUsage } from './agent-token-usage.lib.mjs';
|
|
24
24
|
import { calculateAgentPricing } from './agent.lib.mjs';
|
|
25
|
+
import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
|
|
25
26
|
|
|
26
27
|
export { parseOpenCodeTokenUsage };
|
|
27
28
|
|
|
@@ -184,10 +185,9 @@ export const executeOpenCode = async params => {
|
|
|
184
185
|
};
|
|
185
186
|
|
|
186
187
|
export const executeOpenCodeCommand = async params => {
|
|
187
|
-
const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, opencodePath,
|
|
188
|
+
const { tempDir, branchName, prompt, systemPrompt, argv, log, formatAligned, getResourceSnapshot, forkedRepo, feedbackLines, opencodePath, $, waitForRetryDelay = waitWithCountdown } = params;
|
|
188
189
|
|
|
189
190
|
// Retry configuration
|
|
190
|
-
const maxRetries = 3;
|
|
191
191
|
let retryCount = 0;
|
|
192
192
|
|
|
193
193
|
const executeWithRetry = async () => {
|
|
@@ -195,7 +195,7 @@ export const executeOpenCodeCommand = async params => {
|
|
|
195
195
|
if (retryCount === 0) {
|
|
196
196
|
await log(`\n${formatAligned('š¤', 'Executing OpenCode:', argv.model.toUpperCase())}`);
|
|
197
197
|
} else {
|
|
198
|
-
await log(`\n${formatAligned('š', 'Retry attempt:', `${retryCount}/${
|
|
198
|
+
await log(`\n${formatAligned('š', 'Retry attempt:', `${retryCount}/${retryLimits.maxTransientErrorRetries}`)}`);
|
|
199
199
|
}
|
|
200
200
|
|
|
201
201
|
if (argv.verbose) {
|
|
@@ -265,7 +265,7 @@ export const executeOpenCodeCommand = async params => {
|
|
|
265
265
|
|
|
266
266
|
if (argv.resume) {
|
|
267
267
|
await log(`š Resuming from session: ${argv.resume}`);
|
|
268
|
-
opencodeArgs = `run --format json --
|
|
268
|
+
opencodeArgs = `run --format json --session ${argv.resume} --model ${mappedModel}`;
|
|
269
269
|
}
|
|
270
270
|
|
|
271
271
|
// For OpenCode, we pass the prompt via stdin
|
|
@@ -301,7 +301,7 @@ export const executeOpenCodeCommand = async params => {
|
|
|
301
301
|
cwd: tempDir,
|
|
302
302
|
mirror: false,
|
|
303
303
|
env: opencodeEnv,
|
|
304
|
-
})`cat ${promptFile} | ${opencodePath} run --format json --
|
|
304
|
+
})`cat ${promptFile} | ${opencodePath} run --format json --session ${argv.resume} --model ${mappedModel}`;
|
|
305
305
|
} else {
|
|
306
306
|
execCommand = $({
|
|
307
307
|
cwd: tempDir,
|
|
@@ -470,6 +470,28 @@ export const executeOpenCodeCommand = async params => {
|
|
|
470
470
|
}
|
|
471
471
|
|
|
472
472
|
if (exitCode !== 0) {
|
|
473
|
+
const retryableError = classifyRetryableError(allOutput || lastMessage);
|
|
474
|
+
if (retryableError.isRetryable) {
|
|
475
|
+
const isRequestTimeoutRetry = retryableError.label === 'Request timeout';
|
|
476
|
+
const maxRetries = isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
|
|
477
|
+
if (retryCount < maxRetries) {
|
|
478
|
+
const delay = getRetryDelayMs({
|
|
479
|
+
retryCount,
|
|
480
|
+
initialDelayMs: isRequestTimeoutRetry ? retryLimits.initialRequestTimeoutDelayMs : retryLimits.initialTransientErrorDelayMs,
|
|
481
|
+
maxDelayMs: isRequestTimeoutRetry ? retryLimits.maxRequestTimeoutDelayMs : retryLimits.maxTransientErrorDelayMs,
|
|
482
|
+
});
|
|
483
|
+
const delayLabel = delay >= 60000 ? `${Math.round(delay / 60000)} min` : `${Math.round(delay / 1000)}s`;
|
|
484
|
+
await log(`\nā ļø ${retryableError.label} detected. Retry ${retryCount + 1}/${maxRetries} in ${delayLabel}${sessionId ? ' (session preserved)' : ''}...`, { level: 'warning' });
|
|
485
|
+
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
486
|
+
await maybeSwitchToFallbackModel({ tool: 'opencode', argv, log, errorMessage: retryableError.message });
|
|
487
|
+
await waitForRetryDelay(delay, log);
|
|
488
|
+
await log('\nš Retrying now...');
|
|
489
|
+
retryCount++;
|
|
490
|
+
return await executeWithRetry();
|
|
491
|
+
}
|
|
492
|
+
await log(`\n\nā ${retryableError.label} persisted after ${maxRetries} retries`, { level: 'error' });
|
|
493
|
+
}
|
|
494
|
+
|
|
473
495
|
// Check for usage limit errors first (more specific)
|
|
474
496
|
const limitInfo = detectUsageLimit(lastMessage);
|
|
475
497
|
if (limitInfo.isUsageLimit) {
|
|
@@ -203,6 +203,7 @@ const KNOWN_OPTION_NAMES = [
|
|
|
203
203
|
'allow-to-push-to-contributors-pull-requests-as-maintainer',
|
|
204
204
|
'prefix-fork-name-with-owner-name',
|
|
205
205
|
'auto-restart-max-iterations',
|
|
206
|
+
'auto-resume-max-iterations',
|
|
206
207
|
'auto-continue-only-on-new-comments',
|
|
207
208
|
'auto-restart-on-limit-reset',
|
|
208
209
|
'auto-restart-on-non-updated-pull-request-description',
|
|
@@ -48,6 +48,7 @@ const { extractLinkedIssueNumber } = githubLinking;
|
|
|
48
48
|
|
|
49
49
|
// Import configuration
|
|
50
50
|
import { autoContinue, limitReset } from './config.lib.mjs';
|
|
51
|
+
import { formatAutoIterationLimit, hasReachedAutoIterationLimit, normalizeAutoIterationCounter, normalizeAutoIterationLimit } from './auto-iteration-limits.lib.mjs';
|
|
51
52
|
|
|
52
53
|
// Issue #1574: Interruptible sleep so CTRL+C is never blocked by a lingering timer
|
|
53
54
|
const { interruptibleSleep } = await import('./interruptible-sleep.lib.mjs');
|
|
@@ -79,6 +80,15 @@ const formatWaitTime = ms => {
|
|
|
79
80
|
// See: https://github.com/link-assistant/hive-mind/issues/1152
|
|
80
81
|
export const autoContinueWhenLimitResets = async (issueUrl, sessionId, argv, shouldAttachLogs, tempDir = null, isRestart = false) => {
|
|
81
82
|
try {
|
|
83
|
+
const maxAutoResumeIterations = normalizeAutoIterationLimit(argv.autoResumeMaxIterations);
|
|
84
|
+
const currentAutoResumeIteration = normalizeAutoIterationCounter(argv.autoResumeIteration);
|
|
85
|
+
|
|
86
|
+
if (hasReachedAutoIterationLimit(currentAutoResumeIteration, maxAutoResumeIterations)) {
|
|
87
|
+
await log(`\nā ļø Auto-${isRestart ? 'restart' : 'resume'} limit reached: ${currentAutoResumeIteration}/${formatAutoIterationLimit(maxAutoResumeIterations)}`);
|
|
88
|
+
await safeExit(1, `Auto-${isRestart ? 'restart' : 'resume'} limit reached`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const nextAutoResumeIteration = currentAutoResumeIteration + 1;
|
|
82
92
|
const resetTime = global.limitResetTime;
|
|
83
93
|
const timezone = global.limitTimezone || null;
|
|
84
94
|
const baseWaitMs = calculateWaitTime(resetTime);
|
|
@@ -125,6 +135,7 @@ export const autoContinueWhenLimitResets = async (issueUrl, sessionId, argv, sho
|
|
|
125
135
|
const actionType = isRestart ? 'Restarting' : 'Resuming';
|
|
126
136
|
await log(`\nā
Limit reset time reached (+ ${bufferMinutes} min buffer + ${jitterSeconds}s jitter)! ${actionType} session...`);
|
|
127
137
|
await log(` Current time: ${new Date().toLocaleTimeString()}`);
|
|
138
|
+
await log(` Auto-${isRestart ? 'restart' : 'resume'} iteration: ${maxAutoResumeIterations === 0 ? nextAutoResumeIteration : `${nextAutoResumeIteration}/${maxAutoResumeIterations}`}`);
|
|
128
139
|
|
|
129
140
|
// Recursively call the solve script
|
|
130
141
|
// For resume: use --resume with session ID to maintain context
|
|
@@ -153,6 +164,8 @@ export const autoContinueWhenLimitResets = async (issueUrl, sessionId, argv, sho
|
|
|
153
164
|
if (argv.autoRestartOnLimitReset) {
|
|
154
165
|
resumeArgs.push('--auto-restart-on-limit-reset');
|
|
155
166
|
}
|
|
167
|
+
resumeArgs.push('--auto-resume-iteration', String(nextAutoResumeIteration));
|
|
168
|
+
resumeArgs.push('--auto-resume-max-iterations', String(maxAutoResumeIterations));
|
|
156
169
|
|
|
157
170
|
// Pass session type for proper comment differentiation
|
|
158
171
|
// See: https://github.com/link-assistant/hive-mind/issues/1152
|
|
@@ -162,6 +175,7 @@ export const autoContinueWhenLimitResets = async (issueUrl, sessionId, argv, sho
|
|
|
162
175
|
// Preserve other flags from original invocation
|
|
163
176
|
if (argv.tool && argv.tool !== 'claude') resumeArgs.push('--tool', argv.tool);
|
|
164
177
|
if (argv.model !== 'sonnet') resumeArgs.push('--model', argv.model);
|
|
178
|
+
if (argv.fallbackModel) resumeArgs.push('--fallback-model', argv.fallbackModel);
|
|
165
179
|
if (argv.verbose) resumeArgs.push('--verbose');
|
|
166
180
|
if (argv.fork) resumeArgs.push('--fork');
|
|
167
181
|
if (shouldAttachLogs) resumeArgs.push('--attach-logs');
|