@link-assistant/hive-mind 2.12.5 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/src/claude.lib.mjs +52 -1
- package/src/hive.mjs +39 -0
- package/src/limits-i18n.lib.mjs +8 -0
- package/src/locales/en.lino +9 -0
- package/src/locales/hi.lino +9 -0
- package/src/locales/ru.lino +9 -0
- package/src/locales/zh.lino +9 -0
- package/src/session-monitor.lib.mjs +45 -1
- package/src/solve.mjs +36 -4
- package/src/subscription-block-telegram.lib.mjs +115 -0
- package/src/subscription-error.lib.mjs +328 -0
- package/src/tool-retry.lib.mjs +29 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.13.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- aaf809a: Recognize subscription/account access blocks from every supported CLI (Claude, Codex, Qwen, Gemini, opencode) as their own error class: stop the run instead of retrying or switching model, auto-commit and push the in-flight work first, report what happened and what to do in the terminal, in the `/solve` exit message and in the Telegram completion message (en/ru/zh/hi), and stop the `/hive` queue so the fleet no longer rediscovers the block once per issue.
|
|
8
|
+
|
|
3
9
|
## 2.12.5
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
package/package.json
CHANGED
package/src/claude.lib.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import { isENOSPC, buildToolErrorMessage } from './lib.mjs';
|
|
|
9
9
|
import { reportError } from './sentry.lib.mjs';
|
|
10
10
|
import { timeouts, retryLimits, claudeCode, getClaudeEnv, getMaxOutputTokensForModel } from './config.lib.mjs';
|
|
11
11
|
import { detectUsageLimit, formatUsageLimitMessage, isUsageLimitError } from './usage-limit.lib.mjs';
|
|
12
|
+
import { detectSubscriptionError, SUBSCRIPTION_BLOCKED_MARKER } from './subscription-error.lib.mjs'; // Issue #2161
|
|
12
13
|
import { createInteractiveHandler } from './interactive-mode.lib.mjs';
|
|
13
14
|
import { setupBidirectionalHandler, finalizeBidirectionalHandler, validateBidirectionalModeConfig, attachStreamingInput } from './bidirectional-interactive.lib.mjs';
|
|
14
15
|
import { initProgressMonitoring } from './solve.progress-monitoring.lib.mjs';
|
|
@@ -376,6 +377,10 @@ export const executeClaudeCommand = async params => {
|
|
|
376
377
|
let isInternalServerError = false;
|
|
377
378
|
let isRequestTimeout = false;
|
|
378
379
|
let isRateLimitError = false; // Issue #1924: server-side 429 temporary rate limiting
|
|
380
|
+
// Issue #2161: account/subscription-level block (e.g. oauth_org_not_allowed).
|
|
381
|
+
// Terminal — never retried, never model-switched; carried out to the caller
|
|
382
|
+
// so /solve can stop with a specific diagnosis instead of a generic failure.
|
|
383
|
+
let subscriptionError = null;
|
|
379
384
|
let apiMarkedNotRetryable = false;
|
|
380
385
|
let resultNumTurns = 0;
|
|
381
386
|
let stderrErrors = [];
|
|
@@ -731,6 +736,24 @@ export const executeClaudeCommand = async params => {
|
|
|
731
736
|
isRateLimitError = true;
|
|
732
737
|
await log(`⚠️ Detected server-side rate limiting (429) from Claude CLI (will retry with --resume). request_id=${data.request_id || 'unknown'}`, { verbose: true });
|
|
733
738
|
}
|
|
739
|
+
// Issue #2161: account/subscription block. `data.error` carries the
|
|
740
|
+
// machine-readable code ("oauth_org_not_allowed" for the reported
|
|
741
|
+
// case) alongside api_error_status 403 — a far stronger signal than
|
|
742
|
+
// the rendered sentence, so it is passed to the detector first.
|
|
743
|
+
if (!subscriptionError) {
|
|
744
|
+
subscriptionError = detectSubscriptionError({
|
|
745
|
+
message: lastMessage,
|
|
746
|
+
tool: 'claude',
|
|
747
|
+
errorCode: typeof data.error === 'string' ? data.error : null,
|
|
748
|
+
apiErrorStatus: data.api_error_status,
|
|
749
|
+
terminalReason: data.terminal_reason,
|
|
750
|
+
});
|
|
751
|
+
if (subscriptionError) {
|
|
752
|
+
// Not verbose: this is the reason the whole run is about to end.
|
|
753
|
+
await log(`${SUBSCRIPTION_BLOCKED_MARKER} — ${subscriptionError.label}`);
|
|
754
|
+
await log(` code=${subscriptionError.code || 'n/a'} http=${data.api_error_status || 'n/a'} terminal_reason=${data.terminal_reason || 'n/a'} request_id=${data.request_id || 'unknown'}`, { verbose: true });
|
|
755
|
+
}
|
|
756
|
+
}
|
|
734
757
|
// Issue #1834: Detect corrupted extended-thinking-block 400 (un-resumable session).
|
|
735
758
|
// Capture diagnostics (request id, content path) to aid debugging and upstream reports.
|
|
736
759
|
if ((lastMessage.includes('thinking') || lastMessage.includes('redacted_thinking')) && lastMessage.includes('cannot be modified')) {
|
|
@@ -765,6 +788,26 @@ export const executeClaudeCommand = async params => {
|
|
|
765
788
|
await log(`🤖 Sub-agent "${callEntry.description || 'unknown'}" completed: ${data.usage.total_tokens} total tokens`, { verbose: true });
|
|
766
789
|
}
|
|
767
790
|
}
|
|
791
|
+
// Issue #2161: Claude Code injects API failures as synthetic assistant
|
|
792
|
+
// messages flagged `is_api_error_message` and carrying the error code.
|
|
793
|
+
// In the reported run this arrived ~40s before the terminal result
|
|
794
|
+
// event, so detecting it here surfaces the diagnosis earlier.
|
|
795
|
+
if (data.type === 'assistant' && data.is_api_error_message === true && !subscriptionError) {
|
|
796
|
+
const apiErrorText = getClaudeMessageContent(data)
|
|
797
|
+
.filter(item => item.type === 'text' && item.text)
|
|
798
|
+
.map(item => item.text)
|
|
799
|
+
.join('\n');
|
|
800
|
+
subscriptionError = detectSubscriptionError({
|
|
801
|
+
message: apiErrorText,
|
|
802
|
+
tool: 'claude',
|
|
803
|
+
errorCode: typeof data.error === 'string' ? data.error : null,
|
|
804
|
+
});
|
|
805
|
+
if (subscriptionError) {
|
|
806
|
+
if (apiErrorText) lastMessage = apiErrorText;
|
|
807
|
+
await log(`${SUBSCRIPTION_BLOCKED_MARKER} — ${subscriptionError.label}`);
|
|
808
|
+
await log(` code=${subscriptionError.code || 'n/a'} request_id=${data.request_id || 'unknown'} uuid=${data.uuid || 'unknown'}`, { verbose: true });
|
|
809
|
+
}
|
|
810
|
+
}
|
|
768
811
|
if (data.type === 'assistant' && data.message && data.message.content) {
|
|
769
812
|
const content = getClaudeMessageContent(data);
|
|
770
813
|
for (const item of content) {
|
|
@@ -977,7 +1020,12 @@ export const executeClaudeCommand = async params => {
|
|
|
977
1020
|
}
|
|
978
1021
|
// Issues #1331, #1353, #1472/#1475: Unified transient error retry (exponential backoff, session preservation)
|
|
979
1022
|
const isTransientError = isStartupTimeout || isActivityTimeout || isOverloadError || isInternalServerError || is503Error || isRequestTimeout || isRateLimitError || retryableLastError.isRetryable || (lastMessage.includes('API Error: 500') && (lastMessage.includes('Overloaded') || lastMessage.includes('Internal server error'))) || (lastMessage.includes('API Error: 529') && (lastMessage.includes('overloaded_error') || lastMessage.includes('Overloaded'))) || (lastMessage.includes('api_error') && lastMessage.includes('Overloaded')) || (lastMessage.includes('overloaded_error') && lastMessage.includes('Overloaded')) || lastMessage.includes('API Error: 503') || (lastMessage.includes('503') && (lastMessage.includes('upstream connect error') || lastMessage.includes('remote connection failure'))) || lastMessage === 'Request timed out' || lastMessage.includes('Request timed out');
|
|
980
|
-
|
|
1023
|
+
// Issue #2161: an account/subscription block short-circuits every retry
|
|
1024
|
+
// path. Stale transient flags from earlier in the run (an overload at hour
|
|
1025
|
+
// one, say) must not schedule a retry that is guaranteed to fail the same
|
|
1026
|
+
// way — and each retry would burn another full startup against a provider
|
|
1027
|
+
// that has already refused the credentials.
|
|
1028
|
+
if ((commandFailed || isTransientError) && isTransientError && !subscriptionError) {
|
|
981
1029
|
// Issue #1472/#1475: Startup/activity timeout → 30s–2min backoff; #1353: Request timeout → 5min–1hr; general → 2min–30min
|
|
982
1030
|
const isTimeoutRetry = isStartupTimeout || isActivityTimeout;
|
|
983
1031
|
const maxRetries = isTimeoutRetry ? retryLimits.maxTransientErrorRetries : isRequestTimeout ? retryLimits.maxRequestTimeoutRetries : retryLimits.maxTransientErrorRetries;
|
|
@@ -1005,6 +1053,7 @@ export const executeClaudeCommand = async params => {
|
|
|
1005
1053
|
resultSummary,
|
|
1006
1054
|
// Issue #1845/#1941: surface the actual error, rejecting meaningless fragments (e.g. a lone "}")
|
|
1007
1055
|
errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: 'API explicitly marked error as not retryable', toolLabel: 'Claude' }), exitCode },
|
|
1056
|
+
subscriptionError, // Issue #2161
|
|
1008
1057
|
queuedFeedback, // Issue #817: Bidirectional mode feedback
|
|
1009
1058
|
};
|
|
1010
1059
|
}
|
|
@@ -1055,6 +1104,7 @@ export const executeClaudeCommand = async params => {
|
|
|
1055
1104
|
resultSummary, // Issue #1263: Include result summary
|
|
1056
1105
|
// Issue #1845/#1941: surface the actual error, rejecting meaningless fragments (e.g. a lone "}")
|
|
1057
1106
|
errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: `Transient API error persisted after ${maxRetries} retries`, toolLabel: 'Claude' }), exitCode },
|
|
1107
|
+
subscriptionError, // Issue #2161
|
|
1058
1108
|
queuedFeedback, // Issue #817: Bidirectional mode feedback
|
|
1059
1109
|
};
|
|
1060
1110
|
}
|
|
@@ -1123,6 +1173,7 @@ export const executeClaudeCommand = async params => {
|
|
|
1123
1173
|
// Issue #1845: surface the core error (e.g. "API Error: Output blocked by content filtering policy").
|
|
1124
1174
|
// Issue #1941: a lone "}" fragment at interrupt time must not become "CLAUDE execution failed with }".
|
|
1125
1175
|
errorInfo: { message: buildToolErrorMessage({ lastMessage, exitCode, fallback: `Claude command failed with exit code ${exitCode}`, toolLabel: 'Claude' }), exitCode },
|
|
1176
|
+
subscriptionError, // Issue #2161: terminal account block — /solve stops and preserves the work
|
|
1126
1177
|
queuedFeedback, // Issue #817: Bidirectional mode feedback
|
|
1127
1178
|
};
|
|
1128
1179
|
}
|
package/src/hive.mjs
CHANGED
|
@@ -36,6 +36,7 @@ if (earlyArgs.includes('--help') || earlyArgs.includes('-h')) {
|
|
|
36
36
|
}
|
|
37
37
|
export { createYargsConfig } from './hive.config.lib.mjs';
|
|
38
38
|
import { attachChildExitHandlers } from './child-exit.lib.mjs';
|
|
39
|
+
import { SUBSCRIPTION_BLOCKED_MARKER } from './subscription-error.lib.mjs'; // Issue #2161
|
|
39
40
|
import { isDirectExecution, withTimeout } from './hive.bootstrap.lib.mjs';
|
|
40
41
|
import { createShutdownManager } from './hive.shutdown.lib.mjs';
|
|
41
42
|
const isRunningDirectly = isDirectExecution(process.argv[1], import.meta.url);
|
|
@@ -653,6 +654,22 @@ if (isRunningDirectly) {
|
|
|
653
654
|
// controlled SIGTERM to each (they run in their own detached process group, so the
|
|
654
655
|
// terminal's SIGINT never reaches them); a *second* interrupt force-kills the groups.
|
|
655
656
|
const activeSolveChildren = new Set();
|
|
657
|
+
// Issue #2161: an account/subscription block is hive-wide, not per-issue. The
|
|
658
|
+
// credentials every worker shares have been refused, so each remaining issue
|
|
659
|
+
// would spin up a full solve run only to die the same way — burning clones,
|
|
660
|
+
// containers and PR comments while the queue drains into "failed". The first
|
|
661
|
+
// worker to see the marker in its child's output records it here and stops the
|
|
662
|
+
// queue; the rest exit as soon as their current child returns.
|
|
663
|
+
let subscriptionBlock = null;
|
|
664
|
+
const noteSubscriptionBlock = (workerId, line) => {
|
|
665
|
+
if (subscriptionBlock) return;
|
|
666
|
+
subscriptionBlock = { workerId, line: line.trim() };
|
|
667
|
+
log(`\n${SUBSCRIPTION_BLOCKED_MARKER} — worker ${workerId} reported that the account can no longer use the tool:`, { level: 'error' }).catch(() => {});
|
|
668
|
+
log(` ${subscriptionBlock.line}`, { level: 'error' }).catch(() => {});
|
|
669
|
+
log(' Stopping the hive: every remaining issue would fail the same way until access is restored.', { level: 'error' }).catch(() => {});
|
|
670
|
+
log(' In-flight workers finish (and auto-commit their work) before the run ends.', { level: 'error' }).catch(() => {});
|
|
671
|
+
issueQueue.stop();
|
|
672
|
+
};
|
|
656
673
|
// Worker function to process issues from queue
|
|
657
674
|
async function worker(workerId) {
|
|
658
675
|
await log(`🔧 Worker ${workerId} started`, { verbose: true });
|
|
@@ -758,6 +775,9 @@ if (isRunningDirectly) {
|
|
|
758
775
|
const lines = data.toString().split('\n');
|
|
759
776
|
for (const line of lines) {
|
|
760
777
|
if (line.trim()) {
|
|
778
|
+
// Issue #2161: solve prints SUBSCRIPTION_BLOCKED_MARKER on a terminal
|
|
779
|
+
// account block. Seen here, it stops the whole hive (see noteSubscriptionBlock).
|
|
780
|
+
if (line.includes(SUBSCRIPTION_BLOCKED_MARKER)) noteSubscriptionBlock(workerId, line);
|
|
761
781
|
log(` [${solveCommand} worker-${workerId}] ${line}`).catch(logError => {
|
|
762
782
|
reportError(logError, {
|
|
763
783
|
context: 'worker_stdout_log',
|
|
@@ -777,6 +797,7 @@ if (isRunningDirectly) {
|
|
|
777
797
|
const lines = data.toString().split('\n');
|
|
778
798
|
for (const line of lines) {
|
|
779
799
|
if (line.trim()) {
|
|
800
|
+
if (line.includes(SUBSCRIPTION_BLOCKED_MARKER)) noteSubscriptionBlock(workerId, line); // Issue #2161
|
|
780
801
|
log(` [${solveCommand} worker-${workerId} stderr] ${line}`).catch(logError => {
|
|
781
802
|
reportError(logError, {
|
|
782
803
|
context: 'worker_stderr_log',
|
|
@@ -813,6 +834,14 @@ if (isRunningDirectly) {
|
|
|
813
834
|
await log(` 🛑 Worker ${workerId} stopped gracefully during shutdown on ${issueUrl} (exit ${exitCode}, ${duration}s)`);
|
|
814
835
|
gracefulStop = true;
|
|
815
836
|
break; // stop processing more PRs for this issue
|
|
837
|
+
} else if (subscriptionBlock) {
|
|
838
|
+
// Issue #2161: the run did not fail because of this issue — the account
|
|
839
|
+
// lost access mid-flight. Report the real reason and stop; solve has
|
|
840
|
+
// already auto-committed whatever work existed.
|
|
841
|
+
await log(` ${SUBSCRIPTION_BLOCKED_MARKER} Worker ${workerId} stopped on ${issueUrl} after ${duration}s: the tool account can no longer be used (exit ${exitCode}).`, { level: 'error' });
|
|
842
|
+
await log(` Restore access, then re-run the hive — this issue stays queued, not failed.`, { level: 'error' });
|
|
843
|
+
gracefulStop = true;
|
|
844
|
+
break;
|
|
816
845
|
} else {
|
|
817
846
|
throw new Error(`${solveCommand} exited with code ${exitCode}`);
|
|
818
847
|
}
|
|
@@ -1260,6 +1289,16 @@ if (isRunningDirectly) {
|
|
|
1260
1289
|
}
|
|
1261
1290
|
await log('\n👋 Hive Mind monitoring stopped');
|
|
1262
1291
|
await log(` 📁 Full log file: ${absoluteLogPath}`);
|
|
1292
|
+
// Issue #2161: the hive did not simply "finish" — it was cut short because the
|
|
1293
|
+
// account lost access. Say so last (that is what a human scrolls to) and exit
|
|
1294
|
+
// non-zero so supervisors and the Telegram monitor report a failure, not a
|
|
1295
|
+
// clean completion.
|
|
1296
|
+
if (subscriptionBlock) {
|
|
1297
|
+
await log(`\n${SUBSCRIPTION_BLOCKED_MARKER} Hive stopped early: the tool account can no longer be used.`, { level: 'error' });
|
|
1298
|
+
await log(` Reported by worker ${subscriptionBlock.workerId}: ${subscriptionBlock.line}`, { level: 'error' });
|
|
1299
|
+
await log(' Restore subscription/account access, then start the hive again.', { level: 'error' });
|
|
1300
|
+
await safeExit(1, 'Subscription/account access blocked');
|
|
1301
|
+
}
|
|
1263
1302
|
}
|
|
1264
1303
|
// Issue #1823: Graceful-shutdown + force-kill logic lives in hive.shutdown.lib.mjs.
|
|
1265
1304
|
// gracefulShutdown waits (uncapped) for in-flight solve workers to finish on the first
|
package/src/limits-i18n.lib.mjs
CHANGED
|
@@ -85,6 +85,14 @@ const ENGLISH_LIMITS = {
|
|
|
85
85
|
subscription_detail_trial_ends: 'trial ends {{time}}',
|
|
86
86
|
subscription_detail_trial_ends_in: 'trial ends in {{duration}}; {{time}}',
|
|
87
87
|
subscription_status: 'Subscription: {{status}}',
|
|
88
|
+
subscription_blocked_title: 'Subscription/account access blocked',
|
|
89
|
+
subscription_blocked_provider: 'Provider said',
|
|
90
|
+
subscription_blocked_code: 'Error code',
|
|
91
|
+
subscription_blocked_reason: 'Why the run stopped',
|
|
92
|
+
subscription_blocked_note: 'This is not a usage limit: waiting, retrying or switching model cannot fix it.',
|
|
93
|
+
subscription_blocked_steps: 'What to do',
|
|
94
|
+
subscription_blocked_preserved: 'Uncommitted work was auto-committed before stopping.',
|
|
95
|
+
subscription_blocked_resume: 'Resume after access is restored',
|
|
88
96
|
telegram_api: 'Telegram Bot API',
|
|
89
97
|
telegram_flood_control: 'flood control',
|
|
90
98
|
telegram_last_rate_limit: 'Last 429: {{method}}',
|
package/src/locales/en.lino
CHANGED
|
@@ -322,6 +322,15 @@ en
|
|
|
322
322
|
session "session"
|
|
323
323
|
start "Start"
|
|
324
324
|
subscription
|
|
325
|
+
blocked
|
|
326
|
+
title "Subscription/account access blocked"
|
|
327
|
+
provider "Provider said"
|
|
328
|
+
code "Error code"
|
|
329
|
+
reason "Why the run stopped"
|
|
330
|
+
note "This is not a usage limit: waiting, retrying or switching model cannot fix it."
|
|
331
|
+
steps "What to do"
|
|
332
|
+
preserved "Uncommitted work was auto-committed before stopping."
|
|
333
|
+
resume "Resume after access is restored"
|
|
325
334
|
detail
|
|
326
335
|
ends
|
|
327
336
|
label "ends {{time}}"
|
package/src/locales/hi.lino
CHANGED
|
@@ -322,6 +322,15 @@ hi
|
|
|
322
322
|
session "सत्र"
|
|
323
323
|
start "शुरुआत"
|
|
324
324
|
subscription
|
|
325
|
+
blocked
|
|
326
|
+
title "सदस्यता/खाता पहुँच अवरुद्ध"
|
|
327
|
+
provider "प्रदाता ने कहा"
|
|
328
|
+
code "त्रुटि कोड"
|
|
329
|
+
reason "रन क्यों रुका"
|
|
330
|
+
note "यह उपयोग सीमा नहीं है: प्रतीक्षा, पुनः प्रयास या मॉडल बदलना इसे ठीक नहीं करेगा।"
|
|
331
|
+
steps "क्या करें"
|
|
332
|
+
preserved "रुकने से पहले बिना कमिट किए बदलाव स्वतः कमिट कर दिए गए।"
|
|
333
|
+
resume "पहुँच बहाल होने पर फिर से शुरू करें"
|
|
325
334
|
detail
|
|
326
335
|
ends
|
|
327
336
|
label "{{time}} को समाप्त होगी"
|
package/src/locales/ru.lino
CHANGED
|
@@ -322,6 +322,15 @@ ru
|
|
|
322
322
|
session "сеанс"
|
|
323
323
|
start "Начало"
|
|
324
324
|
subscription
|
|
325
|
+
blocked
|
|
326
|
+
title "Доступ по подписке/аккаунту заблокирован"
|
|
327
|
+
provider "Ответ провайдера"
|
|
328
|
+
code "Код ошибки"
|
|
329
|
+
reason "Почему запуск остановлен"
|
|
330
|
+
note "Это не лимит использования: ожидание, повтор или смена модели не помогут."
|
|
331
|
+
steps "Что делать"
|
|
332
|
+
preserved "Незакоммиченные изменения были автоматически закоммичены перед остановкой."
|
|
333
|
+
resume "Продолжить после восстановления доступа"
|
|
325
334
|
detail
|
|
326
335
|
ends
|
|
327
336
|
label "заканчивается {{time}}"
|
package/src/locales/zh.lino
CHANGED
|
@@ -322,6 +322,15 @@ zh
|
|
|
322
322
|
session "会话"
|
|
323
323
|
start "开始"
|
|
324
324
|
subscription
|
|
325
|
+
blocked
|
|
326
|
+
title "订阅/账号访问被阻止"
|
|
327
|
+
provider "服务方提示"
|
|
328
|
+
code "错误代码"
|
|
329
|
+
reason "运行停止的原因"
|
|
330
|
+
note "这不是用量限制:等待、重试或切换模型都无法解决。"
|
|
331
|
+
steps "如何处理"
|
|
332
|
+
preserved "停止前已自动提交未提交的改动。"
|
|
333
|
+
resume "恢复访问后继续"
|
|
325
334
|
detail
|
|
326
335
|
ends
|
|
327
336
|
label "结束于 {{time}}"
|
|
@@ -389,6 +389,35 @@ export async function buildDiskDiagnosticsExtraSection(logPath, { verbose = fals
|
|
|
389
389
|
return '';
|
|
390
390
|
}
|
|
391
391
|
}
|
|
392
|
+
/**
|
|
393
|
+
* Issue #2161: Replay the `🚫 SUBSCRIPTION/ACCESS BLOCKED` report from the
|
|
394
|
+
* captured solve log into the Telegram completion message, so the operator is
|
|
395
|
+
* told that the account itself lost access (and not that "claude failed") on the
|
|
396
|
+
* surface they actually watch. Returns '' when the session hit no such block.
|
|
397
|
+
*/
|
|
398
|
+
export async function buildSubscriptionBlockedExtraSection(logPath, { verbose = false, readFile = fs.readFile, locale = null } = {}) {
|
|
399
|
+
if (!logPath) return '';
|
|
400
|
+
try {
|
|
401
|
+
let logText = '';
|
|
402
|
+
try {
|
|
403
|
+
logText = await readFile(logPath, 'utf8');
|
|
404
|
+
} catch (readError) {
|
|
405
|
+
if (verbose) {
|
|
406
|
+
console.log(`[VERBOSE] Could not read session log ${logPath} for subscription block: ${readError?.message || readError}`);
|
|
407
|
+
}
|
|
408
|
+
return '';
|
|
409
|
+
}
|
|
410
|
+
const telegramLib = await import('./subscription-block-telegram.lib.mjs');
|
|
411
|
+
const parsed = telegramLib.parseSubscriptionBlockFromLog(logText);
|
|
412
|
+
if (!parsed) return '';
|
|
413
|
+
return telegramLib.formatSubscriptionBlockedSection(parsed, { locale });
|
|
414
|
+
} catch (error) {
|
|
415
|
+
if (verbose) {
|
|
416
|
+
console.log(`[VERBOSE] Could not build subscription block section for ${logPath}: ${error?.message || error}`);
|
|
417
|
+
}
|
|
418
|
+
return '';
|
|
419
|
+
}
|
|
420
|
+
}
|
|
392
421
|
async function getDockerContainerFilesystemSizeForSession(sessionName, sessionInfo, { verbose = false, sizeProvider = null } = {}) {
|
|
393
422
|
if (sessionInfo?.isolationBackend !== 'docker') return null;
|
|
394
423
|
const containerName = sessionInfo.sessionId || sessionName;
|
|
@@ -826,6 +855,21 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
826
855
|
}
|
|
827
856
|
}
|
|
828
857
|
const dockerTaskContainerExtraSections = dockerTaskContainerAction?.extraSection ? [dockerTaskContainerAction.extraSection] : [];
|
|
858
|
+
// Issue #2161: a blocked subscription/account explains every other
|
|
859
|
+
// symptom of the run, so it goes first in the completion message.
|
|
860
|
+
const subscriptionBlockedExtraSections = [];
|
|
861
|
+
try {
|
|
862
|
+
const blockedSection = await buildSubscriptionBlockedExtraSection(statusResult?.logPath || sessionInfo?.logPath || null, {
|
|
863
|
+
verbose,
|
|
864
|
+
readFile: options.readFile,
|
|
865
|
+
locale: sessionInfo?.locale || null,
|
|
866
|
+
});
|
|
867
|
+
if (blockedSection) subscriptionBlockedExtraSections.push(blockedSection);
|
|
868
|
+
} catch (blockedError) {
|
|
869
|
+
if (verbose) {
|
|
870
|
+
console.log(`[VERBOSE] Could not build subscription block section for ${sessionName}: ${blockedError?.message || blockedError}`);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
829
873
|
// Issue #2134: say exactly WHY a session was killed, and warn when a
|
|
830
874
|
// session merely survived a kill event instead of reporting a plain
|
|
831
875
|
// success. The pull request gets the very same report below.
|
|
@@ -869,7 +913,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
869
913
|
infoBlock: sessionInfo?.infoBlock || '',
|
|
870
914
|
pullRequestUrl,
|
|
871
915
|
pullRequestState,
|
|
872
|
-
extraSections: [...limitsExtraSections, ...killReport.sections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
916
|
+
extraSections: [...subscriptionBlockedExtraSections, ...limitsExtraSections, ...killReport.sections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
873
917
|
});
|
|
874
918
|
if (killReport.killed || killReport.recovered) {
|
|
875
919
|
const notice = await announceKillOnPullRequest({
|
package/src/solve.mjs
CHANGED
|
@@ -19,7 +19,9 @@ const fs = (await use('fs')).promises;
|
|
|
19
19
|
const crypto = (await use('crypto')).default;
|
|
20
20
|
const memoryCheck = await import('./memory-check.mjs');
|
|
21
21
|
const lib = await import('./lib.mjs');
|
|
22
|
-
const { log, setLogFile, getLogFile, getAbsoluteLogPath, cleanErrorMessage, formatAligned, formatToolExecutionFailure, getVersionInfo, logSolveStartup, setupVerboseLogInterceptor, setupStdioLogInterceptor } = lib;
|
|
22
|
+
const { log, setLogFile, getLogFile, getAbsoluteLogPath, cleanErrorMessage, formatAligned, formatToolExecutionFailure, extractToolErrorCore, getVersionInfo, logSolveStartup, setupVerboseLogInterceptor, setupStdioLogInterceptor } = lib;
|
|
23
|
+
// Issue #2161: terminal subscription/account-access blocks.
|
|
24
|
+
const { detectSubscriptionError, formatSubscriptionErrorReport, formatSubscriptionErrorSummary, SUBSCRIPTION_BLOCKED_MARKER } = await import('./subscription-error.lib.mjs');
|
|
23
25
|
const githubLib = await import('./github.lib.mjs');
|
|
24
26
|
const { sanitizeLogContent, attachLogToGitHub, getToolDisplayName } = githubLib;
|
|
25
27
|
const validation = await import('./solve.validation.lib.mjs');
|
|
@@ -1000,6 +1002,15 @@ try {
|
|
|
1000
1002
|
const toolForFailure = argv.tool || 'claude';
|
|
1001
1003
|
// Issue #1845: surface the core error instead of just "<TOOL> execution failed" (terminal + comment).
|
|
1002
1004
|
const toolFailureMessage = formatToolExecutionFailure({ tool: toolForFailure, toolResult });
|
|
1005
|
+
// Issue #2161: an account/subscription block ("Your organization has disabled
|
|
1006
|
+
// Claude subscription access for Claude Code", a revoked OAuth token, an
|
|
1007
|
+
// expired plan) is terminal — the run must stop, say precisely what happened
|
|
1008
|
+
// and preserve the work, instead of ending on a bare "<TOOL> execution failed
|
|
1009
|
+
// with <provider sentence>". Adapters that parse structured provider codes
|
|
1010
|
+
// (claude.lib.mjs) hand the classification over directly; for every other tool
|
|
1011
|
+
// the rendered message is re-classified here, so the whole failure surface is
|
|
1012
|
+
// covered by one chokepoint.
|
|
1013
|
+
const subscriptionInfo = toolResult?.subscriptionError || detectSubscriptionError({ message: extractToolErrorCore({ toolResult }) || toolFailureMessage, tool: toolForFailure });
|
|
1003
1014
|
if (sessionId) {
|
|
1004
1015
|
await log('');
|
|
1005
1016
|
await log('💡 To continue this session:');
|
|
@@ -1014,15 +1025,32 @@ try {
|
|
|
1014
1025
|
await log('');
|
|
1015
1026
|
}
|
|
1016
1027
|
// Preserve work before remote diagnostics; issue #2101 ended during log upload.
|
|
1028
|
+
let preservedWork = null;
|
|
1017
1029
|
try {
|
|
1018
1030
|
const { criticalErrorRecovery } = await import('./config.lib.mjs');
|
|
1019
1031
|
if (criticalErrorRecovery.autoCommitUncommittedChanges) {
|
|
1020
1032
|
const { commitUncommittedChangesOnCriticalError } = await import('./critical-error-commit.lib.mjs');
|
|
1021
|
-
|
|
1033
|
+
// Issue #2161: when the subscription is gone it is unknown whether/when it
|
|
1034
|
+
// will be restored, so the emergency commit is the only thing standing
|
|
1035
|
+
// between the operator and hours of lost work — name it as such.
|
|
1036
|
+
preservedWork = await commitUncommittedChangesOnCriticalError({ tempDir, branchName, $, log, reason: subscriptionInfo ? formatSubscriptionErrorSummary(subscriptionInfo, { tool: toolForFailure }) : toolFailureMessage });
|
|
1022
1037
|
}
|
|
1023
1038
|
} catch (preserveError) {
|
|
1024
1039
|
await log(` ⚠️ Could not auto-commit before failure exit: ${preserveError.message}`, { verbose: true });
|
|
1025
1040
|
}
|
|
1041
|
+
// Issue #2161: printed after the emergency commit so the block can state
|
|
1042
|
+
// whether the work was preserved. This is the message the operator reads.
|
|
1043
|
+
if (subscriptionInfo) {
|
|
1044
|
+
const reportLines = formatSubscriptionErrorReport(subscriptionInfo, {
|
|
1045
|
+
tool: toolForFailure,
|
|
1046
|
+
sessionId,
|
|
1047
|
+
tempDir,
|
|
1048
|
+
branchName,
|
|
1049
|
+
committed: preservedWork ? preservedWork.committed : null,
|
|
1050
|
+
resumeCommand: sessionId && argv.url ? buildSolveResumeCommand({ issueUrl: argv.url, sessionId, tool: toolForFailure, model: argv.model, fallbackModel: argv.fallbackModel, tempDir }) : null,
|
|
1051
|
+
});
|
|
1052
|
+
for (const line of reportLines) await log(line, { level: 'error' });
|
|
1053
|
+
}
|
|
1026
1054
|
// Attach failure logs before exiting (Issues #1212, #1462: fall back to issue if no PR)
|
|
1027
1055
|
const hasPR = global.createdPR && global.createdPR.number;
|
|
1028
1056
|
const hasIssue = global.issueNumber;
|
|
@@ -1052,7 +1080,9 @@ try {
|
|
|
1052
1080
|
// Include sessionId so the PR comment can present it
|
|
1053
1081
|
sessionId,
|
|
1054
1082
|
// If not a usage limit case, fall back to generic failure format
|
|
1055
|
-
|
|
1083
|
+
// Issue #2161: the PR/issue comment gets the diagnosis + the remediation
|
|
1084
|
+
// steps too — whoever finds the run in the morning reads that, not the log.
|
|
1085
|
+
errorMessage: limitReached ? undefined : subscriptionInfo ? [formatSubscriptionErrorSummary(subscriptionInfo, { tool: toolForFailure }), '', ...(subscriptionInfo.guidance || []).map(step => `- ${step}`)].join('\n') : toolFailureMessage,
|
|
1056
1086
|
argv,
|
|
1057
1087
|
requestedModel: argv.originalModel || argv.model,
|
|
1058
1088
|
tool: argv.tool || 'claude',
|
|
@@ -1070,7 +1100,9 @@ try {
|
|
|
1070
1100
|
await log(` ⚠️ Error uploading failure logs: ${uploadError.message}`);
|
|
1071
1101
|
}
|
|
1072
1102
|
}
|
|
1073
|
-
|
|
1103
|
+
// Issue #2161: the exit message is what /hive and the session monitor see, so
|
|
1104
|
+
// it carries the marker rather than the generic tool-failure sentence.
|
|
1105
|
+
await safeExit(1, subscriptionInfo ? `${SUBSCRIPTION_BLOCKED_MARKER} — ${formatSubscriptionErrorSummary(subscriptionInfo, { tool: toolForFailure })}` : toolFailureMessage);
|
|
1074
1106
|
}
|
|
1075
1107
|
// Clean up .playwright-mcp/ to prevent browser artifacts from triggering auto-restart (Issue #1124)
|
|
1076
1108
|
if (argv.playwrightMcpAutoCleanup !== false) {
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// Issue #2161: Telegram surface for subscription/account-access blocks.
|
|
2
|
+
//
|
|
3
|
+
// `/solve` prints a SUBSCRIPTION_BLOCKED_MARKER report into the session log when
|
|
4
|
+
// the account can no longer use the agent tool (expired/cancelled Claude MAX
|
|
5
|
+
// subscription, org policy, revoked ChatGPT/Codex entitlement, ...). The session
|
|
6
|
+
// monitor captures that log, so the same block can be replayed into the Telegram
|
|
7
|
+
// completion message without any extra plumbing between processes.
|
|
8
|
+
|
|
9
|
+
import { SUBSCRIPTION_BLOCKED_MARKER } from './subscription-error.lib.mjs';
|
|
10
|
+
import { lt } from './limits-i18n.lib.mjs';
|
|
11
|
+
|
|
12
|
+
const MAX_MESSAGE_LENGTH = 400;
|
|
13
|
+
const MAX_GUIDANCE_STEPS = 4;
|
|
14
|
+
|
|
15
|
+
const truncate = (value, limit = MAX_MESSAGE_LENGTH) => {
|
|
16
|
+
const text = String(value || '').trim();
|
|
17
|
+
if (text.length <= limit) return text;
|
|
18
|
+
return `${text.slice(0, limit - 1)}…`;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const stripPrefix = (line, prefix) => line.slice(line.indexOf(prefix) + prefix.length).trim();
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Parse the last SUBSCRIPTION_BLOCKED_MARKER report out of a captured session log.
|
|
25
|
+
*
|
|
26
|
+
* The report is emitted by formatSubscriptionErrorReport(); every line after the
|
|
27
|
+
* marker line is indented, so the block ends at the first non-indented line.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} logText
|
|
30
|
+
* @returns {null|{tool: string|null, label: string|null, message: string|null, code: string|null, reason: string|null, guidance: string[], committed: boolean|null, resumeCommand: string|null}}
|
|
31
|
+
*/
|
|
32
|
+
export function parseSubscriptionBlockFromLog(logText) {
|
|
33
|
+
if (!logText || typeof logText !== 'string') return null;
|
|
34
|
+
if (!logText.includes(SUBSCRIPTION_BLOCKED_MARKER)) return null;
|
|
35
|
+
|
|
36
|
+
const lines = logText.split('\n');
|
|
37
|
+
// Walk backwards: the richest report (from /solve) is the last one printed.
|
|
38
|
+
let markerIndex = -1;
|
|
39
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
40
|
+
if (lines[i].includes(SUBSCRIPTION_BLOCKED_MARKER)) {
|
|
41
|
+
markerIndex = i;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (markerIndex === -1) return null;
|
|
46
|
+
|
|
47
|
+
const headline = stripPrefix(lines[markerIndex], SUBSCRIPTION_BLOCKED_MARKER).replace(/^—\s*/, '');
|
|
48
|
+
const separator = headline.indexOf(':');
|
|
49
|
+
const parsed = {
|
|
50
|
+
tool: separator > 0 ? headline.slice(0, separator).trim() : null,
|
|
51
|
+
label: separator > 0 ? headline.slice(separator + 1).trim() : headline || null,
|
|
52
|
+
message: null,
|
|
53
|
+
code: null,
|
|
54
|
+
reason: null,
|
|
55
|
+
guidance: [],
|
|
56
|
+
committed: null,
|
|
57
|
+
resumeCommand: null,
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
for (let i = markerIndex + 1; i < lines.length; i += 1) {
|
|
61
|
+
const raw = lines[i];
|
|
62
|
+
if (!raw.trim()) continue;
|
|
63
|
+
if (!/^\s{3}/.test(raw)) break; // end of the indented report block
|
|
64
|
+
const line = raw.trim();
|
|
65
|
+
if (line.startsWith('Provider said:')) parsed.message = stripPrefix(line, 'Provider said:');
|
|
66
|
+
else if (line.startsWith('Error code:')) parsed.code = stripPrefix(line, 'Error code:');
|
|
67
|
+
else if (line.startsWith('HTTP status:')) parsed.code = `HTTP ${stripPrefix(line, 'HTTP status:')}`;
|
|
68
|
+
else if (line.startsWith('Why this stops the run:')) parsed.reason = stripPrefix(line, 'Why this stops the run:');
|
|
69
|
+
else if (line.startsWith('•')) parsed.guidance.push(line.slice(1).trim());
|
|
70
|
+
else if (line.startsWith('💾')) parsed.committed = true;
|
|
71
|
+
else if (line.startsWith('⚠️')) parsed.committed = false;
|
|
72
|
+
else if (line.startsWith('▶️')) parsed.resumeCommand = stripPrefix(line, ':');
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return parsed;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Render the parsed block as a Telegram extraSection (title + fenced body), the
|
|
80
|
+
* same shape formatDiskDiagnosticsBlock() uses.
|
|
81
|
+
*
|
|
82
|
+
* @returns {string} empty string when there is nothing to show
|
|
83
|
+
*/
|
|
84
|
+
export function formatSubscriptionBlockedSection(parsed, { locale = null } = {}) {
|
|
85
|
+
if (!parsed) return '';
|
|
86
|
+
const options = locale ? { locale } : {};
|
|
87
|
+
const body = [];
|
|
88
|
+
|
|
89
|
+
const label = parsed.label || lt('subscription_blocked_title', {}, options);
|
|
90
|
+
body.push(parsed.tool ? `${parsed.tool}: ${label}` : label);
|
|
91
|
+
if (parsed.message) body.push(`${lt('subscription_blocked_provider', {}, options)}: ${truncate(parsed.message)}`);
|
|
92
|
+
if (parsed.code) body.push(`${lt('subscription_blocked_code', {}, options)}: ${parsed.code}`);
|
|
93
|
+
if (parsed.reason) body.push(`${lt('subscription_blocked_reason', {}, options)}: ${parsed.reason}`);
|
|
94
|
+
body.push(lt('subscription_blocked_note', {}, options));
|
|
95
|
+
if (parsed.guidance.length) {
|
|
96
|
+
body.push('');
|
|
97
|
+
body.push(`${lt('subscription_blocked_steps', {}, options)}:`);
|
|
98
|
+
for (const step of parsed.guidance.slice(0, MAX_GUIDANCE_STEPS)) body.push(` • ${step}`);
|
|
99
|
+
}
|
|
100
|
+
if (parsed.committed === true) {
|
|
101
|
+
body.push('');
|
|
102
|
+
body.push(lt('subscription_blocked_preserved', {}, options));
|
|
103
|
+
}
|
|
104
|
+
if (parsed.resumeCommand) {
|
|
105
|
+
body.push('');
|
|
106
|
+
body.push(`${lt('subscription_blocked_resume', {}, options)}: ${parsed.resumeCommand}`);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return `🚫 ${lt('subscription_blocked_title', {}, options)}\n\`\`\`\n${body.join('\n')}\n\`\`\``;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export default {
|
|
113
|
+
parseSubscriptionBlockFromLog,
|
|
114
|
+
formatSubscriptionBlockedSection,
|
|
115
|
+
};
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Subscription / account-access error detection for AI CLI tools.
|
|
3
|
+
*
|
|
4
|
+
* Issue #2161: a `/solve` run died after 4h11m and $31.39 of work with nothing
|
|
5
|
+
* but the generic line
|
|
6
|
+
*
|
|
7
|
+
* ❌ CLAUDE execution failed with Your organization has disabled Claude
|
|
8
|
+
* subscription access for Claude Code · Use an Anthropic API key instead,
|
|
9
|
+
* or ask your admin to enable access
|
|
10
|
+
*
|
|
11
|
+
* That sentence is not a transient API fault and not a usage limit: it means the
|
|
12
|
+
* *account itself* is no longer permitted to use the tool. Waiting does not help,
|
|
13
|
+
* retrying does not help, switching to a fallback model does not help — the run
|
|
14
|
+
* must stop immediately, preserve the work, and tell the operator exactly what to
|
|
15
|
+
* do.
|
|
16
|
+
*
|
|
17
|
+
* This module is the single place that recognises that whole class of errors for
|
|
18
|
+
* every tool hive-mind can drive. Two detection layers are used, strongest first:
|
|
19
|
+
*
|
|
20
|
+
* 1. Machine-readable codes emitted by the tool (Claude Code's `error` field on
|
|
21
|
+
* stream-json `assistant`/`result` events, Codex's auth error codes). These
|
|
22
|
+
* are exact and locale independent.
|
|
23
|
+
* 2. Verbatim user-facing strings, transcribed from the shipped CLI binaries
|
|
24
|
+
* (see docs/case-studies/issue-2161/provider-error-strings.md). Used when
|
|
25
|
+
* only the rendered message survives (most tools give us nothing else).
|
|
26
|
+
*
|
|
27
|
+
* Deliberately NOT matched here:
|
|
28
|
+
* - "Authentication error · This may be a temporary network issue, please try
|
|
29
|
+
* again" — Claude Code's own wording says it is transient, so it belongs to
|
|
30
|
+
* the retry path, not to this terminal path.
|
|
31
|
+
* - Usage/quota limits ("You've hit your usage limit", "resets 5am") — those
|
|
32
|
+
* have a reset time and are handled by usage-limit.lib.mjs.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** Emitted verbatim into the log so /hive, the Telegram monitor and humans can grep for it. */
|
|
36
|
+
export const SUBSCRIPTION_BLOCKED_MARKER = '🚫 SUBSCRIPTION/ACCESS BLOCKED';
|
|
37
|
+
|
|
38
|
+
export const SUBSCRIPTION_ERROR_KINDS = {
|
|
39
|
+
ORG_SUBSCRIPTION_DISABLED: 'org_subscription_disabled',
|
|
40
|
+
ACCOUNT_NO_ACCESS: 'account_no_access',
|
|
41
|
+
LOGIN_REQUIRED: 'login_required',
|
|
42
|
+
BILLING: 'billing',
|
|
43
|
+
PLAN_RESTRICTED: 'plan_restricted',
|
|
44
|
+
API_KEY_INVALID: 'api_key_invalid',
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const K = SUBSCRIPTION_ERROR_KINDS;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Machine-readable codes → kind. Sources:
|
|
51
|
+
* - Claude Code CLI 2.1.233, blocked-state switch (`oauth_org_not_allowed`,
|
|
52
|
+
* `authentication_failed`, `billing_error`, …).
|
|
53
|
+
* - Codex CLI 0.147.0 auth error codes (`missing_codex_entitlement`,
|
|
54
|
+
* `refresh_token_expired`, `disabled_by_admin`, `plan_not_eligible`, …).
|
|
55
|
+
*/
|
|
56
|
+
export const SUBSCRIPTION_ERROR_CODES = Object.freeze({
|
|
57
|
+
// Claude Code
|
|
58
|
+
oauth_org_not_allowed: K.ORG_SUBSCRIPTION_DISABLED,
|
|
59
|
+
authentication_failed: K.LOGIN_REQUIRED,
|
|
60
|
+
token_revoked: K.LOGIN_REQUIRED,
|
|
61
|
+
invalid_api_key: K.API_KEY_INVALID,
|
|
62
|
+
billing_error: K.BILLING,
|
|
63
|
+
credit_balance_low: K.BILLING,
|
|
64
|
+
// Codex
|
|
65
|
+
missing_codex_entitlement: K.ACCOUNT_NO_ACCESS,
|
|
66
|
+
disabled_by_admin: K.ORG_SUBSCRIPTION_DISABLED,
|
|
67
|
+
plan_not_eligible: K.PLAN_RESTRICTED,
|
|
68
|
+
required_app_unavailable: K.ACCOUNT_NO_ACCESS,
|
|
69
|
+
refresh_token_expired: K.LOGIN_REQUIRED,
|
|
70
|
+
refresh_token_invalidated: K.LOGIN_REQUIRED,
|
|
71
|
+
not_chatgpt_auth: K.LOGIN_REQUIRED,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Substrings that look authentication-ish but are explicitly transient. Checked
|
|
76
|
+
* before every other rule so a network blip is never reported as a cancelled
|
|
77
|
+
* subscription (which would stop the whole hive).
|
|
78
|
+
*/
|
|
79
|
+
const TRANSIENT_AUTH_PATTERNS = ['this may be a temporary network issue', 'could not authenticate with its upstream provider', 'temporary failure in name resolution'];
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Verbatim strings from the shipped CLIs, lower-cased. `tool` is informational:
|
|
83
|
+
* a message is matched regardless of which tool produced it, because hive-mind
|
|
84
|
+
* often only sees the rendered text several layers away from its origin.
|
|
85
|
+
*/
|
|
86
|
+
const MESSAGE_RULES = [
|
|
87
|
+
// ---- Claude Code -------------------------------------------------------
|
|
88
|
+
{ kind: K.ORG_SUBSCRIPTION_DISABLED, tool: 'claude', needles: ['organization has disabled claude subscription access'] },
|
|
89
|
+
{ kind: K.ORG_SUBSCRIPTION_DISABLED, tool: 'claude', needles: ['organization has disabled api key authentication'] },
|
|
90
|
+
{ kind: K.ORG_SUBSCRIPTION_DISABLED, tool: 'claude', needles: ['belongs to a disabled organization'] },
|
|
91
|
+
{ kind: K.ORG_SUBSCRIPTION_DISABLED, tool: 'claude', needles: ['organization has been disabled'] },
|
|
92
|
+
{ kind: K.ACCOUNT_NO_ACCESS, tool: 'claude', needles: ['your account does not have access to claude'] },
|
|
93
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'claude', needles: ['oauth token revoked'] },
|
|
94
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'claude', needles: ['login expired'] },
|
|
95
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'claude', needles: ['not logged in'] },
|
|
96
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'claude', needles: ['session expired. please run /login'] },
|
|
97
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'claude', needles: ['oauth session expired and could not be refreshed'] },
|
|
98
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'claude', needles: ['anthropic profile login expired'] },
|
|
99
|
+
{ kind: K.BILLING, tool: 'claude', needles: ['credit balance is too low'] },
|
|
100
|
+
{ kind: K.API_KEY_INVALID, tool: 'claude', needles: ['invalid api key'] },
|
|
101
|
+
{ kind: K.API_KEY_INVALID, tool: 'claude', needles: ['invalid auth token'] },
|
|
102
|
+
{ kind: K.PLAN_RESTRICTED, tool: 'claude', needles: ['is not available with the claude pro plan'] },
|
|
103
|
+
{ kind: K.PLAN_RESTRICTED, tool: 'claude', needles: ['auto mode is unavailable for your plan'] },
|
|
104
|
+
|
|
105
|
+
// ---- Codex -------------------------------------------------------------
|
|
106
|
+
{ kind: K.ACCOUNT_NO_ACCESS, tool: 'codex', needles: ['you do not have access to codex'] },
|
|
107
|
+
{ kind: K.ACCOUNT_NO_ACCESS, tool: 'codex', needles: ['not currently authorized to use codex'] },
|
|
108
|
+
{ kind: K.ACCOUNT_NO_ACCESS, tool: 'codex', needles: ['contact your workspace administrator to request access to codex'] },
|
|
109
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'codex', needles: ['access token could not be refreshed'] },
|
|
110
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'codex', needles: ['oauth refresh token was rejected'] },
|
|
111
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'codex', needles: ["not signed in. please run 'codex login'"] },
|
|
112
|
+
|
|
113
|
+
// ---- Qwen Code ---------------------------------------------------------
|
|
114
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'qwen', needles: ['qwen oauth credentials expired'] },
|
|
115
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'qwen', needles: ['refresh token expired or invalid'] },
|
|
116
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'qwen', needles: ['failed to obtain valid qwen access token'] },
|
|
117
|
+
{ kind: K.PLAN_RESTRICTED, tool: 'qwen', needles: ['coding plan api key not found'] },
|
|
118
|
+
|
|
119
|
+
// ---- Gemini CLI --------------------------------------------------------
|
|
120
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'gemini', needles: ['please re-authenticate with the correct type'] },
|
|
121
|
+
{ kind: K.PLAN_RESTRICTED, tool: 'gemini', needles: ["doesn't have a gemini code assist"] },
|
|
122
|
+
|
|
123
|
+
// ---- OpenCode ----------------------------------------------------------
|
|
124
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'opencode', needles: ['run `opencode auth login` in the terminal'] },
|
|
125
|
+
{ kind: K.LOGIN_REQUIRED, tool: 'opencode', needles: ['oauth token refresh failed and no fallback'] },
|
|
126
|
+
{ kind: K.ACCOUNT_NO_ACCESS, tool: 'opencode', needles: ['your account does not have access to ai features'] },
|
|
127
|
+
|
|
128
|
+
// ---- Generic provider phrasing (any tool) ------------------------------
|
|
129
|
+
{ kind: K.ORG_SUBSCRIPTION_DISABLED, tool: null, needles: ['has disabled', 'subscription access'] },
|
|
130
|
+
{ kind: K.BILLING, tool: null, needles: ['subscription', 'expired'] },
|
|
131
|
+
{ kind: K.BILLING, tool: null, needles: ['subscription', 'cancel'] },
|
|
132
|
+
];
|
|
133
|
+
|
|
134
|
+
const KIND_LABELS = Object.freeze({
|
|
135
|
+
[K.ORG_SUBSCRIPTION_DISABLED]: 'Subscription access disabled for this organization',
|
|
136
|
+
[K.ACCOUNT_NO_ACCESS]: 'Account is not authorized to use this tool',
|
|
137
|
+
[K.LOGIN_REQUIRED]: 'Authentication expired — re-login required',
|
|
138
|
+
[K.BILLING]: 'Subscription/billing problem',
|
|
139
|
+
[K.PLAN_RESTRICTED]: 'Current plan does not allow this request',
|
|
140
|
+
[K.API_KEY_INVALID]: 'Invalid API key or auth token',
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
const KIND_REASONS = Object.freeze({
|
|
144
|
+
[K.ORG_SUBSCRIPTION_DISABLED]: 'The provider rejected the request because the organization/account behind the subscription is no longer allowed to use this CLI. This is an account-level block, not a rate limit — it will not clear on its own.',
|
|
145
|
+
[K.ACCOUNT_NO_ACCESS]: 'The provider accepted the credentials but the account has no entitlement for this product. Access must be granted before any further run can succeed.',
|
|
146
|
+
[K.LOGIN_REQUIRED]: 'The stored OAuth credentials are gone, revoked or unrefreshable. Every request will fail until the tool is logged in again.',
|
|
147
|
+
[K.BILLING]: 'The subscription is expired, cancelled or out of credit. Requests stay rejected until billing is restored.',
|
|
148
|
+
[K.PLAN_RESTRICTED]: 'The account is authenticated, but the requested model/mode is not included in the current plan.',
|
|
149
|
+
[K.API_KEY_INVALID]: 'The configured API key or auth token was rejected by the provider.',
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
/** Per-tool re-authentication commands, used to build actionable guidance. */
|
|
153
|
+
const TOOL_LOGIN_HINTS = Object.freeze({
|
|
154
|
+
claude: 'claude /login (or set ANTHROPIC_API_KEY for API-key billing)',
|
|
155
|
+
codex: 'codex login (add --device-auth on a headless machine)',
|
|
156
|
+
qwen: 'qwen → /auth',
|
|
157
|
+
gemini: 'gemini → /auth',
|
|
158
|
+
opencode: 'opencode auth login',
|
|
159
|
+
agent: 'agent auth login',
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
const TOOL_ACCOUNT_URLS = Object.freeze({
|
|
163
|
+
claude: 'https://claude.ai/settings/billing',
|
|
164
|
+
codex: 'https://chatgpt.com/codex/settings/usage',
|
|
165
|
+
qwen: 'https://chat.qwen.ai',
|
|
166
|
+
gemini: 'https://codeassist.google.com',
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
const toText = value => {
|
|
170
|
+
if (value === null || value === undefined) return '';
|
|
171
|
+
if (typeof value === 'string') return value;
|
|
172
|
+
if (typeof value?.error?.message === 'string') return value.error.message;
|
|
173
|
+
if (typeof value?.message === 'string') return value.message;
|
|
174
|
+
try {
|
|
175
|
+
return JSON.stringify(value);
|
|
176
|
+
} catch {
|
|
177
|
+
return String(value);
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* True when the text is an authentication-flavoured error that the provider
|
|
183
|
+
* itself describes as temporary. Such errors must keep using the retry path.
|
|
184
|
+
*/
|
|
185
|
+
export const isTransientAuthError = value => {
|
|
186
|
+
const lower = toText(value).toLowerCase();
|
|
187
|
+
if (!lower) return false;
|
|
188
|
+
return TRANSIENT_AUTH_PATTERNS.some(p => lower.includes(p));
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
const buildGuidance = (kind, tool) => {
|
|
192
|
+
const loginHint = TOOL_LOGIN_HINTS[tool] || TOOL_LOGIN_HINTS.claude;
|
|
193
|
+
const accountUrl = TOOL_ACCOUNT_URLS[tool] || null;
|
|
194
|
+
const steps = [];
|
|
195
|
+
switch (kind) {
|
|
196
|
+
case K.ORG_SUBSCRIPTION_DISABLED:
|
|
197
|
+
steps.push('Ask the organization/workspace admin to re-enable CLI access for this account.');
|
|
198
|
+
steps.push('Or switch this run to API-key billing instead of the subscription.');
|
|
199
|
+
break;
|
|
200
|
+
case K.ACCOUNT_NO_ACCESS:
|
|
201
|
+
steps.push('Request access for this account from the workspace administrator.');
|
|
202
|
+
steps.push('Verify you are logged in with the account that actually owns the subscription.');
|
|
203
|
+
break;
|
|
204
|
+
case K.LOGIN_REQUIRED:
|
|
205
|
+
steps.push(`Re-authenticate the tool: ${loginHint}`);
|
|
206
|
+
break;
|
|
207
|
+
case K.BILLING:
|
|
208
|
+
steps.push('Renew/reactivate the subscription or top up the credit balance.');
|
|
209
|
+
if (accountUrl) steps.push(`Billing page: ${accountUrl}`);
|
|
210
|
+
break;
|
|
211
|
+
case K.PLAN_RESTRICTED:
|
|
212
|
+
steps.push('Pick a model/mode included in the current plan (see --model), or upgrade the plan.');
|
|
213
|
+
steps.push(`After a plan change, re-login so the new entitlements are picked up: ${loginHint}`);
|
|
214
|
+
break;
|
|
215
|
+
case K.API_KEY_INVALID:
|
|
216
|
+
steps.push('Fix or regenerate the configured API key / auth token, then re-run.');
|
|
217
|
+
break;
|
|
218
|
+
default:
|
|
219
|
+
steps.push(`Re-authenticate the tool: ${loginHint}`);
|
|
220
|
+
}
|
|
221
|
+
steps.push('Once access is restored, resume with the session ID printed above — no work is lost.');
|
|
222
|
+
return steps;
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Detect an account/subscription-level block.
|
|
227
|
+
*
|
|
228
|
+
* @param {string|Object} input - Raw message, or a descriptor:
|
|
229
|
+
* { message, tool, errorCode, apiErrorStatus, terminalReason }
|
|
230
|
+
* @returns {null|{isSubscriptionError: true, kind, code, tool, label, reason, message, guidance, apiErrorStatus}}
|
|
231
|
+
*/
|
|
232
|
+
export const detectSubscriptionError = input => {
|
|
233
|
+
const descriptor = typeof input === 'string' || input === null || input === undefined ? { message: input } : input;
|
|
234
|
+
const message = toText(descriptor.message ?? descriptor);
|
|
235
|
+
const tool = descriptor.tool ? String(descriptor.tool).toLowerCase() : null;
|
|
236
|
+
const rawCode = descriptor.errorCode ? String(descriptor.errorCode).toLowerCase().trim() : null;
|
|
237
|
+
const apiErrorStatus = Number.isFinite(descriptor.apiErrorStatus) ? descriptor.apiErrorStatus : null;
|
|
238
|
+
|
|
239
|
+
// Layer 1: machine-readable code. Trusted even when the message is missing.
|
|
240
|
+
if (rawCode && Object.hasOwn(SUBSCRIPTION_ERROR_CODES, rawCode)) {
|
|
241
|
+
const kind = SUBSCRIPTION_ERROR_CODES[rawCode];
|
|
242
|
+
return {
|
|
243
|
+
isSubscriptionError: true,
|
|
244
|
+
kind,
|
|
245
|
+
code: rawCode,
|
|
246
|
+
tool,
|
|
247
|
+
label: KIND_LABELS[kind],
|
|
248
|
+
reason: KIND_REASONS[kind],
|
|
249
|
+
message,
|
|
250
|
+
apiErrorStatus,
|
|
251
|
+
guidance: buildGuidance(kind, tool),
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
if (!message) return null;
|
|
256
|
+
const lower = message.toLowerCase();
|
|
257
|
+
if (isTransientAuthError(lower)) return null;
|
|
258
|
+
|
|
259
|
+
// Layer 2: verbatim provider strings.
|
|
260
|
+
for (const rule of MESSAGE_RULES) {
|
|
261
|
+
if (!rule.needles.every(n => lower.includes(n))) continue;
|
|
262
|
+
return {
|
|
263
|
+
isSubscriptionError: true,
|
|
264
|
+
kind: rule.kind,
|
|
265
|
+
code: rawCode || null,
|
|
266
|
+
tool: tool || rule.tool || null,
|
|
267
|
+
label: KIND_LABELS[rule.kind],
|
|
268
|
+
reason: KIND_REASONS[rule.kind],
|
|
269
|
+
message,
|
|
270
|
+
apiErrorStatus,
|
|
271
|
+
guidance: buildGuidance(rule.kind, tool || rule.tool),
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
return null;
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
/** Convenience boolean wrapper mirroring isUsageLimitError(). */
|
|
278
|
+
export const isSubscriptionBlockedError = input => detectSubscriptionError(input) !== null;
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Render the terminal/log block. The first line is SUBSCRIPTION_BLOCKED_MARKER so
|
|
282
|
+
* downstream consumers (/hive worker output scanner, Telegram session monitor,
|
|
283
|
+
* `grep`) have one stable anchor.
|
|
284
|
+
*
|
|
285
|
+
* @returns {string[]} lines
|
|
286
|
+
*/
|
|
287
|
+
export const formatSubscriptionErrorReport = (info, { tool = null, sessionId = null, tempDir = null, branchName = null, committed = null, resumeCommand = null } = {}) => {
|
|
288
|
+
if (!info) return [];
|
|
289
|
+
const toolName = (info.tool || tool || 'tool').toUpperCase();
|
|
290
|
+
const lines = [];
|
|
291
|
+
lines.push('');
|
|
292
|
+
lines.push(`${SUBSCRIPTION_BLOCKED_MARKER} — ${toolName}: ${info.label}`);
|
|
293
|
+
lines.push(` Provider said: ${info.message || '(no message)'}`);
|
|
294
|
+
if (info.code) lines.push(` Error code: ${info.code}${info.apiErrorStatus ? ` (HTTP ${info.apiErrorStatus})` : ''}`);
|
|
295
|
+
else if (info.apiErrorStatus) lines.push(` HTTP status: ${info.apiErrorStatus}`);
|
|
296
|
+
lines.push(` Why this stops the run: ${info.reason}`);
|
|
297
|
+
lines.push(' This is NOT a usage limit and NOT a transient API error — retrying, waiting for a reset');
|
|
298
|
+
lines.push(' or switching to a fallback model cannot fix it, so the task is stopped now.');
|
|
299
|
+
lines.push('');
|
|
300
|
+
lines.push(' What to do:');
|
|
301
|
+
for (const step of info.guidance || []) lines.push(` • ${step}`);
|
|
302
|
+
if (committed === true) lines.push(' 💾 Uncommitted changes were auto-committed and pushed before stopping.');
|
|
303
|
+
else if (committed === false) lines.push(' ⚠️ No uncommitted changes to preserve (working tree was clean).');
|
|
304
|
+
if (tempDir) lines.push(` 📁 Working directory: ${tempDir}`);
|
|
305
|
+
if (branchName) lines.push(` 🌿 Branch: ${branchName}`);
|
|
306
|
+
if (sessionId) lines.push(` 📌 Session ID: ${sessionId}`);
|
|
307
|
+
if (resumeCommand) lines.push(` ▶️ Resume after access is restored: ${resumeCommand}`);
|
|
308
|
+
lines.push('');
|
|
309
|
+
return lines;
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
/** One-line summary used for exit messages, PR comments and commit reasons. */
|
|
313
|
+
export const formatSubscriptionErrorSummary = (info, { tool = null } = {}) => {
|
|
314
|
+
if (!info) return '';
|
|
315
|
+
const toolName = (info.tool || tool || 'tool').toUpperCase();
|
|
316
|
+
return `${toolName} stopped: ${info.label}${info.code ? ` [${info.code}]` : ''} — ${info.message || ''}`.trim();
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
export default {
|
|
320
|
+
SUBSCRIPTION_BLOCKED_MARKER,
|
|
321
|
+
SUBSCRIPTION_ERROR_KINDS,
|
|
322
|
+
SUBSCRIPTION_ERROR_CODES,
|
|
323
|
+
detectSubscriptionError,
|
|
324
|
+
isSubscriptionBlockedError,
|
|
325
|
+
isTransientAuthError,
|
|
326
|
+
formatSubscriptionErrorReport,
|
|
327
|
+
formatSubscriptionErrorSummary,
|
|
328
|
+
};
|
package/src/tool-retry.lib.mjs
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { retryLimits } from './config.lib.mjs';
|
|
4
4
|
import { resolveDefaultFallbackModel, resolveModelId } from './models/index.mjs';
|
|
5
|
+
import { detectSubscriptionError, isTransientAuthError } from './subscription-error.lib.mjs';
|
|
5
6
|
|
|
6
7
|
const normalizeMessage = value => {
|
|
7
8
|
if (value === null || value === undefined) return '';
|
|
@@ -27,6 +28,34 @@ export const classifyRetryableError = value => {
|
|
|
27
28
|
const message = normalizeMessage(value);
|
|
28
29
|
const lower = message.toLowerCase();
|
|
29
30
|
|
|
31
|
+
// Issue #2161: account/subscription-level blocks ("Your organization has
|
|
32
|
+
// disabled Claude subscription access for Claude Code", "You do not have
|
|
33
|
+
// access to Codex", revoked OAuth tokens, expired subscriptions, …). These are
|
|
34
|
+
// terminal by nature: the credentials themselves are no longer accepted, so
|
|
35
|
+
// neither a backoff nor a different model can recover — the run must stop and
|
|
36
|
+
// the operator must restore access. isCapacity stays false so
|
|
37
|
+
// maybeSwitchToFallbackModel() never burns a fallback hop on them.
|
|
38
|
+
//
|
|
39
|
+
// Checked first, because several of these messages contain words ("timed
|
|
40
|
+
// out", "rate", "503") that later transient branches would otherwise claim.
|
|
41
|
+
// detectSubscriptionError() itself excludes provider errors that are
|
|
42
|
+
// explicitly described as temporary — see isTransientAuthError below.
|
|
43
|
+
const subscriptionError = detectSubscriptionError(message);
|
|
44
|
+
if (subscriptionError) {
|
|
45
|
+
return { message, isRetryable: false, isCapacity: false, isSubscriptionError: true, subscriptionError, label: 'subscription access blocked' };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Issue #2161: the counterpart — Claude Code's own wording marks this
|
|
49
|
+
// authentication failure as temporary ("This may be a temporary network
|
|
50
|
+
// issue, please try again"). Without an explicit branch it would fall through
|
|
51
|
+
// to the non-retryable default and abort a run that a retry would have saved.
|
|
52
|
+
// The `auth` guard keeps purely network-level members of that list (e.g.
|
|
53
|
+
// "Temporary failure in name resolution") on their own, more specific branches
|
|
54
|
+
// below — this branch only claims the *authentication* wordings.
|
|
55
|
+
if (isTransientAuthError(lower) && lower.includes('auth')) {
|
|
56
|
+
return { message, isRetryable: true, isCapacity: false, label: 'Transient authentication/network error' };
|
|
57
|
+
}
|
|
58
|
+
|
|
30
59
|
// Genuine model-specific capacity: the API explicitly tells us this *particular*
|
|
31
60
|
// model is full and recommends trying a *different* model (e.g. Codex's
|
|
32
61
|
// "Selected model is at capacity. Please try a different model."). Here a model
|