@link-assistant/hive-mind 2.13.4 → 2.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/package.json +1 -1
- package/src/agent.lib.mjs +5 -1
- package/src/claude.lib.mjs +5 -158
- package/src/claude.session-tokens.lib.mjs +180 -0
- package/src/codex.diagnostics.lib.mjs +135 -0
- package/src/codex.lib.mjs +8 -121
- package/src/config.lib.mjs +9 -0
- package/src/gemini.lib.mjs +5 -1
- package/src/git-retry.lib.mjs +97 -0
- package/src/github-pr-idempotency.lib.mjs +83 -0
- package/src/github-rate-limit.lib.mjs +44 -41
- package/src/hive.mjs +8 -150
- package/src/hive.repository-fallback.lib.mjs +125 -0
- package/src/hive.startup-checks.lib.mjs +57 -0
- package/src/isolation-runner.lib.mjs +4 -277
- package/src/isolation-runner.parsers.lib.mjs +292 -0
- package/src/lib.mjs +79 -18
- package/src/opencode.lib.mjs +5 -1
- package/src/qwen.lib.mjs +5 -1
- package/src/session-monitor.lib.mjs +12 -272
- package/src/session-monitor.queries.lib.mjs +304 -0
- package/src/solve.auto-pr-push-sync.lib.mjs +176 -0
- package/src/solve.auto-pr.lib.mjs +40 -154
- package/src/solve.mjs +8 -158
- package/src/solve.mode.lib.mjs +191 -0
- package/src/telegram-solve-queue.lib.mjs +19 -272
- package/src/telegram-solve-queue.throttling.lib.mjs +323 -0
- package/src/transient-errors.lib.mjs +238 -0
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resource and API-limit throttling checks for the Telegram solve queue.
|
|
3
|
+
*
|
|
4
|
+
* These are the two decision functions the queue consults before starting a
|
|
5
|
+
* command: `checkSystemResources` (RAM, CPU, disk) and `checkApiLimits`
|
|
6
|
+
* (Claude/Codex session + weekly usage, GitHub API). Both return the same
|
|
7
|
+
* shape — the reasons a command must wait, whether the queue must fall back to
|
|
8
|
+
* one-at-a-time, and whether the command must be rejected outright.
|
|
9
|
+
*
|
|
10
|
+
* Extracted from telegram-solve-queue.lib.mjs (issue #2175) so that file stays
|
|
11
|
+
* under the 1350-line early-warning threshold of the CI file-headroom check
|
|
12
|
+
* (long files cause concurrent PR merge conflicts — issue #1593). They take the
|
|
13
|
+
* queue instance explicitly instead of `this`; SolveQueue keeps thin methods
|
|
14
|
+
* that delegate here, so every caller and test is unaffected.
|
|
15
|
+
*
|
|
16
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2175
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { getCachedClaudeLimits, getCachedCodexLimits, getCachedGitHubLimits, getCachedMemoryInfo, getCachedCpuInfo, getCachedDiskInfo } from './limits.lib.mjs';
|
|
20
|
+
import { formatWaitingReason } from './telegram-solve-queue.helpers.lib.mjs';
|
|
21
|
+
import { QUEUE_CONFIG } from './queue-config.lib.mjs';
|
|
22
|
+
import { lt } from './limits-i18n.lib.mjs';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Normalize the locale argument, which callers pass either as a bare string or
|
|
26
|
+
* inside an options object.
|
|
27
|
+
* @param {string|{locale?: string}} [options]
|
|
28
|
+
* @returns {string|null}
|
|
29
|
+
*/
|
|
30
|
+
export function getLocale(options = {}) {
|
|
31
|
+
if (typeof options === 'string') return options;
|
|
32
|
+
return options?.locale || null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Suffix a waiting reason with the "waiting for the current command" note.
|
|
37
|
+
* @param {string} reason
|
|
38
|
+
* @param {string|null} locale
|
|
39
|
+
* @returns {string}
|
|
40
|
+
*/
|
|
41
|
+
export function appendWaitingForCurrentCommand(reason, locale) {
|
|
42
|
+
return `${reason} (${lt('queue_waiting_current_command', {}, { locale })})`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Check system resources (RAM, CPU, disk) using cached values
|
|
47
|
+
*
|
|
48
|
+
* Uses 5-minute load average for CPU instead of instantaneous usage.
|
|
49
|
+
* This provides a more stable metric that isn't affected by brief spikes
|
|
50
|
+
* during claude process startup.
|
|
51
|
+
*
|
|
52
|
+
* Resource threshold modes are now configurable via HIVE_MIND_QUEUE_CONFIG:
|
|
53
|
+
* - 'reject': Immediately reject the command, no queueing
|
|
54
|
+
* - 'enqueue': Block all commands unconditionally until metric drops
|
|
55
|
+
* - 'dequeue-one-at-a-time': Allow one command when above threshold
|
|
56
|
+
*
|
|
57
|
+
* Default strategies:
|
|
58
|
+
* - RAM: enqueue
|
|
59
|
+
* - CPU: enqueue
|
|
60
|
+
* - DISK: enqueue (waits until disk drops below the threshold)
|
|
61
|
+
*
|
|
62
|
+
* See: https://github.com/link-assistant/hive-mind/issues/1155
|
|
63
|
+
* See: https://github.com/link-assistant/hive-mind/issues/1253
|
|
64
|
+
* See: https://github.com/link-assistant/hive-mind/issues/1981
|
|
65
|
+
*
|
|
66
|
+
* @param {number} totalProcessing - Total processing count (queue + external claude processes)
|
|
67
|
+
* @returns {Promise<{ok: boolean, reasons: string[], oneAtATime: boolean, rejected: boolean, rejectReason: string|null}>}
|
|
68
|
+
*/
|
|
69
|
+
export async function checkSystemResources(queue, totalProcessing = 0, options = {}) {
|
|
70
|
+
const locale = getLocale(options);
|
|
71
|
+
const reasons = [];
|
|
72
|
+
let oneAtATime = false;
|
|
73
|
+
let rejected = false;
|
|
74
|
+
let rejectReason = null;
|
|
75
|
+
// Check RAM (using cached value)
|
|
76
|
+
const memResult = await getCachedMemoryInfo(queue.verbose);
|
|
77
|
+
if (memResult.success) {
|
|
78
|
+
const usedRatio = memResult.memory.usedPercentage / 100;
|
|
79
|
+
if (usedRatio >= QUEUE_CONFIG.thresholds.ram.value) {
|
|
80
|
+
const reason = formatWaitingReason('ram', memResult.memory.usedPercentage, QUEUE_CONFIG.thresholds.ram.value, { locale });
|
|
81
|
+
const strategy = QUEUE_CONFIG.thresholds.ram.strategy;
|
|
82
|
+
queue.recordThrottle(`ram_${strategy}`);
|
|
83
|
+
if (strategy === 'reject') {
|
|
84
|
+
rejected = true;
|
|
85
|
+
rejectReason = reason;
|
|
86
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
87
|
+
oneAtATime = true;
|
|
88
|
+
if (totalProcessing > 0) {
|
|
89
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
90
|
+
}
|
|
91
|
+
} else {
|
|
92
|
+
// 'enqueue' - block unconditionally
|
|
93
|
+
reasons.push(reason);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
// Check CPU using 5-minute load average (more stable than 1-minute)
|
|
98
|
+
const cpuResult = await getCachedCpuInfo(queue.verbose);
|
|
99
|
+
if (cpuResult.success) {
|
|
100
|
+
// Use loadAvg5 (5-minute average) instead of usagePercentage (1-minute based)
|
|
101
|
+
// This provides a more stable metric that isn't affected by transient spikes
|
|
102
|
+
const loadAvg5 = cpuResult.cpuLoad.loadAvg5;
|
|
103
|
+
const cpuCount = cpuResult.cpuLoad.cpuCount;
|
|
104
|
+
// Calculate usage ratio: loadAvg5 / cpuCount
|
|
105
|
+
// Load average of 1.0 per CPU = 100% utilization
|
|
106
|
+
const usageRatio = loadAvg5 / cpuCount;
|
|
107
|
+
const usagePercent = Math.min(100, Math.round(usageRatio * 100));
|
|
108
|
+
if (queue.verbose) {
|
|
109
|
+
queue.log(`CPU 5m load avg: ${loadAvg5.toFixed(2)}, cpus: ${cpuCount}, usage: ${usagePercent}%`);
|
|
110
|
+
}
|
|
111
|
+
if (usageRatio >= QUEUE_CONFIG.thresholds.cpu.value) {
|
|
112
|
+
const reason = formatWaitingReason('cpu', usagePercent, QUEUE_CONFIG.thresholds.cpu.value, { locale });
|
|
113
|
+
const strategy = QUEUE_CONFIG.thresholds.cpu.strategy;
|
|
114
|
+
queue.recordThrottle(`cpu_${strategy}`);
|
|
115
|
+
if (strategy === 'reject') {
|
|
116
|
+
rejected = true;
|
|
117
|
+
rejectReason = reason;
|
|
118
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
119
|
+
oneAtATime = true;
|
|
120
|
+
if (totalProcessing > 0) {
|
|
121
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
122
|
+
}
|
|
123
|
+
} else {
|
|
124
|
+
// 'enqueue' - block unconditionally
|
|
125
|
+
reasons.push(reason);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
// Check disk space (using cached value)
|
|
130
|
+
// Default strategy changed to 'reject' because queue is lost on restart anyway
|
|
131
|
+
// See: https://github.com/link-assistant/hive-mind/issues/1253
|
|
132
|
+
const diskResult = await getCachedDiskInfo(queue.verbose);
|
|
133
|
+
if (diskResult.success) {
|
|
134
|
+
// Calculate usage from free percentage
|
|
135
|
+
const usedPercent = 100 - diskResult.diskSpace.freePercentage;
|
|
136
|
+
const usedRatio = usedPercent / 100;
|
|
137
|
+
if (usedRatio >= QUEUE_CONFIG.thresholds.disk.value) {
|
|
138
|
+
const reason = formatWaitingReason('disk', usedPercent, QUEUE_CONFIG.thresholds.disk.value, { locale });
|
|
139
|
+
const strategy = QUEUE_CONFIG.thresholds.disk.strategy;
|
|
140
|
+
queue.recordThrottle(`disk_${strategy}`);
|
|
141
|
+
if (strategy === 'reject') {
|
|
142
|
+
rejected = true;
|
|
143
|
+
rejectReason = reason;
|
|
144
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
145
|
+
oneAtATime = true;
|
|
146
|
+
if (totalProcessing > 0) {
|
|
147
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
148
|
+
}
|
|
149
|
+
} else {
|
|
150
|
+
// 'enqueue' - block unconditionally
|
|
151
|
+
reasons.push(reason);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return { ok: reasons.length === 0 && !rejected, reasons, oneAtATime, rejected, rejectReason };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Check API limits (Claude, GitHub) using cached values
|
|
160
|
+
*
|
|
161
|
+
* Logic per issue #1133:
|
|
162
|
+
* - CLAUDE_5_HOUR_SESSION_THRESHOLD and CLAUDE_WEEKLY_THRESHOLD use one-at-a-time mode:
|
|
163
|
+
* when above threshold, allow exactly one command, block if claudeProcessing > 0
|
|
164
|
+
* - GitHub threshold blocks unconditionally when exceeded (ultimate restriction)
|
|
165
|
+
*
|
|
166
|
+
* Logic per issue #1159:
|
|
167
|
+
* - When tool is 'agent', 'gemini', or 'qwen', skip Claude-specific limits entirely since these tools use
|
|
168
|
+
* different rate limiting backends. Only system resources and GitHub limits apply.
|
|
169
|
+
* - For Claude limits, only count Claude-specific processing items, not agent/codex/gemini/qwen items.
|
|
170
|
+
* This allows non-Claude tasks to run in parallel even when Claude limits are reached.
|
|
171
|
+
*
|
|
172
|
+
* Logic per issue #1253:
|
|
173
|
+
* - All thresholds now support configurable strategies (reject, enqueue, dequeue-one-at-a-time)
|
|
174
|
+
* - Configuration via HIVE_MIND_QUEUE_CONFIG or individual env vars
|
|
175
|
+
*
|
|
176
|
+
* @param {boolean} hasRunningToolProcess - Whether matching tool processes are running (from pgrep)
|
|
177
|
+
* @param {number} toolProcessingCount - Count of matching tool items being processed in queue
|
|
178
|
+
* @param {string} tool - The tool being used ('claude', 'agent', 'codex', 'gemini', 'qwen', etc.)
|
|
179
|
+
* @returns {Promise<{ok: boolean, reasons: string[], oneAtATime: boolean, rejected: boolean, rejectReason: string|null}>}
|
|
180
|
+
*/
|
|
181
|
+
export async function checkApiLimits(queue, hasRunningToolProcess = false, toolProcessingCount = 0, tool = 'claude', options = {}) {
|
|
182
|
+
const locale = getLocale(options);
|
|
183
|
+
const reasons = [];
|
|
184
|
+
let oneAtATime = false;
|
|
185
|
+
let rejected = false;
|
|
186
|
+
let rejectReason = null;
|
|
187
|
+
// Apply Claude-specific limits only when tool is 'claude'
|
|
188
|
+
// Other tools (like 'agent', 'gemini', and 'qwen') use different rate limiting backends and are not
|
|
189
|
+
// affected by Claude API limits (5-hour session, weekly limits)
|
|
190
|
+
// See: https://github.com/link-assistant/hive-mind/issues/1159
|
|
191
|
+
const applyClaudeLimits = tool === 'claude';
|
|
192
|
+
const applyCodexLimits = tool === 'codex';
|
|
193
|
+
const totalToolProcessing = toolProcessingCount + (hasRunningToolProcess ? 1 : 0);
|
|
194
|
+
// Check Claude limits (using cached value)
|
|
195
|
+
// Only applied when tool is 'claude'
|
|
196
|
+
if (applyClaudeLimits) {
|
|
197
|
+
const claudeResult = await getCachedClaudeLimits(queue.verbose);
|
|
198
|
+
if (claudeResult.success) {
|
|
199
|
+
const sessionPercent = claudeResult.usage.currentSession.percentage;
|
|
200
|
+
const weeklyPercent = claudeResult.usage.allModels.percentage;
|
|
201
|
+
// Session limit (5-hour)
|
|
202
|
+
// Configurable strategy via HIVE_MIND_QUEUE_CONFIG or HIVE_MIND_CLAUDE_5_HOUR_SESSION_STRATEGY
|
|
203
|
+
// See: https://github.com/link-assistant/hive-mind/issues/1133, #1159, #1253
|
|
204
|
+
if (sessionPercent !== null) {
|
|
205
|
+
const sessionRatio = sessionPercent / 100;
|
|
206
|
+
if (sessionRatio >= QUEUE_CONFIG.thresholds.claude5Hour.value) {
|
|
207
|
+
const reason = formatWaitingReason('claude_5_hour_session', sessionPercent, QUEUE_CONFIG.thresholds.claude5Hour.value, { locale });
|
|
208
|
+
const strategy = QUEUE_CONFIG.thresholds.claude5Hour.strategy;
|
|
209
|
+
queue.recordThrottle(sessionRatio >= 1.0 ? 'claude_5_hour_session_100' : `claude_5_hour_session_${strategy}`);
|
|
210
|
+
if (strategy === 'reject') {
|
|
211
|
+
rejected = true;
|
|
212
|
+
rejectReason = reason;
|
|
213
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
214
|
+
oneAtATime = true;
|
|
215
|
+
if (totalToolProcessing > 0) {
|
|
216
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
217
|
+
}
|
|
218
|
+
} else {
|
|
219
|
+
// 'enqueue' - block unconditionally
|
|
220
|
+
reasons.push(reason);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
// Weekly limit
|
|
225
|
+
// Configurable strategy via HIVE_MIND_QUEUE_CONFIG or HIVE_MIND_CLAUDE_WEEKLY_STRATEGY
|
|
226
|
+
// See: https://github.com/link-assistant/hive-mind/issues/1133, #1159, #1253
|
|
227
|
+
if (weeklyPercent !== null) {
|
|
228
|
+
const weeklyRatio = weeklyPercent / 100;
|
|
229
|
+
if (weeklyRatio >= QUEUE_CONFIG.thresholds.claudeWeekly.value) {
|
|
230
|
+
const reason = formatWaitingReason('claude_weekly', weeklyPercent, QUEUE_CONFIG.thresholds.claudeWeekly.value, { locale });
|
|
231
|
+
const strategy = QUEUE_CONFIG.thresholds.claudeWeekly.strategy;
|
|
232
|
+
queue.recordThrottle(weeklyRatio >= 1.0 ? 'claude_weekly_100' : `claude_weekly_${strategy}`);
|
|
233
|
+
if (strategy === 'reject') {
|
|
234
|
+
rejected = true;
|
|
235
|
+
rejectReason = reason;
|
|
236
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
237
|
+
oneAtATime = true;
|
|
238
|
+
if (totalToolProcessing > 0) {
|
|
239
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
240
|
+
}
|
|
241
|
+
} else {
|
|
242
|
+
// 'enqueue' - block unconditionally
|
|
243
|
+
reasons.push(reason);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
} else if (applyCodexLimits) {
|
|
249
|
+
const codexResult = await getCachedCodexLimits(queue.verbose);
|
|
250
|
+
if (codexResult.success) {
|
|
251
|
+
const sessionPercent = codexResult.usage.currentSession.percentage;
|
|
252
|
+
const weeklyPercent = codexResult.usage.allModels.percentage;
|
|
253
|
+
if (sessionPercent !== null) {
|
|
254
|
+
const sessionRatio = sessionPercent / 100;
|
|
255
|
+
if (sessionRatio >= QUEUE_CONFIG.thresholds.codex5Hour.value) {
|
|
256
|
+
const reason = formatWaitingReason('codex_5_hour_session', sessionPercent, QUEUE_CONFIG.thresholds.codex5Hour.value, { locale });
|
|
257
|
+
const strategy = QUEUE_CONFIG.thresholds.codex5Hour.strategy;
|
|
258
|
+
queue.recordThrottle(sessionRatio >= 1.0 ? 'codex_5_hour_session_100' : `codex_5_hour_session_${strategy}`);
|
|
259
|
+
if (strategy === 'reject') {
|
|
260
|
+
rejected = true;
|
|
261
|
+
rejectReason = reason;
|
|
262
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
263
|
+
oneAtATime = true;
|
|
264
|
+
if (totalToolProcessing > 0) {
|
|
265
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
266
|
+
}
|
|
267
|
+
} else {
|
|
268
|
+
reasons.push(reason);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (weeklyPercent !== null) {
|
|
273
|
+
const weeklyRatio = weeklyPercent / 100;
|
|
274
|
+
if (weeklyRatio >= QUEUE_CONFIG.thresholds.codexWeekly.value) {
|
|
275
|
+
const reason = formatWaitingReason('codex_weekly', weeklyPercent, QUEUE_CONFIG.thresholds.codexWeekly.value, { locale });
|
|
276
|
+
const strategy = QUEUE_CONFIG.thresholds.codexWeekly.strategy;
|
|
277
|
+
queue.recordThrottle(weeklyRatio >= 1.0 ? 'codex_weekly_100' : `codex_weekly_${strategy}`);
|
|
278
|
+
if (strategy === 'reject') {
|
|
279
|
+
rejected = true;
|
|
280
|
+
rejectReason = reason;
|
|
281
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
282
|
+
oneAtATime = true;
|
|
283
|
+
if (totalToolProcessing > 0) {
|
|
284
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
285
|
+
}
|
|
286
|
+
} else {
|
|
287
|
+
reasons.push(reason);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
} else if (queue.verbose) {
|
|
293
|
+
queue.log(`Claude limits not applied for --tool ${tool}`);
|
|
294
|
+
}
|
|
295
|
+
// Check GitHub limits when the active tool already has a running process.
|
|
296
|
+
// This keeps the queue behavior aligned with the existing one-at-a-time throttling model.
|
|
297
|
+
// Configurable strategy via HIVE_MIND_QUEUE_CONFIG or HIVE_MIND_GITHUB_API_STRATEGY
|
|
298
|
+
if (hasRunningToolProcess) {
|
|
299
|
+
const githubResult = await getCachedGitHubLimits(queue.verbose);
|
|
300
|
+
if (githubResult.success) {
|
|
301
|
+
const usedPercent = githubResult.githubRateLimit.usedPercentage;
|
|
302
|
+
const usedRatio = usedPercent / 100;
|
|
303
|
+
if (usedRatio >= QUEUE_CONFIG.thresholds.githubApi.value) {
|
|
304
|
+
const reason = formatWaitingReason('github', usedPercent, QUEUE_CONFIG.thresholds.githubApi.value, { locale });
|
|
305
|
+
const strategy = QUEUE_CONFIG.thresholds.githubApi.strategy;
|
|
306
|
+
queue.recordThrottle(usedRatio >= 1.0 ? 'github_100' : `github_${strategy}`);
|
|
307
|
+
if (strategy === 'reject') {
|
|
308
|
+
rejected = true;
|
|
309
|
+
rejectReason = reason;
|
|
310
|
+
} else if (strategy === 'dequeue-one-at-a-time') {
|
|
311
|
+
oneAtATime = true;
|
|
312
|
+
if (totalToolProcessing > 0) {
|
|
313
|
+
reasons.push(appendWaitingForCurrentCommand(reason, locale));
|
|
314
|
+
}
|
|
315
|
+
} else {
|
|
316
|
+
// 'enqueue' - block unconditionally
|
|
317
|
+
reasons.push(reason);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
return { ok: reasons.length === 0 && !rejected, reasons, oneAtATime, rejected, rejectReason };
|
|
323
|
+
}
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Single source of truth for "is this failure worth retrying?" classification
|
|
5
|
+
* of git and GitHub (`gh`) operations.
|
|
6
|
+
*
|
|
7
|
+
* Issue #2168: `gh pr create` aborted a whole solve run with
|
|
8
|
+
*
|
|
9
|
+
* GraphQL: Something went wrong while executing your query on
|
|
10
|
+
* 2026-08-21T19:28:14Z. Please include `811E:19A5B0:3A5AA9:37C97F:6A88A6CC`
|
|
11
|
+
* when reporting this issue.
|
|
12
|
+
*
|
|
13
|
+
* That is GitHub's generic *server-side* GraphQL failure (the GraphQL analogue
|
|
14
|
+
* of an HTTP 500 — GitHub's own docs tell you to retry it and, if it persists,
|
|
15
|
+
* report the reference id). The call site already went through
|
|
16
|
+
* `execGhWithRetry` (added for issue #1756), but the retry wrapper only
|
|
17
|
+
* recognised TCP/TLS faults and `HTTP 502/503/504`, so this error fell through
|
|
18
|
+
* the `isTransientNetworkError` check and was rethrown on the first attempt.
|
|
19
|
+
*
|
|
20
|
+
* Before this module the pattern list lived in two places — `src/lib.mjs`
|
|
21
|
+
* (`isTransientNetworkError`, superset with git-transport patterns) and
|
|
22
|
+
* `src/github-rate-limit.lib.mjs` (a deliberate copy, to avoid a circular
|
|
23
|
+
* import). Two lists meant a pattern added to one silently did not apply to
|
|
24
|
+
* the other. This module is a leaf (it imports nothing from the project) so
|
|
25
|
+
* both files can depend on it without a cycle.
|
|
26
|
+
*
|
|
27
|
+
* @see docs/case-studies/issue-2168/README.md
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Transport-level faults: the request never reached GitHub, or the connection
|
|
32
|
+
* died mid-flight. Safe to retry for both git and gh.
|
|
33
|
+
*/
|
|
34
|
+
export const NETWORK_TRANSIENT_PATTERNS = Object.freeze(['i/o timeout', 'dial tcp', 'connection refused', 'connection reset', 'econnreset', 'econnaborted', 'epipe', 'etimedout', 'enotfound', 'eai_again', 'ehostunreach', 'enetunreach', 'network is unreachable', 'temporary failure', 'tls handshake timeout', 'ssl_error', 'socket hang up', 'unexpected eof', 'client.timeout exceeded', 'context deadline exceeded', 'request timed out', 'timeout awaiting response headers']);
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* GitHub-side faults: the request reached GitHub and GitHub failed to serve
|
|
38
|
+
* it. These are 5xx-class conditions — retrying is the documented remedy.
|
|
39
|
+
*
|
|
40
|
+
* Issue #2168 added the GraphQL variants. GitHub's GraphQL API answers with
|
|
41
|
+
* HTTP 200 and an `errors[]` payload for internal failures, so no HTTP status
|
|
42
|
+
* pattern can catch them; the message text is the only signal `gh` surfaces.
|
|
43
|
+
*/
|
|
44
|
+
export const GITHUB_SERVER_TRANSIENT_PATTERNS = Object.freeze([
|
|
45
|
+
'http 500',
|
|
46
|
+
'http 502',
|
|
47
|
+
'http 503',
|
|
48
|
+
'http 504',
|
|
49
|
+
'bad gateway',
|
|
50
|
+
'service unavailable',
|
|
51
|
+
'gateway timeout',
|
|
52
|
+
'internal server error',
|
|
53
|
+
// GraphQL internal errors (HTTP 200 + errors[]), issue #2168.
|
|
54
|
+
'something went wrong while executing your query',
|
|
55
|
+
'this may be the result of a timeout',
|
|
56
|
+
'or it could be a github bug',
|
|
57
|
+
'graphql: server error',
|
|
58
|
+
'graphql: timedout',
|
|
59
|
+
'graphql: internal error',
|
|
60
|
+
// REST/GraphQL "try again later" phrasings.
|
|
61
|
+
'please try again later',
|
|
62
|
+
'try again in a few',
|
|
63
|
+
'this diff is temporarily unavailable',
|
|
64
|
+
'temporarily unavailable due to heavy server load',
|
|
65
|
+
'heavy server load',
|
|
66
|
+
'not_available',
|
|
67
|
+
]);
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* git transport faults specific to the pack protocol. `git push`/`git fetch`
|
|
71
|
+
* report a broken transfer through these strings rather than an HTTP status.
|
|
72
|
+
*
|
|
73
|
+
* Issue #1957 introduced them for clone recovery; issue #2168 makes them part
|
|
74
|
+
* of the shared vocabulary so `git push` retries recognise them too.
|
|
75
|
+
*/
|
|
76
|
+
export const GIT_TRANSIENT_PATTERNS = Object.freeze(['unexpected disconnect', 'sideband', 'early eof', 'the remote end hung up', 'remote end hung up unexpectedly', 'rpc failed', 'fetch-pack', 'index-pack failed', 'transfer closed', 'unable to access', 'could not read from remote repository', 'failed to connect to github.com', 'operation timed out after', 'gnutls_handshake() failed', 'the requested url returned error: 5']);
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Union used by the general-purpose `isTransientNetworkError` helpers. Kept as
|
|
80
|
+
* a single flat list so a caller cannot accidentally miss a category.
|
|
81
|
+
*/
|
|
82
|
+
export const ALL_TRANSIENT_PATTERNS = Object.freeze([...NETWORK_TRANSIENT_PATTERNS, ...GITHUB_SERVER_TRANSIENT_PATTERNS, ...GIT_TRANSIENT_PATTERNS]);
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Pull every plausible string out of an error-ish value so pattern matches
|
|
86
|
+
* survive whatever shape the caller produced: `Error`, a plain string, a
|
|
87
|
+
* `child_process.exec` rejection (`stdout`/`stderr`), a command-stream result
|
|
88
|
+
* object (`code`/`stdout`/`stderr`), or a wrapper carrying `cause`.
|
|
89
|
+
*
|
|
90
|
+
* @param {unknown} error
|
|
91
|
+
* @param {number} [depth] - internal recursion guard for `cause` chains.
|
|
92
|
+
* @returns {string}
|
|
93
|
+
*/
|
|
94
|
+
export const collectErrorText = (error, depth = 0) => {
|
|
95
|
+
if (!error || depth > 5) return '';
|
|
96
|
+
if (typeof error === 'string') return error;
|
|
97
|
+
const parts = [];
|
|
98
|
+
const push = value => {
|
|
99
|
+
if (typeof value === 'string') parts.push(value);
|
|
100
|
+
else if (value && typeof value.toString === 'function') parts.push(value.toString());
|
|
101
|
+
};
|
|
102
|
+
if (typeof error.message === 'string') parts.push(error.message);
|
|
103
|
+
push(error.stderr);
|
|
104
|
+
push(error.stdout);
|
|
105
|
+
if (error.cause) parts.push(collectErrorText(error.cause, depth + 1));
|
|
106
|
+
// Last resort: an error-ish object whose text only lives in a custom
|
|
107
|
+
// `toString`. Raw byte payloads are deliberately excluded — issue #1829 pins
|
|
108
|
+
// that a bare Buffer carries no classifiable text (its bytes may be an
|
|
109
|
+
// arbitrary command payload rather than a failure description).
|
|
110
|
+
if (parts.length === 0 && !ArrayBuffer.isView(error) && typeof error.toString === 'function') push(error);
|
|
111
|
+
return parts.join('\n');
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const matchPattern = (error, patterns) => {
|
|
115
|
+
const text = collectErrorText(error).toLowerCase();
|
|
116
|
+
if (!text) return null;
|
|
117
|
+
return patterns.find(pattern => text.includes(pattern)) || null;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* True when `error` is a transient transport/server fault that a retry can fix.
|
|
122
|
+
* Superset covering network, GitHub 5xx/GraphQL-internal, and git-pack faults.
|
|
123
|
+
*
|
|
124
|
+
* @param {unknown} error
|
|
125
|
+
* @returns {boolean}
|
|
126
|
+
*/
|
|
127
|
+
export const isTransientNetworkError = error => matchPattern(error, ALL_TRANSIENT_PATTERNS) !== null;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* True when `error` is a GitHub server-side fault (5xx or GraphQL internal).
|
|
131
|
+
* Narrower than `isTransientNetworkError` — used for logging/classification.
|
|
132
|
+
*
|
|
133
|
+
* @param {unknown} error
|
|
134
|
+
* @returns {boolean}
|
|
135
|
+
*/
|
|
136
|
+
export const isGitHubServerError = error => matchPattern(error, GITHUB_SERVER_TRANSIENT_PATTERNS) !== null;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Return the first transient pattern that matched, or `null` when the error is
|
|
140
|
+
* not classified as transient. Exposed so retry wrappers can log *why* they
|
|
141
|
+
* retried (or, in verbose mode, why they refused to).
|
|
142
|
+
*
|
|
143
|
+
* @param {unknown} error
|
|
144
|
+
* @returns {string|null}
|
|
145
|
+
*/
|
|
146
|
+
export const matchTransientPattern = error => matchPattern(error, ALL_TRANSIENT_PATTERNS);
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Extract GitHub's support reference id from an error.
|
|
150
|
+
*
|
|
151
|
+
* Two shapes are recognised:
|
|
152
|
+
* - the `X-GitHub-Request-Id: AAAA:BBBB:...` response header, and
|
|
153
|
+
* - the GraphQL prose form: ``Please include `811E:19A5B0:...` when
|
|
154
|
+
* reporting this issue.``
|
|
155
|
+
*
|
|
156
|
+
* Issue #2168: without this id a GitHub support report (or an upstream bug
|
|
157
|
+
* report) is unactionable, and the id was previously only visible by reading
|
|
158
|
+
* the raw failure text by hand.
|
|
159
|
+
*
|
|
160
|
+
* @param {unknown} error
|
|
161
|
+
* @returns {string|null}
|
|
162
|
+
*/
|
|
163
|
+
export const parseGitHubRequestId = error => {
|
|
164
|
+
const text = collectErrorText(error);
|
|
165
|
+
if (!text) return null;
|
|
166
|
+
const headerMatch = text.match(/x-github-request-id:\s*([0-9A-Fa-f]{4,}(?::[0-9A-Fa-f]{4,})+)/i);
|
|
167
|
+
if (headerMatch) return headerMatch[1];
|
|
168
|
+
const proseMatch = text.match(/please include\s+`?([0-9A-Fa-f]{4,}(?::[0-9A-Fa-f]{4,})+)`?/i);
|
|
169
|
+
if (proseMatch) return proseMatch[1];
|
|
170
|
+
return null;
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Full classification of a failure, for retry decisions *and* for diagnostics.
|
|
175
|
+
*
|
|
176
|
+
* @param {unknown} error
|
|
177
|
+
* @returns {{transient: boolean, category: 'network'|'github-server'|'git-transport'|null, matchedPattern: string|null, requestId: string|null, text: string}}
|
|
178
|
+
*/
|
|
179
|
+
export const describeTransientError = error => {
|
|
180
|
+
const text = collectErrorText(error);
|
|
181
|
+
const lowered = text.toLowerCase();
|
|
182
|
+
const find = patterns => patterns.find(pattern => lowered.includes(pattern)) || null;
|
|
183
|
+
|
|
184
|
+
const networkPattern = find(NETWORK_TRANSIENT_PATTERNS);
|
|
185
|
+
const serverPattern = find(GITHUB_SERVER_TRANSIENT_PATTERNS);
|
|
186
|
+
const gitPattern = find(GIT_TRANSIENT_PATTERNS);
|
|
187
|
+
|
|
188
|
+
let category = null;
|
|
189
|
+
let matchedPattern = null;
|
|
190
|
+
if (networkPattern) {
|
|
191
|
+
category = 'network';
|
|
192
|
+
matchedPattern = networkPattern;
|
|
193
|
+
} else if (serverPattern) {
|
|
194
|
+
category = 'github-server';
|
|
195
|
+
matchedPattern = serverPattern;
|
|
196
|
+
} else if (gitPattern) {
|
|
197
|
+
category = 'git-transport';
|
|
198
|
+
matchedPattern = gitPattern;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
transient: category !== null,
|
|
203
|
+
category,
|
|
204
|
+
matchedPattern,
|
|
205
|
+
requestId: parseGitHubRequestId(error),
|
|
206
|
+
text,
|
|
207
|
+
};
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* One-line, log-friendly summary of a classification. Used by the retry
|
|
212
|
+
* wrappers so every retry decision (and every give-up) is traceable in the
|
|
213
|
+
* solve log without turning on verbose mode.
|
|
214
|
+
*
|
|
215
|
+
* @param {ReturnType<typeof describeTransientError>} description
|
|
216
|
+
* @returns {string}
|
|
217
|
+
*/
|
|
218
|
+
export const formatTransientDiagnostics = description => {
|
|
219
|
+
if (!description) return '';
|
|
220
|
+
const bits = [];
|
|
221
|
+
bits.push(description.transient ? `transient=yes category=${description.category} pattern="${description.matchedPattern}"` : 'transient=no');
|
|
222
|
+
if (description.requestId) bits.push(`github-request-id=${description.requestId}`);
|
|
223
|
+
return bits.join(' ');
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
export default {
|
|
227
|
+
NETWORK_TRANSIENT_PATTERNS,
|
|
228
|
+
GITHUB_SERVER_TRANSIENT_PATTERNS,
|
|
229
|
+
GIT_TRANSIENT_PATTERNS,
|
|
230
|
+
ALL_TRANSIENT_PATTERNS,
|
|
231
|
+
collectErrorText,
|
|
232
|
+
isTransientNetworkError,
|
|
233
|
+
isGitHubServerError,
|
|
234
|
+
matchTransientPattern,
|
|
235
|
+
parseGitHubRequestId,
|
|
236
|
+
describeTransientError,
|
|
237
|
+
formatTransientDiagnostics,
|
|
238
|
+
};
|