@link-assistant/hive-mind 2.13.4 → 2.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/package.json +1 -1
- package/src/agent.lib.mjs +5 -1
- package/src/claude.lib.mjs +5 -158
- package/src/claude.session-tokens.lib.mjs +180 -0
- package/src/codex.diagnostics.lib.mjs +135 -0
- package/src/codex.lib.mjs +8 -121
- package/src/config.lib.mjs +9 -0
- package/src/gemini.lib.mjs +5 -1
- package/src/git-retry.lib.mjs +97 -0
- package/src/github-pr-idempotency.lib.mjs +83 -0
- package/src/github-rate-limit.lib.mjs +44 -41
- package/src/hive.mjs +8 -150
- package/src/hive.repository-fallback.lib.mjs +125 -0
- package/src/hive.startup-checks.lib.mjs +57 -0
- package/src/isolation-runner.lib.mjs +4 -277
- package/src/isolation-runner.parsers.lib.mjs +292 -0
- package/src/lib.mjs +79 -18
- package/src/opencode.lib.mjs +5 -1
- package/src/qwen.lib.mjs +5 -1
- package/src/session-monitor.lib.mjs +12 -272
- package/src/session-monitor.queries.lib.mjs +304 -0
- package/src/solve.auto-pr-push-sync.lib.mjs +176 -0
- package/src/solve.auto-pr.lib.mjs +40 -154
- package/src/solve.mjs +8 -158
- package/src/solve.mode.lib.mjs +191 -0
- package/src/telegram-solve-queue.lib.mjs +19 -272
- package/src/telegram-solve-queue.throttling.lib.mjs +323 -0
- package/src/transient-errors.lib.mjs +238 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parsers for start-command (`$`) output.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from src/isolation-runner.lib.mjs (issue #2175) so that file stays
|
|
5
|
+
* under the 1350-line early-warning threshold that protects concurrent merges
|
|
6
|
+
* (#1593). Behaviour is unchanged, and isolation-runner.lib.mjs re-exports
|
|
7
|
+
* every symbol so existing importers are unaffected.
|
|
8
|
+
*
|
|
9
|
+
* These are pure functions over `$ --status` / `$ --list` output and execution
|
|
10
|
+
* log tails: no process spawning, and no filesystem access beyond the
|
|
11
|
+
* injectable `fsImpl` used to read a log footer.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import fs from 'fs';
|
|
15
|
+
|
|
16
|
+
// Sentinel start-command's detached docker logger records when it cannot capture the container's real exit code. A terminal `$ --status` carrying this value is ambiguous — the container may still be running — so we cross-check it against a live `docker inspect` before concluding the session finished. See #1939. The upstream emission of this premature sentinel was fixed in start-command 0.29.1 (link-foundation/start#136), which the Hive Mind images now pin; this cross-check is retained as defense-in-depth so an older `$` on an operator's PATH cannot resurrect the bug.
|
|
17
|
+
const DOCKER_UNKNOWN_EXIT_CODE = -1;
|
|
18
|
+
function normalizeProcessIds(value) {
|
|
19
|
+
if (!value || typeof value !== 'object') return {};
|
|
20
|
+
const out = {};
|
|
21
|
+
for (const [key, raw] of Object.entries(value)) {
|
|
22
|
+
const number = Number(raw);
|
|
23
|
+
if (Number.isInteger(number) && number > 0) out[key] = number;
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
28
|
+
/**
|
|
29
|
+
* Extract start-command's own execution UUID from a launch banner.
|
|
30
|
+
*
|
|
31
|
+
* Issue #2154: an isolated task has two UUIDs. Hive Mind generates the session
|
|
32
|
+
* name and passes it as `--session` (it also becomes the container name);
|
|
33
|
+
* start-command mints a separate execution UUID and prints it as the `session`
|
|
34
|
+
* field of its launch banner:
|
|
35
|
+
*
|
|
36
|
+
* ```
|
|
37
|
+
* │ session edc7b051-e12f-4f7b-b677-c885f3208407
|
|
38
|
+
* │ container 0a3627ef-f1f1-4801-a073-3678b9453db7
|
|
39
|
+
* ```
|
|
40
|
+
*
|
|
41
|
+
* `$ --list` shows the execution UUID, while Telegram and the logs showed the
|
|
42
|
+
* session UUID, so the two views could not be joined — which is why three
|
|
43
|
+
* refused tasks and two healthy ones looked equally unaccounted for. Returning
|
|
44
|
+
* it lets the caller record both.
|
|
45
|
+
*
|
|
46
|
+
* Only a well-formed UUID is returned; a banner we do not recognise yields
|
|
47
|
+
* null rather than a guess, because a wrong correlation is worse than none.
|
|
48
|
+
*
|
|
49
|
+
* @param {string} output - Raw stdout from the detached `$` launch
|
|
50
|
+
* @returns {string|null}
|
|
51
|
+
*/
|
|
52
|
+
export function parseStartCommandExecutionUuid(output) {
|
|
53
|
+
const raw = (output || '').trim();
|
|
54
|
+
if (!raw) return null;
|
|
55
|
+
try {
|
|
56
|
+
const parsed = JSON.parse(raw);
|
|
57
|
+
const data = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
58
|
+
const uuid = data?.uuid || data?.session || null;
|
|
59
|
+
if (typeof uuid === 'string' && UUID_PATTERN.test(uuid.trim())) return uuid.trim();
|
|
60
|
+
} catch {
|
|
61
|
+
// Human-readable banner — fall through.
|
|
62
|
+
}
|
|
63
|
+
// The banner is box-drawn (`│ session <uuid>`); tolerate the prefix, an
|
|
64
|
+
// ASCII `|`, or no prefix at all.
|
|
65
|
+
const match = raw.match(/^[\s│|]*session\s+([^\s]+)\s*$/im);
|
|
66
|
+
const candidate = match?.[1]?.trim();
|
|
67
|
+
return candidate && UUID_PATTERN.test(candidate) ? candidate : null;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Parse output from `$ --status <session>`.
|
|
71
|
+
*
|
|
72
|
+
* start-command versions used in the wild may return JSON when
|
|
73
|
+
* `--output-format json` is supported, or human-readable key/value text.
|
|
74
|
+
* Keep the parser tolerant so completion monitoring survives either format.
|
|
75
|
+
*
|
|
76
|
+
* @param {string} output - Raw stdout from `$ --status`
|
|
77
|
+
* @returns {{exists: boolean, uuid: string|null, status: string|null, exitCode: number|null, startTime: string|null, endTime: string|null, currentTime: string|null, logPath: string|null, command: string|null, isolation: string|null, workingDirectory: string|null, sessionName: string|null, processIds: Object, raw: string}}
|
|
78
|
+
*/
|
|
79
|
+
export function parseSessionStatusOutput(output) {
|
|
80
|
+
const raw = (output || '').trim();
|
|
81
|
+
if (!raw) {
|
|
82
|
+
return { exists: false, uuid: null, status: null, exitCode: null, startTime: null, endTime: null, currentTime: null, logPath: null, command: null, isolation: null, workingDirectory: null, sessionName: null, processIds: {}, oomKilled: null, raw: '' };
|
|
83
|
+
}
|
|
84
|
+
const normalizeBooleanField = value => {
|
|
85
|
+
if (typeof value === 'boolean') return value;
|
|
86
|
+
if (value === null || value === undefined) return null;
|
|
87
|
+
const normalized = String(value).trim().toLowerCase();
|
|
88
|
+
if (['true', '1', 'yes'].includes(normalized)) return true;
|
|
89
|
+
if (['false', '0', 'no'].includes(normalized)) return false;
|
|
90
|
+
return null;
|
|
91
|
+
};
|
|
92
|
+
try {
|
|
93
|
+
const parsed = JSON.parse(raw);
|
|
94
|
+
const data = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
95
|
+
// start-command (link-foundation/start) reports the isolation backend at `options.isolated` in both JSON and links-notation output. Older hypothetical layouts used `options.isolation` or a top-level `isolation` field — keep accepting all three so we are tolerant of future renames. See https://github.com/link-assistant/hive-mind/issues/1700.
|
|
96
|
+
const isolationCandidate = (typeof data?.isolation === 'string' && data.isolation) || (typeof data?.options?.isolated === 'string' && data.options.isolated) || (typeof data?.options?.isolation === 'string' && data.options.isolation) || null;
|
|
97
|
+
const topPid = Number(data?.pid);
|
|
98
|
+
const processIds = normalizeProcessIds(data?.processIds);
|
|
99
|
+
if (Number.isInteger(topPid) && topPid > 0 && processIds.pid == null) processIds.pid = topPid;
|
|
100
|
+
return {
|
|
101
|
+
exists: true,
|
|
102
|
+
uuid: data?.uuid || null,
|
|
103
|
+
status: typeof data?.status === 'string' ? data.status.toLowerCase() : null,
|
|
104
|
+
exitCode: data?.exitCode !== undefined && data?.exitCode !== null ? Number(data.exitCode) : null,
|
|
105
|
+
startTime: data?.startTime || null,
|
|
106
|
+
endTime: data?.endTime || null,
|
|
107
|
+
currentTime: data?.currentTime || null,
|
|
108
|
+
logPath: data?.logPath || null,
|
|
109
|
+
command: data?.command || null,
|
|
110
|
+
isolation: isolationCandidate ? isolationCandidate.toLowerCase() : null,
|
|
111
|
+
workingDirectory: data?.workingDirectory || null,
|
|
112
|
+
sessionName: data?.sessionName || data?.options?.sessionName || null,
|
|
113
|
+
processIds,
|
|
114
|
+
oomKilled: normalizeBooleanField(data?.oomKilled ?? data?.OOMKilled ?? data?.options?.oomKilled ?? data?.state?.oomKilled ?? data?.State?.OOMKilled),
|
|
115
|
+
raw,
|
|
116
|
+
};
|
|
117
|
+
} catch {
|
|
118
|
+
// Fall through to text parsing.
|
|
119
|
+
}
|
|
120
|
+
const firstLine =
|
|
121
|
+
raw
|
|
122
|
+
.split('\n')
|
|
123
|
+
.find(line => line.trim() && !line.includes(' '))
|
|
124
|
+
?.trim() || null;
|
|
125
|
+
const readField = name => {
|
|
126
|
+
const match = raw.match(new RegExp(`^\\s*${name}\\s+"?([^"\\n]+)"?\\s*$`, 'mi'));
|
|
127
|
+
return match ? match[1].trim() : null;
|
|
128
|
+
};
|
|
129
|
+
const readBooleanField = name => normalizeBooleanField(readField(name));
|
|
130
|
+
const status = readField('status')?.toLowerCase() || null;
|
|
131
|
+
const exitCodeText = readField('exitCode');
|
|
132
|
+
// `start-command` links-notation output nests the isolation backend under `options` as `isolated <backend>` (not `isolation`). The leading indent varies by depth, but `readField` is anchored with `^\s*` which already matches indented lines. Older code only looked for `isolation`, which
|
|
133
|
+
// returned null for every real session and made /log + /terminal_watch
|
|
134
|
+
// reject screen/tmux/docker sessions. See issue #1700.
|
|
135
|
+
const isolationText = readField('isolated') || readField('isolation');
|
|
136
|
+
const processIds = {};
|
|
137
|
+
for (const name of ['pid', 'wrapperPid', 'childPid', 'processPid', 'commandPid']) {
|
|
138
|
+
const value = readField(name);
|
|
139
|
+
const number = Number(value);
|
|
140
|
+
if (Number.isInteger(number) && number > 0) processIds[name] = number;
|
|
141
|
+
}
|
|
142
|
+
return {
|
|
143
|
+
exists: Boolean(status || firstLine),
|
|
144
|
+
uuid: readField('uuid') || firstLine,
|
|
145
|
+
status,
|
|
146
|
+
exitCode: exitCodeText !== null ? Number(exitCodeText) : null,
|
|
147
|
+
startTime: readField('startTime'),
|
|
148
|
+
endTime: readField('endTime'),
|
|
149
|
+
currentTime: readField('currentTime'),
|
|
150
|
+
logPath: readField('logPath'),
|
|
151
|
+
command: readField('command'),
|
|
152
|
+
isolation: isolationText?.toLowerCase() || null,
|
|
153
|
+
workingDirectory: readField('workingDirectory'),
|
|
154
|
+
sessionName: readField('sessionName'),
|
|
155
|
+
processIds,
|
|
156
|
+
oomKilled: readBooleanField('oomKilled'),
|
|
157
|
+
raw,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Decide whether a detached-docker exit code is "unknown" (not a real result).
|
|
162
|
+
*
|
|
163
|
+
* start-command's detached docker logger writes the exit-code footer only after
|
|
164
|
+
* `docker logs -f` returns, capturing the real code via `docker inspect`. When
|
|
165
|
+
* it cannot capture one it records the sentinel `-1`. A `$ --status` that
|
|
166
|
+
* reports a terminal status ("executed") while still carrying that sentinel — or
|
|
167
|
+
* no exit code at all — is therefore ambiguous: the container may actually still
|
|
168
|
+
* be running. Callers treat such a status as provisional and cross-check the
|
|
169
|
+
* live container before declaring the session finished. See issue #1939.
|
|
170
|
+
*
|
|
171
|
+
* @param {number|null|undefined} exitCode
|
|
172
|
+
* @returns {boolean} True when the exit code carries no real result.
|
|
173
|
+
*/
|
|
174
|
+
export function isUnknownDockerExitCode(exitCode) {
|
|
175
|
+
return exitCode === null || exitCode === undefined || Number(exitCode) === DOCKER_UNKNOWN_EXIT_CODE;
|
|
176
|
+
}
|
|
177
|
+
export function shouldFallbackToScreenStatus(statusResult) {
|
|
178
|
+
return !statusResult?.exists || !statusResult?.status;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Parse the footer start-command appends to every execution log when the wrapped
|
|
182
|
+
* command exits. The footer is authoritative about the terminal exit code even
|
|
183
|
+
* when `$ --status` is wrong: start-command writes it from the command's own
|
|
184
|
+
* `close`/`exited` handler, so its presence proves the command terminated.
|
|
185
|
+
*
|
|
186
|
+
* Footer shape (see start-command spawn-helpers.js):
|
|
187
|
+
*
|
|
188
|
+
* ==================================================
|
|
189
|
+
* Finished: 2026-06-14 19:10:49.822
|
|
190
|
+
* Exit Code: 137
|
|
191
|
+
*
|
|
192
|
+
* Issue #1927: start-command's `enrichDetachedStatus` can flip a completed
|
|
193
|
+
* `executed/137` record back to `executing` (nulling the exit code) when a
|
|
194
|
+
* lingering shell keeps the screen session alive — so `$ --status` reports
|
|
195
|
+
* `executing` forever and the bot never notices the kill. Reading this footer
|
|
196
|
+
* lets hive-mind detect the real terminal exit regardless of that flip.
|
|
197
|
+
*
|
|
198
|
+
* @param {string} text - Log text (typically the tail of the log file)
|
|
199
|
+
* @returns {{finished: boolean, exitCode: number|null, endTime: string|null}}
|
|
200
|
+
*/
|
|
201
|
+
export function parseSessionExitFooter(text) {
|
|
202
|
+
if (!text) return { finished: false, exitCode: null, endTime: null };
|
|
203
|
+
// Match the LAST footer block in the text (a re-run could append more than
|
|
204
|
+
// one). Anchor on the `=` separator so command output that merely prints
|
|
205
|
+
// "Exit Code: N" mid-stream is not mistaken for the footer.
|
|
206
|
+
const re = /={10,}\s*\r?\nFinished:\s*([^\r\n]+)\r?\nExit Code:\s*(-?\d+)/g;
|
|
207
|
+
let match;
|
|
208
|
+
let last = null;
|
|
209
|
+
while ((match = re.exec(text)) !== null) last = match;
|
|
210
|
+
if (!last) return { finished: false, exitCode: null, endTime: null };
|
|
211
|
+
return { finished: true, exitCode: Number(last[2]), endTime: last[1].trim() };
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Read the terminal exit code from the tail of a start-command execution log.
|
|
215
|
+
*
|
|
216
|
+
* Only the last `tailBytes` of the file are read (the footer lives at the end),
|
|
217
|
+
* so this is cheap even for multi-megabyte logs. Never throws — a missing or
|
|
218
|
+
* unreadable log yields `{ finished: false }`.
|
|
219
|
+
*
|
|
220
|
+
* @param {string} logPath
|
|
221
|
+
* @param {Object} [options]
|
|
222
|
+
* @param {Object} [options.fsImpl=fs] - Injectable fs (for tests)
|
|
223
|
+
* @param {number} [options.tailBytes=16384] - How many trailing bytes to scan
|
|
224
|
+
* @param {boolean} [options.verbose]
|
|
225
|
+
* @returns {{finished: boolean, exitCode: number|null, endTime: string|null}}
|
|
226
|
+
*/
|
|
227
|
+
export function readSessionExitFromLog(logPath, options = {}) {
|
|
228
|
+
const { fsImpl = fs, tailBytes = 16384, verbose = false } = options;
|
|
229
|
+
if (!logPath) return { finished: false, exitCode: null, endTime: null };
|
|
230
|
+
try {
|
|
231
|
+
const { size } = fsImpl.statSync(logPath);
|
|
232
|
+
if (!size) return { finished: false, exitCode: null, endTime: null };
|
|
233
|
+
const start = Math.max(0, size - tailBytes);
|
|
234
|
+
const length = size - start;
|
|
235
|
+
const buffer = Buffer.alloc(length);
|
|
236
|
+
const fd = fsImpl.openSync(logPath, 'r');
|
|
237
|
+
try {
|
|
238
|
+
fsImpl.readSync(fd, buffer, 0, length, start);
|
|
239
|
+
} finally {
|
|
240
|
+
fsImpl.closeSync(fd);
|
|
241
|
+
}
|
|
242
|
+
const result = parseSessionExitFooter(buffer.toString('utf8'));
|
|
243
|
+
if (verbose && result.finished) {
|
|
244
|
+
console.log(`[VERBOSE] isolation-runner: log footer for ${logPath} reports exit ${result.exitCode} (finished ${result.endTime})`);
|
|
245
|
+
}
|
|
246
|
+
return result;
|
|
247
|
+
} catch (error) {
|
|
248
|
+
if (verbose) {
|
|
249
|
+
console.log(`[VERBOSE] isolation-runner: could not read exit footer from ${logPath}: ${error.message}`);
|
|
250
|
+
}
|
|
251
|
+
return { finished: false, exitCode: null, endTime: null };
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Parse output from `$ --list --output-format json`.
|
|
256
|
+
*
|
|
257
|
+
* start-command may return a top-level array, or an object with an
|
|
258
|
+
* `executions`/`sessions` array. Each entry is normalized to the same shape used
|
|
259
|
+
* by {@link parseSessionStatusOutput} (uuid/status/exitCode/command/isolation/…).
|
|
260
|
+
* Tolerant of unknown layouts — anything unparseable yields an empty list.
|
|
261
|
+
*
|
|
262
|
+
* @param {string} output - Raw stdout from `$ --list`
|
|
263
|
+
* @returns {Array<{uuid: string|null, status: string|null, exitCode: number|null, startTime: string|null, endTime: string|null, command: string|null, isolation: string|null, workingDirectory: string|null, sessionName: string|null}>}
|
|
264
|
+
*/
|
|
265
|
+
export function parseSessionListOutput(output) {
|
|
266
|
+
const raw = (output || '').trim();
|
|
267
|
+
if (!raw) return [];
|
|
268
|
+
let parsed;
|
|
269
|
+
try {
|
|
270
|
+
parsed = JSON.parse(raw);
|
|
271
|
+
} catch {
|
|
272
|
+
return [];
|
|
273
|
+
}
|
|
274
|
+
const records = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.executions) ? parsed.executions : Array.isArray(parsed?.sessions) ? parsed.sessions : parsed && typeof parsed === 'object' ? [parsed] : [];
|
|
275
|
+
return records
|
|
276
|
+
.map(data => {
|
|
277
|
+
if (!data || typeof data !== 'object') return null;
|
|
278
|
+
const isolationCandidate = (typeof data.isolation === 'string' && data.isolation) || (typeof data.options?.isolated === 'string' && data.options.isolated) || (typeof data.options?.isolation === 'string' && data.options.isolation) || null;
|
|
279
|
+
return {
|
|
280
|
+
uuid: data.uuid || data.session || data.sessionId || null,
|
|
281
|
+
status: typeof data.status === 'string' ? data.status.toLowerCase() : null,
|
|
282
|
+
exitCode: data.exitCode !== undefined && data.exitCode !== null ? Number(data.exitCode) : null,
|
|
283
|
+
startTime: data.startTime || null,
|
|
284
|
+
endTime: data.endTime || null,
|
|
285
|
+
command: data.command || null,
|
|
286
|
+
isolation: isolationCandidate ? isolationCandidate.toLowerCase() : null,
|
|
287
|
+
workingDirectory: data.workingDirectory || null,
|
|
288
|
+
sessionName: data.sessionName || data.options?.sessionName || null,
|
|
289
|
+
};
|
|
290
|
+
})
|
|
291
|
+
.filter(Boolean);
|
|
292
|
+
}
|
package/src/lib.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import { ensureUseM } from './use-m-bootstrap.lib.mjs';
|
|
|
3
3
|
import { createCredentialStreamSanitizer, maskToken, sanitizeCredentialText } from './credential-sanitization-core.lib.mjs';
|
|
4
4
|
import { recordLogBytes, resetLogGrowth } from './log-growth.lib.mjs'; // issue #2135: notice a session log that is running away
|
|
5
5
|
import { isPlaceholderErrorText } from './error-text.lib.mjs'; // issue #2141: never publish "[object Object]" as a reason
|
|
6
|
+
import { describeTransientError, formatTransientDiagnostics, isTransientNetworkError as isTransientNetworkErrorShared } from './transient-errors.lib.mjs'; // issue #2168: one shared transient-fault vocabulary for git + gh
|
|
6
7
|
|
|
7
8
|
export { maskToken };
|
|
8
9
|
|
|
@@ -538,22 +539,18 @@ export const retry = async (fn, options = {}) => {
|
|
|
538
539
|
/**
|
|
539
540
|
* Check if an error is a transient network error that can be retried.
|
|
540
541
|
* Used by validateForkParent to detect network timeouts (Issue #1311).
|
|
542
|
+
*
|
|
543
|
+
* Issue #1536 added 'unexpected eof' (gh CLI dropping mid-response);
|
|
544
|
+
* issue #1957 added the git fetch-pack/sideband disconnect patterns;
|
|
545
|
+
* issue #2168 moved the whole vocabulary into
|
|
546
|
+
* `src/transient-errors.lib.mjs` so `github-rate-limit.lib.mjs` and this
|
|
547
|
+
* module can no longer drift apart. This re-export is kept so the many
|
|
548
|
+
* existing `lib.isTransientNetworkError(...)` call sites keep working.
|
|
549
|
+
*
|
|
541
550
|
* @param {Error|string} error - The error to check
|
|
542
551
|
* @returns {boolean} True if the error is transient and retryable
|
|
543
552
|
*/
|
|
544
|
-
export const isTransientNetworkError = error =>
|
|
545
|
-
const msg = (error?.message || error?.toString() || '').toLowerCase();
|
|
546
|
-
const output = (error?.stderr?.toString() || error?.stdout?.toString() || '').toLowerCase();
|
|
547
|
-
const combined = msg + ' ' + output;
|
|
548
|
-
|
|
549
|
-
// Issue #1536: added 'unexpected eof' — seen in gh CLI when connection drops mid-response
|
|
550
|
-
// Issue #1957: added git fetch-pack/sideband disconnect patterns — seen when a
|
|
551
|
-
// `gh repo clone` / `git clone` connection drops mid-transfer, leaving an incomplete
|
|
552
|
-
// (or missing) working tree even though the wrapper can exit 0.
|
|
553
|
-
const transientPatterns = ['i/o timeout', 'dial tcp', 'connection refused', 'connection reset', 'econnreset', 'etimedout', 'enotfound', 'ehostunreach', 'enetunreach', 'network is unreachable', 'temporary failure', 'http 502', 'http 503', 'http 504', 'bad gateway', 'service unavailable', 'gateway timeout', 'tls handshake timeout', 'ssl_error', 'socket hang up', 'unexpected eof', 'unexpected disconnect', 'sideband', 'early eof', 'the remote end hung up', 'rpc failed', 'fetch-pack', 'index-pack failed', 'remote end hung up unexpectedly', 'transfer closed'];
|
|
554
|
-
|
|
555
|
-
return transientPatterns.some(pattern => combined.includes(pattern));
|
|
556
|
-
};
|
|
553
|
+
export const isTransientNetworkError = error => isTransientNetworkErrorShared(error);
|
|
557
554
|
|
|
558
555
|
/**
|
|
559
556
|
* Retry a GitHub CLI / API operation with exponential backoff on transient network errors.
|
|
@@ -594,12 +591,16 @@ export const ghRetry = async (fn, options = {}) => {
|
|
|
594
591
|
await sleep(waitMs);
|
|
595
592
|
continue;
|
|
596
593
|
}
|
|
597
|
-
|
|
594
|
+
// Issue #2168: always record the classification, so a failure that was
|
|
595
|
+
// NOT retried leaves the reason (and GitHub's request id) in the log.
|
|
596
|
+
const description = describeTransientError(error);
|
|
597
|
+
if (description.transient && attempt < maxAttempts) {
|
|
598
598
|
const waitTime = delay * Math.pow(backoff, attempt - 1);
|
|
599
|
-
await log(`⚠️ ${label}:
|
|
599
|
+
await log(`⚠️ ${label}: transient error (attempt ${attempt}/${maxAttempts}), retrying in ${waitTime / 1000}s... [${formatTransientDiagnostics(description)}]`, { level: 'warn' });
|
|
600
600
|
await sleep(waitTime);
|
|
601
601
|
continue;
|
|
602
602
|
}
|
|
603
|
+
await log(` ${label}: not retrying [${formatTransientDiagnostics(description)}] attempt=${attempt}/${maxAttempts}`, { verbose: true });
|
|
603
604
|
throw error;
|
|
604
605
|
}
|
|
605
606
|
}
|
|
@@ -648,19 +649,79 @@ export const ghCmdRetry = async (cmdFn, options = {}) => {
|
|
|
648
649
|
continue;
|
|
649
650
|
}
|
|
650
651
|
|
|
651
|
-
// Check if this is a transient network error worth retrying
|
|
652
|
-
|
|
652
|
+
// Check if this is a transient network / GitHub-server error worth retrying
|
|
653
|
+
// (issue #2168 — the classification is logged either way so the next
|
|
654
|
+
// failure of this kind is diagnosable from the session log alone).
|
|
655
|
+
const description = describeTransientError(errorLike);
|
|
656
|
+
if (description.transient && attempt < maxAttempts) {
|
|
653
657
|
const waitTime = delay * Math.pow(backoff, attempt - 1);
|
|
654
|
-
await log(`⚠️ ${label}:
|
|
658
|
+
await log(`⚠️ ${label}: transient error (attempt ${attempt}/${maxAttempts}), retrying in ${waitTime / 1000}s... [${formatTransientDiagnostics(description)}]`, { level: 'warn' });
|
|
655
659
|
await sleep(waitTime);
|
|
656
660
|
continue;
|
|
657
661
|
}
|
|
658
662
|
|
|
663
|
+
await log(` ${label}: not retrying [${formatTransientDiagnostics(description)}] attempt=${attempt}/${maxAttempts} exit=${result.code}`, { verbose: true });
|
|
664
|
+
|
|
659
665
|
// Non-transient error or last attempt — return the result as-is
|
|
660
666
|
return result;
|
|
661
667
|
}
|
|
662
668
|
};
|
|
663
669
|
|
|
670
|
+
/**
|
|
671
|
+
* Execute a git command-stream `$` call with retry on transient transport
|
|
672
|
+
* errors. The git analogue of `ghCmdRetry`.
|
|
673
|
+
*
|
|
674
|
+
* Issue #2168: `gh` calls have been retry-wrapped since #1536/#1726/#1756, but
|
|
675
|
+
* the network-facing *git* operations (`git push`, `git fetch`, `git clone`)
|
|
676
|
+
* were still single-shot — a dropped pack transfer or a GitHub 5xx on the
|
|
677
|
+
* smart-HTTP endpoint aborted the run exactly like the GraphQL error did.
|
|
678
|
+
*
|
|
679
|
+
* Semantics match `ghCmdRetry` deliberately so call sites need only wrap the
|
|
680
|
+
* existing expression in a thunk: the resolved command-stream result object is
|
|
681
|
+
* returned unchanged for success, for non-transient failures, and for the
|
|
682
|
+
* final attempt. Nothing throws that would not have thrown before.
|
|
683
|
+
*
|
|
684
|
+
* Retries are only issued for errors classified as transient by
|
|
685
|
+
* `src/transient-errors.lib.mjs`, so genuinely terminal outcomes
|
|
686
|
+
* (`non-fast-forward`, `Permission ... denied`, `repository is archived`)
|
|
687
|
+
* still surface immediately and keep their existing dedicated handling.
|
|
688
|
+
*
|
|
689
|
+
* @param {Function} cmdFn - Thunk returning a command-stream result, e.g. `() => $({ cwd })\`git push origin main\``
|
|
690
|
+
* @param {Object} [options]
|
|
691
|
+
* @param {number} [options.maxAttempts] - Maximum number of attempts (default `retryLimits.maxGitRetries`)
|
|
692
|
+
* @param {number} [options.delay=2000] - Initial delay between retries in ms
|
|
693
|
+
* @param {number} [options.backoff] - Backoff multiplier (default `retryLimits.retryBackoffMultiplier`)
|
|
694
|
+
* @param {string} [options.label='git command'] - Label for log messages
|
|
695
|
+
* @returns {Promise<{stdout: *, stderr: *, code: number}>} Command result
|
|
696
|
+
*/
|
|
697
|
+
export const gitCmdRetry = async (cmdFn, options = {}) => {
|
|
698
|
+
const { retryLimits } = await import('./config.lib.mjs');
|
|
699
|
+
const { maxAttempts = retryLimits.maxGitRetries, delay = 2000, backoff = retryLimits.retryBackoffMultiplier, label = 'git command', log: logFn = log } = options;
|
|
700
|
+
|
|
701
|
+
let result;
|
|
702
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
703
|
+
result = await cmdFn();
|
|
704
|
+
|
|
705
|
+
if (result?.code === 0) return result;
|
|
706
|
+
|
|
707
|
+
// `2>&1` is the prevailing idiom at these call sites, so the diagnosis
|
|
708
|
+
// text can be on either stream.
|
|
709
|
+
const combinedOutput = `${result?.stdout?.toString() || ''}\n${result?.stderr?.toString() || ''}`;
|
|
710
|
+
const description = describeTransientError({ message: combinedOutput });
|
|
711
|
+
|
|
712
|
+
if (description.transient && attempt < maxAttempts) {
|
|
713
|
+
const waitTime = delay * Math.pow(backoff, attempt - 1);
|
|
714
|
+
await logFn(`⚠️ ${label}: transient git error (attempt ${attempt}/${maxAttempts}), retrying in ${Math.round(waitTime / 1000)}s... [${formatTransientDiagnostics(description)}]`, { level: 'warn' });
|
|
715
|
+
await sleep(waitTime);
|
|
716
|
+
continue;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
await logFn(` ${label}: not retrying [${formatTransientDiagnostics(description)}] attempt=${attempt}/${maxAttempts} exit=${result?.code}`, { verbose: true });
|
|
720
|
+
return result;
|
|
721
|
+
}
|
|
722
|
+
return result;
|
|
723
|
+
};
|
|
724
|
+
|
|
664
725
|
/**
|
|
665
726
|
* Format bytes to human readable string
|
|
666
727
|
* @param {number} bytes - Number of bytes
|
package/src/opencode.lib.mjs
CHANGED
|
@@ -8,7 +8,11 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
8
8
|
await ensureUseM();
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
-
const { $ } = await use('command-stream');
|
|
11
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
12
|
+
// Issue #2168: retry transient git network failures (push/fetch/pull) the same
|
|
13
|
+
// way `gh` calls are retried, for every command run through this module's `$`.
|
|
14
|
+
const { wrapDollarWithGitRetry } = await import('./git-retry.lib.mjs');
|
|
15
|
+
const $ = wrapDollarWithGitRetry(__rawDollar$);
|
|
12
16
|
const fs = (await use('fs')).promises;
|
|
13
17
|
const path = (await use('path')).default;
|
|
14
18
|
const os = (await use('os')).default;
|
package/src/qwen.lib.mjs
CHANGED
|
@@ -8,7 +8,11 @@ if (typeof globalThis.use === 'undefined') {
|
|
|
8
8
|
await ensureUseM();
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
-
const { $ } = await use('command-stream');
|
|
11
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
12
|
+
// Issue #2168: retry transient git network failures (push/fetch/pull) the same
|
|
13
|
+
// way `gh` calls are retried, for every command run through this module's `$`.
|
|
14
|
+
const { wrapDollarWithGitRetry } = await import('./git-retry.lib.mjs');
|
|
15
|
+
const $ = wrapDollarWithGitRetry(__rawDollar$);
|
|
12
16
|
const fs = (await use('fs')).promises;
|
|
13
17
|
const path = (await use('path')).default;
|
|
14
18
|
const os = (await use('os')).default;
|