@link-assistant/hive-mind 2.0.23 → 2.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/package.json +16 -16
- package/src/agent-commander.lib.mjs +1 -1
- package/src/auto-language.lib.mjs +1 -1
- package/src/claude-quiet-config.lib.mjs +1 -1
- package/src/claude.lib.mjs +1 -1
- package/src/cleanup.mjs +1 -1
- package/src/cleanup.os.lib.mjs +1 -1
- package/src/codex-health.lib.mjs +191 -0
- package/src/codex.lib.mjs +57 -102
- package/src/gemini.lib.mjs +47 -0
- package/src/git.lib.mjs +2 -2
- package/src/github-repository-names.lib.mjs +1 -1
- package/src/github.lib.mjs +1 -1
- package/src/handoff-skill.lib.mjs +1 -1
- package/src/hive-screens.lib.mjs +1 -1
- package/src/i18n.lib.mjs +1 -1
- package/src/instrument.mjs +1 -1
- package/src/interactive-mode.lib.mjs +3 -3
- package/src/isolation-runner.lib.mjs +100 -3
- package/src/qwen.lib.mjs +43 -0
- package/src/session-monitor.lib.mjs +57 -15
- package/src/session-store.lib.mjs +1 -1
- package/src/solve.auto-pr.lib.mjs +2 -2
- package/src/solve.disk-diagnostics.lib.mjs +50 -29
- package/src/solve.escalate.lib.mjs +0 -3
- package/src/solve.keep-working.lib.mjs +1 -1
- package/src/solve.repository.lib.mjs +1 -1
- package/src/solve.validation.lib.mjs +1 -2
- package/src/telegram-bot-launcher.lib.mjs +2 -6
- package/src/telegram-command-execution.lib.mjs +4 -0
- package/src/telegram-isolation.lib.mjs +1 -0
- package/src/telegram-tokens-command.lib.mjs +1 -1
- package/src/telegram-top-command.lib.mjs +1 -1
- package/src/tool-run-health.lib.mjs +64 -0
- package/src/use-m-bootstrap.lib.mjs +1 -1
- package/src/useless-tools.lib.mjs +1 -1
- package/src/youtrack/solve.youtrack.lib.mjs +2 -2
- package/src/youtrack/youtrack.lib.mjs +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.0.25
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 148ca23: Fix exit-0-but-incomplete runs being reported as success under docker isolation (#1990). A `solve` run whose AI tool exited 0 while its session was cut off mid-run (e.g. the container ran out of disk) is now registered as a failure instead of a false success: codex requires its paired `turn.started`/`turn.completed` lifecycle, and gemini and qwen now require their terminal `result` event (claude already gated on it). A flagged failure preserves the AI session for a context-preserving retry and returns a non-zero exit so the docker container filesystem is kept for inspection. Disk-exhaustion strings are surfaced only as diagnostics, never as an independent failure gate, to avoid the #1955 echo false positive.
|
|
8
|
+
|
|
9
|
+
This also refreshes dependencies and picks up the upstream half of the #1990 fix. The
|
|
10
|
+
`start-command` pin in `Dockerfile`/`Dockerfile.dind` is bumped `0.30.1 → 0.30.2`,
|
|
11
|
+
which delivers [link-foundation/start#144](https://github.com/link-foundation/start/issues/144):
|
|
12
|
+
detached/isolated docker runs now surface the container's `OOMKilled` status and
|
|
13
|
+
preserve an abnormally-terminated container's filesystem for inspection instead of
|
|
14
|
+
auto-removing it. npm dependencies and devDependencies are updated to their latest
|
|
15
|
+
compatible versions (notably ESLint 9 → 10, which enables the `no-useless-assignment`
|
|
16
|
+
and `preserve-caught-error` recommended rules — all newly-flagged sites were fixed).
|
|
17
|
+
`jscpd` is intentionally held at `^4.0.5` because its 5.x line changes the
|
|
18
|
+
duplication baseline (it analyzes a wider file set, reporting 12.2% vs 10.7% on the
|
|
19
|
+
same tree) and would otherwise force weakening the duplication gate; this is a tooling
|
|
20
|
+
behavior change, not new duplication.
|
|
21
|
+
|
|
22
|
+
## 2.0.24
|
|
23
|
+
|
|
24
|
+
### Patch Changes
|
|
25
|
+
|
|
26
|
+
- 4cdff62: Improve task disk usage diagnostics for repository and Docker container filesystem reporting.
|
|
27
|
+
|
|
3
28
|
## 2.0.23
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-assistant/hive-mind",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.25",
|
|
4
4
|
"description": "AI-powered issue solver and hive mind for collaborative problem solving",
|
|
5
5
|
"main": "src/hive.mjs",
|
|
6
6
|
"type": "module",
|
|
@@ -57,30 +57,30 @@
|
|
|
57
57
|
"*.md"
|
|
58
58
|
],
|
|
59
59
|
"devDependencies": {
|
|
60
|
-
"@changesets/cli": "^2.
|
|
61
|
-
"@eslint/js": "^
|
|
62
|
-
"eslint": "^
|
|
60
|
+
"@changesets/cli": "^2.31.0",
|
|
61
|
+
"@eslint/js": "^10.0.1",
|
|
62
|
+
"eslint": "^10.5.0",
|
|
63
63
|
"eslint-config-prettier": "^10.1.8",
|
|
64
|
-
"eslint-plugin-prettier": "^5.5.
|
|
64
|
+
"eslint-plugin-prettier": "^5.5.6",
|
|
65
65
|
"husky": "^9.1.7",
|
|
66
66
|
"jscpd": "^4.0.5",
|
|
67
|
-
"lint-staged": "^
|
|
68
|
-
"prettier": "^3.
|
|
67
|
+
"lint-staged": "^17.0.8",
|
|
68
|
+
"prettier": "^3.8.5",
|
|
69
69
|
"test-anywhere": "^0.9.1"
|
|
70
70
|
},
|
|
71
71
|
"dependencies": {
|
|
72
|
-
"@secretlint/core": "^
|
|
73
|
-
"@secretlint/secretlint-rule-preset-recommend": "^
|
|
74
|
-
"@sentry/node": "^10.
|
|
75
|
-
"@sentry/profiling-node": "^10.
|
|
76
|
-
"agent-commander": "^0.
|
|
77
|
-
"dayjs": "^1.11.
|
|
72
|
+
"@secretlint/core": "^13.0.2",
|
|
73
|
+
"@secretlint/secretlint-rule-preset-recommend": "^13.0.2",
|
|
74
|
+
"@sentry/node": "^10.62.0",
|
|
75
|
+
"@sentry/profiling-node": "^10.62.0",
|
|
76
|
+
"agent-commander": "^0.8.0",
|
|
77
|
+
"dayjs": "^1.11.21",
|
|
78
78
|
"decimal.js-light": "^2.5.1",
|
|
79
79
|
"lino-arguments": "^0.3.0",
|
|
80
80
|
"lino-i18n": "^0.1.1",
|
|
81
|
-
"lino-objects-codec": "^0.
|
|
82
|
-
"secretlint": "^
|
|
83
|
-
"semver": "^7.
|
|
81
|
+
"lino-objects-codec": "^0.4.0",
|
|
82
|
+
"secretlint": "^13.0.2",
|
|
83
|
+
"semver": "^7.8.5",
|
|
84
84
|
"tinyld": "^1.3.4"
|
|
85
85
|
},
|
|
86
86
|
"lint-staged": {
|
|
@@ -91,7 +91,7 @@ const getAgentCommander = async () => {
|
|
|
91
91
|
try {
|
|
92
92
|
return await import('agent-commander');
|
|
93
93
|
} catch (error) {
|
|
94
|
-
throw new Error(`agent-commander is not installed or cannot be loaded. Install it with: npm install agent-commander\nOriginal error: ${error.message}
|
|
94
|
+
throw new Error(`agent-commander is not installed or cannot be loaded. Install it with: npm install agent-commander\nOriginal error: ${error.message}`, { cause: error });
|
|
95
95
|
}
|
|
96
96
|
};
|
|
97
97
|
|
|
@@ -77,7 +77,7 @@ export function resolveClaudeLanguageSetting(locale) {
|
|
|
77
77
|
|
|
78
78
|
export const ensureClaudeQuietConfig = async ({ settingsPath, log, workLanguage = null } = {}) => {
|
|
79
79
|
const resolvedPath = settingsPath || path.join(os.homedir(), '.claude', 'settings.json');
|
|
80
|
-
let settings
|
|
80
|
+
let settings;
|
|
81
81
|
try {
|
|
82
82
|
const content = await fs.readFile(resolvedPath, 'utf-8');
|
|
83
83
|
const parsed = JSON.parse(content);
|
package/src/claude.lib.mjs
CHANGED
|
@@ -528,7 +528,7 @@ export const calculateSessionTokens = async (sessionId, tempDir, resultModelUsag
|
|
|
528
528
|
compactifications: compactifications.length > 0 ? compactifications : null,
|
|
529
529
|
};
|
|
530
530
|
} catch (readError) {
|
|
531
|
-
throw new Error(`Failed to read session file: ${readError.message}
|
|
531
|
+
throw new Error(`Failed to read session file: ${readError.message}`, { cause: readError });
|
|
532
532
|
}
|
|
533
533
|
};
|
|
534
534
|
// Extracted to claude.stderr.lib.mjs (Issue #477, #1337)
|
package/src/cleanup.mjs
CHANGED
|
@@ -372,7 +372,7 @@ async function main() {
|
|
|
372
372
|
if (!options.force) {
|
|
373
373
|
console.log(`\n⚠️ This will permanently delete ${classified.remove.length} entries (${formatBytes(totals.removeBytes)}) and remove ${dockerIsolationPlan.remove.length} Docker isolation containers.`);
|
|
374
374
|
console.log('Type "yes" to confirm, or Ctrl+C to cancel:');
|
|
375
|
-
let answer
|
|
375
|
+
let answer;
|
|
376
376
|
try {
|
|
377
377
|
answer = await readConfirmationLine({ prompt: '> ' });
|
|
378
378
|
} catch {
|
package/src/cleanup.os.lib.mjs
CHANGED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// Codex run-health analysis helpers.
|
|
2
|
+
//
|
|
3
|
+
// Extracted from codex.lib.mjs (which exceeded the 1500-line max-lines budget).
|
|
4
|
+
// These functions inspect the parsed `codex exec --json` event state and decide
|
|
5
|
+
// whether a run genuinely succeeded:
|
|
6
|
+
// - getCodexErrorEventSummary: classifies stray error events, suppressing the
|
|
7
|
+
// #1955 echoed-fixture false positives once a turn has completed.
|
|
8
|
+
// - getCodexCompletionHealth: the #1990 turn-lifecycle gate that flags exit-0
|
|
9
|
+
// runs cut off mid-turn (e.g. by docker disk exhaustion) as unhealthy.
|
|
10
|
+
//
|
|
11
|
+
// Both are re-exported from codex.lib.mjs for backward compatibility, so existing
|
|
12
|
+
// importers (and tests) can keep importing them from either module.
|
|
13
|
+
|
|
14
|
+
import { isENOSPC } from './lib.mjs';
|
|
15
|
+
|
|
16
|
+
const unwrapCodexErrorMessage = value => {
|
|
17
|
+
if (!value) return '';
|
|
18
|
+
if (typeof value !== 'string') {
|
|
19
|
+
if (typeof value?.error?.message === 'string') return unwrapCodexErrorMessage(value.error.message);
|
|
20
|
+
if (typeof value?.message === 'string') return unwrapCodexErrorMessage(value.message);
|
|
21
|
+
return String(value);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
let text = value.trim();
|
|
25
|
+
for (let i = 0; i < 3; i++) {
|
|
26
|
+
if (!text.startsWith('{') && !text.startsWith('[')) break;
|
|
27
|
+
try {
|
|
28
|
+
const parsed = JSON.parse(text);
|
|
29
|
+
if (typeof parsed?.error?.message === 'string') return unwrapCodexErrorMessage(parsed.error.message);
|
|
30
|
+
if (typeof parsed?.message === 'string') {
|
|
31
|
+
text = parsed.message.trim();
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
return JSON.stringify(parsed);
|
|
35
|
+
} catch {
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return text;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const isNonFatalCodexItemErrorMessage = message => /^in-process app-server event stream lagged; dropped \d+ events?$/i.test(message || '');
|
|
43
|
+
|
|
44
|
+
export const getCodexErrorEventSummary = codexJsonState => {
|
|
45
|
+
const events = [];
|
|
46
|
+
const ignoredEvents = [];
|
|
47
|
+
|
|
48
|
+
// Issue #1955: When the codex turn genuinely completed (a `turn.completed`
|
|
49
|
+
// event was observed) and codex never emitted a `turn.failed`, the session
|
|
50
|
+
// SUCCEEDED. Any stray top-level `error` (stream) or nested item `error` event
|
|
51
|
+
// in that case is non-fatal and must not fail the run. Two things produce such
|
|
52
|
+
// strays:
|
|
53
|
+
// 1. A transient error codex itself retried/recovered from before completing
|
|
54
|
+
// the turn (e.g. a momentary stream blip).
|
|
55
|
+
// 2. Echoed content that merely *looks* like a codex protocol event. The
|
|
56
|
+
// codex CLI prints OTEL telemetry (`codex_otel.log_only`,
|
|
57
|
+
// event.name="codex.tool_result") containing a raw `Output:` dump of each
|
|
58
|
+
// command's stdout. When a command prints a line shaped like a protocol
|
|
59
|
+
// event — e.g. a printed NDJSON fixture line
|
|
60
|
+
// `{"type":"error","message":"Network lookup skipped in fixture"}` — our
|
|
61
|
+
// line-by-line parser misreads it as a genuine codex stream error and
|
|
62
|
+
// fails an otherwise-successful run. This was the exact false positive in
|
|
63
|
+
// issue #1955 (codex finished, working tree clean, CI passed, yet the run
|
|
64
|
+
// was reported failed).
|
|
65
|
+
// `turn.failed` is the authoritative failure signal, so it is NEVER suppressed
|
|
66
|
+
// here; only non-`turn` error events are gated on turn completion.
|
|
67
|
+
const turnCompleted = (codexJsonState?.eventCounts?.['turn.completed'] || 0) > 0;
|
|
68
|
+
const turnFailed = (codexJsonState?.turnFailures?.length || 0) > 0;
|
|
69
|
+
const sessionSucceeded = turnCompleted && !turnFailed;
|
|
70
|
+
|
|
71
|
+
const addEvents = (type, items = []) => {
|
|
72
|
+
for (const item of items) {
|
|
73
|
+
const message = unwrapCodexErrorMessage(item?.message);
|
|
74
|
+
const event = { type, message: message || 'Codex emitted an error event' };
|
|
75
|
+
if (type === 'item' && isNonFatalCodexItemErrorMessage(message)) {
|
|
76
|
+
ignoredEvents.push({
|
|
77
|
+
...event,
|
|
78
|
+
reason: 'Codex app-server backpressure warning; the turn can still complete successfully',
|
|
79
|
+
});
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (type !== 'turn' && sessionSucceeded) {
|
|
83
|
+
ignoredEvents.push({
|
|
84
|
+
...event,
|
|
85
|
+
reason: 'Codex turn completed successfully with no turn.failed; stray non-turn error event is non-fatal (Issue #1955)',
|
|
86
|
+
});
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
events.push(event);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
addEvents('item', codexJsonState?.itemErrors);
|
|
94
|
+
addEvents('turn', codexJsonState?.turnFailures);
|
|
95
|
+
addEvents('stream', codexJsonState?.streamErrors);
|
|
96
|
+
|
|
97
|
+
const countByType = items => ({
|
|
98
|
+
item: items.filter(item => item.type === 'item').length,
|
|
99
|
+
turn: items.filter(item => item.type === 'turn').length,
|
|
100
|
+
stream: items.filter(item => item.type === 'stream').length,
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
hasError: events.length > 0,
|
|
105
|
+
message: events[0]?.message || null,
|
|
106
|
+
events,
|
|
107
|
+
ignoredEvents,
|
|
108
|
+
counts: countByType(events),
|
|
109
|
+
ignoredCounts: countByType(ignoredEvents),
|
|
110
|
+
observedCounts: {
|
|
111
|
+
item: codexJsonState?.itemErrors?.length || 0,
|
|
112
|
+
turn: codexJsonState?.turnFailures?.length || 0,
|
|
113
|
+
stream: codexJsonState?.streamErrors?.length || 0,
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// Issue #1990: A Codex run can exit 0 with no fatal `turn.failed`/error event yet
|
|
119
|
+
// still be fundamentally broken. Under docker isolation two long-running
|
|
120
|
+
// `solve --tool codex` tasks reported SUCCESS (Exit Code: 0) while their
|
|
121
|
+
// containers had run out of disk: cargo builds died with "No space left on
|
|
122
|
+
// device" / exit 101, no commits were produced, and — critically — the codex
|
|
123
|
+
// turn was never completed (the process was cut off mid-turn). Because the exit
|
|
124
|
+
// code was 0 and codex emitted no `turn.failed`, executeCodexCommand declared
|
|
125
|
+
// success, which under docker isolation also discarded the container filesystem
|
|
126
|
+
// we needed to inspect and retry from.
|
|
127
|
+
//
|
|
128
|
+
// The authoritative, echo-proof signal is codex's own turn lifecycle: `codex
|
|
129
|
+
// exec` emits a paired `turn.started`/`turn.completed` for every turn (a failed
|
|
130
|
+
// turn emits `turn.failed`). When the process ends with started turns that
|
|
131
|
+
// neither completed nor failed, the session is INCOMPLETE regardless of the exit
|
|
132
|
+
// code. Both captured failures had turn.started=1, turn.completed=0,
|
|
133
|
+
// turn.failed=0 (see docs/case-studies/issue-1990).
|
|
134
|
+
//
|
|
135
|
+
// Disk-exhaustion strings ("No space left on device", ENOSPC) are deliberately
|
|
136
|
+
// NOT used as an independent failure gate: codex echoes the stdout of every
|
|
137
|
+
// command it runs back into its own stream (see issue #1955), so a target repo
|
|
138
|
+
// that merely prints or works on that phrase (e.g. a `sed`/`cat` of a saved log,
|
|
139
|
+
// both observed in the captured runs at exit_code 0) would be wrongly failed.
|
|
140
|
+
// Disk pressure is surfaced only as supporting *diagnostics* explaining why a
|
|
141
|
+
// session was likely cut off, never as the sole reason to fail a completed turn.
|
|
142
|
+
export const getCodexCompletionHealth = (codexJsonState, { lastMessage = '' } = {}) => {
|
|
143
|
+
const eventCounts = codexJsonState?.eventCounts || {};
|
|
144
|
+
const turnStarted = eventCounts['turn.started'] || 0;
|
|
145
|
+
const turnCompleted = eventCounts['turn.completed'] || 0;
|
|
146
|
+
const turnFailed = codexJsonState?.turnFailures?.length || 0;
|
|
147
|
+
const commandExecutions = codexJsonState?.commandExecutions || [];
|
|
148
|
+
|
|
149
|
+
// hadActivity = codex actually began doing work, so a genuinely empty stream
|
|
150
|
+
// (no turns, no commands) is never spuriously flagged — we only fail when work
|
|
151
|
+
// started but never finished.
|
|
152
|
+
const hadActivity = turnStarted > 0 || commandExecutions.length > 0 || (eventCounts['item.completed'] || 0) > 0;
|
|
153
|
+
|
|
154
|
+
// A started turn that never completed or failed = the process was cut off
|
|
155
|
+
// mid-turn (OOM / disk-full / container teardown) even though it exited 0.
|
|
156
|
+
const incompleteSession = hadActivity && turnCompleted + turnFailed < Math.max(turnStarted, 1);
|
|
157
|
+
|
|
158
|
+
// Diagnostic-only disk-pressure hints (never an independent failure gate).
|
|
159
|
+
const diskEvidence = [];
|
|
160
|
+
const addDiskEvidence = (source, text) => {
|
|
161
|
+
if (text && isENOSPC(text)) {
|
|
162
|
+
diskEvidence.push({ source, text: String(text).replace(/\s+/g, ' ').trim().slice(0, 300) });
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
for (const exec of commandExecutions) addDiskEvidence(`command:${exec.command || exec.id || 'unknown'}`, exec.aggregatedOutput);
|
|
166
|
+
for (const streamError of codexJsonState?.streamErrors || []) addDiskEvidence('stream-error', streamError.message);
|
|
167
|
+
for (const itemError of codexJsonState?.itemErrors || []) addDiskEvidence('item-error', itemError.message);
|
|
168
|
+
for (const turnFailure of codexJsonState?.turnFailures || []) addDiskEvidence('turn-failure', turnFailure.message);
|
|
169
|
+
addDiskEvidence('last-message', lastMessage);
|
|
170
|
+
addDiskEvidence('result-summary', codexJsonState?.resultSummary);
|
|
171
|
+
const diskPressureDetected = diskEvidence.length > 0;
|
|
172
|
+
|
|
173
|
+
const reasons = [];
|
|
174
|
+
if (incompleteSession) {
|
|
175
|
+
reasons.push(`Codex session ended without completing its turn (turn.started=${turnStarted}, turn.completed=${turnCompleted}, turn.failed=${turnFailed}); the process exited 0 but was cut off mid-turn.`);
|
|
176
|
+
if (diskPressureDetected) {
|
|
177
|
+
reasons.push(`Disk-exhaustion signals were present in ${diskEvidence.length} location(s) (e.g. "No space left on device") — the likely cause of the interrupted session.`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
healthy: !incompleteSession,
|
|
183
|
+
incompleteSession,
|
|
184
|
+
diskPressureDetected,
|
|
185
|
+
diskEvidence,
|
|
186
|
+
turnStarted,
|
|
187
|
+
turnCompleted,
|
|
188
|
+
turnFailed,
|
|
189
|
+
reasons,
|
|
190
|
+
};
|
|
191
|
+
};
|
package/src/codex.lib.mjs
CHANGED
|
@@ -15,6 +15,10 @@ const os = (await use('os')).default;
|
|
|
15
15
|
|
|
16
16
|
// Import log from general lib
|
|
17
17
|
import { log } from './lib.mjs';
|
|
18
|
+
// Issues #1955 / #1990: run-health analysis lives in its own module to keep this
|
|
19
|
+
// file under the max-lines budget. Re-exported below for backward compatibility.
|
|
20
|
+
import { getCodexErrorEventSummary, getCodexCompletionHealth } from './codex-health.lib.mjs';
|
|
21
|
+
export { getCodexErrorEventSummary, getCodexCompletionHealth };
|
|
18
22
|
import { reportError } from './sentry.lib.mjs';
|
|
19
23
|
import { timeouts, retryLimits } from './config.lib.mjs';
|
|
20
24
|
import { detectUsageLimit, formatUsageLimitMessage } from './usage-limit.lib.mjs';
|
|
@@ -327,108 +331,6 @@ const upsertCodexItemError = (itemErrors, item) => {
|
|
|
327
331
|
});
|
|
328
332
|
};
|
|
329
333
|
|
|
330
|
-
const unwrapCodexErrorMessage = value => {
|
|
331
|
-
if (!value) return '';
|
|
332
|
-
if (typeof value !== 'string') {
|
|
333
|
-
if (typeof value?.error?.message === 'string') return unwrapCodexErrorMessage(value.error.message);
|
|
334
|
-
if (typeof value?.message === 'string') return unwrapCodexErrorMessage(value.message);
|
|
335
|
-
return String(value);
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
let text = value.trim();
|
|
339
|
-
for (let i = 0; i < 3; i++) {
|
|
340
|
-
if (!text.startsWith('{') && !text.startsWith('[')) break;
|
|
341
|
-
try {
|
|
342
|
-
const parsed = JSON.parse(text);
|
|
343
|
-
if (typeof parsed?.error?.message === 'string') return unwrapCodexErrorMessage(parsed.error.message);
|
|
344
|
-
if (typeof parsed?.message === 'string') {
|
|
345
|
-
text = parsed.message.trim();
|
|
346
|
-
continue;
|
|
347
|
-
}
|
|
348
|
-
return JSON.stringify(parsed);
|
|
349
|
-
} catch {
|
|
350
|
-
break;
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
return text;
|
|
354
|
-
};
|
|
355
|
-
|
|
356
|
-
const isNonFatalCodexItemErrorMessage = message => /^in-process app-server event stream lagged; dropped \d+ events?$/i.test(message || '');
|
|
357
|
-
|
|
358
|
-
export const getCodexErrorEventSummary = codexJsonState => {
|
|
359
|
-
const events = [];
|
|
360
|
-
const ignoredEvents = [];
|
|
361
|
-
|
|
362
|
-
// Issue #1955: When the codex turn genuinely completed (a `turn.completed`
|
|
363
|
-
// event was observed) and codex never emitted a `turn.failed`, the session
|
|
364
|
-
// SUCCEEDED. Any stray top-level `error` (stream) or nested item `error` event
|
|
365
|
-
// in that case is non-fatal and must not fail the run. Two things produce such
|
|
366
|
-
// strays:
|
|
367
|
-
// 1. A transient error codex itself retried/recovered from before completing
|
|
368
|
-
// the turn (e.g. a momentary stream blip).
|
|
369
|
-
// 2. Echoed content that merely *looks* like a codex protocol event. The
|
|
370
|
-
// codex CLI prints OTEL telemetry (`codex_otel.log_only`,
|
|
371
|
-
// event.name="codex.tool_result") containing a raw `Output:` dump of each
|
|
372
|
-
// command's stdout. When a command prints a line shaped like a protocol
|
|
373
|
-
// event — e.g. a printed NDJSON fixture line
|
|
374
|
-
// `{"type":"error","message":"Network lookup skipped in fixture"}` — our
|
|
375
|
-
// line-by-line parser misreads it as a genuine codex stream error and
|
|
376
|
-
// fails an otherwise-successful run. This was the exact false positive in
|
|
377
|
-
// issue #1955 (codex finished, working tree clean, CI passed, yet the run
|
|
378
|
-
// was reported failed).
|
|
379
|
-
// `turn.failed` is the authoritative failure signal, so it is NEVER suppressed
|
|
380
|
-
// here; only non-`turn` error events are gated on turn completion.
|
|
381
|
-
const turnCompleted = (codexJsonState?.eventCounts?.['turn.completed'] || 0) > 0;
|
|
382
|
-
const turnFailed = (codexJsonState?.turnFailures?.length || 0) > 0;
|
|
383
|
-
const sessionSucceeded = turnCompleted && !turnFailed;
|
|
384
|
-
|
|
385
|
-
const addEvents = (type, items = []) => {
|
|
386
|
-
for (const item of items) {
|
|
387
|
-
const message = unwrapCodexErrorMessage(item?.message);
|
|
388
|
-
const event = { type, message: message || 'Codex emitted an error event' };
|
|
389
|
-
if (type === 'item' && isNonFatalCodexItemErrorMessage(message)) {
|
|
390
|
-
ignoredEvents.push({
|
|
391
|
-
...event,
|
|
392
|
-
reason: 'Codex app-server backpressure warning; the turn can still complete successfully',
|
|
393
|
-
});
|
|
394
|
-
continue;
|
|
395
|
-
}
|
|
396
|
-
if (type !== 'turn' && sessionSucceeded) {
|
|
397
|
-
ignoredEvents.push({
|
|
398
|
-
...event,
|
|
399
|
-
reason: 'Codex turn completed successfully with no turn.failed; stray non-turn error event is non-fatal (Issue #1955)',
|
|
400
|
-
});
|
|
401
|
-
continue;
|
|
402
|
-
}
|
|
403
|
-
events.push(event);
|
|
404
|
-
}
|
|
405
|
-
};
|
|
406
|
-
|
|
407
|
-
addEvents('item', codexJsonState?.itemErrors);
|
|
408
|
-
addEvents('turn', codexJsonState?.turnFailures);
|
|
409
|
-
addEvents('stream', codexJsonState?.streamErrors);
|
|
410
|
-
|
|
411
|
-
const countByType = items => ({
|
|
412
|
-
item: items.filter(item => item.type === 'item').length,
|
|
413
|
-
turn: items.filter(item => item.type === 'turn').length,
|
|
414
|
-
stream: items.filter(item => item.type === 'stream').length,
|
|
415
|
-
});
|
|
416
|
-
|
|
417
|
-
return {
|
|
418
|
-
hasError: events.length > 0,
|
|
419
|
-
message: events[0]?.message || null,
|
|
420
|
-
events,
|
|
421
|
-
ignoredEvents,
|
|
422
|
-
counts: countByType(events),
|
|
423
|
-
ignoredCounts: countByType(ignoredEvents),
|
|
424
|
-
observedCounts: {
|
|
425
|
-
item: codexJsonState?.itemErrors?.length || 0,
|
|
426
|
-
turn: codexJsonState?.turnFailures?.length || 0,
|
|
427
|
-
stream: codexJsonState?.streamErrors?.length || 0,
|
|
428
|
-
},
|
|
429
|
-
};
|
|
430
|
-
};
|
|
431
|
-
|
|
432
334
|
export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId = null) => {
|
|
433
335
|
const nextState = {
|
|
434
336
|
sessionId: state.sessionId || null,
|
|
@@ -1339,6 +1241,59 @@ export const executeCodexCommand = async params => {
|
|
|
1339
1241
|
};
|
|
1340
1242
|
}
|
|
1341
1243
|
|
|
1244
|
+
// Issue #1990: exit code 0 and the absence of a fatal codex error event are
|
|
1245
|
+
// necessary but NOT sufficient for success. Verify the run actually
|
|
1246
|
+
// completed its turn before declaring success. A broken-but-exit-0 run (the
|
|
1247
|
+
// codex process cut off mid-turn by disk exhaustion / OOM) previously
|
|
1248
|
+
// reported SUCCESS, which under docker isolation also discarded the
|
|
1249
|
+
// container filesystem needed to inspect and retry the failure (#1990).
|
|
1250
|
+
const completionHealth = getCodexCompletionHealth(codexJsonState, { lastMessage });
|
|
1251
|
+
if (!completionHealth.healthy) {
|
|
1252
|
+
await log('\n\n❌ Codex exited 0 but the run did not complete — treating as failure', { level: 'error' });
|
|
1253
|
+
for (const reason of completionHealth.reasons) {
|
|
1254
|
+
await log(` • ${reason}`, { level: 'error' });
|
|
1255
|
+
}
|
|
1256
|
+
await log(` 📊 turn.started=${completionHealth.turnStarted}, turn.completed=${completionHealth.turnCompleted}, turn.failed=${completionHealth.turnFailed}`, { verbose: true });
|
|
1257
|
+
if (completionHealth.diskPressureDetected) {
|
|
1258
|
+
await log(' 💽 Disk-exhaustion evidence (diagnostic):', { level: 'error' });
|
|
1259
|
+
for (const evidence of completionHealth.diskEvidence.slice(0, 5)) {
|
|
1260
|
+
await log(` ↳ [${evidence.source}] ${evidence.text}`, { level: 'error' });
|
|
1261
|
+
}
|
|
1262
|
+
await log(' 💡 Free disk space before retrying. Under docker isolation the container is preserved on failure for inspection.', { level: 'error' });
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
const resourcesAfter = await getResourceSnapshot();
|
|
1266
|
+
await log('\n📈 System resources after execution:', { verbose: true });
|
|
1267
|
+
await log(` Memory: ${resourcesAfter.memory.split('\n')[1]}`, { verbose: true });
|
|
1268
|
+
await log(` Load: ${resourcesAfter.load}`, { verbose: true });
|
|
1269
|
+
|
|
1270
|
+
// Issue #1990: preserve the codex session so an outer full restart can
|
|
1271
|
+
// resume with context (mirrors the transient-error retry above and the
|
|
1272
|
+
// `--tool claude` behavior). We do NOT inline-retry within the same broken
|
|
1273
|
+
// container — the run is registered as a failure so the session and (under
|
|
1274
|
+
// docker isolation) the container filesystem are preserved for a clean
|
|
1275
|
+
// restart at the orchestration level.
|
|
1276
|
+
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
1277
|
+
|
|
1278
|
+
return {
|
|
1279
|
+
success: false,
|
|
1280
|
+
sessionId,
|
|
1281
|
+
limitReached,
|
|
1282
|
+
limitResetTime,
|
|
1283
|
+
pricingInfo,
|
|
1284
|
+
publicPricingEstimate: pricingInfo?.totalCostUSD ?? null,
|
|
1285
|
+
resultModelUsage,
|
|
1286
|
+
subAgentCalls: codexJsonState.subAgentCalls.length > 0 ? codexJsonState.subAgentCalls : null,
|
|
1287
|
+
codexJsonDetails: codexJsonState,
|
|
1288
|
+
errorInfo: getCodexErrorEventSummary(codexJsonState),
|
|
1289
|
+
completionHealth,
|
|
1290
|
+
incompleteSession: completionHealth.incompleteSession,
|
|
1291
|
+
diskPressureDetected: completionHealth.diskPressureDetected,
|
|
1292
|
+
result: completionHealth.reasons.join(' '),
|
|
1293
|
+
resultSummary: lastTextContent || null,
|
|
1294
|
+
};
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1342
1297
|
await log('\n\n✅ Codex command completed');
|
|
1343
1298
|
|
|
1344
1299
|
// Issue #1263: Log if result summary was captured
|
package/src/gemini.lib.mjs
CHANGED
|
@@ -21,6 +21,7 @@ import { defaultModels, geminiModels } from './models/index.mjs';
|
|
|
21
21
|
import { checkPlaywrightMcpPackageAvailability } from './playwright-mcp.lib.mjs';
|
|
22
22
|
import { classifyRetryableError, getRetryDelayMs, maybeSwitchToFallbackModel, waitWithCountdown } from './tool-retry.lib.mjs';
|
|
23
23
|
import { getCumulativeContextInputTokens, toTokenCount } from './context-fill.lib.mjs';
|
|
24
|
+
import { getTerminalEventCompletionHealth } from './tool-run-health.lib.mjs'; // Issue #1990
|
|
24
25
|
|
|
25
26
|
const shellQuote = value => `"${String(value).replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`;
|
|
26
27
|
|
|
@@ -581,6 +582,52 @@ export const executeGeminiCommand = async params => {
|
|
|
581
582
|
};
|
|
582
583
|
}
|
|
583
584
|
|
|
585
|
+
// Issue #1990: exit 0 and a non-empty stream are necessary but NOT
|
|
586
|
+
// sufficient. gemini-cli's stream-json ends with a terminal `result` event;
|
|
587
|
+
// a run that did work but never emitted it was cut off mid-run (e.g. the
|
|
588
|
+
// docker container ran out of disk) and must be registered as a failure so
|
|
589
|
+
// the session is preserved for a context-preserving restart and — under
|
|
590
|
+
// docker isolation — the container filesystem is kept for inspection.
|
|
591
|
+
const completionHealth = getTerminalEventCompletionHealth({
|
|
592
|
+
eventCounts: geminiJsonState.eventCounts,
|
|
593
|
+
terminalEventTypes: ['result'],
|
|
594
|
+
hadActivity: (geminiJsonState.messageCount || 0) > 0 || (geminiJsonState.toolUseCount || 0) > 0,
|
|
595
|
+
diskEvidenceTexts: [
|
|
596
|
+
{ source: 'output', text: allOutput },
|
|
597
|
+
{ source: 'result-summary', text: geminiJsonState.resultSummary },
|
|
598
|
+
],
|
|
599
|
+
});
|
|
600
|
+
if (!completionHealth.healthy) {
|
|
601
|
+
await log('\n\n❌ Gemini exited 0 but the run did not complete — treating as failure', { level: 'error' });
|
|
602
|
+
for (const reason of completionHealth.reasons) {
|
|
603
|
+
await log(` • ${reason}`, { level: 'error' });
|
|
604
|
+
}
|
|
605
|
+
if (completionHealth.diskPressureDetected) {
|
|
606
|
+
await log(' 💽 Disk-exhaustion evidence (diagnostic):', { level: 'error' });
|
|
607
|
+
for (const evidence of completionHealth.diskEvidence.slice(0, 5)) {
|
|
608
|
+
await log(` ↳ [${evidence.source}] ${evidence.text}`, { level: 'error' });
|
|
609
|
+
}
|
|
610
|
+
await log(' 💡 Free disk space before retrying. Under docker isolation the container is preserved on failure for inspection.', { level: 'error' });
|
|
611
|
+
}
|
|
612
|
+
if (sessionId && !argv.resume) argv.resume = sessionId;
|
|
613
|
+
return {
|
|
614
|
+
success: false,
|
|
615
|
+
sessionId,
|
|
616
|
+
limitReached,
|
|
617
|
+
limitResetTime,
|
|
618
|
+
messageCount: geminiJsonState.messageCount || 0,
|
|
619
|
+
toolUseCount: geminiJsonState.toolUseCount || 0,
|
|
620
|
+
resultModelUsage: geminiJsonState.resultModelUsage || buildGeminiResultModelUsage(mappedModel),
|
|
621
|
+
pricingInfo: { modelId: mappedModel, modelName: mappedModel, provider: 'Google', totalCostUSD: null },
|
|
622
|
+
publicPricingEstimate: null,
|
|
623
|
+
resultSummary: geminiJsonState.resultSummary || null,
|
|
624
|
+
completionHealth,
|
|
625
|
+
incompleteSession: completionHealth.incompleteSession,
|
|
626
|
+
diskPressureDetected: completionHealth.diskPressureDetected,
|
|
627
|
+
errorInfo: { message: completionHealth.reasons.join(' ') },
|
|
628
|
+
};
|
|
629
|
+
}
|
|
630
|
+
|
|
584
631
|
await log('\n\n✅ Gemini command completed');
|
|
585
632
|
await log(`📊 Total messages: ${geminiJsonState.messageCount || 0}, Tool uses: ${geminiJsonState.toolUseCount || 0}`);
|
|
586
633
|
if (geminiJsonState.resultSummary) {
|
package/src/git.lib.mjs
CHANGED
|
@@ -233,7 +233,7 @@ export const validateGitIdentity = async ($, options = {}) => {
|
|
|
233
233
|
const { log = console.log } = options;
|
|
234
234
|
|
|
235
235
|
// Check user.name
|
|
236
|
-
let userName
|
|
236
|
+
let userName;
|
|
237
237
|
try {
|
|
238
238
|
const nameResult = await $`git config user.name 2>/dev/null || true`;
|
|
239
239
|
userName = nameResult.stdout.toString().trim() || null;
|
|
@@ -242,7 +242,7 @@ export const validateGitIdentity = async ($, options = {}) => {
|
|
|
242
242
|
}
|
|
243
243
|
|
|
244
244
|
// Check user.email
|
|
245
|
-
let userEmail
|
|
245
|
+
let userEmail;
|
|
246
246
|
try {
|
|
247
247
|
const emailResult = await $`git config user.email 2>/dev/null || true`;
|
|
248
248
|
userEmail = emailResult.stdout.toString().trim() || null;
|
|
@@ -19,7 +19,7 @@ function normalizeRepositoryFullName(owner, repositoryName) {
|
|
|
19
19
|
|
|
20
20
|
function parseGitHubRepositoryUrlToken(token) {
|
|
21
21
|
const cleaned = trimOutputToken(token);
|
|
22
|
-
let pathName
|
|
22
|
+
let pathName;
|
|
23
23
|
|
|
24
24
|
if (cleaned.startsWith('git@github.com:')) {
|
|
25
25
|
pathName = cleaned.slice('git@github.com:'.length);
|
package/src/github.lib.mjs
CHANGED
|
@@ -967,7 +967,7 @@ export async function fetchProjectIssues(projectNumber, owner, statusFilter) {
|
|
|
967
967
|
context: 'github.lib.mjs - GitHub CLI auth status check',
|
|
968
968
|
level: 'error',
|
|
969
969
|
});
|
|
970
|
-
throw new Error('GitHub CLI authentication failed. Please run: gh auth login');
|
|
970
|
+
throw new Error('GitHub CLI authentication failed. Please run: gh auth login', { cause: error });
|
|
971
971
|
}
|
|
972
972
|
// Add delay to respect rate limits
|
|
973
973
|
await log(' ⏰ Waiting 2 seconds before API call to respect rate limits...', { verbose: true });
|
package/src/hive-screens.lib.mjs
CHANGED
|
@@ -135,7 +135,7 @@ export const parseHiveScreensArgs = argv => {
|
|
|
135
135
|
* the typical `NNNNN.name` session names.
|
|
136
136
|
*/
|
|
137
137
|
export const listDetachedSessions = async ({ exec = execAsync } = {}) => {
|
|
138
|
-
let stdout
|
|
138
|
+
let stdout;
|
|
139
139
|
try {
|
|
140
140
|
({ stdout } = await exec('screen -ls'));
|
|
141
141
|
} catch (err) {
|
package/src/i18n.lib.mjs
CHANGED
|
@@ -75,7 +75,7 @@ function refreshI18nRuntime() {
|
|
|
75
75
|
export async function loadTranslations(locale) {
|
|
76
76
|
if (localeCache.has(locale)) return localeCache.get(locale);
|
|
77
77
|
|
|
78
|
-
let translations
|
|
78
|
+
let translations;
|
|
79
79
|
try {
|
|
80
80
|
translations = await readLocaleFile(locale);
|
|
81
81
|
} catch {
|
package/src/instrument.mjs
CHANGED
|
@@ -42,7 +42,7 @@ const shouldDisableSentry = () => {
|
|
|
42
42
|
// Lazily import Sentry only if needed
|
|
43
43
|
// This prevents the Sentry packages from keeping the event loop alive when not needed
|
|
44
44
|
let Sentry = null;
|
|
45
|
-
let nodeProfilingIntegration
|
|
45
|
+
let nodeProfilingIntegration;
|
|
46
46
|
|
|
47
47
|
// Initialize Sentry if not disabled
|
|
48
48
|
if (!shouldDisableSentry()) {
|