@link-assistant/hive-mind 2.10.4 → 2.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/src/exit-handler.lib.mjs +61 -44
- package/src/github-pr-state.lib.mjs +56 -0
- package/src/locales/en.lino +3 -0
- package/src/locales/hi.lino +3 -0
- package/src/locales/ru.lino +3 -0
- package/src/locales/zh.lino +3 -0
- package/src/session-monitor.docker-terminal.lib.mjs +118 -0
- package/src/session-monitor.lib.mjs +71 -158
- package/src/session-monitor.stale-executing.lib.mjs +151 -0
- package/src/solve.finalize.lib.mjs +23 -8
- package/src/solve.mjs +1 -1
- package/src/work-session-formatting.lib.mjs +12 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.10.5
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 9772498: Stop reporting successful docker work sessions as failed. start-command can fabricate a detached-docker exit code from any `Exit Code: N` text the command itself printed (link-foundation/start#150), so the session monitor now trusts its own anchored log footer over `$ --status` and defers an uncorroborated docker failure for up to 60 seconds until the real footer is written.
|
|
8
|
+
|
|
3
9
|
## 2.10.4
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
package/package.json
CHANGED
package/src/exit-handler.lib.mjs
CHANGED
|
@@ -244,11 +244,22 @@ export const logActiveHandles = async (log = null) => {
|
|
|
244
244
|
* guidance for the pre-exit notifier.
|
|
245
245
|
*/
|
|
246
246
|
export const safeExit = async (code = 0, reason = 'Process completed', { skipPreExit = false, failureActionSection = null } = {}) => {
|
|
247
|
-
|
|
247
|
+
// Issue #2117: every best-effort step below is diagnostic housekeeping. It may
|
|
248
|
+
// fail, but it must never change the exit code the caller asked for — neither
|
|
249
|
+
// by masking a failure nor by turning a success into an uncaught exception.
|
|
250
|
+
try {
|
|
251
|
+
await showExitMessage(reason, code);
|
|
252
|
+
} catch (error) {
|
|
253
|
+
console.warn(`⚠️ Could not show exit message: ${error?.message || error}`);
|
|
254
|
+
}
|
|
248
255
|
|
|
249
256
|
// Issue #2090: collect the working session that is still uncollected (and the
|
|
250
257
|
// log tail produced after it) before the process goes away.
|
|
251
|
-
|
|
258
|
+
try {
|
|
259
|
+
await finalizeActiveDevelopmentLog({ force: true });
|
|
260
|
+
} catch {
|
|
261
|
+
// Best-effort finalization must never change the selected process exit.
|
|
262
|
+
}
|
|
252
263
|
|
|
253
264
|
if (!skipPreExit && code !== 0 && preExitFunction && !preExitHandlerRan) {
|
|
254
265
|
preExitHandlerRan = true;
|
|
@@ -267,7 +278,11 @@ export const safeExit = async (code = 0, reason = 'Process completed', { skipPre
|
|
|
267
278
|
// Issue #1431: Drain/unref active handles so the event loop exits naturally.
|
|
268
279
|
// This resolves the root causes of dangling ReadStream (stdin), Socket (undici),
|
|
269
280
|
// ChildProcess (command-stream), and WriteStream (stdout/stderr) handles.
|
|
270
|
-
|
|
281
|
+
try {
|
|
282
|
+
await drainHandles();
|
|
283
|
+
} catch {
|
|
284
|
+
// Best-effort handle draining must never change the selected process exit.
|
|
285
|
+
}
|
|
271
286
|
|
|
272
287
|
// Close Sentry to flush any pending events and allow the process to exit cleanly.
|
|
273
288
|
// Use Promise.race with a hard timeout to guarantee sentry.close() never hangs
|
|
@@ -291,7 +306,7 @@ export const safeExit = async (code = 0, reason = 'Process completed', { skipPre
|
|
|
291
306
|
/**
|
|
292
307
|
* Install global exit handlers to ensure log path is always shown
|
|
293
308
|
*/
|
|
294
|
-
export const installGlobalExitHandlers = () => {
|
|
309
|
+
export const installGlobalExitHandlers = ({ handleProcessErrors = true } = {}) => {
|
|
295
310
|
// Handle normal exit
|
|
296
311
|
process.on('exit', code => {
|
|
297
312
|
// Synchronous fallback - can't use async here
|
|
@@ -427,53 +442,55 @@ export const installGlobalExitHandlers = () => {
|
|
|
427
442
|
process.exit(143);
|
|
428
443
|
});
|
|
429
444
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
445
|
+
if (handleProcessErrors) {
|
|
446
|
+
// Handle uncaught exceptions
|
|
447
|
+
process.on('uncaughtException', async error => {
|
|
448
|
+
if (cleanupFunction) {
|
|
449
|
+
try {
|
|
450
|
+
await cleanupFunction();
|
|
451
|
+
} catch {
|
|
452
|
+
// Ignore cleanup errors on exception
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
if (logFunction) {
|
|
456
|
+
await logFunction(`\n❌ Uncaught Exception: ${error.message}`, { level: 'error' });
|
|
457
|
+
}
|
|
458
|
+
await showExitMessage('Uncaught exception occurred', 1);
|
|
433
459
|
try {
|
|
434
|
-
await
|
|
460
|
+
const sentry = await getSentry();
|
|
461
|
+
if (sentry && sentry.close) {
|
|
462
|
+
await Promise.race([sentry.close(2000), new Promise(resolve => setTimeout(resolve, 3000))]);
|
|
463
|
+
}
|
|
435
464
|
} catch {
|
|
436
|
-
// Ignore
|
|
465
|
+
// Ignore Sentry.close() errors
|
|
437
466
|
}
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
await logFunction(`\n❌ Uncaught Exception: ${error.message}`, { level: 'error' });
|
|
441
|
-
}
|
|
442
|
-
await showExitMessage('Uncaught exception occurred', 1);
|
|
443
|
-
try {
|
|
444
|
-
const sentry = await getSentry();
|
|
445
|
-
if (sentry && sentry.close) {
|
|
446
|
-
await Promise.race([sentry.close(2000), new Promise(resolve => setTimeout(resolve, 3000))]);
|
|
447
|
-
}
|
|
448
|
-
} catch {
|
|
449
|
-
// Ignore Sentry.close() errors
|
|
450
|
-
}
|
|
451
|
-
process.exit(1);
|
|
452
|
-
});
|
|
467
|
+
process.exit(1);
|
|
468
|
+
});
|
|
453
469
|
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
470
|
+
// Handle unhandled rejections
|
|
471
|
+
process.on('unhandledRejection', async reason => {
|
|
472
|
+
if (cleanupFunction) {
|
|
473
|
+
try {
|
|
474
|
+
await cleanupFunction();
|
|
475
|
+
} catch {
|
|
476
|
+
// Ignore cleanup errors on rejection
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
if (logFunction) {
|
|
480
|
+
await logFunction(`\n❌ Unhandled Rejection: ${reason}`, { level: 'error' });
|
|
481
|
+
}
|
|
482
|
+
await showExitMessage('Unhandled rejection occurred', 1);
|
|
457
483
|
try {
|
|
458
|
-
await
|
|
484
|
+
const sentry = await getSentry();
|
|
485
|
+
if (sentry && sentry.close) {
|
|
486
|
+
await Promise.race([sentry.close(2000), new Promise(resolve => setTimeout(resolve, 3000))]);
|
|
487
|
+
}
|
|
459
488
|
} catch {
|
|
460
|
-
// Ignore
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
if (logFunction) {
|
|
464
|
-
await logFunction(`\n❌ Unhandled Rejection: ${reason}`, { level: 'error' });
|
|
465
|
-
}
|
|
466
|
-
await showExitMessage('Unhandled rejection occurred', 1);
|
|
467
|
-
try {
|
|
468
|
-
const sentry = await getSentry();
|
|
469
|
-
if (sentry && sentry.close) {
|
|
470
|
-
await Promise.race([sentry.close(2000), new Promise(resolve => setTimeout(resolve, 3000))]);
|
|
489
|
+
// Ignore Sentry.close() errors
|
|
471
490
|
}
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
process.exit(1);
|
|
476
|
-
});
|
|
491
|
+
process.exit(1);
|
|
492
|
+
});
|
|
493
|
+
}
|
|
477
494
|
};
|
|
478
495
|
|
|
479
496
|
/**
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { execGhWithRetry } from './github-rate-limit.lib.mjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Resolve the externally visible state of a GitHub pull request.
|
|
5
|
+
*
|
|
6
|
+
* A strict URL parser keeps the command limited to GitHub owner/repo/PR
|
|
7
|
+
* identifiers. The REST response exposes `merged` and `merged_at`, which are
|
|
8
|
+
* stronger evidence of goal completion than the detached runner's exit code.
|
|
9
|
+
*
|
|
10
|
+
* @param {string|null} pullRequestUrl
|
|
11
|
+
* @param {Object} [options]
|
|
12
|
+
* @param {Function} [options.lookupPullRequestState] - Test/application override
|
|
13
|
+
* @param {boolean} [options.verbose]
|
|
14
|
+
* @returns {Promise<{merged:boolean, mergedAt:string|null, state:string|null}|null>}
|
|
15
|
+
*/
|
|
16
|
+
export async function resolvePullRequestState(pullRequestUrl, { lookupPullRequestState = null, verbose = false } = {}) {
|
|
17
|
+
if (!pullRequestUrl) return null;
|
|
18
|
+
if (typeof lookupPullRequestState === 'function') {
|
|
19
|
+
return (await lookupPullRequestState(pullRequestUrl)) || null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const match = String(pullRequestUrl).match(/^https:\/\/github\.com\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)\/pull\/(\d+)(?:[/?#].*)?$/i);
|
|
23
|
+
if (!match) {
|
|
24
|
+
if (verbose) console.log(`[VERBOSE] Cannot resolve PR state for unrecognized URL: ${pullRequestUrl}`);
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const [, owner, repo, number] = match;
|
|
29
|
+
try {
|
|
30
|
+
const { stdout } = await execGhWithRetry(`gh api repos/${owner}/${repo}/pulls/${number} --jq '{merged: .merged, mergedAt: .merged_at, state: .state}'`, {
|
|
31
|
+
execOptions: {
|
|
32
|
+
encoding: 'utf8',
|
|
33
|
+
maxBuffer: 1024 * 1024,
|
|
34
|
+
},
|
|
35
|
+
label: `gh api pull request state (${owner}/${repo}#${number})`,
|
|
36
|
+
});
|
|
37
|
+
const state = JSON.parse(stdout);
|
|
38
|
+
return {
|
|
39
|
+
merged: state?.merged === true,
|
|
40
|
+
mergedAt: state?.mergedAt || null,
|
|
41
|
+
state: state?.state || null,
|
|
42
|
+
};
|
|
43
|
+
} catch (error) {
|
|
44
|
+
if (verbose) console.log(`[VERBOSE] Pull request state lookup failed for ${pullRequestUrl}: ${error?.message || error}`);
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function resolveFailedSessionPullRequestState({ pullRequestUrl, outcome, lookupPullRequestState = null, verbose = false, sessionName = 'unknown', exitCode = null, status = null, logPath = null } = {}) {
|
|
50
|
+
if (!outcome?.failed || outcome.killed || !pullRequestUrl) return null;
|
|
51
|
+
const state = await resolvePullRequestState(pullRequestUrl, { lookupPullRequestState, verbose });
|
|
52
|
+
if (verbose && state) {
|
|
53
|
+
console.log(`[VERBOSE] Completion evidence for ${sessionName}: exitCode=${exitCode}, status=${status || 'unknown'}, pullRequest=${pullRequestUrl}, merged=${state.merged}, mergedAt=${state.mergedAt || 'unknown'}, logPath=${logPath || 'unknown'}`);
|
|
54
|
+
}
|
|
55
|
+
return state;
|
|
56
|
+
}
|
package/src/locales/en.lino
CHANGED
|
@@ -649,6 +649,9 @@ en
|
|
|
649
649
|
executing "⏳ Executing..."
|
|
650
650
|
finished "Work session finished successfully"
|
|
651
651
|
failed "Work session failed (exit code: {{exitCode}})"
|
|
652
|
+
merged_but_failed "Pull request merged, but the work session exited with code: {{exitCode}}"
|
|
653
|
+
merged_success "The requested pull request was merged successfully."
|
|
654
|
+
runner_also_failed "The runner also failed; its exit code is preserved for investigation."
|
|
652
655
|
killed "Work session {{reason}}{{exitSuffix}}"
|
|
653
656
|
stopped "Work session stopped by user{{requestedBy}}{{exitSuffix}}"
|
|
654
657
|
duration
|
package/src/locales/hi.lino
CHANGED
|
@@ -649,6 +649,9 @@ hi
|
|
|
649
649
|
executing "⏳ चल रहा है..."
|
|
650
650
|
finished "कार्य सत्र सफलतापूर्वक पूरा हुआ"
|
|
651
651
|
failed "कार्य सत्र विफल हुआ (exit code: {{exitCode}})"
|
|
652
|
+
merged_but_failed "पुल अनुरोध मर्ज हो गया, लेकिन कार्य सत्र कोड {{exitCode}} के साथ समाप्त हुआ"
|
|
653
|
+
merged_success "अनुरोधित पुल अनुरोध सफलतापूर्वक मर्ज हो गया।"
|
|
654
|
+
runner_also_failed "रनर भी विफल हुआ; जाँच के लिए उसका exit code सुरक्षित रखा गया है।"
|
|
652
655
|
killed "कार्य सत्र रोका गया: {{reason}}{{exitSuffix}}"
|
|
653
656
|
stopped "कार्य सत्र उपयोगकर्ता द्वारा रोका गया{{requestedBy}}{{exitSuffix}}"
|
|
654
657
|
duration
|
package/src/locales/ru.lino
CHANGED
|
@@ -649,6 +649,9 @@ ru
|
|
|
649
649
|
executing "⏳ Выполняется..."
|
|
650
650
|
finished "Рабочий сеанс успешно завершен"
|
|
651
651
|
failed "Рабочий сеанс завершился с ошибкой (код выхода: {{exitCode}})"
|
|
652
|
+
merged_but_failed "Пул-реквест объединён, но рабочий сеанс завершился с кодом: {{exitCode}}"
|
|
653
|
+
merged_success "Запрошенный пул-реквест успешно объединён."
|
|
654
|
+
runner_also_failed "Средство запуска также завершилось с ошибкой; код выхода сохранён для расследования."
|
|
652
655
|
killed "Рабочий сеанс остановлен: {{reason}}{{exitSuffix}}"
|
|
653
656
|
stopped "Рабочий сеанс остановлен пользователем{{requestedBy}}{{exitSuffix}}"
|
|
654
657
|
duration
|
package/src/locales/zh.lino
CHANGED
|
@@ -649,6 +649,9 @@ zh
|
|
|
649
649
|
executing "⏳ 正在执行..."
|
|
650
650
|
finished "工作会话已成功完成"
|
|
651
651
|
failed "工作会话失败(退出代码:{{exitCode}})"
|
|
652
|
+
merged_but_failed "拉取请求已合并,但工作会话退出,代码为:{{exitCode}}"
|
|
653
|
+
merged_success "请求的拉取请求已成功合并。"
|
|
654
|
+
runner_also_failed "运行器也失败了;其退出代码已保留以供调查。"
|
|
652
655
|
killed "工作会话已终止:{{reason}}{{exitSuffix}}"
|
|
653
656
|
stopped "工作会话已由用户停止{{requestedBy}}{{exitSuffix}}"
|
|
654
657
|
duration
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue #2117: guard against a *fabricated* terminal exit code for detached
|
|
3
|
+
* docker sessions.
|
|
4
|
+
*
|
|
5
|
+
* start-command derives the exit code of a detached docker session from an
|
|
6
|
+
* unanchored `Exit Code: N` scan over the whole execution log
|
|
7
|
+
* (`status-formatter.js#readExitCodeFromLog`), so any text the wrapped command
|
|
8
|
+
* prints — for example an AI agent echoing the tail of an older execution log —
|
|
9
|
+
* can be mistaken for the terminal footer. That is exactly how a `/solve` run
|
|
10
|
+
* that merged its pull request and exited 0 was announced as "Work session
|
|
11
|
+
* failed (exit code: 1)": the fabricated code entered the log 21 minutes before
|
|
12
|
+
* the command finished, and `$ --status` stamped the record with it the moment
|
|
13
|
+
* the container stopped — about two seconds before the real footer
|
|
14
|
+
* (`Exit Code: 0`) was appended by start-command's detached-docker watcher.
|
|
15
|
+
*
|
|
16
|
+
* The watcher always appends that footer right after the container exits, so a
|
|
17
|
+
* short deferral is enough for the authoritative value to appear. If it never
|
|
18
|
+
* does (e.g. the watcher itself was killed), the reported status is accepted
|
|
19
|
+
* once the grace period expires, preserving the previous behaviour: a real
|
|
20
|
+
* failure is still reported, just a minute later.
|
|
21
|
+
*
|
|
22
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2117
|
|
23
|
+
* @see https://github.com/link-foundation/start/issues/150
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** How long an uncorroborated docker terminal failure stays provisional. */
|
|
27
|
+
export const DOCKER_TERMINAL_FOOTER_GRACE_MS = 60 * 1000;
|
|
28
|
+
|
|
29
|
+
/** Session-snapshot field holding the first sighting of such a status. */
|
|
30
|
+
export const DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD = 'dockerTerminalUnverifiedFirstSeenAt';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Whether the reported end of the session is recent enough for the fabrication
|
|
34
|
+
* race to still be open.
|
|
35
|
+
*
|
|
36
|
+
* start-command only invents a terminal result for a record it still has stored
|
|
37
|
+
* as `executing`, and it stamps that invention with `endTime = new Date()`
|
|
38
|
+
* (`status-formatter.js#enrichDetachedStatus`) — so a fabricated failure always
|
|
39
|
+
* looks like it ended just now, no matter how long ago the container really
|
|
40
|
+
* stopped. A record that reports an end time older than the grace period, on the
|
|
41
|
+
* other hand, has had all the time it needed for the footer to be written;
|
|
42
|
+
* deferring it would only postpone a real failure notification.
|
|
43
|
+
*
|
|
44
|
+
* @param {string|Date|null|undefined} endTime - End time from the status record.
|
|
45
|
+
* @param {number} nowMs - Current epoch milliseconds.
|
|
46
|
+
* @returns {boolean} True when the race cannot be ruled out.
|
|
47
|
+
*/
|
|
48
|
+
function terminalClaimIsRecent(endTime, nowMs) {
|
|
49
|
+
if (!endTime) return true; // No timestamp at all: the race cannot be ruled out.
|
|
50
|
+
const endMs = endTime instanceof Date ? endTime.getTime() : new Date(endTime).getTime();
|
|
51
|
+
if (!Number.isFinite(endMs)) return true;
|
|
52
|
+
return nowMs - endMs < DOCKER_TERMINAL_FOOTER_GRACE_MS;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Drop the deferral marker once the session's outcome is resolved (footer
|
|
57
|
+
* written, session running again, or the status accepted).
|
|
58
|
+
*
|
|
59
|
+
* @param {object|null|undefined} sessionInfo - Tracked session snapshot (mutated).
|
|
60
|
+
* @param {function} persistSnapshot - Callback persisting the updated snapshot.
|
|
61
|
+
*/
|
|
62
|
+
export function clearUnverifiedDockerTerminalMarker(sessionInfo, persistSnapshot) {
|
|
63
|
+
if (!sessionInfo?.[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD]) return;
|
|
64
|
+
delete sessionInfo[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD];
|
|
65
|
+
persistSnapshot();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Decide whether a docker terminal *failure* status that the log footer does not
|
|
70
|
+
* corroborate should be deferred (treated as still running) for now. The first
|
|
71
|
+
* sighting is persisted so the deferral survives a bot restart and cannot be
|
|
72
|
+
* reset forever by repeated polling.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} sessionName - Session identifier, for verbose output only.
|
|
75
|
+
* @param {object|null|undefined} sessionInfo - Tracked session snapshot (mutated).
|
|
76
|
+
* @param {object} options
|
|
77
|
+
* @param {number|null} options.exitCode - Exit code `$ --status` reported.
|
|
78
|
+
* @param {string|Date|null} [options.endTime] - End time the status record reports.
|
|
79
|
+
* @param {boolean} options.verbose - Whether to explain the decision.
|
|
80
|
+
* @param {function} options.persistSnapshot - Callback persisting the snapshot.
|
|
81
|
+
* @returns {boolean} True while the status is still provisional.
|
|
82
|
+
*/
|
|
83
|
+
export function shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime = null, verbose, persistSnapshot }) {
|
|
84
|
+
const nowMs = Date.now();
|
|
85
|
+
|
|
86
|
+
if (!terminalClaimIsRecent(endTime, nowMs)) {
|
|
87
|
+
if (verbose) {
|
|
88
|
+
console.log(`[VERBOSE] Session ${sessionName} reports a terminal failure (exit ${exitCode}) that ended at ${endTime}, longer than ${DOCKER_TERMINAL_FOOTER_GRACE_MS}ms ago; the log footer had time to appear, so the reported exit code is accepted (issue #2117)`);
|
|
89
|
+
}
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const raw = sessionInfo?.[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD];
|
|
94
|
+
const firstSeenMs = raw ? new Date(raw).getTime() : null;
|
|
95
|
+
|
|
96
|
+
if (firstSeenMs === null || !Number.isFinite(firstSeenMs)) {
|
|
97
|
+
if (sessionInfo) {
|
|
98
|
+
sessionInfo[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
99
|
+
persistSnapshot();
|
|
100
|
+
}
|
|
101
|
+
if (verbose) {
|
|
102
|
+
console.log(`[VERBOSE] Session ${sessionName} reports a terminal failure (exit ${exitCode}) that no log footer corroborates; deferring for up to ${DOCKER_TERMINAL_FOOTER_GRACE_MS}ms until start-command writes the real footer (issue #2117)`);
|
|
103
|
+
}
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (nowMs - firstSeenMs < DOCKER_TERMINAL_FOOTER_GRACE_MS) {
|
|
108
|
+
if (verbose) {
|
|
109
|
+
console.log(`[VERBOSE] Session ${sessionName} still reports an unverified terminal failure (exit ${exitCode}); waiting for the log footer (issue #2117)`);
|
|
110
|
+
}
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (verbose) {
|
|
115
|
+
console.log(`[VERBOSE] Session ${sessionName} kept reporting a terminal failure (exit ${exitCode}) for ${DOCKER_TERMINAL_FOOTER_GRACE_MS}ms without a log footer; accepting the reported exit code (issue #2117)`);
|
|
116
|
+
}
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
@@ -26,8 +26,15 @@ import { formatSessionCompletionMessage, getSessionCompletionExitCode, classifyS
|
|
|
26
26
|
import { notifySubscribers, getSubscriberCount } from './telegram-subscribers.lib.mjs';
|
|
27
27
|
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
28
28
|
import { readLastSessionIdFromLog, buildResumeCommand, formatResumeSection } from './session-resume.lib.mjs';
|
|
29
|
+
import { resolveFailedSessionPullRequestState } from './github-pr-state.lib.mjs';
|
|
30
|
+
// Issue #2117: a docker terminal failure that no anchored log footer corroborates
|
|
31
|
+
// may be an exit code start-command fabricated from the command's own output.
|
|
32
|
+
import { clearUnverifiedDockerTerminalMarker as clearUnverifiedDockerTerminalMarkerImpl, shouldDeferUnverifiedDockerTerminal as shouldDeferUnverifiedDockerTerminalImpl } from './session-monitor.docker-terminal.lib.mjs';
|
|
33
|
+
import { isDockerIsolation, sessionStartMs, resolveOomKilledState, resolveStaleExecutingState as resolveStaleExecutingStateImpl } from './session-monitor.stale-executing.lib.mjs';
|
|
29
34
|
|
|
30
35
|
export { formatSessionCompletionMessage, getSessionCompletionExitCode } from './work-session-formatting.lib.mjs';
|
|
36
|
+
export { DOCKER_TERMINAL_FOOTER_GRACE_MS } from './session-monitor.docker-terminal.lib.mjs';
|
|
37
|
+
export { STALE_EXECUTING_MIN_AGE_MS, DOCKER_BACKEND_GONE_GRACE_MS } from './session-monitor.stale-executing.lib.mjs';
|
|
31
38
|
|
|
32
39
|
const exec = promisify(execCallback);
|
|
33
40
|
|
|
@@ -521,134 +528,16 @@ function isNonIsolationSessionActive(sessionName, sessionInfo, verbose = false)
|
|
|
521
528
|
return true;
|
|
522
529
|
}
|
|
523
530
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
* `executing` is allowed to be declared dead purely on a backend-liveness probe
|
|
527
|
-
* (the screen/tmux/docker session is gone). This avoids a race where a session
|
|
528
|
-
* that has just been launched — but whose backend has not registered yet — is
|
|
529
|
-
* falsely reported as killed. The authoritative log-footer check is NOT gated by
|
|
530
|
-
* this, because a written "Exit Code:" footer is proof the command terminated.
|
|
531
|
-
*/
|
|
532
|
-
export const STALE_EXECUTING_MIN_AGE_MS = 90 * 1000;
|
|
533
|
-
export const DOCKER_BACKEND_GONE_GRACE_MS = 2 * 60 * 1000;
|
|
534
|
-
const DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD = 'dockerBackendGoneFirstSeenAt';
|
|
535
|
-
|
|
536
|
-
function sessionStartMs(sessionInfo) {
|
|
537
|
-
const start = sessionInfo?.startTime;
|
|
538
|
-
if (!start) return null;
|
|
539
|
-
const date = start instanceof Date ? start : new Date(start);
|
|
540
|
-
const ms = date.getTime();
|
|
541
|
-
return Number.isFinite(ms) ? ms : null;
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
function isDockerIsolation(sessionInfo, statusResult) {
|
|
545
|
-
return sessionInfo?.isolationBackend === 'docker' || statusResult?.isolation === 'docker';
|
|
531
|
+
function clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo) {
|
|
532
|
+
clearUnverifiedDockerTerminalMarkerImpl(sessionInfo, () => persistSessionSnapshot(sessionName, sessionInfo));
|
|
546
533
|
}
|
|
547
534
|
|
|
548
|
-
function
|
|
549
|
-
|
|
550
|
-
if (!raw) return null;
|
|
551
|
-
const ms = new Date(raw).getTime();
|
|
552
|
-
return Number.isFinite(ms) ? ms : null;
|
|
535
|
+
function shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime, verbose }) {
|
|
536
|
+
return shouldDeferUnverifiedDockerTerminalImpl(sessionName, sessionInfo, { exitCode, endTime, verbose, persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo) });
|
|
553
537
|
}
|
|
554
538
|
|
|
555
|
-
function
|
|
556
|
-
|
|
557
|
-
delete sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
558
|
-
persistSessionSnapshot(sessionName, sessionInfo);
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
/**
|
|
562
|
-
* Cross-check whether a session that `$ --status` still reports as `executing`
|
|
563
|
-
* has actually terminated. Issue #1927: start-command's status can get stuck on
|
|
564
|
-
* `executing` after the process was killed (a lingering shell keeps the screen
|
|
565
|
-
* session alive, flipping executed→executing), so a SIGKILLed /solve was never
|
|
566
|
-
* reported. Two independent signals are consulted, strongest first:
|
|
567
|
-
*
|
|
568
|
-
* 1. The execution log FOOTER. When start-command wrote "Exit Code: N" the
|
|
569
|
-
* command terminated, full stop — regardless of what `--status` claims.
|
|
570
|
-
* This is authoritative and catches the dominant lingering-shell case.
|
|
571
|
-
* 2. Backend LIVENESS. If no footer was written (e.g. the wrapper itself was
|
|
572
|
-
* hard-killed) but the backing screen/tmux/docker session is gone, the
|
|
573
|
-
* process cannot still be executing. Gated by STALE_EXECUTING_MIN_AGE_MS to
|
|
574
|
-
* avoid a just-launched-not-yet-registered race.
|
|
575
|
-
*
|
|
576
|
-
* @returns {Promise<{exitCode: number|null, status: string, reason: string}|null>}
|
|
577
|
-
* Terminal details when the session is actually dead, else null (still running).
|
|
578
|
-
*/
|
|
579
|
-
async function resolveStaleExecutingState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive }) {
|
|
580
|
-
// 1. Authoritative: the log footer.
|
|
581
|
-
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
582
|
-
if (logPath) {
|
|
583
|
-
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
584
|
-
const footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
585
|
-
if (footer?.finished) {
|
|
586
|
-
const status = classifyExitStatus(footer.exitCode) || (footer.exitCode === 0 ? 'executed' : 'failed');
|
|
587
|
-
return { exitCode: footer.exitCode, status, reason: `log-footer(exit ${footer.exitCode})` };
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
// 2. Liveness probe, only once the session is old enough to have registered.
|
|
592
|
-
const startMs = sessionStartMs(sessionInfo);
|
|
593
|
-
const ageMs = startMs != null ? Date.now() - startMs : Infinity;
|
|
594
|
-
if (ageMs >= STALE_EXECUTING_MIN_AGE_MS && sessionInfo?.isolationBackend) {
|
|
595
|
-
const probe = backendAlive || runner.checkBackendSessionAlive;
|
|
596
|
-
const alive = probe ? await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose) : null;
|
|
597
|
-
// Only `false` (definitively gone) counts as killed; `null` (unknown backend)
|
|
598
|
-
// is treated as "no signal" so we don't kill on an indeterminate probe.
|
|
599
|
-
if (alive === false) {
|
|
600
|
-
if (isDockerIsolation(sessionInfo, statusResult)) {
|
|
601
|
-
const nowMs = Date.now();
|
|
602
|
-
const firstSeenMs = getDockerBackendGoneFirstSeenMs(sessionInfo);
|
|
603
|
-
if (firstSeenMs === null) {
|
|
604
|
-
sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
605
|
-
persistSessionSnapshot(sessionName, sessionInfo);
|
|
606
|
-
if (verbose) {
|
|
607
|
-
console.log(`[VERBOSE] Session ${sessionName} docker backend is gone but no terminal status/footer is available yet; deferring killed classification for ${DOCKER_BACKEND_GONE_GRACE_MS}ms`);
|
|
608
|
-
}
|
|
609
|
-
return null;
|
|
610
|
-
}
|
|
611
|
-
if (nowMs - firstSeenMs < DOCKER_BACKEND_GONE_GRACE_MS) {
|
|
612
|
-
if (verbose) {
|
|
613
|
-
console.log(`[VERBOSE] Session ${sessionName} docker backend is still gone; waiting for terminal status/footer before reporting killed`);
|
|
614
|
-
}
|
|
615
|
-
return null;
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
return { exitCode: null, status: 'killed', reason: 'backend-gone' };
|
|
619
|
-
}
|
|
620
|
-
if (alive === true) {
|
|
621
|
-
clearDockerBackendGoneMarker(sessionName, sessionInfo);
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
return null;
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
|
|
629
|
-
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
630
|
-
let footer = null;
|
|
631
|
-
if (logPath) {
|
|
632
|
-
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
633
|
-
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
637
|
-
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
638
|
-
let exitCode = 137;
|
|
639
|
-
if (statusExitCode !== null && statusExitCode > 0) {
|
|
640
|
-
exitCode = statusExitCode;
|
|
641
|
-
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
642
|
-
exitCode = footerExitCode;
|
|
643
|
-
}
|
|
644
|
-
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
645
|
-
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
646
|
-
|
|
647
|
-
if (verbose) {
|
|
648
|
-
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
539
|
+
function resolveStaleExecutingState(sessionName, sessionInfo, statusResult, options) {
|
|
540
|
+
return resolveStaleExecutingStateImpl(sessionName, sessionInfo, statusResult, { ...options, persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo) });
|
|
652
541
|
}
|
|
653
542
|
|
|
654
543
|
async function getIsolationSessionState(sessionName, sessionInfo, options = {}) {
|
|
@@ -678,25 +567,30 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
678
567
|
const corrected = { ...statusResult, status: correctedStatus, exitCode: stale.exitCode, endTime: statusResult.endTime || stale.endTime || null };
|
|
679
568
|
return { running: false, exitCode: stale.exitCode, status: correctedStatus, statusResult: corrected, stale: true };
|
|
680
569
|
}
|
|
570
|
+
// Back to a plain `executing` report: any earlier unverified terminal
|
|
571
|
+
// failure was provisional and is now moot (issue #2117).
|
|
572
|
+
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
681
573
|
return { running: true, exitCode: null, status: statusResult.status, statusResult };
|
|
682
574
|
}
|
|
683
575
|
if (runner.isTerminalSessionStatus(statusResult.status)) {
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
//
|
|
687
|
-
//
|
|
688
|
-
//
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
576
|
+
const exitCode = statusResult.exitCode !== undefined ? statusResult.exitCode : null;
|
|
577
|
+
const logPath = statusResult.logPath || sessionInfo?.logPath || null;
|
|
578
|
+
// The log FOOTER is the authoritative terminal result. It is anchored on
|
|
579
|
+
// the `=====` separator (see parseSessionExitFooter), so — unlike the
|
|
580
|
+
// exit code `$ --status` derives from an unanchored full-log scan — it
|
|
581
|
+
// cannot be forged by output the wrapped command printed (issue #2117).
|
|
582
|
+
// Prefer it whenever it exists: that both recovers a real code from a
|
|
583
|
+
// missing/sentinel status (issue #1927) and overrides a fabricated one.
|
|
584
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
585
|
+
const footer = logPath && readFooter ? readFooter(logPath, { verbose }) : null;
|
|
586
|
+
if (footer?.finished) {
|
|
587
|
+
const footerExitCode = footer.exitCode;
|
|
588
|
+
const correctedStatus = classifyExitStatus(footerExitCode) || statusResult.status;
|
|
589
|
+
if (verbose && normalizeExitCode(footerExitCode) !== normalizeExitCode(exitCode)) {
|
|
590
|
+
console.log(`[VERBOSE] Session ${sessionName} reported terminal '${statusResult.status}' with exit ${exitCode}; the log footer says exit ${footerExitCode} (${correctedStatus}) and wins (issues #1927/#2117)`);
|
|
699
591
|
}
|
|
592
|
+
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
593
|
+
return { running: false, exitCode: footerExitCode, status: correctedStatus, statusResult: { ...statusResult, status: correctedStatus, exitCode: footerExitCode } };
|
|
700
594
|
}
|
|
701
595
|
// Issue #1939: a native docker session can report a terminal status
|
|
702
596
|
// ("executed") with the unknown exit-code sentinel (-1) while the
|
|
@@ -704,8 +598,21 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
704
598
|
// a real terminal exit, such a status is provisional — fall through to
|
|
705
599
|
// isSessionRunning() below, which cross-checks the live container via
|
|
706
600
|
// `docker inspect` before we notify the user the work finished.
|
|
707
|
-
const
|
|
601
|
+
const dockerSession = isDockerIsolation(sessionInfo, statusResult);
|
|
602
|
+
const ambiguousDockerTerminal = dockerSession && typeof runner.isUnknownDockerExitCode === 'function' && runner.isUnknownDockerExitCode(exitCode);
|
|
603
|
+
// Issue #2117: a docker terminal FAILURE with no corroborating footer is
|
|
604
|
+
// provisional too — start-command can fabricate that exit code from the
|
|
605
|
+
// command's own output. Give the real footer a moment to appear instead
|
|
606
|
+
// of announcing a failure the run never had. Only a *freshly* reported
|
|
607
|
+
// end time can still be in that race, so an older terminal record is
|
|
608
|
+
// still reported without delay.
|
|
609
|
+
const normalizedExitCode = normalizeExitCode(exitCode);
|
|
610
|
+
const unverifiedDockerFailure = dockerSession && !ambiguousDockerTerminal && normalizedExitCode !== null && normalizedExitCode !== 0;
|
|
611
|
+
if (unverifiedDockerFailure && shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime: statusResult.endTime || null, verbose })) {
|
|
612
|
+
return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
|
|
613
|
+
}
|
|
708
614
|
if (!ambiguousDockerTerminal) {
|
|
615
|
+
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
709
616
|
return { running: false, exitCode, status: statusResult.status, statusResult };
|
|
710
617
|
}
|
|
711
618
|
}
|
|
@@ -844,11 +751,8 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
844
751
|
verbose,
|
|
845
752
|
});
|
|
846
753
|
|
|
847
|
-
// Issue #1688/#1905:
|
|
848
|
-
//
|
|
849
|
-
// `Issue:` and a `Pull request:` line. The linked-issue API can lag
|
|
850
|
-
// behind the solver's own verification log, so we also inspect the
|
|
851
|
-
// completed session log before giving up.
|
|
754
|
+
// Issue #1688/#1905: Resolve the created PR from GitHub or, when its
|
|
755
|
+
// linked-issue API lags, from the completed solve log.
|
|
852
756
|
let pullRequestUrl = null;
|
|
853
757
|
try {
|
|
854
758
|
pullRequestUrl = await resolvePullRequestUrlForSession(sessionInfo, {
|
|
@@ -863,10 +767,25 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
863
767
|
}
|
|
864
768
|
}
|
|
865
769
|
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
770
|
+
let pullRequestState = null;
|
|
771
|
+
const completionOutcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
|
|
772
|
+
try {
|
|
773
|
+
pullRequestState = await resolveFailedSessionPullRequestState({
|
|
774
|
+
pullRequestUrl,
|
|
775
|
+
outcome: completionOutcome,
|
|
776
|
+
lookupPullRequestState: options.lookupPullRequestState,
|
|
777
|
+
verbose,
|
|
778
|
+
sessionName,
|
|
779
|
+
exitCode: finalExitCode,
|
|
780
|
+
status: resolvedStatus,
|
|
781
|
+
logPath: statusResult?.logPath || sessionInfo?.logPath,
|
|
782
|
+
});
|
|
783
|
+
} catch (stateError) {
|
|
784
|
+
if (verbose) console.log(`[VERBOSE] Pull request state resolution failed for ${sessionName}: ${stateError?.message || stateError}`);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
// Issue #594: append an end-of-task limits snapshot/delta. Cached
|
|
788
|
+
// helpers prevent parallel sessions from stampeding the upstream API.
|
|
870
789
|
const limitsExtraSections = [];
|
|
871
790
|
if (sessionInfo?.showLimits) {
|
|
872
791
|
try {
|
|
@@ -895,15 +814,8 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
895
814
|
}
|
|
896
815
|
}
|
|
897
816
|
|
|
898
|
-
// Issue #1927
|
|
899
|
-
//
|
|
900
|
-
// ready-to-run `--resume <lastSessionId>` command so the surviving
|
|
901
|
-
// parent (the operator, or an automation watching the bot) can pick the
|
|
902
|
-
// work back up. We deliberately do NOT auto-relaunch here: a job that
|
|
903
|
-
// reliably OOMs would storm. The rule "use the LAST of multiple
|
|
904
|
-
// sessions" is honored by reading the last `Session ID:` marker from
|
|
905
|
-
// the captured log. Purely additive — failures never block the
|
|
906
|
-
// completion notification, preserving backward compatibility.
|
|
817
|
+
// Issue #1927: for a killed /solve, offer a command using the last tool
|
|
818
|
+
// session ID in the log. Do not auto-relaunch work that may reliably OOM.
|
|
907
819
|
const resumeExtraSections = [];
|
|
908
820
|
try {
|
|
909
821
|
const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
|
|
@@ -961,6 +873,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
961
873
|
exitCode: finalExitCode,
|
|
962
874
|
infoBlock: sessionInfo?.infoBlock || '',
|
|
963
875
|
pullRequestUrl,
|
|
876
|
+
pullRequestState,
|
|
964
877
|
extraSections: [...limitsExtraSections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
965
878
|
});
|
|
966
879
|
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Terminal-state reconciliation helpers for tracked isolation sessions.
|
|
3
|
+
*
|
|
4
|
+
* These live next to session-monitor.lib.mjs (which is at its `max-lines`
|
|
5
|
+
* budget) and cover two independent defects:
|
|
6
|
+
*
|
|
7
|
+
* - issue #1927: `$ --status` can stay stuck on `executing` after the process
|
|
8
|
+
* was killed, so the monitor has to cross-check the log footer and the
|
|
9
|
+
* backing screen/tmux/docker session.
|
|
10
|
+
* - issue #2015: a docker session reported with `oomKilled=true` is terminal
|
|
11
|
+
* and must be surfaced as such instead of being polled forever.
|
|
12
|
+
*
|
|
13
|
+
* @see https://github.com/link-assistant/hive-mind/issues/1927
|
|
14
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2015
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Issue #1927: minimum age before a session that `$ --status` still reports as
|
|
21
|
+
* `executing` is allowed to be declared dead purely on a backend-liveness probe
|
|
22
|
+
* (the screen/tmux/docker session is gone). This avoids a race where a session
|
|
23
|
+
* that has just been launched — but whose backend has not registered yet — is
|
|
24
|
+
* falsely reported as killed. The authoritative log-footer check is NOT gated by
|
|
25
|
+
* this, because a written "Exit Code:" footer is proof the command terminated.
|
|
26
|
+
*/
|
|
27
|
+
export const STALE_EXECUTING_MIN_AGE_MS = 90 * 1000;
|
|
28
|
+
export const DOCKER_BACKEND_GONE_GRACE_MS = 2 * 60 * 1000;
|
|
29
|
+
const DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD = 'dockerBackendGoneFirstSeenAt';
|
|
30
|
+
|
|
31
|
+
export function sessionStartMs(sessionInfo) {
|
|
32
|
+
const start = sessionInfo?.startTime;
|
|
33
|
+
if (!start) return null;
|
|
34
|
+
const date = start instanceof Date ? start : new Date(start);
|
|
35
|
+
const ms = date.getTime();
|
|
36
|
+
return Number.isFinite(ms) ? ms : null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function isDockerIsolation(sessionInfo, statusResult) {
|
|
40
|
+
return sessionInfo?.isolationBackend === 'docker' || statusResult?.isolation === 'docker';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function getDockerBackendGoneFirstSeenMs(sessionInfo) {
|
|
44
|
+
const raw = sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
45
|
+
if (!raw) return null;
|
|
46
|
+
const ms = new Date(raw).getTime();
|
|
47
|
+
return Number.isFinite(ms) ? ms : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function clearDockerBackendGoneMarker(sessionInfo, persistSnapshot) {
|
|
51
|
+
if (!sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD]) return;
|
|
52
|
+
delete sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
53
|
+
persistSnapshot();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Cross-check whether a session that `$ --status` still reports as `executing`
|
|
58
|
+
* has actually terminated. Issue #1927: start-command's status can get stuck on
|
|
59
|
+
* `executing` after the process was killed (a lingering shell keeps the screen
|
|
60
|
+
* session alive, flipping executed→executing), so a SIGKILLed /solve was never
|
|
61
|
+
* reported. Two independent signals are consulted, strongest first:
|
|
62
|
+
*
|
|
63
|
+
* 1. The execution log FOOTER. When start-command wrote "Exit Code: N" the
|
|
64
|
+
* command terminated, full stop — regardless of what `--status` claims.
|
|
65
|
+
* This is authoritative and catches the dominant lingering-shell case.
|
|
66
|
+
* 2. Backend LIVENESS. If no footer was written (e.g. the wrapper itself was
|
|
67
|
+
* hard-killed) but the backing screen/tmux/docker session is gone, the
|
|
68
|
+
* process cannot still be executing. Gated by STALE_EXECUTING_MIN_AGE_MS to
|
|
69
|
+
* avoid a just-launched-not-yet-registered race.
|
|
70
|
+
*
|
|
71
|
+
* @returns {Promise<{exitCode: number|null, status: string, reason: string}|null>}
|
|
72
|
+
* Terminal details when the session is actually dead, else null (still running).
|
|
73
|
+
*/
|
|
74
|
+
export async function resolveStaleExecutingState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive, persistSnapshot }) {
|
|
75
|
+
// 1. Authoritative: the log footer.
|
|
76
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
77
|
+
if (logPath) {
|
|
78
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
79
|
+
const footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
80
|
+
if (footer?.finished) {
|
|
81
|
+
const status = classifyExitStatus(footer.exitCode) || (footer.exitCode === 0 ? 'executed' : 'failed');
|
|
82
|
+
return { exitCode: footer.exitCode, status, reason: `log-footer(exit ${footer.exitCode})` };
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// 2. Liveness probe, only once the session is old enough to have registered.
|
|
87
|
+
const startMs = sessionStartMs(sessionInfo);
|
|
88
|
+
const ageMs = startMs != null ? Date.now() - startMs : Infinity;
|
|
89
|
+
if (ageMs >= STALE_EXECUTING_MIN_AGE_MS && sessionInfo?.isolationBackend) {
|
|
90
|
+
const probe = backendAlive || runner.checkBackendSessionAlive;
|
|
91
|
+
const alive = probe ? await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose) : null;
|
|
92
|
+
// Only `false` (definitively gone) counts as killed; `null` (unknown backend)
|
|
93
|
+
// is treated as "no signal" so we don't kill on an indeterminate probe.
|
|
94
|
+
if (alive === false) {
|
|
95
|
+
if (isDockerIsolation(sessionInfo, statusResult)) {
|
|
96
|
+
const nowMs = Date.now();
|
|
97
|
+
const firstSeenMs = getDockerBackendGoneFirstSeenMs(sessionInfo);
|
|
98
|
+
if (firstSeenMs === null) {
|
|
99
|
+
sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
100
|
+
persistSnapshot();
|
|
101
|
+
if (verbose) {
|
|
102
|
+
console.log(`[VERBOSE] Session ${sessionName} docker backend is gone but no terminal status/footer is available yet; deferring killed classification for ${DOCKER_BACKEND_GONE_GRACE_MS}ms`);
|
|
103
|
+
}
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
if (nowMs - firstSeenMs < DOCKER_BACKEND_GONE_GRACE_MS) {
|
|
107
|
+
if (verbose) {
|
|
108
|
+
console.log(`[VERBOSE] Session ${sessionName} docker backend is still gone; waiting for terminal status/footer before reporting killed`);
|
|
109
|
+
}
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return { exitCode: null, status: 'killed', reason: 'backend-gone' };
|
|
114
|
+
}
|
|
115
|
+
if (alive === true) {
|
|
116
|
+
clearDockerBackendGoneMarker(sessionInfo, persistSnapshot);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Issue #2015: `oomKilled` is terminal — the container was killed by the kernel,
|
|
125
|
+
* so no further polling can change the outcome.
|
|
126
|
+
*/
|
|
127
|
+
export function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
|
|
128
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
129
|
+
let footer = null;
|
|
130
|
+
if (logPath) {
|
|
131
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
132
|
+
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
136
|
+
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
137
|
+
let exitCode = 137;
|
|
138
|
+
if (statusExitCode !== null && statusExitCode > 0) {
|
|
139
|
+
exitCode = statusExitCode;
|
|
140
|
+
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
141
|
+
exitCode = footerExitCode;
|
|
142
|
+
}
|
|
143
|
+
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
144
|
+
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
145
|
+
|
|
146
|
+
if (verbose) {
|
|
147
|
+
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
151
|
+
}
|
|
@@ -1,20 +1,35 @@
|
|
|
1
1
|
export async function finalizeSolveProcess({ tempDir, argv, limitReached, path, getLogFile, log, closeSentry, logActiveHandles, cleanupTempDirectory, safeExit }) {
|
|
2
|
-
|
|
2
|
+
const runFinalizationStep = async (label, step) => {
|
|
3
|
+
try {
|
|
4
|
+
await step();
|
|
5
|
+
} catch (error) {
|
|
6
|
+
const message = error?.message || String(error);
|
|
7
|
+
try {
|
|
8
|
+
await log(`⚠️ Finalization step failed (${label}): ${message}`, { level: 'warning' });
|
|
9
|
+
} catch {
|
|
10
|
+
console.warn(`⚠️ Finalization step failed (${label}): ${message}`);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
};
|
|
3
14
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
15
|
+
await runFinalizationStep('temporary directory cleanup', () => cleanupTempDirectory(tempDir, argv, limitReached));
|
|
16
|
+
|
|
17
|
+
await runFinalizationStep('final log reference', async () => {
|
|
18
|
+
// Show final log file reference so users always know where to find the complete log
|
|
19
|
+
if (getLogFile()) {
|
|
20
|
+
const finalLogPath = path.resolve(getLogFile());
|
|
21
|
+
await log(`\n📁 Complete log file: ${finalLogPath}`);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
9
24
|
|
|
10
25
|
// Issue #1346: Flush Sentry events before exit.
|
|
11
26
|
// closeSentry() uses a hard Promise.race deadline so it cannot block indefinitely.
|
|
12
|
-
await closeSentry
|
|
27
|
+
await runFinalizationStep('Sentry close', closeSentry);
|
|
13
28
|
|
|
14
29
|
// Issue #1431: Log active handles before draining.
|
|
15
30
|
// Always logged to file and console so future hangs are immediately visible in logs.
|
|
16
31
|
// drainHandles() inside safeExit() will unref/close these before process.exit().
|
|
17
|
-
await logActiveHandles(msg => log(msg));
|
|
32
|
+
await runFinalizationStep('active handle diagnostics', () => logActiveHandles(msg => log(msg)));
|
|
18
33
|
|
|
19
34
|
// Issue #1431: safeExit() unrefs handles so the event loop exits naturally, then calls process.exit(0)
|
|
20
35
|
await safeExit(0, 'Process completed');
|
package/src/solve.mjs
CHANGED
|
@@ -160,7 +160,7 @@ const cleanupWrapper = async () => {
|
|
|
160
160
|
};
|
|
161
161
|
const interruptWrapper = createInterruptWrapper({ cleanupContext, checkForUncommittedChanges, shouldAttachLogs, attachLogToGitHub, getLogFile, sanitizeLogContent, $, log });
|
|
162
162
|
initializeExitHandler(getAbsoluteLogPath, log, cleanupWrapper, interruptWrapper, ({ code, reason, failureActionSection }) => notifyIssueAboutPrePullRequestFailure({ code, reason, failureActionSection, argv, globalState: global, $, log, getLogFile, shouldAttachLogs, attachLogToGitHub, sanitizeLogContent, rawCommand }));
|
|
163
|
-
installGlobalExitHandlers();
|
|
163
|
+
installGlobalExitHandlers({ handleProcessErrors: false }); // #2117: solve's richer process-error handlers below must not race a duplicate pair.
|
|
164
164
|
// Issue #1823: Configure the working-session guard. When the experimental
|
|
165
165
|
// --do-not-shutdown-in-the-middle-of-working-session flag is set (hive passes it to every
|
|
166
166
|
// worker), an interrupt received during an AI working session is deferred: solve lets the AI
|
|
@@ -128,7 +128,7 @@ export function appendPullRequestLine(infoBlock, pullRequestUrl, { locale = null
|
|
|
128
128
|
return [...before, prLine, ...after].join('\n');
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
-
export function formatSessionCompletionMessage({ sessionName, sessionInfo, statusResult = null, observedEndTime = new Date(), exitCode = null, infoBlock = '', pullRequestUrl = null, extraSections = [], locale = null } = {}) {
|
|
131
|
+
export function formatSessionCompletionMessage({ sessionName, sessionInfo, statusResult = null, observedEndTime = new Date(), exitCode = null, infoBlock = '', pullRequestUrl = null, pullRequestState = null, extraSections = [], locale = null } = {}) {
|
|
132
132
|
const finalExitCode = getSessionCompletionExitCode({ exitCode, statusResult });
|
|
133
133
|
const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: statusResult?.status || null });
|
|
134
134
|
const { failed, killed, signal } = outcome;
|
|
@@ -142,6 +142,7 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
|
|
|
142
142
|
// is an orderly, intentional termination, so surface it as such regardless of
|
|
143
143
|
// which signal actually delivered the kill.
|
|
144
144
|
const stopRequestedByUser = Boolean(sessionInfo?.stopRequestedByUser);
|
|
145
|
+
const pullRequestMerged = pullRequestState?.merged === true || Boolean(pullRequestState?.mergedAt);
|
|
145
146
|
let statusEmojiOverride = null;
|
|
146
147
|
let statusText;
|
|
147
148
|
if (killed && stopRequestedByUser) {
|
|
@@ -158,6 +159,14 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
|
|
|
158
159
|
const exitSuffix = showCode ? ` (exit code: ${finalExitCode})` : '';
|
|
159
160
|
const reason = signal ? signal.reason : 'killed';
|
|
160
161
|
statusText = text(messageLocale, 'telegram.work_session_killed', `Work session ${reason}${exitSuffix}`, { reason, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '', exitSuffix });
|
|
162
|
+
} else if (failed && pullRequestMerged) {
|
|
163
|
+
// Issue #2117: the runner's exit code is still authoritative and must not
|
|
164
|
+
// be hidden, but calling the entire session "failed" contradicts the
|
|
165
|
+
// externally verified result when its pull request has already merged.
|
|
166
|
+
// Describe both outcomes so operators know the requested goal completed
|
|
167
|
+
// and that a later orchestration failure still needs investigation.
|
|
168
|
+
statusEmojiOverride = '⚠️';
|
|
169
|
+
statusText = text(messageLocale, 'telegram.work_session_merged_but_failed', `Pull request merged, but the work session exited with code: ${finalExitCode}`, { exitCode: finalExitCode });
|
|
161
170
|
} else if (failed) {
|
|
162
171
|
statusText = text(messageLocale, 'telegram.work_session_failed', `Work session failed (exit code: ${finalExitCode})`, { exitCode: finalExitCode });
|
|
163
172
|
} else {
|
|
@@ -183,7 +192,8 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
|
|
|
183
192
|
|
|
184
193
|
// Issue #594: --show-limits virtual option appends snapshot/delta sections
|
|
185
194
|
// (Markdown code blocks) below the standard completion details.
|
|
186
|
-
const
|
|
195
|
+
const mergedWithRunnerFailureSection = failed && !killed && pullRequestMerged ? [`✅ ${text(messageLocale, 'telegram.work_session_merged_success', 'The requested pull request was merged successfully.')}`, `⚠️ ${text(messageLocale, 'telegram.work_session_runner_also_failed', 'The runner also failed; its exit code is preserved for investigation.')}`].join('\n') : null;
|
|
196
|
+
const extras = [mergedWithRunnerFailureSection, ...(Array.isArray(extraSections) ? extraSections : [])].filter(Boolean);
|
|
187
197
|
if (extras.length > 0) {
|
|
188
198
|
message += `\n\n${extras.join('\n\n')}`;
|
|
189
199
|
}
|