@link-assistant/hive-mind 2.10.4 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.hi.md +5 -0
- package/README.md +5 -0
- package/README.ru.md +5 -0
- package/README.zh.md +4 -0
- package/package.json +1 -1
- package/src/exit-handler.lib.mjs +61 -44
- package/src/fix.ci-cd-issue.lib.mjs +137 -0
- package/src/fix.mjs +6 -116
- package/src/github-pr-state.lib.mjs +56 -0
- package/src/locales/en.lino +6 -3
- package/src/locales/hi.lino +6 -3
- package/src/locales/ru.lino +6 -3
- package/src/locales/zh.lino +6 -3
- package/src/session-monitor.docker-terminal.lib.mjs +118 -0
- package/src/session-monitor.lib.mjs +71 -158
- package/src/session-monitor.stale-executing.lib.mjs +151 -0
- package/src/solve.finalize.lib.mjs +23 -8
- package/src/solve.mjs +1 -1
- package/src/telegram-task-command.lib.mjs +47 -1
- package/src/work-session-formatting.lib.mjs +12 -2
package/src/locales/zh.lino
CHANGED
|
@@ -556,9 +556,9 @@ zh
|
|
|
556
556
|
locked
|
|
557
557
|
options "🔒 锁定选项:`{{options}}`"
|
|
558
558
|
task
|
|
559
|
-
enabled "*/task* -
|
|
560
|
-
usage "用法:`/task <github-repository-url>` 后接 issue
|
|
561
|
-
example "示例:`/task https://github.com/owner/repo
|
|
559
|
+
enabled "*/task* - 根据文本或 CI/CD 上下文创建 GitHub issue"
|
|
560
|
+
usage "用法:`/task <github-repository-url>` 后接 issue 文本,或 `/task --ci-cd <github-repository-url>`"
|
|
561
|
+
example "示例:`/task --ci-cd https://github.com/owner/repo` 只创建 CI/CD 修复 issue"
|
|
562
562
|
disabled "*/task* / */split* - ❌ 已禁用"
|
|
563
563
|
split
|
|
564
564
|
enabled "*/split* - 将 GitHub issue 拆分为更小的 issue"
|
|
@@ -649,6 +649,9 @@ zh
|
|
|
649
649
|
executing "⏳ 正在执行..."
|
|
650
650
|
finished "工作会话已成功完成"
|
|
651
651
|
failed "工作会话失败(退出代码:{{exitCode}})"
|
|
652
|
+
merged_but_failed "拉取请求已合并,但工作会话退出,代码为:{{exitCode}}"
|
|
653
|
+
merged_success "请求的拉取请求已成功合并。"
|
|
654
|
+
runner_also_failed "运行器也失败了;其退出代码已保留以供调查。"
|
|
652
655
|
killed "工作会话已终止:{{reason}}{{exitSuffix}}"
|
|
653
656
|
stopped "工作会话已由用户停止{{requestedBy}}{{exitSuffix}}"
|
|
654
657
|
duration
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue #2117: guard against a *fabricated* terminal exit code for detached
|
|
3
|
+
* docker sessions.
|
|
4
|
+
*
|
|
5
|
+
* start-command derives the exit code of a detached docker session from an
|
|
6
|
+
* unanchored `Exit Code: N` scan over the whole execution log
|
|
7
|
+
* (`status-formatter.js#readExitCodeFromLog`), so any text the wrapped command
|
|
8
|
+
* prints — for example an AI agent echoing the tail of an older execution log —
|
|
9
|
+
* can be mistaken for the terminal footer. That is exactly how a `/solve` run
|
|
10
|
+
* that merged its pull request and exited 0 was announced as "Work session
|
|
11
|
+
* failed (exit code: 1)": the fabricated code entered the log 21 minutes before
|
|
12
|
+
* the command finished, and `$ --status` stamped the record with it the moment
|
|
13
|
+
* the container stopped — about two seconds before the real footer
|
|
14
|
+
* (`Exit Code: 0`) was appended by start-command's detached-docker watcher.
|
|
15
|
+
*
|
|
16
|
+
* The watcher always appends that footer right after the container exits, so a
|
|
17
|
+
* short deferral is enough for the authoritative value to appear. If it never
|
|
18
|
+
* does (e.g. the watcher itself was killed), the reported status is accepted
|
|
19
|
+
* once the grace period expires, preserving the previous behaviour: a real
|
|
20
|
+
* failure is still reported, just a minute later.
|
|
21
|
+
*
|
|
22
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2117
|
|
23
|
+
* @see https://github.com/link-foundation/start/issues/150
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** How long an uncorroborated docker terminal failure stays provisional. */
|
|
27
|
+
export const DOCKER_TERMINAL_FOOTER_GRACE_MS = 60 * 1000;
|
|
28
|
+
|
|
29
|
+
/** Session-snapshot field holding the first sighting of such a status. */
|
|
30
|
+
export const DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD = 'dockerTerminalUnverifiedFirstSeenAt';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Whether the reported end of the session is recent enough for the fabrication
|
|
34
|
+
* race to still be open.
|
|
35
|
+
*
|
|
36
|
+
* start-command only invents a terminal result for a record it still has stored
|
|
37
|
+
* as `executing`, and it stamps that invention with `endTime = new Date()`
|
|
38
|
+
* (`status-formatter.js#enrichDetachedStatus`) — so a fabricated failure always
|
|
39
|
+
* looks like it ended just now, no matter how long ago the container really
|
|
40
|
+
* stopped. A record that reports an end time older than the grace period, on the
|
|
41
|
+
* other hand, has had all the time it needed for the footer to be written;
|
|
42
|
+
* deferring it would only postpone a real failure notification.
|
|
43
|
+
*
|
|
44
|
+
* @param {string|Date|null|undefined} endTime - End time from the status record.
|
|
45
|
+
* @param {number} nowMs - Current epoch milliseconds.
|
|
46
|
+
* @returns {boolean} True when the race cannot be ruled out.
|
|
47
|
+
*/
|
|
48
|
+
function terminalClaimIsRecent(endTime, nowMs) {
|
|
49
|
+
if (!endTime) return true; // No timestamp at all: the race cannot be ruled out.
|
|
50
|
+
const endMs = endTime instanceof Date ? endTime.getTime() : new Date(endTime).getTime();
|
|
51
|
+
if (!Number.isFinite(endMs)) return true;
|
|
52
|
+
return nowMs - endMs < DOCKER_TERMINAL_FOOTER_GRACE_MS;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Drop the deferral marker once the session's outcome is resolved (footer
|
|
57
|
+
* written, session running again, or the status accepted).
|
|
58
|
+
*
|
|
59
|
+
* @param {object|null|undefined} sessionInfo - Tracked session snapshot (mutated).
|
|
60
|
+
* @param {function} persistSnapshot - Callback persisting the updated snapshot.
|
|
61
|
+
*/
|
|
62
|
+
export function clearUnverifiedDockerTerminalMarker(sessionInfo, persistSnapshot) {
|
|
63
|
+
if (!sessionInfo?.[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD]) return;
|
|
64
|
+
delete sessionInfo[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD];
|
|
65
|
+
persistSnapshot();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Decide whether a docker terminal *failure* status that the log footer does not
|
|
70
|
+
* corroborate should be deferred (treated as still running) for now. The first
|
|
71
|
+
* sighting is persisted so the deferral survives a bot restart and cannot be
|
|
72
|
+
* reset forever by repeated polling.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} sessionName - Session identifier, for verbose output only.
|
|
75
|
+
* @param {object|null|undefined} sessionInfo - Tracked session snapshot (mutated).
|
|
76
|
+
* @param {object} options
|
|
77
|
+
* @param {number|null} options.exitCode - Exit code `$ --status` reported.
|
|
78
|
+
* @param {string|Date|null} [options.endTime] - End time the status record reports.
|
|
79
|
+
* @param {boolean} options.verbose - Whether to explain the decision.
|
|
80
|
+
* @param {function} options.persistSnapshot - Callback persisting the snapshot.
|
|
81
|
+
* @returns {boolean} True while the status is still provisional.
|
|
82
|
+
*/
|
|
83
|
+
export function shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime = null, verbose, persistSnapshot }) {
|
|
84
|
+
const nowMs = Date.now();
|
|
85
|
+
|
|
86
|
+
if (!terminalClaimIsRecent(endTime, nowMs)) {
|
|
87
|
+
if (verbose) {
|
|
88
|
+
console.log(`[VERBOSE] Session ${sessionName} reports a terminal failure (exit ${exitCode}) that ended at ${endTime}, longer than ${DOCKER_TERMINAL_FOOTER_GRACE_MS}ms ago; the log footer had time to appear, so the reported exit code is accepted (issue #2117)`);
|
|
89
|
+
}
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const raw = sessionInfo?.[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD];
|
|
94
|
+
const firstSeenMs = raw ? new Date(raw).getTime() : null;
|
|
95
|
+
|
|
96
|
+
if (firstSeenMs === null || !Number.isFinite(firstSeenMs)) {
|
|
97
|
+
if (sessionInfo) {
|
|
98
|
+
sessionInfo[DOCKER_TERMINAL_UNVERIFIED_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
99
|
+
persistSnapshot();
|
|
100
|
+
}
|
|
101
|
+
if (verbose) {
|
|
102
|
+
console.log(`[VERBOSE] Session ${sessionName} reports a terminal failure (exit ${exitCode}) that no log footer corroborates; deferring for up to ${DOCKER_TERMINAL_FOOTER_GRACE_MS}ms until start-command writes the real footer (issue #2117)`);
|
|
103
|
+
}
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (nowMs - firstSeenMs < DOCKER_TERMINAL_FOOTER_GRACE_MS) {
|
|
108
|
+
if (verbose) {
|
|
109
|
+
console.log(`[VERBOSE] Session ${sessionName} still reports an unverified terminal failure (exit ${exitCode}); waiting for the log footer (issue #2117)`);
|
|
110
|
+
}
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (verbose) {
|
|
115
|
+
console.log(`[VERBOSE] Session ${sessionName} kept reporting a terminal failure (exit ${exitCode}) for ${DOCKER_TERMINAL_FOOTER_GRACE_MS}ms without a log footer; accepting the reported exit code (issue #2117)`);
|
|
116
|
+
}
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
@@ -26,8 +26,15 @@ import { formatSessionCompletionMessage, getSessionCompletionExitCode, classifyS
|
|
|
26
26
|
import { notifySubscribers, getSubscriberCount } from './telegram-subscribers.lib.mjs';
|
|
27
27
|
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
28
28
|
import { readLastSessionIdFromLog, buildResumeCommand, formatResumeSection } from './session-resume.lib.mjs';
|
|
29
|
+
import { resolveFailedSessionPullRequestState } from './github-pr-state.lib.mjs';
|
|
30
|
+
// Issue #2117: a docker terminal failure that no anchored log footer corroborates
|
|
31
|
+
// may be an exit code start-command fabricated from the command's own output.
|
|
32
|
+
import { clearUnverifiedDockerTerminalMarker as clearUnverifiedDockerTerminalMarkerImpl, shouldDeferUnverifiedDockerTerminal as shouldDeferUnverifiedDockerTerminalImpl } from './session-monitor.docker-terminal.lib.mjs';
|
|
33
|
+
import { isDockerIsolation, sessionStartMs, resolveOomKilledState, resolveStaleExecutingState as resolveStaleExecutingStateImpl } from './session-monitor.stale-executing.lib.mjs';
|
|
29
34
|
|
|
30
35
|
export { formatSessionCompletionMessage, getSessionCompletionExitCode } from './work-session-formatting.lib.mjs';
|
|
36
|
+
export { DOCKER_TERMINAL_FOOTER_GRACE_MS } from './session-monitor.docker-terminal.lib.mjs';
|
|
37
|
+
export { STALE_EXECUTING_MIN_AGE_MS, DOCKER_BACKEND_GONE_GRACE_MS } from './session-monitor.stale-executing.lib.mjs';
|
|
31
38
|
|
|
32
39
|
const exec = promisify(execCallback);
|
|
33
40
|
|
|
@@ -521,134 +528,16 @@ function isNonIsolationSessionActive(sessionName, sessionInfo, verbose = false)
|
|
|
521
528
|
return true;
|
|
522
529
|
}
|
|
523
530
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
* `executing` is allowed to be declared dead purely on a backend-liveness probe
|
|
527
|
-
* (the screen/tmux/docker session is gone). This avoids a race where a session
|
|
528
|
-
* that has just been launched — but whose backend has not registered yet — is
|
|
529
|
-
* falsely reported as killed. The authoritative log-footer check is NOT gated by
|
|
530
|
-
* this, because a written "Exit Code:" footer is proof the command terminated.
|
|
531
|
-
*/
|
|
532
|
-
export const STALE_EXECUTING_MIN_AGE_MS = 90 * 1000;
|
|
533
|
-
export const DOCKER_BACKEND_GONE_GRACE_MS = 2 * 60 * 1000;
|
|
534
|
-
const DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD = 'dockerBackendGoneFirstSeenAt';
|
|
535
|
-
|
|
536
|
-
function sessionStartMs(sessionInfo) {
|
|
537
|
-
const start = sessionInfo?.startTime;
|
|
538
|
-
if (!start) return null;
|
|
539
|
-
const date = start instanceof Date ? start : new Date(start);
|
|
540
|
-
const ms = date.getTime();
|
|
541
|
-
return Number.isFinite(ms) ? ms : null;
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
function isDockerIsolation(sessionInfo, statusResult) {
|
|
545
|
-
return sessionInfo?.isolationBackend === 'docker' || statusResult?.isolation === 'docker';
|
|
531
|
+
function clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo) {
|
|
532
|
+
clearUnverifiedDockerTerminalMarkerImpl(sessionInfo, () => persistSessionSnapshot(sessionName, sessionInfo));
|
|
546
533
|
}
|
|
547
534
|
|
|
548
|
-
function
|
|
549
|
-
|
|
550
|
-
if (!raw) return null;
|
|
551
|
-
const ms = new Date(raw).getTime();
|
|
552
|
-
return Number.isFinite(ms) ? ms : null;
|
|
535
|
+
function shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime, verbose }) {
|
|
536
|
+
return shouldDeferUnverifiedDockerTerminalImpl(sessionName, sessionInfo, { exitCode, endTime, verbose, persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo) });
|
|
553
537
|
}
|
|
554
538
|
|
|
555
|
-
function
|
|
556
|
-
|
|
557
|
-
delete sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
558
|
-
persistSessionSnapshot(sessionName, sessionInfo);
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
/**
|
|
562
|
-
* Cross-check whether a session that `$ --status` still reports as `executing`
|
|
563
|
-
* has actually terminated. Issue #1927: start-command's status can get stuck on
|
|
564
|
-
* `executing` after the process was killed (a lingering shell keeps the screen
|
|
565
|
-
* session alive, flipping executed→executing), so a SIGKILLed /solve was never
|
|
566
|
-
* reported. Two independent signals are consulted, strongest first:
|
|
567
|
-
*
|
|
568
|
-
* 1. The execution log FOOTER. When start-command wrote "Exit Code: N" the
|
|
569
|
-
* command terminated, full stop — regardless of what `--status` claims.
|
|
570
|
-
* This is authoritative and catches the dominant lingering-shell case.
|
|
571
|
-
* 2. Backend LIVENESS. If no footer was written (e.g. the wrapper itself was
|
|
572
|
-
* hard-killed) but the backing screen/tmux/docker session is gone, the
|
|
573
|
-
* process cannot still be executing. Gated by STALE_EXECUTING_MIN_AGE_MS to
|
|
574
|
-
* avoid a just-launched-not-yet-registered race.
|
|
575
|
-
*
|
|
576
|
-
* @returns {Promise<{exitCode: number|null, status: string, reason: string}|null>}
|
|
577
|
-
* Terminal details when the session is actually dead, else null (still running).
|
|
578
|
-
*/
|
|
579
|
-
async function resolveStaleExecutingState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive }) {
|
|
580
|
-
// 1. Authoritative: the log footer.
|
|
581
|
-
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
582
|
-
if (logPath) {
|
|
583
|
-
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
584
|
-
const footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
585
|
-
if (footer?.finished) {
|
|
586
|
-
const status = classifyExitStatus(footer.exitCode) || (footer.exitCode === 0 ? 'executed' : 'failed');
|
|
587
|
-
return { exitCode: footer.exitCode, status, reason: `log-footer(exit ${footer.exitCode})` };
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
// 2. Liveness probe, only once the session is old enough to have registered.
|
|
592
|
-
const startMs = sessionStartMs(sessionInfo);
|
|
593
|
-
const ageMs = startMs != null ? Date.now() - startMs : Infinity;
|
|
594
|
-
if (ageMs >= STALE_EXECUTING_MIN_AGE_MS && sessionInfo?.isolationBackend) {
|
|
595
|
-
const probe = backendAlive || runner.checkBackendSessionAlive;
|
|
596
|
-
const alive = probe ? await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose) : null;
|
|
597
|
-
// Only `false` (definitively gone) counts as killed; `null` (unknown backend)
|
|
598
|
-
// is treated as "no signal" so we don't kill on an indeterminate probe.
|
|
599
|
-
if (alive === false) {
|
|
600
|
-
if (isDockerIsolation(sessionInfo, statusResult)) {
|
|
601
|
-
const nowMs = Date.now();
|
|
602
|
-
const firstSeenMs = getDockerBackendGoneFirstSeenMs(sessionInfo);
|
|
603
|
-
if (firstSeenMs === null) {
|
|
604
|
-
sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
605
|
-
persistSessionSnapshot(sessionName, sessionInfo);
|
|
606
|
-
if (verbose) {
|
|
607
|
-
console.log(`[VERBOSE] Session ${sessionName} docker backend is gone but no terminal status/footer is available yet; deferring killed classification for ${DOCKER_BACKEND_GONE_GRACE_MS}ms`);
|
|
608
|
-
}
|
|
609
|
-
return null;
|
|
610
|
-
}
|
|
611
|
-
if (nowMs - firstSeenMs < DOCKER_BACKEND_GONE_GRACE_MS) {
|
|
612
|
-
if (verbose) {
|
|
613
|
-
console.log(`[VERBOSE] Session ${sessionName} docker backend is still gone; waiting for terminal status/footer before reporting killed`);
|
|
614
|
-
}
|
|
615
|
-
return null;
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
return { exitCode: null, status: 'killed', reason: 'backend-gone' };
|
|
619
|
-
}
|
|
620
|
-
if (alive === true) {
|
|
621
|
-
clearDockerBackendGoneMarker(sessionName, sessionInfo);
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
return null;
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
|
|
629
|
-
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
630
|
-
let footer = null;
|
|
631
|
-
if (logPath) {
|
|
632
|
-
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
633
|
-
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
637
|
-
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
638
|
-
let exitCode = 137;
|
|
639
|
-
if (statusExitCode !== null && statusExitCode > 0) {
|
|
640
|
-
exitCode = statusExitCode;
|
|
641
|
-
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
642
|
-
exitCode = footerExitCode;
|
|
643
|
-
}
|
|
644
|
-
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
645
|
-
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
646
|
-
|
|
647
|
-
if (verbose) {
|
|
648
|
-
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
539
|
+
function resolveStaleExecutingState(sessionName, sessionInfo, statusResult, options) {
|
|
540
|
+
return resolveStaleExecutingStateImpl(sessionName, sessionInfo, statusResult, { ...options, persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo) });
|
|
652
541
|
}
|
|
653
542
|
|
|
654
543
|
async function getIsolationSessionState(sessionName, sessionInfo, options = {}) {
|
|
@@ -678,25 +567,30 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
678
567
|
const corrected = { ...statusResult, status: correctedStatus, exitCode: stale.exitCode, endTime: statusResult.endTime || stale.endTime || null };
|
|
679
568
|
return { running: false, exitCode: stale.exitCode, status: correctedStatus, statusResult: corrected, stale: true };
|
|
680
569
|
}
|
|
570
|
+
// Back to a plain `executing` report: any earlier unverified terminal
|
|
571
|
+
// failure was provisional and is now moot (issue #2117).
|
|
572
|
+
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
681
573
|
return { running: true, exitCode: null, status: statusResult.status, statusResult };
|
|
682
574
|
}
|
|
683
575
|
if (runner.isTerminalSessionStatus(statusResult.status)) {
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
//
|
|
687
|
-
//
|
|
688
|
-
//
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
576
|
+
const exitCode = statusResult.exitCode !== undefined ? statusResult.exitCode : null;
|
|
577
|
+
const logPath = statusResult.logPath || sessionInfo?.logPath || null;
|
|
578
|
+
// The log FOOTER is the authoritative terminal result. It is anchored on
|
|
579
|
+
// the `=====` separator (see parseSessionExitFooter), so — unlike the
|
|
580
|
+
// exit code `$ --status` derives from an unanchored full-log scan — it
|
|
581
|
+
// cannot be forged by output the wrapped command printed (issue #2117).
|
|
582
|
+
// Prefer it whenever it exists: that both recovers a real code from a
|
|
583
|
+
// missing/sentinel status (issue #1927) and overrides a fabricated one.
|
|
584
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
585
|
+
const footer = logPath && readFooter ? readFooter(logPath, { verbose }) : null;
|
|
586
|
+
if (footer?.finished) {
|
|
587
|
+
const footerExitCode = footer.exitCode;
|
|
588
|
+
const correctedStatus = classifyExitStatus(footerExitCode) || statusResult.status;
|
|
589
|
+
if (verbose && normalizeExitCode(footerExitCode) !== normalizeExitCode(exitCode)) {
|
|
590
|
+
console.log(`[VERBOSE] Session ${sessionName} reported terminal '${statusResult.status}' with exit ${exitCode}; the log footer says exit ${footerExitCode} (${correctedStatus}) and wins (issues #1927/#2117)`);
|
|
699
591
|
}
|
|
592
|
+
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
593
|
+
return { running: false, exitCode: footerExitCode, status: correctedStatus, statusResult: { ...statusResult, status: correctedStatus, exitCode: footerExitCode } };
|
|
700
594
|
}
|
|
701
595
|
// Issue #1939: a native docker session can report a terminal status
|
|
702
596
|
// ("executed") with the unknown exit-code sentinel (-1) while the
|
|
@@ -704,8 +598,21 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
704
598
|
// a real terminal exit, such a status is provisional — fall through to
|
|
705
599
|
// isSessionRunning() below, which cross-checks the live container via
|
|
706
600
|
// `docker inspect` before we notify the user the work finished.
|
|
707
|
-
const
|
|
601
|
+
const dockerSession = isDockerIsolation(sessionInfo, statusResult);
|
|
602
|
+
const ambiguousDockerTerminal = dockerSession && typeof runner.isUnknownDockerExitCode === 'function' && runner.isUnknownDockerExitCode(exitCode);
|
|
603
|
+
// Issue #2117: a docker terminal FAILURE with no corroborating footer is
|
|
604
|
+
// provisional too — start-command can fabricate that exit code from the
|
|
605
|
+
// command's own output. Give the real footer a moment to appear instead
|
|
606
|
+
// of announcing a failure the run never had. Only a *freshly* reported
|
|
607
|
+
// end time can still be in that race, so an older terminal record is
|
|
608
|
+
// still reported without delay.
|
|
609
|
+
const normalizedExitCode = normalizeExitCode(exitCode);
|
|
610
|
+
const unverifiedDockerFailure = dockerSession && !ambiguousDockerTerminal && normalizedExitCode !== null && normalizedExitCode !== 0;
|
|
611
|
+
if (unverifiedDockerFailure && shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime: statusResult.endTime || null, verbose })) {
|
|
612
|
+
return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
|
|
613
|
+
}
|
|
708
614
|
if (!ambiguousDockerTerminal) {
|
|
615
|
+
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
709
616
|
return { running: false, exitCode, status: statusResult.status, statusResult };
|
|
710
617
|
}
|
|
711
618
|
}
|
|
@@ -844,11 +751,8 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
844
751
|
verbose,
|
|
845
752
|
});
|
|
846
753
|
|
|
847
|
-
// Issue #1688/#1905:
|
|
848
|
-
//
|
|
849
|
-
// `Issue:` and a `Pull request:` line. The linked-issue API can lag
|
|
850
|
-
// behind the solver's own verification log, so we also inspect the
|
|
851
|
-
// completed session log before giving up.
|
|
754
|
+
// Issue #1688/#1905: Resolve the created PR from GitHub or, when its
|
|
755
|
+
// linked-issue API lags, from the completed solve log.
|
|
852
756
|
let pullRequestUrl = null;
|
|
853
757
|
try {
|
|
854
758
|
pullRequestUrl = await resolvePullRequestUrlForSession(sessionInfo, {
|
|
@@ -863,10 +767,25 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
863
767
|
}
|
|
864
768
|
}
|
|
865
769
|
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
770
|
+
let pullRequestState = null;
|
|
771
|
+
const completionOutcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
|
|
772
|
+
try {
|
|
773
|
+
pullRequestState = await resolveFailedSessionPullRequestState({
|
|
774
|
+
pullRequestUrl,
|
|
775
|
+
outcome: completionOutcome,
|
|
776
|
+
lookupPullRequestState: options.lookupPullRequestState,
|
|
777
|
+
verbose,
|
|
778
|
+
sessionName,
|
|
779
|
+
exitCode: finalExitCode,
|
|
780
|
+
status: resolvedStatus,
|
|
781
|
+
logPath: statusResult?.logPath || sessionInfo?.logPath,
|
|
782
|
+
});
|
|
783
|
+
} catch (stateError) {
|
|
784
|
+
if (verbose) console.log(`[VERBOSE] Pull request state resolution failed for ${sessionName}: ${stateError?.message || stateError}`);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
// Issue #594: append an end-of-task limits snapshot/delta. Cached
|
|
788
|
+
// helpers prevent parallel sessions from stampeding the upstream API.
|
|
870
789
|
const limitsExtraSections = [];
|
|
871
790
|
if (sessionInfo?.showLimits) {
|
|
872
791
|
try {
|
|
@@ -895,15 +814,8 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
895
814
|
}
|
|
896
815
|
}
|
|
897
816
|
|
|
898
|
-
// Issue #1927
|
|
899
|
-
//
|
|
900
|
-
// ready-to-run `--resume <lastSessionId>` command so the surviving
|
|
901
|
-
// parent (the operator, or an automation watching the bot) can pick the
|
|
902
|
-
// work back up. We deliberately do NOT auto-relaunch here: a job that
|
|
903
|
-
// reliably OOMs would storm. The rule "use the LAST of multiple
|
|
904
|
-
// sessions" is honored by reading the last `Session ID:` marker from
|
|
905
|
-
// the captured log. Purely additive — failures never block the
|
|
906
|
-
// completion notification, preserving backward compatibility.
|
|
817
|
+
// Issue #1927: for a killed /solve, offer a command using the last tool
|
|
818
|
+
// session ID in the log. Do not auto-relaunch work that may reliably OOM.
|
|
907
819
|
const resumeExtraSections = [];
|
|
908
820
|
try {
|
|
909
821
|
const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
|
|
@@ -961,6 +873,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
961
873
|
exitCode: finalExitCode,
|
|
962
874
|
infoBlock: sessionInfo?.infoBlock || '',
|
|
963
875
|
pullRequestUrl,
|
|
876
|
+
pullRequestState,
|
|
964
877
|
extraSections: [...limitsExtraSections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
965
878
|
});
|
|
966
879
|
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Terminal-state reconciliation helpers for tracked isolation sessions.
|
|
3
|
+
*
|
|
4
|
+
* These live next to session-monitor.lib.mjs (which is at its `max-lines`
|
|
5
|
+
* budget) and cover two independent defects:
|
|
6
|
+
*
|
|
7
|
+
* - issue #1927: `$ --status` can stay stuck on `executing` after the process
|
|
8
|
+
* was killed, so the monitor has to cross-check the log footer and the
|
|
9
|
+
* backing screen/tmux/docker session.
|
|
10
|
+
* - issue #2015: a docker session reported with `oomKilled=true` is terminal
|
|
11
|
+
* and must be surfaced as such instead of being polled forever.
|
|
12
|
+
*
|
|
13
|
+
* @see https://github.com/link-assistant/hive-mind/issues/1927
|
|
14
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2015
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Issue #1927: minimum age before a session that `$ --status` still reports as
|
|
21
|
+
* `executing` is allowed to be declared dead purely on a backend-liveness probe
|
|
22
|
+
* (the screen/tmux/docker session is gone). This avoids a race where a session
|
|
23
|
+
* that has just been launched — but whose backend has not registered yet — is
|
|
24
|
+
* falsely reported as killed. The authoritative log-footer check is NOT gated by
|
|
25
|
+
* this, because a written "Exit Code:" footer is proof the command terminated.
|
|
26
|
+
*/
|
|
27
|
+
export const STALE_EXECUTING_MIN_AGE_MS = 90 * 1000;
|
|
28
|
+
export const DOCKER_BACKEND_GONE_GRACE_MS = 2 * 60 * 1000;
|
|
29
|
+
const DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD = 'dockerBackendGoneFirstSeenAt';
|
|
30
|
+
|
|
31
|
+
export function sessionStartMs(sessionInfo) {
|
|
32
|
+
const start = sessionInfo?.startTime;
|
|
33
|
+
if (!start) return null;
|
|
34
|
+
const date = start instanceof Date ? start : new Date(start);
|
|
35
|
+
const ms = date.getTime();
|
|
36
|
+
return Number.isFinite(ms) ? ms : null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function isDockerIsolation(sessionInfo, statusResult) {
|
|
40
|
+
return sessionInfo?.isolationBackend === 'docker' || statusResult?.isolation === 'docker';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function getDockerBackendGoneFirstSeenMs(sessionInfo) {
|
|
44
|
+
const raw = sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
45
|
+
if (!raw) return null;
|
|
46
|
+
const ms = new Date(raw).getTime();
|
|
47
|
+
return Number.isFinite(ms) ? ms : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function clearDockerBackendGoneMarker(sessionInfo, persistSnapshot) {
|
|
51
|
+
if (!sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD]) return;
|
|
52
|
+
delete sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
53
|
+
persistSnapshot();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Cross-check whether a session that `$ --status` still reports as `executing`
|
|
58
|
+
* has actually terminated. Issue #1927: start-command's status can get stuck on
|
|
59
|
+
* `executing` after the process was killed (a lingering shell keeps the screen
|
|
60
|
+
* session alive, flipping executed→executing), so a SIGKILLed /solve was never
|
|
61
|
+
* reported. Two independent signals are consulted, strongest first:
|
|
62
|
+
*
|
|
63
|
+
* 1. The execution log FOOTER. When start-command wrote "Exit Code: N" the
|
|
64
|
+
* command terminated, full stop — regardless of what `--status` claims.
|
|
65
|
+
* This is authoritative and catches the dominant lingering-shell case.
|
|
66
|
+
* 2. Backend LIVENESS. If no footer was written (e.g. the wrapper itself was
|
|
67
|
+
* hard-killed) but the backing screen/tmux/docker session is gone, the
|
|
68
|
+
* process cannot still be executing. Gated by STALE_EXECUTING_MIN_AGE_MS to
|
|
69
|
+
* avoid a just-launched-not-yet-registered race.
|
|
70
|
+
*
|
|
71
|
+
* @returns {Promise<{exitCode: number|null, status: string, reason: string}|null>}
|
|
72
|
+
* Terminal details when the session is actually dead, else null (still running).
|
|
73
|
+
*/
|
|
74
|
+
export async function resolveStaleExecutingState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive, persistSnapshot }) {
|
|
75
|
+
// 1. Authoritative: the log footer.
|
|
76
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
77
|
+
if (logPath) {
|
|
78
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
79
|
+
const footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
80
|
+
if (footer?.finished) {
|
|
81
|
+
const status = classifyExitStatus(footer.exitCode) || (footer.exitCode === 0 ? 'executed' : 'failed');
|
|
82
|
+
return { exitCode: footer.exitCode, status, reason: `log-footer(exit ${footer.exitCode})` };
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// 2. Liveness probe, only once the session is old enough to have registered.
|
|
87
|
+
const startMs = sessionStartMs(sessionInfo);
|
|
88
|
+
const ageMs = startMs != null ? Date.now() - startMs : Infinity;
|
|
89
|
+
if (ageMs >= STALE_EXECUTING_MIN_AGE_MS && sessionInfo?.isolationBackend) {
|
|
90
|
+
const probe = backendAlive || runner.checkBackendSessionAlive;
|
|
91
|
+
const alive = probe ? await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose) : null;
|
|
92
|
+
// Only `false` (definitively gone) counts as killed; `null` (unknown backend)
|
|
93
|
+
// is treated as "no signal" so we don't kill on an indeterminate probe.
|
|
94
|
+
if (alive === false) {
|
|
95
|
+
if (isDockerIsolation(sessionInfo, statusResult)) {
|
|
96
|
+
const nowMs = Date.now();
|
|
97
|
+
const firstSeenMs = getDockerBackendGoneFirstSeenMs(sessionInfo);
|
|
98
|
+
if (firstSeenMs === null) {
|
|
99
|
+
sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
100
|
+
persistSnapshot();
|
|
101
|
+
if (verbose) {
|
|
102
|
+
console.log(`[VERBOSE] Session ${sessionName} docker backend is gone but no terminal status/footer is available yet; deferring killed classification for ${DOCKER_BACKEND_GONE_GRACE_MS}ms`);
|
|
103
|
+
}
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
if (nowMs - firstSeenMs < DOCKER_BACKEND_GONE_GRACE_MS) {
|
|
107
|
+
if (verbose) {
|
|
108
|
+
console.log(`[VERBOSE] Session ${sessionName} docker backend is still gone; waiting for terminal status/footer before reporting killed`);
|
|
109
|
+
}
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return { exitCode: null, status: 'killed', reason: 'backend-gone' };
|
|
114
|
+
}
|
|
115
|
+
if (alive === true) {
|
|
116
|
+
clearDockerBackendGoneMarker(sessionInfo, persistSnapshot);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Issue #2015: `oomKilled` is terminal — the container was killed by the kernel,
|
|
125
|
+
* so no further polling can change the outcome.
|
|
126
|
+
*/
|
|
127
|
+
export function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
|
|
128
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
129
|
+
let footer = null;
|
|
130
|
+
if (logPath) {
|
|
131
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
132
|
+
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
136
|
+
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
137
|
+
let exitCode = 137;
|
|
138
|
+
if (statusExitCode !== null && statusExitCode > 0) {
|
|
139
|
+
exitCode = statusExitCode;
|
|
140
|
+
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
141
|
+
exitCode = footerExitCode;
|
|
142
|
+
}
|
|
143
|
+
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
144
|
+
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
145
|
+
|
|
146
|
+
if (verbose) {
|
|
147
|
+
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
151
|
+
}
|
|
@@ -1,20 +1,35 @@
|
|
|
1
1
|
export async function finalizeSolveProcess({ tempDir, argv, limitReached, path, getLogFile, log, closeSentry, logActiveHandles, cleanupTempDirectory, safeExit }) {
|
|
2
|
-
|
|
2
|
+
const runFinalizationStep = async (label, step) => {
|
|
3
|
+
try {
|
|
4
|
+
await step();
|
|
5
|
+
} catch (error) {
|
|
6
|
+
const message = error?.message || String(error);
|
|
7
|
+
try {
|
|
8
|
+
await log(`⚠️ Finalization step failed (${label}): ${message}`, { level: 'warning' });
|
|
9
|
+
} catch {
|
|
10
|
+
console.warn(`⚠️ Finalization step failed (${label}): ${message}`);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
};
|
|
3
14
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
15
|
+
await runFinalizationStep('temporary directory cleanup', () => cleanupTempDirectory(tempDir, argv, limitReached));
|
|
16
|
+
|
|
17
|
+
await runFinalizationStep('final log reference', async () => {
|
|
18
|
+
// Show final log file reference so users always know where to find the complete log
|
|
19
|
+
if (getLogFile()) {
|
|
20
|
+
const finalLogPath = path.resolve(getLogFile());
|
|
21
|
+
await log(`\n📁 Complete log file: ${finalLogPath}`);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
9
24
|
|
|
10
25
|
// Issue #1346: Flush Sentry events before exit.
|
|
11
26
|
// closeSentry() uses a hard Promise.race deadline so it cannot block indefinitely.
|
|
12
|
-
await closeSentry
|
|
27
|
+
await runFinalizationStep('Sentry close', closeSentry);
|
|
13
28
|
|
|
14
29
|
// Issue #1431: Log active handles before draining.
|
|
15
30
|
// Always logged to file and console so future hangs are immediately visible in logs.
|
|
16
31
|
// drainHandles() inside safeExit() will unref/close these before process.exit().
|
|
17
|
-
await logActiveHandles(msg => log(msg));
|
|
32
|
+
await runFinalizationStep('active handle diagnostics', () => logActiveHandles(msg => log(msg)));
|
|
18
33
|
|
|
19
34
|
// Issue #1431: safeExit() unrefs handles so the event loop exits naturally, then calls process.exit(0)
|
|
20
35
|
await safeExit(0, 'Process completed');
|
package/src/solve.mjs
CHANGED
|
@@ -160,7 +160,7 @@ const cleanupWrapper = async () => {
|
|
|
160
160
|
};
|
|
161
161
|
const interruptWrapper = createInterruptWrapper({ cleanupContext, checkForUncommittedChanges, shouldAttachLogs, attachLogToGitHub, getLogFile, sanitizeLogContent, $, log });
|
|
162
162
|
initializeExitHandler(getAbsoluteLogPath, log, cleanupWrapper, interruptWrapper, ({ code, reason, failureActionSection }) => notifyIssueAboutPrePullRequestFailure({ code, reason, failureActionSection, argv, globalState: global, $, log, getLogFile, shouldAttachLogs, attachLogToGitHub, sanitizeLogContent, rawCommand }));
|
|
163
|
-
installGlobalExitHandlers();
|
|
163
|
+
installGlobalExitHandlers({ handleProcessErrors: false }); // #2117: solve's richer process-error handlers below must not race a duplicate pair.
|
|
164
164
|
// Issue #1823: Configure the working-session guard. When the experimental
|
|
165
165
|
// --do-not-shutdown-in-the-middle-of-working-session flag is set (hive passes it to every
|
|
166
166
|
// worker), an interrupt received during an AI working session is deferred: solve lets the AI
|