@link-assistant/hive-mind 2.11.6 → 2.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/package.json +1 -1
- package/src/budget-stats-policy.lib.mjs +31 -0
- package/src/locales/en.lino +10 -0
- package/src/locales/hi.lino +10 -0
- package/src/locales/ru.lino +10 -0
- package/src/locales/zh.lino +10 -0
- package/src/option-suggestions.lib.mjs +2 -0
- package/src/session-kill-diagnostics.lib.mjs +388 -0
- package/src/session-kill-policy.lib.mjs +96 -0
- package/src/session-kill-recovery.lib.mjs +252 -0
- package/src/session-kill-resume.lib.mjs +175 -0
- package/src/session-monitor.kill-sections.lib.mjs +198 -0
- package/src/session-monitor.lib.mjs +97 -2
- package/src/session-monitor.oom.lib.mjs +148 -0
- package/src/session-monitor.stale-executing.lib.mjs +6 -27
- package/src/session-resume.lib.mjs +28 -2
- package/src/solve.auto-merge.lib.mjs +14 -15
- package/src/solve.config.lib.mjs +16 -2
- package/src/solve.mjs +0 -2
- package/src/solve.results.lib.mjs +29 -24
- package/src/solve.watch.lib.mjs +15 -15
- package/src/telegram-bot.mjs +3 -1
|
@@ -31,6 +31,9 @@ import { resolveFailedSessionPullRequestState } from './github-pr-state.lib.mjs'
|
|
|
31
31
|
// may be an exit code start-command fabricated from the command's own output.
|
|
32
32
|
import { clearUnverifiedDockerTerminalMarker as clearUnverifiedDockerTerminalMarkerImpl, shouldDeferUnverifiedDockerTerminal as shouldDeferUnverifiedDockerTerminalImpl } from './session-monitor.docker-terminal.lib.mjs';
|
|
33
33
|
import { isDockerIsolation, sessionStartMs, resolveOomKilledState, resolveStaleExecutingState as resolveStaleExecutingStateImpl } from './session-monitor.stale-executing.lib.mjs';
|
|
34
|
+
// Issue #2134: kill-cause diagnostics + the matching pull-request notice.
|
|
35
|
+
import { buildKillCompletionSections, announceKillOnPullRequest } from './session-monitor.kill-sections.lib.mjs';
|
|
36
|
+
import { runKillRecoveryForCompletion } from './session-kill-resume.lib.mjs';
|
|
34
37
|
|
|
35
38
|
export { formatSessionCompletionMessage, getSessionCompletionExitCode } from './work-session-formatting.lib.mjs';
|
|
36
39
|
export { DOCKER_TERMINAL_FOOTER_GRACE_MS } from './session-monitor.docker-terminal.lib.mjs';
|
|
@@ -550,7 +553,16 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
550
553
|
|
|
551
554
|
if (statusResult?.exists && statusResult.status) {
|
|
552
555
|
if (statusResult.oomKilled === true) {
|
|
553
|
-
|
|
556
|
+
// Issue #2134: `oomKilled` is a *container* flag — the kernel sets it when
|
|
557
|
+
// any process in the cgroup is OOM-killed — so it is verified against the
|
|
558
|
+
// log footer and container liveness before a kill is announced.
|
|
559
|
+
return await resolveOomKilledState(sessionName, sessionInfo, statusResult, {
|
|
560
|
+
verbose,
|
|
561
|
+
runner,
|
|
562
|
+
exitFromLog,
|
|
563
|
+
backendAlive,
|
|
564
|
+
persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
|
|
565
|
+
});
|
|
554
566
|
}
|
|
555
567
|
if (runner.isExecutingSessionStatus(statusResult.status)) {
|
|
556
568
|
// Issue #1927: an `executing` status is not trusted blindly — verify the
|
|
@@ -611,6 +623,27 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
611
623
|
if (unverifiedDockerFailure && shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime: statusResult.endTime || null, verbose })) {
|
|
612
624
|
return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
|
|
613
625
|
}
|
|
626
|
+
// Issue #2134: even after the grace window, a container that is verifiably
|
|
627
|
+
// still alive cannot have produced a terminal failure — the same liveness
|
|
628
|
+
// ladder used for `oomKilled` applies here, so no kill is announced while
|
|
629
|
+
// the working session keeps running (that is exactly what #2134 reported).
|
|
630
|
+
if (unverifiedDockerFailure) {
|
|
631
|
+
const probe = backendAlive || runner.checkBackendSessionAlive;
|
|
632
|
+
let alive = null;
|
|
633
|
+
if (probe && sessionInfo?.isolationBackend) {
|
|
634
|
+
try {
|
|
635
|
+
alive = await probe(sessionId, sessionInfo.isolationBackend, verbose);
|
|
636
|
+
} catch {
|
|
637
|
+
alive = null;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
if (alive === true) {
|
|
641
|
+
if (verbose) {
|
|
642
|
+
console.log(`[VERBOSE] Session ${sessionName} reported terminal '${statusResult.status}' with exit ${exitCode}, but its docker backend is still alive; keeping the session tracked (issue #2134)`);
|
|
643
|
+
}
|
|
644
|
+
return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
|
|
645
|
+
}
|
|
646
|
+
}
|
|
614
647
|
if (!ambiguousDockerTerminal) {
|
|
615
648
|
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
616
649
|
return { running: false, exitCode, status: statusResult.status, statusResult };
|
|
@@ -817,6 +850,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
817
850
|
// Issue #1927: for a killed /solve, offer a command using the last tool
|
|
818
851
|
// session ID in the log. Do not auto-relaunch work that may reliably OOM.
|
|
819
852
|
const resumeExtraSections = [];
|
|
853
|
+
let killResumeCommand = null;
|
|
820
854
|
try {
|
|
821
855
|
const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
|
|
822
856
|
const isResumableCommand = (sessionInfo?.command || 'solve') === 'solve';
|
|
@@ -831,6 +865,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
831
865
|
const lastSessionId = readLastSessionIdFromLog(logPath, { verbose });
|
|
832
866
|
const resumeCommand = buildResumeCommand({ sessionInfo, lastSessionId });
|
|
833
867
|
const resumeSection = formatResumeSection({ lastSessionId, command: resumeCommand });
|
|
868
|
+
killResumeCommand = resumeCommand || null;
|
|
834
869
|
if (resumeSection) {
|
|
835
870
|
resumeExtraSections.push(resumeSection);
|
|
836
871
|
if (verbose) {
|
|
@@ -865,6 +900,42 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
865
900
|
}
|
|
866
901
|
const dockerTaskContainerExtraSections = dockerTaskContainerAction?.extraSection ? [dockerTaskContainerAction.extraSection] : [];
|
|
867
902
|
|
|
903
|
+
// Issue #2134: say exactly WHY a session was killed, and warn when a
|
|
904
|
+
// session merely survived a kill event instead of reporting a plain
|
|
905
|
+
// success. The pull request gets the very same report below.
|
|
906
|
+
const killReport = await buildKillCompletionSections({
|
|
907
|
+
sessionName,
|
|
908
|
+
sessionInfo,
|
|
909
|
+
statusResult,
|
|
910
|
+
exitCode: finalExitCode,
|
|
911
|
+
status: resolvedStatus,
|
|
912
|
+
verbose,
|
|
913
|
+
readFile: options.readFile,
|
|
914
|
+
env: options.env || process.env,
|
|
915
|
+
});
|
|
916
|
+
|
|
917
|
+
// Issue #2134: `--on-session-kill=resume` must actually start a new
|
|
918
|
+
// working session, and both surfaces must say so. Done before the
|
|
919
|
+
// message is built so the Telegram report and the pull-request notice
|
|
920
|
+
// below name the very same recovery session.
|
|
921
|
+
let killRecovery = { resumed: false, sessionId: null, attempt: 0, maxAttempts: 0 };
|
|
922
|
+
if (killReport.killed) {
|
|
923
|
+
const recovered = await runKillRecoveryForCompletion({
|
|
924
|
+
sessionName,
|
|
925
|
+
sessionInfo,
|
|
926
|
+
logPath: statusResult?.logPath || sessionInfo?.logPath || null,
|
|
927
|
+
killed: true,
|
|
928
|
+
env: options.env || process.env,
|
|
929
|
+
runner: options.isolationRunner || null,
|
|
930
|
+
trackSession: options.trackSession || trackSession,
|
|
931
|
+
persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
|
|
932
|
+
locale: sessionInfo?.locale || null,
|
|
933
|
+
verbose,
|
|
934
|
+
});
|
|
935
|
+
killRecovery = recovered.recovery;
|
|
936
|
+
if (recovered.section) killReport.sections.push(recovered.section);
|
|
937
|
+
}
|
|
938
|
+
|
|
868
939
|
const message = formatSessionCompletionMessage({
|
|
869
940
|
sessionName,
|
|
870
941
|
sessionInfo,
|
|
@@ -874,9 +945,33 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
874
945
|
infoBlock: sessionInfo?.infoBlock || '',
|
|
875
946
|
pullRequestUrl,
|
|
876
947
|
pullRequestState,
|
|
877
|
-
extraSections: [...limitsExtraSections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
948
|
+
extraSections: [...limitsExtraSections, ...killReport.sections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
878
949
|
});
|
|
879
950
|
|
|
951
|
+
if (killReport.killed || killReport.recovered) {
|
|
952
|
+
const notice = await announceKillOnPullRequest({
|
|
953
|
+
pullRequestUrl,
|
|
954
|
+
sessionName,
|
|
955
|
+
sessionInfo,
|
|
956
|
+
diagnosis: killReport.diagnosis,
|
|
957
|
+
exitCode: finalExitCode,
|
|
958
|
+
observedAt: killReport.observedAt,
|
|
959
|
+
policy: killReport.policy,
|
|
960
|
+
recovered: killReport.recovered,
|
|
961
|
+
resumed: killRecovery.resumed,
|
|
962
|
+
recoverySessionId: killRecovery.resumed ? killRecovery.sessionId : null,
|
|
963
|
+
attempt: killRecovery.resumed ? killRecovery.attempt : null,
|
|
964
|
+
maxAttempts: killRecovery.resumed ? killRecovery.maxAttempts : null,
|
|
965
|
+
resumeCommand: killResumeCommand,
|
|
966
|
+
runCommand: options.runCommand || undefined,
|
|
967
|
+
attachLog: options.attachLog || undefined,
|
|
968
|
+
verbose,
|
|
969
|
+
});
|
|
970
|
+
if (verbose && !notice.posted) {
|
|
971
|
+
console.log(`[VERBOSE] Killed-session notice not posted for ${sessionName}: ${notice.skipped || 'unknown reason'}`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// Update the original reply message if messageId is available, otherwise send new message
|
|
881
976
|
let notifyFromChatId = null;
|
|
882
977
|
let notifyMessageId = null;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verified out-of-memory classification for tracked isolation sessions.
|
|
3
|
+
*
|
|
4
|
+
* Issue #2015 made `oomKilled: true` in a `$ --status` record terminal so an
|
|
5
|
+
* OOM-killed session could not be polled forever. Issue #2134 showed the other
|
|
6
|
+
* half of the problem: Docker's `State.OOMKilled` is a *container* flag, not a
|
|
7
|
+
* statement about the container's main process. The kernel sets it when ANY
|
|
8
|
+
* process in the container cgroup is OOM-killed, and it stays `true` afterwards
|
|
9
|
+
* — so a container can be flagged, keep running, and exit 0 (see
|
|
10
|
+
* https://github.com/moby/moby/issues/47618).
|
|
11
|
+
*
|
|
12
|
+
* That is exactly what happened in #2134: the host ran out of memory, the OOM
|
|
13
|
+
* killer terminated a child process, `$ --status` flipped to
|
|
14
|
+
* `executed / exitCode 137`, and Hive Mind announced
|
|
15
|
+
* "Work session killed — out of memory or forced kill (SIGKILL)" while
|
|
16
|
+
* `docker inspect` still reported the container **running**. The session went on
|
|
17
|
+
* for another 3.5 hours and auto-merged its pull request, unmonitored.
|
|
18
|
+
*
|
|
19
|
+
* This module keeps #2015's guarantee (a truly OOM-killed session is terminal)
|
|
20
|
+
* while refusing to declare a kill that is contradicted by stronger evidence,
|
|
21
|
+
* following the same ladder the rest of the monitor already uses:
|
|
22
|
+
*
|
|
23
|
+
* 1. The log FOOTER wins. A written `Exit Code: N` is proof of how the command
|
|
24
|
+
* actually ended — including `Exit Code: 0`, which means the session merely
|
|
25
|
+
* *survived* an OOM event.
|
|
26
|
+
* 2. LIVENESS beats the status record. No footer + the backing container is
|
|
27
|
+
* still alive → the session is still running; keep polling and remember the
|
|
28
|
+
* OOM event so the eventual completion can report the recovery.
|
|
29
|
+
* 3. Otherwise report `oom-killed`, exactly as issue #2015 requires.
|
|
30
|
+
*
|
|
31
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2015
|
|
32
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Field written on the persisted session snapshot the first time an OOM event is
|
|
39
|
+
* observed for a session that is still alive. It survives a bot restart, so the
|
|
40
|
+
* completion message can still say "recovered from out of memory" hours later.
|
|
41
|
+
*/
|
|
42
|
+
export const OOM_EVENT_OBSERVED_FIELD = 'oomEventObservedAt';
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Remember that the kernel OOM killer fired inside this session's container.
|
|
46
|
+
* Idempotent: only the FIRST observation timestamp is kept.
|
|
47
|
+
*
|
|
48
|
+
* @param {Object} sessionInfo - Mutable persisted session info
|
|
49
|
+
* @param {Function} [persistSnapshot] - Callback that mirrors sessionInfo to disk
|
|
50
|
+
* @returns {boolean} True when this call recorded a new observation
|
|
51
|
+
*/
|
|
52
|
+
export function markOomEventObserved(sessionInfo, persistSnapshot) {
|
|
53
|
+
if (!sessionInfo || sessionInfo[OOM_EVENT_OBSERVED_FIELD]) return false;
|
|
54
|
+
sessionInfo[OOM_EVENT_OBSERVED_FIELD] = new Date().toISOString();
|
|
55
|
+
if (typeof persistSnapshot === 'function') persistSnapshot();
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Whether an OOM event was observed for this session while it was running.
|
|
61
|
+
*
|
|
62
|
+
* @param {Object} sessionInfo
|
|
63
|
+
* @returns {string|null} ISO timestamp of the first observation, or null
|
|
64
|
+
*/
|
|
65
|
+
export function getOomEventObservedAt(sessionInfo) {
|
|
66
|
+
return sessionInfo?.[OOM_EVENT_OBSERVED_FIELD] || null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function probeBackendAlive(sessionName, sessionInfo, { verbose, runner, backendAlive }) {
|
|
70
|
+
if (!sessionInfo?.isolationBackend) return null;
|
|
71
|
+
const probe = backendAlive || runner?.checkBackendSessionAlive;
|
|
72
|
+
if (!probe) return null;
|
|
73
|
+
try {
|
|
74
|
+
return await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
if (verbose) {
|
|
77
|
+
console.log(`[VERBOSE] Session ${sessionName} OOM liveness probe failed: ${error?.message || error}`);
|
|
78
|
+
}
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Resolve the real state of a session whose status record carries
|
|
85
|
+
* `oomKilled: true`.
|
|
86
|
+
*
|
|
87
|
+
* @param {string} sessionName
|
|
88
|
+
* @param {Object} sessionInfo
|
|
89
|
+
* @param {Object} statusResult - Parsed `$ --status` payload
|
|
90
|
+
* @param {Object} deps
|
|
91
|
+
* @param {boolean} [deps.verbose]
|
|
92
|
+
* @param {Object} deps.runner - Isolation runner (for its default probes)
|
|
93
|
+
* @param {Function} [deps.exitFromLog] - Injectable footer reader
|
|
94
|
+
* @param {Function} [deps.backendAlive] - Injectable liveness probe
|
|
95
|
+
* @param {Function} [deps.persistSnapshot] - Persist the session snapshot
|
|
96
|
+
* @returns {Promise<Object>} Monitor state object
|
|
97
|
+
*/
|
|
98
|
+
export async function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive, persistSnapshot } = {}) {
|
|
99
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
100
|
+
let footer = null;
|
|
101
|
+
if (logPath) {
|
|
102
|
+
const readFooter = exitFromLog || runner?.readSessionExitFromLog;
|
|
103
|
+
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 1. The authoritative log footer.
|
|
107
|
+
if (footer?.finished) {
|
|
108
|
+
const footerExitCode = normalizeExitCode(footer.exitCode);
|
|
109
|
+
const correctedStatus = classifyExitStatus(footerExitCode) || (footerExitCode === 0 ? 'executed' : 'failed');
|
|
110
|
+
const survivedOom = footerExitCode === 0;
|
|
111
|
+
if (survivedOom) markOomEventObserved(sessionInfo, persistSnapshot);
|
|
112
|
+
if (verbose) {
|
|
113
|
+
console.log(`[VERBOSE] Session ${sessionName} reported oomKilled=true, but its log footer says exit ${footerExitCode} (${correctedStatus}) and wins${survivedOom ? ' — the session SURVIVED the out-of-memory event (issue #2134)' : ''}`);
|
|
114
|
+
}
|
|
115
|
+
return {
|
|
116
|
+
running: false,
|
|
117
|
+
exitCode: footerExitCode,
|
|
118
|
+
status: correctedStatus,
|
|
119
|
+
statusResult: { ...statusResult, status: correctedStatus, exitCode: footerExitCode, endTime: statusResult?.endTime || footer.endTime || null },
|
|
120
|
+
oomEventObserved: true,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// 2. Liveness: an alive container cannot have had its command killed.
|
|
125
|
+
const alive = await probeBackendAlive(sessionName, sessionInfo, { verbose, runner, backendAlive });
|
|
126
|
+
if (alive === true) {
|
|
127
|
+
markOomEventObserved(sessionInfo, persistSnapshot);
|
|
128
|
+
if (verbose) {
|
|
129
|
+
console.log(`[VERBOSE] Session ${sessionName} reported oomKilled=true but its ${sessionInfo.isolationBackend} backend is still alive; an OOM event hit the container, not the command — keeping the session tracked (issue #2134)`);
|
|
130
|
+
}
|
|
131
|
+
return { running: true, exitCode: null, status: statusResult?.status || 'executing', statusResult, deferred: true, oomEventObserved: true };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// 3. Nothing contradicts the status record: this really is an OOM kill (#2015).
|
|
135
|
+
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
136
|
+
let exitCode = 137;
|
|
137
|
+
if (statusExitCode !== null && statusExitCode > 0) {
|
|
138
|
+
exitCode = statusExitCode;
|
|
139
|
+
}
|
|
140
|
+
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
141
|
+
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
142
|
+
|
|
143
|
+
if (verbose) {
|
|
144
|
+
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true (backend alive: ${alive === null ? 'unknown' : alive}); treating it as terminal oom-killed (exit ${exitCode})`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true, oomEventObserved: true };
|
|
148
|
+
}
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* @see https://github.com/link-assistant/hive-mind/issues/2015
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import { classifyExitStatus
|
|
17
|
+
import { classifyExitStatus } from './session-status.lib.mjs';
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Issue #1927: minimum age before a session that `$ --status` still reports as
|
|
@@ -122,30 +122,9 @@ export async function resolveStaleExecutingState(sessionName, sessionInfo, statu
|
|
|
122
122
|
|
|
123
123
|
/**
|
|
124
124
|
* Issue #2015: `oomKilled` is terminal — the container was killed by the kernel,
|
|
125
|
-
* so no further polling can change the outcome.
|
|
125
|
+
* so no further polling can change the outcome. Issue #2134 refined that into a
|
|
126
|
+
* *verified* classification (footer/liveness ladder); the implementation now
|
|
127
|
+
* lives in session-monitor.oom.lib.mjs and is re-exported here so existing
|
|
128
|
+
* importers keep working.
|
|
126
129
|
*/
|
|
127
|
-
export
|
|
128
|
-
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
129
|
-
let footer = null;
|
|
130
|
-
if (logPath) {
|
|
131
|
-
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
132
|
-
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
136
|
-
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
137
|
-
let exitCode = 137;
|
|
138
|
-
if (statusExitCode !== null && statusExitCode > 0) {
|
|
139
|
-
exitCode = statusExitCode;
|
|
140
|
-
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
141
|
-
exitCode = footerExitCode;
|
|
142
|
-
}
|
|
143
|
-
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
144
|
-
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
145
|
-
|
|
146
|
-
if (verbose) {
|
|
147
|
-
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
151
|
-
}
|
|
130
|
+
export { resolveOomKilledState, markOomEventObserved, getOomEventObservedAt, OOM_EVENT_OBSERVED_FIELD } from './session-monitor.oom.lib.mjs';
|
|
@@ -43,6 +43,30 @@ const SESSION_LOG_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0
|
|
|
43
43
|
const DEFAULT_LOG_SCAN_CHUNK_BYTES = 262144;
|
|
44
44
|
const LOG_SCAN_OVERLAP_BYTES = 1024;
|
|
45
45
|
|
|
46
|
+
// Issue #2134: the marker scan runs over a log that can contain *anything* the
|
|
47
|
+
// run printed — including Hive Mind's own source code. `solve` prints its id
|
|
48
|
+
// with `` console.log(`📌 Session ID: ${sessionId}`) ``, so a run that `cat`-ed
|
|
49
|
+
// that source made the literal string `${sessionId}` the "last session id" and
|
|
50
|
+
// the bot offered `--resume "${sessionId}"` to the user. Tool session ids are
|
|
51
|
+
// opaque tokens (UUIDs for claude/codex, slugs elsewhere): a shape check plus a
|
|
52
|
+
// small vocabulary of placeholder words rejects that whole class of false hit
|
|
53
|
+
// without narrowing the ids that are legitimately accepted today.
|
|
54
|
+
const SESSION_ID_SHAPE_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{6,199}$/;
|
|
55
|
+
const SESSION_ID_PLACEHOLDERS = new Set(['unknown', 'n/a', 'na', 'none', 'null', 'undefined', 'sessionid', 'session_id', 'session-id', 'id']);
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Whether a token captured after a `Session ID:` label can be a real tool
|
|
59
|
+
* session id (issue #2134).
|
|
60
|
+
*
|
|
61
|
+
* @param {*} id - Captured token.
|
|
62
|
+
* @returns {boolean} True when the token has a plausible session-id shape.
|
|
63
|
+
*/
|
|
64
|
+
export function isPlausibleSessionId(id) {
|
|
65
|
+
if (!id || typeof id !== 'string') return false;
|
|
66
|
+
if (SESSION_ID_PLACEHOLDERS.has(id.toLowerCase())) return false;
|
|
67
|
+
return SESSION_ID_SHAPE_RE.test(id);
|
|
68
|
+
}
|
|
69
|
+
|
|
46
70
|
/**
|
|
47
71
|
* Extract every tool session id printed to a log, in the order they appear.
|
|
48
72
|
*
|
|
@@ -60,8 +84,10 @@ export function extractSessionIds(text) {
|
|
|
60
84
|
SESSION_ID_MARKER_RE.lastIndex = 0;
|
|
61
85
|
while ((match = SESSION_ID_MARKER_RE.exec(text)) !== null) {
|
|
62
86
|
const id = match[1];
|
|
63
|
-
// Skip obvious non-ids that can follow the label in prose/log output
|
|
64
|
-
|
|
87
|
+
// Skip obvious non-ids that can follow the label in prose/log output, and
|
|
88
|
+
// template placeholders such as `${sessionId}` printed by quoted source
|
|
89
|
+
// code (issue #2134).
|
|
90
|
+
if (!isPlausibleSessionId(id)) continue;
|
|
65
91
|
if (ids[ids.length - 1] !== id) ids.push(id);
|
|
66
92
|
}
|
|
67
93
|
return ids;
|
|
@@ -1052,19 +1052,20 @@ Once the billing issue is resolved, you can re-run the CI checks or push a new c
|
|
|
1052
1052
|
latestAnthropicCost = toolResult.anthropicTotalCostUSD;
|
|
1053
1053
|
}
|
|
1054
1054
|
|
|
1055
|
-
// Issue #1508: Compute budget stats for auto-restart-until-mergeable log comment
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1055
|
+
// Issue #1508: Compute budget stats for auto-restart-until-mergeable log comment.
|
|
1056
|
+
// Issue #2132: shared with the top-level run and the watch loop via
|
|
1057
|
+
// buildSessionBudgetStatsData, so every working session derives its own
|
|
1058
|
+
// stats the same way (and skips them when `--attach-logs` is disabled).
|
|
1059
|
+
const { buildSessionBudgetStatsData } = await import('./solve.results.lib.mjs');
|
|
1060
|
+
const autoMergeBudgetStatsData = await buildSessionBudgetStatsData({
|
|
1061
|
+
argv,
|
|
1062
|
+
sessionId: latestSessionId,
|
|
1063
|
+
tempDir,
|
|
1064
|
+
resultModelUsage: toolResult.resultModelUsage,
|
|
1065
|
+
streamTokenUsage: toolResult.streamTokenUsage || null,
|
|
1066
|
+
subAgentCalls: toolResult.subAgentCalls || null,
|
|
1067
|
+
pricingInfo: toolResult.pricingInfo || null,
|
|
1068
|
+
});
|
|
1068
1069
|
|
|
1069
1070
|
// Issue #1761: Post the working session **summary** BEFORE uploading
|
|
1070
1071
|
// the working session **log** so the summary always appears above
|
|
@@ -1090,8 +1091,6 @@ Once the billing issue is resolved, you can re-run the CI checks or push a new c
|
|
|
1090
1091
|
prNumber,
|
|
1091
1092
|
issueNumber,
|
|
1092
1093
|
success: true,
|
|
1093
|
-
publicPricingEstimate: toolResult.publicPricingEstimate,
|
|
1094
|
-
anthropicTotalCostUSD: latestAnthropicCost,
|
|
1095
1094
|
pricingInfo: toolResult.pricingInfo,
|
|
1096
1095
|
budgetStatsData: autoMergeBudgetStatsData,
|
|
1097
1096
|
});
|
package/src/solve.config.lib.mjs
CHANGED
|
@@ -136,7 +136,7 @@ export const SOLVE_OPTION_DEFINITIONS = {
|
|
|
136
136
|
},
|
|
137
137
|
'attach-logs': {
|
|
138
138
|
type: 'boolean',
|
|
139
|
-
description:
|
|
139
|
+
description: "Upload the solution draft log file to the Pull Request on completion, together with that working session's cost estimation and context/token budget statistics (⚠️ WARNING: May expose sensitive data). Disabled means no log comment and no published budget statistics.",
|
|
140
140
|
default: false,
|
|
141
141
|
},
|
|
142
142
|
'dangerously-skip-output-sanitization': {
|
|
@@ -154,6 +154,20 @@ export const SOLVE_OPTION_DEFINITIONS = {
|
|
|
154
154
|
description: 'DANGEROUS: skip masking known active local tokens in output. This is separate from other sanitization skip flags and should only be used for controlled debugging.',
|
|
155
155
|
default: false,
|
|
156
156
|
},
|
|
157
|
+
// Issue #2134: one option decides what happens when a working session is
|
|
158
|
+
// killed (out of memory, disk exhaustion, forced kill). The Telegram bot and
|
|
159
|
+
// solve honour the exact same value, so the report never differs by surface.
|
|
160
|
+
'on-session-kill': {
|
|
161
|
+
type: 'string',
|
|
162
|
+
description: 'What to do when a working session is killed (out of memory, disk full, forced kill): "report" describes exactly what happened in the pull request and in Telegram, "resume" additionally starts a new working session to recover and says so in both places. Can also be set with HIVE_MIND_ON_SESSION_KILL.',
|
|
163
|
+
choices: ['report', 'resume'],
|
|
164
|
+
default: 'report',
|
|
165
|
+
},
|
|
166
|
+
'session-kill-resume-attempts': {
|
|
167
|
+
type: 'number',
|
|
168
|
+
description: 'Maximum number of automatic recovery working sessions started for one killed session when --on-session-kill=resume. Can also be set with HIVE_MIND_SESSION_KILL_RESUME_ATTEMPTS.',
|
|
169
|
+
default: 1,
|
|
170
|
+
},
|
|
157
171
|
'auto-close-pull-request-on-fail': {
|
|
158
172
|
type: 'boolean',
|
|
159
173
|
description: 'Automatically close the pull request if execution fails',
|
|
@@ -505,7 +519,7 @@ export const SOLVE_OPTION_DEFINITIONS = {
|
|
|
505
519
|
},
|
|
506
520
|
'tokens-budget-stats': {
|
|
507
521
|
type: 'boolean',
|
|
508
|
-
description: 'Show detailed token budget statistics including context window usage and ratios (enabled by default, use --no-tokens-budget-stats to disable). Supported for --tool claude, --tool codex, and any tool that returns detailed token usage.',
|
|
522
|
+
description: 'Show detailed token budget statistics including context window usage and ratios (enabled by default, use --no-tokens-budget-stats to disable). Shown in the terminal; publishing them to the pull request additionally requires --attach-logs, and they appear only in the working session log comment (never in the working session summary). Supported for --tool claude, --tool codex, and any tool that returns detailed token usage.',
|
|
509
523
|
default: true,
|
|
510
524
|
},
|
|
511
525
|
'prompt-issue-reporting': {
|
package/src/solve.mjs
CHANGED
|
@@ -29,8 +29,6 @@ import { safeExit } from './exit-handler.lib.mjs';
|
|
|
29
29
|
// Import GitHub-related functions
|
|
30
30
|
const githubLib = await import('./github.lib.mjs');
|
|
31
31
|
const { sanitizeLogContent, attachLogToGitHub } = githubLib;
|
|
32
|
-
const { buildCostInfoString } = await import('./github-cost-info.lib.mjs');
|
|
33
|
-
const { buildBudgetStatsString } = await import('./claude.budget-stats.lib.mjs');
|
|
34
32
|
|
|
35
33
|
// Issue #1745: process-wide sanitization counters used to print a one-line
|
|
36
34
|
// "we masked N secrets" summary at the end of each run.
|
|
@@ -674,7 +672,18 @@ export const showSessionSummary = async (sessionId, limitReached, argv, issueUrl
|
|
|
674
672
|
// same observed facts (working-session summary and attached log alike).
|
|
675
673
|
export const buildSessionBudgetStatsData = async ({ argv, sessionId = null, tempDir = null, resultModelUsage = null, streamTokenUsage = null, subAgentCalls = null, pricingInfo = null }) => {
|
|
676
674
|
let budgetStatsData = null;
|
|
677
|
-
|
|
675
|
+
// Issue #2132: budget stats are a property of the working session **log**.
|
|
676
|
+
// With `--attach-logs` disabled there is no log comment, so they must not be
|
|
677
|
+
// computed or published anywhere.
|
|
678
|
+
const { shouldPublishBudgetStats, isAttachLogsEnabled, isTokensBudgetStatsEnabled } = await import('./budget-stats-policy.lib.mjs');
|
|
679
|
+
if (!shouldPublishBudgetStats(argv)) {
|
|
680
|
+
if (argv?.verbose) {
|
|
681
|
+
const reason = !isTokensBudgetStatsEnabled(argv) ? '--no-tokens-budget-stats' : !isAttachLogsEnabled(argv) ? '--attach-logs is disabled' : 'unknown';
|
|
682
|
+
await log(` ℹ️ Skipping context/cost budget stats publication (${reason})`, { verbose: true });
|
|
683
|
+
}
|
|
684
|
+
return null;
|
|
685
|
+
}
|
|
686
|
+
if (sessionId && tempDir) {
|
|
678
687
|
try {
|
|
679
688
|
const { calculateSessionTokens } = await import('./claude.lib.mjs');
|
|
680
689
|
const tokenUsage = await calculateSessionTokens(sessionId, tempDir, resultModelUsage);
|
|
@@ -686,7 +695,7 @@ export const buildSessionBudgetStatsData = async ({ argv, sessionId = null, temp
|
|
|
686
695
|
}
|
|
687
696
|
}
|
|
688
697
|
// Issue #1526: Build budget stats from Agent CLI token/context data when no JSONL session available
|
|
689
|
-
if (!budgetStatsData &&
|
|
698
|
+
if (!budgetStatsData && pricingInfo?.tokenUsage) {
|
|
690
699
|
try {
|
|
691
700
|
const { buildAgentBudgetStats } = await import('./claude.budget-stats.lib.mjs');
|
|
692
701
|
const agentBudgetData = buildAgentBudgetStats(pricingInfo.tokenUsage, pricingInfo);
|
|
@@ -1264,15 +1273,19 @@ export const checkForAiCreatedComments = async (sessionStartTime, owner, repo, p
|
|
|
1264
1273
|
* @param {string} options.repo - Repository name
|
|
1265
1274
|
* @returns {Promise<boolean>} - True if comment was posted successfully
|
|
1266
1275
|
*/
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1276
|
+
/**
|
|
1277
|
+
* Issue #2132: the working session summary must describe *what the AI did* and
|
|
1278
|
+
* nothing else. Cost estimation and context/token budget statistics belong to
|
|
1279
|
+
* the working session log comment (`--attach-logs`), where they are already
|
|
1280
|
+
* published once per working session. Rendering them in the summary as well
|
|
1281
|
+
* duplicated the very same block in two consecutive comments.
|
|
1282
|
+
*
|
|
1283
|
+
* Kept as an exported function returning an empty string so the invariant is
|
|
1284
|
+
* directly testable and any future caller cannot silently re-add the block.
|
|
1285
|
+
*/
|
|
1286
|
+
export const buildWorkingSessionSummaryDetails = () => '';
|
|
1274
1287
|
|
|
1275
|
-
export const attachSolutionSummary = async ({ resultSummary, prNumber, issueNumber, owner, repo,
|
|
1288
|
+
export const attachSolutionSummary = async ({ resultSummary, prNumber, issueNumber, owner, repo, changeStats = null }) => {
|
|
1276
1289
|
if (!resultSummary || typeof resultSummary !== 'string') {
|
|
1277
1290
|
await log('⚠️ No working session summary available to attach', { verbose: true });
|
|
1278
1291
|
return false;
|
|
@@ -1287,12 +1300,6 @@ export const attachSolutionSummary = async ({ resultSummary, prNumber, issueNumb
|
|
|
1287
1300
|
}
|
|
1288
1301
|
|
|
1289
1302
|
try {
|
|
1290
|
-
const usageDetails = buildWorkingSessionSummaryDetails({
|
|
1291
|
-
publicPricingEstimate,
|
|
1292
|
-
anthropicTotalCostUSD,
|
|
1293
|
-
pricingInfo,
|
|
1294
|
-
budgetStatsData,
|
|
1295
|
-
});
|
|
1296
1303
|
// Issue #2119: publish what the session actually produced. The reported
|
|
1297
1304
|
// summary said "The `pwd` command completed" and printed the solver's own
|
|
1298
1305
|
// /tmp workspace, on a pull request that was still empty.
|
|
@@ -1302,7 +1309,7 @@ export const attachSolutionSummary = async ({ resultSummary, prNumber, issueNumb
|
|
|
1302
1309
|
const comment = `${toolComments.WORKING_SESSION_SUMMARY_AUTOMATION_MARKER}
|
|
1303
1310
|
## ${toolComments.WORKING_SESSION_SUMMARY_MARKER}
|
|
1304
1311
|
|
|
1305
|
-
${summaryBody}${noChangesNotice ? `\n\n${noChangesNotice}` : ''}
|
|
1312
|
+
${summaryBody}${noChangesNotice ? `\n\n${noChangesNotice}` : ''}
|
|
1306
1313
|
|
|
1307
1314
|
---
|
|
1308
1315
|
*${toolComments.WORKING_SESSION_SUMMARY_AUTOMATED_FOOTER}*`;
|
|
@@ -1359,7 +1366,7 @@ ${summaryBody}${noChangesNotice ? `\n\n${noChangesNotice}` : ''}${usageDetails ?
|
|
|
1359
1366
|
* @param {boolean} [options.success=true] - skip attachment for failed iterations
|
|
1360
1367
|
* @returns {Promise<{attached: boolean, reason: string}>}
|
|
1361
1368
|
*/
|
|
1362
|
-
export const maybeAttachWorkingSessionSummary = async ({ argv, resultSummary, workStartTime, owner, repo, prNumber, issueNumber, success = true,
|
|
1369
|
+
export const maybeAttachWorkingSessionSummary = async ({ argv, resultSummary, workStartTime, owner, repo, prNumber, issueNumber, success = true, pricingInfo = null, budgetStatsData = null, sessionUsage = null }) => {
|
|
1363
1370
|
if (!success) {
|
|
1364
1371
|
return { attached: false, reason: 'iteration_failed' };
|
|
1365
1372
|
}
|
|
@@ -1408,16 +1415,14 @@ export const maybeAttachWorkingSessionSummary = async ({ argv, resultSummary, wo
|
|
|
1408
1415
|
// say so, instead of reading as a report of completed work.
|
|
1409
1416
|
const changeStats = prNumber ? await getPullRequestChangeStats({ owner, repo, prNumber, $ }) : null;
|
|
1410
1417
|
|
|
1418
|
+
// Issue #2132: the summary carries no cost/budget block. `resolvedBudgetStatsData`
|
|
1419
|
+
// is computed only so the caller can reuse it for this session's log comment.
|
|
1411
1420
|
const ok = await attachSolutionSummary({
|
|
1412
1421
|
resultSummary,
|
|
1413
1422
|
prNumber,
|
|
1414
1423
|
issueNumber,
|
|
1415
1424
|
owner,
|
|
1416
1425
|
repo,
|
|
1417
|
-
publicPricingEstimate,
|
|
1418
|
-
anthropicTotalCostUSD,
|
|
1419
|
-
pricingInfo,
|
|
1420
|
-
budgetStatsData: resolvedBudgetStatsData,
|
|
1421
1426
|
changeStats,
|
|
1422
1427
|
});
|
|
1423
1428
|
return { attached: !!ok, reason: ok ? 'attached' : 'post_failed', budgetStatsData: resolvedBudgetStatsData };
|
package/src/solve.watch.lib.mjs
CHANGED
|
@@ -567,19 +567,21 @@ export const watchForFeedback = async params => {
|
|
|
567
567
|
latestResultModelUsage = toolResult.resultModelUsage;
|
|
568
568
|
}
|
|
569
569
|
|
|
570
|
-
// Issue #1508: Compute budget stats for auto-restart log comment
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
570
|
+
// Issue #1508: Compute budget stats for auto-restart log comment.
|
|
571
|
+
// Issue #2132: shared with the top-level run and the
|
|
572
|
+
// auto-restart-until-mergeable loop via buildSessionBudgetStatsData,
|
|
573
|
+
// so every working session derives its own stats the same way (and
|
|
574
|
+
// skips them entirely when `--attach-logs` is disabled).
|
|
575
|
+
const { buildSessionBudgetStatsData } = await import('./solve.results.lib.mjs');
|
|
576
|
+
const autoRestartBudgetStatsData = await buildSessionBudgetStatsData({
|
|
577
|
+
argv,
|
|
578
|
+
sessionId: latestSessionId,
|
|
579
|
+
tempDir,
|
|
580
|
+
resultModelUsage: toolResult.resultModelUsage,
|
|
581
|
+
streamTokenUsage: toolResult.streamTokenUsage || null,
|
|
582
|
+
subAgentCalls: toolResult.subAgentCalls || null,
|
|
583
|
+
pricingInfo: toolResult.pricingInfo || null,
|
|
584
|
+
});
|
|
583
585
|
|
|
584
586
|
// Issue #1761: Post the working session **summary** BEFORE uploading
|
|
585
587
|
// the working session **log** so the summary always appears above
|
|
@@ -604,8 +606,6 @@ export const watchForFeedback = async params => {
|
|
|
604
606
|
prNumber,
|
|
605
607
|
issueNumber,
|
|
606
608
|
success: true,
|
|
607
|
-
publicPricingEstimate: toolResult.publicPricingEstimate,
|
|
608
|
-
anthropicTotalCostUSD: latestAnthropicCost,
|
|
609
609
|
pricingInfo: toolResult.pricingInfo,
|
|
610
610
|
budgetStatsData: autoRestartBudgetStatsData,
|
|
611
611
|
});
|
package/src/telegram-bot.mjs
CHANGED
|
@@ -1248,7 +1248,9 @@ let launchAnnouncementShown = false;
|
|
|
1248
1248
|
|
|
1249
1249
|
function startSessionMonitoringOnce() {
|
|
1250
1250
|
if (sessionMonitoringTimer) return;
|
|
1251
|
-
|
|
1251
|
+
// Issue #2134: the monitor needs the isolation runner to start a recovery
|
|
1252
|
+
// working session when `--on-session-kill=resume` is in effect.
|
|
1253
|
+
sessionMonitoringTimer = startSessionMonitoring(bot, VERBOSE, 30000, { isolationRunner });
|
|
1252
1254
|
}
|
|
1253
1255
|
|
|
1254
1256
|
// Issue #1927 (requirements #3/#4): a periodic timestamped heartbeat so the "last
|