@link-assistant/hive-mind 2.11.7 → 2.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/package.json +1 -1
- package/src/bidirectional-interactive.lib.mjs +6 -2
- package/src/child-exit.lib.mjs +107 -0
- package/src/contributing-guidelines.lib.mjs +19 -6
- package/src/development-log.lib.mjs +82 -5
- package/src/fix.ci-cd-issue.lib.mjs +5 -3
- package/src/fix.mjs +5 -2
- package/src/github-entity-validation.lib.mjs +4 -1
- package/src/github.lib.mjs +3 -3
- package/src/hive.mjs +15 -21
- package/src/isolation-runner.lib.mjs +5 -2
- package/src/lib.mjs +21 -0
- package/src/locales/en.lino +10 -0
- package/src/locales/hi.lino +10 -0
- package/src/locales/ru.lino +10 -0
- package/src/locales/zh.lino +10 -0
- package/src/log-growth.lib.mjs +94 -0
- package/src/option-suggestions.lib.mjs +2 -0
- package/src/pull-request-changes.lib.mjs +94 -24
- package/src/review.mjs +12 -3
- package/src/session-kill-diagnostics.lib.mjs +388 -0
- package/src/session-kill-policy.lib.mjs +96 -0
- package/src/session-kill-recovery.lib.mjs +256 -0
- package/src/session-kill-resume.lib.mjs +175 -0
- package/src/session-monitor.kill-sections.lib.mjs +198 -0
- package/src/session-monitor.lib.mjs +97 -2
- package/src/session-monitor.oom.lib.mjs +148 -0
- package/src/session-monitor.stale-executing.lib.mjs +6 -27
- package/src/session-resume.lib.mjs +28 -2
- package/src/solve.auto-continue.lib.mjs +6 -2
- package/src/solve.auto-merge.lib.mjs +1 -1
- package/src/solve.config.lib.mjs +14 -0
- package/src/solve.keep-working.lib.mjs +7 -2
- package/src/solve.minimal-restart-prompt.lib.mjs +11 -3
- package/src/solve.preparation.lib.mjs +5 -1
- package/src/solve.progress-monitoring.lib.mjs +5 -1
- package/src/solve.repository.lib.mjs +5 -2
- package/src/solve.results.lib.mjs +13 -8
- package/src/task.mjs +5 -3
- package/src/telegram-bot.mjs +3 -1
- package/src/telegram-command-execution.lib.mjs +5 -2
|
@@ -31,6 +31,9 @@ import { resolveFailedSessionPullRequestState } from './github-pr-state.lib.mjs'
|
|
|
31
31
|
// may be an exit code start-command fabricated from the command's own output.
|
|
32
32
|
import { clearUnverifiedDockerTerminalMarker as clearUnverifiedDockerTerminalMarkerImpl, shouldDeferUnverifiedDockerTerminal as shouldDeferUnverifiedDockerTerminalImpl } from './session-monitor.docker-terminal.lib.mjs';
|
|
33
33
|
import { isDockerIsolation, sessionStartMs, resolveOomKilledState, resolveStaleExecutingState as resolveStaleExecutingStateImpl } from './session-monitor.stale-executing.lib.mjs';
|
|
34
|
+
// Issue #2134: kill-cause diagnostics + the matching pull-request notice.
|
|
35
|
+
import { buildKillCompletionSections, announceKillOnPullRequest } from './session-monitor.kill-sections.lib.mjs';
|
|
36
|
+
import { runKillRecoveryForCompletion } from './session-kill-resume.lib.mjs';
|
|
34
37
|
|
|
35
38
|
export { formatSessionCompletionMessage, getSessionCompletionExitCode } from './work-session-formatting.lib.mjs';
|
|
36
39
|
export { DOCKER_TERMINAL_FOOTER_GRACE_MS } from './session-monitor.docker-terminal.lib.mjs';
|
|
@@ -550,7 +553,16 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
550
553
|
|
|
551
554
|
if (statusResult?.exists && statusResult.status) {
|
|
552
555
|
if (statusResult.oomKilled === true) {
|
|
553
|
-
|
|
556
|
+
// Issue #2134: `oomKilled` is a *container* flag — the kernel sets it when
|
|
557
|
+
// any process in the cgroup is OOM-killed — so it is verified against the
|
|
558
|
+
// log footer and container liveness before a kill is announced.
|
|
559
|
+
return await resolveOomKilledState(sessionName, sessionInfo, statusResult, {
|
|
560
|
+
verbose,
|
|
561
|
+
runner,
|
|
562
|
+
exitFromLog,
|
|
563
|
+
backendAlive,
|
|
564
|
+
persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
|
|
565
|
+
});
|
|
554
566
|
}
|
|
555
567
|
if (runner.isExecutingSessionStatus(statusResult.status)) {
|
|
556
568
|
// Issue #1927: an `executing` status is not trusted blindly — verify the
|
|
@@ -611,6 +623,27 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
611
623
|
if (unverifiedDockerFailure && shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime: statusResult.endTime || null, verbose })) {
|
|
612
624
|
return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
|
|
613
625
|
}
|
|
626
|
+
// Issue #2134: even after the grace window, a container that is verifiably
|
|
627
|
+
// still alive cannot have produced a terminal failure — the same liveness
|
|
628
|
+
// ladder used for `oomKilled` applies here, so no kill is announced while
|
|
629
|
+
// the working session keeps running (that is exactly what #2134 reported).
|
|
630
|
+
if (unverifiedDockerFailure) {
|
|
631
|
+
const probe = backendAlive || runner.checkBackendSessionAlive;
|
|
632
|
+
let alive = null;
|
|
633
|
+
if (probe && sessionInfo?.isolationBackend) {
|
|
634
|
+
try {
|
|
635
|
+
alive = await probe(sessionId, sessionInfo.isolationBackend, verbose);
|
|
636
|
+
} catch {
|
|
637
|
+
alive = null;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
if (alive === true) {
|
|
641
|
+
if (verbose) {
|
|
642
|
+
console.log(`[VERBOSE] Session ${sessionName} reported terminal '${statusResult.status}' with exit ${exitCode}, but its docker backend is still alive; keeping the session tracked (issue #2134)`);
|
|
643
|
+
}
|
|
644
|
+
return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
|
|
645
|
+
}
|
|
646
|
+
}
|
|
614
647
|
if (!ambiguousDockerTerminal) {
|
|
615
648
|
clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
|
|
616
649
|
return { running: false, exitCode, status: statusResult.status, statusResult };
|
|
@@ -817,6 +850,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
817
850
|
// Issue #1927: for a killed /solve, offer a command using the last tool
|
|
818
851
|
// session ID in the log. Do not auto-relaunch work that may reliably OOM.
|
|
819
852
|
const resumeExtraSections = [];
|
|
853
|
+
let killResumeCommand = null;
|
|
820
854
|
try {
|
|
821
855
|
const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
|
|
822
856
|
const isResumableCommand = (sessionInfo?.command || 'solve') === 'solve';
|
|
@@ -831,6 +865,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
831
865
|
const lastSessionId = readLastSessionIdFromLog(logPath, { verbose });
|
|
832
866
|
const resumeCommand = buildResumeCommand({ sessionInfo, lastSessionId });
|
|
833
867
|
const resumeSection = formatResumeSection({ lastSessionId, command: resumeCommand });
|
|
868
|
+
killResumeCommand = resumeCommand || null;
|
|
834
869
|
if (resumeSection) {
|
|
835
870
|
resumeExtraSections.push(resumeSection);
|
|
836
871
|
if (verbose) {
|
|
@@ -865,6 +900,42 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
865
900
|
}
|
|
866
901
|
const dockerTaskContainerExtraSections = dockerTaskContainerAction?.extraSection ? [dockerTaskContainerAction.extraSection] : [];
|
|
867
902
|
|
|
903
|
+
// Issue #2134: say exactly WHY a session was killed, and warn when a
|
|
904
|
+
// session merely survived a kill event instead of reporting a plain
|
|
905
|
+
// success. The pull request gets the very same report below.
|
|
906
|
+
const killReport = await buildKillCompletionSections({
|
|
907
|
+
sessionName,
|
|
908
|
+
sessionInfo,
|
|
909
|
+
statusResult,
|
|
910
|
+
exitCode: finalExitCode,
|
|
911
|
+
status: resolvedStatus,
|
|
912
|
+
verbose,
|
|
913
|
+
readFile: options.readFile,
|
|
914
|
+
env: options.env || process.env,
|
|
915
|
+
});
|
|
916
|
+
|
|
917
|
+
// Issue #2134: `--on-session-kill=resume` must actually start a new
|
|
918
|
+
// working session, and both surfaces must say so. Done before the
|
|
919
|
+
// message is built so the Telegram report and the pull-request notice
|
|
920
|
+
// below name the very same recovery session.
|
|
921
|
+
let killRecovery = { resumed: false, sessionId: null, attempt: 0, maxAttempts: 0 };
|
|
922
|
+
if (killReport.killed) {
|
|
923
|
+
const recovered = await runKillRecoveryForCompletion({
|
|
924
|
+
sessionName,
|
|
925
|
+
sessionInfo,
|
|
926
|
+
logPath: statusResult?.logPath || sessionInfo?.logPath || null,
|
|
927
|
+
killed: true,
|
|
928
|
+
env: options.env || process.env,
|
|
929
|
+
runner: options.isolationRunner || null,
|
|
930
|
+
trackSession: options.trackSession || trackSession,
|
|
931
|
+
persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
|
|
932
|
+
locale: sessionInfo?.locale || null,
|
|
933
|
+
verbose,
|
|
934
|
+
});
|
|
935
|
+
killRecovery = recovered.recovery;
|
|
936
|
+
if (recovered.section) killReport.sections.push(recovered.section);
|
|
937
|
+
}
|
|
938
|
+
|
|
868
939
|
const message = formatSessionCompletionMessage({
|
|
869
940
|
sessionName,
|
|
870
941
|
sessionInfo,
|
|
@@ -874,9 +945,33 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
874
945
|
infoBlock: sessionInfo?.infoBlock || '',
|
|
875
946
|
pullRequestUrl,
|
|
876
947
|
pullRequestState,
|
|
877
|
-
extraSections: [...limitsExtraSections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
948
|
+
extraSections: [...limitsExtraSections, ...killReport.sections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
|
|
878
949
|
});
|
|
879
950
|
|
|
951
|
+
if (killReport.killed || killReport.recovered) {
|
|
952
|
+
const notice = await announceKillOnPullRequest({
|
|
953
|
+
pullRequestUrl,
|
|
954
|
+
sessionName,
|
|
955
|
+
sessionInfo,
|
|
956
|
+
diagnosis: killReport.diagnosis,
|
|
957
|
+
exitCode: finalExitCode,
|
|
958
|
+
observedAt: killReport.observedAt,
|
|
959
|
+
policy: killReport.policy,
|
|
960
|
+
recovered: killReport.recovered,
|
|
961
|
+
resumed: killRecovery.resumed,
|
|
962
|
+
recoverySessionId: killRecovery.resumed ? killRecovery.sessionId : null,
|
|
963
|
+
attempt: killRecovery.resumed ? killRecovery.attempt : null,
|
|
964
|
+
maxAttempts: killRecovery.resumed ? killRecovery.maxAttempts : null,
|
|
965
|
+
resumeCommand: killResumeCommand,
|
|
966
|
+
runCommand: options.runCommand || undefined,
|
|
967
|
+
attachLog: options.attachLog || undefined,
|
|
968
|
+
verbose,
|
|
969
|
+
});
|
|
970
|
+
if (verbose && !notice.posted) {
|
|
971
|
+
console.log(`[VERBOSE] Killed-session notice not posted for ${sessionName}: ${notice.skipped || 'unknown reason'}`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
880
975
|
// Update the original reply message if messageId is available, otherwise send new message
|
|
881
976
|
let notifyFromChatId = null;
|
|
882
977
|
let notifyMessageId = null;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verified out-of-memory classification for tracked isolation sessions.
|
|
3
|
+
*
|
|
4
|
+
* Issue #2015 made `oomKilled: true` in a `$ --status` record terminal so an
|
|
5
|
+
* OOM-killed session could not be polled forever. Issue #2134 showed the other
|
|
6
|
+
* half of the problem: Docker's `State.OOMKilled` is a *container* flag, not a
|
|
7
|
+
* statement about the container's main process. The kernel sets it when ANY
|
|
8
|
+
* process in the container cgroup is OOM-killed, and it stays `true` afterwards
|
|
9
|
+
* — so a container can be flagged, keep running, and exit 0 (see
|
|
10
|
+
* https://github.com/moby/moby/issues/47618).
|
|
11
|
+
*
|
|
12
|
+
* That is exactly what happened in #2134: the host ran out of memory, the OOM
|
|
13
|
+
* killer terminated a child process, `$ --status` flipped to
|
|
14
|
+
* `executed / exitCode 137`, and Hive Mind announced
|
|
15
|
+
* "Work session killed — out of memory or forced kill (SIGKILL)" while
|
|
16
|
+
* `docker inspect` still reported the container **running**. The session went on
|
|
17
|
+
* for another 3.5 hours and auto-merged its pull request, unmonitored.
|
|
18
|
+
*
|
|
19
|
+
* This module keeps #2015's guarantee (a truly OOM-killed session is terminal)
|
|
20
|
+
* while refusing to declare a kill that is contradicted by stronger evidence,
|
|
21
|
+
* following the same ladder the rest of the monitor already uses:
|
|
22
|
+
*
|
|
23
|
+
* 1. The log FOOTER wins. A written `Exit Code: N` is proof of how the command
|
|
24
|
+
* actually ended — including `Exit Code: 0`, which means the session merely
|
|
25
|
+
* *survived* an OOM event.
|
|
26
|
+
* 2. LIVENESS beats the status record. No footer + the backing container is
|
|
27
|
+
* still alive → the session is still running; keep polling and remember the
|
|
28
|
+
* OOM event so the eventual completion can report the recovery.
|
|
29
|
+
* 3. Otherwise report `oom-killed`, exactly as issue #2015 requires.
|
|
30
|
+
*
|
|
31
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2015
|
|
32
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Field written on the persisted session snapshot the first time an OOM event is
|
|
39
|
+
* observed for a session that is still alive. It survives a bot restart, so the
|
|
40
|
+
* completion message can still say "recovered from out of memory" hours later.
|
|
41
|
+
*/
|
|
42
|
+
export const OOM_EVENT_OBSERVED_FIELD = 'oomEventObservedAt';
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Remember that the kernel OOM killer fired inside this session's container.
|
|
46
|
+
* Idempotent: only the FIRST observation timestamp is kept.
|
|
47
|
+
*
|
|
48
|
+
* @param {Object} sessionInfo - Mutable persisted session info
|
|
49
|
+
* @param {Function} [persistSnapshot] - Callback that mirrors sessionInfo to disk
|
|
50
|
+
* @returns {boolean} True when this call recorded a new observation
|
|
51
|
+
*/
|
|
52
|
+
export function markOomEventObserved(sessionInfo, persistSnapshot) {
|
|
53
|
+
if (!sessionInfo || sessionInfo[OOM_EVENT_OBSERVED_FIELD]) return false;
|
|
54
|
+
sessionInfo[OOM_EVENT_OBSERVED_FIELD] = new Date().toISOString();
|
|
55
|
+
if (typeof persistSnapshot === 'function') persistSnapshot();
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Whether an OOM event was observed for this session while it was running.
|
|
61
|
+
*
|
|
62
|
+
* @param {Object} sessionInfo
|
|
63
|
+
* @returns {string|null} ISO timestamp of the first observation, or null
|
|
64
|
+
*/
|
|
65
|
+
export function getOomEventObservedAt(sessionInfo) {
|
|
66
|
+
return sessionInfo?.[OOM_EVENT_OBSERVED_FIELD] || null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function probeBackendAlive(sessionName, sessionInfo, { verbose, runner, backendAlive }) {
|
|
70
|
+
if (!sessionInfo?.isolationBackend) return null;
|
|
71
|
+
const probe = backendAlive || runner?.checkBackendSessionAlive;
|
|
72
|
+
if (!probe) return null;
|
|
73
|
+
try {
|
|
74
|
+
return await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
if (verbose) {
|
|
77
|
+
console.log(`[VERBOSE] Session ${sessionName} OOM liveness probe failed: ${error?.message || error}`);
|
|
78
|
+
}
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Resolve the real state of a session whose status record carries
|
|
85
|
+
* `oomKilled: true`.
|
|
86
|
+
*
|
|
87
|
+
* @param {string} sessionName
|
|
88
|
+
* @param {Object} sessionInfo
|
|
89
|
+
* @param {Object} statusResult - Parsed `$ --status` payload
|
|
90
|
+
* @param {Object} deps
|
|
91
|
+
* @param {boolean} [deps.verbose]
|
|
92
|
+
* @param {Object} deps.runner - Isolation runner (for its default probes)
|
|
93
|
+
* @param {Function} [deps.exitFromLog] - Injectable footer reader
|
|
94
|
+
* @param {Function} [deps.backendAlive] - Injectable liveness probe
|
|
95
|
+
* @param {Function} [deps.persistSnapshot] - Persist the session snapshot
|
|
96
|
+
* @returns {Promise<Object>} Monitor state object
|
|
97
|
+
*/
|
|
98
|
+
export async function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive, persistSnapshot } = {}) {
|
|
99
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
100
|
+
let footer = null;
|
|
101
|
+
if (logPath) {
|
|
102
|
+
const readFooter = exitFromLog || runner?.readSessionExitFromLog;
|
|
103
|
+
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 1. The authoritative log footer.
|
|
107
|
+
if (footer?.finished) {
|
|
108
|
+
const footerExitCode = normalizeExitCode(footer.exitCode);
|
|
109
|
+
const correctedStatus = classifyExitStatus(footerExitCode) || (footerExitCode === 0 ? 'executed' : 'failed');
|
|
110
|
+
const survivedOom = footerExitCode === 0;
|
|
111
|
+
if (survivedOom) markOomEventObserved(sessionInfo, persistSnapshot);
|
|
112
|
+
if (verbose) {
|
|
113
|
+
console.log(`[VERBOSE] Session ${sessionName} reported oomKilled=true, but its log footer says exit ${footerExitCode} (${correctedStatus}) and wins${survivedOom ? ' — the session SURVIVED the out-of-memory event (issue #2134)' : ''}`);
|
|
114
|
+
}
|
|
115
|
+
return {
|
|
116
|
+
running: false,
|
|
117
|
+
exitCode: footerExitCode,
|
|
118
|
+
status: correctedStatus,
|
|
119
|
+
statusResult: { ...statusResult, status: correctedStatus, exitCode: footerExitCode, endTime: statusResult?.endTime || footer.endTime || null },
|
|
120
|
+
oomEventObserved: true,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// 2. Liveness: an alive container cannot have had its command killed.
|
|
125
|
+
const alive = await probeBackendAlive(sessionName, sessionInfo, { verbose, runner, backendAlive });
|
|
126
|
+
if (alive === true) {
|
|
127
|
+
markOomEventObserved(sessionInfo, persistSnapshot);
|
|
128
|
+
if (verbose) {
|
|
129
|
+
console.log(`[VERBOSE] Session ${sessionName} reported oomKilled=true but its ${sessionInfo.isolationBackend} backend is still alive; an OOM event hit the container, not the command — keeping the session tracked (issue #2134)`);
|
|
130
|
+
}
|
|
131
|
+
return { running: true, exitCode: null, status: statusResult?.status || 'executing', statusResult, deferred: true, oomEventObserved: true };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// 3. Nothing contradicts the status record: this really is an OOM kill (#2015).
|
|
135
|
+
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
136
|
+
let exitCode = 137;
|
|
137
|
+
if (statusExitCode !== null && statusExitCode > 0) {
|
|
138
|
+
exitCode = statusExitCode;
|
|
139
|
+
}
|
|
140
|
+
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
141
|
+
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
142
|
+
|
|
143
|
+
if (verbose) {
|
|
144
|
+
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true (backend alive: ${alive === null ? 'unknown' : alive}); treating it as terminal oom-killed (exit ${exitCode})`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true, oomEventObserved: true };
|
|
148
|
+
}
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* @see https://github.com/link-assistant/hive-mind/issues/2015
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import { classifyExitStatus
|
|
17
|
+
import { classifyExitStatus } from './session-status.lib.mjs';
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Issue #1927: minimum age before a session that `$ --status` still reports as
|
|
@@ -122,30 +122,9 @@ export async function resolveStaleExecutingState(sessionName, sessionInfo, statu
|
|
|
122
122
|
|
|
123
123
|
/**
|
|
124
124
|
* Issue #2015: `oomKilled` is terminal — the container was killed by the kernel,
|
|
125
|
-
* so no further polling can change the outcome.
|
|
125
|
+
* so no further polling can change the outcome. Issue #2134 refined that into a
|
|
126
|
+
* *verified* classification (footer/liveness ladder); the implementation now
|
|
127
|
+
* lives in session-monitor.oom.lib.mjs and is re-exported here so existing
|
|
128
|
+
* importers keep working.
|
|
126
129
|
*/
|
|
127
|
-
export
|
|
128
|
-
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
129
|
-
let footer = null;
|
|
130
|
-
if (logPath) {
|
|
131
|
-
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
132
|
-
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
136
|
-
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
137
|
-
let exitCode = 137;
|
|
138
|
-
if (statusExitCode !== null && statusExitCode > 0) {
|
|
139
|
-
exitCode = statusExitCode;
|
|
140
|
-
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
141
|
-
exitCode = footerExitCode;
|
|
142
|
-
}
|
|
143
|
-
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
144
|
-
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
145
|
-
|
|
146
|
-
if (verbose) {
|
|
147
|
-
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
151
|
-
}
|
|
130
|
+
export { resolveOomKilledState, markOomEventObserved, getOomEventObservedAt, OOM_EVENT_OBSERVED_FIELD } from './session-monitor.oom.lib.mjs';
|
|
@@ -43,6 +43,30 @@ const SESSION_LOG_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0
|
|
|
43
43
|
const DEFAULT_LOG_SCAN_CHUNK_BYTES = 262144;
|
|
44
44
|
const LOG_SCAN_OVERLAP_BYTES = 1024;
|
|
45
45
|
|
|
46
|
+
// Issue #2134: the marker scan runs over a log that can contain *anything* the
|
|
47
|
+
// run printed — including Hive Mind's own source code. `solve` prints its id
|
|
48
|
+
// with `` console.log(`📌 Session ID: ${sessionId}`) ``, so a run that `cat`-ed
|
|
49
|
+
// that source made the literal string `${sessionId}` the "last session id" and
|
|
50
|
+
// the bot offered `--resume "${sessionId}"` to the user. Tool session ids are
|
|
51
|
+
// opaque tokens (UUIDs for claude/codex, slugs elsewhere): a shape check plus a
|
|
52
|
+
// small vocabulary of placeholder words rejects that whole class of false hit
|
|
53
|
+
// without narrowing the ids that are legitimately accepted today.
|
|
54
|
+
const SESSION_ID_SHAPE_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{6,199}$/;
|
|
55
|
+
const SESSION_ID_PLACEHOLDERS = new Set(['unknown', 'n/a', 'na', 'none', 'null', 'undefined', 'sessionid', 'session_id', 'session-id', 'id']);
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Whether a token captured after a `Session ID:` label can be a real tool
|
|
59
|
+
* session id (issue #2134).
|
|
60
|
+
*
|
|
61
|
+
* @param {*} id - Captured token.
|
|
62
|
+
* @returns {boolean} True when the token has a plausible session-id shape.
|
|
63
|
+
*/
|
|
64
|
+
export function isPlausibleSessionId(id) {
|
|
65
|
+
if (!id || typeof id !== 'string') return false;
|
|
66
|
+
if (SESSION_ID_PLACEHOLDERS.has(id.toLowerCase())) return false;
|
|
67
|
+
return SESSION_ID_SHAPE_RE.test(id);
|
|
68
|
+
}
|
|
69
|
+
|
|
46
70
|
/**
|
|
47
71
|
* Extract every tool session id printed to a log, in the order they appear.
|
|
48
72
|
*
|
|
@@ -60,8 +84,10 @@ export function extractSessionIds(text) {
|
|
|
60
84
|
SESSION_ID_MARKER_RE.lastIndex = 0;
|
|
61
85
|
while ((match = SESSION_ID_MARKER_RE.exec(text)) !== null) {
|
|
62
86
|
const id = match[1];
|
|
63
|
-
// Skip obvious non-ids that can follow the label in prose/log output
|
|
64
|
-
|
|
87
|
+
// Skip obvious non-ids that can follow the label in prose/log output, and
|
|
88
|
+
// template placeholders such as `${sessionId}` printed by quoted source
|
|
89
|
+
// code (issue #2134).
|
|
90
|
+
if (!isPlausibleSessionId(id)) continue;
|
|
65
91
|
if (ids[ids.length - 1] !== id) ids.push(id);
|
|
66
92
|
}
|
|
67
93
|
return ids;
|
|
@@ -537,7 +537,10 @@ export const processAutoContinueForIssue = async (argv, isIssueUrl, urlNumber, o
|
|
|
537
537
|
|
|
538
538
|
// List all branches in the fork that match the pattern issue-{issueNumber}-* (supports both 8-char and 12-char formats)
|
|
539
539
|
const branchPattern = getIssueBranchPrefix(issueNumber);
|
|
540
|
-
|
|
540
|
+
// Issue #2135: `mirror: false`. The list grows with the repository -
|
|
541
|
+
// a repository worked on by the solver accumulates one branch per
|
|
542
|
+
// issue - and only the matching ones are reported below.
|
|
543
|
+
const branchListResult = await $(QUIET_PROBE)`gh api --paginate repos/${forkRepo}/branches --jq '.[].name'`;
|
|
541
544
|
|
|
542
545
|
if (branchListResult.code === 0) {
|
|
543
546
|
const allBranches = branchListResult.stdout
|
|
@@ -575,7 +578,8 @@ export const processAutoContinueForIssue = async (argv, isIssueUrl, urlNumber, o
|
|
|
575
578
|
|
|
576
579
|
// List all branches in the main repo that match the pattern issue-{issueNumber}-* (supports both 8-char and 12-char formats)
|
|
577
580
|
const branchPattern = getIssueBranchPrefix(issueNumber);
|
|
578
|
-
|
|
581
|
+
// Issue #2135: `mirror: false` - see the fork branch listing above.
|
|
582
|
+
const branchListResult = await $(QUIET_PROBE)`gh api --paginate repos/${owner}/${repo}/branches --jq '.[].name'`;
|
|
579
583
|
|
|
580
584
|
if (branchListResult.code === 0) {
|
|
581
585
|
const allBranches = branchListResult.stdout
|
|
@@ -293,7 +293,7 @@ export const watchUntilMergeable = async params => {
|
|
|
293
293
|
// reproduction run posted "✅ Ready to merge - No pending changes" for a
|
|
294
294
|
// pull request whose net diff was empty, so merging it would have closed
|
|
295
295
|
// the issue without implementing anything.
|
|
296
|
-
const changeStats = await getPullRequestChangeStats({ owner, repo, prNumber,
|
|
296
|
+
const changeStats = await getPullRequestChangeStats({ owner, repo, prNumber, $, log });
|
|
297
297
|
const isEmptyPullRequest = changeStats.measured && !changeStats.hasChanges;
|
|
298
298
|
const emptyPullRequestBlocker = buildEmptyPullRequestBlocker(changeStats);
|
|
299
299
|
if (isEmptyPullRequest) {
|
package/src/solve.config.lib.mjs
CHANGED
|
@@ -154,6 +154,20 @@ export const SOLVE_OPTION_DEFINITIONS = {
|
|
|
154
154
|
description: 'DANGEROUS: skip masking known active local tokens in output. This is separate from other sanitization skip flags and should only be used for controlled debugging.',
|
|
155
155
|
default: false,
|
|
156
156
|
},
|
|
157
|
+
// Issue #2134: one option decides what happens when a working session is
|
|
158
|
+
// killed (out of memory, disk exhaustion, forced kill). The Telegram bot and
|
|
159
|
+
// solve honour the exact same value, so the report never differs by surface.
|
|
160
|
+
'on-session-kill': {
|
|
161
|
+
type: 'string',
|
|
162
|
+
description: 'What to do when a working session is killed (out of memory, disk full, forced kill): "report" describes exactly what happened in the pull request and in Telegram, "resume" additionally starts a new working session to recover and says so in both places. Can also be set with HIVE_MIND_ON_SESSION_KILL.',
|
|
163
|
+
choices: ['report', 'resume'],
|
|
164
|
+
default: 'report',
|
|
165
|
+
},
|
|
166
|
+
'session-kill-resume-attempts': {
|
|
167
|
+
type: 'number',
|
|
168
|
+
description: 'Maximum number of automatic recovery working sessions started for one killed session when --on-session-kill=resume. Can also be set with HIVE_MIND_SESSION_KILL_RESUME_ATTEMPTS.',
|
|
169
|
+
default: 1,
|
|
170
|
+
},
|
|
157
171
|
'auto-close-pull-request-on-fail': {
|
|
158
172
|
type: 'boolean',
|
|
159
173
|
description: 'Automatically close the pull request if execution fails',
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { QUIET_PROBE } from './quiet-probe.lib.mjs';
|
|
2
3
|
import { ensureUseM } from './use-m-bootstrap.lib.mjs';
|
|
3
4
|
|
|
4
5
|
/**
|
|
@@ -68,7 +69,8 @@ export const collectDeferredWorkSources = async ({ owner, repo, prNumber, result
|
|
|
68
69
|
|
|
69
70
|
// 1. Pull request description
|
|
70
71
|
try {
|
|
71
|
-
|
|
72
|
+
// Issue #2135: `mirror: false` - the description is scanned here, not shown.
|
|
73
|
+
const prResult = await $(QUIET_PROBE)`gh api repos/${owner}/${repo}/pulls/${prNumber} --jq '.body // ""'`;
|
|
72
74
|
if (prResult.code === 0) {
|
|
73
75
|
const body = prResult.stdout.toString();
|
|
74
76
|
if (body && body.trim()) {
|
|
@@ -86,7 +88,10 @@ export const collectDeferredWorkSources = async ({ owner, repo, prNumber, result
|
|
|
86
88
|
|
|
87
89
|
// 3. Changed markdown documents (scan only added lines from the diff)
|
|
88
90
|
try {
|
|
89
|
-
|
|
91
|
+
// Issue #2135: `mirror: false`. Every entry carries the file's patch, so
|
|
92
|
+
// this answer is as large as the pull request's diff - and it was being
|
|
93
|
+
// echoed into the log that gets attached to that same pull request.
|
|
94
|
+
const filesResult = await $(QUIET_PROBE)`gh api repos/${owner}/${repo}/pulls/${prNumber}/files --paginate`;
|
|
90
95
|
if (filesResult.code === 0) {
|
|
91
96
|
const files = JSON.parse(filesResult.stdout.toString() || '[]');
|
|
92
97
|
for (const file of files) {
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
* @see case-studies/issue-661-session-resume-cost-optimization/
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { QUIET_PROBE } from './quiet-probe.lib.mjs';
|
|
13
|
+
|
|
12
14
|
// Note: This module does not import $ directly
|
|
13
15
|
// Functions receive $ as a parameter from the calling module
|
|
14
16
|
// This ensures consistent command executor usage across the codebase
|
|
@@ -28,9 +30,11 @@ export const generateMinimalRestartPrompt = async (tempDir, $) => {
|
|
|
28
30
|
const uncommittedFiles = gitStatus.stdout.toString().trim();
|
|
29
31
|
|
|
30
32
|
// Get brief diff summaries (not full diffs to keep the prompt minimal)
|
|
31
|
-
|
|
33
|
+
// Issue #2135: `mirror: false` - the summaries go into the prompt below, so
|
|
34
|
+
// echoing them into the log only duplicates them into the attached log file.
|
|
35
|
+
const gitDiffStat = await $({ cwd: tempDir, ...QUIET_PROBE })`git diff --stat`;
|
|
32
36
|
const unstagedDiffSummary = gitDiffStat.stdout.toString().trim();
|
|
33
|
-
const gitCachedDiffStat = await $({ cwd: tempDir })`git diff --cached --stat`;
|
|
37
|
+
const gitCachedDiffStat = await $({ cwd: tempDir, ...QUIET_PROBE })`git diff --cached --stat`;
|
|
34
38
|
const stagedDiffSummary = gitCachedDiffStat.stdout.toString().trim();
|
|
35
39
|
const summarySections = [];
|
|
36
40
|
if (unstagedDiffSummary) summarySections.push(`Unstaged changes:\n${unstagedDiffSummary}`);
|
|
@@ -69,7 +73,11 @@ export const generateFullRestartPrompt = async (issueUrl, issueBody, prNumber, f
|
|
|
69
73
|
const gitStatus = await $({ cwd: tempDir })`git status --porcelain`;
|
|
70
74
|
const uncommittedFiles = gitStatus.stdout.toString().trim();
|
|
71
75
|
|
|
72
|
-
|
|
76
|
+
// Issue #2135: `mirror: false`. This is the working tree's whole diff and it
|
|
77
|
+
// is embedded in the prompt below; mirroring it wrote a second copy into the
|
|
78
|
+
// session log, which is attached to the pull request and (with
|
|
79
|
+
// --development-log) committed into the branch the diff is taken from.
|
|
80
|
+
const gitDiff = await $({ cwd: tempDir, ...QUIET_PROBE })`git diff`;
|
|
73
81
|
const fullDiff = gitDiff.stdout.toString();
|
|
74
82
|
|
|
75
83
|
let prompt = `
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
7
|
+
import { quietProbe } from './quiet-probe.lib.mjs';
|
|
7
8
|
// Import feedback detection functionality
|
|
8
9
|
const feedback = await import('./solve.feedback.lib.mjs');
|
|
9
10
|
const { detectAndCountFeedback } = feedback;
|
|
@@ -45,7 +46,10 @@ export async function prepareFeedbackAndTimestamps({ tempDir = null, prNumber, b
|
|
|
45
46
|
|
|
46
47
|
// Get the last comment's timestamp (if any)
|
|
47
48
|
// Use --paginate to get all comments - GitHub API returns max 30 per page by default
|
|
48
|
-
|
|
49
|
+
// Issue #2135: `mirror: false`. Only the last comment's timestamp is read
|
|
50
|
+
// from this answer, but the answer itself is every comment body on the
|
|
51
|
+
// issue - tens of kilobytes echoed into the log on every run.
|
|
52
|
+
const commentsResult = await quietProbe($)`gh api repos/${owner}/${repo}/issues/${issueNumber}/comments --paginate`;
|
|
49
53
|
|
|
50
54
|
if (commentsResult.code !== 0) {
|
|
51
55
|
await log(`Warning: Failed to get comments: ${commentsResult.stderr ? commentsResult.stderr.toString() : 'Unknown error'}`, { level: 'warning' });
|
|
@@ -30,6 +30,7 @@ import { LIVE_PROGRESS_SECTION_START_MARKER, LIVE_PROGRESS_SECTION_END_MARKER, p
|
|
|
30
30
|
import { writeSanitizedPublicationFile } from './token-sanitization.lib.mjs';
|
|
31
31
|
|
|
32
32
|
import { wrapDollarWithGhRetry as _wrapDollarWithGhRetry } from './github-rate-limit.lib.mjs'; // rate-limit marker (#1726): gh API calls flow through $ wrapped by caller
|
|
33
|
+
import { quietProbe } from './quiet-probe.lib.mjs'; // issue #2135: keep large read-only probe payloads out of the attached log
|
|
33
34
|
/**
|
|
34
35
|
* Configuration constants for progress monitoring
|
|
35
36
|
*/
|
|
@@ -280,7 +281,10 @@ export const createProgressMonitor = ({ owner, repo, prNumber, $, log, verbose =
|
|
|
280
281
|
state.currentTodos = todos;
|
|
281
282
|
|
|
282
283
|
// Fetch current PR description
|
|
283
|
-
|
|
284
|
+
// Issue #2135: `mirror: false`. This runs on every progress update and
|
|
285
|
+
// the answer is the whole pull-request description, which by then holds
|
|
286
|
+
// the progress section itself.
|
|
287
|
+
const prData = await quietProbe($)`gh pr view ${prNumber} --repo ${owner}/${repo} --json body`;
|
|
284
288
|
const prInfo = JSON.parse(prData.stdout);
|
|
285
289
|
let currentBody = prInfo.body || '';
|
|
286
290
|
|
|
@@ -65,7 +65,8 @@ export const checkExistingForkOfRoot = async rootRepo => {
|
|
|
65
65
|
// not to the shell, and command-stream quotes interpolated values itself - so
|
|
66
66
|
// interpolating inside the quotes would leak shell quotes into the comparison.
|
|
67
67
|
const forkFilter = `.[] | select(.owner.login == ${JSON.stringify(currentUser)}) | .full_name`;
|
|
68
|
-
|
|
68
|
+
// Issue #2135: `mirror: false` - see the fork-name lookup below.
|
|
69
|
+
const forksResult = await lib.ghCmdRetry(() => $(QUIET_PROBE)`gh api repos/${rootRepo}/forks --paginate --jq ${forkFilter}`, { label: `check forks of ${rootRepo}` });
|
|
69
70
|
if (forksResult.code !== 0) return null;
|
|
70
71
|
|
|
71
72
|
const forks = forksResult.stdout
|
|
@@ -1225,7 +1226,9 @@ export const setupPrForkRemote = async (tempDir, argv, prForkOwner, repo, isCont
|
|
|
1225
1226
|
// Issue #2119: the double quotes here are jq syntax, so the expression is
|
|
1226
1227
|
// built in JS and interpolated as one already-escaped argument.
|
|
1227
1228
|
const forkNameFilter = `.[] | select(.owner.login == ${JSON.stringify(prForkOwner)}) | .name`;
|
|
1228
|
-
|
|
1229
|
+
// Issue #2135: `mirror: false` - a popular repository has thousands of
|
|
1230
|
+
// forks, and only the matching name is used.
|
|
1231
|
+
const forksResult = await $(QUIET_PROBE)`gh api repos/${owner}/${repo}/forks --paginate --jq ${forkNameFilter}`;
|
|
1229
1232
|
if (forksResult.code === 0 && forksResult.stdout) {
|
|
1230
1233
|
const forkName = forksResult.stdout.toString().trim().split('\n')[0]; // Take first match
|
|
1231
1234
|
if (forkName) {
|