@link-assistant/hive-mind 2.11.7 → 2.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/package.json +1 -1
- package/src/bidirectional-interactive.lib.mjs +6 -2
- package/src/child-exit.lib.mjs +107 -0
- package/src/contributing-guidelines.lib.mjs +19 -6
- package/src/development-log.lib.mjs +82 -5
- package/src/fix.ci-cd-issue.lib.mjs +5 -3
- package/src/fix.mjs +5 -2
- package/src/github-entity-validation.lib.mjs +4 -1
- package/src/github.lib.mjs +3 -3
- package/src/hive.mjs +15 -21
- package/src/isolation-runner.lib.mjs +5 -2
- package/src/lib.mjs +21 -0
- package/src/locales/en.lino +10 -0
- package/src/locales/hi.lino +10 -0
- package/src/locales/ru.lino +10 -0
- package/src/locales/zh.lino +10 -0
- package/src/log-growth.lib.mjs +94 -0
- package/src/option-suggestions.lib.mjs +2 -0
- package/src/pull-request-changes.lib.mjs +94 -24
- package/src/review.mjs +12 -3
- package/src/session-kill-diagnostics.lib.mjs +388 -0
- package/src/session-kill-policy.lib.mjs +96 -0
- package/src/session-kill-recovery.lib.mjs +256 -0
- package/src/session-kill-resume.lib.mjs +175 -0
- package/src/session-monitor.kill-sections.lib.mjs +198 -0
- package/src/session-monitor.lib.mjs +97 -2
- package/src/session-monitor.oom.lib.mjs +148 -0
- package/src/session-monitor.stale-executing.lib.mjs +6 -27
- package/src/session-resume.lib.mjs +28 -2
- package/src/solve.auto-continue.lib.mjs +6 -2
- package/src/solve.auto-merge.lib.mjs +1 -1
- package/src/solve.config.lib.mjs +14 -0
- package/src/solve.keep-working.lib.mjs +7 -2
- package/src/solve.minimal-restart-prompt.lib.mjs +11 -3
- package/src/solve.preparation.lib.mjs +5 -1
- package/src/solve.progress-monitoring.lib.mjs +5 -1
- package/src/solve.repository.lib.mjs +5 -2
- package/src/solve.results.lib.mjs +13 -8
- package/src/task.mjs +5 -3
- package/src/telegram-bot.mjs +3 -1
- package/src/telegram-command-execution.lib.mjs +5 -2
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Kill-cause diagnostics for work sessions (issue #2134).
|
|
3
|
+
*
|
|
4
|
+
* "Work session killed — out of memory **or** forced kill (SIGKILL)" was as much
|
|
5
|
+
* as Hive Mind could say, because nothing ever looked at the machine state — even
|
|
6
|
+
* though every ingredient was already being collected and thrown away:
|
|
7
|
+
*
|
|
8
|
+
* - `src/solve.resource-diagnostics.lib.mjs` writes `📈 [RESOURCES]` markers
|
|
9
|
+
* (memory available/total, disk used percent) into every session log;
|
|
10
|
+
* - the bot records the same snapshot every heartbeat;
|
|
11
|
+
* - on Linux the kernel exposes the ground truth in
|
|
12
|
+
* `/sys/fs/cgroup/memory.events` (`oom`, `oom_kill`), `/proc/meminfo`,
|
|
13
|
+
* `/proc/pressure/memory` and the OOM killer's `dmesg` lines, which name the
|
|
14
|
+
* exact victim process.
|
|
15
|
+
*
|
|
16
|
+
* This module turns those into ONE explicit verdict — out of memory / disk full /
|
|
17
|
+
* forced kill — with the evidence that produced it, so both the Telegram message
|
|
18
|
+
* and the pull-request notice can say what actually happened. Every probe is
|
|
19
|
+
* injectable and never throws: a diagnosis is a best-effort enrichment, never a
|
|
20
|
+
* reason to fail a completion notification.
|
|
21
|
+
*
|
|
22
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import fsPromises from 'fs/promises';
|
|
26
|
+
import { exec as execCallback } from 'child_process';
|
|
27
|
+
import { promisify } from 'util';
|
|
28
|
+
import { t } from './i18n.lib.mjs';
|
|
29
|
+
import { formatBytes, parseResourceMarkers } from './solve.resource-diagnostics.lib.mjs';
|
|
30
|
+
|
|
31
|
+
const exec = promisify(execCallback);
|
|
32
|
+
|
|
33
|
+
export const KILL_CAUSE_OUT_OF_MEMORY = 'out-of-memory';
|
|
34
|
+
export const KILL_CAUSE_DISK_FULL = 'disk-full';
|
|
35
|
+
export const KILL_CAUSE_FORCED_KILL = 'forced-kill';
|
|
36
|
+
export const KILL_CAUSE_UNKNOWN = 'unknown';
|
|
37
|
+
|
|
38
|
+
/** Memory is considered exhausted below this share of total RAM still available. */
|
|
39
|
+
export const MEMORY_EXHAUSTED_AVAILABLE_RATIO = 0.1;
|
|
40
|
+
/** Disk is considered full at or above this used percentage… */
|
|
41
|
+
export const DISK_FULL_USED_PERCENT = 95;
|
|
42
|
+
/** …or below this much free space, whichever triggers first. */
|
|
43
|
+
export const DISK_FULL_AVAILABLE_BYTES = 512 * 1024 * 1024;
|
|
44
|
+
|
|
45
|
+
function text(locale, key, fallback, params = {}) {
|
|
46
|
+
if (!locale) return fallback;
|
|
47
|
+
return t(key, params, { locale });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function finite(value) {
|
|
51
|
+
return Number.isFinite(value) ? value : null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The most recent `📈 [RESOURCES]` marker that carries usable memory data —
|
|
56
|
+
* i.e. the closest reading to the moment the session died.
|
|
57
|
+
*
|
|
58
|
+
* @param {{markers: Array}|null} parsed - Output of parseResourceMarkers()
|
|
59
|
+
* @returns {Object|null}
|
|
60
|
+
*/
|
|
61
|
+
export function selectLastMemoryResourceMarker(parsed) {
|
|
62
|
+
const markers = Array.isArray(parsed?.markers) ? parsed.markers : [];
|
|
63
|
+
for (let i = markers.length - 1; i >= 0; i--) {
|
|
64
|
+
if (finite(markers[i]?.memory?.availableBytes) !== null) return markers[i];
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The most recent marker that carries usable disk data.
|
|
71
|
+
*
|
|
72
|
+
* @param {{markers: Array}|null} parsed
|
|
73
|
+
* @returns {Object|null}
|
|
74
|
+
*/
|
|
75
|
+
export function selectLastDiskResourceMarker(parsed) {
|
|
76
|
+
const markers = Array.isArray(parsed?.markers) ? parsed.markers : [];
|
|
77
|
+
for (let i = markers.length - 1; i >= 0; i--) {
|
|
78
|
+
if (finite(markers[i]?.disk?.usedPercent) !== null || finite(markers[i]?.disk?.availableBytes) !== null) return markers[i];
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function parseCgroupKeyedFile(content) {
|
|
84
|
+
const out = {};
|
|
85
|
+
for (const line of String(content || '').split(/\r?\n/)) {
|
|
86
|
+
const match = /^(\S+)\s+(\d+)$/.exec(line.trim());
|
|
87
|
+
if (match) out[match[1]] = Number(match[2]);
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function parseMeminfo(content) {
|
|
93
|
+
const fields = {};
|
|
94
|
+
for (const line of String(content || '').split(/\r?\n/)) {
|
|
95
|
+
const match = /^(\w+):\s+(\d+)\s*kB$/.exec(line.trim());
|
|
96
|
+
if (match) fields[match[1]] = Number(match[2]) * 1024;
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
totalBytes: finite(fields.MemTotal),
|
|
100
|
+
availableBytes: finite(fields.MemAvailable),
|
|
101
|
+
swapTotalBytes: finite(fields.SwapTotal),
|
|
102
|
+
swapFreeBytes: finite(fields.SwapFree),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Parse the kernel OOM killer's own report out of `dmesg` output. Both the
|
|
108
|
+
* modern `oom-kill:...,task=<comm>,pid=<pid>` summary and the classic
|
|
109
|
+
* `Killed process <pid> (<comm>)` line are recognised, because which one appears
|
|
110
|
+
* depends on the kernel version.
|
|
111
|
+
*
|
|
112
|
+
* @param {string} dmesgText
|
|
113
|
+
* @returns {Array<{pid: number|null, comm: string|null, line: string}>}
|
|
114
|
+
*/
|
|
115
|
+
export function parseOomVictims(dmesgText) {
|
|
116
|
+
const victims = [];
|
|
117
|
+
for (const line of String(dmesgText || '').split(/\r?\n/)) {
|
|
118
|
+
const killed = /Killed process (\d+)\s+\(([^)]+)\)/.exec(line);
|
|
119
|
+
if (killed) {
|
|
120
|
+
victims.push({ pid: Number(killed[1]), comm: killed[2], line: line.trim() });
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const oomKill = /oom-kill:.*?task=([^,\s]+).*?pid=(\d+)/.exec(line);
|
|
124
|
+
if (oomKill) {
|
|
125
|
+
victims.push({ pid: Number(oomKill[2]), comm: oomKill[1], line: line.trim() });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return victims;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Read every kernel-side kill signal this machine exposes. Linux-only in
|
|
133
|
+
* practice; on other platforms (or in a sandbox where the files are not
|
|
134
|
+
* readable) the corresponding fields stay null and the reason is recorded in
|
|
135
|
+
* `errors` so a `--verbose` run can show why a diagnosis was thin (R8/#2134).
|
|
136
|
+
*
|
|
137
|
+
* @param {Object} [options]
|
|
138
|
+
* @param {Function} [options.readFile] - Injectable fs.promises.readFile
|
|
139
|
+
* @param {Function} [options.execImpl] - Injectable promisified exec (for dmesg)
|
|
140
|
+
* @param {boolean} [options.verbose]
|
|
141
|
+
* @param {boolean} [options.includeDmesg=true]
|
|
142
|
+
* @returns {Promise<Object>} System diagnostics (never throws)
|
|
143
|
+
*/
|
|
144
|
+
export async function collectSystemKillDiagnostics({ readFile = fsPromises.readFile, execImpl = exec, verbose = false, includeDmesg = true, cgroupPath = '/sys/fs/cgroup' } = {}) {
|
|
145
|
+
const result = { cgroup: { oom: null, oomKill: null, maxBytes: null, currentBytes: null }, memory: { totalBytes: null, availableBytes: null, swapTotalBytes: null, swapFreeBytes: null }, pressure: null, victims: [], errors: [] };
|
|
146
|
+
|
|
147
|
+
const readOptional = async (file, label) => {
|
|
148
|
+
try {
|
|
149
|
+
return await readFile(file, 'utf8');
|
|
150
|
+
} catch (error) {
|
|
151
|
+
result.errors.push(`${label}: ${error?.message || error}`);
|
|
152
|
+
if (verbose) console.log(`[VERBOSE] kill-diagnostics: could not read ${file}: ${error?.message || error}`);
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
const events = await readOptional(`${cgroupPath}/memory.events`, 'cgroup memory.events');
|
|
158
|
+
if (events) {
|
|
159
|
+
const parsed = parseCgroupKeyedFile(events);
|
|
160
|
+
result.cgroup.oom = finite(parsed.oom);
|
|
161
|
+
result.cgroup.oomKill = finite(parsed.oom_kill);
|
|
162
|
+
}
|
|
163
|
+
const max = await readOptional(`${cgroupPath}/memory.max`, 'cgroup memory.max');
|
|
164
|
+
if (max) {
|
|
165
|
+
const trimmed = max.trim();
|
|
166
|
+
result.cgroup.maxBytes = trimmed === 'max' ? null : finite(Number(trimmed));
|
|
167
|
+
}
|
|
168
|
+
const current = await readOptional(`${cgroupPath}/memory.current`, 'cgroup memory.current');
|
|
169
|
+
if (current) result.cgroup.currentBytes = finite(Number(current.trim()));
|
|
170
|
+
|
|
171
|
+
const meminfo = await readOptional('/proc/meminfo', '/proc/meminfo');
|
|
172
|
+
if (meminfo) result.memory = parseMeminfo(meminfo);
|
|
173
|
+
|
|
174
|
+
const pressure = await readOptional('/proc/pressure/memory', '/proc/pressure/memory');
|
|
175
|
+
if (pressure) result.pressure = pressure.trim().split(/\r?\n/)[0] || null;
|
|
176
|
+
|
|
177
|
+
if (includeDmesg) {
|
|
178
|
+
try {
|
|
179
|
+
const { stdout } = await execImpl('dmesg -T 2>/dev/null | grep -iE "out of memory|oom-kill|killed process" | tail -20', { timeout: 10000, maxBuffer: 1024 * 1024 });
|
|
180
|
+
result.victims = parseOomVictims(stdout);
|
|
181
|
+
} catch (error) {
|
|
182
|
+
// A non-root container usually cannot read the kernel ring buffer, and
|
|
183
|
+
// grep exits 1 when nothing matched. Both are normal, not failures.
|
|
184
|
+
result.errors.push(`dmesg: ${error?.message || error}`);
|
|
185
|
+
if (verbose) console.log(`[VERBOSE] kill-diagnostics: dmesg probe unavailable: ${error?.message || error}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
if (verbose) {
|
|
190
|
+
console.log(`[VERBOSE] kill-diagnostics: cgroup oom=${result.cgroup.oom} oom_kill=${result.cgroup.oomKill}, mem available=${formatBytes(result.memory.availableBytes)}/${formatBytes(result.memory.totalBytes)}, victims=${result.victims.length}`);
|
|
191
|
+
}
|
|
192
|
+
return result;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function memoryRatio(memory) {
|
|
196
|
+
const available = finite(memory?.availableBytes);
|
|
197
|
+
const total = finite(memory?.totalBytes);
|
|
198
|
+
if (available === null || total === null || total <= 0) return null;
|
|
199
|
+
return available / total;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function describeMemory(memory, timestamp) {
|
|
203
|
+
const available = finite(memory?.availableBytes);
|
|
204
|
+
const total = finite(memory?.totalBytes);
|
|
205
|
+
if (available === null || total === null) return null;
|
|
206
|
+
const usedPercent = total > 0 ? (100 * (total - available)) / total : null;
|
|
207
|
+
const at = timestamp ? ` at ${timestamp}` : '';
|
|
208
|
+
return `${formatBytes(available)} of ${formatBytes(total)} RAM available${usedPercent === null ? '' : ` (${usedPercent.toFixed(1)}% used)`}${at}`;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function describeDisk(disk, timestamp) {
|
|
212
|
+
const usedPercent = finite(disk?.usedPercent);
|
|
213
|
+
const available = finite(disk?.availableBytes);
|
|
214
|
+
if (usedPercent === null && available === null) return null;
|
|
215
|
+
const at = timestamp ? ` at ${timestamp}` : '';
|
|
216
|
+
const parts = [];
|
|
217
|
+
if (available !== null) parts.push(`${formatBytes(available)} free`);
|
|
218
|
+
if (finite(disk?.totalBytes) !== null) parts.push(`of ${formatBytes(disk.totalBytes)}`);
|
|
219
|
+
if (usedPercent !== null) parts.push(`(${usedPercent.toFixed(1)}% used)`);
|
|
220
|
+
return `disk ${disk?.path || '/'}: ${parts.join(' ')}${at}`;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Decide WHY a session was killed, from the evidence available.
|
|
225
|
+
*
|
|
226
|
+
* The order encodes confidence: a kernel OOM report or a container OOM flag is
|
|
227
|
+
* proof; a resource marker showing exhausted memory/disk is strong
|
|
228
|
+
* circumstantial evidence; a signal exit with healthy resources is a forced
|
|
229
|
+
* kill (an operator, a supervisor, or `docker stop`).
|
|
230
|
+
*
|
|
231
|
+
* @param {Object} [params]
|
|
232
|
+
* @param {string} [params.logText] - Session log text (parsed for markers)
|
|
233
|
+
* @param {Object} [params.resourceMarkers] - Pre-parsed parseResourceMarkers() output
|
|
234
|
+
* @param {boolean} [params.oomKilled] - Docker `State.OOMKilled`
|
|
235
|
+
* @param {number|null} [params.exitCode]
|
|
236
|
+
* @param {Object|null} [params.system] - collectSystemKillDiagnostics() result
|
|
237
|
+
* @param {boolean} [params.stopRequestedByUser] - The operator asked for the stop
|
|
238
|
+
* @returns {{cause: string, summary: string, evidence: string[], memory: Object|null, disk: Object|null, victims: Array}}
|
|
239
|
+
*/
|
|
240
|
+
export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false } = {}) {
|
|
241
|
+
const parsed = resourceMarkers || (logText ? parseResourceMarkers(logText) : { markers: [], byPhase: {} });
|
|
242
|
+
const memoryMarker = selectLastMemoryResourceMarker(parsed);
|
|
243
|
+
const diskMarker = selectLastDiskResourceMarker(parsed);
|
|
244
|
+
const memory = memoryMarker?.memory || null;
|
|
245
|
+
const disk = diskMarker?.disk || null;
|
|
246
|
+
const victims = Array.isArray(system?.victims) ? system.victims : [];
|
|
247
|
+
const cgroupOomKills = finite(system?.cgroup?.oomKill);
|
|
248
|
+
|
|
249
|
+
const evidence = [];
|
|
250
|
+
const memoryLine = describeMemory(memory, memoryMarker?.timestamp || null);
|
|
251
|
+
if (memoryLine) evidence.push(`last session memory reading — ${memoryLine} (phase \`${memoryMarker.phase}\`)`);
|
|
252
|
+
const diskLine = describeDisk(disk, diskMarker?.timestamp || null);
|
|
253
|
+
if (diskLine) evidence.push(`last session ${diskLine} (phase \`${diskMarker.phase}\`)`);
|
|
254
|
+
if (oomKilled) evidence.push('container reports `State.OOMKilled = true` (an OOM event hit the container cgroup)');
|
|
255
|
+
if (cgroupOomKills !== null && cgroupOomKills > 0) evidence.push(`cgroup \`memory.events\` reports ${cgroupOomKills} OOM kill(s)`);
|
|
256
|
+
const systemMemoryLine = describeMemory(system?.memory, null);
|
|
257
|
+
if (systemMemoryLine) evidence.push(`host memory now — ${systemMemoryLine}`);
|
|
258
|
+
if (system?.pressure) evidence.push(`\`/proc/pressure/memory\`: ${system.pressure}`);
|
|
259
|
+
for (const victim of victims.slice(-3)) {
|
|
260
|
+
evidence.push(`kernel OOM killer terminated \`${victim.comm || 'unknown'}\` (pid ${victim.pid ?? '?'})`);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const ratio = memoryRatio(memory);
|
|
264
|
+
const memoryExhausted = ratio !== null && ratio <= MEMORY_EXHAUSTED_AVAILABLE_RATIO;
|
|
265
|
+
const diskUsedPercent = finite(disk?.usedPercent);
|
|
266
|
+
const diskAvailable = finite(disk?.availableBytes);
|
|
267
|
+
const diskFull = (diskUsedPercent !== null && diskUsedPercent >= DISK_FULL_USED_PERCENT) || (diskAvailable !== null && diskAvailable <= DISK_FULL_AVAILABLE_BYTES);
|
|
268
|
+
|
|
269
|
+
let cause = KILL_CAUSE_UNKNOWN;
|
|
270
|
+
if (stopRequestedByUser) {
|
|
271
|
+
cause = KILL_CAUSE_FORCED_KILL;
|
|
272
|
+
} else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted) {
|
|
273
|
+
cause = KILL_CAUSE_OUT_OF_MEMORY;
|
|
274
|
+
} else if (diskFull) {
|
|
275
|
+
cause = KILL_CAUSE_DISK_FULL;
|
|
276
|
+
} else if (exitCode !== null && exitCode > 128) {
|
|
277
|
+
cause = KILL_CAUSE_FORCED_KILL;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
let summary;
|
|
281
|
+
if (cause === KILL_CAUSE_OUT_OF_MEMORY) {
|
|
282
|
+
const victim = victims.length > 0 ? `, kernel OOM killer terminated \`${victims[victims.length - 1].comm || 'unknown'}\` (pid ${victims[victims.length - 1].pid ?? '?'})` : '';
|
|
283
|
+
summary = `out of memory${memoryLine ? ` — ${memoryLine}` : ''}${victim}`;
|
|
284
|
+
} else if (cause === KILL_CAUSE_DISK_FULL) {
|
|
285
|
+
summary = `disk full${diskLine ? ` — ${diskLine}` : ''}`;
|
|
286
|
+
} else if (cause === KILL_CAUSE_FORCED_KILL) {
|
|
287
|
+
const healthy = [memoryLine ? `memory (${memoryLine})` : null, diskLine ? diskLine : null].filter(Boolean).join(', ');
|
|
288
|
+
summary = stopRequestedByUser ? 'forced kill — an operator requested the stop' : `forced kill${healthy ? ` — ${healthy} were within normal limits` : ' — no resource exhaustion was observed'}`;
|
|
289
|
+
} else {
|
|
290
|
+
summary = 'unknown — no resource marker, cgroup counter or kernel OOM report was available';
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
return { cause, summary, evidence, memory, disk, victims };
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Telegram/PR section describing why the session was killed.
|
|
298
|
+
*
|
|
299
|
+
* @param {Object} diagnosis - describeKillCause() result
|
|
300
|
+
* @param {Object} [options]
|
|
301
|
+
* @param {string|null} [options.locale]
|
|
302
|
+
* @returns {string} Markdown block, or '' when there is nothing to report
|
|
303
|
+
*/
|
|
304
|
+
export function formatKillDiagnosticsSection(diagnosis, { locale = null } = {}) {
|
|
305
|
+
if (!diagnosis || diagnosis.cause === KILL_CAUSE_UNKNOWN) {
|
|
306
|
+
if (!diagnosis?.evidence?.length) return '';
|
|
307
|
+
}
|
|
308
|
+
const title = text(locale, 'telegram.session_kill_diagnostics', 'Kill diagnostics');
|
|
309
|
+
const causeLabel = text(locale, 'telegram.session_kill_cause', 'Cause');
|
|
310
|
+
const lines = [`🔎 *${title}*`, `${causeLabel}: ${diagnosis.summary}`];
|
|
311
|
+
for (const item of diagnosis.evidence) lines.push(`• ${item}`);
|
|
312
|
+
return lines.join('\n');
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* The warning the issue asks for: a session that hit an out-of-memory event (or
|
|
317
|
+
* any other kill) and nevertheless completed must say so, instead of reading as
|
|
318
|
+
* an ordinary success.
|
|
319
|
+
*
|
|
320
|
+
* @param {Object} [options]
|
|
321
|
+
* @param {string} [options.cause] - Kill cause constant
|
|
322
|
+
* @param {string|null} [options.observedAt] - When the event was observed
|
|
323
|
+
* @param {string|null} [options.locale]
|
|
324
|
+
* @param {boolean} [options.resumed] - A new working session was started
|
|
325
|
+
* @returns {string} Markdown block, or '' when nothing was recovered from
|
|
326
|
+
*/
|
|
327
|
+
export function formatKillRecoverySection({ cause = KILL_CAUSE_OUT_OF_MEMORY, observedAt = null, locale = null, resumed = false } = {}) {
|
|
328
|
+
const key = cause === KILL_CAUSE_OUT_OF_MEMORY ? 'telegram.session_recovered_oom' : 'telegram.session_recovered_kill';
|
|
329
|
+
const fallback = cause === KILL_CAUSE_OUT_OF_MEMORY ? 'recovered from out of memory' : 'recovered from forced kill';
|
|
330
|
+
const lines = [`⚠️ *${text(locale, key, fallback)}*`];
|
|
331
|
+
if (observedAt) {
|
|
332
|
+
lines.push(text(locale, 'telegram.session_recovered_at', `The event was observed at ${observedAt}; the work session kept running and completed.`, { observedAt }));
|
|
333
|
+
}
|
|
334
|
+
if (resumed) {
|
|
335
|
+
lines.push(text(locale, 'telegram.session_recovered_resumed', 'A new working session was started to recover from the kill.'));
|
|
336
|
+
}
|
|
337
|
+
return lines.join('\n');
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* The counterpart for a session that did NOT survive: `--on-session-kill=resume`
|
|
342
|
+
* started a fresh working session, and the Telegram report must say so with the
|
|
343
|
+
* same words the pull-request notice uses — that is the "consistent in ALL
|
|
344
|
+
* places" requirement of issue #2134.
|
|
345
|
+
*
|
|
346
|
+
* @param {Object} [options]
|
|
347
|
+
* @param {string|null} [options.sessionId] - Id of the recovery working session
|
|
348
|
+
* @param {number|null} [options.attempt]
|
|
349
|
+
* @param {number|null} [options.maxAttempts]
|
|
350
|
+
* @param {string|null} [options.locale]
|
|
351
|
+
* @returns {string} Markdown block, or '' when no recovery session was started
|
|
352
|
+
*/
|
|
353
|
+
export function formatKillResumeSection({ sessionId = null, attempt = null, maxAttempts = null, locale = null } = {}) {
|
|
354
|
+
if (!sessionId) return '';
|
|
355
|
+
const withAttempt = Number.isFinite(attempt) && Number.isFinite(maxAttempts);
|
|
356
|
+
const attemptText = withAttempt ? `${attempt}/${maxAttempts}` : '';
|
|
357
|
+
const key = withAttempt ? 'telegram.session_kill_resumed_attempt' : 'telegram.session_kill_resumed';
|
|
358
|
+
const fallback = withAttempt ? `🔄 A new working session was started to recover from this kill (attempt ${attemptText}): ${sessionId}` : `🔄 A new working session was started to recover from this kill: ${sessionId}`;
|
|
359
|
+
return text(locale, key, fallback, { attempt: attemptText, sessionId });
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Read a session log, diagnose the kill and render the Telegram section in one
|
|
364
|
+
* call. Never throws — returns '' when nothing can be said.
|
|
365
|
+
*
|
|
366
|
+
* @param {string|null} logPath
|
|
367
|
+
* @param {Object} [options]
|
|
368
|
+
* @returns {Promise<{section: string, diagnosis: Object|null}>}
|
|
369
|
+
*/
|
|
370
|
+
export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics } = {}) {
|
|
371
|
+
try {
|
|
372
|
+
let logText = '';
|
|
373
|
+
if (logPath) {
|
|
374
|
+
try {
|
|
375
|
+
logText = await readFile(logPath, 'utf8');
|
|
376
|
+
} catch (readError) {
|
|
377
|
+
if (verbose) console.log(`[VERBOSE] kill-diagnostics: could not read session log ${logPath}: ${readError?.message || readError}`);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
const system = await collectSystem({ verbose });
|
|
381
|
+
const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser });
|
|
382
|
+
if (verbose) console.log(`[VERBOSE] kill-diagnostics: cause=${diagnosis.cause} — ${diagnosis.summary}`);
|
|
383
|
+
return { section: formatKillDiagnosticsSection(diagnosis, { locale }), diagnosis };
|
|
384
|
+
} catch (error) {
|
|
385
|
+
if (verbose) console.log(`[VERBOSE] kill-diagnostics: diagnosis failed: ${error?.message || error}`);
|
|
386
|
+
return { section: '', diagnosis: null };
|
|
387
|
+
}
|
|
388
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How Hive Mind reacts when a detached work session is killed (issue #2134).
|
|
3
|
+
*
|
|
4
|
+
* The issue asks for one configurable behaviour that every surface honours
|
|
5
|
+
* identically — the Telegram completion message and the pull-request notice must
|
|
6
|
+
* never disagree about what happened:
|
|
7
|
+
*
|
|
8
|
+
* - `report` (default, today's behaviour): the kill is terminal. The Telegram
|
|
9
|
+
* message says the session was killed, with the diagnosed cause, and offers
|
|
10
|
+
* the resume command. The pull request gets the same notice.
|
|
11
|
+
* - `resume`: the kill is treated as recoverable. A new working session is
|
|
12
|
+
* started from the last tool session id, and BOTH surfaces say so
|
|
13
|
+
* ("recovered from out of memory" / "a new working session was started").
|
|
14
|
+
*
|
|
15
|
+
* Selected by `--on-session-kill=<policy>` or `HIVE_MIND_ON_SESSION_KILL`, with
|
|
16
|
+
* the CLI flag winning over the environment. Nothing is removed by choosing one
|
|
17
|
+
* over the other: `resume` still reports the kill and its cause, it just adds
|
|
18
|
+
* the recovery, and log uploads stay gated on `--attach-logs` in both modes.
|
|
19
|
+
*
|
|
20
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export const ON_SESSION_KILL_REPORT = 'report';
|
|
24
|
+
export const ON_SESSION_KILL_RESUME = 'resume';
|
|
25
|
+
export const ON_SESSION_KILL_POLICIES = [ON_SESSION_KILL_REPORT, ON_SESSION_KILL_RESUME];
|
|
26
|
+
export const DEFAULT_ON_SESSION_KILL_POLICY = ON_SESSION_KILL_REPORT;
|
|
27
|
+
|
|
28
|
+
export const ON_SESSION_KILL_ENV_VAR = 'HIVE_MIND_ON_SESSION_KILL';
|
|
29
|
+
|
|
30
|
+
/** Hard cap on automatic resumes per session, so a reliably OOM-ing job cannot storm. */
|
|
31
|
+
export const DEFAULT_SESSION_KILL_RESUME_ATTEMPTS = 1;
|
|
32
|
+
export const SESSION_KILL_RESUME_ATTEMPTS_ENV_VAR = 'HIVE_MIND_SESSION_KILL_RESUME_ATTEMPTS';
|
|
33
|
+
|
|
34
|
+
function normalize(value) {
|
|
35
|
+
return String(value ?? '')
|
|
36
|
+
.trim()
|
|
37
|
+
.toLowerCase();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Resolve the configured on-kill policy.
|
|
42
|
+
*
|
|
43
|
+
* @param {Object} [options]
|
|
44
|
+
* @param {Object} [options.argv] - yargs argv (`onSessionKill` / `on-session-kill`)
|
|
45
|
+
* @param {Object} [options.env=process.env]
|
|
46
|
+
* @param {Object} [options.sessionInfo] - Persisted session info (per-session override)
|
|
47
|
+
* @param {boolean} [options.verbose]
|
|
48
|
+
* @returns {string} One of ON_SESSION_KILL_POLICIES
|
|
49
|
+
*/
|
|
50
|
+
export function resolveOnSessionKillPolicy({ argv = null, env = process.env, sessionInfo = null, verbose = false } = {}) {
|
|
51
|
+
const candidates = [
|
|
52
|
+
{ source: 'session', raw: sessionInfo?.onSessionKill },
|
|
53
|
+
{ source: '--on-session-kill', raw: argv?.onSessionKill ?? argv?.['on-session-kill'] },
|
|
54
|
+
{ source: ON_SESSION_KILL_ENV_VAR, raw: env?.[ON_SESSION_KILL_ENV_VAR] },
|
|
55
|
+
];
|
|
56
|
+
for (const { source, raw } of candidates) {
|
|
57
|
+
const normalized = normalize(raw);
|
|
58
|
+
if (!normalized) continue;
|
|
59
|
+
if (ON_SESSION_KILL_POLICIES.includes(normalized)) return normalized;
|
|
60
|
+
if (verbose) {
|
|
61
|
+
console.log(`[VERBOSE] Invalid ${source}='${raw}', using '${DEFAULT_ON_SESSION_KILL_POLICY}' (valid: ${ON_SESSION_KILL_POLICIES.join(', ')})`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return DEFAULT_ON_SESSION_KILL_POLICY;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Maximum number of automatic resumes for one killed session.
|
|
69
|
+
*
|
|
70
|
+
* @param {Object} [options]
|
|
71
|
+
* @param {Object} [options.argv]
|
|
72
|
+
* @param {Object} [options.env=process.env]
|
|
73
|
+
* @returns {number} A non-negative integer
|
|
74
|
+
*/
|
|
75
|
+
export function resolveSessionKillResumeAttempts({ argv = null, env = process.env } = {}) {
|
|
76
|
+
const raw = argv?.sessionKillResumeAttempts ?? argv?.['session-kill-resume-attempts'] ?? env?.[SESSION_KILL_RESUME_ATTEMPTS_ENV_VAR];
|
|
77
|
+
const text = String(raw ?? '').trim();
|
|
78
|
+
// An unset flag/variable is an empty string, and `Number('')` is 0 — which
|
|
79
|
+
// would silently disable resuming instead of using the default.
|
|
80
|
+
if (text === '') return DEFAULT_SESSION_KILL_RESUME_ATTEMPTS;
|
|
81
|
+
const parsed = Number(text);
|
|
82
|
+
if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_SESSION_KILL_RESUME_ATTEMPTS;
|
|
83
|
+
return Math.floor(parsed);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Whether a killed session should be auto-resumed under the resolved policy.
|
|
88
|
+
*
|
|
89
|
+
* @param {Object} [options]
|
|
90
|
+
* @param {string} [options.policy]
|
|
91
|
+
* @param {boolean} [options.killed] - The completion outcome is a kill
|
|
92
|
+
* @returns {boolean}
|
|
93
|
+
*/
|
|
94
|
+
export function shouldResumeKilledSession({ policy = DEFAULT_ON_SESSION_KILL_POLICY, killed = false } = {}) {
|
|
95
|
+
return killed === true && policy === ON_SESSION_KILL_RESUME;
|
|
96
|
+
}
|