@link-assistant/hive-mind 2.11.7 → 2.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/package.json +1 -1
- package/src/locales/en.lino +10 -0
- package/src/locales/hi.lino +10 -0
- package/src/locales/ru.lino +10 -0
- package/src/locales/zh.lino +10 -0
- package/src/option-suggestions.lib.mjs +2 -0
- package/src/session-kill-diagnostics.lib.mjs +388 -0
- package/src/session-kill-policy.lib.mjs +96 -0
- package/src/session-kill-recovery.lib.mjs +252 -0
- package/src/session-kill-resume.lib.mjs +175 -0
- package/src/session-monitor.kill-sections.lib.mjs +198 -0
- package/src/session-monitor.lib.mjs +97 -2
- package/src/session-monitor.oom.lib.mjs +148 -0
- package/src/session-monitor.stale-executing.lib.mjs +6 -27
- package/src/session-resume.lib.mjs +28 -2
- package/src/solve.config.lib.mjs +14 -0
- package/src/telegram-bot.mjs +3 -1
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pull-request side of killed-session handling (issue #2134).
|
|
3
|
+
*
|
|
4
|
+
* The issue's screenshot showed the two surfaces disagreeing: Telegram announced
|
|
5
|
+
* "Work session killed — out of memory or forced kill (SIGKILL) (exit code: 137)"
|
|
6
|
+
* while the pull request silently carried on, with nothing to tell a reader that
|
|
7
|
+
* a kill had happened at all — let alone that a new working session had been
|
|
8
|
+
* started to recover from it.
|
|
9
|
+
*
|
|
10
|
+
* This module makes the pull request say exactly what the bot says:
|
|
11
|
+
*
|
|
12
|
+
* - `buildKillRecoveryNotice()` renders the Markdown notice (kill cause,
|
|
13
|
+
* diagnostics evidence, what happens next, resume command).
|
|
14
|
+
* - `postKillRecoveryNotice()` posts it with `gh pr comment --body-file`.
|
|
15
|
+
* - `attachIntermediateSessionLog()` uploads the intermediate working-session
|
|
16
|
+
* log — ONLY when `--attach-logs` is enabled, per the issue: "Logs are
|
|
17
|
+
* uploaded as usual only if --attach-logs is enabled."
|
|
18
|
+
*
|
|
19
|
+
* Every external dependency is injectable so the builders can be unit-tested
|
|
20
|
+
* without touching GitHub.
|
|
21
|
+
*
|
|
22
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { spawn } from 'child_process';
|
|
26
|
+
import { KILL_CAUSE_DISK_FULL, KILL_CAUSE_FORCED_KILL, KILL_CAUSE_OUT_OF_MEMORY } from './session-kill-diagnostics.lib.mjs';
|
|
27
|
+
import { ON_SESSION_KILL_RESUME } from './session-kill-policy.lib.mjs';
|
|
28
|
+
|
|
29
|
+
/** Marker used to recognise (and avoid duplicating) our own notices. */
|
|
30
|
+
export const KILL_RECOVERY_NOTICE_MARKER = '<!-- hive-mind:session-kill-notice -->';
|
|
31
|
+
|
|
32
|
+
const CAUSE_HEADLINES = {
|
|
33
|
+
[KILL_CAUSE_OUT_OF_MEMORY]: 'recovered from out of memory',
|
|
34
|
+
[KILL_CAUSE_FORCED_KILL]: 'recovered from forced kill',
|
|
35
|
+
[KILL_CAUSE_DISK_FULL]: 'recovered from disk exhaustion',
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const CAUSE_TITLES = {
|
|
39
|
+
[KILL_CAUSE_OUT_OF_MEMORY]: 'Working session was killed: out of memory',
|
|
40
|
+
[KILL_CAUSE_DISK_FULL]: 'Working session was killed: disk full',
|
|
41
|
+
[KILL_CAUSE_FORCED_KILL]: 'Working session was force-killed',
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Human-readable headline for a recovered session, matching the wording the
|
|
46
|
+
* issue asks the Telegram bot to use.
|
|
47
|
+
*
|
|
48
|
+
* @param {string} cause - One of the KILL_CAUSE_* constants
|
|
49
|
+
* @returns {string}
|
|
50
|
+
*/
|
|
51
|
+
export function killRecoveryHeadline(cause) {
|
|
52
|
+
return CAUSE_HEADLINES[cause] || 'recovered from an unexpected kill';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Render the pull-request notice for a killed (and possibly resumed) session.
|
|
57
|
+
*
|
|
58
|
+
* @param {Object} options
|
|
59
|
+
* @param {Object} [options.diagnosis] - Result of describeKillCause()
|
|
60
|
+
* @param {number|null} [options.exitCode]
|
|
61
|
+
* @param {string} [options.sessionName]
|
|
62
|
+
* @param {string|null} [options.observedAt] - ISO timestamp of the kill/OOM event
|
|
63
|
+
* @param {string} [options.policy] - Resolved --on-session-kill policy
|
|
64
|
+
* @param {boolean} [options.resumed] - A new working session was started
|
|
65
|
+
* @param {string|null} [options.recoverySessionId] - Id of that working session
|
|
66
|
+
* @param {string|null} [options.resumeCommand] - Command to resume manually
|
|
67
|
+
* @param {number|null} [options.attempt] - Resume attempt number
|
|
68
|
+
* @param {number|null} [options.maxAttempts]
|
|
69
|
+
* @param {boolean} [options.attachLogs] - Whether --attach-logs is enabled
|
|
70
|
+
* @param {string|null} [options.logUrl] - URL of the uploaded intermediate log
|
|
71
|
+
* @returns {string} Markdown body
|
|
72
|
+
*/
|
|
73
|
+
export function buildKillRecoveryNotice({ diagnosis = null, exitCode = null, sessionName = null, observedAt = null, policy = null, resumed = false, recoverySessionId = null, resumeCommand = null, attempt = null, maxAttempts = null, attachLogs = false, logAttached = false, logUrl = null } = {}) {
|
|
74
|
+
const cause = diagnosis?.cause || null;
|
|
75
|
+
const title = resumed ? `⚠️ Working session ${killRecoveryHeadline(cause)}` : `❌ ${CAUSE_TITLES[cause] || 'Working session was killed'}`;
|
|
76
|
+
|
|
77
|
+
const lines = [KILL_RECOVERY_NOTICE_MARKER, `## ${title}`, ''];
|
|
78
|
+
|
|
79
|
+
if (diagnosis?.summary) lines.push(diagnosis.summary, '');
|
|
80
|
+
|
|
81
|
+
const facts = [];
|
|
82
|
+
if (exitCode !== null && exitCode !== undefined) facts.push(`- **Exit code:** ${exitCode}`);
|
|
83
|
+
if (observedAt) facts.push(`- **Detected at:** ${observedAt}`);
|
|
84
|
+
if (sessionName) facts.push(`- **Working session:** \`${sessionName}\``);
|
|
85
|
+
if (policy) facts.push(`- **On-kill policy:** \`${policy}\`${policy === ON_SESSION_KILL_RESUME ? ' (`--on-session-kill=resume`)' : ' (`--on-session-kill=report`)'}`);
|
|
86
|
+
if (facts.length > 0) lines.push(...facts, '');
|
|
87
|
+
|
|
88
|
+
const evidence = Array.isArray(diagnosis?.evidence) ? diagnosis.evidence.filter(Boolean) : [];
|
|
89
|
+
if (evidence.length > 0) {
|
|
90
|
+
lines.push('<details><summary>Kill diagnostics</summary>', '');
|
|
91
|
+
for (const item of evidence) lines.push(`- ${item}`);
|
|
92
|
+
lines.push('', '</details>', '');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (resumed) {
|
|
96
|
+
const attemptSuffix = attempt && maxAttempts ? ` (attempt ${attempt}/${maxAttempts})` : '';
|
|
97
|
+
const sessionSuffix = recoverySessionId ? ` Its working session is \`${recoverySessionId}\`.` : '';
|
|
98
|
+
lines.push(`🔄 A **new working session was started** to recover from this event${attemptSuffix}. Progress below continues in that session.${sessionSuffix}`, '');
|
|
99
|
+
} else {
|
|
100
|
+
lines.push('This working session did not continue. Nothing below this comment was produced by it.', '');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (logUrl) {
|
|
104
|
+
lines.push(`📎 Intermediate working-session log: ${logUrl}`, '');
|
|
105
|
+
} else if (logAttached) {
|
|
106
|
+
lines.push('📎 The intermediate working-session log was uploaded as a separate comment.', '');
|
|
107
|
+
} else if (!attachLogs) {
|
|
108
|
+
lines.push('_The intermediate working-session log was not uploaded because `--attach-logs` is disabled._', '');
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if (resumeCommand) {
|
|
112
|
+
lines.push('To continue manually:', '', '```bash', resumeCommand, '```', '');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
lines.push(`<sub>Reported by Hive Mind — [issue #2134](https://github.com/link-assistant/hive-mind/issues/2134)</sub>`);
|
|
116
|
+
return lines.join('\n');
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Run a command and capture its output, without a shell (the notice body never
|
|
121
|
+
* touches the command line — it is passed via `--body-file`).
|
|
122
|
+
*
|
|
123
|
+
* @param {string} command
|
|
124
|
+
* @param {string[]} args
|
|
125
|
+
* @param {Object} [options] - Forwarded to child_process.spawn
|
|
126
|
+
* @returns {Promise<{code: number, stdout: string, stderr: string}>}
|
|
127
|
+
*/
|
|
128
|
+
export function spawnCapture(command, args, options = {}) {
|
|
129
|
+
return new Promise(resolve => {
|
|
130
|
+
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'], env: process.env, ...options });
|
|
131
|
+
let stdout = '';
|
|
132
|
+
let stderr = '';
|
|
133
|
+
child.stdout.on('data', data => {
|
|
134
|
+
stdout += data.toString();
|
|
135
|
+
});
|
|
136
|
+
child.stderr.on('data', data => {
|
|
137
|
+
stderr += data.toString();
|
|
138
|
+
});
|
|
139
|
+
child.on('error', error => resolve({ code: 1, stdout, stderr: stderr || error.message }));
|
|
140
|
+
child.on('close', code => resolve({ code, stdout, stderr }));
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const defaultWriteFile = async (filePath, content) => {
|
|
145
|
+
const fs = (await import('fs')).promises;
|
|
146
|
+
await fs.writeFile(filePath, content, 'utf8');
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
const defaultUnlink = async filePath => {
|
|
150
|
+
const fs = (await import('fs')).promises;
|
|
151
|
+
await fs.unlink(filePath).catch(() => {});
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Post the notice to a pull request via `gh pr comment --body-file`.
|
|
156
|
+
*
|
|
157
|
+
* `--body-file` (not `--body`) is used deliberately: the notice contains
|
|
158
|
+
* backticks and newlines that would otherwise have to survive shell quoting.
|
|
159
|
+
*
|
|
160
|
+
* @param {Object} options
|
|
161
|
+
* @param {string} options.pullRequestUrl
|
|
162
|
+
* @param {string} options.body
|
|
163
|
+
* @param {Function} options.runCommand - async (cmd, args) => { code, stdout, stderr }
|
|
164
|
+
* @param {string} [options.tempDir=/tmp]
|
|
165
|
+
* @param {string} [options.fileSuffix] - Deterministic suffix for the temp file
|
|
166
|
+
* @param {Function} [options.writeFile]
|
|
167
|
+
* @param {Function} [options.unlink]
|
|
168
|
+
* @param {boolean} [options.verbose]
|
|
169
|
+
* @returns {Promise<{posted: boolean, url: string|null, error: string|null}>}
|
|
170
|
+
*/
|
|
171
|
+
export async function postKillRecoveryNotice({ pullRequestUrl, body, runCommand = spawnCapture, tempDir = '/tmp', fileSuffix = 'notice', writeFile = defaultWriteFile, unlink = defaultUnlink, verbose = false }) {
|
|
172
|
+
if (!pullRequestUrl) return { posted: false, url: null, error: 'no pull request url' };
|
|
173
|
+
if (typeof runCommand !== 'function') return { posted: false, url: null, error: 'no command runner' };
|
|
174
|
+
|
|
175
|
+
const bodyFile = `${tempDir.replace(/\/$/, '')}/hive-mind-kill-notice-${fileSuffix}.md`;
|
|
176
|
+
try {
|
|
177
|
+
await writeFile(bodyFile, body);
|
|
178
|
+
const result = await runCommand('gh', ['pr', 'comment', pullRequestUrl, '--body-file', bodyFile]);
|
|
179
|
+
if (result?.code === 0) {
|
|
180
|
+
const url = String(result.stdout || '').trim() || null;
|
|
181
|
+
if (verbose) console.log(`[VERBOSE] Posted killed-session notice to ${pullRequestUrl}${url ? ` (${url})` : ''}`);
|
|
182
|
+
return { posted: true, url, error: null };
|
|
183
|
+
}
|
|
184
|
+
const error = String(result?.stderr || result?.stdout || `gh pr comment exited with code ${result?.code}`).trim();
|
|
185
|
+
if (verbose) console.log(`[VERBOSE] Failed to post killed-session notice: ${error}`);
|
|
186
|
+
return { posted: false, url: null, error };
|
|
187
|
+
} catch (error) {
|
|
188
|
+
if (verbose) console.log(`[VERBOSE] Failed to post killed-session notice: ${error?.message || error}`);
|
|
189
|
+
return { posted: false, url: null, error: error?.message || String(error) };
|
|
190
|
+
} finally {
|
|
191
|
+
await unlink(bodyFile);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Upload the intermediate working-session log to the pull request.
|
|
197
|
+
*
|
|
198
|
+
* Per the issue, this is gated on `--attach-logs` exactly like every other log
|
|
199
|
+
* upload — a killed session never becomes a reason to publish logs the user did
|
|
200
|
+
* not ask for.
|
|
201
|
+
*
|
|
202
|
+
* @param {Object} options
|
|
203
|
+
* @param {boolean} options.attachLogs - Resolved `--attach-logs` value
|
|
204
|
+
* @param {string|null} options.logPath
|
|
205
|
+
* @param {string|null} options.pullRequestUrl
|
|
206
|
+
* @param {Function} options.attachLog - attachLogToGitHub-compatible uploader
|
|
207
|
+
* @param {Object} [options.attachOptions] - Extra options forwarded to the uploader
|
|
208
|
+
* @param {string} [options.customTitle]
|
|
209
|
+
* @param {boolean} [options.verbose]
|
|
210
|
+
* @returns {Promise<{uploaded: boolean, skipped: string|null}>}
|
|
211
|
+
*/
|
|
212
|
+
export async function attachIntermediateSessionLog({ attachLogs, logPath, pullRequestUrl, attachLog, attachOptions = {}, customTitle = '📎 Intermediate working-session log (killed session)', verbose = false }) {
|
|
213
|
+
if (!attachLogs) {
|
|
214
|
+
if (verbose) console.log('[VERBOSE] Skipping intermediate log upload: --attach-logs is disabled');
|
|
215
|
+
return { uploaded: false, skipped: 'attach-logs-disabled' };
|
|
216
|
+
}
|
|
217
|
+
if (!logPath) return { uploaded: false, skipped: 'no-log-path' };
|
|
218
|
+
if (!pullRequestUrl) return { uploaded: false, skipped: 'no-pull-request' };
|
|
219
|
+
if (typeof attachLog !== 'function') return { uploaded: false, skipped: 'no-uploader' };
|
|
220
|
+
|
|
221
|
+
const parsed = parsePullRequestUrl(pullRequestUrl);
|
|
222
|
+
if (!parsed) return { uploaded: false, skipped: 'unparsable-pull-request-url' };
|
|
223
|
+
|
|
224
|
+
try {
|
|
225
|
+
const ok = await attachLog({
|
|
226
|
+
...attachOptions,
|
|
227
|
+
logFile: logPath,
|
|
228
|
+
targetType: 'pr',
|
|
229
|
+
targetNumber: parsed.number,
|
|
230
|
+
owner: parsed.owner,
|
|
231
|
+
repo: parsed.repo,
|
|
232
|
+
customTitle,
|
|
233
|
+
verbose,
|
|
234
|
+
});
|
|
235
|
+
return { uploaded: ok !== false, skipped: null };
|
|
236
|
+
} catch (error) {
|
|
237
|
+
if (verbose) console.log(`[VERBOSE] Intermediate log upload failed: ${error?.message || error}`);
|
|
238
|
+
return { uploaded: false, skipped: `error: ${error?.message || error}` };
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Minimal `https://github.com/<owner>/<repo>/pull/<number>` parser.
|
|
244
|
+
*
|
|
245
|
+
* @param {string} url
|
|
246
|
+
* @returns {{owner: string, repo: string, number: number}|null}
|
|
247
|
+
*/
|
|
248
|
+
export function parsePullRequestUrl(url) {
|
|
249
|
+
const match = /github\.com\/([^/\s]+)\/([^/\s]+)\/pull\/(\d+)/u.exec(String(url || ''));
|
|
250
|
+
if (!match) return null;
|
|
251
|
+
return { owner: match[1], repo: match[2], number: Number(match[3]) };
|
|
252
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Automatic recovery of a killed working session (issue #2134).
|
|
3
|
+
*
|
|
4
|
+
* `--on-session-kill=resume` (env `HIVE_MIND_ON_SESSION_KILL`) asks for a new
|
|
5
|
+
* working session to be started when the previous one was killed by the kernel
|
|
6
|
+
* (out of memory), by a full disk, or by a forced kill. This module performs
|
|
7
|
+
* that restart, and — just as importantly — reports it: the caller puts the
|
|
8
|
+
* returned facts into the Telegram completion message AND into the pull request
|
|
9
|
+
* notice, so a reader of either surface knows a recovery session exists.
|
|
10
|
+
*
|
|
11
|
+
* The restart is bounded by `--session-kill-resume-attempts` (default 1), so a
|
|
12
|
+
* job that reliably runs the host out of memory cannot storm the queue.
|
|
13
|
+
*
|
|
14
|
+
* Nothing here runs under the default `report` policy — behaviour is unchanged
|
|
15
|
+
* unless the operator opts in.
|
|
16
|
+
*
|
|
17
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { readLastSessionIdFromLog, planKilledSessionResume } from './session-resume.lib.mjs';
|
|
21
|
+
import { resolveOnSessionKillPolicy, resolveSessionKillResumeAttempts, shouldResumeKilledSession, ON_SESSION_KILL_RESUME } from './session-kill-policy.lib.mjs';
|
|
22
|
+
import { argvFromSessionArgs } from './session-monitor.kill-sections.lib.mjs';
|
|
23
|
+
import { formatKillResumeSection } from './session-kill-diagnostics.lib.mjs';
|
|
24
|
+
|
|
25
|
+
/** Field recording how many automatic recovery sessions this session produced. */
|
|
26
|
+
export const KILL_RESUME_ATTEMPTS_FIELD = 'killRecoveryAttempts';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Decide whether a killed session should be auto-resumed, and with which
|
|
30
|
+
* command. Pure — no process is started here, so the decision is testable on
|
|
31
|
+
* its own and the caller can report a skipped resume just as precisely.
|
|
32
|
+
*
|
|
33
|
+
* @param {Object} options
|
|
34
|
+
* @param {Object} options.sessionInfo - Persisted session info
|
|
35
|
+
* @param {string|null} [options.logPath] - Working-session log to scan for the tool session id
|
|
36
|
+
* @param {boolean} options.killed - The completion outcome is a kill
|
|
37
|
+
* @param {Object} [options.env]
|
|
38
|
+
* @param {boolean} [options.verbose]
|
|
39
|
+
* @param {Function} [options.readLastSessionId] - Override for tests
|
|
40
|
+
* @returns {{shouldResume: boolean, reason: string, policy: string, command: Object|null, attempt: number, maxAttempts: number, lastSessionId: string|null}}
|
|
41
|
+
*/
|
|
42
|
+
export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = false, env = process.env, verbose = false, readLastSessionId = readLastSessionIdFromLog } = {}) {
|
|
43
|
+
const argv = argvFromSessionArgs(sessionInfo?.args);
|
|
44
|
+
const policy = resolveOnSessionKillPolicy({ argv, env, sessionInfo, verbose });
|
|
45
|
+
const maxAttempts = resolveSessionKillResumeAttempts({ argv, env });
|
|
46
|
+
const attempts = Number.isFinite(sessionInfo?.[KILL_RESUME_ATTEMPTS_FIELD]) ? sessionInfo[KILL_RESUME_ATTEMPTS_FIELD] : 0;
|
|
47
|
+
const base = { shouldResume: false, policy, command: null, attempt: attempts, maxAttempts, lastSessionId: null };
|
|
48
|
+
|
|
49
|
+
if (!shouldResumeKilledSession({ policy, killed })) {
|
|
50
|
+
return { ...base, reason: killed ? 'policy-report' : 'not-killed' };
|
|
51
|
+
}
|
|
52
|
+
if (sessionInfo?.stopRequestedByUser === true) {
|
|
53
|
+
// A `/stop` is a kill the operator asked for; restarting it would fight them.
|
|
54
|
+
return { ...base, reason: 'stopped-by-user' };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const lastSessionId = readLastSessionId(logPath, { verbose });
|
|
58
|
+
const plan = planKilledSessionResume({ sessionInfo, lastSessionId, attempts, maxAttempts });
|
|
59
|
+
return { ...base, shouldResume: plan.resumable, reason: plan.reason, command: plan.command, attempt: plan.attempt, lastSessionId: lastSessionId || null };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Start the recovery working session decided by {@link planKillRecovery}.
|
|
64
|
+
*
|
|
65
|
+
* The new session is launched through the same isolation runner the original
|
|
66
|
+
* used and is tracked like any other session, so it reports its own completion
|
|
67
|
+
* (and, if it is killed too, its own diagnosis) through the normal path.
|
|
68
|
+
*
|
|
69
|
+
* @param {Object} options
|
|
70
|
+
* @param {string} options.sessionName - The killed session's name
|
|
71
|
+
* @param {Object} options.sessionInfo - Persisted session info
|
|
72
|
+
* @param {Object} options.plan - Result of planKillRecovery()
|
|
73
|
+
* @param {Object} options.runner - Isolation runner (executeWithIsolation/generateSessionId)
|
|
74
|
+
* @param {Function} options.trackSession - Tracker for the new session
|
|
75
|
+
* @param {Function} [options.persistSnapshot] - Persist the attempt counter
|
|
76
|
+
* @param {boolean} [options.verbose]
|
|
77
|
+
* @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null}>}
|
|
78
|
+
*/
|
|
79
|
+
export async function startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot = null, verbose = false } = {}) {
|
|
80
|
+
const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null });
|
|
81
|
+
if (!plan?.shouldResume || !plan.command) return fail(plan?.reason || 'no-plan');
|
|
82
|
+
if (!runner || typeof runner.executeWithIsolation !== 'function' || typeof runner.generateSessionId !== 'function') return fail('no-isolation-runner');
|
|
83
|
+
if (typeof trackSession !== 'function') return fail('no-tracker');
|
|
84
|
+
const backend = sessionInfo?.isolationBackend || null;
|
|
85
|
+
if (!backend) return fail('no-isolation-backend');
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
const newSessionId = runner.generateSessionId();
|
|
89
|
+
const tool = sessionInfo?.tool || 'claude';
|
|
90
|
+
const result = await runner.executeWithIsolation(sessionInfo?.command || 'solve', plan.command.args, { backend, sessionId: newSessionId, tool, verbose });
|
|
91
|
+
if (!result?.success) return fail('start-failed');
|
|
92
|
+
|
|
93
|
+
trackSession(
|
|
94
|
+
newSessionId,
|
|
95
|
+
{
|
|
96
|
+
...sessionInfo,
|
|
97
|
+
startTime: new Date(),
|
|
98
|
+
sessionId: newSessionId,
|
|
99
|
+
args: [...plan.command.args],
|
|
100
|
+
// Carry the counter forward so attempt N+1 is bounded by the same cap,
|
|
101
|
+
// and remember what this session is recovering from for its own report.
|
|
102
|
+
[KILL_RESUME_ATTEMPTS_FIELD]: plan.attempt,
|
|
103
|
+
killRecoveryResumed: true,
|
|
104
|
+
killRecoveryOfSession: sessionName,
|
|
105
|
+
oomEventObservedAt: undefined,
|
|
106
|
+
dockerBackendGoneFirstSeenAt: undefined,
|
|
107
|
+
containerFilesystemStartBytes: Number.isFinite(result.containerFilesystemStartBytes) ? result.containerFilesystemStartBytes : null,
|
|
108
|
+
},
|
|
109
|
+
verbose
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
if (sessionInfo) sessionInfo[KILL_RESUME_ATTEMPTS_FIELD] = plan.attempt;
|
|
113
|
+
if (typeof persistSnapshot === 'function') {
|
|
114
|
+
try {
|
|
115
|
+
persistSnapshot();
|
|
116
|
+
} catch {
|
|
117
|
+
// Persisting the counter is best effort; the recovery session is started.
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (verbose) {
|
|
122
|
+
console.log(`[VERBOSE] Session ${sessionName} was killed; started recovery session ${newSessionId} (attempt ${plan.attempt}/${plan.maxAttempts}): ${plan.command.display}`);
|
|
123
|
+
}
|
|
124
|
+
return { resumed: true, reason: 'started', sessionId: newSessionId, display: plan.command.display };
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (verbose) {
|
|
127
|
+
console.log(`[VERBOSE] Could not start recovery session for ${sessionName}: ${error?.message || error}`);
|
|
128
|
+
}
|
|
129
|
+
return fail('start-error');
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Plan and, when the policy asks for it, perform the recovery in one call.
|
|
135
|
+
* Never throws — a failed recovery must still leave a correct kill report.
|
|
136
|
+
*
|
|
137
|
+
* @param {Object} options - See planKillRecovery() and startKillRecoverySession()
|
|
138
|
+
* @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number}>}
|
|
139
|
+
*/
|
|
140
|
+
export async function recoverKilledSession({ sessionName, sessionInfo, logPath = null, killed = false, env = process.env, runner = null, trackSession = null, persistSnapshot = null, verbose = false, readLastSessionId = readLastSessionIdFromLog } = {}) {
|
|
141
|
+
let plan;
|
|
142
|
+
try {
|
|
143
|
+
plan = planKillRecovery({ sessionInfo, logPath, killed, env, verbose, readLastSessionId });
|
|
144
|
+
} catch (error) {
|
|
145
|
+
if (verbose) console.log(`[VERBOSE] Could not plan kill recovery for ${sessionName}: ${error?.message || error}`);
|
|
146
|
+
return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0 };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (!plan.shouldResume) {
|
|
150
|
+
return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const started = await startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot, verbose });
|
|
154
|
+
return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Completion-time entry point: recover the killed session and render the
|
|
159
|
+
* Telegram section describing what was started, in one call — the monitor is at
|
|
160
|
+
* its `max-lines` budget and this keeps the two facts (what happened, what is
|
|
161
|
+
* said about it) together.
|
|
162
|
+
*
|
|
163
|
+
* @param {Object} options - See recoverKilledSession(); plus `locale`
|
|
164
|
+
* @returns {Promise<{recovery: Object, section: string}>}
|
|
165
|
+
*/
|
|
166
|
+
export async function runKillRecoveryForCompletion({ locale = null, ...options } = {}) {
|
|
167
|
+
const recovery = await recoverKilledSession(options);
|
|
168
|
+
const section = formatKillResumeSection({
|
|
169
|
+
sessionId: recovery.resumed ? recovery.sessionId : null,
|
|
170
|
+
attempt: recovery.attempt,
|
|
171
|
+
maxAttempts: recovery.maxAttempts,
|
|
172
|
+
locale,
|
|
173
|
+
});
|
|
174
|
+
return { recovery, section };
|
|
175
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Completion-time reporting for killed and recovered work sessions (issue #2134).
|
|
3
|
+
*
|
|
4
|
+
* Glue between the pieces that do the actual work — kill diagnostics, the
|
|
5
|
+
* on-kill policy, and the pull-request notice — so `session-monitor.lib.mjs`
|
|
6
|
+
* (already at its `max-lines` budget) gains one call instead of a hundred lines.
|
|
7
|
+
*
|
|
8
|
+
* Two situations are reported, and the Telegram message and the pull request are
|
|
9
|
+
* always given the SAME facts, which is the core complaint in the issue:
|
|
10
|
+
*
|
|
11
|
+
* - The session was killed → say exactly why (out of memory / disk full /
|
|
12
|
+
* forced kill) with the evidence behind that verdict.
|
|
13
|
+
* - The session survived a kill event → warn "recovered from out of memory" /
|
|
14
|
+
* "recovered from forced kill" instead of reading as a plain success.
|
|
15
|
+
*
|
|
16
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2134
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import fs from 'fs/promises';
|
|
20
|
+
import { classifySessionOutcome } from './work-session-formatting.lib.mjs';
|
|
21
|
+
import { buildKillDiagnosticsSection, formatKillRecoverySection, KILL_CAUSE_FORCED_KILL, KILL_CAUSE_OUT_OF_MEMORY } from './session-kill-diagnostics.lib.mjs';
|
|
22
|
+
import { getOomEventObservedAt } from './session-monitor.oom.lib.mjs';
|
|
23
|
+
import { resolveOnSessionKillPolicy } from './session-kill-policy.lib.mjs';
|
|
24
|
+
import { buildKillRecoveryNotice, postKillRecoveryNotice, attachIntermediateSessionLog, spawnCapture } from './session-kill-recovery.lib.mjs';
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* `--attach-logs` as it reaches the bot: a raw CLI argument array recorded on
|
|
28
|
+
* the tracked session (see telegram-isolation.lib.mjs `args`).
|
|
29
|
+
*
|
|
30
|
+
* @param {string[]|null} args
|
|
31
|
+
* @returns {boolean}
|
|
32
|
+
*/
|
|
33
|
+
export function argsIncludeAttachLogs(args) {
|
|
34
|
+
if (!Array.isArray(args)) return false;
|
|
35
|
+
return args.some(arg => {
|
|
36
|
+
const value = String(arg || '').trim();
|
|
37
|
+
return value === '--attach-logs' || value.startsWith('--attach-logs=');
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Turn a recorded CLI argument array into a minimal argv-like object. */
|
|
42
|
+
export function argvFromSessionArgs(args) {
|
|
43
|
+
const argv = {};
|
|
44
|
+
if (!Array.isArray(args)) return argv;
|
|
45
|
+
for (let i = 0; i < args.length; i++) {
|
|
46
|
+
const raw = String(args[i] || '');
|
|
47
|
+
if (!raw.startsWith('--')) continue;
|
|
48
|
+
const eq = raw.indexOf('=');
|
|
49
|
+
if (eq > -1) {
|
|
50
|
+
argv[raw.slice(2, eq)] = raw.slice(eq + 1);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const next = args[i + 1];
|
|
54
|
+
if (next !== undefined && !String(next).startsWith('--')) {
|
|
55
|
+
argv[raw.slice(2)] = String(next);
|
|
56
|
+
i++;
|
|
57
|
+
} else {
|
|
58
|
+
argv[raw.slice(2)] = true;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return argv;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Build the kill/recovery sections for a completed session.
|
|
66
|
+
*
|
|
67
|
+
* Never throws: a failed diagnosis must never block a completion notification.
|
|
68
|
+
*
|
|
69
|
+
* @param {Object} options
|
|
70
|
+
* @param {string} options.sessionName
|
|
71
|
+
* @param {Object} options.sessionInfo
|
|
72
|
+
* @param {Object|null} [options.statusResult]
|
|
73
|
+
* @param {number|null} [options.exitCode]
|
|
74
|
+
* @param {string|null} [options.status]
|
|
75
|
+
* @param {boolean} [options.verbose]
|
|
76
|
+
* @param {Function} [options.readFile]
|
|
77
|
+
* @param {Object} [options.env]
|
|
78
|
+
* @returns {Promise<{sections: string[], diagnosis: Object|null, killed: boolean, recovered: boolean, policy: string|null, observedAt: string|null}>}
|
|
79
|
+
*/
|
|
80
|
+
export async function buildKillCompletionSections({ sessionName, sessionInfo, statusResult = null, exitCode = null, status = null, verbose = false, readFile = fs.readFile, env = process.env } = {}) {
|
|
81
|
+
const empty = { sections: [], diagnosis: null, killed: false, recovered: false, policy: null, observedAt: null };
|
|
82
|
+
try {
|
|
83
|
+
const outcome = classifySessionOutcome({ exitCode, status });
|
|
84
|
+
const observedAt = getOomEventObservedAt(sessionInfo);
|
|
85
|
+
const killed = outcome.killed === true;
|
|
86
|
+
const recovered = !killed && Boolean(observedAt);
|
|
87
|
+
if (!killed && !recovered) return empty;
|
|
88
|
+
|
|
89
|
+
const locale = sessionInfo?.locale || null;
|
|
90
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
91
|
+
const { section, diagnosis } = await buildKillDiagnosticsSection(logPath, {
|
|
92
|
+
verbose,
|
|
93
|
+
readFile,
|
|
94
|
+
oomKilled: statusResult?.oomKilled === true || recovered,
|
|
95
|
+
exitCode,
|
|
96
|
+
stopRequestedByUser: sessionInfo?.stopRequestedByUser === true,
|
|
97
|
+
locale,
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
const argv = argvFromSessionArgs(sessionInfo?.args);
|
|
101
|
+
const policy = resolveOnSessionKillPolicy({ argv, env, sessionInfo, verbose });
|
|
102
|
+
|
|
103
|
+
const sections = [];
|
|
104
|
+
if (recovered) {
|
|
105
|
+
// The session outlived the event — this is the warning the issue asks for.
|
|
106
|
+
sections.push(formatKillRecoverySection({ cause: diagnosis?.cause || KILL_CAUSE_OUT_OF_MEMORY, observedAt, locale, resumed: sessionInfo?.killRecoveryResumed === true }));
|
|
107
|
+
}
|
|
108
|
+
if (section) sections.push(section);
|
|
109
|
+
|
|
110
|
+
if (verbose) {
|
|
111
|
+
console.log(`[VERBOSE] Session ${sessionName} kill reporting: killed=${killed} recovered=${recovered} cause=${diagnosis?.cause || 'n/a'} policy=${policy}`);
|
|
112
|
+
}
|
|
113
|
+
return { sections: sections.filter(Boolean), diagnosis, killed, recovered, policy, observedAt };
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (verbose) {
|
|
116
|
+
console.log(`[VERBOSE] Could not build kill sections for ${sessionName}: ${error?.message || error}`);
|
|
117
|
+
}
|
|
118
|
+
return empty;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Default log uploader: `attachLogToGitHub` with the dependencies the bot does
|
|
124
|
+
* not otherwise carry (`$`, `log`, `sanitizeLogContent`). Imported lazily so the
|
|
125
|
+
* monitor keeps starting on machines where the heavy GitHub helpers are unused.
|
|
126
|
+
*
|
|
127
|
+
* @param {Object} options - attachLogToGitHub options
|
|
128
|
+
* @returns {Promise<boolean>}
|
|
129
|
+
*/
|
|
130
|
+
export async function defaultAttachLog(options) {
|
|
131
|
+
if (typeof globalThis.use === 'undefined') {
|
|
132
|
+
const { ensureUseM } = await import('./use-m-bootstrap.lib.mjs');
|
|
133
|
+
await ensureUseM();
|
|
134
|
+
}
|
|
135
|
+
const [{ attachLogToGitHub }, { sanitizeLogContent }] = await Promise.all([import('./github.lib.mjs'), import('./token-sanitization.lib.mjs')]);
|
|
136
|
+
const { $ } = await globalThis.use('command-stream');
|
|
137
|
+
return attachLogToGitHub({
|
|
138
|
+
$,
|
|
139
|
+
log: async message => console.log(message),
|
|
140
|
+
sanitizeLogContent,
|
|
141
|
+
...options,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Post the same kill/recovery report to the pull request, so a reader of the PR
|
|
147
|
+
* is never left believing an unattended session simply carried on.
|
|
148
|
+
*
|
|
149
|
+
* The intermediate working-session log is uploaded only when `--attach-logs` is
|
|
150
|
+
* enabled, exactly as the issue requires.
|
|
151
|
+
*
|
|
152
|
+
* @param {Object} options
|
|
153
|
+
* @returns {Promise<{posted: boolean, url: string|null, skipped: string|null, logUploaded: boolean}>}
|
|
154
|
+
*/
|
|
155
|
+
export async function announceKillOnPullRequest({ pullRequestUrl, sessionName, sessionInfo, diagnosis, exitCode = null, observedAt = null, policy = null, recovered = false, resumed = false, recoverySessionId = null, attempt = null, maxAttempts = null, resumeCommand = null, runCommand = spawnCapture, attachLog = defaultAttachLog, attachOptions = {}, verbose = false } = {}) {
|
|
156
|
+
const skip = reason => ({ posted: false, url: null, skipped: reason, logUploaded: false });
|
|
157
|
+
if (!pullRequestUrl) return skip('no-pull-request');
|
|
158
|
+
if (typeof runCommand !== 'function') return skip('no-command-runner');
|
|
159
|
+
|
|
160
|
+
const attachLogs = argsIncludeAttachLogs(sessionInfo?.args);
|
|
161
|
+
const logPath = sessionInfo?.logPath || null;
|
|
162
|
+
|
|
163
|
+
const upload = await attachIntermediateSessionLog({
|
|
164
|
+
attachLogs,
|
|
165
|
+
logPath,
|
|
166
|
+
pullRequestUrl,
|
|
167
|
+
attachLog,
|
|
168
|
+
attachOptions,
|
|
169
|
+
verbose,
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
const body = buildKillRecoveryNotice({
|
|
173
|
+
diagnosis,
|
|
174
|
+
exitCode,
|
|
175
|
+
sessionName,
|
|
176
|
+
observedAt,
|
|
177
|
+
policy,
|
|
178
|
+
resumed: recovered || resumed || sessionInfo?.killRecoveryResumed === true,
|
|
179
|
+
recoverySessionId,
|
|
180
|
+
attempt,
|
|
181
|
+
maxAttempts,
|
|
182
|
+
resumeCommand,
|
|
183
|
+
attachLogs,
|
|
184
|
+
logAttached: upload.uploaded,
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
const result = await postKillRecoveryNotice({
|
|
188
|
+
pullRequestUrl,
|
|
189
|
+
body,
|
|
190
|
+
runCommand,
|
|
191
|
+
fileSuffix: String(sessionName || 'session').replace(/[^A-Za-z0-9._-]/g, '-'),
|
|
192
|
+
verbose,
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
return { posted: result.posted, url: result.url, skipped: result.posted ? null : result.error, logUploaded: upload.uploaded };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export { KILL_CAUSE_FORCED_KILL, KILL_CAUSE_OUT_OF_MEMORY };
|