@link-assistant/hive-mind 2.16.0 → 2.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,263 @@
1
+ /**
2
+ * `$ --resume` / `$ --resume-all` wrappers (start-command >= 0.33.0).
3
+ *
4
+ * Issue #2189 reported that a killed working session was only ever *offered*
5
+ * for resume, and that when Hive Mind did resume one it had to start a fresh
6
+ * isolated run — the container the work had happened in, with its clone, its
7
+ * build cache and its half-finished branch, was thrown away. The missing
8
+ * capability was filed upstream as link-foundation/start#162 and delivered in
9
+ * `start-command@0.33.0`:
10
+ *
11
+ * - `$ --resume <id> -- <command>` re-enters an existing execution. For a
12
+ * stopped docker session it commits the container filesystem and runs the
13
+ * new command in a container derived from that snapshot, so the workspace
14
+ * survives. The execution UUID is preserved, so `--status`, `--list` and
15
+ * `--upload-log` keep addressing one logical session across restarts.
16
+ * - `$ --resume-all` re-attaches a completion watcher to every execution
17
+ * still marked running and reconciles the ones that ended unsupervised. It
18
+ * never restarts work silently.
19
+ *
20
+ * Both are additive: a Hive Mind talking to an older `$` gets a clean
21
+ * `unsupported` result and the caller falls back to its previous behaviour.
22
+ * Neither wrapper throws.
23
+ *
24
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
25
+ * @see https://github.com/link-foundation/start/issues/162
26
+ */
27
+
28
+ import { describeChildExit } from './child-exit.lib.mjs';
29
+ import { findStartCommandBinary, getCommandStreamDollar, START_COMMAND_MISSING_ERROR } from './start-command-cli.lib.mjs';
30
+
31
+ /** Strategies `$ --resume` can pick, mirroring upstream `ResumeMode`. */
32
+ export const RESUME_MODES = Object.freeze({
33
+ DOCKER_START: 'docker-start',
34
+ DOCKER_SNAPSHOT: 'docker-snapshot',
35
+ RELAUNCH: 'relaunch',
36
+ });
37
+
38
+ /** Outcomes `$ --resume-all` reports per execution, mirroring `ResumeAllAction`. */
39
+ export const RESUME_ALL_ACTIONS = Object.freeze({
40
+ REATTACHED: 'reattached',
41
+ RUNNING: 'running',
42
+ RECONCILED: 'reconciled',
43
+ UNKNOWN: 'unknown',
44
+ });
45
+
46
+ /**
47
+ * Does this `$` failure mean the verb does not exist yet?
48
+ *
49
+ * An older binary rejects the flag while parsing, long before it looks at the
50
+ * store. Distinguishing that from a real refusal ("session is still running")
51
+ * is what lets the caller degrade gracefully instead of reporting a bug.
52
+ *
53
+ * @param {string} message - stderr/message from the failed invocation
54
+ * @returns {boolean}
55
+ */
56
+ export function isUnsupportedStartCommandVerb(message) {
57
+ const text = String(message || '').toLowerCase();
58
+ // 0.32.1 answers `$ --resume-all` with `Error: Unknown wrapper option:
59
+ // --resume-all` (verified against the pinned pre-0.33.0 binary), and other
60
+ // argument parsers word it differently; match the family, not one string.
61
+ return /unknown (\w+ )?(option|argument|flag)|unrecognized option|invalid option|no such option/.test(text);
62
+ }
63
+
64
+ /**
65
+ * Parse the `executionResume` block `$ --resume --output-format json` prints.
66
+ *
67
+ * Tolerates links notation too (`executionResume` followed by indented
68
+ * `key value` pairs), because an operator's `$` may default to it.
69
+ *
70
+ * @param {string} output - Raw stdout
71
+ * @returns {{uuid: string|null, mode: string|null, backend: string|null, sessionName: string|null, previousSessionName: string|null, snapshotImage: string|null, command: string|null, message: string|null}}
72
+ */
73
+ export function parseExecutionResumeOutput(output) {
74
+ const empty = { uuid: null, mode: null, backend: null, sessionName: null, previousSessionName: null, snapshotImage: null, command: null, message: null };
75
+ const raw = (output || '').trim();
76
+ if (!raw) return empty;
77
+ const str = value => (typeof value === 'string' && value.trim() ? value.trim() : null);
78
+ try {
79
+ const parsed = JSON.parse(raw);
80
+ const data = Array.isArray(parsed) ? parsed[0] : parsed;
81
+ return {
82
+ uuid: str(data?.uuid),
83
+ mode: str(data?.mode),
84
+ backend: str(data?.backend),
85
+ sessionName: str(data?.sessionName),
86
+ previousSessionName: str(data?.previousSessionName),
87
+ snapshotImage: str(data?.snapshotImage),
88
+ command: str(data?.command),
89
+ message: str(data?.message),
90
+ };
91
+ } catch {
92
+ // Links notation — fall through.
93
+ }
94
+ // Links notation indents `key value`; `--output-format text` prints
95
+ // `Label: value`. The optional colon covers both with one expression.
96
+ const readField = name => {
97
+ const match = raw.match(new RegExp(`^\\s*${name}\\s*:?\\s+"?([^"\\n]+)"?\\s*$`, 'mi'));
98
+ return str(match?.[1]);
99
+ };
100
+ return {
101
+ uuid: readField('uuid'),
102
+ mode: readField('mode') || readField('Resume Mode'),
103
+ backend: readField('backend'),
104
+ sessionName: readField('sessionName') || readField('Session Name'),
105
+ previousSessionName: readField('previousSessionName'),
106
+ snapshotImage: readField('snapshotImage'),
107
+ command: readField('command'),
108
+ message: readField('message'),
109
+ };
110
+ }
111
+
112
+ /**
113
+ * Parse the `executionResumeAll` block `$ --resume-all --output-format json`
114
+ * prints. Anything unparseable yields an empty list rather than a throw — a
115
+ * startup reconciliation must never be able to stop the bot from starting.
116
+ *
117
+ * @param {string} output - Raw stdout
118
+ * @returns {Array<{uuid: string|null, backend: string|null, sessionName: string|null, state: string|null, action: string|null, exitCode: number|null, message: string|null}>}
119
+ */
120
+ export function parseExecutionResumeAllOutput(output) {
121
+ const raw = (output || '').trim();
122
+ if (!raw) return [];
123
+ let parsed;
124
+ try {
125
+ parsed = JSON.parse(raw);
126
+ } catch {
127
+ return [];
128
+ }
129
+ const records = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.executions) ? parsed.executions : [];
130
+ const str = value => (typeof value === 'string' && value.trim() ? value.trim() : null);
131
+ return records
132
+ .map(entry => {
133
+ if (!entry || typeof entry !== 'object') return null;
134
+ const exitCode = entry.exitCode === null || entry.exitCode === undefined ? null : Number(entry.exitCode);
135
+ return {
136
+ uuid: str(entry.uuid),
137
+ backend: str(entry.backend),
138
+ sessionName: str(entry.sessionName),
139
+ state: str(entry.state),
140
+ action: str(entry.action),
141
+ exitCode: Number.isFinite(exitCode) ? exitCode : null,
142
+ message: str(entry.message),
143
+ };
144
+ })
145
+ .filter(Boolean);
146
+ }
147
+
148
+ /**
149
+ * Normalize what `command-stream`'s `$` hands back for one invocation.
150
+ *
151
+ * `$` does *not* throw on a non-zero exit — it resolves with `code` set (checked
152
+ * against command-stream in experiments/issue-2189-start-command-resume.mjs).
153
+ * Reading only the resolved value would therefore report every refusal
154
+ * ("session is still running", "no execution found", "unknown wrapper option")
155
+ * as a successful resume. Both shapes are folded into one verdict here.
156
+ *
157
+ * @param {object|null} result - Resolved value from `$`
158
+ * @param {*} [error] - Rejection from `$`, when it threw instead
159
+ * @returns {{ok: boolean, stdout: string, message: string|null, unsupported: boolean}}
160
+ */
161
+ function interpretStartCommandResult(result, error = null) {
162
+ const source = error || result || {};
163
+ const stdout = source.stdout?.toString?.().trim() || '';
164
+ const stderr = source.stderr?.toString?.().trim() || '';
165
+ const code = error ? (Number.isFinite(source.code) ? source.code : 1) : Number.isFinite(source.code) ? source.code : 0;
166
+ if (!error && code === 0) return { ok: true, stdout, message: null, unsupported: false };
167
+ // describeChildExit is the repository's single vocabulary for "how a child
168
+ // ended" (issue #2135); command-stream has already normalized a signalled
169
+ // exit to 128+signum by this point, so the code is all there is to say.
170
+ const message = stderr || source.message || describeChildExit({ command: 'start-command', code });
171
+ return { ok: false, stdout, message, unsupported: isUnsupportedStartCommandVerb(`${message}\n${stdout}`) };
172
+ }
173
+
174
+ /**
175
+ * Re-enter an existing execution via `$ --resume <identifier> [-- <command>]`.
176
+ *
177
+ * With a `command`, a stopped docker session is snapshotted and the command runs
178
+ * against that snapshot, so the work already on disk is preserved — this is the
179
+ * "re-enter the same container" half of issue #2189. Without one, the stored
180
+ * command is re-run in place.
181
+ *
182
+ * The command is passed as a single argument after `--`, exactly like the launch
183
+ * path does: start-command 0.33.0 preserves argv boundaries and runs a lone
184
+ * argument verbatim as a shell script, so no quoting is lost.
185
+ *
186
+ * @param {string} identifier - Execution UUID or session name
187
+ * @param {Object} [options]
188
+ * @param {string|null} [options.command] - Command to run against the resumed session
189
+ * @param {boolean} [options.verbose]
190
+ * @returns {Promise<{success: boolean, unsupported: boolean, uuid: string|null, mode: string|null, backend: string|null, sessionName: string|null, previousSessionName: string|null, snapshotImage: string|null, message: string|null, output: string, error: string|null}>}
191
+ */
192
+ export async function resumeIsolatedSession(identifier, { command = null, verbose = false } = {}) {
193
+ const base = { success: false, unsupported: false, uuid: null, mode: null, backend: null, sessionName: null, previousSessionName: null, snapshotImage: null, message: null, output: '', error: null };
194
+ if (!identifier) return { ...base, error: 'No execution identifier was given to resume.' };
195
+
196
+ const binPath = await findStartCommandBinary();
197
+ if (!binPath) {
198
+ if (verbose) console.log('[VERBOSE] isolation-runner: cannot resume - $ binary not found');
199
+ return { ...base, unsupported: true, error: START_COMMAND_MISSING_ERROR };
200
+ }
201
+
202
+ try {
203
+ const $ = await getCommandStreamDollar();
204
+ const raw = command ? await $({ mirror: false })`${binPath} --resume ${identifier} --output-format json -- ${command}` : await $({ mirror: false })`${binPath} --resume ${identifier} --output-format json`;
205
+ const { ok, stdout, message, unsupported } = interpretStartCommandResult(raw);
206
+ if (!ok) {
207
+ if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume ${identifier} refused${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
208
+ return { ...base, unsupported, output: stdout, error: message };
209
+ }
210
+ const parsed = parseExecutionResumeOutput(stdout);
211
+ if (verbose) {
212
+ console.log(`[VERBOSE] isolation-runner: $ --resume ${identifier} → mode=${parsed.mode || '(unknown)'} session=${parsed.sessionName || '(unknown)'} uuid=${parsed.uuid || '(unknown)'}`);
213
+ }
214
+ return { ...base, ...parsed, success: true, output: stdout };
215
+ } catch (error) {
216
+ const { stdout, message, unsupported } = interpretStartCommandResult(null, error);
217
+ if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume ${identifier} failed${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
218
+ return { ...base, unsupported, output: stdout, error: message };
219
+ }
220
+ }
221
+
222
+ /**
223
+ * Reconcile every execution still marked running via `$ --resume-all`.
224
+ *
225
+ * Run at bot startup: the detached-docker completion watchers are children of
226
+ * the process that launched them, so a bot restart leaves every running
227
+ * container unsupervised — its exit would never be written to the log footer,
228
+ * which is one of the ways issue #2189's session stayed in limbo. `--resume-all`
229
+ * re-attaches a watcher to what is alive and finalizes what died meanwhile. It
230
+ * starts no work on its own.
231
+ *
232
+ * @param {Object} [options]
233
+ * @param {boolean} [options.verbose]
234
+ * @returns {Promise<{success: boolean, unsupported: boolean, executions: Array<Object>, output: string, error: string|null}>}
235
+ */
236
+ export async function resumeAllIsolationSessions({ verbose = false } = {}) {
237
+ const base = { success: false, unsupported: false, executions: [], output: '', error: null };
238
+ const binPath = await findStartCommandBinary();
239
+ if (!binPath) {
240
+ if (verbose) console.log('[VERBOSE] isolation-runner: cannot run $ --resume-all - $ binary not found');
241
+ return { ...base, unsupported: true, error: START_COMMAND_MISSING_ERROR };
242
+ }
243
+
244
+ try {
245
+ const $ = await getCommandStreamDollar();
246
+ const raw = await $({ mirror: false })`${binPath} --resume-all --output-format json`;
247
+ const { ok, stdout, message, unsupported } = interpretStartCommandResult(raw);
248
+ if (!ok) {
249
+ if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume-all refused${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
250
+ return { ...base, unsupported, output: stdout, error: message };
251
+ }
252
+ const executions = parseExecutionResumeAllOutput(stdout);
253
+ if (verbose) {
254
+ const summary = executions.map(entry => `${entry.action}:${entry.sessionName || entry.uuid}`).join(', ') || '(none)';
255
+ console.log(`[VERBOSE] isolation-runner: $ --resume-all reconciled ${executions.length} execution(s): ${summary}`);
256
+ }
257
+ return { ...base, success: true, executions, output: stdout };
258
+ } catch (error) {
259
+ const { stdout, message, unsupported } = interpretStartCommandResult(null, error);
260
+ if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume-all failed${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
261
+ return { ...base, unsupported, output: stdout, error: message };
262
+ }
263
+ }
@@ -106,7 +106,7 @@ const measureDiff = diff => {
106
106
  section = null;
107
107
  };
108
108
 
109
- for (let start = 0; start < diff.length; ) {
109
+ for (let start = 0; start < diff.length;) {
110
110
  let end = diff.indexOf('\n', start);
111
111
  if (end === -1) end = diff.length;
112
112
  const line = diff.slice(start, end);
@@ -54,6 +54,16 @@ export const KILL_DIAGNOSTICS_LOG_BYTES = 1024 * 1024;
54
54
  */
55
55
  export const HEAP_EXHAUSTED_PERCENT = 90;
56
56
 
57
+ /**
58
+ * Prefix start-command 0.33.0 puts on every `exitReason` that reports memory
59
+ * exhaustion, whatever the mechanism — `memory-exhaustion (v8-heap-limit)`,
60
+ * `memory-exhaustion (kernel-oom-killer)`, `memory-exhaustion (go-runtime)`,
61
+ * `memory-exhaustion (allocation-failure)`. Matching the prefix rather than the
62
+ * exact strings means a new upstream mechanism is classified correctly without
63
+ * a Hive Mind release. See link-foundation/start#164 and #165.
64
+ */
65
+ export const UPSTREAM_MEMORY_EXHAUSTION_PREFIX = 'memory-exhaustion';
66
+
57
67
  /** Disk is considered full at or above this used percentage… */
58
68
  export const DISK_FULL_USED_PERCENT = 95;
59
69
  /** …or below this much free space, whichever triggers first. */
@@ -268,9 +278,12 @@ function describeDisk(disk, timestamp) {
268
278
  * @param {number|null} [params.exitCode]
269
279
  * @param {Object|null} [params.system] - collectSystemKillDiagnostics() result
270
280
  * @param {boolean} [params.stopRequestedByUser] - The operator asked for the stop
281
+ * @param {boolean|null} [params.reportedMemoryExhausted] - `$ --status` `memoryExhausted` (start-command >= 0.33.0)
282
+ * @param {string|null} [params.reportedMemoryExhaustedReason] - `$ --status` `memoryExhaustedReason` (the evidence line)
283
+ * @param {string|null} [params.reportedExitReason] - `$ --status` `exitReason` hint, e.g. `memory-exhaustion (v8-heap-limit)`
271
284
  * @returns {{cause: string, summary: string, evidence: string[], memory: Object|null, disk: Object|null, victims: Array}}
272
285
  */
273
- export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false } = {}) {
286
+ export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false, reportedMemoryExhausted = null, reportedMemoryExhaustedReason = null, reportedExitReason = null } = {}) {
274
287
  const parsed = resourceMarkers || (logText ? parseResourceMarkers(logText) : { markers: [], byPhase: {} });
275
288
  const memoryMarker = selectLastMemoryResourceMarker(parsed);
276
289
  const heapMarker = selectLastHeapResourceMarker(parsed);
@@ -310,6 +323,30 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
310
323
  evidence.push(`the ${fatalMemoryMarker.runtime} runtime aborted on its own heap limit: \`${fatalMemoryMarker.line}\` (a self-abort is invisible to \`docker inspect\` and to cgroup OOM counters)`);
311
324
  }
312
325
 
326
+ // Issue #2189 also went upstream: start-command 0.33.0 (link-foundation/start
327
+ // #164, #165) performs the same tail scan inside `$` and reports it as
328
+ // `memoryExhausted` / `memoryExhaustedReason` / `exitReason`. Consuming it
329
+ // catches the case our own scan cannot: the fatal line scrolled out of the
330
+ // bounded window we read, but `$` saw it when the command exited. The local
331
+ // scan stays as defense in depth — these fields are absent on an older `$`.
332
+ const reportedExitReasonText = typeof reportedExitReason === 'string' && reportedExitReason.trim() ? reportedExitReason.trim() : null;
333
+ const reportedMemoryExhaustion = abnormalExit && (reportedMemoryExhausted === true || (reportedExitReasonText !== null && reportedExitReasonText.startsWith(UPSTREAM_MEMORY_EXHAUSTION_PREFIX)));
334
+ // `memory-exhaustion (v8-heap-limit)` → `v8-heap-limit`: the prefix is already
335
+ // said in words by the surrounding sentence, so only the mechanism is new.
336
+ const reportedMechanism =
337
+ reportedMemoryExhaustion && reportedExitReasonText
338
+ ? reportedExitReasonText
339
+ .slice(UPSTREAM_MEMORY_EXHAUSTION_PREFIX.length)
340
+ .trim()
341
+ .replace(/^\((.*)\)$/, '$1') || null
342
+ : null;
343
+ if (reportedMemoryExhaustion) {
344
+ const detail = reportedMemoryExhaustedReason ? `: \`${reportedMemoryExhaustedReason}\`` : '';
345
+ evidence.push(`\`$ --status\` reports memory exhaustion${reportedMechanism ? ` (\`${reportedMechanism}\`)` : ''}${detail}`);
346
+ } else if (reportedExitReasonText && abnormalExit) {
347
+ evidence.push(`\`$ --status\` reports \`exitReason = ${reportedExitReasonText}\``);
348
+ }
349
+
313
350
  const ratio = memoryRatio(memory);
314
351
  const memoryExhausted = ratio !== null && ratio <= MEMORY_EXHAUSTED_AVAILABLE_RATIO;
315
352
  // A heap already at the limit is only evidence of a kill when the session
@@ -322,7 +359,7 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
322
359
  let cause = KILL_CAUSE_UNKNOWN;
323
360
  if (stopRequestedByUser) {
324
361
  cause = KILL_CAUSE_FORCED_KILL;
325
- } else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted || fatalMemoryMarker || heapExhausted) {
362
+ } else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted || fatalMemoryMarker || heapExhausted || reportedMemoryExhaustion) {
326
363
  cause = KILL_CAUSE_OUT_OF_MEMORY;
327
364
  } else if (diskFull) {
328
365
  cause = KILL_CAUSE_DISK_FULL;
@@ -340,6 +377,11 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
340
377
  // Same shape as the fatal-marker case, but reconstructed from telemetry when
341
378
  // the fatal line itself was lost (truncated tail, killed before flushing).
342
379
  summary = `out of memory — the runtime's own heap was exhausted: ${heapLine}${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
380
+ } else if (cause === KILL_CAUSE_OUT_OF_MEMORY && reportedMemoryExhaustion && victims.length === 0 && !oomKilled && !(cgroupOomKills > 0) && !memoryExhausted) {
381
+ // Only `$` saw the evidence (our bounded window missed the fatal line).
382
+ // Quote what it saw rather than falling back to the host-memory phrasing,
383
+ // which would again read as a contradiction on a healthy machine.
384
+ summary = `out of memory — start-command reported memory exhaustion${reportedMechanism ? ` (${reportedMechanism})` : ''}${reportedMemoryExhaustedReason ? `: \`${reportedMemoryExhaustedReason}\`` : ''}${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
343
385
  } else if (cause === KILL_CAUSE_OUT_OF_MEMORY) {
344
386
  const victim = victims.length > 0 ? `, kernel OOM killer terminated \`${victims[victims.length - 1].comm || 'unknown'}\` (pid ${victims[victims.length - 1].pid ?? '?'})` : '';
345
387
  summary = `out of memory${memoryLine ? ` — ${memoryLine}` : ''}${victim}`;
@@ -352,7 +394,7 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
352
394
  summary = 'unknown — no resource marker, cgroup counter or kernel OOM report was available';
353
395
  }
354
396
 
355
- return { cause, summary, evidence, memory, heap: heapMemory, heapUsedPercent, disk, victims, fatalMemoryMarker };
397
+ return { cause, summary, evidence, memory, heap: heapMemory, heapUsedPercent, disk, victims, fatalMemoryMarker, reportedMemoryExhaustion, reportedExitReason: reportedExitReasonText };
356
398
  }
357
399
 
358
400
  /**
@@ -429,7 +471,7 @@ export function formatKillResumeSection({ sessionId = null, attempt = null, maxA
429
471
  * @param {Object} [options]
430
472
  * @returns {Promise<{section: string, diagnosis: Object|null}>}
431
473
  */
432
- export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, maxLogBytes = KILL_DIAGNOSTICS_LOG_BYTES, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics } = {}) {
474
+ export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, maxLogBytes = KILL_DIAGNOSTICS_LOG_BYTES, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics, reportedMemoryExhausted = null, reportedMemoryExhaustedReason = null, reportedExitReason = null } = {}) {
433
475
  try {
434
476
  let logText = '';
435
477
  if (logPath) {
@@ -441,7 +483,7 @@ export async function buildKillDiagnosticsSection(logPath, { verbose = false, re
441
483
  logText = await readLogTextBounded(logPath, { readFile, maxBytes: maxLogBytes, verbose });
442
484
  }
443
485
  const system = await collectSystem({ verbose });
444
- const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser });
486
+ const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser, reportedMemoryExhausted, reportedMemoryExhaustedReason, reportedExitReason });
445
487
  if (verbose) console.log(`[VERBOSE] kill-diagnostics: cause=${diagnosis.cause} — ${diagnosis.summary}`);
446
488
  return { section: formatKillDiagnosticsSection(diagnosis, { locale }), diagnosis };
447
489
  } catch (error) {
@@ -0,0 +1,136 @@
1
+ /**
2
+ * Re-enter the *same* container when recovering a killed session (issue #2189).
3
+ *
4
+ * The incident behind #2189 ended with a session that was killed 10 minutes
5
+ * after the AI tool had already finished its work. Recovery, when it finally
6
+ * happened, threw that container away: a fresh isolated run re-cloned the
7
+ * repository, re-installed everything and re-did work that was sitting on disk.
8
+ * The issue asks for the opposite — "ideally re-entering the same `$` session
9
+ * id / container".
10
+ *
11
+ * `start-command@0.33.0` (upstream link-foundation/start#162, filed from this
12
+ * very issue) makes that possible: `$ --resume <id> -- <command>` commits the
13
+ * stopped container's filesystem and runs the recovery command in a container
14
+ * derived from that snapshot, keeping the original execution UUID and log.
15
+ *
16
+ * Not every session may take that path, and the two exceptions are deliberate:
17
+ *
18
+ * - **Formal AI tasks** (issue #2146) reach their sidecar over an *internal*
19
+ * Docker network that Hive Mind attaches with `docker network connect` after
20
+ * the container is created. `$` knows nothing about that network, so a
21
+ * resumed container would come up without it and the task would silently talk
22
+ * to nothing. #2146 requires Formal AI to fail closed, so these fall back to
23
+ * the normal launch path, which re-acquires the sidecar lease properly.
24
+ * - **`--use-router` tasks** are attached to the router network the same way,
25
+ * with a freshly minted token, and have the same problem.
26
+ *
27
+ * Everything else — the overwhelming majority, and every session in the
28
+ * original incident — resumes in place.
29
+ *
30
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
31
+ * @see https://github.com/link-foundation/start/issues/162
32
+ * @see https://github.com/link-assistant/hive-mind/issues/2146
33
+ */
34
+
35
+ import { isFormalAiTask } from './formal-ai-sidecar.lib.mjs';
36
+ import { hasUseRouterFlag } from './router-isolation.lib.mjs';
37
+ import { RESUME_MODES } from './isolation-runner.resume.lib.mjs';
38
+
39
+ /** Why a killed session cannot be re-entered in place. Reported, never thrown. */
40
+ export const IN_PLACE_SKIP_REASONS = Object.freeze({
41
+ NOT_DOCKER: 'not-docker',
42
+ NO_IDENTIFIER: 'no-identifier',
43
+ FORMAL_AI_TASK: 'formal-ai-task',
44
+ ROUTER_TASK: 'router-task',
45
+ NO_RESUME_SUPPORT: 'no-resume-support',
46
+ CONTAINER_GONE: 'container-gone',
47
+ UNSUPPORTED: 'resume-unsupported',
48
+ REFUSED: 'resume-refused',
49
+ ERROR: 'resume-error',
50
+ });
51
+
52
+ /**
53
+ * Decide — purely, from persisted facts — whether a killed session is a
54
+ * candidate for a same-container resume.
55
+ *
56
+ * Kept separate from the Docker probe below so the policy is testable without a
57
+ * daemon, and so a caller can report precisely *why* a session was relaunched
58
+ * from scratch instead of resumed.
59
+ *
60
+ * @param {Object} options
61
+ * @param {string} options.sessionName - The killed session's name (= container name)
62
+ * @param {Object} options.sessionInfo - Persisted session info
63
+ * @returns {{eligible: boolean, reason: string, identifier: string|null, containerName: string|null}}
64
+ */
65
+ export function planSameContainerResume({ sessionName = null, sessionInfo = {} } = {}) {
66
+ const containerName = sessionInfo?.sessionId || sessionName || null;
67
+ const identifier = sessionInfo?.executionUuid || containerName || null;
68
+ const base = { eligible: false, identifier, containerName };
69
+
70
+ if (sessionInfo?.isolationBackend !== 'docker') {
71
+ // screen/tmux sessions have no filesystem to preserve: their work happens
72
+ // on the host, which a fresh run already sees.
73
+ return { ...base, reason: IN_PLACE_SKIP_REASONS.NOT_DOCKER };
74
+ }
75
+ if (!identifier) return { ...base, reason: IN_PLACE_SKIP_REASONS.NO_IDENTIFIER };
76
+
77
+ const args = Array.isArray(sessionInfo?.args) ? sessionInfo.args : [];
78
+ if (isFormalAiTask({ args, model: sessionInfo?.model || null })) {
79
+ return { ...base, reason: IN_PLACE_SKIP_REASONS.FORMAL_AI_TASK };
80
+ }
81
+ if (hasUseRouterFlag(args)) return { ...base, reason: IN_PLACE_SKIP_REASONS.ROUTER_TASK };
82
+
83
+ return { ...base, eligible: true, reason: 'ready' };
84
+ }
85
+
86
+ /**
87
+ * Attempt the same-container resume. Never throws, and never leaves work
88
+ * running that it does not report: the caller may only fall back to a fresh
89
+ * launch when `resumed` is false.
90
+ *
91
+ * @param {Object} options
92
+ * @param {string} options.sessionName - The killed session's name
93
+ * @param {Object} options.sessionInfo - Persisted session info
94
+ * @param {Object} options.plan - Result of planKillRecovery() (needs `command.display`)
95
+ * @param {Object} options.runner - Isolation runner module
96
+ * @param {boolean} [options.verbose]
97
+ * @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, executionUuid: string|null, mode: string|null, snapshotImage: string|null}>}
98
+ */
99
+ export async function resumeKilledSessionInPlace({ sessionName, sessionInfo, plan, runner, verbose = false } = {}) {
100
+ const decision = planSameContainerResume({ sessionName, sessionInfo });
101
+ const miss = reason => ({ resumed: false, reason, sessionId: null, executionUuid: decision.identifier, mode: null, snapshotImage: null });
102
+ if (!decision.eligible) return miss(decision.reason);
103
+ if (typeof runner?.resumeIsolatedSession !== 'function' || typeof runner?.checkDockerContainerExists !== 'function') {
104
+ return miss(IN_PLACE_SKIP_REASONS.NO_RESUME_SUPPORT);
105
+ }
106
+
107
+ // A container that no longer exists has nothing left to re-enter; `$` would
108
+ // fall back to a full relaunch, which is what the caller does anyway — but
109
+ // through the path that also re-acquires leases.
110
+ const exists = await runner.checkDockerContainerExists(decision.containerName, verbose);
111
+ if (!exists) return miss(IN_PLACE_SKIP_REASONS.CONTAINER_GONE);
112
+
113
+ const result = await runner.resumeIsolatedSession(decision.identifier, { command: plan?.command?.display || null, verbose });
114
+ if (!result?.success) {
115
+ const reason = result?.unsupported ? IN_PLACE_SKIP_REASONS.UNSUPPORTED : IN_PLACE_SKIP_REASONS.REFUSED;
116
+ if (verbose) console.log(`[VERBOSE] In-place resume of ${sessionName} was not possible (${reason}): ${result?.error || 'no reason given'}`);
117
+ return miss(reason);
118
+ }
119
+
120
+ // `docker-snapshot` names the new container `<session>-resume-<attempt>`; the
121
+ // old name stays addressable through upstream's `sessionNameHistory`, but the
122
+ // *new* one is what `$ --status` reports on now, so that is what the monitor
123
+ // has to track. A resume that somehow reports the old name (a `docker-start`
124
+ // race, say) is tracked under the execution UUID instead, which upstream
125
+ // resolves just as well and cannot collide with the dying session's entry.
126
+ const returnedName = result.sessionName && result.sessionName !== sessionName ? result.sessionName : null;
127
+ const sessionId = returnedName || decision.identifier;
128
+ return {
129
+ resumed: true,
130
+ reason: result.mode === RESUME_MODES.DOCKER_SNAPSHOT ? 'resumed-in-place' : `resumed-${result.mode || 'unknown'}`,
131
+ sessionId,
132
+ executionUuid: result.uuid || decision.identifier,
133
+ mode: result.mode || null,
134
+ snapshotImage: result.snapshotImage || null,
135
+ };
136
+ }
@@ -24,6 +24,7 @@ import { readLastSessionIdFromLog, planKilledSessionResume } from './session-res
24
24
  import { resolveOnSessionKillPolicy, resolveSessionKillResumeAttempts, shouldResumeKilledSession, ON_SESSION_KILL_RESUME } from './session-kill-policy.lib.mjs';
25
25
  import { argvFromSessionArgs } from './session-monitor.kill-sections.lib.mjs';
26
26
  import { formatKillResumeSection } from './session-kill-diagnostics.lib.mjs';
27
+ import { resumeKilledSessionInPlace } from './session-kill-resume.in-place.lib.mjs';
27
28
 
28
29
  /** Field recording how many automatic recovery sessions this session produced. */
29
30
  export const KILL_RESUME_ATTEMPTS_FIELD = 'killRecoveryAttempts';
@@ -65,9 +66,18 @@ export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = fa
65
66
  /**
66
67
  * Start the recovery working session decided by {@link planKillRecovery}.
67
68
  *
68
- * The new session is launched through the same isolation runner the original
69
- * used and is tracked like any other session, so it reports its own completion
70
- * (and, if it is killed too, its own diagnosis) through the normal path.
69
+ * Two ways in, in order of preference:
70
+ *
71
+ * 1. **Same container** (issue #2189) `$ --resume` re-enters the killed
72
+ * session's own filesystem, so the clone, the caches and the half-finished
73
+ * branch survive. See `./session-kill-resume.in-place.lib.mjs` for the
74
+ * cases that are deliberately excluded.
75
+ * 2. **A fresh isolated run** — the original behaviour, used whenever (1) is
76
+ * not available or refuses. Correct, just more expensive.
77
+ *
78
+ * Either way the new session is tracked like any other, so it reports its own
79
+ * completion (and, if it is killed too, its own diagnosis) through the normal
80
+ * path.
71
81
  *
72
82
  * @param {Object} options
73
83
  * @param {string} options.sessionName - The killed session's name
@@ -77,10 +87,10 @@ export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = fa
77
87
  * @param {Function} options.trackSession - Tracker for the new session
78
88
  * @param {Function} [options.persistSnapshot] - Persist the attempt counter
79
89
  * @param {boolean} [options.verbose]
80
- * @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null}>}
90
+ * @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null, inPlace: boolean}>}
81
91
  */
82
92
  export async function startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot = null, verbose = false } = {}) {
83
- const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null });
93
+ const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null, inPlace: false });
84
94
  if (!plan?.shouldResume || !plan.command) return fail(plan?.reason || 'no-plan');
85
95
  if (!runner || typeof runner.executeWithIsolation !== 'function' || typeof runner.generateSessionId !== 'function') return fail('no-isolation-runner');
86
96
  if (typeof trackSession !== 'function') return fail('no-tracker');
@@ -88,10 +98,20 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
88
98
  if (!backend) return fail('no-isolation-backend');
89
99
 
90
100
  try {
91
- const newSessionId = runner.generateSessionId();
92
- const tool = sessionInfo?.tool || 'claude';
93
- const result = await runner.executeWithIsolation(sessionInfo?.command || 'solve', plan.command.args, { backend, sessionId: newSessionId, tool, verbose });
94
- if (!result?.success) return fail('start-failed');
101
+ // Preferred path: re-enter the container the work already happened in.
102
+ const inPlace = await resumeKilledSessionInPlace({ sessionName, sessionInfo, plan, runner, verbose });
103
+ let newSessionId = inPlace.sessionId;
104
+ let executionUuid = inPlace.executionUuid;
105
+ let containerFilesystemStartBytes = null;
106
+
107
+ if (!inPlace.resumed) {
108
+ newSessionId = runner.generateSessionId();
109
+ const tool = sessionInfo?.tool || 'claude';
110
+ const result = await runner.executeWithIsolation(sessionInfo?.command || 'solve', plan.command.args, { backend, sessionId: newSessionId, tool, verbose });
111
+ if (!result?.success) return fail('start-failed');
112
+ executionUuid = result.executionUuid || null;
113
+ containerFilesystemStartBytes = Number.isFinite(result.containerFilesystemStartBytes) ? result.containerFilesystemStartBytes : null;
114
+ }
95
115
 
96
116
  trackSession(
97
117
  newSessionId,
@@ -105,9 +125,15 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
105
125
  [KILL_RESUME_ATTEMPTS_FIELD]: plan.attempt,
106
126
  killRecoveryResumed: true,
107
127
  killRecoveryOfSession: sessionName,
128
+ // A resumed execution keeps its UUID; a fresh launch gets a new one, and
129
+ // inheriting the dead session's would make `$ --status` answer about the
130
+ // wrong execution until the monitor happened to correct it.
131
+ executionUuid,
132
+ killRecoveryInPlace: inPlace.resumed,
133
+ killRecoveryResumeMode: inPlace.mode || null,
108
134
  oomEventObservedAt: undefined,
109
135
  dockerBackendGoneFirstSeenAt: undefined,
110
- containerFilesystemStartBytes: Number.isFinite(result.containerFilesystemStartBytes) ? result.containerFilesystemStartBytes : null,
136
+ containerFilesystemStartBytes,
111
137
  },
112
138
  verbose
113
139
  );
@@ -122,9 +148,10 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
122
148
  }
123
149
 
124
150
  if (verbose) {
125
- console.log(`[VERBOSE] Session ${sessionName} was killed; started recovery session ${newSessionId} (attempt ${plan.attempt}/${plan.maxAttempts}): ${plan.command.display}`);
151
+ const how = inPlace.resumed ? `resumed in place (${inPlace.mode || 'unknown mode'})` : `started fresh (in-place resume skipped: ${inPlace.reason})`;
152
+ console.log(`[VERBOSE] Session ${sessionName} was killed; recovery session ${newSessionId} ${how} (attempt ${plan.attempt}/${plan.maxAttempts}): ${plan.command.display}`);
126
153
  }
127
- return { resumed: true, reason: 'started', sessionId: newSessionId, display: plan.command.display };
154
+ return { resumed: true, reason: inPlace.resumed ? inPlace.reason : 'started', sessionId: newSessionId, display: plan.command.display, inPlace: inPlace.resumed };
128
155
  } catch (error) {
129
156
  if (verbose) {
130
157
  console.log(`[VERBOSE] Could not start recovery session for ${sessionName}: ${error?.message || error}`);
@@ -138,7 +165,7 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
138
165
  * Never throws — a failed recovery must still leave a correct kill report.
139
166
  *
140
167
  * @param {Object} options - See planKillRecovery() and startKillRecoverySession()
141
- * @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number}>}
168
+ * @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number, inPlace: boolean}>}
142
169
  */
143
170
  export async function recoverKilledSession({ sessionName, sessionInfo, logPath = null, killed = false, env = process.env, runner = null, trackSession = null, persistSnapshot = null, verbose = false, readLastSessionId = readLastSessionIdFromLog } = {}) {
144
171
  let plan;
@@ -146,15 +173,15 @@ export async function recoverKilledSession({ sessionName, sessionInfo, logPath =
146
173
  plan = planKillRecovery({ sessionInfo, logPath, killed, env, verbose, readLastSessionId });
147
174
  } catch (error) {
148
175
  if (verbose) console.log(`[VERBOSE] Could not plan kill recovery for ${sessionName}: ${error?.message || error}`);
149
- return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0 };
176
+ return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0, inPlace: false };
150
177
  }
151
178
 
152
179
  if (!plan.shouldResume) {
153
- return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
180
+ return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts, inPlace: false };
154
181
  }
155
182
 
156
183
  const started = await startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot, verbose });
157
- return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
184
+ return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts, inPlace: started.inPlace === true };
158
185
  }
159
186
 
160
187
  /**