@link-assistant/hive-mind 2.16.0 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +106 -0
- package/README.hi.md +12 -0
- package/README.md +15 -0
- package/README.ru.md +15 -0
- package/README.zh.md +24 -12
- package/package.json +24 -17
- package/src/agent-snapshot-store.lib.mjs +252 -0
- package/src/agent.lib.mjs +25 -25
- package/src/agent.version-gates.lib.mjs +73 -0
- package/src/bot-lifecycle.lib.mjs +55 -4
- package/src/cleanup.mjs +57 -3
- package/src/disk-guard.lib.mjs +21 -1
- package/src/formal-ai-version.lib.mjs +10 -6
- package/src/instrument.mjs +12 -14
- package/src/instrument.sanitize.lib.mjs +52 -0
- package/src/isolation-runner.lib.mjs +56 -33
- package/src/isolation-runner.parsers.lib.mjs +29 -3
- package/src/isolation-runner.resume.lib.mjs +263 -0
- package/src/pull-request-changes.lib.mjs +1 -1
- package/src/session-kill-diagnostics.lib.mjs +47 -5
- package/src/session-kill-resume.in-place.lib.mjs +136 -0
- package/src/session-kill-resume.lib.mjs +43 -16
- package/src/session-monitor.kill-sections.lib.mjs +8 -0
- package/src/session-store.lib.mjs +1 -1
- package/src/solve.clone-errors.lib.mjs +86 -0
- package/src/solve.repository.lib.mjs +36 -63
- package/src/solve.resource-diagnostics.lib.mjs +34 -1
- package/src/start-command-cli.lib.mjs +60 -0
- package/src/telegram-bot.mjs +31 -91
- package/src/telegram-overrides-validation.lib.mjs +73 -0
- package/src/working-session-summary.lib.mjs +1 -1
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `$ --resume` / `$ --resume-all` wrappers (start-command >= 0.33.0).
|
|
3
|
+
*
|
|
4
|
+
* Issue #2189 reported that a killed working session was only ever *offered*
|
|
5
|
+
* for resume, and that when Hive Mind did resume one it had to start a fresh
|
|
6
|
+
* isolated run — the container the work had happened in, with its clone, its
|
|
7
|
+
* build cache and its half-finished branch, was thrown away. The missing
|
|
8
|
+
* capability was filed upstream as link-foundation/start#162 and delivered in
|
|
9
|
+
* `start-command@0.33.0`:
|
|
10
|
+
*
|
|
11
|
+
* - `$ --resume <id> -- <command>` re-enters an existing execution. For a
|
|
12
|
+
* stopped docker session it commits the container filesystem and runs the
|
|
13
|
+
* new command in a container derived from that snapshot, so the workspace
|
|
14
|
+
* survives. The execution UUID is preserved, so `--status`, `--list` and
|
|
15
|
+
* `--upload-log` keep addressing one logical session across restarts.
|
|
16
|
+
* - `$ --resume-all` re-attaches a completion watcher to every execution
|
|
17
|
+
* still marked running and reconciles the ones that ended unsupervised. It
|
|
18
|
+
* never restarts work silently.
|
|
19
|
+
*
|
|
20
|
+
* Both are additive: a Hive Mind talking to an older `$` gets a clean
|
|
21
|
+
* `unsupported` result and the caller falls back to its previous behaviour.
|
|
22
|
+
* Neither wrapper throws.
|
|
23
|
+
*
|
|
24
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2189
|
|
25
|
+
* @see https://github.com/link-foundation/start/issues/162
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { describeChildExit } from './child-exit.lib.mjs';
|
|
29
|
+
import { findStartCommandBinary, getCommandStreamDollar, START_COMMAND_MISSING_ERROR } from './start-command-cli.lib.mjs';
|
|
30
|
+
|
|
31
|
+
/** Strategies `$ --resume` can pick, mirroring upstream `ResumeMode`. */
|
|
32
|
+
export const RESUME_MODES = Object.freeze({
|
|
33
|
+
DOCKER_START: 'docker-start',
|
|
34
|
+
DOCKER_SNAPSHOT: 'docker-snapshot',
|
|
35
|
+
RELAUNCH: 'relaunch',
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
/** Outcomes `$ --resume-all` reports per execution, mirroring `ResumeAllAction`. */
|
|
39
|
+
export const RESUME_ALL_ACTIONS = Object.freeze({
|
|
40
|
+
REATTACHED: 'reattached',
|
|
41
|
+
RUNNING: 'running',
|
|
42
|
+
RECONCILED: 'reconciled',
|
|
43
|
+
UNKNOWN: 'unknown',
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Does this `$` failure mean the verb does not exist yet?
|
|
48
|
+
*
|
|
49
|
+
* An older binary rejects the flag while parsing, long before it looks at the
|
|
50
|
+
* store. Distinguishing that from a real refusal ("session is still running")
|
|
51
|
+
* is what lets the caller degrade gracefully instead of reporting a bug.
|
|
52
|
+
*
|
|
53
|
+
* @param {string} message - stderr/message from the failed invocation
|
|
54
|
+
* @returns {boolean}
|
|
55
|
+
*/
|
|
56
|
+
export function isUnsupportedStartCommandVerb(message) {
|
|
57
|
+
const text = String(message || '').toLowerCase();
|
|
58
|
+
// 0.32.1 answers `$ --resume-all` with `Error: Unknown wrapper option:
|
|
59
|
+
// --resume-all` (verified against the pinned pre-0.33.0 binary), and other
|
|
60
|
+
// argument parsers word it differently; match the family, not one string.
|
|
61
|
+
return /unknown (\w+ )?(option|argument|flag)|unrecognized option|invalid option|no such option/.test(text);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Parse the `executionResume` block `$ --resume --output-format json` prints.
|
|
66
|
+
*
|
|
67
|
+
* Tolerates links notation too (`executionResume` followed by indented
|
|
68
|
+
* `key value` pairs), because an operator's `$` may default to it.
|
|
69
|
+
*
|
|
70
|
+
* @param {string} output - Raw stdout
|
|
71
|
+
* @returns {{uuid: string|null, mode: string|null, backend: string|null, sessionName: string|null, previousSessionName: string|null, snapshotImage: string|null, command: string|null, message: string|null}}
|
|
72
|
+
*/
|
|
73
|
+
export function parseExecutionResumeOutput(output) {
|
|
74
|
+
const empty = { uuid: null, mode: null, backend: null, sessionName: null, previousSessionName: null, snapshotImage: null, command: null, message: null };
|
|
75
|
+
const raw = (output || '').trim();
|
|
76
|
+
if (!raw) return empty;
|
|
77
|
+
const str = value => (typeof value === 'string' && value.trim() ? value.trim() : null);
|
|
78
|
+
try {
|
|
79
|
+
const parsed = JSON.parse(raw);
|
|
80
|
+
const data = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
81
|
+
return {
|
|
82
|
+
uuid: str(data?.uuid),
|
|
83
|
+
mode: str(data?.mode),
|
|
84
|
+
backend: str(data?.backend),
|
|
85
|
+
sessionName: str(data?.sessionName),
|
|
86
|
+
previousSessionName: str(data?.previousSessionName),
|
|
87
|
+
snapshotImage: str(data?.snapshotImage),
|
|
88
|
+
command: str(data?.command),
|
|
89
|
+
message: str(data?.message),
|
|
90
|
+
};
|
|
91
|
+
} catch {
|
|
92
|
+
// Links notation — fall through.
|
|
93
|
+
}
|
|
94
|
+
// Links notation indents `key value`; `--output-format text` prints
|
|
95
|
+
// `Label: value`. The optional colon covers both with one expression.
|
|
96
|
+
const readField = name => {
|
|
97
|
+
const match = raw.match(new RegExp(`^\\s*${name}\\s*:?\\s+"?([^"\\n]+)"?\\s*$`, 'mi'));
|
|
98
|
+
return str(match?.[1]);
|
|
99
|
+
};
|
|
100
|
+
return {
|
|
101
|
+
uuid: readField('uuid'),
|
|
102
|
+
mode: readField('mode') || readField('Resume Mode'),
|
|
103
|
+
backend: readField('backend'),
|
|
104
|
+
sessionName: readField('sessionName') || readField('Session Name'),
|
|
105
|
+
previousSessionName: readField('previousSessionName'),
|
|
106
|
+
snapshotImage: readField('snapshotImage'),
|
|
107
|
+
command: readField('command'),
|
|
108
|
+
message: readField('message'),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Parse the `executionResumeAll` block `$ --resume-all --output-format json`
|
|
114
|
+
* prints. Anything unparseable yields an empty list rather than a throw — a
|
|
115
|
+
* startup reconciliation must never be able to stop the bot from starting.
|
|
116
|
+
*
|
|
117
|
+
* @param {string} output - Raw stdout
|
|
118
|
+
* @returns {Array<{uuid: string|null, backend: string|null, sessionName: string|null, state: string|null, action: string|null, exitCode: number|null, message: string|null}>}
|
|
119
|
+
*/
|
|
120
|
+
export function parseExecutionResumeAllOutput(output) {
|
|
121
|
+
const raw = (output || '').trim();
|
|
122
|
+
if (!raw) return [];
|
|
123
|
+
let parsed;
|
|
124
|
+
try {
|
|
125
|
+
parsed = JSON.parse(raw);
|
|
126
|
+
} catch {
|
|
127
|
+
return [];
|
|
128
|
+
}
|
|
129
|
+
const records = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.executions) ? parsed.executions : [];
|
|
130
|
+
const str = value => (typeof value === 'string' && value.trim() ? value.trim() : null);
|
|
131
|
+
return records
|
|
132
|
+
.map(entry => {
|
|
133
|
+
if (!entry || typeof entry !== 'object') return null;
|
|
134
|
+
const exitCode = entry.exitCode === null || entry.exitCode === undefined ? null : Number(entry.exitCode);
|
|
135
|
+
return {
|
|
136
|
+
uuid: str(entry.uuid),
|
|
137
|
+
backend: str(entry.backend),
|
|
138
|
+
sessionName: str(entry.sessionName),
|
|
139
|
+
state: str(entry.state),
|
|
140
|
+
action: str(entry.action),
|
|
141
|
+
exitCode: Number.isFinite(exitCode) ? exitCode : null,
|
|
142
|
+
message: str(entry.message),
|
|
143
|
+
};
|
|
144
|
+
})
|
|
145
|
+
.filter(Boolean);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Normalize what `command-stream`'s `$` hands back for one invocation.
|
|
150
|
+
*
|
|
151
|
+
* `$` does *not* throw on a non-zero exit — it resolves with `code` set (checked
|
|
152
|
+
* against command-stream in experiments/issue-2189-start-command-resume.mjs).
|
|
153
|
+
* Reading only the resolved value would therefore report every refusal
|
|
154
|
+
* ("session is still running", "no execution found", "unknown wrapper option")
|
|
155
|
+
* as a successful resume. Both shapes are folded into one verdict here.
|
|
156
|
+
*
|
|
157
|
+
* @param {object|null} result - Resolved value from `$`
|
|
158
|
+
* @param {*} [error] - Rejection from `$`, when it threw instead
|
|
159
|
+
* @returns {{ok: boolean, stdout: string, message: string|null, unsupported: boolean}}
|
|
160
|
+
*/
|
|
161
|
+
function interpretStartCommandResult(result, error = null) {
|
|
162
|
+
const source = error || result || {};
|
|
163
|
+
const stdout = source.stdout?.toString?.().trim() || '';
|
|
164
|
+
const stderr = source.stderr?.toString?.().trim() || '';
|
|
165
|
+
const code = error ? (Number.isFinite(source.code) ? source.code : 1) : Number.isFinite(source.code) ? source.code : 0;
|
|
166
|
+
if (!error && code === 0) return { ok: true, stdout, message: null, unsupported: false };
|
|
167
|
+
// describeChildExit is the repository's single vocabulary for "how a child
|
|
168
|
+
// ended" (issue #2135); command-stream has already normalized a signalled
|
|
169
|
+
// exit to 128+signum by this point, so the code is all there is to say.
|
|
170
|
+
const message = stderr || source.message || describeChildExit({ command: 'start-command', code });
|
|
171
|
+
return { ok: false, stdout, message, unsupported: isUnsupportedStartCommandVerb(`${message}\n${stdout}`) };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Re-enter an existing execution via `$ --resume <identifier> [-- <command>]`.
|
|
176
|
+
*
|
|
177
|
+
* With a `command`, a stopped docker session is snapshotted and the command runs
|
|
178
|
+
* against that snapshot, so the work already on disk is preserved — this is the
|
|
179
|
+
* "re-enter the same container" half of issue #2189. Without one, the stored
|
|
180
|
+
* command is re-run in place.
|
|
181
|
+
*
|
|
182
|
+
* The command is passed as a single argument after `--`, exactly like the launch
|
|
183
|
+
* path does: start-command 0.33.0 preserves argv boundaries and runs a lone
|
|
184
|
+
* argument verbatim as a shell script, so no quoting is lost.
|
|
185
|
+
*
|
|
186
|
+
* @param {string} identifier - Execution UUID or session name
|
|
187
|
+
* @param {Object} [options]
|
|
188
|
+
* @param {string|null} [options.command] - Command to run against the resumed session
|
|
189
|
+
* @param {boolean} [options.verbose]
|
|
190
|
+
* @returns {Promise<{success: boolean, unsupported: boolean, uuid: string|null, mode: string|null, backend: string|null, sessionName: string|null, previousSessionName: string|null, snapshotImage: string|null, message: string|null, output: string, error: string|null}>}
|
|
191
|
+
*/
|
|
192
|
+
export async function resumeIsolatedSession(identifier, { command = null, verbose = false } = {}) {
|
|
193
|
+
const base = { success: false, unsupported: false, uuid: null, mode: null, backend: null, sessionName: null, previousSessionName: null, snapshotImage: null, message: null, output: '', error: null };
|
|
194
|
+
if (!identifier) return { ...base, error: 'No execution identifier was given to resume.' };
|
|
195
|
+
|
|
196
|
+
const binPath = await findStartCommandBinary();
|
|
197
|
+
if (!binPath) {
|
|
198
|
+
if (verbose) console.log('[VERBOSE] isolation-runner: cannot resume - $ binary not found');
|
|
199
|
+
return { ...base, unsupported: true, error: START_COMMAND_MISSING_ERROR };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
try {
|
|
203
|
+
const $ = await getCommandStreamDollar();
|
|
204
|
+
const raw = command ? await $({ mirror: false })`${binPath} --resume ${identifier} --output-format json -- ${command}` : await $({ mirror: false })`${binPath} --resume ${identifier} --output-format json`;
|
|
205
|
+
const { ok, stdout, message, unsupported } = interpretStartCommandResult(raw);
|
|
206
|
+
if (!ok) {
|
|
207
|
+
if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume ${identifier} refused${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
|
|
208
|
+
return { ...base, unsupported, output: stdout, error: message };
|
|
209
|
+
}
|
|
210
|
+
const parsed = parseExecutionResumeOutput(stdout);
|
|
211
|
+
if (verbose) {
|
|
212
|
+
console.log(`[VERBOSE] isolation-runner: $ --resume ${identifier} → mode=${parsed.mode || '(unknown)'} session=${parsed.sessionName || '(unknown)'} uuid=${parsed.uuid || '(unknown)'}`);
|
|
213
|
+
}
|
|
214
|
+
return { ...base, ...parsed, success: true, output: stdout };
|
|
215
|
+
} catch (error) {
|
|
216
|
+
const { stdout, message, unsupported } = interpretStartCommandResult(null, error);
|
|
217
|
+
if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume ${identifier} failed${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
|
|
218
|
+
return { ...base, unsupported, output: stdout, error: message };
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Reconcile every execution still marked running via `$ --resume-all`.
|
|
224
|
+
*
|
|
225
|
+
* Run at bot startup: the detached-docker completion watchers are children of
|
|
226
|
+
* the process that launched them, so a bot restart leaves every running
|
|
227
|
+
* container unsupervised — its exit would never be written to the log footer,
|
|
228
|
+
* which is one of the ways issue #2189's session stayed in limbo. `--resume-all`
|
|
229
|
+
* re-attaches a watcher to what is alive and finalizes what died meanwhile. It
|
|
230
|
+
* starts no work on its own.
|
|
231
|
+
*
|
|
232
|
+
* @param {Object} [options]
|
|
233
|
+
* @param {boolean} [options.verbose]
|
|
234
|
+
* @returns {Promise<{success: boolean, unsupported: boolean, executions: Array<Object>, output: string, error: string|null}>}
|
|
235
|
+
*/
|
|
236
|
+
export async function resumeAllIsolationSessions({ verbose = false } = {}) {
|
|
237
|
+
const base = { success: false, unsupported: false, executions: [], output: '', error: null };
|
|
238
|
+
const binPath = await findStartCommandBinary();
|
|
239
|
+
if (!binPath) {
|
|
240
|
+
if (verbose) console.log('[VERBOSE] isolation-runner: cannot run $ --resume-all - $ binary not found');
|
|
241
|
+
return { ...base, unsupported: true, error: START_COMMAND_MISSING_ERROR };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
try {
|
|
245
|
+
const $ = await getCommandStreamDollar();
|
|
246
|
+
const raw = await $({ mirror: false })`${binPath} --resume-all --output-format json`;
|
|
247
|
+
const { ok, stdout, message, unsupported } = interpretStartCommandResult(raw);
|
|
248
|
+
if (!ok) {
|
|
249
|
+
if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume-all refused${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
|
|
250
|
+
return { ...base, unsupported, output: stdout, error: message };
|
|
251
|
+
}
|
|
252
|
+
const executions = parseExecutionResumeAllOutput(stdout);
|
|
253
|
+
if (verbose) {
|
|
254
|
+
const summary = executions.map(entry => `${entry.action}:${entry.sessionName || entry.uuid}`).join(', ') || '(none)';
|
|
255
|
+
console.log(`[VERBOSE] isolation-runner: $ --resume-all reconciled ${executions.length} execution(s): ${summary}`);
|
|
256
|
+
}
|
|
257
|
+
return { ...base, success: true, executions, output: stdout };
|
|
258
|
+
} catch (error) {
|
|
259
|
+
const { stdout, message, unsupported } = interpretStartCommandResult(null, error);
|
|
260
|
+
if (verbose) console.log(`[VERBOSE] isolation-runner: $ --resume-all failed${unsupported ? ' (verb not supported by this $ build)' : ''}: ${message}`);
|
|
261
|
+
return { ...base, unsupported, output: stdout, error: message };
|
|
262
|
+
}
|
|
263
|
+
}
|
|
@@ -106,7 +106,7 @@ const measureDiff = diff => {
|
|
|
106
106
|
section = null;
|
|
107
107
|
};
|
|
108
108
|
|
|
109
|
-
for (let start = 0; start < diff.length;
|
|
109
|
+
for (let start = 0; start < diff.length;) {
|
|
110
110
|
let end = diff.indexOf('\n', start);
|
|
111
111
|
if (end === -1) end = diff.length;
|
|
112
112
|
const line = diff.slice(start, end);
|
|
@@ -54,6 +54,16 @@ export const KILL_DIAGNOSTICS_LOG_BYTES = 1024 * 1024;
|
|
|
54
54
|
*/
|
|
55
55
|
export const HEAP_EXHAUSTED_PERCENT = 90;
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Prefix start-command 0.33.0 puts on every `exitReason` that reports memory
|
|
59
|
+
* exhaustion, whatever the mechanism — `memory-exhaustion (v8-heap-limit)`,
|
|
60
|
+
* `memory-exhaustion (kernel-oom-killer)`, `memory-exhaustion (go-runtime)`,
|
|
61
|
+
* `memory-exhaustion (allocation-failure)`. Matching the prefix rather than the
|
|
62
|
+
* exact strings means a new upstream mechanism is classified correctly without
|
|
63
|
+
* a Hive Mind release. See link-foundation/start#164 and #165.
|
|
64
|
+
*/
|
|
65
|
+
export const UPSTREAM_MEMORY_EXHAUSTION_PREFIX = 'memory-exhaustion';
|
|
66
|
+
|
|
57
67
|
/** Disk is considered full at or above this used percentage… */
|
|
58
68
|
export const DISK_FULL_USED_PERCENT = 95;
|
|
59
69
|
/** …or below this much free space, whichever triggers first. */
|
|
@@ -268,9 +278,12 @@ function describeDisk(disk, timestamp) {
|
|
|
268
278
|
* @param {number|null} [params.exitCode]
|
|
269
279
|
* @param {Object|null} [params.system] - collectSystemKillDiagnostics() result
|
|
270
280
|
* @param {boolean} [params.stopRequestedByUser] - The operator asked for the stop
|
|
281
|
+
* @param {boolean|null} [params.reportedMemoryExhausted] - `$ --status` `memoryExhausted` (start-command >= 0.33.0)
|
|
282
|
+
* @param {string|null} [params.reportedMemoryExhaustedReason] - `$ --status` `memoryExhaustedReason` (the evidence line)
|
|
283
|
+
* @param {string|null} [params.reportedExitReason] - `$ --status` `exitReason` hint, e.g. `memory-exhaustion (v8-heap-limit)`
|
|
271
284
|
* @returns {{cause: string, summary: string, evidence: string[], memory: Object|null, disk: Object|null, victims: Array}}
|
|
272
285
|
*/
|
|
273
|
-
export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false } = {}) {
|
|
286
|
+
export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false, reportedMemoryExhausted = null, reportedMemoryExhaustedReason = null, reportedExitReason = null } = {}) {
|
|
274
287
|
const parsed = resourceMarkers || (logText ? parseResourceMarkers(logText) : { markers: [], byPhase: {} });
|
|
275
288
|
const memoryMarker = selectLastMemoryResourceMarker(parsed);
|
|
276
289
|
const heapMarker = selectLastHeapResourceMarker(parsed);
|
|
@@ -310,6 +323,30 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
|
|
|
310
323
|
evidence.push(`the ${fatalMemoryMarker.runtime} runtime aborted on its own heap limit: \`${fatalMemoryMarker.line}\` (a self-abort is invisible to \`docker inspect\` and to cgroup OOM counters)`);
|
|
311
324
|
}
|
|
312
325
|
|
|
326
|
+
// Issue #2189 also went upstream: start-command 0.33.0 (link-foundation/start
|
|
327
|
+
// #164, #165) performs the same tail scan inside `$` and reports it as
|
|
328
|
+
// `memoryExhausted` / `memoryExhaustedReason` / `exitReason`. Consuming it
|
|
329
|
+
// catches the case our own scan cannot: the fatal line scrolled out of the
|
|
330
|
+
// bounded window we read, but `$` saw it when the command exited. The local
|
|
331
|
+
// scan stays as defense in depth — these fields are absent on an older `$`.
|
|
332
|
+
const reportedExitReasonText = typeof reportedExitReason === 'string' && reportedExitReason.trim() ? reportedExitReason.trim() : null;
|
|
333
|
+
const reportedMemoryExhaustion = abnormalExit && (reportedMemoryExhausted === true || (reportedExitReasonText !== null && reportedExitReasonText.startsWith(UPSTREAM_MEMORY_EXHAUSTION_PREFIX)));
|
|
334
|
+
// `memory-exhaustion (v8-heap-limit)` → `v8-heap-limit`: the prefix is already
|
|
335
|
+
// said in words by the surrounding sentence, so only the mechanism is new.
|
|
336
|
+
const reportedMechanism =
|
|
337
|
+
reportedMemoryExhaustion && reportedExitReasonText
|
|
338
|
+
? reportedExitReasonText
|
|
339
|
+
.slice(UPSTREAM_MEMORY_EXHAUSTION_PREFIX.length)
|
|
340
|
+
.trim()
|
|
341
|
+
.replace(/^\((.*)\)$/, '$1') || null
|
|
342
|
+
: null;
|
|
343
|
+
if (reportedMemoryExhaustion) {
|
|
344
|
+
const detail = reportedMemoryExhaustedReason ? `: \`${reportedMemoryExhaustedReason}\`` : '';
|
|
345
|
+
evidence.push(`\`$ --status\` reports memory exhaustion${reportedMechanism ? ` (\`${reportedMechanism}\`)` : ''}${detail}`);
|
|
346
|
+
} else if (reportedExitReasonText && abnormalExit) {
|
|
347
|
+
evidence.push(`\`$ --status\` reports \`exitReason = ${reportedExitReasonText}\``);
|
|
348
|
+
}
|
|
349
|
+
|
|
313
350
|
const ratio = memoryRatio(memory);
|
|
314
351
|
const memoryExhausted = ratio !== null && ratio <= MEMORY_EXHAUSTED_AVAILABLE_RATIO;
|
|
315
352
|
// A heap already at the limit is only evidence of a kill when the session
|
|
@@ -322,7 +359,7 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
|
|
|
322
359
|
let cause = KILL_CAUSE_UNKNOWN;
|
|
323
360
|
if (stopRequestedByUser) {
|
|
324
361
|
cause = KILL_CAUSE_FORCED_KILL;
|
|
325
|
-
} else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted || fatalMemoryMarker || heapExhausted) {
|
|
362
|
+
} else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted || fatalMemoryMarker || heapExhausted || reportedMemoryExhaustion) {
|
|
326
363
|
cause = KILL_CAUSE_OUT_OF_MEMORY;
|
|
327
364
|
} else if (diskFull) {
|
|
328
365
|
cause = KILL_CAUSE_DISK_FULL;
|
|
@@ -340,6 +377,11 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
|
|
|
340
377
|
// Same shape as the fatal-marker case, but reconstructed from telemetry when
|
|
341
378
|
// the fatal line itself was lost (truncated tail, killed before flushing).
|
|
342
379
|
summary = `out of memory — the runtime's own heap was exhausted: ${heapLine}${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
|
|
380
|
+
} else if (cause === KILL_CAUSE_OUT_OF_MEMORY && reportedMemoryExhaustion && victims.length === 0 && !oomKilled && !(cgroupOomKills > 0) && !memoryExhausted) {
|
|
381
|
+
// Only `$` saw the evidence (our bounded window missed the fatal line).
|
|
382
|
+
// Quote what it saw rather than falling back to the host-memory phrasing,
|
|
383
|
+
// which would again read as a contradiction on a healthy machine.
|
|
384
|
+
summary = `out of memory — start-command reported memory exhaustion${reportedMechanism ? ` (${reportedMechanism})` : ''}${reportedMemoryExhaustedReason ? `: \`${reportedMemoryExhaustedReason}\`` : ''}${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
|
|
343
385
|
} else if (cause === KILL_CAUSE_OUT_OF_MEMORY) {
|
|
344
386
|
const victim = victims.length > 0 ? `, kernel OOM killer terminated \`${victims[victims.length - 1].comm || 'unknown'}\` (pid ${victims[victims.length - 1].pid ?? '?'})` : '';
|
|
345
387
|
summary = `out of memory${memoryLine ? ` — ${memoryLine}` : ''}${victim}`;
|
|
@@ -352,7 +394,7 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
|
|
|
352
394
|
summary = 'unknown — no resource marker, cgroup counter or kernel OOM report was available';
|
|
353
395
|
}
|
|
354
396
|
|
|
355
|
-
return { cause, summary, evidence, memory, heap: heapMemory, heapUsedPercent, disk, victims, fatalMemoryMarker };
|
|
397
|
+
return { cause, summary, evidence, memory, heap: heapMemory, heapUsedPercent, disk, victims, fatalMemoryMarker, reportedMemoryExhaustion, reportedExitReason: reportedExitReasonText };
|
|
356
398
|
}
|
|
357
399
|
|
|
358
400
|
/**
|
|
@@ -429,7 +471,7 @@ export function formatKillResumeSection({ sessionId = null, attempt = null, maxA
|
|
|
429
471
|
* @param {Object} [options]
|
|
430
472
|
* @returns {Promise<{section: string, diagnosis: Object|null}>}
|
|
431
473
|
*/
|
|
432
|
-
export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, maxLogBytes = KILL_DIAGNOSTICS_LOG_BYTES, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics } = {}) {
|
|
474
|
+
export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, maxLogBytes = KILL_DIAGNOSTICS_LOG_BYTES, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics, reportedMemoryExhausted = null, reportedMemoryExhaustedReason = null, reportedExitReason = null } = {}) {
|
|
433
475
|
try {
|
|
434
476
|
let logText = '';
|
|
435
477
|
if (logPath) {
|
|
@@ -441,7 +483,7 @@ export async function buildKillDiagnosticsSection(logPath, { verbose = false, re
|
|
|
441
483
|
logText = await readLogTextBounded(logPath, { readFile, maxBytes: maxLogBytes, verbose });
|
|
442
484
|
}
|
|
443
485
|
const system = await collectSystem({ verbose });
|
|
444
|
-
const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser });
|
|
486
|
+
const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser, reportedMemoryExhausted, reportedMemoryExhaustedReason, reportedExitReason });
|
|
445
487
|
if (verbose) console.log(`[VERBOSE] kill-diagnostics: cause=${diagnosis.cause} — ${diagnosis.summary}`);
|
|
446
488
|
return { section: formatKillDiagnosticsSection(diagnosis, { locale }), diagnosis };
|
|
447
489
|
} catch (error) {
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Re-enter the *same* container when recovering a killed session (issue #2189).
|
|
3
|
+
*
|
|
4
|
+
* The incident behind #2189 ended with a session that was killed 10 minutes
|
|
5
|
+
* after the AI tool had already finished its work. Recovery, when it finally
|
|
6
|
+
* happened, threw that container away: a fresh isolated run re-cloned the
|
|
7
|
+
* repository, re-installed everything and re-did work that was sitting on disk.
|
|
8
|
+
* The issue asks for the opposite — "ideally re-entering the same `$` session
|
|
9
|
+
* id / container".
|
|
10
|
+
*
|
|
11
|
+
* `start-command@0.33.0` (upstream link-foundation/start#162, filed from this
|
|
12
|
+
* very issue) makes that possible: `$ --resume <id> -- <command>` commits the
|
|
13
|
+
* stopped container's filesystem and runs the recovery command in a container
|
|
14
|
+
* derived from that snapshot, keeping the original execution UUID and log.
|
|
15
|
+
*
|
|
16
|
+
* Not every session may take that path, and the two exceptions are deliberate:
|
|
17
|
+
*
|
|
18
|
+
* - **Formal AI tasks** (issue #2146) reach their sidecar over an *internal*
|
|
19
|
+
* Docker network that Hive Mind attaches with `docker network connect` after
|
|
20
|
+
* the container is created. `$` knows nothing about that network, so a
|
|
21
|
+
* resumed container would come up without it and the task would silently talk
|
|
22
|
+
* to nothing. #2146 requires Formal AI to fail closed, so these fall back to
|
|
23
|
+
* the normal launch path, which re-acquires the sidecar lease properly.
|
|
24
|
+
* - **`--use-router` tasks** are attached to the router network the same way,
|
|
25
|
+
* with a freshly minted token, and have the same problem.
|
|
26
|
+
*
|
|
27
|
+
* Everything else — the overwhelming majority, and every session in the
|
|
28
|
+
* original incident — resumes in place.
|
|
29
|
+
*
|
|
30
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2189
|
|
31
|
+
* @see https://github.com/link-foundation/start/issues/162
|
|
32
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2146
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { isFormalAiTask } from './formal-ai-sidecar.lib.mjs';
|
|
36
|
+
import { hasUseRouterFlag } from './router-isolation.lib.mjs';
|
|
37
|
+
import { RESUME_MODES } from './isolation-runner.resume.lib.mjs';
|
|
38
|
+
|
|
39
|
+
/** Why a killed session cannot be re-entered in place. Reported, never thrown. */
|
|
40
|
+
export const IN_PLACE_SKIP_REASONS = Object.freeze({
|
|
41
|
+
NOT_DOCKER: 'not-docker',
|
|
42
|
+
NO_IDENTIFIER: 'no-identifier',
|
|
43
|
+
FORMAL_AI_TASK: 'formal-ai-task',
|
|
44
|
+
ROUTER_TASK: 'router-task',
|
|
45
|
+
NO_RESUME_SUPPORT: 'no-resume-support',
|
|
46
|
+
CONTAINER_GONE: 'container-gone',
|
|
47
|
+
UNSUPPORTED: 'resume-unsupported',
|
|
48
|
+
REFUSED: 'resume-refused',
|
|
49
|
+
ERROR: 'resume-error',
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Decide — purely, from persisted facts — whether a killed session is a
|
|
54
|
+
* candidate for a same-container resume.
|
|
55
|
+
*
|
|
56
|
+
* Kept separate from the Docker probe below so the policy is testable without a
|
|
57
|
+
* daemon, and so a caller can report precisely *why* a session was relaunched
|
|
58
|
+
* from scratch instead of resumed.
|
|
59
|
+
*
|
|
60
|
+
* @param {Object} options
|
|
61
|
+
* @param {string} options.sessionName - The killed session's name (= container name)
|
|
62
|
+
* @param {Object} options.sessionInfo - Persisted session info
|
|
63
|
+
* @returns {{eligible: boolean, reason: string, identifier: string|null, containerName: string|null}}
|
|
64
|
+
*/
|
|
65
|
+
export function planSameContainerResume({ sessionName = null, sessionInfo = {} } = {}) {
|
|
66
|
+
const containerName = sessionInfo?.sessionId || sessionName || null;
|
|
67
|
+
const identifier = sessionInfo?.executionUuid || containerName || null;
|
|
68
|
+
const base = { eligible: false, identifier, containerName };
|
|
69
|
+
|
|
70
|
+
if (sessionInfo?.isolationBackend !== 'docker') {
|
|
71
|
+
// screen/tmux sessions have no filesystem to preserve: their work happens
|
|
72
|
+
// on the host, which a fresh run already sees.
|
|
73
|
+
return { ...base, reason: IN_PLACE_SKIP_REASONS.NOT_DOCKER };
|
|
74
|
+
}
|
|
75
|
+
if (!identifier) return { ...base, reason: IN_PLACE_SKIP_REASONS.NO_IDENTIFIER };
|
|
76
|
+
|
|
77
|
+
const args = Array.isArray(sessionInfo?.args) ? sessionInfo.args : [];
|
|
78
|
+
if (isFormalAiTask({ args, model: sessionInfo?.model || null })) {
|
|
79
|
+
return { ...base, reason: IN_PLACE_SKIP_REASONS.FORMAL_AI_TASK };
|
|
80
|
+
}
|
|
81
|
+
if (hasUseRouterFlag(args)) return { ...base, reason: IN_PLACE_SKIP_REASONS.ROUTER_TASK };
|
|
82
|
+
|
|
83
|
+
return { ...base, eligible: true, reason: 'ready' };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Attempt the same-container resume. Never throws, and never leaves work
|
|
88
|
+
* running that it does not report: the caller may only fall back to a fresh
|
|
89
|
+
* launch when `resumed` is false.
|
|
90
|
+
*
|
|
91
|
+
* @param {Object} options
|
|
92
|
+
* @param {string} options.sessionName - The killed session's name
|
|
93
|
+
* @param {Object} options.sessionInfo - Persisted session info
|
|
94
|
+
* @param {Object} options.plan - Result of planKillRecovery() (needs `command.display`)
|
|
95
|
+
* @param {Object} options.runner - Isolation runner module
|
|
96
|
+
* @param {boolean} [options.verbose]
|
|
97
|
+
* @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, executionUuid: string|null, mode: string|null, snapshotImage: string|null}>}
|
|
98
|
+
*/
|
|
99
|
+
export async function resumeKilledSessionInPlace({ sessionName, sessionInfo, plan, runner, verbose = false } = {}) {
|
|
100
|
+
const decision = planSameContainerResume({ sessionName, sessionInfo });
|
|
101
|
+
const miss = reason => ({ resumed: false, reason, sessionId: null, executionUuid: decision.identifier, mode: null, snapshotImage: null });
|
|
102
|
+
if (!decision.eligible) return miss(decision.reason);
|
|
103
|
+
if (typeof runner?.resumeIsolatedSession !== 'function' || typeof runner?.checkDockerContainerExists !== 'function') {
|
|
104
|
+
return miss(IN_PLACE_SKIP_REASONS.NO_RESUME_SUPPORT);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// A container that no longer exists has nothing left to re-enter; `$` would
|
|
108
|
+
// fall back to a full relaunch, which is what the caller does anyway — but
|
|
109
|
+
// through the path that also re-acquires leases.
|
|
110
|
+
const exists = await runner.checkDockerContainerExists(decision.containerName, verbose);
|
|
111
|
+
if (!exists) return miss(IN_PLACE_SKIP_REASONS.CONTAINER_GONE);
|
|
112
|
+
|
|
113
|
+
const result = await runner.resumeIsolatedSession(decision.identifier, { command: plan?.command?.display || null, verbose });
|
|
114
|
+
if (!result?.success) {
|
|
115
|
+
const reason = result?.unsupported ? IN_PLACE_SKIP_REASONS.UNSUPPORTED : IN_PLACE_SKIP_REASONS.REFUSED;
|
|
116
|
+
if (verbose) console.log(`[VERBOSE] In-place resume of ${sessionName} was not possible (${reason}): ${result?.error || 'no reason given'}`);
|
|
117
|
+
return miss(reason);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// `docker-snapshot` names the new container `<session>-resume-<attempt>`; the
|
|
121
|
+
// old name stays addressable through upstream's `sessionNameHistory`, but the
|
|
122
|
+
// *new* one is what `$ --status` reports on now, so that is what the monitor
|
|
123
|
+
// has to track. A resume that somehow reports the old name (a `docker-start`
|
|
124
|
+
// race, say) is tracked under the execution UUID instead, which upstream
|
|
125
|
+
// resolves just as well and cannot collide with the dying session's entry.
|
|
126
|
+
const returnedName = result.sessionName && result.sessionName !== sessionName ? result.sessionName : null;
|
|
127
|
+
const sessionId = returnedName || decision.identifier;
|
|
128
|
+
return {
|
|
129
|
+
resumed: true,
|
|
130
|
+
reason: result.mode === RESUME_MODES.DOCKER_SNAPSHOT ? 'resumed-in-place' : `resumed-${result.mode || 'unknown'}`,
|
|
131
|
+
sessionId,
|
|
132
|
+
executionUuid: result.uuid || decision.identifier,
|
|
133
|
+
mode: result.mode || null,
|
|
134
|
+
snapshotImage: result.snapshotImage || null,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -24,6 +24,7 @@ import { readLastSessionIdFromLog, planKilledSessionResume } from './session-res
|
|
|
24
24
|
import { resolveOnSessionKillPolicy, resolveSessionKillResumeAttempts, shouldResumeKilledSession, ON_SESSION_KILL_RESUME } from './session-kill-policy.lib.mjs';
|
|
25
25
|
import { argvFromSessionArgs } from './session-monitor.kill-sections.lib.mjs';
|
|
26
26
|
import { formatKillResumeSection } from './session-kill-diagnostics.lib.mjs';
|
|
27
|
+
import { resumeKilledSessionInPlace } from './session-kill-resume.in-place.lib.mjs';
|
|
27
28
|
|
|
28
29
|
/** Field recording how many automatic recovery sessions this session produced. */
|
|
29
30
|
export const KILL_RESUME_ATTEMPTS_FIELD = 'killRecoveryAttempts';
|
|
@@ -65,9 +66,18 @@ export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = fa
|
|
|
65
66
|
/**
|
|
66
67
|
* Start the recovery working session decided by {@link planKillRecovery}.
|
|
67
68
|
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
69
|
+
* Two ways in, in order of preference:
|
|
70
|
+
*
|
|
71
|
+
* 1. **Same container** (issue #2189) — `$ --resume` re-enters the killed
|
|
72
|
+
* session's own filesystem, so the clone, the caches and the half-finished
|
|
73
|
+
* branch survive. See `./session-kill-resume.in-place.lib.mjs` for the
|
|
74
|
+
* cases that are deliberately excluded.
|
|
75
|
+
* 2. **A fresh isolated run** — the original behaviour, used whenever (1) is
|
|
76
|
+
* not available or refuses. Correct, just more expensive.
|
|
77
|
+
*
|
|
78
|
+
* Either way the new session is tracked like any other, so it reports its own
|
|
79
|
+
* completion (and, if it is killed too, its own diagnosis) through the normal
|
|
80
|
+
* path.
|
|
71
81
|
*
|
|
72
82
|
* @param {Object} options
|
|
73
83
|
* @param {string} options.sessionName - The killed session's name
|
|
@@ -77,10 +87,10 @@ export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = fa
|
|
|
77
87
|
* @param {Function} options.trackSession - Tracker for the new session
|
|
78
88
|
* @param {Function} [options.persistSnapshot] - Persist the attempt counter
|
|
79
89
|
* @param {boolean} [options.verbose]
|
|
80
|
-
* @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null}>}
|
|
90
|
+
* @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null, inPlace: boolean}>}
|
|
81
91
|
*/
|
|
82
92
|
export async function startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot = null, verbose = false } = {}) {
|
|
83
|
-
const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null });
|
|
93
|
+
const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null, inPlace: false });
|
|
84
94
|
if (!plan?.shouldResume || !plan.command) return fail(plan?.reason || 'no-plan');
|
|
85
95
|
if (!runner || typeof runner.executeWithIsolation !== 'function' || typeof runner.generateSessionId !== 'function') return fail('no-isolation-runner');
|
|
86
96
|
if (typeof trackSession !== 'function') return fail('no-tracker');
|
|
@@ -88,10 +98,20 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
|
|
|
88
98
|
if (!backend) return fail('no-isolation-backend');
|
|
89
99
|
|
|
90
100
|
try {
|
|
91
|
-
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
101
|
+
// Preferred path: re-enter the container the work already happened in.
|
|
102
|
+
const inPlace = await resumeKilledSessionInPlace({ sessionName, sessionInfo, plan, runner, verbose });
|
|
103
|
+
let newSessionId = inPlace.sessionId;
|
|
104
|
+
let executionUuid = inPlace.executionUuid;
|
|
105
|
+
let containerFilesystemStartBytes = null;
|
|
106
|
+
|
|
107
|
+
if (!inPlace.resumed) {
|
|
108
|
+
newSessionId = runner.generateSessionId();
|
|
109
|
+
const tool = sessionInfo?.tool || 'claude';
|
|
110
|
+
const result = await runner.executeWithIsolation(sessionInfo?.command || 'solve', plan.command.args, { backend, sessionId: newSessionId, tool, verbose });
|
|
111
|
+
if (!result?.success) return fail('start-failed');
|
|
112
|
+
executionUuid = result.executionUuid || null;
|
|
113
|
+
containerFilesystemStartBytes = Number.isFinite(result.containerFilesystemStartBytes) ? result.containerFilesystemStartBytes : null;
|
|
114
|
+
}
|
|
95
115
|
|
|
96
116
|
trackSession(
|
|
97
117
|
newSessionId,
|
|
@@ -105,9 +125,15 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
|
|
|
105
125
|
[KILL_RESUME_ATTEMPTS_FIELD]: plan.attempt,
|
|
106
126
|
killRecoveryResumed: true,
|
|
107
127
|
killRecoveryOfSession: sessionName,
|
|
128
|
+
// A resumed execution keeps its UUID; a fresh launch gets a new one, and
|
|
129
|
+
// inheriting the dead session's would make `$ --status` answer about the
|
|
130
|
+
// wrong execution until the monitor happened to correct it.
|
|
131
|
+
executionUuid,
|
|
132
|
+
killRecoveryInPlace: inPlace.resumed,
|
|
133
|
+
killRecoveryResumeMode: inPlace.mode || null,
|
|
108
134
|
oomEventObservedAt: undefined,
|
|
109
135
|
dockerBackendGoneFirstSeenAt: undefined,
|
|
110
|
-
containerFilesystemStartBytes
|
|
136
|
+
containerFilesystemStartBytes,
|
|
111
137
|
},
|
|
112
138
|
verbose
|
|
113
139
|
);
|
|
@@ -122,9 +148,10 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
|
|
|
122
148
|
}
|
|
123
149
|
|
|
124
150
|
if (verbose) {
|
|
125
|
-
|
|
151
|
+
const how = inPlace.resumed ? `resumed in place (${inPlace.mode || 'unknown mode'})` : `started fresh (in-place resume skipped: ${inPlace.reason})`;
|
|
152
|
+
console.log(`[VERBOSE] Session ${sessionName} was killed; recovery session ${newSessionId} ${how} (attempt ${plan.attempt}/${plan.maxAttempts}): ${plan.command.display}`);
|
|
126
153
|
}
|
|
127
|
-
return { resumed: true, reason: 'started', sessionId: newSessionId, display: plan.command.display };
|
|
154
|
+
return { resumed: true, reason: inPlace.resumed ? inPlace.reason : 'started', sessionId: newSessionId, display: plan.command.display, inPlace: inPlace.resumed };
|
|
128
155
|
} catch (error) {
|
|
129
156
|
if (verbose) {
|
|
130
157
|
console.log(`[VERBOSE] Could not start recovery session for ${sessionName}: ${error?.message || error}`);
|
|
@@ -138,7 +165,7 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
|
|
|
138
165
|
* Never throws — a failed recovery must still leave a correct kill report.
|
|
139
166
|
*
|
|
140
167
|
* @param {Object} options - See planKillRecovery() and startKillRecoverySession()
|
|
141
|
-
* @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number}>}
|
|
168
|
+
* @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number, inPlace: boolean}>}
|
|
142
169
|
*/
|
|
143
170
|
export async function recoverKilledSession({ sessionName, sessionInfo, logPath = null, killed = false, env = process.env, runner = null, trackSession = null, persistSnapshot = null, verbose = false, readLastSessionId = readLastSessionIdFromLog } = {}) {
|
|
144
171
|
let plan;
|
|
@@ -146,15 +173,15 @@ export async function recoverKilledSession({ sessionName, sessionInfo, logPath =
|
|
|
146
173
|
plan = planKillRecovery({ sessionInfo, logPath, killed, env, verbose, readLastSessionId });
|
|
147
174
|
} catch (error) {
|
|
148
175
|
if (verbose) console.log(`[VERBOSE] Could not plan kill recovery for ${sessionName}: ${error?.message || error}`);
|
|
149
|
-
return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0 };
|
|
176
|
+
return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0, inPlace: false };
|
|
150
177
|
}
|
|
151
178
|
|
|
152
179
|
if (!plan.shouldResume) {
|
|
153
|
-
return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
|
|
180
|
+
return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts, inPlace: false };
|
|
154
181
|
}
|
|
155
182
|
|
|
156
183
|
const started = await startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot, verbose });
|
|
157
|
-
return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
|
|
184
|
+
return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts, inPlace: started.inPlace === true };
|
|
158
185
|
}
|
|
159
186
|
|
|
160
187
|
/**
|