@link-assistant/hive-mind 2.11.7 → 2.11.8

package/src/session-monitor.lib.mjs

@@ -31,6 +31,9 @@ import { resolveFailedSessionPullRequestState } from './github-pr-state.lib.mjs'
 // may be an exit code start-command fabricated from the command's own output.
 import { clearUnverifiedDockerTerminalMarker as clearUnverifiedDockerTerminalMarkerImpl, shouldDeferUnverifiedDockerTerminal as shouldDeferUnverifiedDockerTerminalImpl } from './session-monitor.docker-terminal.lib.mjs';
 import { isDockerIsolation, sessionStartMs, resolveOomKilledState, resolveStaleExecutingState as resolveStaleExecutingStateImpl } from './session-monitor.stale-executing.lib.mjs';
+// Issue #2134: kill-cause diagnostics + the matching pull-request notice.
+import { buildKillCompletionSections, announceKillOnPullRequest } from './session-monitor.kill-sections.lib.mjs';
+import { runKillRecoveryForCompletion } from './session-kill-resume.lib.mjs';
 
 export { formatSessionCompletionMessage, getSessionCompletionExitCode } from './work-session-formatting.lib.mjs';
 export { DOCKER_TERMINAL_FOOTER_GRACE_MS } from './session-monitor.docker-terminal.lib.mjs';
@@ -550,7 +553,16 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
 
     if (statusResult?.exists && statusResult.status) {
       if (statusResult.oomKilled === true) {
-        return resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog });
+        // Issue #2134: `oomKilled` is a *container* flag — the kernel sets it when
+        // any process in the cgroup is OOM-killed — so it is verified against the
+        // log footer and container liveness before a kill is announced.
+        return await resolveOomKilledState(sessionName, sessionInfo, statusResult, {
+          verbose,
+          runner,
+          exitFromLog,
+          backendAlive,
+          persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
+        });
       }
       if (runner.isExecutingSessionStatus(statusResult.status)) {
         // Issue #1927: an `executing` status is not trusted blindly — verify the
@@ -611,6 +623,27 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
         if (unverifiedDockerFailure && shouldDeferUnverifiedDockerTerminal(sessionName, sessionInfo, { exitCode, endTime: statusResult.endTime || null, verbose })) {
           return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
         }
+        // Issue #2134: even after the grace window, a container that is verifiably
+        // still alive cannot have produced a terminal failure — the same liveness
+        // ladder used for `oomKilled` applies here, so no kill is announced while
+        // the working session keeps running (that is exactly what #2134 reported).
+        if (unverifiedDockerFailure) {
+          const probe = backendAlive || runner.checkBackendSessionAlive;
+          let alive = null;
+          if (probe && sessionInfo?.isolationBackend) {
+            try {
+              alive = await probe(sessionId, sessionInfo.isolationBackend, verbose);
+            } catch {
+              alive = null;
+            }
+          }
+          if (alive === true) {
+            if (verbose) {
+              console.log(`[VERBOSE] Session ${sessionName} reported terminal '${statusResult.status}' with exit ${exitCode}, but its docker backend is still alive; keeping the session tracked (issue #2134)`);
+            }
+            return { running: true, exitCode: null, status: statusResult.status, statusResult, deferred: true };
+          }
+        }
         if (!ambiguousDockerTerminal) {
           clearUnverifiedDockerTerminalMarker(sessionName, sessionInfo);
           return { running: false, exitCode, status: statusResult.status, statusResult };
@@ -817,6 +850,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
         // Issue #1927: for a killed /solve, offer a command using the last tool
         // session ID in the log. Do not auto-relaunch work that may reliably OOM.
         const resumeExtraSections = [];
+        let killResumeCommand = null;
         try {
           const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: resolvedStatus });
           const isResumableCommand = (sessionInfo?.command || 'solve') === 'solve';
@@ -831,6 +865,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
             const lastSessionId = readLastSessionIdFromLog(logPath, { verbose });
             const resumeCommand = buildResumeCommand({ sessionInfo, lastSessionId });
             const resumeSection = formatResumeSection({ lastSessionId, command: resumeCommand });
+            killResumeCommand = resumeCommand || null;
             if (resumeSection) {
               resumeExtraSections.push(resumeSection);
               if (verbose) {
@@ -865,6 +900,42 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
         }
         const dockerTaskContainerExtraSections = dockerTaskContainerAction?.extraSection ? [dockerTaskContainerAction.extraSection] : [];
 
+        // Issue #2134: say exactly WHY a session was killed, and warn when a
+        // session merely survived a kill event instead of reporting a plain
+        // success. The pull request gets the very same report below.
+        const killReport = await buildKillCompletionSections({
+          sessionName,
+          sessionInfo,
+          statusResult,
+          exitCode: finalExitCode,
+          status: resolvedStatus,
+          verbose,
+          readFile: options.readFile,
+          env: options.env || process.env,
+        });
+
+        // Issue #2134: `--on-session-kill=resume` must actually start a new
+        // working session, and both surfaces must say so. Done before the
+        // message is built so the Telegram report and the pull-request notice
+        // below name the very same recovery session.
+        let killRecovery = { resumed: false, sessionId: null, attempt: 0, maxAttempts: 0 };
+        if (killReport.killed) {
+          const recovered = await runKillRecoveryForCompletion({
+            sessionName,
+            sessionInfo,
+            logPath: statusResult?.logPath || sessionInfo?.logPath || null,
+            killed: true,
+            env: options.env || process.env,
+            runner: options.isolationRunner || null,
+            trackSession: options.trackSession || trackSession,
+            persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
+            locale: sessionInfo?.locale || null,
+            verbose,
+          });
+          killRecovery = recovered.recovery;
+          if (recovered.section) killReport.sections.push(recovered.section);
+        }
+
         const message = formatSessionCompletionMessage({
           sessionName,
           sessionInfo,
@@ -874,9 +945,33 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
           infoBlock: sessionInfo?.infoBlock || '',
           pullRequestUrl,
           pullRequestState,
-          extraSections: [...limitsExtraSections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
+          extraSections: [...limitsExtraSections, ...killReport.sections, ...resumeExtraSections, ...diskExtraSections, ...dockerTaskContainerExtraSections],
         });
 
+        if (killReport.killed || killReport.recovered) {
+          const notice = await announceKillOnPullRequest({
+            pullRequestUrl,
+            sessionName,
+            sessionInfo,
+            diagnosis: killReport.diagnosis,
+            exitCode: finalExitCode,
+            observedAt: killReport.observedAt,
+            policy: killReport.policy,
+            recovered: killReport.recovered,
+            resumed: killRecovery.resumed,
+            recoverySessionId: killRecovery.resumed ? killRecovery.sessionId : null,
+            attempt: killRecovery.resumed ? killRecovery.attempt : null,
+            maxAttempts: killRecovery.resumed ? killRecovery.maxAttempts : null,
+            resumeCommand: killResumeCommand,
+            runCommand: options.runCommand || undefined,
+            attachLog: options.attachLog || undefined,
+            verbose,
+          });
+          if (verbose && !notice.posted) {
+            console.log(`[VERBOSE] Killed-session notice not posted for ${sessionName}: ${notice.skipped || 'unknown reason'}`);
+          }
+        }
+
         // Update the original reply message if messageId is available, otherwise send new message
         let notifyFromChatId = null;
         let notifyMessageId = null;

package/src/session-monitor.oom.lib.mjs

@@ -0,0 +1,148 @@
+/**
+ * Verified out-of-memory classification for tracked isolation sessions.
+ *
+ * Issue #2015 made `oomKilled: true` in a `$ --status` record terminal so an
+ * OOM-killed session could not be polled forever. Issue #2134 showed the other
+ * half of the problem: Docker's `State.OOMKilled` is a *container* flag, not a
+ * statement about the container's main process. The kernel sets it when ANY
+ * process in the container cgroup is OOM-killed, and it stays `true` afterwards
+ * — so a container can be flagged, keep running, and exit 0 (see
+ * https://github.com/moby/moby/issues/47618).
+ *
+ * That is exactly what happened in #2134: the host ran out of memory, the OOM
+ * killer terminated a child process, `$ --status` flipped to
+ * `executed / exitCode 137`, and Hive Mind announced
+ * "Work session killed — out of memory or forced kill (SIGKILL)" while
+ * `docker inspect` still reported the container **running**. The session went on
+ * for another 3.5 hours and auto-merged its pull request, unmonitored.
+ *
+ * This module keeps #2015's guarantee (a truly OOM-killed session is terminal)
+ * while refusing to declare a kill that is contradicted by stronger evidence,
+ * following the same ladder the rest of the monitor already uses:
+ *
+ *   1. The log FOOTER wins. A written `Exit Code: N` is proof of how the command
+ *      actually ended — including `Exit Code: 0`, which means the session merely
+ *      *survived* an OOM event.
+ *   2. LIVENESS beats the status record. No footer + the backing container is
+ *      still alive → the session is still running; keep polling and remember the
+ *      OOM event so the eventual completion can report the recovery.
+ *   3. Otherwise report `oom-killed`, exactly as issue #2015 requires.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2015
+ * @see https://github.com/link-assistant/hive-mind/issues/2134
+ */
+
+import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
+
+/**
+ * Field written on the persisted session snapshot the first time an OOM event is
+ * observed for a session that is still alive. It survives a bot restart, so the
+ * completion message can still say "recovered from out of memory" hours later.
+ */
+export const OOM_EVENT_OBSERVED_FIELD = 'oomEventObservedAt';
+
+/**
+ * Remember that the kernel OOM killer fired inside this session's container.
+ * Idempotent: only the FIRST observation timestamp is kept.
+ *
+ * @param {Object} sessionInfo - Mutable persisted session info
+ * @param {Function} [persistSnapshot] - Callback that mirrors sessionInfo to disk
+ * @returns {boolean} True when this call recorded a new observation
+ */
+export function markOomEventObserved(sessionInfo, persistSnapshot) {
+  if (!sessionInfo || sessionInfo[OOM_EVENT_OBSERVED_FIELD]) return false;
+  sessionInfo[OOM_EVENT_OBSERVED_FIELD] = new Date().toISOString();
+  if (typeof persistSnapshot === 'function') persistSnapshot();
+  return true;
+}
+
+/**
+ * Whether an OOM event was observed for this session while it was running.
+ *
+ * @param {Object} sessionInfo
+ * @returns {string|null} ISO timestamp of the first observation, or null
+ */
+export function getOomEventObservedAt(sessionInfo) {
+  return sessionInfo?.[OOM_EVENT_OBSERVED_FIELD] || null;
+}
+
+async function probeBackendAlive(sessionName, sessionInfo, { verbose, runner, backendAlive }) {
+  if (!sessionInfo?.isolationBackend) return null;
+  const probe = backendAlive || runner?.checkBackendSessionAlive;
+  if (!probe) return null;
+  try {
+    return await probe(sessionInfo.sessionId || sessionName, sessionInfo.isolationBackend, verbose);
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] Session ${sessionName} OOM liveness probe failed: ${error?.message || error}`);
+    }
+    return null;
+  }
+}
+
+/**
+ * Resolve the real state of a session whose status record carries
+ * `oomKilled: true`.
+ *
+ * @param {string} sessionName
+ * @param {Object} sessionInfo
+ * @param {Object} statusResult - Parsed `$ --status` payload
+ * @param {Object} deps
+ * @param {boolean} [deps.verbose]
+ * @param {Object} deps.runner - Isolation runner (for its default probes)
+ * @param {Function} [deps.exitFromLog] - Injectable footer reader
+ * @param {Function} [deps.backendAlive] - Injectable liveness probe
+ * @param {Function} [deps.persistSnapshot] - Persist the session snapshot
+ * @returns {Promise<Object>} Monitor state object
+ */
+export async function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog, backendAlive, persistSnapshot } = {}) {
+  const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
+  let footer = null;
+  if (logPath) {
+    const readFooter = exitFromLog || runner?.readSessionExitFromLog;
+    footer = readFooter ? readFooter(logPath, { verbose }) : null;
+  }
+
+  // 1. The authoritative log footer.
+  if (footer?.finished) {
+    const footerExitCode = normalizeExitCode(footer.exitCode);
+    const correctedStatus = classifyExitStatus(footerExitCode) || (footerExitCode === 0 ? 'executed' : 'failed');
+    const survivedOom = footerExitCode === 0;
+    if (survivedOom) markOomEventObserved(sessionInfo, persistSnapshot);
+    if (verbose) {
+      console.log(`[VERBOSE] Session ${sessionName} reported oomKilled=true, but its log footer says exit ${footerExitCode} (${correctedStatus}) and wins${survivedOom ? ' — the session SURVIVED the out-of-memory event (issue #2134)' : ''}`);
+    }
+    return {
+      running: false,
+      exitCode: footerExitCode,
+      status: correctedStatus,
+      statusResult: { ...statusResult, status: correctedStatus, exitCode: footerExitCode, endTime: statusResult?.endTime || footer.endTime || null },
+      oomEventObserved: true,
+    };
+  }
+
+  // 2. Liveness: an alive container cannot have had its command killed.
+  const alive = await probeBackendAlive(sessionName, sessionInfo, { verbose, runner, backendAlive });
+  if (alive === true) {
+    markOomEventObserved(sessionInfo, persistSnapshot);
+    if (verbose) {
+      console.log(`[VERBOSE] Session ${sessionName} reported oomKilled=true but its ${sessionInfo.isolationBackend} backend is still alive; an OOM event hit the container, not the command — keeping the session tracked (issue #2134)`);
+    }
+    return { running: true, exitCode: null, status: statusResult?.status || 'executing', statusResult, deferred: true, oomEventObserved: true };
+  }
+
+  // 3. Nothing contradicts the status record: this really is an OOM kill (#2015).
+  const statusExitCode = normalizeExitCode(statusResult?.exitCode);
+  let exitCode = 137;
+  if (statusExitCode !== null && statusExitCode > 0) {
+    exitCode = statusExitCode;
+  }
+  const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
+  const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
+
+  if (verbose) {
+    console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true (backend alive: ${alive === null ? 'unknown' : alive}); treating it as terminal oom-killed (exit ${exitCode})`);
+  }
+
+  return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true, oomEventObserved: true };
+}

package/src/session-monitor.stale-executing.lib.mjs

@@ -14,7 +14,7 @@
  * @see https://github.com/link-assistant/hive-mind/issues/2015
  */
 
-import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
+import { classifyExitStatus } from './session-status.lib.mjs';
 
 /**
  * Issue #1927: minimum age before a session that `$ --status` still reports as
@@ -122,30 +122,9 @@ export async function resolveStaleExecutingState(sessionName, sessionInfo, statu
 
 /**
  * Issue #2015: `oomKilled` is terminal — the container was killed by the kernel,
- * so no further polling can change the outcome.
+ * so no further polling can change the outcome. Issue #2134 refined that into a
+ * *verified* classification (footer/liveness ladder); the implementation now
+ * lives in session-monitor.oom.lib.mjs and is re-exported here so existing
+ * importers keep working.
  */
-export function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
-  const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
-  let footer = null;
-  if (logPath) {
-    const readFooter = exitFromLog || runner.readSessionExitFromLog;
-    footer = readFooter ? readFooter(logPath, { verbose }) : null;
-  }
-
-  const statusExitCode = normalizeExitCode(statusResult?.exitCode);
-  const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
-  let exitCode = 137;
-  if (statusExitCode !== null && statusExitCode > 0) {
-    exitCode = statusExitCode;
-  } else if (footerExitCode !== null && footerExitCode > 0) {
-    exitCode = footerExitCode;
-  }
-  const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
-  const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
-
-  if (verbose) {
-    console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
-  }
-
-  return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
-}
+export { resolveOomKilledState, markOomEventObserved, getOomEventObservedAt, OOM_EVENT_OBSERVED_FIELD } from './session-monitor.oom.lib.mjs';

package/src/session-resume.lib.mjs

@@ -43,6 +43,30 @@ const SESSION_LOG_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0
 const DEFAULT_LOG_SCAN_CHUNK_BYTES = 262144;
 const LOG_SCAN_OVERLAP_BYTES = 1024;
 
+// Issue #2134: the marker scan runs over a log that can contain *anything* the
+// run printed — including Hive Mind's own source code. `solve` prints its id
+// with `` console.log(`📌 Session ID: ${sessionId}`) ``, so a run that `cat`-ed
+// that source made the literal string `${sessionId}` the "last session id" and
+// the bot offered `--resume "${sessionId}"` to the user. Tool session ids are
+// opaque tokens (UUIDs for claude/codex, slugs elsewhere): a shape check plus a
+// small vocabulary of placeholder words rejects that whole class of false hit
+// without narrowing the ids that are legitimately accepted today.
+const SESSION_ID_SHAPE_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{6,199}$/;
+const SESSION_ID_PLACEHOLDERS = new Set(['unknown', 'n/a', 'na', 'none', 'null', 'undefined', 'sessionid', 'session_id', 'session-id', 'id']);
+
+/**
+ * Whether a token captured after a `Session ID:` label can be a real tool
+ * session id (issue #2134).
+ *
+ * @param {*} id - Captured token.
+ * @returns {boolean} True when the token has a plausible session-id shape.
+ */
+export function isPlausibleSessionId(id) {
+  if (!id || typeof id !== 'string') return false;
+  if (SESSION_ID_PLACEHOLDERS.has(id.toLowerCase())) return false;
+  return SESSION_ID_SHAPE_RE.test(id);
+}
+
 /**
  * Extract every tool session id printed to a log, in the order they appear.
  *
@@ -60,8 +84,10 @@ export function extractSessionIds(text) {
   SESSION_ID_MARKER_RE.lastIndex = 0;
   while ((match = SESSION_ID_MARKER_RE.exec(text)) !== null) {
     const id = match[1];
-    // Skip obvious non-ids that can follow the label in prose/log output.
-    if (!id || id.toLowerCase() === 'unknown' || id.toLowerCase() === 'n/a') continue;
+    // Skip obvious non-ids that can follow the label in prose/log output, and
+    // template placeholders such as `${sessionId}` printed by quoted source
+    // code (issue #2134).
+    if (!isPlausibleSessionId(id)) continue;
     if (ids[ids.length - 1] !== id) ids.push(id);
   }
   return ids;

package/src/solve.config.lib.mjs

@@ -154,6 +154,20 @@ export const SOLVE_OPTION_DEFINITIONS = {
     description: 'DANGEROUS: skip masking known active local tokens in output. This is separate from other sanitization skip flags and should only be used for controlled debugging.',
     default: false,
   },
+  // Issue #2134: one option decides what happens when a working session is
+  // killed (out of memory, disk exhaustion, forced kill). The Telegram bot and
+  // solve honour the exact same value, so the report never differs by surface.
+  'on-session-kill': {
+    type: 'string',
+    description: 'What to do when a working session is killed (out of memory, disk full, forced kill): "report" describes exactly what happened in the pull request and in Telegram, "resume" additionally starts a new working session to recover and says so in both places. Can also be set with HIVE_MIND_ON_SESSION_KILL.',
+    choices: ['report', 'resume'],
+    default: 'report',
+  },
+  'session-kill-resume-attempts': {
+    type: 'number',
+    description: 'Maximum number of automatic recovery working sessions started for one killed session when --on-session-kill=resume. Can also be set with HIVE_MIND_SESSION_KILL_RESUME_ATTEMPTS.',
+    default: 1,
+  },
   'auto-close-pull-request-on-fail': {
     type: 'boolean',
     description: 'Automatically close the pull request if execution fails',

package/src/telegram-bot.mjs

@@ -1248,7 +1248,9 @@ let launchAnnouncementShown = false;
 
 function startSessionMonitoringOnce() {
   if (sessionMonitoringTimer) return;
-  sessionMonitoringTimer = startSessionMonitoring(bot, VERBOSE);
+  // Issue #2134: the monitor needs the isolation runner to start a recovery
+  // working session when `--on-session-kill=resume` is in effect.
+  sessionMonitoringTimer = startSessionMonitoring(bot, VERBOSE, 30000, { isolationRunner });
 }
 
 // Issue #1927 (requirements #3/#4): a periodic timestamped heartbeat so the "last