@link-assistant/hive-mind 2.15.2 → 2.17.0

package/src/session-kill-resume.in-place.lib.mjs

@@ -0,0 +1,136 @@
+/**
+ * Re-enter the *same* container when recovering a killed session (issue #2189).
+ *
+ * The incident behind #2189 ended with a session that was killed 10 minutes
+ * after the AI tool had already finished its work. Recovery, when it finally
+ * happened, threw that container away: a fresh isolated run re-cloned the
+ * repository, re-installed everything and re-did work that was sitting on disk.
+ * The issue asks for the opposite — "ideally re-entering the same `$` session
+ * id / container".
+ *
+ * `start-command@0.33.0` (upstream link-foundation/start#162, filed from this
+ * very issue) makes that possible: `$ --resume <id> -- <command>` commits the
+ * stopped container's filesystem and runs the recovery command in a container
+ * derived from that snapshot, keeping the original execution UUID and log.
+ *
+ * Not every session may take that path, and the two exceptions are deliberate:
+ *
+ * - **Formal AI tasks** (issue #2146) reach their sidecar over an *internal*
+ *   Docker network that Hive Mind attaches with `docker network connect` after
+ *   the container is created. `$` knows nothing about that network, so a
+ *   resumed container would come up without it and the task would silently talk
+ *   to nothing. #2146 requires Formal AI to fail closed, so these fall back to
+ *   the normal launch path, which re-acquires the sidecar lease properly.
+ * - **`--use-router` tasks** are attached to the router network the same way,
+ *   with a freshly minted token, and have the same problem.
+ *
+ * Everything else — the overwhelming majority, and every session in the
+ * original incident — resumes in place.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
+ * @see https://github.com/link-foundation/start/issues/162
+ * @see https://github.com/link-assistant/hive-mind/issues/2146
+ */
+
+import { isFormalAiTask } from './formal-ai-sidecar.lib.mjs';
+import { hasUseRouterFlag } from './router-isolation.lib.mjs';
+import { RESUME_MODES } from './isolation-runner.resume.lib.mjs';
+
+/** Why a killed session cannot be re-entered in place. Reported, never thrown. */
+export const IN_PLACE_SKIP_REASONS = Object.freeze({
+  NOT_DOCKER: 'not-docker',
+  NO_IDENTIFIER: 'no-identifier',
+  FORMAL_AI_TASK: 'formal-ai-task',
+  ROUTER_TASK: 'router-task',
+  NO_RESUME_SUPPORT: 'no-resume-support',
+  CONTAINER_GONE: 'container-gone',
+  UNSUPPORTED: 'resume-unsupported',
+  REFUSED: 'resume-refused',
+  ERROR: 'resume-error',
+});
+
+/**
+ * Decide — purely, from persisted facts — whether a killed session is a
+ * candidate for a same-container resume.
+ *
+ * Kept separate from the Docker probe below so the policy is testable without a
+ * daemon, and so a caller can report precisely *why* a session was relaunched
+ * from scratch instead of resumed.
+ *
+ * @param {Object} options
+ * @param {string} options.sessionName - The killed session's name (= container name)
+ * @param {Object} options.sessionInfo - Persisted session info
+ * @returns {{eligible: boolean, reason: string, identifier: string|null, containerName: string|null}}
+ */
+export function planSameContainerResume({ sessionName = null, sessionInfo = {} } = {}) {
+  const containerName = sessionInfo?.sessionId || sessionName || null;
+  const identifier = sessionInfo?.executionUuid || containerName || null;
+  const base = { eligible: false, identifier, containerName };
+
+  if (sessionInfo?.isolationBackend !== 'docker') {
+    // screen/tmux sessions have no filesystem to preserve: their work happens
+    // on the host, which a fresh run already sees.
+    return { ...base, reason: IN_PLACE_SKIP_REASONS.NOT_DOCKER };
+  }
+  if (!identifier) return { ...base, reason: IN_PLACE_SKIP_REASONS.NO_IDENTIFIER };
+
+  const args = Array.isArray(sessionInfo?.args) ? sessionInfo.args : [];
+  if (isFormalAiTask({ args, model: sessionInfo?.model || null })) {
+    return { ...base, reason: IN_PLACE_SKIP_REASONS.FORMAL_AI_TASK };
+  }
+  if (hasUseRouterFlag(args)) return { ...base, reason: IN_PLACE_SKIP_REASONS.ROUTER_TASK };
+
+  return { ...base, eligible: true, reason: 'ready' };
+}
+
+/**
+ * Attempt the same-container resume. Never throws, and never leaves work
+ * running that it does not report: the caller may only fall back to a fresh
+ * launch when `resumed` is false.
+ *
+ * @param {Object} options
+ * @param {string} options.sessionName - The killed session's name
+ * @param {Object} options.sessionInfo - Persisted session info
+ * @param {Object} options.plan - Result of planKillRecovery() (needs `command.display`)
+ * @param {Object} options.runner - Isolation runner module
+ * @param {boolean} [options.verbose]
+ * @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, executionUuid: string|null, mode: string|null, snapshotImage: string|null}>}
+ */
+export async function resumeKilledSessionInPlace({ sessionName, sessionInfo, plan, runner, verbose = false } = {}) {
+  const decision = planSameContainerResume({ sessionName, sessionInfo });
+  const miss = reason => ({ resumed: false, reason, sessionId: null, executionUuid: decision.identifier, mode: null, snapshotImage: null });
+  if (!decision.eligible) return miss(decision.reason);
+  if (typeof runner?.resumeIsolatedSession !== 'function' || typeof runner?.checkDockerContainerExists !== 'function') {
+    return miss(IN_PLACE_SKIP_REASONS.NO_RESUME_SUPPORT);
+  }
+
+  // A container that no longer exists has nothing left to re-enter; `$` would
+  // fall back to a full relaunch, which is what the caller does anyway — but
+  // through the path that also re-acquires leases.
+  const exists = await runner.checkDockerContainerExists(decision.containerName, verbose);
+  if (!exists) return miss(IN_PLACE_SKIP_REASONS.CONTAINER_GONE);
+
+  const result = await runner.resumeIsolatedSession(decision.identifier, { command: plan?.command?.display || null, verbose });
+  if (!result?.success) {
+    const reason = result?.unsupported ? IN_PLACE_SKIP_REASONS.UNSUPPORTED : IN_PLACE_SKIP_REASONS.REFUSED;
+    if (verbose) console.log(`[VERBOSE] In-place resume of ${sessionName} was not possible (${reason}): ${result?.error || 'no reason given'}`);
+    return miss(reason);
+  }
+
+  // `docker-snapshot` names the new container `<session>-resume-<attempt>`; the
+  // old name stays addressable through upstream's `sessionNameHistory`, but the
+  // *new* one is what `$ --status` reports on now, so that is what the monitor
+  // has to track. A resume that somehow reports the old name (a `docker-start`
+  // race, say) is tracked under the execution UUID instead, which upstream
+  // resolves just as well and cannot collide with the dying session's entry.
+  const returnedName = result.sessionName && result.sessionName !== sessionName ? result.sessionName : null;
+  const sessionId = returnedName || decision.identifier;
+  return {
+    resumed: true,
+    reason: result.mode === RESUME_MODES.DOCKER_SNAPSHOT ? 'resumed-in-place' : `resumed-${result.mode || 'unknown'}`,
+    sessionId,
+    executionUuid: result.uuid || decision.identifier,
+    mode: result.mode || null,
+    snapshotImage: result.snapshotImage || null,
+  };
+}

package/src/session-kill-resume.lib.mjs

@@ -11,16 +11,20 @@
  * The restart is bounded by `--session-kill-resume-attempts` (default 1), so a
  * job that reliably runs the host out of memory cannot storm the queue.
  *
- * Nothing here runs under the default `report` policy — behaviour is unchanged
- * unless the operator opts in.
+ * Issue #2189 made `resume` the default: a killed session that is only ever
+ * *offered* for resume is a session nobody resumes. `--on-session-kill=report`
+ * turns everything below back off, and `planKillRecovery` still returns
+ * `reason: 'policy-report'` in that case.
  *
  * @see https://github.com/link-assistant/hive-mind/issues/2134
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
  */
 
 import { readLastSessionIdFromLog, planKilledSessionResume } from './session-resume.lib.mjs';
 import { resolveOnSessionKillPolicy, resolveSessionKillResumeAttempts, shouldResumeKilledSession, ON_SESSION_KILL_RESUME } from './session-kill-policy.lib.mjs';
 import { argvFromSessionArgs } from './session-monitor.kill-sections.lib.mjs';
 import { formatKillResumeSection } from './session-kill-diagnostics.lib.mjs';
+import { resumeKilledSessionInPlace } from './session-kill-resume.in-place.lib.mjs';
 
 /** Field recording how many automatic recovery sessions this session produced. */
 export const KILL_RESUME_ATTEMPTS_FIELD = 'killRecoveryAttempts';
@@ -62,9 +66,18 @@ export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = fa
 /**
  * Start the recovery working session decided by {@link planKillRecovery}.
  *
- * The new session is launched through the same isolation runner the original
- * used and is tracked like any other session, so it reports its own completion
- * (and, if it is killed too, its own diagnosis) through the normal path.
+ * Two ways in, in order of preference:
+ *
+ *   1. **Same container** (issue #2189) — `$ --resume` re-enters the killed
+ *      session's own filesystem, so the clone, the caches and the half-finished
+ *      branch survive. See `./session-kill-resume.in-place.lib.mjs` for the
+ *      cases that are deliberately excluded.
+ *   2. **A fresh isolated run** — the original behaviour, used whenever (1) is
+ *      not available or refuses. Correct, just more expensive.
+ *
+ * Either way the new session is tracked like any other, so it reports its own
+ * completion (and, if it is killed too, its own diagnosis) through the normal
+ * path.
  *
  * @param {Object} options
  * @param {string} options.sessionName - The killed session's name
@@ -74,10 +87,10 @@ export function planKillRecovery({ sessionInfo = {}, logPath = null, killed = fa
  * @param {Function} options.trackSession - Tracker for the new session
  * @param {Function} [options.persistSnapshot] - Persist the attempt counter
  * @param {boolean} [options.verbose]
- * @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null}>}
+ * @returns {Promise<{resumed: boolean, reason: string, sessionId: string|null, display: string|null, inPlace: boolean}>}
  */
 export async function startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot = null, verbose = false } = {}) {
-  const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null });
+  const fail = reason => ({ resumed: false, reason, sessionId: null, display: plan?.command?.display || null, inPlace: false });
   if (!plan?.shouldResume || !plan.command) return fail(plan?.reason || 'no-plan');
   if (!runner || typeof runner.executeWithIsolation !== 'function' || typeof runner.generateSessionId !== 'function') return fail('no-isolation-runner');
   if (typeof trackSession !== 'function') return fail('no-tracker');
@@ -85,10 +98,20 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
   if (!backend) return fail('no-isolation-backend');
 
   try {
-    const newSessionId = runner.generateSessionId();
-    const tool = sessionInfo?.tool || 'claude';
-    const result = await runner.executeWithIsolation(sessionInfo?.command || 'solve', plan.command.args, { backend, sessionId: newSessionId, tool, verbose });
-    if (!result?.success) return fail('start-failed');
+    // Preferred path: re-enter the container the work already happened in.
+    const inPlace = await resumeKilledSessionInPlace({ sessionName, sessionInfo, plan, runner, verbose });
+    let newSessionId = inPlace.sessionId;
+    let executionUuid = inPlace.executionUuid;
+    let containerFilesystemStartBytes = null;
+
+    if (!inPlace.resumed) {
+      newSessionId = runner.generateSessionId();
+      const tool = sessionInfo?.tool || 'claude';
+      const result = await runner.executeWithIsolation(sessionInfo?.command || 'solve', plan.command.args, { backend, sessionId: newSessionId, tool, verbose });
+      if (!result?.success) return fail('start-failed');
+      executionUuid = result.executionUuid || null;
+      containerFilesystemStartBytes = Number.isFinite(result.containerFilesystemStartBytes) ? result.containerFilesystemStartBytes : null;
+    }
 
     trackSession(
       newSessionId,
@@ -102,9 +125,15 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
         [KILL_RESUME_ATTEMPTS_FIELD]: plan.attempt,
         killRecoveryResumed: true,
         killRecoveryOfSession: sessionName,
+        // A resumed execution keeps its UUID; a fresh launch gets a new one, and
+        // inheriting the dead session's would make `$ --status` answer about the
+        // wrong execution until the monitor happened to correct it.
+        executionUuid,
+        killRecoveryInPlace: inPlace.resumed,
+        killRecoveryResumeMode: inPlace.mode || null,
         oomEventObservedAt: undefined,
         dockerBackendGoneFirstSeenAt: undefined,
-        containerFilesystemStartBytes: Number.isFinite(result.containerFilesystemStartBytes) ? result.containerFilesystemStartBytes : null,
+        containerFilesystemStartBytes,
       },
       verbose
     );
@@ -119,9 +148,10 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
     }
 
     if (verbose) {
-      console.log(`[VERBOSE] Session ${sessionName} was killed; started recovery session ${newSessionId} (attempt ${plan.attempt}/${plan.maxAttempts}): ${plan.command.display}`);
+      const how = inPlace.resumed ? `resumed in place (${inPlace.mode || 'unknown mode'})` : `started fresh (in-place resume skipped: ${inPlace.reason})`;
+      console.log(`[VERBOSE] Session ${sessionName} was killed; recovery session ${newSessionId} ${how} (attempt ${plan.attempt}/${plan.maxAttempts}): ${plan.command.display}`);
     }
-    return { resumed: true, reason: 'started', sessionId: newSessionId, display: plan.command.display };
+    return { resumed: true, reason: inPlace.resumed ? inPlace.reason : 'started', sessionId: newSessionId, display: plan.command.display, inPlace: inPlace.resumed };
   } catch (error) {
     if (verbose) {
       console.log(`[VERBOSE] Could not start recovery session for ${sessionName}: ${error?.message || error}`);
@@ -135,7 +165,7 @@ export async function startKillRecoverySession({ sessionName, sessionInfo, plan,
  * Never throws — a failed recovery must still leave a correct kill report.
  *
  * @param {Object} options - See planKillRecovery() and startKillRecoverySession()
- * @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number}>}
+ * @returns {Promise<{resumed: boolean, reason: string, policy: string, sessionId: string|null, display: string|null, attempt: number, maxAttempts: number, inPlace: boolean}>}
  */
 export async function recoverKilledSession({ sessionName, sessionInfo, logPath = null, killed = false, env = process.env, runner = null, trackSession = null, persistSnapshot = null, verbose = false, readLastSessionId = readLastSessionIdFromLog } = {}) {
   let plan;
@@ -143,15 +173,15 @@ export async function recoverKilledSession({ sessionName, sessionInfo, logPath =
     plan = planKillRecovery({ sessionInfo, logPath, killed, env, verbose, readLastSessionId });
   } catch (error) {
     if (verbose) console.log(`[VERBOSE] Could not plan kill recovery for ${sessionName}: ${error?.message || error}`);
-    return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0 };
+    return { resumed: false, reason: 'plan-error', policy: ON_SESSION_KILL_RESUME, sessionId: null, display: null, attempt: 0, maxAttempts: 0, inPlace: false };
   }
 
   if (!plan.shouldResume) {
-    return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
+    return { resumed: false, reason: plan.reason, policy: plan.policy, sessionId: null, display: plan.command?.display || null, attempt: plan.attempt, maxAttempts: plan.maxAttempts, inPlace: false };
   }
 
   const started = await startKillRecoverySession({ sessionName, sessionInfo, plan, runner, trackSession, persistSnapshot, verbose });
-  return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts };
+  return { resumed: started.resumed, reason: started.reason, policy: plan.policy, sessionId: started.sessionId, display: started.display, attempt: plan.attempt, maxAttempts: plan.maxAttempts, inPlace: started.inPlace === true };
 }
 
 /**

package/src/session-monitor.kill-sections.lib.mjs

@@ -95,6 +95,14 @@ export async function buildKillCompletionSections({ sessionName, sessionInfo, st
       exitCode,
       stopRequestedByUser: sessionInfo?.stopRequestedByUser === true,
       locale,
+      // start-command 0.33.0 scans the log tail for fatal markers when the
+      // command exits and reports what it found (link-foundation/start#164,
+      // #165 — both filed from this issue). Pass it through: `$` saw the exit
+      // live, so it can carry evidence our bounded re-read of a huge log may
+      // have missed. Absent on an older `$`, which is why the local scan stays.
+      reportedMemoryExhausted: statusResult?.memoryExhausted ?? null,
+      reportedMemoryExhaustedReason: statusResult?.memoryExhaustedReason ?? null,
+      reportedExitReason: statusResult?.exitReason ?? null,
     });
 
     const argv = argvFromSessionArgs(sessionInfo?.args);

package/src/session-monitor.lib.mjs

@@ -25,7 +25,8 @@ import { formatSessionCompletionMessage, getSessionCompletionExitCode, classifyS
 import { notifySubscribers, getSubscriberCount } from './telegram-subscribers.lib.mjs';
 import { safeSendMessage, safeEditMessageText } from './telegram-safe-reply.lib.mjs';
 import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
-import { readLastSessionIdFromLog, buildResumeCommand, formatResumeSection } from './session-resume.lib.mjs';
+import { buildResumeCommand, formatResumeSection } from './session-resume.lib.mjs';
+import { readLogMarkerLines, readLogTextBounded, scanLogTextChunks } from './log-bounded-read.lib.mjs';
 import { resolveFailedSessionPullRequestState } from './github-pr-state.lib.mjs';
 // Issue #2117: a docker terminal failure that no anchored log footer corroborates may be an exit code start-command fabricated from the command's own output.
 import { clearUnverifiedDockerTerminalMarker as clearUnverifiedDockerTerminalMarkerImpl, shouldDeferUnverifiedDockerTerminal as shouldDeferUnverifiedDockerTerminalImpl } from './session-monitor.docker-terminal.lib.mjs';
@@ -33,6 +34,8 @@ import { isDockerIsolation, sessionStartMs, resolveOomKilledState, resolveStaleE
 // Issue #2134: kill-cause diagnostics + the matching pull-request notice.
 import { buildKillCompletionSections, announceKillOnPullRequest } from './session-monitor.kill-sections.lib.mjs';
 import { runKillRecoveryForCompletion } from './session-kill-resume.lib.mjs';
+// Issue #2189: the handled latch + the memoized last-tool-session-id read that keep a completed session from replaying its whole completion pipeline on every poll.
+import { isCompletionHandled, markCompletionHandled, resolveCachedLastToolSessionId } from './session-completion-state.lib.mjs';
 import { createSessionRegistryQueries } from './session-monitor.queries.lib.mjs';
 export { formatSessionCompletionMessage, getSessionCompletionExitCode } from './work-session-formatting.lib.mjs';
 export { DOCKER_TERMINAL_FOOTER_GRACE_MS } from './session-monitor.docker-terminal.lib.mjs';
@@ -326,6 +329,8 @@ function normalizeSessionUrl(url) {
   return url.replace(/#.*$/, '').replace(/\/+$/, '').toLowerCase();
 }
 const GITHUB_PULL_REQUEST_URL_RE = /https:\/\/github\.com\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)\/pull\/([0-9]+)/g;
+/** Marker prefix parsed by `parseDiskMarkers` (issue #1945/#1988), used to collect just those lines. */
+const DISK_MARKER_LINE_RE = /📊 \[DISK\] /;
 export function extractPullRequestUrlFromText(text, { owner = null, repo = null } = {}) {
   if (!text) return null;
   const expectedOwner = owner ? String(owner).toLowerCase() : null;
@@ -344,8 +349,12 @@ export function extractPullRequestUrlFromText(text, { owner = null, repo = null
 async function resolvePullRequestUrlFromSessionLog(logPath, ctx, { verbose = false, readFile = fs.readFile } = {}) {
   if (!logPath) return null;
   try {
-    const logText = await readFile(logPath, 'utf8');
-    const pullRequestUrl = extractPullRequestUrlFromText(logText, { owner: ctx.owner, repo: ctx.repo });
+    // Issue #2189: this used to be `readFile(logPath, 'utf8')`, run once per
+    // monitor tick for a session the bot never marked handled — a 134 MB
+    // transcript pulled into the bot's own heap over and over. The chunked scan
+    // still covers the whole log, one chunk at a time, and stops at the first
+    // matching URL (which is printed early, when the PR is created).
+    const pullRequestUrl = await scanLogTextChunks(logPath, text => extractPullRequestUrlFromText(text, { owner: ctx.owner, repo: ctx.repo }), { readFile, verbose });
     if (pullRequestUrl && verbose) {
       console.log(`[VERBOSE] Found PR ${pullRequestUrl} in completed session log ${logPath}`);
     }
@@ -370,7 +379,10 @@ export async function buildDiskDiagnosticsExtraSection(logPath, { verbose = fals
     let logText = '';
     if (logPath) {
       try {
-        logText = await readFile(logPath, 'utf8');
+        // Issue #2189: `parseDiskMarkers` only ever looks at `📊 [DISK]` lines,
+        // so collect those lines instead of the whole transcript. Cost is one
+        // chunk plus a few kilobytes of markers, whatever the log's size.
+        logText = await readLogMarkerLines(logPath, DISK_MARKER_LINE_RE, { readFile, verbose });
       } catch (readError) {
         if (verbose) {
           console.log(`[VERBOSE] Could not read session log ${logPath} for disk diagnostics: ${readError?.message || readError}`);
@@ -402,7 +414,10 @@ export async function buildSubscriptionBlockedExtraSection(logPath, { verbose =
   try {
     let logText = '';
     try {
-      logText = await readFile(logPath, 'utf8');
+      // Issue #2189: the blocked report is printed as the run stops, so the
+      // bounded head+tail excerpt always contains it; the middle of a multi-
+      // gigabyte transcript never has to be resident to find it.
+      logText = await readLogTextBounded(logPath, { readFile, verbose });
     } catch (readError) {
       if (verbose) {
         console.log(`[VERBOSE] Could not read session log ${logPath} for subscription block: ${readError?.message || readError}`);
@@ -439,6 +454,35 @@ async function getDockerContainerFilesystemSizeForSession(sessionName, sessionIn
     return null;
   }
 }
+/**
+ * How long a docker writable-layer measurement stays good enough while the
+ * session is still running (issue #2189).
+ *
+ * `docker ps --size` walks the container's whole writable layer. On the 27 GB
+ * layer in the captured incident that is expensive, and the monitor was paying
+ * it once per session per 30-second poll for no gain: the number is only ever
+ * *reported* at completion. The completion measurement is never throttled, so
+ * the number in the report is still fresh.
+ */
+export const DOCKER_FILESYSTEM_REFRESH_INTERVAL_MS = 5 * 60 * 1000;
+/**
+ * Whether the writable-layer size should be measured on this poll.
+ *
+ * @param {Object} sessionInfo
+ * @param {Object} [options]
+ * @param {boolean} [options.stillRunning] - The session is still executing
+ * @param {number} [options.now] - Epoch ms (injectable for tests)
+ * @param {number} [options.intervalMs]
+ * @returns {boolean}
+ */
+export function shouldRefreshDockerFilesystemSize(sessionInfo, { stillRunning = true, now = Date.now(), intervalMs = DOCKER_FILESYSTEM_REFRESH_INTERVAL_MS } = {}) {
+  if (sessionInfo?.isolationBackend !== 'docker') return false;
+  // The completion report quotes this number, so it is always measured fresh.
+  if (!stillRunning) return true;
+  const observedAt = Date.parse(sessionInfo?.containerFilesystemLastObservedAt || '');
+  if (!Number.isFinite(observedAt)) return true;
+  return now - observedAt >= intervalMs;
+}
 async function refreshDockerContainerFilesystemSizeForSession(sessionName, sessionInfo, { verbose = false, sizeProvider = null } = {}) {
   const bytes = await getDockerContainerFilesystemSizeForSession(sessionName, sessionInfo, { verbose, sizeProvider });
   if (!Number.isFinite(bytes)) return null;
@@ -665,6 +709,20 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
     console.log(`[VERBOSE] Checking ${sessions.length} active session(s)...`);
   }
   for (const { sessionName, sessionInfo } of sessions) {
+    // Issue #2189: a session whose completion notification was already
+    // delivered is terminal. Without this latch the monitor re-entered the whole
+    // completion pipeline on every poll — re-resolving the linked pull request,
+    // re-scanning a 134 MB log, re-walking a 27 GB writable layer and re-sending
+    // a notification the user already had — because a late failure in that
+    // pipeline (or a bot restart) left the session tracked. Finalize it here,
+    // before any status probe, and do none of that work again.
+    if (isCompletionHandled(sessionInfo)) {
+      if (verbose) {
+        console.log(`[VERBOSE] Session ${sessionName} was already reported at ${sessionInfo.completionNotifiedAt}; finalizing without repeating the completion work (issue #2189)`);
+      }
+      completeSession(sessionName, sessionInfo.completionExitCode ?? 0, verbose, sessionInfo.completionStatus ?? null);
+      continue;
+    }
     let stillRunning;
     let exitCode = null;
     let statusResult = null;
@@ -727,11 +785,13 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
         }
       }
     }
-    if (sessionInfo?.isolationBackend === 'docker') {
+    if (shouldRefreshDockerFilesystemSize(sessionInfo, { stillRunning })) {
       observedContainerFilesystemBytes = await refreshDockerContainerFilesystemSizeForSession(sessionName, sessionInfo, {
         verbose,
         sizeProvider: options.dockerContainerSizeProvider,
       });
+    } else if (sessionInfo?.isolationBackend === 'docker' && verbose) {
+      console.log(`[VERBOSE] Session ${sessionName}: reusing the writable-layer size observed at ${sessionInfo.containerFilesystemLastObservedAt} (issue #2189: not re-walking the layer every poll)`);
     }
     if (!stillRunning) {
       console.log(`Session ${sessionName} has finished. Sending notification to chat ${sessionInfo.chatId}`);
@@ -756,6 +816,10 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
             statusResult,
             readFile: options.readFile,
           });
+          if (pullRequestUrl && sessionInfo.resolvedPullRequestUrl !== pullRequestUrl) {
+            sessionInfo.resolvedPullRequestUrl = pullRequestUrl;
+            persistSessionSnapshot(sessionName, sessionInfo);
+          }
         } catch (lookupError) {
           if (verbose) {
             console.log(`[VERBOSE] Pull request lookup failed for ${sessionName}: ${lookupError?.message || lookupError}`);
@@ -806,8 +870,20 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
             }
           }
         }
+        // Issue #2189: the last tool session id is read through the session
+        // record's cache, so the working-session log is scanned once per session
+        // instead of once per poll per consumer (the resume section below and the
+        // automatic recovery further down both need it, and both used to scan).
+        const resolveLastToolSessionId = logPath => {
+          const resolution = resolveCachedLastToolSessionId({ sessionInfo, logPath, verbose });
+          if (resolution.scanned) persistSessionSnapshot(sessionName, sessionInfo);
+          return resolution.id;
+        };
         // Issue #1927: for a killed /solve, offer a command using the last tool
-        // session ID in the log. Do not auto-relaunch work that may reliably OOM.
+        // session ID in the log. Issue #2189 additionally *starts* that command
+        // by default (`--on-session-kill=resume`), bounded by
+        // `--session-kill-resume-attempts`, so the work is not left for a human
+        // to notice hours later.
         const resumeExtraSections = [];
         let killResumeCommand = null;
         try {
@@ -821,7 +897,7 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
             //   `Session ID:` marker. Do not guess from neighboring UUID-named
             //   logs: start-command stores unrelated tasks in the same backend
             //   directory, which caused issue #2109's invalid resume id.
-            const lastSessionId = readLastSessionIdFromLog(logPath, { verbose });
+            const lastSessionId = resolveLastToolSessionId(logPath);
             const resumeCommand = buildResumeCommand({ sessionInfo, lastSessionId });
             const resumeSection = formatResumeSection({ lastSessionId, command: resumeCommand });
             killResumeCommand = resumeCommand || null;
@@ -900,10 +976,21 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
             runner: options.isolationRunner || null,
             trackSession: options.trackSession || trackSession,
             persistSnapshot: () => persistSessionSnapshot(sessionName, sessionInfo),
+            // Issue #2189: reuse the id already read above instead of scanning
+            // the same (possibly multi-gigabyte) log a second time.
+            readLastSessionId: logPath => resolveLastToolSessionId(logPath),
             locale: sessionInfo?.locale || null,
             verbose,
           });
           killRecovery = recovered.recovery;
+          if (killRecovery.resumed && killRecovery.sessionId) {
+            // Issue #2189: remember which session took over, so the durable
+            // history says what happened to this work and a restart cannot start
+            // a second recovery for the same kill.
+            sessionInfo.killRecoverySessionId = killRecovery.sessionId;
+            persistSessionSnapshot(sessionName, sessionInfo);
+            logEvent('session_kill_recovered', { sessionName, recoverySessionId: killRecovery.sessionId, attempt: killRecovery.attempt, maxAttempts: killRecovery.maxAttempts, policy: killRecovery.policy || null });
+          }
           if (recovered.section) killReport.sections.push(recovered.section);
         }
         const message = formatSessionCompletionMessage({
@@ -952,6 +1039,16 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
           notifyFromChatId = sent?.chat?.id || sessionInfo.chatId;
           notifyMessageId = sent?.message_id || null;
         }
+        // Issue #2189: the user has now been told. Latch that fact durably
+        // BEFORE the remaining best-effort work (subscriber fan-out, container
+        // cleanup), because any failure after this point used to send the whole
+        // completion pipeline — and this notification — round again on the next
+        // poll. The snapshot write means even a bot restart in this window
+        // finalizes the session silently instead of re-notifying.
+        if (markCompletionHandled(sessionInfo, { exitCode: finalExitCode, status: resolvedStatus })) {
+          persistSessionSnapshot(sessionName, sessionInfo);
+          logEvent('session_completion_notified', { sessionName, exitCode: finalExitCode ?? null, status: resolvedStatus || null, notifiedAt: sessionInfo.completionNotifiedAt });
+        }
         // Issue #1688: forward the same completion message to every /subscribe-d user
         //   in their private chat with the bot. Failures are logged but don't block
         //   completion of the parent session.
@@ -1024,6 +1121,13 @@ async function resolvePullRequestUrlForSession(sessionInfo, { verbose = false, l
   if (!ctx || ctx.type !== 'issue' || !ctx.owner || !ctx.repo || !ctx.number) {
     return null;
   }
+  // Issue #2189: a completion that has to be retried must not re-run the linked-PR
+  // lookup (an API round trip, then a scan of the session log). The answer cannot
+  // change for a session that has already finished, so remember it.
+  if (typeof sessionInfo.resolvedPullRequestUrl === 'string' && sessionInfo.resolvedPullRequestUrl) {
+    if (verbose) console.log(`[VERBOSE] Reusing resolved pull request ${sessionInfo.resolvedPullRequestUrl} for this session (not looked up again)`);
+    return sessionInfo.resolvedPullRequestUrl;
+  }
   if (typeof lookupLinkedPullRequest === 'function') {
     const linkedPullRequestUrl = await lookupLinkedPullRequest(ctx);
     if (linkedPullRequestUrl) return linkedPullRequestUrl;
@@ -1089,6 +1193,22 @@ export function startSessionMonitoring(bot, verbose = false, intervalMs = 30000,
  * record whose startTime is after the current bot start (it cannot belong to a
  * previous run), satisfying requirement #2's "started before bot start time".
  *
+ * Issue #2189 asks for the other half of that sentence — "on startup resume all
+ * still-running / interrupted commands". Both cases are handled here plus the
+ * first monitor tick, which runs synchronously after this function:
+ *
+ *   - a session that is **still running** keeps running; re-registering it is
+ *     exactly what resumes it, and the bot reports it when it ends;
+ *   - a session that was **interrupted** (its backend is gone, or its log footer
+ *     records a kill) is detected as finished on that first tick and, under the
+ *     now-default `--on-session-kill=resume`, a recovery working session is
+ *     started from its last tool session id — bounded by
+ *     `--session-kill-resume-attempts`, whose counter is persisted, so a job
+ *     that dies every time cannot be relaunched once per bot restart forever;
+ *   - a session that was already **reported** before the previous process died
+ *     carries the persisted `completionNotifiedAt` latch and is finalized
+ *     silently, so a restart never re-notifies.
+ *
  * @param {object} [options]
  * @param {object} [options.store] - Session store to load from (default: the store set via setSessionStore).
  * @param {number} [options.botStartTime] - Epoch seconds; only sessions started strictly before this are resumed. Defaults to now.
@@ -1137,7 +1257,10 @@ export async function resumeTrackedSessions(options = {}) {
     }
   }
   if (resumed.length > 0) {
-    console.log(`♻️  Resumed monitoring of ${resumed.length} session(s) from durable store after restart`);
+    // Issue #2189: say how many of these are already-reported leftovers, so the
+    // startup line is not read as "N sessions are still working".
+    const alreadyReported = resumed.filter(({ sessionInfo }) => isCompletionHandled(sessionInfo)).length;
+    console.log(`♻️  Resumed monitoring of ${resumed.length} session(s) from durable store after restart${alreadyReported > 0 ? ` (${alreadyReported} already reported, will be finalized silently)` : ''}`);
   } else if (verbose) {
     console.log('[VERBOSE] resumeTrackedSessions: no eligible sessions to resume');
   }

package/src/session-store.lib.mjs

@@ -38,7 +38,21 @@ import path from 'node:path';
 // `executionUuid` (#2154) is start-command's own identifier for the execution —
 // the one `$ --list` prints. It differs from `sessionId`, so persisting it is
 // what lets a restarted bot still correlate its sessions with the session list.
-const PERSISTABLE_FIELDS = ['chatId', 'messageId', 'startTime', 'url', 'command', 'commandAlias', 'isolationBackend', 'sessionId', 'executionUuid', 'containerFilesystemStartBytes', 'containerFilesystemLastBytes', 'containerFilesystemLastObservedAt', 'tool', 'infoBlock', 'urlContext', 'requesterUserId', 'showLimits', 'locale', 'logPath', 'args'];
+// Issue #2189 adds the fields that keep a completed session terminal and its
+// per-poll cost constant:
+//   - `completionNotifiedAt`/`completionExitCode`/`completionStatus` latch the
+//     one delivered notification, so a restart between "notified" and
+//     "untracked" finalizes silently instead of re-running the whole completion
+//     pipeline and notifying the user again.
+//   - `lastToolSessionId` caches the marker found by scanning the working
+//     session log, so that scan is never O(log size) per poll.
+//   - `killRecoveryAttempts`/`killRecoverySessionId`/`killRecoveryOfSession`
+//     bound and record automatic recovery across restarts — without them a
+//     reliably-crashing job could restart once per bot launch forever.
+//   - `stopRequestedByUser`/`stopRequestedBy` must survive a restart too: with
+//     `--on-session-kill=resume` now the default, forgetting that an operator
+//     asked for the stop would relaunch the very work they cancelled.
+const PERSISTABLE_FIELDS = ['chatId', 'messageId', 'startTime', 'url', 'command', 'commandAlias', 'isolationBackend', 'sessionId', 'executionUuid', 'containerFilesystemStartBytes', 'containerFilesystemLastBytes', 'containerFilesystemLastObservedAt', 'tool', 'infoBlock', 'urlContext', 'requesterUserId', 'showLimits', 'locale', 'logPath', 'args', 'completionNotifiedAt', 'completionExitCode', 'completionStatus', 'lastToolSessionId', 'killRecoveryAttempts', 'killRecoverySessionId', 'killRecoveryOfSession', 'killRecoveryResumed', 'killRecoveryInPlace', 'killRecoveryResumeMode', 'stopRequestedByUser', 'stopRequestedBy', 'onSessionKill', 'resolvedPullRequestUrl'];
 
 /**
  * Resolve the directory durable bot state is written to. Honors