@link-assistant/hive-mind 2.15.2 → 2.17.0

package/src/log-sanitize-worker-entry.mjs

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+/**
+ * Worker entry point for {@link sanitizeLogFileToFileBounded} (issue #2189).
+ *
+ * Runs the streaming publication sanitizer inside a worker thread whose old
+ * generation is capped by `resourceLimits`, so a residual blow-up in any of the
+ * sanitizer's regular expressions terminates *this thread* with
+ * `ERR_WORKER_OUT_OF_MEMORY` instead of aborting the whole run with
+ * `FATAL ERROR: Reached heap limit`.
+ *
+ * Protocol: post `{type:'ready'}` once the sanitizer module graph is loaded (the
+ * parent uses it to tell a start-up failure from a sanitize failure), then
+ * `{type:'done', stats}` or `{type:'error', ...}`.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
+ */
+
+import { parentPort, workerData } from 'node:worker_threads';
+import { sanitizeLogFileToFile } from './log-sanitize-stream.lib.mjs';
+
+if (!parentPort) throw new Error('log-sanitize-worker-entry must be started as a worker thread');
+
+parentPort.postMessage({ type: 'ready' });
+
+try {
+  const stats = await sanitizeLogFileToFile({ ...(workerData || {}) });
+  parentPort.postMessage({ type: 'done', stats });
+} catch (error) {
+  parentPort.postMessage({ type: 'error', message: error?.message || String(error), name: error?.name || 'Error', code: error?.code || null });
+}

package/src/log-sanitize-worker.lib.mjs

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+
+/**
+ * Bounded-heap execution of the streaming log sanitizer (issue #2189).
+ *
+ * {@link sanitizeLogFileToFile} is already memory-bounded by construction: it
+ * holds one block at a time and caps its hold-back. This module is the second
+ * line of defence issue #2189 asks for —
+ *
+ * > A worker with a bounded heap for the sanitize step would also contain any
+ * > residual blow-up instead of taking the whole run down.
+ *
+ * Node's `worker_threads` gives each worker its own V8 isolate with its own
+ * `resourceLimits`. Exceeding them terminates the worker with
+ * `ERR_WORKER_OUT_OF_MEMORY` and leaves the parent running, verified in
+ * `experiments/issue-2189-bounded-sanitize-worker.mjs`. So even if a future
+ * pattern, a pathological log, or a dependency upgrade reintroduces an
+ * unbounded allocation inside the sanitizer, the blast radius is one thread and
+ * one failed log upload — not the working session that already did its job.
+ *
+ * The worker is only worth its start-up cost on logs big enough to matter, so
+ * small logs run in-process ({@link DEFAULT_WORKER_THRESHOLD_BYTES}). If the
+ * worker cannot even start (no `worker_threads`, a module resolution failure,
+ * a restricted runtime) the sanitize falls back in-process, which is exactly
+ * the behaviour before this module existed. Once the worker has reported
+ * `ready`, failures are propagated instead: falling back in-process after the
+ * worker hit its heap limit would run the same blow-up in the parent.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
+ */
+
+import fsPromises from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { sanitizeLogFileToFile } from './log-sanitize-stream.lib.mjs';
+
+/** Old-generation cap for the sanitize worker, in MiB. */
+export const DEFAULT_WORKER_HEAP_MB = 512;
+
+/** Logs at or above this size are sanitized in a bounded worker. */
+export const DEFAULT_WORKER_THRESHOLD_BYTES = 16 * 1024 * 1024;
+
+/** Environment variable that forces the worker off (`0`/`false`/`off`). */
+export const WORKER_DISABLE_ENV = 'HIVE_MIND_SANITIZE_WORKER';
+
+const WORKER_ENTRY_URL = new URL('./log-sanitize-worker-entry.mjs', import.meta.url);
+
+/** Options whose values cannot be structured-cloned into a worker. */
+const NON_CLONABLE_OPTIONS = ['sanitize', 'transform', 'onProgress', 'fsImpl'];
+
+/**
+ * Whether the bounded worker should be used for a source of this size.
+ *
+ * @param {number} sourceSize - Bytes to sanitize
+ * @param {object} [options]
+ * @param {number} [options.thresholdBytes=DEFAULT_WORKER_THRESHOLD_BYTES]
+ * @param {object} [options.env=process.env]
+ * @returns {boolean}
+ */
+export function shouldUseSanitizeWorker(sourceSize, options = {}) {
+  const { thresholdBytes = DEFAULT_WORKER_THRESHOLD_BYTES, env = process.env } = options;
+  const disabled = String(env?.[WORKER_DISABLE_ENV] ?? '')
+    .trim()
+    .toLowerCase();
+  if (disabled === '0' || disabled === 'false' || disabled === 'off' || disabled === 'no') return false;
+  if (!Number.isFinite(sourceSize)) return false;
+  return sourceSize >= thresholdBytes;
+}
+
+/**
+ * Run {@link sanitizeLogFileToFile} inside a worker with a bounded old generation.
+ *
+ * @param {object} options - Forwarded to {@link sanitizeLogFileToFile}
+ * @param {number} [options.workerHeapMb=DEFAULT_WORKER_HEAP_MB]
+ * @param {Function} [options.workerFactory] - `(url, opts) => Worker`, for tests
+ * @returns {Promise<object>} Sanitize stats, plus `{worker: true}`
+ */
+export async function sanitizeLogFileInWorker(options = {}) {
+  const { workerHeapMb = DEFAULT_WORKER_HEAP_MB, workerFactory = null, ...sanitizeOptions } = options;
+  const createWorker = workerFactory || (await defaultWorkerFactory());
+
+  return await new Promise((resolve, reject) => {
+    let ready = false;
+    let settled = false;
+    const settle = (fn, value) => {
+      if (settled) return;
+      settled = true;
+      fn(value);
+    };
+
+    let worker;
+    try {
+      worker = createWorker(WORKER_ENTRY_URL, {
+        workerData: sanitizeOptions,
+        resourceLimits: { maxOldGenerationSizeMb: workerHeapMb },
+      });
+    } catch (error) {
+      error.sanitizeWorkerStarted = false;
+      settle(reject, error);
+      return;
+    }
+
+    worker.on('message', message => {
+      if (message?.type === 'ready') {
+        ready = true;
+        return;
+      }
+      if (message?.type === 'done') {
+        settle(resolve, { ...message.stats, worker: true });
+        worker.terminate().catch(() => {});
+        return;
+      }
+      if (message?.type === 'error') {
+        const error = new Error(message.message);
+        error.name = message.name || 'Error';
+        if (message.code) error.code = message.code;
+        error.sanitizeWorkerStarted = true;
+        settle(reject, error);
+        worker.terminate().catch(() => {});
+      }
+    });
+    worker.on('error', error => {
+      error.sanitizeWorkerStarted = ready;
+      settle(reject, error);
+    });
+    worker.on('exit', code => {
+      const error = new Error(`log sanitize worker exited with code ${code} before reporting a result`);
+      error.sanitizeWorkerStarted = ready;
+      settle(reject, error);
+    });
+  });
+}
+
+/**
+ * Sanitize a log file to another file, in a bounded worker when it is large.
+ *
+ * Drop-in replacement for {@link sanitizeLogFileToFile}: same options, same
+ * stats, same fail-closed guarantee about the destination.
+ *
+ * @param {object} options - {@link sanitizeLogFileToFile} options
+ * @param {number} [options.thresholdBytes=DEFAULT_WORKER_THRESHOLD_BYTES]
+ * @param {number} [options.workerHeapMb=DEFAULT_WORKER_HEAP_MB]
+ * @param {Function} [options.onWorkerFallback] - `({reason, error}) => void`
+ * @returns {Promise<object>} Sanitize stats; `worker` is true when the worker ran
+ */
+export async function sanitizeLogFileToFileBounded(options = {}) {
+  const { thresholdBytes = DEFAULT_WORKER_THRESHOLD_BYTES, workerHeapMb = DEFAULT_WORKER_HEAP_MB, workerFactory = null, onWorkerFallback = null, env = process.env, fsImpl = fsPromises, ...sanitizeOptions } = options;
+
+  let sourceSize = null;
+  try {
+    sourceSize = (await fsImpl.stat(sanitizeOptions.sourcePath)).size;
+  } catch {
+    // An unknown size routes in-process, which is the conservative choice.
+  }
+
+  // `workerData` crosses the thread boundary by structured clone, which cannot
+  // carry functions. A caller that customises the sanitize, the transform or the
+  // progress hook keeps the in-process path (still streaming, still bounded).
+  const clonable = NON_CLONABLE_OPTIONS.every(name => typeof sanitizeOptions[name] !== 'function');
+
+  if (!clonable || !shouldUseSanitizeWorker(sourceSize, { thresholdBytes, env })) {
+    return { ...(await sanitizeLogFileToFile({ ...sanitizeOptions, fsImpl })), worker: false };
+  }
+
+  try {
+    return await sanitizeLogFileInWorker({ ...sanitizeOptions, workerHeapMb, workerFactory });
+  } catch (error) {
+    if (error?.sanitizeWorkerStarted) throw error;
+    // The worker never got as far as loading the sanitizer, so nothing about
+    // memory has been learned; run in-process exactly as before.
+    if (onWorkerFallback) onWorkerFallback({ reason: 'worker-unavailable', error });
+    await fsImpl.unlink(sanitizeOptions.destPath).catch(() => {});
+    return { ...(await sanitizeLogFileToFile({ ...sanitizeOptions, fsImpl })), worker: false };
+  }
+}
+
+/**
+ * Resolve the real `worker_threads` Worker constructor, lazily.
+ *
+ * @returns {Promise<Function>} `(url, options) => Worker`
+ */
+async function defaultWorkerFactory() {
+  const { Worker } = await import('node:worker_threads');
+  return (url, workerOptions) => new Worker(fileURLToPath(url), workerOptions);
+}
+
+export { WORKER_ENTRY_URL };

package/src/log-upload.lib.mjs

@@ -4,6 +4,7 @@ import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 import { sanitizeForPublication } from './token-sanitization.lib.mjs';
+import { sanitizeLogFileToFileBounded } from './log-sanitize-worker.lib.mjs';
 
 // Log upload module for hive-mind
 // Uses gh-upload-log for uploading log files to GitHub
@@ -151,14 +152,25 @@ export const uploadLogWithGhUploadLog = async ({ logFile, isPublic, description,
   let privateTempDirectory = null;
 
   try {
-    const exactSourceBytes = await fs.readFile(logFile, 'utf8');
-    const sanitizedLog = await sanitizeForPublication(exactSourceBytes);
     const sanitizedDescription = description ? await sanitizeForPublication(description) : description;
     privateTempDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'hive-mind-log-upload-'));
     await fs.chmod(privateTempDirectory, 0o700);
     const privateLogFile = path.join(privateTempDirectory, 'sanitized.log');
-    await fs.writeFile(privateLogFile, sanitizedLog, { encoding: 'utf8', mode: 0o600 });
-    await fs.chmod(privateLogFile, 0o600);
+    // Issue #2189: this used to be readFile → sanitizeForPublication → writeFile,
+    // i.e. three full-size copies of the log in the heap at once. A 134 MB
+    // transcript reliably killed the run with "Reached heap limit". The streaming
+    // sanitizer holds one block (1 MiB) at a time, so cost no longer scales with
+    // log size. This is now the ONLY place `--attach-logs` sanitizes the log.
+    // Large logs additionally run in a heap-capped worker, so a residual blow-up
+    // costs one thread and one failed upload instead of the whole session.
+    const sanitizeStats = await sanitizeLogFileToFileBounded({
+      sourcePath: logFile,
+      destPath: privateLogFile,
+      onWorkerFallback: ({ error }) => log(`  ⚠️  Sanitize worker unavailable (${error?.message || error}); sanitizing in-process`, { verbose: true }),
+    });
+    if (verbose) {
+      await log(`  🧼 Streamed sanitize: ${sanitizeStats.sourceSize} bytes in ${sanitizeStats.blocks} block(s)${sanitizeStats.forcedReleases > 0 ? `, ${sanitizeStats.forcedReleases} forced release(s)` : ''}${sanitizeStats.worker ? ' (bounded worker)' : ''}`, { verbose: true });
+    }
 
     const commandArgs = buildGhUploadLogArgs({
       logFile: privateLogFile,

package/src/pull-request-changes.lib.mjs

@@ -106,7 +106,7 @@ const measureDiff = diff => {
     section = null;
   };
 
-  for (let start = 0; start < diff.length; ) {
+  for (let start = 0; start < diff.length;) {
     let end = diff.indexOf('\n', start);
     if (end === -1) end = diff.length;
     const line = diff.slice(start, end);

package/src/session-completion-state.lib.mjs

@@ -0,0 +1,124 @@
+/**
+ * Terminal, persisted "this session has been reported" state (issue #2189).
+ *
+ * The captured incident shows what happens without it. A `/solve` session was
+ * killed at 14:07:49Z; the monitor detected the completion, ran the full
+ * completion pipeline, and something late in that pipeline threw. The session
+ * was therefore kept in memory "so the completion notification can be retried",
+ * and every subsequent poll started the whole pipeline again — four
+ * `Session … has finished. Sending notification` lines, three
+ * `was killed; offering resume from last session` lines. Each of those cycles
+ *
+ *   - re-resolved the linked pull request over the network,
+ *   - re-scanned a 134 MB working-session log for the last tool session id,
+ *   - re-stat'ed a 27 GB docker writable layer,
+ *   - and re-sent the notification the user had already received.
+ *
+ * The bot's RSS climbed from 1.78 GB to 1.84 GB against a ~2 GB heap cap while
+ * it did so. The requirement from the issue is exact: "a killed session must
+ * reach a terminal, persisted, handled state after its notification is delivered
+ * once", and "per-cycle work must not be O(log size) — cache the recovered
+ * session id in the session record".
+ *
+ * This module owns both halves of that:
+ *
+ *   1. {@link markCompletionHandled} / {@link isCompletionHandled} — the
+ *      handled latch. It is written to the durable session snapshot, so it also
+ *      survives a bot restart between "notification delivered" and "session
+ *      untracked": a reloaded session that was already reported is finalized
+ *      silently instead of notifying the user a second time.
+ *   2. {@link resolveCachedLastToolSessionId} — a memoized, snapshot-backed
+ *      read of the last `Session ID:` marker in the working-session log. The
+ *      log is scanned at most once per session, and a scan that found nothing
+ *      is remembered as such (an empty string) so a fruitless multi-gigabyte
+ *      scan is never repeated either.
+ *
+ * Pure and dependency-light: the log reader and the clock are injectable, so
+ * every branch is testable without a real log or a real bot.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
+ */
+
+import { readLastSessionIdFromLog } from './session-resume.lib.mjs';
+
+/**
+ * Fields this module writes on a tracked session. They are persisted (see
+ * `PERSISTABLE_FIELDS` in session-store.lib.mjs) precisely so the latch and the
+ * cache survive the restart that would otherwise replay the whole pipeline.
+ */
+export const COMPLETION_STATE_FIELDS = ['completionNotifiedAt', 'completionExitCode', 'completionStatus', 'lastToolSessionId'];
+
+/**
+ * Whether this session's completion notification was already delivered.
+ *
+ * @param {Object|null} sessionInfo
+ * @returns {boolean}
+ */
+export function isCompletionHandled(sessionInfo) {
+  return typeof sessionInfo?.completionNotifiedAt === 'string' && sessionInfo.completionNotifiedAt.length > 0;
+}
+
+/**
+ * Latch this session as reported, recording the outcome that was reported so a
+ * later finalize does not have to re-derive it.
+ *
+ * Idempotent: a second call keeps the first timestamp, because the first
+ * delivery is the one the user saw.
+ *
+ * @param {Object} sessionInfo - Tracked session info (mutated in place)
+ * @param {Object} [options]
+ * @param {number|null} [options.exitCode] - Exit code that was reported
+ * @param {string|null} [options.status] - Terminal status that was reported
+ * @param {Function} [options.now] - Injectable clock returning a Date
+ * @returns {boolean} True when this call is the one that latched it
+ */
+export function markCompletionHandled(sessionInfo, { exitCode = null, status = null, now = () => new Date() } = {}) {
+  if (!sessionInfo || typeof sessionInfo !== 'object') return false;
+  if (isCompletionHandled(sessionInfo)) return false;
+  sessionInfo.completionNotifiedAt = now().toISOString();
+  sessionInfo.completionExitCode = Number.isFinite(exitCode) ? exitCode : null;
+  sessionInfo.completionStatus = status || null;
+  return true;
+}
+
+/**
+ * The last tool session id for this session, scanning the working-session log at
+ * most once.
+ *
+ * A session that has been scanned carries a string: the id, or `''` for "the log
+ * has no usable marker". Both are answers, and neither is worth paying for
+ * twice — the scan walks the log backwards in bounded chunks, but on a 134 MB
+ * log with no marker in the tail that is still a full read of the file, per
+ * poll, per session.
+ *
+ * @param {Object} options
+ * @param {Object|null} options.sessionInfo - Tracked session info (mutated in place)
+ * @param {string|null} [options.logPath] - Working-session log to scan
+ * @param {boolean} [options.verbose]
+ * @param {Function} [options.readLastSessionId] - Override for tests
+ * @returns {{id: string|null, cached: boolean, scanned: boolean}}
+ */
+export function resolveCachedLastToolSessionId({ sessionInfo = null, logPath = null, verbose = false, readLastSessionId = readLastSessionIdFromLog } = {}) {
+  const cached = sessionInfo?.lastToolSessionId;
+  if (typeof cached === 'string') {
+    if (verbose && cached) {
+      console.log(`[VERBOSE] session-completion-state: reusing cached last tool session id ${cached} (log not re-scanned)`);
+    }
+    return { id: cached || null, cached: true, scanned: false };
+  }
+  let id;
+  try {
+    id = readLastSessionId(logPath, { verbose }) || null;
+  } catch (error) {
+    if (verbose) {
+      console.log(`[VERBOSE] session-completion-state: could not read last tool session id from ${logPath}: ${error?.message || error}`);
+    }
+    // A read error is not an answer — leave the cache unset so a later poll,
+    // once the log is readable again, can still find the id.
+    return { id: null, cached: false, scanned: false };
+  }
+  if (sessionInfo && typeof sessionInfo === 'object') sessionInfo.lastToolSessionId = id || '';
+  return { id, cached: false, scanned: true };
+}
+
+export default { COMPLETION_STATE_FIELDS, isCompletionHandled, markCompletionHandled, resolveCachedLastToolSessionId };

package/src/session-kill-diagnostics.lib.mjs

@@ -26,7 +26,9 @@ import fsPromises from 'fs/promises';
 import { exec as execCallback } from 'child_process';
 import { promisify } from 'util';
 import { t } from './i18n.lib.mjs';
-import { formatBytes, parseResourceMarkers } from './solve.resource-diagnostics.lib.mjs';
+import { formatBytes, formatHeapUsage, parseResourceMarkers } from './solve.resource-diagnostics.lib.mjs';
+import { findFatalMemoryMarker } from './child-exit.lib.mjs';
+import { readLogTextBounded } from './log-bounded-read.lib.mjs';
 
 const exec = promisify(execCallback);
 
@@ -37,6 +39,31 @@ export const KILL_CAUSE_UNKNOWN = 'unknown';
 
 /** Memory is considered exhausted below this share of total RAM still available. */
 export const MEMORY_EXHAUSTED_AVAILABLE_RATIO = 0.1;
+/**
+ * How much of a session log the kill diagnosis is allowed to read (issue #2189).
+ * Split head/tail by {@link readLogTextBounded}: 512 KiB of the beginning and
+ * 512 KiB of the end, which covers both the resource markers and the fatal exit.
+ */
+export const KILL_DIAGNOSTICS_LOG_BYTES = 1024 * 1024;
+/**
+ * A V8 heap this far into its own limit counts as memory exhaustion on its own
+ * (issue #2189). The runtime aborts with "Reached heap limit" while the machine
+ * still reports gigabytes free, so the host-memory ratio above never fires; the
+ * heap markers are the only in-band signal, and the last one written before the
+ * abort is typically already deep in the red.
+ */
+export const HEAP_EXHAUSTED_PERCENT = 90;
+
+/**
+ * Prefix start-command 0.33.0 puts on every `exitReason` that reports memory
+ * exhaustion, whatever the mechanism — `memory-exhaustion (v8-heap-limit)`,
+ * `memory-exhaustion (kernel-oom-killer)`, `memory-exhaustion (go-runtime)`,
+ * `memory-exhaustion (allocation-failure)`. Matching the prefix rather than the
+ * exact strings means a new upstream mechanism is classified correctly without
+ * a Hive Mind release. See link-foundation/start#164 and #165.
+ */
+export const UPSTREAM_MEMORY_EXHAUSTION_PREFIX = 'memory-exhaustion';
+
 /** Disk is considered full at or above this used percentage… */
 export const DISK_FULL_USED_PERCENT = 95;
 /** …or below this much free space, whichever triggers first. */
@@ -66,6 +93,22 @@ export function selectLastMemoryResourceMarker(parsed) {
   return null;
 }
 
+/**
+ * The most recent marker that carries a V8 heap reading (issue #2189). Older
+ * logs have none — the heap fields were added with this fix — so every caller
+ * must tolerate `null`.
+ *
+ * @param {{markers: Array}|null} parsed
+ * @returns {Object|null}
+ */
+export function selectLastHeapResourceMarker(parsed) {
+  const markers = Array.isArray(parsed?.markers) ? parsed.markers : [];
+  for (let i = markers.length - 1; i >= 0; i--) {
+    if (finite(markers[i]?.memory?.processHeapUsedBytes) !== null) return markers[i];
+  }
+  return null;
+}
+
 /**
  * The most recent marker that carries usable disk data.
  *
@@ -235,11 +278,15 @@ function describeDisk(disk, timestamp) {
  * @param {number|null} [params.exitCode]
  * @param {Object|null} [params.system] - collectSystemKillDiagnostics() result
  * @param {boolean} [params.stopRequestedByUser] - The operator asked for the stop
+ * @param {boolean|null} [params.reportedMemoryExhausted] - `$ --status` `memoryExhausted` (start-command >= 0.33.0)
+ * @param {string|null} [params.reportedMemoryExhaustedReason] - `$ --status` `memoryExhaustedReason` (the evidence line)
+ * @param {string|null} [params.reportedExitReason] - `$ --status` `exitReason` hint, e.g. `memory-exhaustion (v8-heap-limit)`
  * @returns {{cause: string, summary: string, evidence: string[], memory: Object|null, disk: Object|null, victims: Array}}
  */
-export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false } = {}) {
+export function describeKillCause({ logText = null, resourceMarkers = null, oomKilled = false, exitCode = null, system = null, stopRequestedByUser = false, reportedMemoryExhausted = null, reportedMemoryExhaustedReason = null, reportedExitReason = null } = {}) {
   const parsed = resourceMarkers || (logText ? parseResourceMarkers(logText) : { markers: [], byPhase: {} });
   const memoryMarker = selectLastMemoryResourceMarker(parsed);
+  const heapMarker = selectLastHeapResourceMarker(parsed);
   const diskMarker = selectLastDiskResourceMarker(parsed);
   const memory = memoryMarker?.memory || null;
   const disk = diskMarker?.disk || null;
@@ -249,6 +296,12 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
   const evidence = [];
   const memoryLine = describeMemory(memory, memoryMarker?.timestamp || null);
   if (memoryLine) evidence.push(`last session memory reading — ${memoryLine} (phase \`${memoryMarker.phase}\`)`);
+  // Issue #2189: report the heap against its own limit, which is the number that
+  // decides a "JavaScript heap out of memory" abort — host RAM does not.
+  const heapMemory = heapMarker?.memory || null;
+  const heapUsedPercent = finite(heapMemory?.processHeapUsedPercent);
+  const heapLine = heapMemory ? `${formatHeapUsage(heapMemory)}${heapMarker?.timestamp ? ` at ${heapMarker.timestamp}` : ''}` : null;
+  if (heapLine) evidence.push(`last session V8 heap reading — ${heapLine} (phase \`${heapMarker.phase}\`)`);
   const diskLine = describeDisk(disk, diskMarker?.timestamp || null);
   if (diskLine) evidence.push(`last session ${diskLine} (phase \`${diskMarker.phase}\`)`);
   if (oomKilled) evidence.push('container reports `State.OOMKilled = true` (an OOM event hit the container cgroup)');
@@ -260,8 +313,45 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
     evidence.push(`kernel OOM killer terminated \`${victim.comm || 'unknown'}\` (pid ${victim.pid ?? '?'})`);
   }
 
+  // Issue #2189: the ground truth for a runtime that exhausted its OWN heap is
+  // the fatal line it printed, not the cgroup counters — those legitimately read
+  // zero for a V8 self-abort. Only an abnormal ending is allowed to be upgraded,
+  // so a log that merely mentions the phrase cannot manufacture a diagnosis.
+  const abnormalExit = exitCode === null || exitCode !== 0;
+  const fatalMemoryMarker = abnormalExit ? findFatalMemoryMarker(logText) : null;
+  if (fatalMemoryMarker) {
+    evidence.push(`the ${fatalMemoryMarker.runtime} runtime aborted on its own heap limit: \`${fatalMemoryMarker.line}\` (a self-abort is invisible to \`docker inspect\` and to cgroup OOM counters)`);
+  }
+
+  // Issue #2189 also went upstream: start-command 0.33.0 (link-foundation/start
+  // #164, #165) performs the same tail scan inside `$` and reports it as
+  // `memoryExhausted` / `memoryExhaustedReason` / `exitReason`. Consuming it
+  // catches the case our own scan cannot: the fatal line scrolled out of the
+  // bounded window we read, but `$` saw it when the command exited. The local
+  // scan stays as defense in depth — these fields are absent on an older `$`.
+  const reportedExitReasonText = typeof reportedExitReason === 'string' && reportedExitReason.trim() ? reportedExitReason.trim() : null;
+  const reportedMemoryExhaustion = abnormalExit && (reportedMemoryExhausted === true || (reportedExitReasonText !== null && reportedExitReasonText.startsWith(UPSTREAM_MEMORY_EXHAUSTION_PREFIX)));
+  // `memory-exhaustion (v8-heap-limit)` → `v8-heap-limit`: the prefix is already
+  // said in words by the surrounding sentence, so only the mechanism is new.
+  const reportedMechanism =
+    reportedMemoryExhaustion && reportedExitReasonText
+      ? reportedExitReasonText
+          .slice(UPSTREAM_MEMORY_EXHAUSTION_PREFIX.length)
+          .trim()
+          .replace(/^\((.*)\)$/, '$1') || null
+      : null;
+  if (reportedMemoryExhaustion) {
+    const detail = reportedMemoryExhaustedReason ? `: \`${reportedMemoryExhaustedReason}\`` : '';
+    evidence.push(`\`$ --status\` reports memory exhaustion${reportedMechanism ? ` (\`${reportedMechanism}\`)` : ''}${detail}`);
+  } else if (reportedExitReasonText && abnormalExit) {
+    evidence.push(`\`$ --status\` reports \`exitReason = ${reportedExitReasonText}\``);
+  }
+
   const ratio = memoryRatio(memory);
   const memoryExhausted = ratio !== null && ratio <= MEMORY_EXHAUSTED_AVAILABLE_RATIO;
+  // A heap already at the limit is only evidence of a kill when the session
+  // actually ended abnormally — a healthy run may legitimately end near its cap.
+  const heapExhausted = abnormalExit && heapUsedPercent !== null && heapUsedPercent >= HEAP_EXHAUSTED_PERCENT;
   const diskUsedPercent = finite(disk?.usedPercent);
   const diskAvailable = finite(disk?.availableBytes);
   const diskFull = (diskUsedPercent !== null && diskUsedPercent >= DISK_FULL_USED_PERCENT) || (diskAvailable !== null && diskAvailable <= DISK_FULL_AVAILABLE_BYTES);
@@ -269,7 +359,7 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
   let cause = KILL_CAUSE_UNKNOWN;
   if (stopRequestedByUser) {
     cause = KILL_CAUSE_FORCED_KILL;
-  } else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted) {
+  } else if (victims.length > 0 || (cgroupOomKills !== null && cgroupOomKills > 0) || oomKilled || memoryExhausted || fatalMemoryMarker || heapExhausted || reportedMemoryExhaustion) {
     cause = KILL_CAUSE_OUT_OF_MEMORY;
   } else if (diskFull) {
     cause = KILL_CAUSE_DISK_FULL;
@@ -278,7 +368,21 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
   }
 
   let summary;
-  if (cause === KILL_CAUSE_OUT_OF_MEMORY) {
+  if (cause === KILL_CAUSE_OUT_OF_MEMORY && fatalMemoryMarker && victims.length === 0 && !oomKilled && !(cgroupOomKills > 0)) {
+    // Issue #2189: appending "10.3 GB of 11.7 GB RAM available" to "out of
+    // memory" reads as a contradiction. For a runtime self-abort the host being
+    // healthy is the whole point, so say which limit was actually hit.
+    summary = `out of memory — the ${fatalMemoryMarker.runtime} runtime hit its own heap limit, not the machine's: \`${fatalMemoryMarker.line}\`${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
+  } else if (cause === KILL_CAUSE_OUT_OF_MEMORY && heapExhausted && victims.length === 0 && !oomKilled && !(cgroupOomKills > 0) && !memoryExhausted) {
+    // Same shape as the fatal-marker case, but reconstructed from telemetry when
+    // the fatal line itself was lost (truncated tail, killed before flushing).
+    summary = `out of memory — the runtime's own heap was exhausted: ${heapLine}${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
+  } else if (cause === KILL_CAUSE_OUT_OF_MEMORY && reportedMemoryExhaustion && victims.length === 0 && !oomKilled && !(cgroupOomKills > 0) && !memoryExhausted) {
+    // Only `$` saw the evidence (our bounded window missed the fatal line).
+    // Quote what it saw rather than falling back to the host-memory phrasing,
+    // which would again read as a contradiction on a healthy machine.
+    summary = `out of memory — start-command reported memory exhaustion${reportedMechanism ? ` (${reportedMechanism})` : ''}${reportedMemoryExhaustedReason ? `: \`${reportedMemoryExhaustedReason}\`` : ''}${memoryLine ? ` (host memory was fine: ${memoryLine})` : ''}`;
+  } else if (cause === KILL_CAUSE_OUT_OF_MEMORY) {
     const victim = victims.length > 0 ? `, kernel OOM killer terminated \`${victims[victims.length - 1].comm || 'unknown'}\` (pid ${victims[victims.length - 1].pid ?? '?'})` : '';
     summary = `out of memory${memoryLine ? ` — ${memoryLine}` : ''}${victim}`;
   } else if (cause === KILL_CAUSE_DISK_FULL) {
@@ -290,7 +394,7 @@ export function describeKillCause({ logText = null, resourceMarkers = null, oomK
     summary = 'unknown — no resource marker, cgroup counter or kernel OOM report was available';
   }
 
-  return { cause, summary, evidence, memory, disk, victims };
+  return { cause, summary, evidence, memory, heap: heapMemory, heapUsedPercent, disk, victims, fatalMemoryMarker, reportedMemoryExhaustion, reportedExitReason: reportedExitReasonText };
 }
 
 /**
@@ -367,18 +471,19 @@ export function formatKillResumeSection({ sessionId = null, attempt = null, maxA
  * @param {Object} [options]
  * @returns {Promise<{section: string, diagnosis: Object|null}>}
  */
-export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics } = {}) {
+export async function buildKillDiagnosticsSection(logPath, { verbose = false, readFile = fsPromises.readFile, maxLogBytes = KILL_DIAGNOSTICS_LOG_BYTES, oomKilled = false, exitCode = null, stopRequestedByUser = false, locale = null, collectSystem = collectSystemKillDiagnostics, reportedMemoryExhausted = null, reportedMemoryExhaustedReason = null, reportedExitReason = null } = {}) {
   try {
     let logText = '';
     if (logPath) {
-      try {
-        logText = await readFile(logPath, 'utf8');
-      } catch (readError) {
-        if (verbose) console.log(`[VERBOSE] kill-diagnostics: could not read session log ${logPath}: ${readError?.message || readError}`);
-      }
+      // Issue #2189: this used to be `readFile(logPath, 'utf8')`. The bot ran it
+      // once per monitor tick for a session it never marked handled, so a 134 MB
+      // transcript was pulled into the bot's own heap over and over. Everything
+      // this function needs — the `📈 [RESOURCES]` markers and the runtime's
+      // dying `FATAL ERROR` line — lives at the two ends of the transcript.
+      logText = await readLogTextBounded(logPath, { readFile, maxBytes: maxLogBytes, verbose });
     }
     const system = await collectSystem({ verbose });
-    const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser });
+    const diagnosis = describeKillCause({ logText, oomKilled, exitCode, system, stopRequestedByUser, reportedMemoryExhausted, reportedMemoryExhaustedReason, reportedExitReason });
     if (verbose) console.log(`[VERBOSE] kill-diagnostics: cause=${diagnosis.cause} — ${diagnosis.summary}`);
     return { section: formatKillDiagnosticsSection(diagnosis, { locale }), diagnosis };
   } catch (error) {

package/src/session-kill-policy.lib.mjs

@@ -5,25 +5,36 @@
  * identically — the Telegram completion message and the pull-request notice must
  * never disagree about what happened:
  *
- *   - `report` (default, today's behaviour): the kill is terminal. The Telegram
- *     message says the session was killed, with the diagnosed cause, and offers
- *     the resume command. The pull request gets the same notice.
- *   - `resume`: the kill is treated as recoverable. A new working session is
- *     started from the last tool session id, and BOTH surfaces say so
- *     ("recovered from out of memory" / "a new working session was started").
+ *   - `report`: the kill is terminal. The Telegram message says the session was
+ *     killed, with the diagnosed cause, and offers the resume command. The pull
+ *     request gets the same notice.
+ *   - `resume` (default since issue #2189): the kill is treated as recoverable.
+ *     A new working session is started from the last tool session id, and BOTH
+ *     surfaces say so ("recovered from out of memory" / "a new working session
+ *     was started").
  *
  * Selected by `--on-session-kill=<policy>` or `HIVE_MIND_ON_SESSION_KILL`, with
  * the CLI flag winning over the environment. Nothing is removed by choosing one
  * over the other: `resume` still reports the kill and its cause, it just adds
  * the recovery, and log uploads stay gated on `--attach-logs` in both modes.
  *
+ * Why `resume` is the default (issue #2189): under `report` the bot only ever
+ * *offered* a resume command that a human had to notice and paste. In the
+ * captured incident the offer reached the operator six hours after the crash,
+ * and the work sat abandoned in between. "The bot should initiate the resume
+ * itself with context preserved" — so it does, bounded by
+ * `--session-kill-resume-attempts` (default 1) so a job that reliably dies still
+ * cannot storm. `--on-session-kill=report` restores the announce-only
+ * behaviour verbatim for anyone who wants it.
+ *
  * @see https://github.com/link-assistant/hive-mind/issues/2134
+ * @see https://github.com/link-assistant/hive-mind/issues/2189
  */
 
 export const ON_SESSION_KILL_REPORT = 'report';
 export const ON_SESSION_KILL_RESUME = 'resume';
 export const ON_SESSION_KILL_POLICIES = [ON_SESSION_KILL_REPORT, ON_SESSION_KILL_RESUME];
-export const DEFAULT_ON_SESSION_KILL_POLICY = ON_SESSION_KILL_REPORT;
+export const DEFAULT_ON_SESSION_KILL_POLICY = ON_SESSION_KILL_RESUME;
 
 export const ON_SESSION_KILL_ENV_VAR = 'HIVE_MIND_ON_SESSION_KILL';