@link-assistant/hive-mind 2.1.5 → 2.1.6

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/hive-mind
 
+## 2.1.6
+
+### Patch Changes
+
+- efcbcce: Handle Docker `oomKilled` status markers as terminal Telegram work-session failures, delay Docker backend-gone killed notifications long enough for start-command to publish a real terminal status or log footer, pace queued task startups at a minimum 10-minute interval, and cap system resource cache freshness at 1 minute.
+
 ## 2.1.5
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "2.1.5",
+  "version": "2.1.6",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/config.lib.mjs

@@ -687,8 +687,9 @@ export const cacheTtl = {
   // because users still hit "Resets in 3m xs" rate-limit responses. The API
   // returns null values or 429 when called too frequently.
   usageApi: parseIntWithDefault('HIVE_MIND_USAGE_API_CACHE_TTL_MS', 13 * 60 * 1000), // 13 minutes
-  // System metrics cache TTL (RAM, CPU, disk)
-  system: parseIntWithDefault('HIVE_MIND_SYSTEM_CACHE_TTL_MS', 2 * 60 * 1000), // 2 minutes
+  // System metrics cache TTL (RAM, CPU, disk). Issue #2015 caps this at
+  // 1 minute so queue decisions do not use stale host pressure data.
+  system: Math.min(parseIntWithDefault('HIVE_MIND_SYSTEM_CACHE_TTL_MS', 60 * 1000), 60 * 1000), // max 1 minute
 };
 
 // File and path configurations

package/src/isolation-runner.lib.mjs

@@ -321,9 +321,18 @@ export function generateSessionId() {
 export function parseSessionStatusOutput(output) {
   const raw = (output || '').trim();
   if (!raw) {
-    return { exists: false, uuid: null, status: null, exitCode: null, startTime: null, endTime: null, currentTime: null, logPath: null, command: null, isolation: null, workingDirectory: null, sessionName: null, processIds: {}, raw: '' };
+    return { exists: false, uuid: null, status: null, exitCode: null, startTime: null, endTime: null, currentTime: null, logPath: null, command: null, isolation: null, workingDirectory: null, sessionName: null, processIds: {}, oomKilled: null, raw: '' };
   }
 
+  const normalizeBooleanField = value => {
+    if (typeof value === 'boolean') return value;
+    if (value === null || value === undefined) return null;
+    const normalized = String(value).trim().toLowerCase();
+    if (['true', '1', 'yes'].includes(normalized)) return true;
+    if (['false', '0', 'no'].includes(normalized)) return false;
+    return null;
+  };
+
   try {
     const parsed = JSON.parse(raw);
     const data = Array.isArray(parsed) ? parsed[0] : parsed;
@@ -350,6 +359,7 @@ export function parseSessionStatusOutput(output) {
       workingDirectory: data?.workingDirectory || null,
       sessionName: data?.sessionName || data?.options?.sessionName || null,
       processIds,
+      oomKilled: normalizeBooleanField(data?.oomKilled ?? data?.OOMKilled ?? data?.options?.oomKilled ?? data?.state?.oomKilled ?? data?.State?.OOMKilled),
       raw,
     };
   } catch {
@@ -365,6 +375,7 @@ export function parseSessionStatusOutput(output) {
     const match = raw.match(new RegExp(`^\\s*${name}\\s+"?([^"\\n]+)"?\\s*$`, 'mi'));
     return match ? match[1].trim() : null;
   };
+  const readBooleanField = name => normalizeBooleanField(readField(name));
 
   const status = readField('status')?.toLowerCase() || null;
   const exitCodeText = readField('exitCode');
@@ -396,6 +407,7 @@ export function parseSessionStatusOutput(output) {
     workingDirectory: readField('workingDirectory'),
     sessionName: readField('sessionName'),
     processIds,
+    oomKilled: readBooleanField('oomKilled'),
     raw,
   };
 }

package/src/limits.lib.mjs

@@ -1239,12 +1239,12 @@ export function formatCodexLimitsSection(codexLimits, codexError = null, options
  * Configurable via environment variables:
  * - HIVE_MIND_API_CACHE_TTL_MS: General API cache TTL (default: 180000 = 3 minutes)
  * - HIVE_MIND_USAGE_API_CACHE_TTL_MS: Claude Usage API cache TTL (default: 780000 = 13 minutes)
- * - HIVE_MIND_SYSTEM_CACHE_TTL_MS: System metrics cache TTL (default: 120000 = 2 minutes)
+ * - HIVE_MIND_SYSTEM_CACHE_TTL_MS: System metrics cache TTL (default: 60000 = 1 minute, capped at 1 minute)
  */
 export const CACHE_TTL = {
   API: cacheTtl.api, // 3 minutes for regular API calls (GitHub)
   USAGE_API: cacheTtl.usageApi, // 13 minutes for Claude Usage API (rate limited)
-  SYSTEM: cacheTtl.system, // 2 minutes for system metrics (RAM, CPU, disk)
+  SYSTEM: cacheTtl.system, // max 1 minute for system metrics (RAM, CPU, disk)
 };
 
 /**

package/src/locales/en.lino

@@ -623,6 +623,7 @@ en
         executing "⏳ Executing..."
         finished "Work session finished successfully"
         failed "Work session failed (exit code: {{exitCode}})"
+        killed "Work session {{reason}}{{exitSuffix}}"
     duration
       label "Duration"
     session

package/src/locales/hi.lino

@@ -623,6 +623,7 @@ hi
         executing "⏳ चल रहा है..."
         finished "कार्य सत्र सफलतापूर्वक पूरा हुआ"
         failed "कार्य सत्र विफल हुआ (exit code: {{exitCode}})"
+        killed "कार्य सत्र रोका गया: {{reason}}{{exitSuffix}}"
     duration
       label "अवधि"
     session

package/src/locales/ru.lino

@@ -623,6 +623,7 @@ ru
         executing "⏳ Выполняется..."
         finished "Рабочий сеанс успешно завершен"
         failed "Рабочий сеанс завершился с ошибкой (код выхода: {{exitCode}})"
+        killed "Рабочий сеанс остановлен: {{reason}}{{exitSuffix}}"
     duration
       label "Длительность"
     session

package/src/locales/zh.lino

@@ -623,6 +623,7 @@ zh
         executing "⏳ 正在执行..."
         finished "工作会话已成功完成"
         failed "工作会话失败（退出代码：{{exitCode}}）"
+        killed "工作会话已终止：{{reason}}{{exitSuffix}}"
     duration
       label "耗时"
     session

package/src/queue-config.lib.mjs

@@ -192,6 +192,8 @@ const parseIntWithDefault = (envVar, defaultValue) => {
   return isNaN(parsed) ? defaultValue : parsed;
 };
 
+const MINIMUM_START_INTERVAL_MS = 10 * 60 * 1000;
+
 // Parse links notation config from environment variable (if provided)
 const linoConfig = parseQueueConfig(getenv('HIVE_MIND_QUEUE_CONFIG', ''));
 
@@ -271,9 +273,10 @@ export const QUEUE_CONFIG = {
   GITHUB_API_THRESHOLD: getThresholdConfig('githubApi', 'HIVE_MIND_GITHUB_API_THRESHOLD', 'HIVE_MIND_GITHUB_API_STRATEGY', 0.5, 'enqueue').value,
 
   // Timing
-  // MIN_START_INTERVAL_MS: Time to allow solve command to start actual claude process
-  // This ensures that when API limits are checked, the running process is counted
-  MIN_START_INTERVAL_MS: parseIntWithDefault('HIVE_MIND_MIN_START_INTERVAL_MS', 60000), // 1 minute between starts
+  // MIN_START_INTERVAL_MS: Minimum global spacing between task startups.
+  // Issue #2015: after resource thresholds clear, starting a backlog in a burst
+  // can kill the next batch before host metrics have time to settle.
+  MIN_START_INTERVAL_MS: Math.max(parseIntWithDefault('HIVE_MIND_MIN_START_INTERVAL_MS', MINIMUM_START_INTERVAL_MS), MINIMUM_START_INTERVAL_MS), // at least 10 minutes between starts
   CONSUMER_POLL_INTERVAL_MS: parseIntWithDefault('HIVE_MIND_CONSUMER_POLL_INTERVAL_MS', 60000), // 1 minute between queue checks
   MESSAGE_UPDATE_INTERVAL_MS: parseIntWithDefault('HIVE_MIND_MESSAGE_UPDATE_INTERVAL_MS', 60000), // 1 minute between status message updates
 

package/src/session-monitor.lib.mjs

@@ -24,7 +24,7 @@ import fs from 'fs/promises';
 import { promisify } from 'util';
 import { formatSessionCompletionMessage, getSessionCompletionExitCode, classifySessionOutcome } from './work-session-formatting.lib.mjs';
 import { notifySubscribers, getSubscriberCount } from './telegram-subscribers.lib.mjs';
-import { classifyExitStatus } from './session-status.lib.mjs';
+import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
 import path from 'node:path';
 import { readLastSessionIdFromLog, findLatestSessionLogId, buildResumeCommand, formatResumeSection } from './session-resume.lib.mjs';
 
@@ -504,6 +504,8 @@ function isNonIsolationSessionActive(sessionName, sessionInfo, verbose = false)
  * this, because a written "Exit Code:" footer is proof the command terminated.
  */
 export const STALE_EXECUTING_MIN_AGE_MS = 90 * 1000;
+export const DOCKER_BACKEND_GONE_GRACE_MS = 2 * 60 * 1000;
+const DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD = 'dockerBackendGoneFirstSeenAt';
 
 function sessionStartMs(sessionInfo) {
   const start = sessionInfo?.startTime;
@@ -513,6 +515,23 @@ function sessionStartMs(sessionInfo) {
   return Number.isFinite(ms) ? ms : null;
 }
 
+function isDockerIsolation(sessionInfo, statusResult) {
+  return sessionInfo?.isolationBackend === 'docker' || statusResult?.isolation === 'docker';
+}
+
+function getDockerBackendGoneFirstSeenMs(sessionInfo) {
+  const raw = sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
+  if (!raw) return null;
+  const ms = new Date(raw).getTime();
+  return Number.isFinite(ms) ? ms : null;
+}
+
+function clearDockerBackendGoneMarker(sessionName, sessionInfo) {
+  if (!sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD]) return;
+  delete sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
+  persistSessionSnapshot(sessionName, sessionInfo);
+}
+
 /**
  * Cross-check whether a session that `$ --status` still reports as `executing`
  * has actually terminated. Issue #1927: start-command's status can get stuck on
@@ -552,13 +571,60 @@ async function resolveStaleExecutingState(sessionName, sessionInfo, statusResult
     // Only `false` (definitively gone) counts as killed; `null` (unknown backend)
     // is treated as "no signal" so we don't kill on an indeterminate probe.
     if (alive === false) {
+      if (isDockerIsolation(sessionInfo, statusResult)) {
+        const nowMs = Date.now();
+        const firstSeenMs = getDockerBackendGoneFirstSeenMs(sessionInfo);
+        if (firstSeenMs === null) {
+          sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
+          persistSessionSnapshot(sessionName, sessionInfo);
+          if (verbose) {
+            console.log(`[VERBOSE] Session ${sessionName} docker backend is gone but no terminal status/footer is available yet; deferring killed classification for ${DOCKER_BACKEND_GONE_GRACE_MS}ms`);
+          }
+          return null;
+        }
+        if (nowMs - firstSeenMs < DOCKER_BACKEND_GONE_GRACE_MS) {
+          if (verbose) {
+            console.log(`[VERBOSE] Session ${sessionName} docker backend is still gone; waiting for terminal status/footer before reporting killed`);
+          }
+          return null;
+        }
+      }
       return { exitCode: null, status: 'killed', reason: 'backend-gone' };
     }
+    if (alive === true) {
+      clearDockerBackendGoneMarker(sessionName, sessionInfo);
+    }
   }
 
   return null;
 }
 
+function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
+  const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
+  let footer = null;
+  if (logPath) {
+    const readFooter = exitFromLog || runner.readSessionExitFromLog;
+    footer = readFooter ? readFooter(logPath, { verbose }) : null;
+  }
+
+  const statusExitCode = normalizeExitCode(statusResult?.exitCode);
+  const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
+  let exitCode = 137;
+  if (statusExitCode !== null && statusExitCode > 0) {
+    exitCode = statusExitCode;
+  } else if (footerExitCode !== null && footerExitCode > 0) {
+    exitCode = footerExitCode;
+  }
+  const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
+  const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
+
+  if (verbose) {
+    console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
+  }
+
+  return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
+}
+
 async function getIsolationSessionState(sessionName, sessionInfo, options = {}) {
   const { verbose = false, statusProvider = null, exitFromLog = null, backendAlive = null, sessionRunning = null } = options;
   const sessionId = sessionInfo.sessionId || sessionName;
@@ -568,6 +634,9 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
     const statusResult = statusProvider ? await statusProvider(sessionId, sessionInfo) : await runner.querySessionStatus(sessionId, verbose);
 
     if (statusResult?.exists && statusResult.status) {
+      if (statusResult.oomKilled === true) {
+        return resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog });
+      }
       if (runner.isExecutingSessionStatus(statusResult.status)) {
         // Issue #1927: an `executing` status is not trusted blindly — verify the
         // process is really alive. start-command can keep reporting `executing`

package/src/telegram-solve-queue.lib.mjs

@@ -374,9 +374,11 @@ export class SolveQueue {
   }
 
   /**
-   * Find startable items from each tool queue
-   * Returns the first item from each tool queue that can start.
-   * With separate queues, each tool is checked independently so they don't block each other.
+   * Find the next startable item across all tool queues.
+   * With separate queues, each tool is checked independently so tool-specific
+   * limits do not block unrelated tools. Issue #2015 adds a global startup
+   * interval, so even when multiple tools are startable this returns only the
+   * oldest startable item to prevent burst launches.
    *
    * Also immediately rejects all queued items when a 'reject' strategy threshold
    * is exceeded, instead of leaving them waiting indefinitely.
@@ -417,7 +419,8 @@ export class SolveQueue {
       }
     }
 
-    return startableItems;
+    startableItems.sort((a, b) => a.item.createdAt - b.item.createdAt);
+    return startableItems.slice(0, 1);
   }
 
   /**
@@ -571,10 +574,10 @@ export class SolveQueue {
     let rejected = false;
     let rejectReason = null;
 
-    // Check minimum interval since last start FOR THIS TOOL
-    // Each tool queue has independent timing to prevent cross-blocking
-    // See: https://github.com/link-assistant/hive-mind/issues/1159
-    const lastStartTime = this.lastStartTimeByTool[tool] || null;
+    // Check minimum interval since the last task start globally.
+    // Issue #2015: do not let another tool queue bypass startup pacing; host
+    // CPU/RAM/disk metrics need time to settle before any next task starts.
+    const lastStartTime = this.lastStartTime || null;
     if (lastStartTime) {
       const timeSinceLastStart = Date.now() - lastStartTime;
       if (timeSinceLastStart < QUEUE_CONFIG.MIN_START_INTERVAL_MS) {
@@ -765,7 +768,6 @@ export class SolveQueue {
     }
 
     // Check CPU using 5-minute load average (more stable than 1-minute)
-    // Cache TTL is 2 minutes, which is appropriate for this metric
     const cpuResult = await getCachedCpuInfo(this.verbose);
     if (cpuResult.success) {
       // Use loadAvg5 (5-minute average) instead of usagePercentage (1-minute based)

package/src/work-session-formatting.lib.mjs

@@ -145,7 +145,7 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
     const showCode = finalExitCode !== null && !(!signal && finalExitCode === 1);
     const exitSuffix = showCode ? ` (exit code: ${finalExitCode})` : '';
     const reason = signal ? signal.reason : 'killed';
-    statusText = text(messageLocale, 'telegram.work_session_killed', `Work session ${reason}${exitSuffix}`, { reason, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '' });
+    statusText = text(messageLocale, 'telegram.work_session_killed', `Work session ${reason}${exitSuffix}`, { reason, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '', exitSuffix });
   } else if (failed) {
     statusText = text(messageLocale, 'telegram.work_session_failed', `Work session failed (exit code: ${finalExitCode})`, { exitCode: finalExitCode });
   } else {