@link-assistant/hive-mind 2.5.2 → 2.5.4

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/hive-mind
 
+## 2.5.4
+
+### Patch Changes
+
+- 2ebdb3d: Add dequeue-decision diagnostics to the Telegram solve queue so global FIFO ordering across the per-tool queues can be audited in production (issue #2051). The oldest startable task still wins the single, globally-paced startup slot; when an older task is skipped because it cannot start, the queue now logs a concise, deduplicated "FIFO queue-jump" line naming the older task and the exact reason it is blocked (Claude/Codex limits, RAM/CPU/disk, min-interval, or one-at-a-time), and records it on `stats.lastQueueJump`. Verbose mode additionally prints a per-tool head snapshot each cycle. No change to ordering, pacing, or the minimum start interval.
+
+## 2.5.3
+
+### Patch Changes
+
+- eb755b5: Recognize an operator-initiated `/stop` as "🛑 Work session stopped by user" instead of "killed — out of memory or forced kill (SIGKILL)" (issue #2052). The Telegram `/stop <uuid>` flow now records the stop via `markSessionStopRequested` before forwarding CTRL+C, so the resulting SIGTERM/SIGKILL exit (143/137) is reported as an intentional user stop. Adds `--verbose` interrupt timing traces (auto-commit vs log-upload) to make the `docker stop` grace-period race behind "no log uploaded on stop" measurable, plus a case study under `docs/case-studies/issue-2052`.
+
 ## 2.5.2
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "2.5.2",
+  "version": "2.5.4",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/locales/en.lino

@@ -624,6 +624,7 @@ en
         finished "Work session finished successfully"
         failed "Work session failed (exit code: {{exitCode}})"
         killed "Work session {{reason}}{{exitSuffix}}"
+        stopped "Work session stopped by user{{requestedBy}}{{exitSuffix}}"
     duration
       label "Duration"
     session

package/src/locales/hi.lino

@@ -624,6 +624,7 @@ hi
         finished "कार्य सत्र सफलतापूर्वक पूरा हुआ"
         failed "कार्य सत्र विफल हुआ (exit code: {{exitCode}})"
         killed "कार्य सत्र रोका गया: {{reason}}{{exitSuffix}}"
+        stopped "कार्य सत्र उपयोगकर्ता द्वारा रोका गया{{requestedBy}}{{exitSuffix}}"
     duration
       label "अवधि"
     session

package/src/locales/ru.lino

@@ -624,6 +624,7 @@ ru
         finished "Рабочий сеанс успешно завершен"
         failed "Рабочий сеанс завершился с ошибкой (код выхода: {{exitCode}})"
         killed "Рабочий сеанс остановлен: {{reason}}{{exitSuffix}}"
+        stopped "Рабочий сеанс остановлен пользователем{{requestedBy}}{{exitSuffix}}"
     duration
       label "Длительность"
     session

package/src/locales/zh.lino

@@ -624,6 +624,7 @@ zh
         finished "工作会话已成功完成"
         failed "工作会话失败（退出代码：{{exitCode}}）"
         killed "工作会话已终止：{{reason}}{{exitSuffix}}"
+        stopped "工作会话已由用户停止{{requestedBy}}{{exitSuffix}}"
     duration
       label "耗时"
     session

package/src/session-monitor.lib.mjs

@@ -217,6 +217,33 @@ export function getTrackedSessionInfo(sessionName) {
   return activeSessions.get(sessionName) || null;
 }
 
+/**
+ * Issue #2052: record that an operator explicitly requested a session stop
+ * (e.g. Telegram `/stop <uuid>`). The subsequent SIGTERM/SIGKILL exit (143/137,
+ * delivered by `docker stop`) is then reported as "🛑 Stopped by user" instead
+ * of the misleading "out of memory or forced kill (SIGKILL)". Matches by
+ * tracking key OR by the isolation `sessionId` UUID (which is how `/stop`
+ * addresses sessions).
+ * @param {string} sessionId - UUID or session name of the session being stopped
+ * @param {{requestedBy?: string|null, verbose?: boolean}} [opts]
+ * @returns {boolean} True when a tracked session was marked.
+ */
+export function markSessionStopRequested(sessionId, { requestedBy = null, verbose = false } = {}) {
+  if (!sessionId) return false;
+  const target = activeSessions.get(sessionId) || Array.from(activeSessions.values()).find(info => info?.sessionId === sessionId) || null;
+  if (!target) {
+    if (verbose) console.log(`[VERBOSE] markSessionStopRequested: no tracked session found for ${sessionId}`);
+    return false;
+  }
+  target.stopRequestedByUser = true;
+  if (requestedBy) target.stopRequestedBy = requestedBy;
+  const key = target.sessionId || sessionId;
+  persistSessionSnapshot(key, target);
+  if (verbose) console.log(`[VERBOSE] markSessionStopRequested: ${sessionId} marked stopped by user${requestedBy ? ` (${requestedBy})` : ''}`);
+  logEvent('session_stop_requested', { sessionName: key, sessionId, requestedBy: requestedBy || null });
+  return true;
+}
+
 /**
  * Stop tracking a session that was registered optimistically but never actually
  * started (e.g. the start-command launch failed). Removes it from the in-memory

package/src/solve.interrupt.lib.mjs

@@ -22,10 +22,24 @@ export const createInterruptWrapper = ({ cleanupContext, checkForUncommittedChan
     const ctx = cleanupContext;
     if (!ctx.tempDir || !ctx.argv) return;
 
+    // Issue #2052: the interrupt handler races the isolation backend's grace
+    // period — `docker stop` sends SIGTERM, waits ~10s, then SIGKILL. Auto-commit
+    // is fast, but attaching a multi-MB log to a Gist/PR can take several seconds
+    // and may be cut off by SIGKILL, which is exactly "no log uploaded on stop".
+    // These verbose timing traces make the race measurable on the next iteration
+    // (they are silent unless --verbose is set).
+    const verbose = ctx.argv.verbose || false;
+    const startedAt = Date.now();
+    const trace = async message => {
+      if (verbose) await log(`[interrupt] +${Date.now() - startedAt}ms ${message}`, { verbose: true });
+    };
+    await trace('handler entered');
+
     await log('\n⚠️  Session interrupted by user (CTRL+C)');
 
     // Always auto-commit uncommitted changes on CTRL+C to preserve work
     if (ctx.branchName) {
+      await trace('auto-commit: start');
       try {
         await checkForUncommittedChanges(
           ctx.tempDir,
@@ -37,6 +51,7 @@ export const createInterruptWrapper = ({ cleanupContext, checkForUncommittedChan
           true, // always autoCommit on CTRL+C to preserve work
           false // no autoRestart
         );
+        await trace('auto-commit: done');
       } catch (commitError) {
         await log(`⚠️  Could not auto-commit changes on interrupt: ${commitError.message}`, {
           level: 'warning',
@@ -47,6 +62,7 @@ export const createInterruptWrapper = ({ cleanupContext, checkForUncommittedChan
     // Upload logs if --attach-logs is enabled and we have a PR
     if (shouldAttachLogs && ctx.prNumber && ctx.owner && ctx.repo) {
       await log('📎 Uploading interrupted session logs to Pull Request...');
+      await trace('log-upload: start');
       try {
         await attachLogToGitHub({
           logFile: getLogFile(),
@@ -63,6 +79,7 @@ export const createInterruptWrapper = ({ cleanupContext, checkForUncommittedChan
           requestedModel: ctx.argv.model,
           tool: ctx.argv.tool || 'claude',
         });
+        await trace('log-upload: done');
       } catch (uploadError) {
         await log(`⚠️  Could not upload logs on interrupt: ${uploadError.message}`, {
           level: 'warning',

package/src/telegram-solve-queue.helpers.lib.mjs

@@ -492,3 +492,61 @@ export function formatWaitingReason(metric, currentValue, threshold, options = {
       return `${metric} threshold exceeded`;
   }
 }
+
+/**
+ * Report the dequeue decision across all tool queues for a SolveQueue.
+ *
+ * Provides the observability that issue #2051 needs to confirm whether a
+ * long-waiting task is blocked legitimately (limits/resources/pacing) or by an
+ * ordering defect:
+ * - In verbose mode: prints a per-tool head snapshot (age + startable +
+ *   blocking reasons) and the selected item each cycle.
+ * - Always-on: emits a concise "FIFO queue-jump" line whenever the
+ *   globally-oldest queued head is skipped in favor of a younger startable head,
+ *   naming the older task and the exact reason it is blocked. Deduplicated by
+ *   (task id + block reasons) so a persistently-blocked task is only reported
+ *   when its situation changes.
+ *
+ * @param {Object} queue - SolveQueue-like instance (uses verbose, log, recordThrottle, stats).
+ * @param {Array<{tool: string, item: Object, ageMs: number, startable: boolean, blockReasons: string[]}>} headDiagnostics
+ * @param {{item: Object, tool: string}|undefined} selected
+ * @see https://github.com/link-assistant/hive-mind/issues/2051
+ */
+export function reportDequeueDecision(queue, headDiagnostics, selected) {
+  if (!headDiagnostics || headDiagnostics.length === 0) return;
+
+  if (queue.verbose) {
+    queue.log('Dequeue decision (global FIFO across tool queues):');
+    for (const d of headDiagnostics) {
+      const detail = d.startable ? 'STARTABLE' : `blocked by [${d.blockReasons.join('; ') || 'unknown'}]`;
+      queue.log(`  ${d.tool}: waited ${formatDuration(d.ageMs)} - ${detail}`);
+    }
+    queue.log(selected ? `  -> selected ${selected.tool} (${selected.item.url})` : '  -> nothing startable this cycle');
+  }
+
+  if (!selected) return;
+
+  // Identify the globally-oldest queued head regardless of startability.
+  const oldest = headDiagnostics.reduce((a, b) => (a.item.createdAt <= b.item.createdAt ? a : b));
+  if (oldest.item.id === selected.item.id) return; // Strict FIFO honored - nothing to report.
+
+  const waitedMs = Date.now() - oldest.item.createdAt;
+  const blockedBy = oldest.blockReasons.join('; ') || 'unknown';
+  queue.recordThrottle('fifo_queue_jump');
+  queue.stats.lastQueueJump = {
+    skippedTool: oldest.tool,
+    skippedUrl: oldest.item.url,
+    waitedMs,
+    blockedBy: oldest.blockReasons,
+    startedTool: selected.tool,
+    startedUrl: selected.item.url,
+  };
+
+  // Deduplicate the always-on notice so a persistently-blocked task does not
+  // spam the log on every poll; only report when the reason changes.
+  const signature = `${oldest.item.id}|${blockedBy}`;
+  if (queue._lastQueueJumpSignature !== signature) {
+    queue._lastQueueJumpSignature = signature;
+    console.log(`[solve_queue] FIFO queue-jump: ${selected.tool} task started ahead of an older ${oldest.tool} task waiting ${formatDuration(waitedMs)} (${oldest.item.url}) — older task blocked by: ${blockedBy}`);
+  }
+}

package/src/telegram-solve-queue.lib.mjs

@@ -2,22 +2,17 @@
 /**
  * Telegram Solve Queue Library
  *
- * Producer/consumer queue for /solve commands in the Telegram bot.
- * Implements resource-aware throttling to prevent system overload.
- *
- * Features:
- * - Resource checking (RAM, CPU, disk)
- * - API limit checking (Claude, GitHub)
- * - Minimum interval between command starts
- * - Running process detection
- * - Status tracking: Queued -> Waiting -> Starting -> Started
+ * Producer/consumer queue for /solve commands in the Telegram bot. Implements
+ * resource-aware throttling (RAM, CPU, disk), API limit checking (Claude,
+ * GitHub), a minimum interval between command starts, running-process
+ * detection, and status tracking (Queued -> Waiting -> Starting -> Started).
  *
  * @see https://github.com/link-assistant/hive-mind/issues/1041
  */
 
 import { getCachedClaudeLimits, getCachedCodexLimits, getCachedGitHubLimits, getCachedMemoryInfo, getCachedCpuInfo, getCachedDiskInfo, getLimitCache } from './limits.lib.mjs';
 export { formatDuration, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses } from './telegram-solve-queue.helpers.lib.mjs';
-import { collectExecutingItems, formatDuration, formatQueueToolSection, formatWaitingReason, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses, getRunningSessionItems, groupQueueItemsByTool } from './telegram-solve-queue.helpers.lib.mjs';
+import { collectExecutingItems, formatDuration, formatQueueToolSection, formatWaitingReason, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses, getRunningSessionItems, groupQueueItemsByTool, reportDequeueDecision } from './telegram-solve-queue.helpers.lib.mjs';
 export { QUEUE_CONFIG, THRESHOLD_STRATEGIES } from './queue-config.lib.mjs';
 import { QUEUE_CONFIG } from './queue-config.lib.mjs';
 import { reserveStartSlotForQueue } from './queue-start-reservation.lib.mjs';
@@ -190,8 +185,7 @@ export class SolveQueue {
       qwen: null,
       gemini: null,
     };
-    // Legacy: keep for compatibility with existing code that uses lastStartTime
-    this.lastStartTime = null;
+    this.lastStartTime = null; // Legacy: global last-start timestamp
 
     // Consumer task reference
     this.consumerTask = null;
@@ -385,14 +379,12 @@ export class SolveQueue {
   }
 
   /**
-   * Find the next startable item across all tool queues.
-   * With separate queues, each tool is checked independently so tool-specific
-   * limits do not block unrelated tools. Issue #2015 adds a global startup
-   * interval, so even when multiple tools are startable this returns only the
-   * oldest startable item to prevent burst launches.
-   *
-   * Also immediately rejects all queued items when a 'reject' strategy threshold
-   * is exceeded, instead of leaving them waiting indefinitely.
+   * Find the next startable item across all tool queues. Each tool is checked
+   * independently so tool-specific limits do not block unrelated tools; issue
+   * #2015 adds a global startup interval, so even when multiple tools are
+   * startable this returns only the oldest startable item (global FIFO) to
+   * prevent burst launches. Queued items are rejected immediately when a
+   * 'reject' strategy threshold is exceeded rather than left waiting.
    *
    * @returns {Promise<Array<{item: SolveQueueItem, tool: string, index: number, check: Object}>>}
    * @see https://github.com/link-assistant/hive-mind/issues/1159
@@ -400,6 +392,9 @@ export class SolveQueue {
    */
   async findStartableItems() {
     const startableItems = [];
+    // Per-tool head diagnostics: why each queue head is/isn't startable, so
+    // global FIFO ordering can be audited in production (issue #2051).
+    const headDiagnostics = [];
 
     for (const [tool, toolQueue] of Object.entries(this.queues)) {
       if (toolQueue.length === 0) continue;
@@ -415,23 +410,36 @@ export class SolveQueue {
         continue;
       }
 
+      const item = toolQueue[0];
+      if (!item) continue;
+
+      // Determine startability and capture the blocking reason(s) for diagnostics.
+      // For tool-specific one-at-a-time, only count that tool's processing items.
+      const toolProcessingCount = this.getProcessingCountByTool(tool);
+      let startable = false;
+      const blockReasons = Array.isArray(check.reasons) ? [...check.reasons] : [];
       if (check.canStart) {
-        const item = toolQueue[0];
-        if (!item) continue;
-        // Also check one-at-a-time mode for this specific tool
-        // For tool-specific one-at-a-time, only count that tool's processing items
-        const toolProcessingCount = this.getProcessingCountByTool(tool);
         if (check.oneAtATime && toolProcessingCount > 0) {
-          // This tool is in one-at-a-time mode and has items processing
-          // Skip but don't block other tools
-          continue;
+          // One-at-a-time for this tool with a task already processing: skip it
+          // but don't block other tools.
+          blockReasons.push(`one-at-a-time: ${toolProcessingCount} ${tool} task(s) already processing`);
+        } else {
+          startable = true;
+          startableItems.push({ item, tool, index: 0, check });
         }
-        startableItems.push({ item, tool, index: 0, check });
       }
+
+      headDiagnostics.push({ tool, item, ageMs: Date.now() - item.createdAt, startable, blockReasons });
     }
 
+    // Global FIFO: the oldest startable head wins the (globally paced) startup slot.
     startableItems.sort((a, b) => a.item.createdAt - b.item.createdAt);
-    return startableItems.slice(0, 1);
+    const selected = startableItems.slice(0, 1);
+
+    // Observe the dequeue decision (issue #2051): see reportDequeueDecision().
+    reportDequeueDecision(this, headDiagnostics, selected[0]);
+
+    return selected;
   }
 
   /**
@@ -555,23 +563,16 @@ export class SolveQueue {
   }
 
   /**
-   * Check if a new command can start
-   *
-   * Logic per issue #1061:
-   * 1. "Claude process is already running" is NOT a limit by itself - it's a metric
-   * 2. Commands can run in parallel as long as actual limits are not exceeded
-   * 3. When any limit >= threshold, allow exactly one claude command to pass
-   *
-   * Logic per issue #1159:
-   * - Different tools have different limits. Claude limits only apply to 'claude' tool.
-   * - Processing count for Claude limits only includes Claude items, not agent/codex/gemini/qwen items.
-   * - This allows non-Claude tasks to run in parallel when Claude limits are reached.
+   * Check if a new command can start.
    *
-   * Logic per issue #1253:
-   * - All thresholds now support configurable strategies (reject, enqueue, dequeue-one-at-a-time)
-   * - 'reject' strategy immediately rejects the command without queueing
-   * - 'enqueue' blocks and waits in queue until metric drops
-   * - 'dequeue-one-at-a-time' allows one command while blocking subsequent
+   * - #1061: a running Claude process is a metric, not a limit by itself;
+   *   commands run in parallel while actual limits are not exceeded.
+   * - #1159: tools have independent limits — Claude limits (and Claude
+   *   processing counts) apply only to the 'claude' tool, so non-Claude tasks
+   *   run in parallel when Claude limits are reached.
+   * - #1253: every threshold supports a configurable strategy — 'reject'
+   *   (reject without queueing), 'enqueue' (block until the metric drops), or
+   *   'dequeue-one-at-a-time' (allow one, block subsequent).
    *
    * @param {Object} options - Options for the check
    * @param {string} options.tool - The tool being used ('claude', 'agent', 'codex', 'gemini', 'qwen', etc.)

package/src/telegram-start-stop-command.lib.mjs

@@ -285,6 +285,24 @@ export function registerStartStopCommands(bot, options) {
     return mod.getTrackedSessionInfo(sessionId);
   }
 
+  // Issue #2052: record that this stop was operator-initiated, so the eventual
+  // SIGTERM/SIGKILL exit (delivered by `docker stop`) is reported as
+  // "🛑 Stopped by user" instead of "out of memory or forced kill (SIGKILL)".
+  // Tolerant of a sync or async stub, and never lets a marking failure block
+  // the actual stop.
+  async function markSessionStopRequestedSafe(sessionId, requestedBy) {
+    try {
+      if (typeof options.markSessionStopRequested === 'function') {
+        return await options.markSessionStopRequested(sessionId, { requestedBy, verbose: VERBOSE });
+      }
+      const mod = await import('./session-monitor.lib.mjs');
+      return mod.markSessionStopRequested(sessionId, { requestedBy, verbose: VERBOSE });
+    } catch (error) {
+      console.error('[ERROR] /stop: markSessionStopRequested failed:', error);
+      return false;
+    }
+  }
+
   // Issue #1871: look a URL up in the session-monitor registry of running
   // detached sessions. A /solve or /codex that started immediately (queue
   // empty) is dispatched straight to an isolation session and removed from the
@@ -425,6 +443,12 @@ export function registerStartStopCommands(bot, options) {
       reply_to_message_id: message.message_id,
     });
 
+    // Issue #2052: mark the stop as user-initiated BEFORE forwarding CTRL+C, so
+    // even a fast SIGKILL race still finds the flag when the completion message
+    // is formatted.
+    const requestedBy = ctx.from?.username ? `@${ctx.from.username}` : ctx.from?.first_name || null;
+    await markSessionStopRequestedSafe(sessionId, requestedBy);
+
     let result;
     try {
       result = await stopIsolatedSessionImpl(sessionId, VERBOSE);

package/src/work-session-formatting.lib.mjs

@@ -132,13 +132,25 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
   const finalExitCode = getSessionCompletionExitCode({ exitCode, statusResult });
   const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: statusResult?.status || null });
   const { failed, killed, signal } = outcome;
-  const statusEmoji = failed ? '❌' : '✅';
   const messageLocale = locale || sessionInfo?.locale || null;
   // Issue #1927: a killed session (OOM/SIGKILL/SIGTERM) must never read as a
   // success, and the signal/reason is surfaced explicitly so an operator can
   // tell an out-of-memory kill apart from an ordinary non-zero exit.
+  // Issue #2052: when the operator explicitly requested a stop (e.g. Telegram
+  // `/stop <uuid>` → `docker stop` → SIGTERM then SIGKILL), the resulting signal
+  // exit (143/137) must NOT read as "out of memory or forced kill". A user stop
+  // is an orderly, intentional termination, so surface it as such regardless of
+  // which signal actually delivered the kill.
+  const stopRequestedByUser = Boolean(sessionInfo?.stopRequestedByUser);
+  let statusEmojiOverride = null;
   let statusText;
-  if (killed) {
+  if (killed && stopRequestedByUser) {
+    const showCode = finalExitCode !== null && !(!signal && finalExitCode === 1);
+    const exitSuffix = showCode ? ` (exit code: ${finalExitCode})` : '';
+    const requestedBy = sessionInfo?.stopRequestedBy ? ` by ${sessionInfo.stopRequestedBy}` : '';
+    statusEmojiOverride = '🛑';
+    statusText = text(messageLocale, 'telegram.work_session_stopped', `Work session stopped by user${requestedBy}${exitSuffix}`, { requestedBy, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '', exitSuffix });
+  } else if (killed) {
     // A real signal exit is always >128; an exit code of exactly 1 on a
     // status-only kill (process vanished, code unknown) is a synthesized failure
     // sentinel, so suppress the misleading "(exit code: 1)" in that case.
@@ -164,6 +176,7 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
   if (pullRequestUrl) resolvedInfoBlock = appendPullRequestLine(resolvedInfoBlock, pullRequestUrl, { locale: messageLocale });
   const details = resolvedInfoBlock ? `\n\n${resolvedInfoBlock}` : '';
 
+  const statusEmoji = statusEmojiOverride || (failed ? '❌' : '✅');
   let message = `${statusEmoji} *${statusText}*\n\n`;
   message += `⏱️ ${durationLabel}: ${formatSessionDurationSeconds(durationSeconds)}\n`;
   message += `📊 ${sessionLabel}: \`${sessionName || 'unknown'}\`${isolationInfo}${details}`;