@link-assistant/hive-mind 2.5.3 → 2.5.4

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/hive-mind
 
+## 2.5.4
+
+### Patch Changes
+
+- 2ebdb3d: Add dequeue-decision diagnostics to the Telegram solve queue so global FIFO ordering across the per-tool queues can be audited in production (issue #2051). The oldest startable task still wins the single, globally-paced startup slot; when an older task is skipped because it cannot start, the queue now logs a concise, deduplicated "FIFO queue-jump" line naming the older task and the exact reason it is blocked (Claude/Codex limits, RAM/CPU/disk, min-interval, or one-at-a-time), and records it on `stats.lastQueueJump`. Verbose mode additionally prints a per-tool head snapshot each cycle. No change to ordering, pacing, or the minimum start interval.
+
 ## 2.5.3
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "2.5.3",
+  "version": "2.5.4",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/telegram-solve-queue.helpers.lib.mjs

@@ -492,3 +492,61 @@ export function formatWaitingReason(metric, currentValue, threshold, options = {
       return `${metric} threshold exceeded`;
   }
 }
+
+/**
+ * Report the dequeue decision across all tool queues for a SolveQueue.
+ *
+ * Provides the observability that issue #2051 needs to confirm whether a
+ * long-waiting task is blocked legitimately (limits/resources/pacing) or by an
+ * ordering defect:
+ * - In verbose mode: prints a per-tool head snapshot (age + startable +
+ *   blocking reasons) and the selected item each cycle.
+ * - Always-on: emits a concise "FIFO queue-jump" line whenever the
+ *   globally-oldest queued head is skipped in favor of a younger startable head,
+ *   naming the older task and the exact reason it is blocked. Deduplicated by
+ *   (task id + block reasons) so a persistently-blocked task is only reported
+ *   when its situation changes.
+ *
+ * @param {Object} queue - SolveQueue-like instance (uses verbose, log, recordThrottle, stats).
+ * @param {Array<{tool: string, item: Object, ageMs: number, startable: boolean, blockReasons: string[]}>} headDiagnostics
+ * @param {{item: Object, tool: string}|undefined} selected
+ * @see https://github.com/link-assistant/hive-mind/issues/2051
+ */
+export function reportDequeueDecision(queue, headDiagnostics, selected) {
+  if (!headDiagnostics || headDiagnostics.length === 0) return;
+
+  if (queue.verbose) {
+    queue.log('Dequeue decision (global FIFO across tool queues):');
+    for (const d of headDiagnostics) {
+      const detail = d.startable ? 'STARTABLE' : `blocked by [${d.blockReasons.join('; ') || 'unknown'}]`;
+      queue.log(`  ${d.tool}: waited ${formatDuration(d.ageMs)} - ${detail}`);
+    }
+    queue.log(selected ? `  -> selected ${selected.tool} (${selected.item.url})` : '  -> nothing startable this cycle');
+  }
+
+  if (!selected) return;
+
+  // Identify the globally-oldest queued head regardless of startability.
+  const oldest = headDiagnostics.reduce((a, b) => (a.item.createdAt <= b.item.createdAt ? a : b));
+  if (oldest.item.id === selected.item.id) return; // Strict FIFO honored - nothing to report.
+
+  const waitedMs = Date.now() - oldest.item.createdAt;
+  const blockedBy = oldest.blockReasons.join('; ') || 'unknown';
+  queue.recordThrottle('fifo_queue_jump');
+  queue.stats.lastQueueJump = {
+    skippedTool: oldest.tool,
+    skippedUrl: oldest.item.url,
+    waitedMs,
+    blockedBy: oldest.blockReasons,
+    startedTool: selected.tool,
+    startedUrl: selected.item.url,
+  };
+
+  // Deduplicate the always-on notice so a persistently-blocked task does not
+  // spam the log on every poll; only report when the reason changes.
+  const signature = `${oldest.item.id}|${blockedBy}`;
+  if (queue._lastQueueJumpSignature !== signature) {
+    queue._lastQueueJumpSignature = signature;
+    console.log(`[solve_queue] FIFO queue-jump: ${selected.tool} task started ahead of an older ${oldest.tool} task waiting ${formatDuration(waitedMs)} (${oldest.item.url}) — older task blocked by: ${blockedBy}`);
+  }
+}

package/src/telegram-solve-queue.lib.mjs

@@ -2,22 +2,17 @@
 /**
  * Telegram Solve Queue Library
  *
- * Producer/consumer queue for /solve commands in the Telegram bot.
- * Implements resource-aware throttling to prevent system overload.
- *
- * Features:
- * - Resource checking (RAM, CPU, disk)
- * - API limit checking (Claude, GitHub)
- * - Minimum interval between command starts
- * - Running process detection
- * - Status tracking: Queued -> Waiting -> Starting -> Started
+ * Producer/consumer queue for /solve commands in the Telegram bot. Implements
+ * resource-aware throttling (RAM, CPU, disk), API limit checking (Claude,
+ * GitHub), a minimum interval between command starts, running-process
+ * detection, and status tracking (Queued -> Waiting -> Starting -> Started).
  *
  * @see https://github.com/link-assistant/hive-mind/issues/1041
  */
 
 import { getCachedClaudeLimits, getCachedCodexLimits, getCachedGitHubLimits, getCachedMemoryInfo, getCachedCpuInfo, getCachedDiskInfo, getLimitCache } from './limits.lib.mjs';
 export { formatDuration, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses } from './telegram-solve-queue.helpers.lib.mjs';
-import { collectExecutingItems, formatDuration, formatQueueToolSection, formatWaitingReason, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses, getRunningSessionItems, groupQueueItemsByTool } from './telegram-solve-queue.helpers.lib.mjs';
+import { collectExecutingItems, formatDuration, formatQueueToolSection, formatWaitingReason, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses, getRunningSessionItems, groupQueueItemsByTool, reportDequeueDecision } from './telegram-solve-queue.helpers.lib.mjs';
 export { QUEUE_CONFIG, THRESHOLD_STRATEGIES } from './queue-config.lib.mjs';
 import { QUEUE_CONFIG } from './queue-config.lib.mjs';
 import { reserveStartSlotForQueue } from './queue-start-reservation.lib.mjs';
@@ -190,8 +185,7 @@ export class SolveQueue {
       qwen: null,
       gemini: null,
     };
-    // Legacy: keep for compatibility with existing code that uses lastStartTime
-    this.lastStartTime = null;
+    this.lastStartTime = null; // Legacy: global last-start timestamp
 
     // Consumer task reference
     this.consumerTask = null;
@@ -385,14 +379,12 @@ export class SolveQueue {
   }
 
   /**
-   * Find the next startable item across all tool queues.
-   * With separate queues, each tool is checked independently so tool-specific
-   * limits do not block unrelated tools. Issue #2015 adds a global startup
-   * interval, so even when multiple tools are startable this returns only the
-   * oldest startable item to prevent burst launches.
-   *
-   * Also immediately rejects all queued items when a 'reject' strategy threshold
-   * is exceeded, instead of leaving them waiting indefinitely.
+   * Find the next startable item across all tool queues. Each tool is checked
+   * independently so tool-specific limits do not block unrelated tools; issue
+   * #2015 adds a global startup interval, so even when multiple tools are
+   * startable this returns only the oldest startable item (global FIFO) to
+   * prevent burst launches. Queued items are rejected immediately when a
+   * 'reject' strategy threshold is exceeded rather than left waiting.
    *
    * @returns {Promise<Array<{item: SolveQueueItem, tool: string, index: number, check: Object}>>}
    * @see https://github.com/link-assistant/hive-mind/issues/1159
@@ -400,6 +392,9 @@ export class SolveQueue {
    */
   async findStartableItems() {
     const startableItems = [];
+    // Per-tool head diagnostics: why each queue head is/isn't startable, so
+    // global FIFO ordering can be audited in production (issue #2051).
+    const headDiagnostics = [];
 
     for (const [tool, toolQueue] of Object.entries(this.queues)) {
       if (toolQueue.length === 0) continue;
@@ -415,23 +410,36 @@ export class SolveQueue {
         continue;
       }
 
+      const item = toolQueue[0];
+      if (!item) continue;
+
+      // Determine startability and capture the blocking reason(s) for diagnostics.
+      // For tool-specific one-at-a-time, only count that tool's processing items.
+      const toolProcessingCount = this.getProcessingCountByTool(tool);
+      let startable = false;
+      const blockReasons = Array.isArray(check.reasons) ? [...check.reasons] : [];
       if (check.canStart) {
-        const item = toolQueue[0];
-        if (!item) continue;
-        // Also check one-at-a-time mode for this specific tool
-        // For tool-specific one-at-a-time, only count that tool's processing items
-        const toolProcessingCount = this.getProcessingCountByTool(tool);
         if (check.oneAtATime && toolProcessingCount > 0) {
-          // This tool is in one-at-a-time mode and has items processing
-          // Skip but don't block other tools
-          continue;
+          // One-at-a-time for this tool with a task already processing: skip it
+          // but don't block other tools.
+          blockReasons.push(`one-at-a-time: ${toolProcessingCount} ${tool} task(s) already processing`);
+        } else {
+          startable = true;
+          startableItems.push({ item, tool, index: 0, check });
         }
-        startableItems.push({ item, tool, index: 0, check });
       }
+
+      headDiagnostics.push({ tool, item, ageMs: Date.now() - item.createdAt, startable, blockReasons });
     }
 
+    // Global FIFO: the oldest startable head wins the (globally paced) startup slot.
     startableItems.sort((a, b) => a.item.createdAt - b.item.createdAt);
-    return startableItems.slice(0, 1);
+    const selected = startableItems.slice(0, 1);
+
+    // Observe the dequeue decision (issue #2051): see reportDequeueDecision().
+    reportDequeueDecision(this, headDiagnostics, selected[0]);
+
+    return selected;
   }
 
   /**
@@ -555,23 +563,16 @@ export class SolveQueue {
   }
 
   /**
-   * Check if a new command can start
-   *
-   * Logic per issue #1061:
-   * 1. "Claude process is already running" is NOT a limit by itself - it's a metric
-   * 2. Commands can run in parallel as long as actual limits are not exceeded
-   * 3. When any limit >= threshold, allow exactly one claude command to pass
-   *
-   * Logic per issue #1159:
-   * - Different tools have different limits. Claude limits only apply to 'claude' tool.
-   * - Processing count for Claude limits only includes Claude items, not agent/codex/gemini/qwen items.
-   * - This allows non-Claude tasks to run in parallel when Claude limits are reached.
+   * Check if a new command can start.
    *
-   * Logic per issue #1253:
-   * - All thresholds now support configurable strategies (reject, enqueue, dequeue-one-at-a-time)
-   * - 'reject' strategy immediately rejects the command without queueing
-   * - 'enqueue' blocks and waits in queue until metric drops
-   * - 'dequeue-one-at-a-time' allows one command while blocking subsequent
+   * - #1061: a running Claude process is a metric, not a limit by itself;
+   *   commands run in parallel while actual limits are not exceeded.
+   * - #1159: tools have independent limits — Claude limits (and Claude
+   *   processing counts) apply only to the 'claude' tool, so non-Claude tasks
+   *   run in parallel when Claude limits are reached.
+   * - #1253: every threshold supports a configurable strategy — 'reject'
+   *   (reject without queueing), 'enqueue' (block until the metric drops), or
+   *   'dequeue-one-at-a-time' (allow one, block subsequent).
    *
    * @param {Object} options - Options for the check
    * @param {string} options.tool - The tool being used ('claude', 'agent', 'codex', 'gemini', 'qwen', etc.)