@link-assistant/hive-mind 2.11.10 → 2.11.11

package/CHANGELOG.md

@@ -1,5 +1,39 @@
 # @link-assistant/hive-mind
 
+## 2.11.11
+
+### Patch Changes
+
+- 36e7976: Keep stream provenance on mirrored agent output and make the Codex completion
+  gate explain itself (issue #2140).
+
+  A `solve --tool codex` run that had finished its work — PR updated, all 46
+  check-runs green, `turn.completed` received — was still failed by the completion
+  gate with `turn.started=3, turn.completed=1`. Replaying the real 96k-line log
+  through the parser shows the two extra `turn.started` records arrived on Codex's
+  **stderr**, inside an OTEL `codex.tool_result` dump of a command that merely read
+  a stored NDJSON log file from disk. This is the issue #2136 defect on a binary
+  released minutes before that fix shipped; current builds already gate correctly.
+  Three residual gaps remain, and this change closes them:
+
+  - `log()` accepted an `options.stream` hint and silently discarded it, so every
+    agent CLI's mirrored output — both streams, all five tools — was written as
+    `[INFO]` on our stdout. It is now tagged `[STDOUT]` / `[STDERR]`, matching the
+    tags the stdio interceptor already uses, and mirrored stderr goes to our
+    stderr so piping stdout yields only what the child wrote there. An explicit
+    `level` still wins, and callers that pass no stream are unchanged.
+  - `codex exec` starts exactly one thread, so a `thread.started` on the protocol
+    stream announcing a different `thread_id` is proof of echoed output. Those ids
+    are now collected and reported (`🧬 Foreign thread IDs seen on the codex
+protocol stream`).
+  - The completion-failure reason carried counts and nothing else, which made a
+    false positive impossible to refute from the posted comment. It now also
+    states the ordered turn lifecycle, how many `turn.started` records were
+    discarded as echoed telemetry, and any foreign thread id seen.
+
+  No gate outcome changes: a genuinely truncated turn still fails, and a completed
+  run with echoed events still passes.
+
 ## 2.11.10
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "2.11.10",
+  "version": "2.11.11",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/claude.lib.mjs

@@ -1009,7 +1009,7 @@ export const executeClaudeCommand = async params => {
               }
               // Not JSON or parsing failed, output as-is if it's not empty
               if (line.trim() && !line.includes('node:internal')) {
-                await log(line, { stream: 'raw' });
+                await log(line, { stream: 'stdout' });
                 lastMessage = line;
                 // Issue #1015: Detect terms acceptance prompt (non-JSON "[ACTION REQUIRED]..." message)
                 const termsAcceptancePattern = /\[ACTION REQUIRED\].*terms|must run.*claude.*review.*terms/i;
@@ -1080,7 +1080,7 @@ export const executeClaudeCommand = async params => {
           }
           if (progressMonitor) await progressMonitor.processStreamEvent(data, true).catch(e => log(`⚠️ Progress: ${e.message}`, { verbose: true }));
         } catch {
-          if (!stdoutLineBuffer.includes('node:internal')) await log(stdoutLineBuffer, { stream: 'raw' });
+          if (!stdoutLineBuffer.includes('node:internal')) await log(stdoutLineBuffer, { stream: 'stdout' });
         }
       }
       if (startupTimeoutId) {

package/src/codex-health.lib.mjs

@@ -269,9 +269,28 @@ export const getCodexCompletionHealth = (codexJsonState, { lastMessage = '' } =
   addDiskEvidence('result-summary', codexJsonState?.resultSummary);
   const diskPressureDetected = diskEvidence.length > 0;
 
+  // Issue #2140: the counts alone made the #2136 false positive unfalsifiable —
+  // "turn.started=3, turn.completed=1" was posted to the PR with nothing to say
+  // whether those starts were codex's own. Carry the evidence that decides it:
+  // the ordered lifecycle, what was discarded as echoed telemetry, and any
+  // foreign thread id that reached the protocol stream.
+  const turnLifecycle = Array.isArray(codexJsonState?.turnLifecycle) ? codexJsonState.turnLifecycle : [];
+  const telemetryEventCounts = codexJsonState?.telemetryEventCounts || {};
+  const foreignThreadIds = codexJsonState?.foreignThreadIds || [];
+  const echoedTurnStarts = telemetryEventCounts['turn.started'] || 0;
+
   const reasons = [];
   if (incompleteSession) {
     reasons.push(`Codex session ended without completing its turn (turn.started=${turnStarted}, turn.completed=${turnCompleted}, turn.failed=${turnFailed}); the process exited 0 but was cut off mid-turn.`);
+    if (turnLifecycle.length > 0) {
+      reasons.push(`Turn lifecycle in order: ${turnLifecycle.join(' → ')} — the stream ends on a start, so the last turn never finished.`);
+    }
+    if (echoedTurnStarts > 0 || foreignThreadIds.length > 0) {
+      const echoParts = [];
+      if (echoedTurnStarts > 0) echoParts.push(`${echoedTurnStarts} echoed turn.started on codex stderr (excluded from the counts above)`);
+      if (foreignThreadIds.length > 0) echoParts.push(`foreign thread id(s) on the protocol stream: ${foreignThreadIds.join(', ')}`);
+      reasons.push(`Echo diagnostics (issues #2136/#2140): ${echoParts.join('; ')}.`);
+    }
     if (diskPressureDetected) {
       reasons.push(`Disk-exhaustion signals were present in ${diskEvidence.length} location(s) (e.g. "No space left on device") — the likely cause of the interrupted session.`);
     }
@@ -285,6 +304,9 @@ export const getCodexCompletionHealth = (codexJsonState, { lastMessage = '' } =
     turnStarted,
     turnCompleted,
     turnFailed,
+    turnLifecycle,
+    echoedTurnStarts,
+    foreignThreadIds,
     reasons,
   };
 };
@@ -305,6 +327,9 @@ export const reportCodexCompletionFailure = async ({ completionHealth, log, getR
     await log(`   • ${reason}`, { level: 'error' });
   }
   await log(`   📊 turn.started=${completionHealth.turnStarted}, turn.completed=${completionHealth.turnCompleted}, turn.failed=${completionHealth.turnFailed}`, { verbose: true });
+  if (completionHealth.turnLifecycle?.length) {
+    await log(`   🔁 turn lifecycle: ${completionHealth.turnLifecycle.join(' → ')}`, { verbose: true });
+  }
   if (completionHealth.diskPressureDetected) {
     await log('   💽 Disk-exhaustion evidence (diagnostic):', { level: 'error' });
     for (const evidence of completionHealth.diskEvidence.slice(0, 5)) {

package/src/codex.lib.mjs

@@ -43,52 +43,11 @@ import { applyCodexCapabilityEnv, runCodexCapabilityPreflight } from './codex-ca
 import { createPullRequestBaseBranchCommandIntervention } from './solve.pr-base-command-intervention.lib.mjs';
 import Decimal from 'decimal.js-light';
 import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
+import { CODEX_CACHE_READ_USAGE_PATHS, CODEX_CACHE_WRITE_USAGE_PATHS, CODEX_MODEL_DIAGNOSTIC_PATHS, CODEX_REASONING_USAGE_PATHS, CODEX_USAGE_FIELD_NAMES, createCodexTokenFieldAvailability, getFirstObservedNumber, hasAnyObservedPath, hasOwnPath } from './codex.usage-fields.lib.mjs';
 
-const CODEX_USAGE_FIELD_NAMES = ['input_tokens', 'cached_input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_creation_input_tokens', 'reasoning_tokens', 'reasoning_output_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens', 'output_tokens_details.reasoning_tokens'];
 const CODEX_LONG_CONTEXT_PRICE_THRESHOLD = 272000;
 const CODEX_COMPACT_API_ENDPOINT = '/responses/compact';
 const getCodexExecEnv = (verbose = false) => (verbose ? { ...process.env, RUST_LOG: 'debug' } : { ...process.env });
-const CODEX_MODEL_DIAGNOSTIC_PATHS = [
-  ['model', data => data?.model],
-  ['model_name', data => data?.model_name],
-  ['from_model', data => data?.from_model],
-  ['to_model', data => data?.to_model],
-  ['message.model', data => data?.message?.model],
-];
-
-const createCodexTokenFieldAvailability = () => ({
-  inputTokens: false,
-  outputTokens: false,
-  reasoningTokens: false,
-  cacheReadTokens: false,
-  cacheWriteTokens: false,
-});
-
-const hasOwnPath = (object, pathName) => {
-  let cursor = object;
-  for (const part of pathName.split('.')) {
-    if (!cursor || typeof cursor !== 'object' || !Object.hasOwn(cursor, part)) return false;
-    cursor = cursor[part];
-  }
-  return true;
-};
-
-const getPathValue = (object, pathName) => pathName.split('.').reduce((cursor, part) => cursor?.[part], object);
-
-const getFirstObservedNumber = (object, pathNames) => {
-  for (const pathName of pathNames) {
-    if (!hasOwnPath(object, pathName)) continue;
-    const value = getPathValue(object, pathName);
-    return Number.isFinite(value) ? value : 0;
-  }
-  return 0;
-};
-
-const hasAnyObservedPath = (object, pathNames) => pathNames.some(pathName => hasOwnPath(object, pathName));
-
-const CODEX_CACHE_READ_USAGE_PATHS = ['cached_input_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens'];
-const CODEX_CACHE_WRITE_USAGE_PATHS = ['cache_write_tokens', 'cache_creation_input_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens'];
-const CODEX_REASONING_USAGE_PATHS = ['reasoning_tokens', 'reasoning_output_tokens', 'output_tokens_details.reasoning_tokens'];
 
 const escapeRegExp = value => String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 
@@ -380,6 +339,12 @@ export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId =
     // completion gate can ask "did the last turn finish?" instead of comparing
     // counts that an echoed `turn.started` can skew.
     turnLifecycle: state.turnLifecycle || [],
+    // Issue #2140: `thread.started` records seen on the protocol stream that
+    // announce a thread id other than this session's. Codex only starts one
+    // thread per `codex exec`, so a second id is proof that something echoed
+    // another agent's protocol into ours — the one turn event that carries an
+    // identity we can check. Diagnostics only; the gate stays order-based.
+    foreignThreadIds: state.foreignThreadIds || [],
   };
 
   nextState.tokenUsage.tokenFieldAvailability ||= createCodexTokenFieldAvailability();
@@ -432,6 +397,11 @@ export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId =
 
     if (eventType === 'thread.started' && typeof data.thread_id === 'string' && !nextState.sessionId) {
       nextState.sessionId = data.thread_id;
+    } else if (eventType === 'thread.started' && typeof data.thread_id === 'string' && data.thread_id !== nextState.sessionId) {
+      // Issue #2140: a foreign thread id on the protocol stream is echoed
+      // output, not a second codex session. Record it once so a run that ends
+      // up disputed can be settled from the log alone.
+      if (!nextState.foreignThreadIds.includes(data.thread_id)) nextState.foreignThreadIds.push(data.thread_id);
     } else if (!nextState.sessionId && typeof data.session_id === 'string') {
       nextState.sessionId = data.session_id;
     }
@@ -1030,7 +1000,7 @@ export const executeCodexCommand = async params => {
         if (chunk.type === 'stdout') {
           const raw = chunk.data.toString();
           if (argv.verbose) {
-            await log(raw);
+            await log(raw, { stream: 'stdout' });
           }
           lastMessage = raw;
           const output = codexStdoutLines.write(raw);

package/src/codex.run-diagnostics.lib.mjs

@@ -97,6 +97,13 @@ export const buildCodexRunDiagnostics = ({ state = {}, exitCode = null, mappedMo
   }
   if (state.turnLifecycle?.length > 0) push(`🔁 Codex turn lifecycle: ${state.turnLifecycle.join(' → ')}`);
 
+  // Issue #2140: codex starts exactly one thread per `codex exec`, so any other
+  // thread id on the protocol stream is echoed output that leaked past the
+  // stream separation above. Always worth saying out loud.
+  if (state.foreignThreadIds?.length > 0) {
+    push(`🧬 Foreign thread IDs seen on the codex protocol stream (echoed, not codex sessions): ${state.foreignThreadIds.join(', ')}`, { level: 'warning', verbose: true });
+  }
+
   const usage = state.tokenUsage || {};
   if (usage.stepCount > 0) {
     push(`📈 Codex usage from turn.completed: ${usage.inputTokens.toLocaleString()} input, ${usage.cacheReadTokens.toLocaleString()} cache read, ${usage.outputTokens.toLocaleString()} output across ${usage.stepCount} turn(s)`);

package/src/codex.usage-fields.lib.mjs

@@ -0,0 +1,59 @@
+/**
+ * Usage-field vocabulary and JSON-path helpers for the `codex exec --json`
+ * parser.
+ *
+ * Split out of codex.lib.mjs to keep that file inside the max-lines budget
+ * (issues #1730 / #1990 / #2140). Everything here is pure data plus pure
+ * lookups: codex has renamed and re-nested its usage fields several times
+ * across releases, so the parser reads whichever spelling is *present* rather
+ * than assuming one shape, and reports what it actually observed.
+ */
+
+/** Every usage field name we know codex has used, for observability reporting. */
+export const CODEX_USAGE_FIELD_NAMES = ['input_tokens', 'cached_input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_creation_input_tokens', 'reasoning_tokens', 'reasoning_output_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens', 'output_tokens_details.reasoning_tokens'];
+
+/** Places a codex event has been seen to name a model, in preference order. */
+export const CODEX_MODEL_DIAGNOSTIC_PATHS = [
+  ['model', data => data?.model],
+  ['model_name', data => data?.model_name],
+  ['from_model', data => data?.from_model],
+  ['to_model', data => data?.to_model],
+  ['message.model', data => data?.message?.model],
+];
+
+export const CODEX_CACHE_READ_USAGE_PATHS = ['cached_input_tokens', 'input_tokens_details.cached_tokens', 'input_tokens_details.cache_read_tokens'];
+export const CODEX_CACHE_WRITE_USAGE_PATHS = ['cache_write_tokens', 'cache_creation_input_tokens', 'input_tokens_details.cache_write_tokens', 'input_tokens_details.cache_creation_tokens', 'input_tokens_details.cache_creation_input_tokens'];
+export const CODEX_REASONING_USAGE_PATHS = ['reasoning_tokens', 'reasoning_output_tokens', 'output_tokens_details.reasoning_tokens'];
+
+/** Which token kinds this run has actually seen codex report. */
+export const createCodexTokenFieldAvailability = () => ({
+  inputTokens: false,
+  outputTokens: false,
+  reasoningTokens: false,
+  cacheReadTokens: false,
+  cacheWriteTokens: false,
+});
+
+/** Own-property check along a dotted path — absent ≠ present-and-zero. */
+export const hasOwnPath = (object, pathName) => {
+  let cursor = object;
+  for (const part of pathName.split('.')) {
+    if (!cursor || typeof cursor !== 'object' || !Object.hasOwn(cursor, part)) return false;
+    cursor = cursor[part];
+  }
+  return true;
+};
+
+export const getPathValue = (object, pathName) => pathName.split('.').reduce((cursor, part) => cursor?.[part], object);
+
+/** First path that is actually present wins; a non-finite value counts as 0. */
+export const getFirstObservedNumber = (object, pathNames) => {
+  for (const pathName of pathNames) {
+    if (!hasOwnPath(object, pathName)) continue;
+    const value = getPathValue(object, pathName);
+    return Number.isFinite(value) ? value : 0;
+  }
+  return 0;
+};
+
+export const hasAnyObservedPath = (object, pathNames) => pathNames.some(pathName => hasOwnPath(object, pathName));

package/src/gemini.lib.mjs

@@ -462,7 +462,7 @@ export const executeGeminiCommand = async params => {
       for await (const chunk of execCommand.stream()) {
         if (chunk.type === 'stdout') {
           const output = chunk.data.toString();
-          await log(output);
+          await log(output, { stream: 'stdout' });
           allOutput += output;
           geminiJsonState = parseGeminiJsonOutput(output, geminiJsonState, mappedModel);
           if (geminiJsonState.sessionId) {

package/src/lib.mjs

@@ -95,10 +95,14 @@ export const getAbsoluteLogPath = async () => {
  * @param {Object} options - Logging options
  * @param {string} [options.level='info'] - Log level (info, warn, error)
  * @param {boolean} [options.verbose=false] - Whether this is a verbose log
+ * @param {string} [options.stream] - Provenance of the message when it is raw
+ *   output mirrored from a child process: 'stdout' or 'stderr'. Tags the log
+ *   file lines [STDOUT]/[STDERR] (matching the process.stdout/stderr
+ *   interceptor below) and routes the console write to the same stream.
  * @returns {Promise<void>}
  */
 export const log = async (message, options = {}) => {
-  const { level = 'info', verbose = false } = options;
+  const { level = 'info', verbose = false, stream = null } = options;
 
   // Skip verbose logs unless --verbose is enabled
   if (verbose && !global.verboseMode) {
@@ -107,12 +111,20 @@ export const log = async (message, options = {}) => {
 
   const sanitizedMessage = sanitizeCredentialText(message);
 
+  // Issue #2140: mirrored child output must stay attributable to the stream it
+  // came from. Both Codex streams used to be written as plain [INFO], so a run
+  // log could not answer "did Codex emit this protocol line, or did its stderr
+  // merely echo one?" — the exact question a false completion failure hinges on.
+  // An explicit level still wins, so warnings/errors keep their own tag.
+  const mirroredStream = stream === 'stdout' || stream === 'stderr' ? stream : null;
+  const tag = mirroredStream && level === 'info' ? mirroredStream.toUpperCase() : level.toUpperCase();
+
   // Write to file if log file is set
   // Issue #1572: Handle multi-line messages by timestamping each line,
   // so continuation lines don't appear without timestamps in the log file
   if (logFile) {
     const timestamp = new Date().toISOString();
-    const prefix = `[${timestamp}] [${level.toUpperCase()}]`;
+    const prefix = `[${timestamp}] [${tag}]`;
     const lines = sanitizedMessage.split('\n');
     const logMessage = lines.map(line => `${prefix} ${line}`).join('\n');
     try {
@@ -146,7 +158,10 @@ export const log = async (message, options = {}) => {
         break;
       case 'info':
       default:
-        console.log(sanitizedMessage);
+        // Mirrored child stderr goes to our stderr, so piping stdout to a
+        // consumer keeps yielding only what the child wrote to stdout.
+        if (mirroredStream === 'stderr') console.error(sanitizedMessage);
+        else console.log(sanitizedMessage);
         break;
     }
   } finally {

package/src/opencode.lib.mjs

@@ -394,7 +394,7 @@ export const executeOpenCodeCommand = async params => {
       for await (const chunk of execCommand.stream()) {
         if (chunk.type === 'stdout') {
           const output = chunk.data.toString();
-          await log(output);
+          await log(output, { stream: 'stdout' });
           lastMessage = output;
           allOutput += output;
 

package/src/qwen.lib.mjs

@@ -566,7 +566,7 @@ export const executeQwenCommand = async params => {
       for await (const chunk of execCommand.stream()) {
         if (chunk.type === 'stdout') {
           const output = chunk.data.toString();
-          await log(output);
+          await log(output, { stream: 'stdout' });
           allOutput += output;
           qwenState = parseQwenStreamJsonOutput(output, qwenState);
         }