@link-assistant/hive-mind 2.11.9 → 2.11.10

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @link-assistant/hive-mind
 
+## 2.11.10
+
+### Patch Changes
+
+- 8c8844a: Stop treating agent CLI stderr as a JSON protocol stream (issue #2136).
+
+  `codex exec --json` writes its NDJSON protocol to stdout only; its stderr carries
+  OTEL tracing whose `codex.tool_result` records dump the raw stdout of every
+  command Codex runs. When the task itself drove another agent CLI, that dump
+  replayed NDJSON byte-identical to Codex's own protocol, so an echoed
+  `turn.started` was counted as Codex's own and the completion gate failed a run
+  that had actually finished — posting a "Solution Draft Failed" comment on a pull
+  request that was complete and later merged.
+
+  Codex now parses only stdout as protocol; protocol-shaped JSON seen on stderr is
+  reported separately (`🪞 Echoed protocol-shaped lines on codex stderr`) and never
+  affects event counts, session id, token usage or error detection. The completion
+  gate additionally uses the ordered turn lifecycle (`🔁 Codex turn lifecycle`)
+  instead of comparing counts, so a stray `turn.started` can no longer fail a
+  completed run while a genuinely truncated turn still does. The same stream
+  separation is applied to qwen (whose stderr echo could raise a false error and
+  whose two streams shared one line buffer), and OpenCode now reports how many JSON
+  records it parsed from stderr.
+
 ## 2.11.9
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/hive-mind",
-  "version": "2.11.9",
+  "version": "2.11.10",
   "description": "AI-powered issue solver and hive mind for collaborative problem solving",
   "main": "src/hive.mjs",
   "type": "module",

package/src/codex-health.lib.mjs

@@ -218,6 +218,26 @@ export const getCodexPluginProvisioningHealth = (codexJsonState, { capabilityPre
 // both observed in the captured runs at exit_code 0) would be wrongly failed.
 // Disk pressure is surfaced only as supporting *diagnostics* explaining why a
 // session was likely cut off, never as the sole reason to fail a completed turn.
+// Issue #2136: the count comparison above is only sound when every counted
+// `turn.started` is codex's own. It is not: under `--verbose` codex's stderr
+// carries OTEL records that dump the raw stdout of each command it ran, so a task
+// that drives another agent CLI replays that agent's `turn.started` into our
+// stream. One echoed line was enough to make turn.started=2 vs turn.completed=1
+// and fail a run that had finished successfully (formal-ai PR #913 was open with
+// green CI). codex.lib.mjs now keeps stderr out of the protocol counters, and the
+// gate itself asks the order-aware question — "was the LAST lifecycle event a
+// start?" — so a stray extra `turn.started` from any future echo path can no
+// longer flip a completed session to failed, while a genuine cut-off mid-turn
+// (the #1990 shape: the stream ends on `turn.started`) still fails.
+const isIncompleteTurnLifecycle = (turnLifecycle, { turnStarted, turnCompleted, turnFailed }) => {
+  if (!Array.isArray(turnLifecycle) || turnLifecycle.length === 0) {
+    // Callers that hand-build a state without an ordered lifecycle keep the
+    // original count rule.
+    return turnCompleted + turnFailed < Math.max(turnStarted, 1);
+  }
+  return turnLifecycle.at(-1) === 'turn.started';
+};
+
 export const getCodexCompletionHealth = (codexJsonState, { lastMessage = '' } = {}) => {
   const eventCounts = codexJsonState?.eventCounts || {};
   const turnStarted = eventCounts['turn.started'] || 0;
@@ -232,7 +252,7 @@ export const getCodexCompletionHealth = (codexJsonState, { lastMessage = '' } =
 
   // A started turn that never completed or failed = the process was cut off
   // mid-turn (OOM / disk-full / container teardown) even though it exited 0.
-  const incompleteSession = hadActivity && turnCompleted + turnFailed < Math.max(turnStarted, 1);
+  const incompleteSession = hadActivity && isIncompleteTurnLifecycle(codexJsonState?.turnLifecycle, { turnStarted, turnCompleted, turnFailed });
 
   // Diagnostic-only disk-pressure hints (never an independent failure gate).
   const diskEvidence = [];

package/src/codex.lib.mjs

@@ -338,7 +338,21 @@ const upsertCodexItemError = (itemErrors, item) => {
   });
 };
 
-export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId = null) => {
+// Issue #2136: `codex exec --json` writes its NDJSON protocol to **stdout** only.
+// Its stderr carries OTEL tracing text (RUST_LOG=debug under --verbose), and each
+// `codex.tool_result` record dumps the raw stdout of the command codex just ran —
+// so a task driving another agent CLI replays that agent's NDJSON verbatim inside
+// the trace. Feeding stderr through this parser counted the nested agent's
+// `turn.started` as codex's own, and the #1990 completion gate then failed a run
+// that had actually completed (see docs/case-studies/issue-2136).
+//
+// Fix: only stdout is trusted as protocol. Stderr is still scanned for the
+// text-only diagnostics that legitimately live there (token/model diagnostics,
+// the #2102 plugin-install rejection), but protocol-shaped JSON found there is
+// counted into `telemetryEventCounts` and otherwise ignored — it never touches
+// eventCounts, sessionId, usage or the error buckets.
+export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId = null, { source = 'stdout' } = {}) => {
+  const isProtocolStream = source === 'stdout';
   const nextState = {
     sessionId: state.sessionId || null,
     authError: state.authError || false,
@@ -359,6 +373,13 @@ export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId =
     pluginInstallRejections: state.pluginInstallRejections || [],
     observedUsageFieldSets: state.observedUsageFieldSets || [],
     observedModelDiagnosticPaths: state.observedModelDiagnosticPaths || [],
+    // Issue #2136: protocol-shaped JSON seen on a non-protocol stream (telemetry
+    // echo), kept for diagnostics only.
+    telemetryEventCounts: state.telemetryEventCounts || {},
+    // Issue #2136: ordered turn lifecycle from the protocol stream, so the
+    // completion gate can ask "did the last turn finish?" instead of comparing
+    // counts that an echoed `turn.started` can skew.
+    turnLifecycle: state.turnLifecycle || [],
   };
 
   nextState.tokenUsage.tokenFieldAvailability ||= createCodexTokenFieldAvailability();
@@ -396,7 +417,18 @@ export const parseCodexExecJsonOutput = (output, state = {}, requestedModelId =
     if (data === null || typeof data !== 'object') continue;
 
     const eventType = typeof data.type === 'string' ? data.type : 'unknown';
+
+    // Issue #2136: a protocol-shaped object on a non-protocol stream is echoed
+    // telemetry, never a codex event. Count it for diagnostics and move on.
+    if (!isProtocolStream) {
+      nextState.telemetryEventCounts[eventType] = (nextState.telemetryEventCounts[eventType] || 0) + 1;
+      continue;
+    }
+
     nextState.eventCounts[eventType] = (nextState.eventCounts[eventType] || 0) + 1;
+    if (eventType === 'turn.started' || eventType === 'turn.completed' || eventType === 'turn.failed') {
+      nextState.turnLifecycle.push(eventType);
+    }
 
     if (eventType === 'thread.started' && typeof data.thread_id === 'string' && !nextState.sessionId) {
       nextState.sessionId = data.thread_id;
@@ -983,6 +1015,8 @@ export const executeCodexCommand = async params => {
         pluginInstallRejections: [],
         observedUsageFieldSets: [],
         observedModelDiagnosticPaths: [],
+        telemetryEventCounts: {},
+        turnLifecycle: [],
       };
 
       // Issue #2119: a process chunk boundary can fall in the middle of an
@@ -1001,7 +1035,7 @@ export const executeCodexCommand = async params => {
           lastMessage = raw;
           const output = codexStdoutLines.write(raw);
 
-          codexJsonState = parseCodexExecJsonOutput(output, codexJsonState, mappedModel);
+          codexJsonState = parseCodexExecJsonOutput(output, codexJsonState, mappedModel, { source: 'stdout' });
           await baseBranchCommandIntervention.handleCommandExecutions(codexJsonState.commandExecutions);
 
           if (interactiveHandler || progressMonitor) {
@@ -1044,7 +1078,8 @@ export const executeCodexCommand = async params => {
             await log(rawError, { stream: 'stderr' });
           }
           const errorOutput = codexStderrLines.write(rawError);
-          codexJsonState = parseCodexExecJsonOutput(errorOutput, codexJsonState, mappedModel);
+          // Issue #2136: stderr is telemetry/tracing text, not the codex protocol.
+          codexJsonState = parseCodexExecJsonOutput(errorOutput, codexJsonState, mappedModel, { source: 'stderr' });
           await baseBranchCommandIntervention.handleCommandExecutions(codexJsonState.commandExecutions);
         } else if (chunk.type === 'exit') {
           exitCode = chunk.code;
@@ -1052,9 +1087,12 @@ export const executeCodexCommand = async params => {
       }
 
       // Release any line that was still being assembled when the stream ended.
-      for (const remaining of [codexStdoutLines.flush(), codexStderrLines.flush()]) {
+      for (const [source, remaining] of [
+        ['stdout', codexStdoutLines.flush()],
+        ['stderr', codexStderrLines.flush()],
+      ]) {
         if (!remaining.trim()) continue;
-        codexJsonState = parseCodexExecJsonOutput(remaining, codexJsonState, mappedModel);
+        codexJsonState = parseCodexExecJsonOutput(remaining, codexJsonState, mappedModel, { source });
         await baseBranchCommandIntervention.handleCommandExecutions(codexJsonState.commandExecutions);
       }
 

package/src/codex.run-diagnostics.lib.mjs

@@ -86,6 +86,17 @@ export const buildCodexRunDiagnostics = ({ state = {}, exitCode = null, mappedMo
   if (Object.keys(state.eventCounts || {}).length > 0) push(`📊 Codex JSON events: ${counts(state.eventCounts)}`);
   if (Object.keys(state.itemTypeCounts || {}).length > 0) push(`📦 Codex item types: ${counts(state.itemTypeCounts)}`);
 
+  // Issue #2136: protocol-shaped JSON that arrived on codex's stderr (OTEL
+  // `codex.tool_result` records replay the raw stdout of every command codex
+  // runs, so a task driving another agent CLI replays that agent's NDJSON). These
+  // lines are deliberately NOT counted as codex events; surfacing them keeps the
+  // discrepancy between "events codex emitted" and "protocol lines seen in the
+  // log" explainable when a run is investigated after the fact.
+  if (Object.keys(state.telemetryEventCounts || {}).length > 0) {
+    push(`🪞 Echoed protocol-shaped lines on codex stderr (ignored, not codex events): ${counts(state.telemetryEventCounts)}`);
+  }
+  if (state.turnLifecycle?.length > 0) push(`🔁 Codex turn lifecycle: ${state.turnLifecycle.join(' → ')}`);
+
   const usage = state.tokenUsage || {};
   if (usage.stepCount > 0) {
     push(`📈 Codex usage from turn.completed: ${usage.inputTokens.toLocaleString()} input, ${usage.cacheReadTokens.toLocaleString()} cache read, ${usage.outputTokens.toLocaleString()} output across ${usage.stepCount} turn(s)`);

package/src/opencode.lib.mjs

@@ -343,9 +343,19 @@ export const executeOpenCodeCommand = async params => {
       const stdoutScanner = createJsonStreamScanner();
       const stderrScanner = createJsonStreamScanner();
 
-      const handleOpenCodeRecords = events => {
+      // Issue #2136: count the JSON records that arrive on stderr. OpenCode has
+      // no terminal-event completion gate (success is decided by the exit code),
+      // so unlike codex and qwen these records cannot fail a healthy run — but
+      // they do feed `lastTextContent` and token usage, and a CLI that echoes a
+      // nested agent's stream would silently skew both. Issue #1263 added stderr
+      // parsing deliberately (some OpenCode-derived CLIs emit their records
+      // there), so the behaviour is kept and only made visible.
+      let stderrJsonRecordCount = 0;
+
+      const handleOpenCodeRecords = (events, { source = 'stdout' } = {}) => {
         for (const event of events) {
           if (event.type !== 'json') continue;
+          if (source === 'stderr') stderrJsonRecordCount++;
           const data = sanitizeObjectStrings(event.value);
           // Issue #1968: a bare `null`/primitive record must not abort the
           // rest of the chunk (data.type access would throw on null).
@@ -399,7 +409,7 @@ export const executeOpenCodeCommand = async params => {
             allOutput += errorOutput;
 
             // Issue #1263: Also parse stderr for text content
-            handleOpenCodeRecords(stderrScanner.write(errorOutput));
+            handleOpenCodeRecords(stderrScanner.write(errorOutput), { source: 'stderr' });
           }
         } else if (chunk.type === 'exit') {
           exitCode = chunk.code;
@@ -408,7 +418,11 @@ export const executeOpenCodeCommand = async params => {
 
       // Release any record that was still being assembled when the stream ended.
       handleOpenCodeRecords(stdoutScanner.flush());
-      handleOpenCodeRecords(stderrScanner.flush());
+      handleOpenCodeRecords(stderrScanner.flush(), { source: 'stderr' });
+
+      if (stderrJsonRecordCount > 0) {
+        await log(`🪞 JSON records parsed from OpenCode stderr: ${stderrJsonRecordCount} (issue #2136: stderr is not a protocol stream — check these before trusting usage/summary)`, { verbose: true });
+      }
 
       // Clean up the opencode.json config file to avoid polluting the repository
       try {

package/src/qwen.lib.mjs

@@ -143,6 +143,10 @@ const extractTextFragments = value => {
 
 const createQwenParserState = state => ({
   buffer: state?.buffer || '',
+  // Issue #2136: stderr is framed independently of stdout — mixing the two into
+  // one buffer would splice half of a stdout record onto a stderr line.
+  telemetryBuffer: state?.telemetryBuffer || '',
+  telemetryEventCounts: { ...(state?.telemetryEventCounts || {}) },
   plainText: state?.plainText || '',
   parsedEvents: Array.isArray(state?.parsedEvents) ? [...state.parsedEvents] : [],
   eventCounts: { ...(state?.eventCounts || {}) },
@@ -324,24 +328,45 @@ const addQwenEventToState = (state, rawEvent) => {
   applyQwenUsageToState(state, event);
 };
 
-export const parseQwenStreamJsonOutput = (output, state = {}) => {
+// Issue #2136: qwen-code writes its stream-json protocol to stdout; stderr is
+// human/diagnostic text. Feeding stderr through the protocol parser made every
+// JSON object a CLI happened to print there — including output a task's own
+// commands echoed back — a genuine qwen event, which could invent an `errors`
+// entry (an instant run failure), hijack the session id, add phantom token usage
+// or satisfy the #1990 terminal-event gate. Codex was bitten by exactly this
+// (see docs/case-studies/issue-2136), so qwen now marks non-stdout records as
+// telemetry: they are counted for diagnostics and otherwise ignored. Plain-text
+// stderr signals are unaffected — auth/usage-limit/retry classification still
+// reads the raw combined output.
+export const parseQwenStreamJsonOutput = (output, state = {}, { source = 'stdout' } = {}) => {
   const nextState = createQwenParserState(state);
   const text = output?.toString?.() ?? String(output || '');
   nextState.plainText += text;
 
+  const isProtocolStream = source === 'stdout';
+
   // Issue #2119: frame the stream by balanced JSON values instead of by lines.
   // `formal-ai with qwen` emits pretty-printed, multi-line records, so every
   // line failed to parse and every event - including the token usage - was
   // dropped. Scanning for balanced values also covers records concatenated
   // without a separator and records split across two process chunks.
-  const { records, rest } = takeJsonRecords(`${nextState.buffer}${text}`);
-  nextState.buffer = rest;
+  const pendingBuffer = isProtocolStream ? nextState.buffer : nextState.telemetryBuffer;
+  const { records, rest } = takeJsonRecords(`${pendingBuffer}${text}`);
+  if (isProtocolStream) {
+    nextState.buffer = rest;
+  } else {
+    nextState.telemetryBuffer = rest;
+  }
 
   for (const record of records) {
-    if (Array.isArray(record)) {
-      for (const item of record) addQwenEventToState(nextState, item);
-    } else {
-      addQwenEventToState(nextState, record);
+    const items = Array.isArray(record) ? record : [record];
+    for (const item of items) {
+      if (isProtocolStream) {
+        addQwenEventToState(nextState, item);
+        continue;
+      }
+      const eventType = item?.type || item?.event || 'unknown';
+      nextState.telemetryEventCounts[eventType] = (nextState.telemetryEventCounts[eventType] || 0) + 1;
     }
   }
 
@@ -551,7 +576,8 @@ export const executeQwenCommand = async params => {
           if (errorOutput) {
             await log(errorOutput, { stream: 'stderr' });
             allOutput += errorOutput;
-            qwenState = parseQwenStreamJsonOutput(errorOutput, qwenState);
+            // Issue #2136: stderr is diagnostics, not the qwen protocol stream.
+            qwenState = parseQwenStreamJsonOutput(errorOutput, qwenState, { source: 'stderr' });
           }
         } else if (chunk.type === 'exit') {
           exitCode = chunk.code;
@@ -562,6 +588,14 @@ export const executeQwenCommand = async params => {
         qwenState = parseQwenStreamJsonOutput(`${qwenState.buffer}\n`, { ...qwenState, buffer: '' });
       }
 
+      // Issue #2136: make the ignored non-protocol JSON visible, so a future
+      // investigation can tell "qwen emitted no result event" apart from "the
+      // records in the log came from stderr and were correctly ignored".
+      const qwenTelemetryTypes = Object.entries(qwenState.telemetryEventCounts || {});
+      if (qwenTelemetryTypes.length > 0) {
+        await log(`🪞 JSON records on qwen stderr (ignored, not protocol events): ${qwenTelemetryTypes.map(([type, count]) => `${type}=${count}`).join(', ')}`, { verbose: true });
+      }
+
       const sessionId = qwenState.sessionId || null;
       const resultSummary = qwenState.lastTextContent || null;
       const errorMessage = qwenState.errors