@yemi33/minions 0.1.2004 → 0.1.2005

package/dashboard.js

@@ -1937,6 +1937,21 @@ const CC_LIVE_STREAM_MAX_AGE_MS = shared.ENGINE_DEFAULTS.ccLiveStreamMaxAgeMs;
 // edits aren't killed mid-stream and the backend timeout never beats the user's reading
 // time. The doc-chat handlers still abort on client disconnect.
 const DOC_CHAT_TIMEOUT_MS = 60 * 60 * 1000;
+// W-mpetru71000re5de — bound the SSE per-tab queue and force-close streams
+// whose consumer has been backpressured >30s. writeCcEvent used to log
+// [cc-sse-backpressure] and silently return true on res.write()===false;
+// Node's WritableState.buffered[] has no upper bound, so a backgrounded tab
+// whose socket is half-open (Windows default TCP keepalive 7200s) can
+// accumulate bytes forever — one tab can push the dashboard past V8's 4 GB
+// heap and silently OOM-kill. Smoking gun: a single CC stream sat open
+// 8.5 min with 45 tool events queued and 0 bytes flushed
+// ([cc-stream] reason=heartbeat-write-failed duration=511442ms chunks=0
+// tools=45 bytes=0). Shedding is safe because liveState.text/.tools are
+// populated BEFORE the writer({...}) call (dashboard.js:6753, 6760, 6854,
+// 6864), so dropped wire frames are fully recoverable via the
+// reconnect-replay protocol (dashboard.js:7048-7083).
+const SSE_MAX_QUEUE_BYTES = 4 * 1024 * 1024;   // 4 MB per-tab — conservative, tunable
+const SSE_STUCK_KILL_MS = 30 * 1000;           // 30s of continuous backpressure → res.destroy()
 function _releaseCCTab(tabId) { ccInFlightTabs.delete(tabId); ccInFlightAborts.delete(tabId); }
 function _getCcLiveStream(tabId) {
   return ccLiveStreams.get(tabId) || null;
@@ -6372,6 +6387,13 @@ What would you like to discuss or change? When you're happy, say "approve" and I
     let _docStreamEnded = false;
     let _docHeartbeatTimer = null;
     const writeDocEvent = (payload) => {
+      // TODO(W-mpetru71000re5de): doc-chat SSE has the same unbounded-queue
+      // failure mode as CC's writeCcEvent — res.write() returning false from
+      // backpressure silently queues bytes in Node's WritableState.buffered[].
+      // Out of scope for this fix (task is scoped to CC only). When this is
+      // addressed, mirror the SSE_MAX_QUEUE_BYTES shed + SSE_STUCK_KILL_MS
+      // heartbeat force-close pattern from the writeCcEvent closure
+      // (dashboard.js, search for SSE_MAX_QUEUE_BYTES).
       try {
         res.write('data: ' + JSON.stringify(payload) + '\n\n');
         return true;
@@ -7315,6 +7337,17 @@ What would you like to discuss or change? When you're happy, say "approve" and I
     let _ccStreamEnded = false;
     let _ccHeartbeatTimer = null;
     let _ccLastHeartbeatAt = Date.now();
+    // W-mpetru71000re5de — per-stream backpressure clock + approximate queued
+    // bytes. _bpStartedAt timestamps the first write that returned false (and
+    // is reset on 'drain'); _queuedBytes accumulates write sizes pushed past
+    // res.writable's highWaterMark (best-effort — Node's internal buffer
+    // length is private). Used by writeCcEvent (shed) + the heartbeat tick
+    // (force-close stuck streams).
+    let _bpStartedAt = null;
+    let _queuedBytes = 0;
+    try {
+      res.on('drain', () => { _bpStartedAt = null; _queuedBytes = 0; });
+    } catch { /* listener registration is best-effort */ }
     // W-mpdavudb000v8446 — SSE delivery telemetry. Previously writeCcEvent
     // swallowed all write failures (res.destroyed / res.write returning false
     // for backpressure / sync throw), and the [cc-timing] log only proved
@@ -7351,6 +7384,30 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         _logFail('json-serialize-failed', { error: String((err && err.message) || err).slice(0, 200) });
         return false;
       }
+      // W-mpetru71000re5de — shed wire frames once the per-tab queue exceeds
+      // the cap. Safe ONLY because liveState.text / liveState.tools are
+      // populated BEFORE writer({...}) is called at every call site:
+      //   - dashboard.js:6753-6754  (legacy direct path: text  → writer)
+      //   - dashboard.js:6760-6761  (legacy direct path: tools → writer)
+      //   - dashboard.js:6854-6855  (pool path: text  → writer)
+      //   - dashboard.js:6864-6865  (pool path: tools → writer)
+      // The reconnect path (dashboard.js:7048-7083) replays from liveState.*,
+      // so dropped wire frames are fully recoverable on reattach. We still
+      // bump _ccTelemetry counters so the [cc-stream] outcome log line stays
+      // truthful about what the orchestrator produced — only the wire was
+      // shed, the work happened.
+      if (_queuedBytes > SSE_MAX_QUEUE_BYTES) {
+        try {
+          shared.log('warn', `[cc-sse-shed] tab=${tabId || _ccTelemetry.tabId || 'unknown'} type=${type} queuedBytes=${_queuedBytes} wireBytes=${wire.length}`);
+        } catch { /* telemetry is best-effort */ }
+        if (type === 'chunk') {
+          _ccTelemetry.chunks++;
+          _ccTelemetry.bytes += Buffer.byteLength(String((payload && payload.text) || ''), 'utf8');
+        } else if (type === 'tool') {
+          _ccTelemetry.tools++;
+        }
+        return true;
+      }
       let writeOk;
       try { writeOk = res.write(wire); }
       catch (err) {
@@ -7362,8 +7419,16 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         // The write IS still queued, so don't treat this as a failure, but
         // surface it so a slow consumer is visible in telemetry. Most CC
         // chunks are small enough that we never hit this in practice.
+        // W-mpetru71000re5de — also start (or extend) the per-stream
+        // backpressure clock and accumulate approximate queued bytes. The
+        // heartbeat tick force-closes the stream once the clock exceeds
+        // SSE_STUCK_KILL_MS; writeCcEvent above sheds further frames once
+        // _queuedBytes exceeds SSE_MAX_QUEUE_BYTES. _bpStartedAt + _queuedBytes
+        // are reset by the res.on('drain') listener registered above.
+        if (_bpStartedAt == null) _bpStartedAt = Date.now();
+        _queuedBytes += wire.length;
         try {
-          shared.log('warn', `[cc-sse-backpressure] tab=${tabId || _ccTelemetry.tabId || 'unknown'} type=${type} bytes=${wire.length}`);
+          shared.log('warn', `[cc-sse-backpressure] tab=${tabId || _ccTelemetry.tabId || 'unknown'} type=${type} bytes=${wire.length} queuedBytes=${_queuedBytes} bpMs=${Date.now() - _bpStartedAt}`);
         } catch { /* telemetry is best-effort */ }
       }
       if (payload && payload.type === 'chunk') {
@@ -7405,6 +7470,22 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           stopCcHeartbeat();
           return;
         }
+        // W-mpetru71000re5de — force-close streams stuck on backpressure.
+        // res.destroy() fires req.on('close'), which triggers the existing
+        // teardown path (detach writer → schedule abort → cleanup), so the
+        // queued bytes are sheddable via GC and the LLM is aborted. Catch-22
+        // workaround: _scheduleCcLiveAbort bails while state.writer is
+        // non-null and writer is non-null until req.close fires; that's
+        // exactly what res.destroy() triggers.
+        if (_bpStartedAt && Date.now() - _bpStartedAt > SSE_STUCK_KILL_MS) {
+          const stuckMs = Date.now() - _bpStartedAt;
+          try {
+            shared.log('warn', `[cc-sse-stuck-close] tab=${tabId || _ccTelemetry.tabId || 'unknown'} stuckMs=${stuckMs} queuedBytes=${_queuedBytes}`);
+          } catch { /* telemetry is best-effort */ }
+          stopCcHeartbeat();
+          try { res.destroy(); } catch { /* swallow — req.on('close') will still fire */ }
+          return;
+        }
         _checkStall();
         if (!writeCcEvent({ type: 'heartbeat' })) {
           stopCcHeartbeat();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2004",
+  "version": "0.1.2005",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"