cc-viewer 1.7.0 → 1.7.2

package/server/lib/v2/live-feed.js

@@ -26,8 +26,11 @@ import { watch, existsSync, readdirSync, statSync, openSync, readSync, closeSync
 import { join } from 'node:path';
 import { reportSwallowed } from '../error-report.js';
 import { createIncrementalReconstructor } from '../delta-reconstructor.js';
-import { processWatchedEntry } from '../log-watcher.js';
+import { processWatchedEntry, sendEventToClients, sendEventRawToClients } from '../log-watcher.js';
 import { SessionSynthesizer } from './adapter.js';
+import { isDiscardableSession } from './session-select.js';
+import { computeCacheLoss } from './meta-rows.js';
+import { classifyRequest } from '../../../src/utils/requestType.js';
 
 const FSWATCH_DEBOUNCE_MS = 80;
 const SAFETY_POLL_MS = 5000;
@@ -36,11 +39,26 @@ const ATTACH_RECENT_MS = 5 * 60 * 1000; // pre-existing dirs considered "live"
 const TICK_RETRY_MS = 250;      // in-process tick raced the first queue drain
 const TICK_RETRY_MAX = 8;
 
-/** Read newly appended complete lines from a file cursor {path, offset,
- *  pending}. Returns an array of raw line strings (possibly empty). A file
+const READ_CHUNK_BYTES = 8 * 1024 * 1024;
+// Node's max string length (~512MiB) — a longer line can never be decoded.
+const MAX_LINE_BYTES = 0x1fffffe8;
+
+/** Read newly appended complete lines from a file cursor {path, offset}
+ *  (partial-line carry lives on lazily-initialized _pendBufs/_pendBytes/
+ *  _skipLine cursor fields). Returns an array of raw line strings (possibly
+ *  empty). A file
  *  that shrank (should never happen — v2 files are append-only) returns null
- *  so the caller can rebuild the cursor. */
-function readNewLines(cursor) {
+ *  so the caller can rebuild the cursor.
+ *
+ *  Chunked + byte-level newline split (issue #129 twin): the first attach to
+ *  a session seeds from offset 0, so one whole-file read + decode would throw
+ *  ERR_STRING_TOO_LONG on an oversized journal and crash-loop the boot path
+ *  exactly like the cold-scan crash the streaming reader fixed. The partial
+ *  tail line is carried on the cursor as Buffer fragments (never decoded
+ *  until its newline lands); a single line past the string cap is skipped
+ *  with a report instead of thrown. Exported for tests (chunkBytes seam).
+ */
+export function readNewLines(cursor, chunkBytes = READ_CHUNK_BYTES, maxLineBytes = MAX_LINE_BYTES) {
   let size;
   try {
     size = statSync(cursor.path).size;
@@ -49,23 +67,74 @@ function readNewLines(cursor) {
   }
   if (size < cursor.offset) return null;
   if (size === cursor.offset) return [];
-  const toRead = size - cursor.offset;
-  const buf = Buffer.alloc(toRead);
+  if (cursor._pendBufs === undefined) {
+    cursor._pendBufs = [];
+    cursor._pendBytes = 0;
+    cursor._skipLine = false;
+  }
+  const out = [];
   let fd;
   try {
     fd = openSync(cursor.path, 'r');
-    readSync(fd, buf, 0, toRead, cursor.offset);
+    const chunk = Buffer.alloc(Math.min(chunkBytes, size - cursor.offset));
+    while (cursor.offset < size) {
+      const toRead = Math.min(chunk.length, size - cursor.offset);
+      const n = readSync(fd, chunk, 0, toRead, cursor.offset);
+      if (n === 0) break;
+      cursor.offset += n;
+      const view = chunk.subarray(0, n);
+      let from = 0;
+      while (from < n) {
+        const nl = view.indexOf(10, from);
+        if (nl === -1) {
+          if (!cursor._skipLine) {
+            const restLen = n - from;
+            if (cursor._pendBytes + restLen > maxLineBytes) {
+              cursor._skipLine = true;
+              cursor._pendBufs = [];
+              cursor._pendBytes = 0;
+              reportSwallowed('v2-live.read-line-too-long', new Error(`${cursor.path}: line exceeds ${maxLineBytes} bytes — skipped`));
+            } else {
+              // chunk is reused next readSync — the carried slice must own its bytes
+              cursor._pendBufs.push(Buffer.from(view.subarray(from)));
+              cursor._pendBytes += restLen;
+            }
+          }
+          break;
+        }
+        if (cursor._skipLine) {
+          cursor._skipLine = false; // the oversized line ends at this newline
+        } else {
+          const seg = view.subarray(from, nl);
+          let text = null;
+          try {
+            if (cursor._pendBufs.length > 0) {
+              cursor._pendBufs.push(seg);
+              text = Buffer.concat(cursor._pendBufs).toString('utf-8');
+            } else {
+              text = seg.toString('utf-8');
+            }
+          } catch (err) {
+            reportSwallowed('v2-live.read-line-too-long', err);
+          }
+          cursor._pendBufs = [];
+          cursor._pendBytes = 0;
+          if (text !== null) {
+            const t = text.trim();
+            if (t) out.push(t);
+          }
+        }
+        from = nl + 1;
+      }
+    }
   } catch (err) {
+    // Deliver what was already parsed; offset only advanced past read bytes,
+    // so the next poll resumes from the failure point.
     reportSwallowed('v2-live.read', err);
-    return [];
   } finally {
     if (fd !== undefined) { try { closeSync(fd); } catch { /* already closed */ } }
   }
-  cursor.offset = size;
-  const chunk = cursor.pending + buf.toString('utf-8');
-  const parts = chunk.split('\n');
-  cursor.pending = parts.pop() || '';
-  return parts.map((l) => l.trim()).filter(Boolean);
+  return out;
 }
 
 export class V2LiveFeed {
@@ -91,8 +160,13 @@ export class V2LiveFeed {
     this._safetyPollMs = typeof opts.safetyPollMs === 'number' ? opts.safetyPollMs : SAFETY_POLL_MS;
     this._deferMs = typeof opts.deferMs === 'number' ? opts.deferMs : DEFER_MS;
     this._now = opts.now || Date.now;
+    // Wire v3 (V3.S2): when on, every emitted item ALSO broadcasts a metadata
+    // row (v2_requests_delta). Explicit ctor param — this module has no access
+    // to the server deps object.
+    this._wireV3 = !!opts.wireV3;
     this._sessions = new Map(); // dir → cursor bundle
     this._seenDirs = new Map(); // dir → last observed journal mtimeMs (attached or not)
+    this._discardGated = new Set(); // dirs refused by the discard gate — first real attach must NOT suppress history
     this._sessionsRoot = null;
     this._rootWatcher = null;
     this._safetyTimer = null;
@@ -120,6 +194,7 @@ export class V2LiveFeed {
     for (const cur of this._sessions.values()) this._closeCursor(cur);
     this._sessions.clear();
     this._seenDirs.clear();
+    this._discardGated.clear();
     this._sessionsRoot = null;
   }
 
@@ -240,6 +315,23 @@ export class V2LiveFeed {
   _attach(dir, { suppressExisting }) {
     if (this._sessions.has(dir)) return this._sessions.get(dir);
     if (!existsSync(join(dir, 'journal.jsonl'))) return null;
+    // Discardable sessions (quota-probe orphans — no main/teammate req, no
+    // meta.leader) are never followed: single choke point, every attach path
+    // (_initialScan / _maybeAttachNew / tick / _safetyTick / _rebuildCursor)
+    // funnels here and handles null. Self-healing when a dir later gains its
+    // first main: the leader's own dir re-attaches via tick's retry chain
+    // (sub-second); cross-process dirs via the 5s safety poll's mtime-bump
+    // scan (_seenDirs bookkeeping stays in the callers, so the bump fires).
+    if (isDiscardableSession(dir)) {
+      this._discardGated.add(dir);
+      return null;
+    }
+    // A dir previously refused by the discard gate is attaching for the FIRST
+    // time — its first renderable turn was never broadcast, so the safety
+    // poll's suppressExisting:true (meant for "old stale dir resumed") must
+    // not swallow it: cross-process producers (IM worker, second ccv) have no
+    // cold-load fallback for a connected client.
+    if (this._discardGated.delete(dir)) suppressExisting = false;
     const cur = {
       dir,
       synth: new SessionSynthesizer(dir, { deferMs: this._deferMs, now: this._now }),
@@ -248,9 +340,9 @@ export class V2LiveFeed {
       // rebase each other's accumulated baseline — v1's tail followed exactly
       // one file, so a single shared reconstructor was a new interleave surface.
       reconstructor: createIncrementalReconstructor(),
-      journal: { path: join(dir, 'journal.jsonl'), offset: 0, pending: '' },
-      responses: { path: join(dir, 'responses.jsonl'), offset: 0, pending: '' },
-      convFiles: new Map(), // path → {key, offset, pending}
+      journal: { path: join(dir, 'journal.jsonl'), offset: 0 },
+      responses: { path: join(dir, 'responses.jsonl'), offset: 0 },
+      convFiles: new Map(), // path → {key, offset} (+ lazy _pendBufs carry)
       watcher: null,
       debounce: null,
       reading: false,
@@ -299,7 +391,13 @@ export class V2LiveFeed {
         this._feedConvFiles(cur);
         const respLines = readNewLines(cur.responses);
         if (respLines === null) { this._rebuildCursor(cur); return; }
-        for (const raw of respLines) cur.synth.ingestResponseLine(raw);
+        for (const raw of respLines) {
+          cur.synth.ingestResponseLine(raw);
+          // Wire v3 (V3.S4): forward the raw stored line — the flagged client
+          // assembles v1-shape entries from native lines locally. Suppressed
+          // during seed (history is covered by the cold-load channel).
+          if (this._wireV3 && !cur.suppress) this._forwardNative(cur, 'resp', null, raw);
+        }
         const journalLines = readNewLines(cur.journal);
         if (journalLines === null) { this._rebuildCursor(cur); return; }
         for (const raw of journalLines) {
@@ -341,7 +439,7 @@ export class V2LiveFeed {
         const p = join(convRoot, key, f);
         let fc = cur.convFiles.get(p);
         if (!fc) {
-          fc = { key, path: p, offset: 0, pending: '' };
+          fc = { key, path: p, offset: 0 };
           cur.convFiles.set(p, fc);
         }
         const lines = readNewLines(fc);
@@ -351,7 +449,10 @@ export class V2LiveFeed {
           try { ev = JSON.parse(raw); } catch (err) {
             reportSwallowed('v2-live.conv-parse', new Error(`${cur.dir}/${key}: ${err.message}`));
           }
-          if (ev) cur.synth.ingestConvLine(key, ev);
+          if (ev) {
+            cur.synth.ingestConvLine(key, ev);
+            if (this._wireV3 && !cur.suppress) this._forwardNative(cur, 'conv', key, raw);
+          }
         }
       }
     }
@@ -403,17 +504,118 @@ export class V2LiveFeed {
           clients: this._clients,
           getClaudePid: this._getClaudePid,
           runParallelHook: this._runParallelHook,
+          suppressEntryBroadcast: this._wireV3,
         });
         emitted++;
       } catch (err) {
         reportSwallowed('v2-live.emit', err);
       }
+      // Wire v3: derive the metadata row from the post-reconstruction entry
+      // (main deltas carry full messages here — classification/cacheLoss get
+      // the same inputs the legacy list sees). Row-level errors must not
+      // disturb the legacy broadcast above.
+      if (this._wireV3) {
+        try {
+          this._emitRow(cur, item, parsed);
+        } catch (err) {
+          reportSwallowed('v2-live.row', err);
+        }
+      }
     }
     if (emitted > 0 && this._notifyStatsWorker) {
       try { this._notifyStatsWorker(cur.dir); } catch (err) { reportSwallowed('v2-live.stats-notify', err); }
     }
   }
 
+  /** Wire v3 (V3.S4): broadcast one raw native line (conv event or responses
+   *  line) — the flagged client's assembler replays these into v1-shape
+   *  entries. `raw` is forwarded verbatim inside a JSON envelope carrying the
+   *  session identity + channel. sessionId comes from the synthesizer (the
+   *  UUID, matching the rows' sessionId and the entries' _seqEpoch). */
+  _forwardNative(cur, kind, channel, raw) {
+    try {
+      const payload = kind === 'conv'
+        ? `{"sessionId":${JSON.stringify(cur.synth.sessionId)},"channel":${JSON.stringify(channel)},"line":${raw}}`
+        : `{"sessionId":${JSON.stringify(cur.synth.sessionId)},"line":${raw}}`;
+      const eventName = kind === 'conv' ? 'v3_conv' : 'v3_resp';
+      sendEventRawToClients(this._clients, eventName, payload);
+    } catch (err) {
+      reportSwallowed('v2-live.native-forward', err);
+    }
+  }
+
+  /** Wire v3 (V3.S2): one metadata row per emitted item, derived from the
+   *  post-reconstruction entry. Rows upsert client-side by (sessionId, seq):
+   *  a placeholder row precedes its completed row, and a correction re-send
+   *  follows when the NEXT request's arrival changes the previous row's
+   *  classification (classifyRequest's Preflight/Plan cases read nextReq). */
+  _rowFrom(item, parsed) {
+    const usage = parsed.response?.body?.usage || null;
+    return {
+      seq: item.seq,
+      sessionId: item.sessionId,
+      timestamp: parsed.timestamp || '',
+      url: parsed.url || '',
+      method: parsed.method || 'POST',
+      // conv/evt/kind/mainAgent mirror the cold fold (meta-rows.js foldDir) —
+      // journal truth, NOT re-derivation. conv is load-bearing: the client
+      // assembler's buildEntry is `if (row.conv)`-gated, so a conv-less live
+      // row rebuilt every entry with EMPTY messages (chat vanished at stream
+      // end). mainAgent must be kind-derived like cold: parsed.mainAgent
+      // re-derives from the body and mis-tags countTokens (main system/tools
+      // aboard) as true, which would merge its turn into the chat live-only.
+      conv: item.conv,
+      evt: item.evt,
+      kind: item.kind || (item.isMain ? 'main' : (parsed.teammate ? 'teammate' : 'sub')),
+      mainAgent: item.isMain === true,
+      teammate: parsed.teammate || undefined,
+      model: parsed.body?.model,
+      proxyUrl: parsed.proxyUrl || undefined,
+      status: parsed.response?.status,
+      duration: parsed.duration,
+      usage: usage ? {
+        input_tokens: usage.input_tokens || 0,
+        output_tokens: usage.output_tokens || 0,
+        cache_read_input_tokens: usage.cache_read_input_tokens || 0,
+        cache_creation_input_tokens: usage.cache_creation_input_tokens || 0,
+      } : null,
+      inProgress: !!parsed.inProgress,
+      typeTag: null,
+      cacheLoss: null,
+    };
+  }
+
+  _emitRow(cur, item, parsed) {
+    const row = this._rowFrom(item, parsed);
+    try {
+      const tag = classifyRequest(parsed, null);
+      row.typeTag = tag ? { type: tag.type, subType: tag.subType ?? null } : null;
+    } catch (err) { reportSwallowed('v2-live.row-classify', err); }
+    if (row.mainAgent && !row.inProgress) {
+      const u = parsed.response?.body?.usage;
+      const cw = (u && u.cache_creation_input_tokens) || 0;
+      const cr = (u && u.cache_read_input_tokens) || 0;
+      if (cur._v3PrevMain && cw > 0 && cw > cr) {
+        try { row.cacheLoss = computeCacheLoss(cur._v3PrevMain, parsed); } catch (err) { reportSwallowed('v2-live.row-cacheloss', err); }
+      }
+      cur._v3PrevMain = parsed; // one full entry retained per cursor (bounded)
+    }
+    // Lookahead correction: this entry is the previous row's nextReq.
+    const prev = cur._v3Last;
+    if (prev && (prev.row.sessionId !== row.sessionId || prev.row.seq !== row.seq)) {
+      try {
+        const tag = classifyRequest(prev.entry, parsed);
+        const corrected = tag ? { type: tag.type, subType: tag.subType ?? null } : null;
+        if (JSON.stringify(corrected) !== JSON.stringify(prev.row.typeTag)) {
+          prev.row.typeTag = corrected;
+          sendEventToClients(this._clients, 'v2_requests_delta', prev.row);
+        }
+      } catch (err) { reportSwallowed('v2-live.row-classify', err); }
+    }
+    cur._v3Last = { row, entry: parsed };
+    sendEventToClients(this._clients, 'v2_requests_delta', row);
+  }
+
   // ── safety poll ────────────────────────────────────────────────────────────
 
   _safetyTick() {