cc-viewer 1.7.1 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{App-gUSmU6Ny.js → App-B224r-8M.js} +1 -1
- package/dist/assets/{MdxEditorPanel-DAR39A3S.js → MdxEditorPanel-BUubiE6Y.js} +1 -1
- package/dist/assets/{Mobile-qfRsRth0.js → Mobile-CKmfRRvo.js} +1 -1
- package/dist/assets/{index-B51Ci0GL.js → index-CjsQEQc2.js} +2 -2
- package/dist/assets/{seqResourceLoaders-DZJxMMaR.js → seqResourceLoaders-C-T_EMiO.js} +2 -2
- package/dist/index.html +1 -1
- package/package.json +1 -1
- package/server/lib/log-management.js +1 -0
- package/server/lib/stats-worker.js +21 -2
- package/server/lib/v2/adapter.js +52 -12
- package/server/lib/v2/journal.js +9 -5
- package/server/lib/v2/jsonl-read.js +127 -0
- package/server/lib/v2/live-feed.js +114 -19
- package/server/lib/v2/meta-rows.js +22 -21
- package/server/lib/v2/replay.js +5 -5
- package/server/lib/v2/session-select.js +69 -2
- package/server/routes/im.js +1 -1
- package/server/workspace-registry.js +5 -0
package/dist/index.html
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
// 整体显示大小已弃用 CSS zoom:Electron 改用 webFrame.setZoomFactor(首屏抢占见
|
|
22
22
|
// electron/tab-content-preload.js),纯浏览器交由用户用浏览器自带快捷键缩放,故此处不再设 zoom。
|
|
23
23
|
</script>
|
|
24
|
-
<script type="module" crossorigin src="./assets/index-
|
|
24
|
+
<script type="module" crossorigin src="./assets/index-CjsQEQc2.js"></script>
|
|
25
25
|
<link rel="modulepreload" crossorigin href="./assets/vendor-antd-DI7JL-mE.js">
|
|
26
26
|
<link rel="modulepreload" crossorigin href="./assets/vendor-codemirror-B9c49dtM.js">
|
|
27
27
|
<link rel="modulepreload" crossorigin href="./assets/vendor-mdxeditor-B6cpBtIE.js">
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cc-viewer",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.2",
|
|
4
4
|
"description": "Claude Code logging, visualization, and management toolkit — launch a web viewer alongside Claude Code with full request/response tracing, proxy, and mobile support",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"main": "server.js",
|
|
@@ -99,6 +99,7 @@ export function listV2Logs(logDir, currentProjectName) {
|
|
|
99
99
|
for (const s of listV2Sessions(join(logDir, project))) {
|
|
100
100
|
if (s.leader) continue;
|
|
101
101
|
if (s.size === 0) continue;
|
|
102
|
+
if (s.discard) continue; // quota-probe orphans: never listed (2026-07-16)
|
|
102
103
|
if (!grouped[project]) grouped[project] = [];
|
|
103
104
|
grouped[project].push({
|
|
104
105
|
file: `v2:${project}/${s.sid}`,
|
|
@@ -9,6 +9,8 @@ import { readFileSync, writeFileSync, existsSync, readdirSync, statSync } from '
|
|
|
9
9
|
import { join, basename } from 'node:path';
|
|
10
10
|
import { readJsonlTolerant, listSessionIds } from './v2/replay.js';
|
|
11
11
|
import { dirSizeSync } from './v2/layout.js';
|
|
12
|
+
import { isDiscardableSession } from './v2/session-select.js';
|
|
13
|
+
import { reportSwallowed } from './error-report.js';
|
|
12
14
|
import {
|
|
13
15
|
INTER_SESSION_TYPES, isSystemText, extractUserTexts, isSuggestionMode,
|
|
14
16
|
collectPromptsFromEvents, sortEpochFiles,
|
|
@@ -23,7 +25,11 @@ export { INTER_SESSION_TYPES, isSystemText, extractUserTexts };
|
|
|
23
25
|
// v9: v2 session-dir units (files map keyed `sessions/<sid>`), journal-based counts
|
|
24
26
|
// v10: per-session `size` = recursive session-dir bytes (was journal-only);
|
|
25
27
|
// `journalSize` carries the incremental-cache key
|
|
26
|
-
|
|
28
|
+
// v11: discardable sessions (quota-probe orphans) excluded — the bump
|
|
29
|
+
// invalidates pre-discard caches so their probe units can't be reused
|
|
30
|
+
// back into filesStats, which lets the discard check sit AFTER the
|
|
31
|
+
// cache-reuse branch (cache hits skip the journal head scan entirely)
|
|
32
|
+
const STATS_VERSION = 11;
|
|
27
33
|
|
|
28
34
|
/**
|
|
29
35
|
* Parse one v2 session directory into the same stats shape parseJsonlFile
|
|
@@ -199,7 +205,20 @@ function generateProjectStats(projectDir, projectName, onlyFile) {
|
|
|
199
205
|
}
|
|
200
206
|
|
|
201
207
|
const sessionDir = join(projectDir, 'sessions', sid);
|
|
202
|
-
|
|
208
|
+
// Discardable sessions (quota-probe orphans — no main/teammate req, no
|
|
209
|
+
// leader) never count toward stats. Runs only on cache misses: v11+
|
|
210
|
+
// caches are written exclusively by post-discard code, so a cache hit can
|
|
211
|
+
// never resurrect a probe unit (the v10→v11 bump invalidated older ones).
|
|
212
|
+
if (isDiscardableSession(sessionDir)) continue;
|
|
213
|
+
let parsed;
|
|
214
|
+
try {
|
|
215
|
+
parsed = parseSessionDir(sessionDir);
|
|
216
|
+
} catch (err) {
|
|
217
|
+
// One unreadable session must not kill the whole worker (issue #129) —
|
|
218
|
+
// skip its stats and keep counting the healthy ones.
|
|
219
|
+
reportSwallowed('stats-worker.session-parse-failed', new Error(`${sid}: ${err.message}`));
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
203
222
|
filesStats[unitKey] = {
|
|
204
223
|
models: parsed.models,
|
|
205
224
|
summary: parsed.summary,
|
package/server/lib/v2/adapter.js
CHANGED
|
@@ -33,6 +33,8 @@ import { reportSwallowed } from '../error-report.js';
|
|
|
33
33
|
import { isMainAgentRequest } from '../interceptor-core.js';
|
|
34
34
|
import { readPromptsHead, collectPromptsFromEvents } from '../user-prompt-extract.js';
|
|
35
35
|
import { readSession, readJsonlTolerant, listSessionIds } from './replay.js';
|
|
36
|
+
import { iterateJsonlLines } from './jsonl-read.js';
|
|
37
|
+
import { isDiscardableSession } from './session-select.js';
|
|
36
38
|
import { blobPath, isSupportedWireFormat, dirSizeSync } from './layout.js';
|
|
37
39
|
import { SingleFlight } from './singleflight.js';
|
|
38
40
|
|
|
@@ -59,16 +61,17 @@ export function isV2SessionDir(p) {
|
|
|
59
61
|
function readResponsesRaw(sessionDir) {
|
|
60
62
|
const bySeq = new Map();
|
|
61
63
|
const p = join(sessionDir, 'responses.jsonl');
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
64
|
+
try {
|
|
65
|
+
// streamed line-by-line (issue #129): responses.jsonl is the most likely
|
|
66
|
+
// file to outgrow Node's string cap (full response bodies, one per request)
|
|
67
|
+
for (const t of iterateJsonlLines(p)) {
|
|
68
|
+
const m = t.match(/"seq":\s*(\d+)/);
|
|
69
|
+
if (!m) continue;
|
|
70
|
+
const seq = Number(m[1]);
|
|
71
|
+
if (!bySeq.has(seq)) bySeq.set(seq, t);
|
|
72
|
+
}
|
|
73
|
+
} catch (err) {
|
|
74
|
+
reportSwallowed('v2-read.responses-read-failed', err);
|
|
72
75
|
}
|
|
73
76
|
return bySeq;
|
|
74
77
|
}
|
|
@@ -491,6 +494,14 @@ export class SessionSynthesizer {
|
|
|
491
494
|
phase: 'placeholder',
|
|
492
495
|
entry,
|
|
493
496
|
isMain: isMainKind,
|
|
497
|
+
// Journal-truth request identity for the live row builder (live-feed
|
|
498
|
+
// _rowFrom): conv/evt are the V3.S5 assembler inputs the cold fold
|
|
499
|
+
// (meta-rows foldDir) already carries — without them on live rows the
|
|
500
|
+
// client rebuilds entries with EMPTY messages and the chat vanished at
|
|
501
|
+
// stream end until a cold reload (2026-07-16 live-render regression).
|
|
502
|
+
kind: req.kind,
|
|
503
|
+
...(req.conv && { conv: req.conv }),
|
|
504
|
+
...(req.evt && { evt: req.evt }),
|
|
494
505
|
stateRef: conv ? conv.state : null,
|
|
495
506
|
};
|
|
496
507
|
this._await.set(seq, item);
|
|
@@ -563,7 +574,15 @@ export class SessionSynthesizer {
|
|
|
563
574
|
function* iterateSessionItems(sessionDir, opts = {}) {
|
|
564
575
|
const projectDir = dirname(dirname(sessionDir));
|
|
565
576
|
const sessionId = basename(sessionDir);
|
|
566
|
-
|
|
577
|
+
let session;
|
|
578
|
+
try {
|
|
579
|
+
session = readSession(projectDir, sessionId);
|
|
580
|
+
} catch (err) {
|
|
581
|
+
// One unreadable session must degrade to "not rendered", never crash the
|
|
582
|
+
// whole scan (issue #129: an oversized file crash-looped every startup).
|
|
583
|
+
reportSwallowed('v2-read.session-read-failed', new Error(`${sessionId}: ${err.message}`));
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
567
586
|
if (session.unsupported) {
|
|
568
587
|
reportSwallowed('v2-read.unsupported-wire-format', new Error(`${sessionId}: wireFormat=${session.wireFormat}`));
|
|
569
588
|
return;
|
|
@@ -628,6 +647,12 @@ export function findTeammateSessionDirs(sessionDir, leaderUuid) {
|
|
|
628
647
|
continue;
|
|
629
648
|
}
|
|
630
649
|
if (!l) {
|
|
650
|
+
// Discardable dirs (quota-probe orphans — no main/teammate req) must
|
|
651
|
+
// never act as leader candidates: a probe fires at process startup,
|
|
652
|
+
// so its startTs sits right next to the real leader's and can win the
|
|
653
|
+
// orphan tie-break below, stealing the teammate's traffic from the
|
|
654
|
+
// real leader's folded stream.
|
|
655
|
+
if (isDiscardableSession(dir, meta)) continue;
|
|
631
656
|
leaderStarts.push({ sid: name, startTs: (meta && meta.startTs) || '' });
|
|
632
657
|
continue;
|
|
633
658
|
}
|
|
@@ -1081,8 +1106,12 @@ export function listV2Sessions(projectDir) {
|
|
|
1081
1106
|
const reqKind = new Map();
|
|
1082
1107
|
let turns = 0;
|
|
1083
1108
|
let sentinelVersion = null;
|
|
1109
|
+
let hasMainOrTeammate = false;
|
|
1084
1110
|
for (const line of readJsonlTolerant(join(dir, 'journal.jsonl'))) {
|
|
1085
|
-
if (line.ph === 'req')
|
|
1111
|
+
if (line.ph === 'req') {
|
|
1112
|
+
reqKind.set(line.seq, line.kind);
|
|
1113
|
+
if (line.kind === 'main' || line.kind === 'teammate') hasMainOrTeammate = true;
|
|
1114
|
+
}
|
|
1086
1115
|
else if (line.ph === 'done' && reqKind.get(line.seq) === 'main') {
|
|
1087
1116
|
turns++;
|
|
1088
1117
|
reqKind.delete(line.seq); // fold duplicate done lines (§14)
|
|
@@ -1117,6 +1146,17 @@ export function listV2Sessions(projectDir) {
|
|
|
1117
1146
|
turns,
|
|
1118
1147
|
size: dirSizeSync(dir),
|
|
1119
1148
|
preview,
|
|
1149
|
+
// Discardable-session verdict. KEEP IN SYNC: session-select.js
|
|
1150
|
+
// isDiscardableSession is the canonical rule; this fold pre-computes
|
|
1151
|
+
// it for free over the FULL journal (the canonical scan is 8MB-
|
|
1152
|
+
// budgeted — intentional asymmetry, a first main sits at the head).
|
|
1153
|
+
// When the fold says discard, the canonical predicate CONFIRMS it:
|
|
1154
|
+
// readJsonlTolerant swallows an I/O error (Windows EBUSY/EPERM lock)
|
|
1155
|
+
// into zero lines, which must KEEP the session, not hide it — the
|
|
1156
|
+
// canonical path carries that error→keep direction (ioErrorResult).
|
|
1157
|
+
// Main-bearing sessions never pay the extra read; probe journals are
|
|
1158
|
+
// ~3 lines.
|
|
1159
|
+
discard: !(meta && meta.leader) && !hasMainOrTeammate && isDiscardableSession(dir, meta),
|
|
1120
1160
|
});
|
|
1121
1161
|
} catch { /* one unreadable session must not break the list */ }
|
|
1122
1162
|
}
|
package/server/lib/v2/journal.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// The reader folds the two phases by seq; a req without a done is in-flight
|
|
9
9
|
// (or the process died) — that IS the v1 placeholder, minus the duplicated body.
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { iterateJsonlLines } from './jsonl-read.js';
|
|
12
12
|
|
|
13
13
|
export class Journal {
|
|
14
14
|
/**
|
|
@@ -29,11 +29,15 @@ export class Journal {
|
|
|
29
29
|
|
|
30
30
|
static _maxSeqIn(journalPath) {
|
|
31
31
|
try {
|
|
32
|
-
if (!existsSync(journalPath)) return 0;
|
|
33
32
|
let max = 0;
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
|
|
33
|
+
// Per-line regex instead of JSON.parse: tolerant of a truncated tail
|
|
34
|
+
// line and ~10x cheaper on large journals. Streamed line-by-line
|
|
35
|
+
// (issue #129): a whole-file read past Node's string cap would land in
|
|
36
|
+
// the catch below and silently RESET seq to 0 — colliding with every
|
|
37
|
+
// existing line, which the reader folds into corrupted sessions.
|
|
38
|
+
for (const line of iterateJsonlLines(journalPath)) {
|
|
39
|
+
const m = /"seq":\s*(\d+)/.exec(line);
|
|
40
|
+
if (!m) continue;
|
|
37
41
|
const n = Number(m[1]);
|
|
38
42
|
if (n > max) max = n;
|
|
39
43
|
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Wire Format v2 — streaming JSONL line reader (issue #129).
|
|
2
|
+
//
|
|
3
|
+
// `readFileSync(path, 'utf-8')` materializes the WHOLE file as one JS string
|
|
4
|
+
// and throws ERR_STRING_TOO_LONG past Node's ~512MiB string cap — a single
|
|
5
|
+
// oversized session file (giant conv epoch / journal / responses.jsonl, seen
|
|
6
|
+
// in the wild after migrating very large v1 logs) crash-looped the server on
|
|
7
|
+
// every startup scan. This reader never builds a whole-file string: it reads
|
|
8
|
+
// fixed-size chunks and splits lines at the BYTE level (scanning for 0x0A
|
|
9
|
+
// before decoding), so multi-byte UTF-8 characters straddling a chunk
|
|
10
|
+
// boundary are never torn, and only one line at a time ever becomes a string.
|
|
11
|
+
//
|
|
12
|
+
// A single line at/above the string cap can never be decoded at all — it is
|
|
13
|
+
// skipped (reported once per file via reportSwallowed) instead of throwing,
|
|
14
|
+
// mirroring readJsonlTolerant's torn-tail tolerance (spec §14).
|
|
15
|
+
|
|
16
|
+
import { existsSync, openSync, readSync, closeSync, statSync } from 'node:fs';
|
|
17
|
+
import { reportSwallowed } from '../error-report.js';
|
|
18
|
+
|
|
19
|
+
const DEFAULT_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
20
|
+
// Node's max string length is 0x1fffffe8 (~512MiB) — a line this long can
|
|
21
|
+
// never become a JS string, so it is unreadable by construction.
|
|
22
|
+
const DEFAULT_MAX_LINE_BYTES = 0x1fffffe8;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Iterate a JSONL file's lines as trimmed strings (blank lines skipped),
|
|
26
|
+
* without ever holding the whole file in one string.
|
|
27
|
+
*
|
|
28
|
+
* @param {string} path
|
|
29
|
+
* @param {{maxLineBytes?: number, chunkBytes?: number}} [opts] - test seams;
|
|
30
|
+
* production callers use the defaults.
|
|
31
|
+
* @yields {string} one trimmed non-empty line
|
|
32
|
+
*/
|
|
33
|
+
export function* iterateJsonlLines(path, opts = {}) {
|
|
34
|
+
const maxLineBytes = opts.maxLineBytes || DEFAULT_MAX_LINE_BYTES;
|
|
35
|
+
const chunkBytes = opts.chunkBytes || DEFAULT_CHUNK_BYTES;
|
|
36
|
+
if (!existsSync(path)) return;
|
|
37
|
+
let fd;
|
|
38
|
+
try {
|
|
39
|
+
fd = openSync(path, 'r');
|
|
40
|
+
} catch (err) {
|
|
41
|
+
// ENOENT = vanished mid-scan (benign TOCTOU); anything else (EBUSY/EPERM
|
|
42
|
+
// Windows locks, EMFILE) silently rendering a session empty is
|
|
43
|
+
// diagnostic-worthy — CLAUDE.md swallow rule.
|
|
44
|
+
if (err && err.code !== 'ENOENT') reportSwallowed('v2-read.open-failed', err);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
try {
|
|
48
|
+
// Cap the chunk to the file's actual size (like the sibling readers):
|
|
49
|
+
// a fixed 8MB zero-filled alloc per call is a ~2600x regression over the
|
|
50
|
+
// old readFileSync for the KB-scale journals startup scans walk.
|
|
51
|
+
let capBytes = chunkBytes;
|
|
52
|
+
try { capBytes = Math.min(chunkBytes, Math.max(statSync(path).size, 1)); }
|
|
53
|
+
catch { /* stat raced a delete — keep the full chunk; the read loop decides */ }
|
|
54
|
+
const chunk = Buffer.alloc(capBytes);
|
|
55
|
+
let carry = []; // Buffer fragments of the current (incomplete) line
|
|
56
|
+
let carryBytes = 0;
|
|
57
|
+
let skipping = false; // inside an oversized line — drop bytes until \n
|
|
58
|
+
let reported = false; // one report per file, not per oversized line
|
|
59
|
+
let pos = 0;
|
|
60
|
+
const reportOnce = (err) => {
|
|
61
|
+
if (reported) return;
|
|
62
|
+
reported = true;
|
|
63
|
+
reportSwallowed('v2-read.jsonl-line-too-long', err);
|
|
64
|
+
};
|
|
65
|
+
while (true) {
|
|
66
|
+
const n = readSync(fd, chunk, 0, chunk.length, pos);
|
|
67
|
+
if (n === 0) break;
|
|
68
|
+
pos += n;
|
|
69
|
+
const view = chunk.subarray(0, n);
|
|
70
|
+
let from = 0;
|
|
71
|
+
while (from < n) {
|
|
72
|
+
const nl = view.indexOf(10, from);
|
|
73
|
+
if (nl === -1) {
|
|
74
|
+
// no newline in the rest of this chunk — carry (or keep skipping)
|
|
75
|
+
if (!skipping) {
|
|
76
|
+
const restLen = n - from;
|
|
77
|
+
if (carryBytes + restLen > maxLineBytes) {
|
|
78
|
+
skipping = true;
|
|
79
|
+
carry = [];
|
|
80
|
+
carryBytes = 0;
|
|
81
|
+
reportOnce(new Error(`${path}: line exceeds ${maxLineBytes} bytes — skipped`));
|
|
82
|
+
} else {
|
|
83
|
+
// chunk is reused by the next readSync — the carried slice must own its bytes
|
|
84
|
+
carry.push(Buffer.from(view.subarray(from)));
|
|
85
|
+
carryBytes += restLen;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
if (skipping) {
|
|
91
|
+
skipping = false; // the oversized line ends at this newline
|
|
92
|
+
} else {
|
|
93
|
+
const seg = view.subarray(from, nl);
|
|
94
|
+
let text = null;
|
|
95
|
+
try {
|
|
96
|
+
if (carry.length > 0) {
|
|
97
|
+
carry.push(seg);
|
|
98
|
+
text = Buffer.concat(carry).toString('utf-8');
|
|
99
|
+
} else {
|
|
100
|
+
text = seg.toString('utf-8');
|
|
101
|
+
}
|
|
102
|
+
} catch (err) {
|
|
103
|
+
reportOnce(err); // decode failed (line at the string cap) — skip it
|
|
104
|
+
}
|
|
105
|
+
carry = [];
|
|
106
|
+
carryBytes = 0;
|
|
107
|
+
if (text !== null) {
|
|
108
|
+
const t = text.trim();
|
|
109
|
+
if (t) yield t;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
from = nl + 1;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// trailing line without a final newline (torn tail is the caller's concern)
|
|
116
|
+
if (!skipping && carry.length > 0) {
|
|
117
|
+
let text = null;
|
|
118
|
+
try { text = Buffer.concat(carry).toString('utf-8'); } catch (err) { reportOnce(err); }
|
|
119
|
+
if (text !== null) {
|
|
120
|
+
const t = text.trim();
|
|
121
|
+
if (t) yield t;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
} finally {
|
|
125
|
+
closeSync(fd);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -28,6 +28,7 @@ import { reportSwallowed } from '../error-report.js';
|
|
|
28
28
|
import { createIncrementalReconstructor } from '../delta-reconstructor.js';
|
|
29
29
|
import { processWatchedEntry, sendEventToClients, sendEventRawToClients } from '../log-watcher.js';
|
|
30
30
|
import { SessionSynthesizer } from './adapter.js';
|
|
31
|
+
import { isDiscardableSession } from './session-select.js';
|
|
31
32
|
import { computeCacheLoss } from './meta-rows.js';
|
|
32
33
|
import { classifyRequest } from '../../../src/utils/requestType.js';
|
|
33
34
|
|
|
@@ -38,11 +39,26 @@ const ATTACH_RECENT_MS = 5 * 60 * 1000; // pre-existing dirs considered "live"
|
|
|
38
39
|
const TICK_RETRY_MS = 250; // in-process tick raced the first queue drain
|
|
39
40
|
const TICK_RETRY_MAX = 8;
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
const READ_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
43
|
+
// Node's max string length (~512MiB) — a longer line can never be decoded.
|
|
44
|
+
const MAX_LINE_BYTES = 0x1fffffe8;
|
|
45
|
+
|
|
46
|
+
/** Read newly appended complete lines from a file cursor {path, offset}
|
|
47
|
+
* (partial-line carry lives on lazily-initialized _pendBufs/_pendBytes/
|
|
48
|
+
* _skipLine cursor fields). Returns an array of raw line strings (possibly
|
|
49
|
+
* empty). A file
|
|
43
50
|
* that shrank (should never happen — v2 files are append-only) returns null
|
|
44
|
-
* so the caller can rebuild the cursor.
|
|
45
|
-
|
|
51
|
+
* so the caller can rebuild the cursor.
|
|
52
|
+
*
|
|
53
|
+
* Chunked + byte-level newline split (issue #129 twin): the first attach to
|
|
54
|
+
* a session seeds from offset 0, so one whole-file read + decode would throw
|
|
55
|
+
* ERR_STRING_TOO_LONG on an oversized journal and crash-loop the boot path
|
|
56
|
+
* exactly like the cold-scan crash the streaming reader fixed. The partial
|
|
57
|
+
* tail line is carried on the cursor as Buffer fragments (never decoded
|
|
58
|
+
* until its newline lands); a single line past the string cap is skipped
|
|
59
|
+
* with a report instead of thrown. Exported for tests (chunkBytes seam).
|
|
60
|
+
*/
|
|
61
|
+
export function readNewLines(cursor, chunkBytes = READ_CHUNK_BYTES, maxLineBytes = MAX_LINE_BYTES) {
|
|
46
62
|
let size;
|
|
47
63
|
try {
|
|
48
64
|
size = statSync(cursor.path).size;
|
|
@@ -51,23 +67,74 @@ function readNewLines(cursor) {
|
|
|
51
67
|
}
|
|
52
68
|
if (size < cursor.offset) return null;
|
|
53
69
|
if (size === cursor.offset) return [];
|
|
54
|
-
|
|
55
|
-
|
|
70
|
+
if (cursor._pendBufs === undefined) {
|
|
71
|
+
cursor._pendBufs = [];
|
|
72
|
+
cursor._pendBytes = 0;
|
|
73
|
+
cursor._skipLine = false;
|
|
74
|
+
}
|
|
75
|
+
const out = [];
|
|
56
76
|
let fd;
|
|
57
77
|
try {
|
|
58
78
|
fd = openSync(cursor.path, 'r');
|
|
59
|
-
|
|
79
|
+
const chunk = Buffer.alloc(Math.min(chunkBytes, size - cursor.offset));
|
|
80
|
+
while (cursor.offset < size) {
|
|
81
|
+
const toRead = Math.min(chunk.length, size - cursor.offset);
|
|
82
|
+
const n = readSync(fd, chunk, 0, toRead, cursor.offset);
|
|
83
|
+
if (n === 0) break;
|
|
84
|
+
cursor.offset += n;
|
|
85
|
+
const view = chunk.subarray(0, n);
|
|
86
|
+
let from = 0;
|
|
87
|
+
while (from < n) {
|
|
88
|
+
const nl = view.indexOf(10, from);
|
|
89
|
+
if (nl === -1) {
|
|
90
|
+
if (!cursor._skipLine) {
|
|
91
|
+
const restLen = n - from;
|
|
92
|
+
if (cursor._pendBytes + restLen > maxLineBytes) {
|
|
93
|
+
cursor._skipLine = true;
|
|
94
|
+
cursor._pendBufs = [];
|
|
95
|
+
cursor._pendBytes = 0;
|
|
96
|
+
reportSwallowed('v2-live.read-line-too-long', new Error(`${cursor.path}: line exceeds ${maxLineBytes} bytes — skipped`));
|
|
97
|
+
} else {
|
|
98
|
+
// chunk is reused next readSync — the carried slice must own its bytes
|
|
99
|
+
cursor._pendBufs.push(Buffer.from(view.subarray(from)));
|
|
100
|
+
cursor._pendBytes += restLen;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
if (cursor._skipLine) {
|
|
106
|
+
cursor._skipLine = false; // the oversized line ends at this newline
|
|
107
|
+
} else {
|
|
108
|
+
const seg = view.subarray(from, nl);
|
|
109
|
+
let text = null;
|
|
110
|
+
try {
|
|
111
|
+
if (cursor._pendBufs.length > 0) {
|
|
112
|
+
cursor._pendBufs.push(seg);
|
|
113
|
+
text = Buffer.concat(cursor._pendBufs).toString('utf-8');
|
|
114
|
+
} else {
|
|
115
|
+
text = seg.toString('utf-8');
|
|
116
|
+
}
|
|
117
|
+
} catch (err) {
|
|
118
|
+
reportSwallowed('v2-live.read-line-too-long', err);
|
|
119
|
+
}
|
|
120
|
+
cursor._pendBufs = [];
|
|
121
|
+
cursor._pendBytes = 0;
|
|
122
|
+
if (text !== null) {
|
|
123
|
+
const t = text.trim();
|
|
124
|
+
if (t) out.push(t);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
from = nl + 1;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
60
130
|
} catch (err) {
|
|
131
|
+
// Deliver what was already parsed; offset only advanced past read bytes,
|
|
132
|
+
// so the next poll resumes from the failure point.
|
|
61
133
|
reportSwallowed('v2-live.read', err);
|
|
62
|
-
return [];
|
|
63
134
|
} finally {
|
|
64
135
|
if (fd !== undefined) { try { closeSync(fd); } catch { /* already closed */ } }
|
|
65
136
|
}
|
|
66
|
-
|
|
67
|
-
const chunk = cursor.pending + buf.toString('utf-8');
|
|
68
|
-
const parts = chunk.split('\n');
|
|
69
|
-
cursor.pending = parts.pop() || '';
|
|
70
|
-
return parts.map((l) => l.trim()).filter(Boolean);
|
|
137
|
+
return out;
|
|
71
138
|
}
|
|
72
139
|
|
|
73
140
|
export class V2LiveFeed {
|
|
@@ -99,6 +166,7 @@ export class V2LiveFeed {
|
|
|
99
166
|
this._wireV3 = !!opts.wireV3;
|
|
100
167
|
this._sessions = new Map(); // dir → cursor bundle
|
|
101
168
|
this._seenDirs = new Map(); // dir → last observed journal mtimeMs (attached or not)
|
|
169
|
+
this._discardGated = new Set(); // dirs refused by the discard gate — first real attach must NOT suppress history
|
|
102
170
|
this._sessionsRoot = null;
|
|
103
171
|
this._rootWatcher = null;
|
|
104
172
|
this._safetyTimer = null;
|
|
@@ -126,6 +194,7 @@ export class V2LiveFeed {
|
|
|
126
194
|
for (const cur of this._sessions.values()) this._closeCursor(cur);
|
|
127
195
|
this._sessions.clear();
|
|
128
196
|
this._seenDirs.clear();
|
|
197
|
+
this._discardGated.clear();
|
|
129
198
|
this._sessionsRoot = null;
|
|
130
199
|
}
|
|
131
200
|
|
|
@@ -246,6 +315,23 @@ export class V2LiveFeed {
|
|
|
246
315
|
_attach(dir, { suppressExisting }) {
|
|
247
316
|
if (this._sessions.has(dir)) return this._sessions.get(dir);
|
|
248
317
|
if (!existsSync(join(dir, 'journal.jsonl'))) return null;
|
|
318
|
+
// Discardable sessions (quota-probe orphans — no main/teammate req, no
|
|
319
|
+
// meta.leader) are never followed: single choke point, every attach path
|
|
320
|
+
// (_initialScan / _maybeAttachNew / tick / _safetyTick / _rebuildCursor)
|
|
321
|
+
// funnels here and handles null. Self-healing when a dir later gains its
|
|
322
|
+
// first main: the leader's own dir re-attaches via tick's retry chain
|
|
323
|
+
// (sub-second); cross-process dirs via the 5s safety poll's mtime-bump
|
|
324
|
+
// scan (_seenDirs bookkeeping stays in the callers, so the bump fires).
|
|
325
|
+
if (isDiscardableSession(dir)) {
|
|
326
|
+
this._discardGated.add(dir);
|
|
327
|
+
return null;
|
|
328
|
+
}
|
|
329
|
+
// A dir previously refused by the discard gate is attaching for the FIRST
|
|
330
|
+
// time — its first renderable turn was never broadcast, so the safety
|
|
331
|
+
// poll's suppressExisting:true (meant for "old stale dir resumed") must
|
|
332
|
+
// not swallow it: cross-process producers (IM worker, second ccv) have no
|
|
333
|
+
// cold-load fallback for a connected client.
|
|
334
|
+
if (this._discardGated.delete(dir)) suppressExisting = false;
|
|
249
335
|
const cur = {
|
|
250
336
|
dir,
|
|
251
337
|
synth: new SessionSynthesizer(dir, { deferMs: this._deferMs, now: this._now }),
|
|
@@ -254,9 +340,9 @@ export class V2LiveFeed {
|
|
|
254
340
|
// rebase each other's accumulated baseline — v1's tail followed exactly
|
|
255
341
|
// one file, so a single shared reconstructor was a new interleave surface.
|
|
256
342
|
reconstructor: createIncrementalReconstructor(),
|
|
257
|
-
journal: { path: join(dir, 'journal.jsonl'), offset: 0
|
|
258
|
-
responses: { path: join(dir, 'responses.jsonl'), offset: 0
|
|
259
|
-
convFiles: new Map(), // path → {key, offset
|
|
343
|
+
journal: { path: join(dir, 'journal.jsonl'), offset: 0 },
|
|
344
|
+
responses: { path: join(dir, 'responses.jsonl'), offset: 0 },
|
|
345
|
+
convFiles: new Map(), // path → {key, offset} (+ lazy _pendBufs carry)
|
|
260
346
|
watcher: null,
|
|
261
347
|
debounce: null,
|
|
262
348
|
reading: false,
|
|
@@ -353,7 +439,7 @@ export class V2LiveFeed {
|
|
|
353
439
|
const p = join(convRoot, key, f);
|
|
354
440
|
let fc = cur.convFiles.get(p);
|
|
355
441
|
if (!fc) {
|
|
356
|
-
fc = { key, path: p, offset: 0
|
|
442
|
+
fc = { key, path: p, offset: 0 };
|
|
357
443
|
cur.convFiles.set(p, fc);
|
|
358
444
|
}
|
|
359
445
|
const lines = readNewLines(fc);
|
|
@@ -471,8 +557,17 @@ export class V2LiveFeed {
|
|
|
471
557
|
timestamp: parsed.timestamp || '',
|
|
472
558
|
url: parsed.url || '',
|
|
473
559
|
method: parsed.method || 'POST',
|
|
474
|
-
kind
|
|
475
|
-
|
|
560
|
+
// conv/evt/kind/mainAgent mirror the cold fold (meta-rows.js foldDir) —
|
|
561
|
+
// journal truth, NOT re-derivation. conv is load-bearing: the client
|
|
562
|
+
// assembler's buildEntry is `if (row.conv)`-gated, so a conv-less live
|
|
563
|
+
// row rebuilt every entry with EMPTY messages (chat vanished at stream
|
|
564
|
+
// end). mainAgent must be kind-derived like cold: parsed.mainAgent
|
|
565
|
+
// re-derives from the body and mis-tags countTokens (main system/tools
|
|
566
|
+
// aboard) as true, which would merge its turn into the chat live-only.
|
|
567
|
+
conv: item.conv,
|
|
568
|
+
evt: item.evt,
|
|
569
|
+
kind: item.kind || (item.isMain ? 'main' : (parsed.teammate ? 'teammate' : 'sub')),
|
|
570
|
+
mainAgent: item.isMain === true,
|
|
476
571
|
teammate: parsed.teammate || undefined,
|
|
477
572
|
model: parsed.body?.model,
|
|
478
573
|
proxyUrl: parsed.proxyUrl || undefined,
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { readFileSync, readdirSync } from 'node:fs';
|
|
19
19
|
import { join, basename } from 'node:path';
|
|
20
20
|
import { readJsonlTolerant } from './replay.js';
|
|
21
|
+
import { iterateJsonlLines } from './jsonl-read.js';
|
|
21
22
|
import { iterateV2Items, findTeammateSessionDirs } from './adapter.js';
|
|
22
23
|
import { SingleFlight } from './singleflight.js';
|
|
23
24
|
import { reportSwallowed } from '../error-report.js';
|
|
@@ -246,13 +247,11 @@ export async function readV2NativeCold(sessionDir, rows) {
|
|
|
246
247
|
try { files = readdirSync(join(convRoot, key)).filter((f) => /^e\d+\.jsonl$/.test(f)).sort((a, b) => Number(a.match(/\d+/)[0]) - Number(b.match(/\d+/)[0])); } catch { continue; }
|
|
247
248
|
const raws = [];
|
|
248
249
|
for (const f of files) {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
for (const
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
}
|
|
255
|
-
await yieldLoop(); // one yield per conv file read
|
|
250
|
+
// streamed line-by-line (issue #129): a conv epoch past Node's string
|
|
251
|
+
// cap must degrade to skipped lines, not lose the whole file silently
|
|
252
|
+
try { for (const trimmed of iterateJsonlLines(join(convRoot, key, f))) raws.push(trimmed); }
|
|
253
|
+
catch (err) { reportSwallowed('v2-native.conv-read-failed', err); }
|
|
254
|
+
await yieldLoop(); // one yield per conv file read (error path included)
|
|
256
255
|
}
|
|
257
256
|
// last snapshot at-or-before the window start for this session
|
|
258
257
|
let start = 0;
|
|
@@ -271,17 +270,15 @@ export async function readV2NativeCold(sessionDir, rows) {
|
|
|
271
270
|
// boundaries: the browser paints and the byte meter ticks between them.
|
|
272
271
|
pushChunked(convPayloads, windowRaws, (linesJson) => `{"sessionId":${JSON.stringify(sessionId)},"channel":${JSON.stringify(key)},"lines":[${linesJson}]}`);
|
|
273
272
|
}
|
|
274
|
-
// responses: exactly the window member seqs
|
|
275
|
-
let respText = '';
|
|
276
|
-
try { respText = readFileSync(join(dir, 'responses.jsonl'), 'utf-8'); } catch { /* none yet */ }
|
|
277
|
-
await yieldLoop();
|
|
273
|
+
// responses: exactly the window member seqs — streamed (issue #129)
|
|
278
274
|
const respRaws = [];
|
|
279
|
-
|
|
280
|
-
const trimmed
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
}
|
|
275
|
+
try {
|
|
276
|
+
for (const trimmed of iterateJsonlLines(join(dir, 'responses.jsonl'))) {
|
|
277
|
+
const m = /"seq":\s*(\d+)/.exec(trimmed);
|
|
278
|
+
if (m && memberSeqs && memberSeqs.has(Number(m[1]))) respRaws.push(trimmed);
|
|
279
|
+
}
|
|
280
|
+
} catch (err) { reportSwallowed('v2-native.responses-read-failed', err); }
|
|
281
|
+
await yieldLoop();
|
|
285
282
|
pushChunked(respPayloads, respRaws, (linesJson) => `{"sessionId":${JSON.stringify(sessionId)},"lines":[${linesJson}]}`);
|
|
286
283
|
}
|
|
287
284
|
return { convPayloads, respPayloads };
|
|
@@ -315,7 +312,7 @@ function pushChunked(out, raws, wrap) {
|
|
|
315
312
|
async function attachBodyFields(sessionDir, rows) {
|
|
316
313
|
const rowByKey = new Map(rows.map((r) => [itemKey(r.sessionId, r.seq), r]));
|
|
317
314
|
const materialize = (sessionId, seq) => rowByKey.has(itemKey(sessionId, seq));
|
|
318
|
-
let pending = null; // { row, entry } awaiting its nextReq for classify
|
|
315
|
+
let pending = null; // { row, entry, isMain } awaiting its nextReq for classify
|
|
319
316
|
let prevMainFull = null; // previous mainAgent full-body entry (cacheLoss)
|
|
320
317
|
let n = 0;
|
|
321
318
|
const finish = (slot, nextEntry) => {
|
|
@@ -329,8 +326,12 @@ async function attachBodyFields(sessionDir, rows) {
|
|
|
329
326
|
reportSwallowed('v2-meta.row-classify', err);
|
|
330
327
|
row.typeTag = null; // journal-derived kind still renders
|
|
331
328
|
}
|
|
332
|
-
//
|
|
333
|
-
|
|
329
|
+
// Authoritative synthesis-level markers. mainAgent stays KIND-derived
|
|
330
|
+
// (item.isMain = journal kind==='main' && !leader) like foldDir and the
|
|
331
|
+
// live rows — entry.mainAgent re-derives from the blob-backfilled body
|
|
332
|
+
// and mis-tags a main-shaped countTokens probe as true, which would merge
|
|
333
|
+
// its turn into the chat after a cold reload (2026-07-16 review P1).
|
|
334
|
+
row.mainAgent = slot.isMain === true;
|
|
334
335
|
if (entry.teammate) row.teammate = entry.teammate;
|
|
335
336
|
if (entry.body && entry.body.model) row.model = entry.body.model;
|
|
336
337
|
if (row.mainAgent) {
|
|
@@ -353,7 +354,7 @@ async function attachBodyFields(sessionDir, rows) {
|
|
|
353
354
|
entry = { ...entry, body: { ...entry.body, messages: item.stateRef } };
|
|
354
355
|
}
|
|
355
356
|
if (pending) finish(pending, entry);
|
|
356
|
-
pending = { row, entry };
|
|
357
|
+
pending = { row, entry, isMain: item.isMain };
|
|
357
358
|
if (++n % 20 === 0) await new Promise((resolve) => setImmediate(resolve));
|
|
358
359
|
}
|
|
359
360
|
if (pending) finish(pending, null);
|
package/server/lib/v2/replay.js
CHANGED
|
@@ -14,14 +14,14 @@ import { join } from 'node:path';
|
|
|
14
14
|
import { createHash } from 'node:crypto';
|
|
15
15
|
import { messageFingerprint, normalizeMsgForEquality } from '../session-boundary.js';
|
|
16
16
|
import { isSupportedWireFormat } from './layout.js';
|
|
17
|
+
import { iterateJsonlLines } from './jsonl-read.js';
|
|
17
18
|
|
|
18
|
-
/** Parse a JSONL file tolerating a truncated tail line (spec §14).
|
|
19
|
+
/** Parse a JSONL file tolerating a truncated tail line (spec §14). Streams
|
|
20
|
+
* line-by-line (issue #129): a file past Node's ~512MiB string cap must not
|
|
21
|
+
* throw ERR_STRING_TOO_LONG out of the whole read path. */
|
|
19
22
|
export function readJsonlTolerant(path) {
|
|
20
|
-
if (!existsSync(path)) return [];
|
|
21
23
|
const out = [];
|
|
22
|
-
for (const
|
|
23
|
-
const t = line.trim();
|
|
24
|
-
if (!t) continue;
|
|
24
|
+
for (const t of iterateJsonlLines(path)) {
|
|
25
25
|
try { out.push(JSON.parse(t)); } catch { /* truncated tail — drop */ }
|
|
26
26
|
}
|
|
27
27
|
return out;
|