agent-dag 1.44.1 → 1.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/bin/agent-dag.js +25 -1
- package/bin/deck.js +98 -15
- package/dist/web/assets/{index-XtT5NdJI.css → index-Bdl1LX0-.css} +1 -1
- package/dist/web/assets/index-jXBjwwZC.js +89 -0
- package/dist/web/index.html +2 -2
- package/hook/hook.js +120 -31
- package/package.json +1 -1
- package/src/server/args.mjs +113 -15
- package/src/server/codex-auth.mjs +9 -4
- package/src/server/codex-usage.mjs +65 -7
- package/src/server/cswap-admin.mjs +166 -34
- package/src/server/cswap-auto.mjs +156 -2
- package/src/server/exec.mjs +103 -15
- package/src/server/index.mjs +1793 -163
- package/src/server/installer.mjs +119 -30
- package/src/server/retire-sound-hook.mjs +315 -0
- package/src/server/supervisor.mjs +67 -0
- package/dist/web/assets/index-DBsxIfdM.js +0 -78
- package/hook/notify.mjs +0 -104
- package/src/server/sound-hook.mjs +0 -518
package/src/server/index.mjs
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
// agent-dag server: HTTP ingest + SSE broadcast + static file serving.
|
|
2
2
|
// Single-file pure Node HTTP server, zero deps.
|
|
3
|
-
|
|
3
|
+
// `request` is the client half, and it has exactly one caller: challengeDeck,
|
|
4
|
+
// which asks another deck's port to prove it is the deck its discovery record
|
|
5
|
+
// describes. Nothing else in this file talks to anything but 127.0.0.1 clients.
|
|
6
|
+
import { createServer, request as httpRequest } from "node:http";
|
|
4
7
|
import { readFile, stat, mkdir, open, truncate, readdir, unlink } from "node:fs/promises";
|
|
5
|
-
import { createReadStream, existsSync, readFileSync, realpathSync } from "node:fs";
|
|
8
|
+
import { createReadStream, existsSync, readFileSync, realpath as realpathCb, realpathSync } from "node:fs";
|
|
9
|
+
import { extname, join, resolve, sep, dirname as pdirname } from "node:path";
|
|
6
10
|
import { homedir } from "node:os";
|
|
7
|
-
import { extname, join, resolve, dirname as pdirname } from "node:path";
|
|
8
11
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
9
12
|
import { dirname } from "node:path";
|
|
10
13
|
import { createInterface } from "node:readline";
|
|
11
14
|
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
|
15
|
+
import { promisify } from "node:util";
|
|
12
16
|
import { claudeConfigDir } from "./claude-dir.mjs";
|
|
13
17
|
import { CODEX_HOME, CODEX_SESSIONS_DIR, STOP, walkRolloutDays } from "./codex-dir.mjs";
|
|
14
18
|
import { PRODUCT } from "./brand.mjs";
|
|
15
19
|
import { invokedName, renameNotice } from "./invoked-as.mjs";
|
|
16
|
-
import { appendLogLine, codexCwdInWorkspace, writesCodexLog } from "./log-writer.mjs";
|
|
20
|
+
import { appendLogLine, codexCwdInWorkspace, electWriters, foldsCase, writesCodexLog } from "./log-writer.mjs";
|
|
17
21
|
import { readProcesses, startSystemMetrics, systemSnapshot } from "./system-metrics.mjs";
|
|
18
22
|
|
|
19
23
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -41,8 +45,221 @@ const MIME = {
|
|
|
41
45
|
".map": "application/json",
|
|
42
46
|
};
|
|
43
47
|
|
|
44
|
-
|
|
48
|
+
// ─── The event ring buffer, and the two bounds it keeps ───────────────────
|
|
49
|
+
// MAX_BUFFER is how many events a late SSE subscriber can be replayed.
|
|
50
|
+
// MAX_BUFFER_CHARS is how much they are allowed to weigh, and until #625 there
|
|
51
|
+
// was no such thing — which is the whole defect, because a count is not a bound
|
|
52
|
+
// on memory when the thing being counted has no size of its own. `POST
|
|
53
|
+
// /api/event` admits a body of 5,000,000 characters and nothing between that
|
|
54
|
+
// door and this array shrinks it: hook/hook.js forwards the payload whole, and
|
|
55
|
+
// log-writer.mjs already says in as many words that a PostToolUse carrying a
|
|
56
|
+
// large Read or Bash response is routinely a good fraction of that. So the real
|
|
57
|
+
// ceiling was MAX_BUFFER multiplied by the largest event ingest accepts.
|
|
58
|
+
//
|
|
59
|
+
// Measured on Node 22.14 / macOS, posting 4,900,000-character bodies to a real
|
|
60
|
+
// server on loopback and reading process.memoryUsage() after a forced GC:
|
|
61
|
+
//
|
|
62
|
+
// after 20 events: heapUsed 107MB rss 292MB per-event 4.94MB
|
|
63
|
+
// after 100 events: heapUsed 481MB rss 739MB per-event 4.73MB
|
|
64
|
+
// after 200 events: heapUsed 947MB rss 1231MB per-event 4.69MB
|
|
65
|
+
//
|
|
66
|
+
// 4.69 MB retained per buffered event, flat as the ring fills. A full ring of
|
|
67
|
+
// 2000 of those is 9.4 GB against the 4144 MB heap limit V8 picks on a 32 GB
|
|
68
|
+
// machine, so the count cap was not reachable on any developer machine — and
|
|
69
|
+
// what happened instead of reaching it was not degradation. The same harness
|
|
70
|
+
// under `--max-old-space-size=2048`, roughly the heap an 8 GB laptop gives
|
|
71
|
+
// itself:
|
|
72
|
+
//
|
|
73
|
+
// after 420 events: heapUsed 1975MB rss 2247MB
|
|
74
|
+
// FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
|
|
75
|
+
//
|
|
76
|
+
// 429 events in, 1571 short of the cap. That abort is not catchable, so the SSE
|
|
77
|
+
// stream, the hook ingest and the log all stop together — and `/api/event` is a
|
|
78
|
+
// deliberate OPEN_MUTATION, so about 430 posts from a local process holding no
|
|
79
|
+
// credential at all end the deck. The comment at OPEN_MUTATIONS says the worst
|
|
80
|
+
// a caller does with that route is draw a session that is not there; this is
|
|
81
|
+
// the sentence that made it false.
|
|
82
|
+
//
|
|
83
|
+
// Both bounds are now enforced on every push, evicting oldest-first until each
|
|
84
|
+
// holds. See the eviction in pushEvent for why it is one splice and why the
|
|
85
|
+
// newest event is never the one evicted.
|
|
86
|
+
export const MAX_BUFFER = 2000; // recent events kept for late SSE subscribers
|
|
87
|
+
|
|
88
|
+
// The byte budget, counted in CHARACTERS — the unit this file already measures
|
|
89
|
+
// payloads in, and the unit MAX_CLIENT_BUFFER_BYTES is really written in
|
|
90
|
+
// despite its name (there is a long note there about why, which applies here
|
|
91
|
+
// unchanged: a character is one byte of heap while the string stays one-byte
|
|
92
|
+
// and two once it does not).
|
|
93
|
+
//
|
|
94
|
+
// 128 MiB, and the three readings that pick it:
|
|
95
|
+
//
|
|
96
|
+
// - It is 26 times the largest single event ingest can admit (5,000,000
|
|
97
|
+
// characters plus this deck's envelope), so a burst of maximum-size tool
|
|
98
|
+
// responses — eight subagents each returning a big Read — is held whole
|
|
99
|
+
// rather than collapsing the ring to nothing.
|
|
100
|
+
// - It is thirteen times a completely FULL 2000-event ring of ordinary
|
|
101
|
+
// traffic. The mean serialized event is about 5 KB, measured over 4.7k real
|
|
102
|
+
// payloads in a 21 MB events.jsonl (the same sample redactDeckToken's note
|
|
103
|
+
// quotes), so 2000 of them are 10 MB. Ordinary traffic therefore never
|
|
104
|
+
// meets this bound at all and keeps the full count-based replay depth; only
|
|
105
|
+
// the traffic that used to kill the process ever sees it.
|
|
106
|
+
// - Its worst case is a bounded fraction of the heap rather than a multiple
|
|
107
|
+
// of it. 128 MiB of charged characters is at most about 320 MiB of retained
|
|
108
|
+
// heap — the charge below tracks real retention within 2.5x across every
|
|
109
|
+
// payload shape measured — which is 16% of the 2 GB heap an 8 GB laptop
|
|
110
|
+
// picks, against the 9.4 GB the count alone permitted.
|
|
111
|
+
//
|
|
112
|
+
// Exported for the same reason MAX_CLIENT_BUFFER_BYTES is: a bound whose only
|
|
113
|
+
// observable failure is the process running out of memory is a bound no test
|
|
114
|
+
// can assert. See event-ring-byte-cap.test.ts, which pins all three readings.
|
|
115
|
+
export const MAX_BUFFER_CHARS = 128 * 1024 * 1024;
|
|
116
|
+
|
|
117
|
+
// What one envelope costs on top of its payload — seq, epoch, receivedAt,
|
|
118
|
+
// source and the JSON around them. Measured at 127 characters, the same figure
|
|
119
|
+
// MAX_CLIENT_BUFFER_BYTES is sized against; 128 here so that an event with an
|
|
120
|
+
// empty payload still costs something and a flood of them cannot be free.
|
|
121
|
+
const ENVELOPE_CHARS = 128;
|
|
122
|
+
|
|
45
123
|
const events = []; // ring buffer
|
|
124
|
+
// The running sum of what `events` holds, in the units payloadChars charges.
|
|
125
|
+
// Kept beside the array rather than recomputed, because the alternative is
|
|
126
|
+
// walking every buffered payload on every push. It and `events` have to move
|
|
127
|
+
// together and nothing outside this section may touch either — see
|
|
128
|
+
// clearEventBuffer for the one place that empties both, and what went wrong the
|
|
129
|
+
// day only one of them was emptied.
|
|
130
|
+
let bufferedChars = 0;
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* What this payload will cost the ring, charged in characters.
|
|
134
|
+
*
|
|
135
|
+
* Not `JSON.stringify(raw).length`, which is the obvious answer and is wrong
|
|
136
|
+
* here twice over. It allocates a full copy of a payload that can be five
|
|
137
|
+
* megabytes, on the hottest path in the process; and it would defeat the
|
|
138
|
+
* deliberate optimization in pushEvent that skips serializing ENTIRELY when
|
|
139
|
+
* nothing is subscribed and nothing is being logged — a headless deck, and the
|
|
140
|
+
* boot replay of a log that only rotates at 50 MB. That optimization is pinned
|
|
141
|
+
* by sse-serialize-once.test.ts, so serializing here would fail the suite as
|
|
142
|
+
* well as the machine.
|
|
143
|
+
*
|
|
144
|
+
* So it is a walk, allocating nothing, in the same iterative shape and for the
|
|
145
|
+
* same stack-overflow reason as redactDeckToken below: a body from JSON.parse
|
|
146
|
+
* is free to nest as deeply as it likes and a recursive scan would be a new way
|
|
147
|
+
* to blow the stack inside the request listener, where nothing catches it.
|
|
148
|
+
*
|
|
149
|
+
* WHAT IT CHARGES, and why it is not just the string lengths. Measured on
|
|
150
|
+
* Node 22.14 by parsing twenty copies of a 4.9M-character body of each shape
|
|
151
|
+
* and reading heapUsed after a forced GC — retained per copy, against what
|
|
152
|
+
* string characters alone would have charged:
|
|
153
|
+
*
|
|
154
|
+
* one long string 4.67MB retained 4.67M charged 1.0x
|
|
155
|
+
* array of 8-char strings 3.91MB retained 3.40M charged 1.2x
|
|
156
|
+
* array of small numbers 4.67MB retained 0.00M charged ∞
|
|
157
|
+
* array of tiny objects 10.20MB retained 0.85M charged 12.0x
|
|
158
|
+
* object of distinct keys 12.38MB retained 2.86M charged 4.3x
|
|
159
|
+
*
|
|
160
|
+
* A payload of numbers is INVISIBLE to a string-length charge while retaining
|
|
161
|
+
* 4.67 MB, and an array of small objects is under-charged twelvefold — so a
|
|
162
|
+
* ring bounded that way would have been the same OOM behind a different
|
|
163
|
+
* payload shape. Adding 8 characters per value (V8 spends a tagged slot on
|
|
164
|
+
* each, and small objects and packed arrays measured at 8–46 bytes an entry)
|
|
165
|
+
* and the length of every key brings the same five shapes to 1.0x, 0.6x, 1.0x,
|
|
166
|
+
* 1.7x and 2.5x — i.e. never blind, and never more than 2.5x under the truth.
|
|
167
|
+
* That 2.5x is what MAX_BUFFER_CHARS is sized against. The one direction it
|
|
168
|
+
* over-charges is arrays of short strings, which shortens replay depth and
|
|
169
|
+
* never the other way.
|
|
170
|
+
*
|
|
171
|
+
* Two-byte strings cost twice what they are charged, exactly as they do for
|
|
172
|
+
* MAX_CLIENT_BUFFER_BYTES: a 4.8M-character CJK payload retains 9.35 MB and is
|
|
173
|
+
* charged 4.67M. Folded into the same 2.5x.
|
|
174
|
+
*
|
|
175
|
+
* Cost, measured against the JSON.parse the same event already pays for:
|
|
176
|
+
* 0.21 µs for a realistic 5 KB hook payload, 0.04 µs for a 4.9M-character
|
|
177
|
+
* single string (it is one node), and 22.6 ms for the pathological 222k-tiny-
|
|
178
|
+
* object body — against 96.4 ms to JSON.parse that same body. So the walk is a
|
|
179
|
+
* fifth of a parse the ingest path is paying anyway, and cheaper than
|
|
180
|
+
* redactDeckToken's walk, which does substring searches this one does not.
|
|
181
|
+
*
|
|
182
|
+
* Exported so the charge itself can be asserted rather than inferred from the
|
|
183
|
+
* ring's behaviour.
|
|
184
|
+
*/
|
|
185
|
+
export function payloadChars(raw) {
|
|
186
|
+
if (typeof raw === "string") return raw.length;
|
|
187
|
+
// A top-level primitive is a legal body for `POST /api/event`, same as it is
|
|
188
|
+
// for redactDeckToken. One slot's worth.
|
|
189
|
+
if (raw === null || typeof raw !== "object") return 8;
|
|
190
|
+
|
|
191
|
+
let n = 0;
|
|
192
|
+
const stack = [raw];
|
|
193
|
+
while (stack.length > 0) {
|
|
194
|
+
const node = stack.pop();
|
|
195
|
+
// Arrays and objects walked separately for the reason redactDeckToken gives:
|
|
196
|
+
// an indexed loop is markedly cheaper than `for…in`, and a single
|
|
197
|
+
// PostToolUse response can be an array of thousands.
|
|
198
|
+
if (Array.isArray(node)) {
|
|
199
|
+
n += 8 * node.length;
|
|
200
|
+
for (let i = 0; i < node.length; i++) {
|
|
201
|
+
const v = node[i];
|
|
202
|
+
if (typeof v === "string") n += v.length;
|
|
203
|
+
else if (v !== null && typeof v === "object") stack.push(v);
|
|
204
|
+
}
|
|
205
|
+
} else {
|
|
206
|
+
for (const k in node) {
|
|
207
|
+
const v = node[k];
|
|
208
|
+
n += 8 + k.length;
|
|
209
|
+
if (typeof v === "string") n += v.length;
|
|
210
|
+
else if (v !== null && typeof v === "object") stack.push(v);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return n;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// Where the charge rides. A Symbol key rather than an ordinary field, because
|
|
218
|
+
// the envelope is JSON.stringify'd on the hot path into both the SSE frame and
|
|
219
|
+
// the events.jsonl line, and JSON.stringify ignores symbol-keyed properties
|
|
220
|
+
// entirely. So the number stays welded to the envelope it describes — which is
|
|
221
|
+
// what makes it impossible for `bufferedChars` and `events` to drift apart —
|
|
222
|
+
// without reaching the wire, the log, the client's HookEnvelope type, or the
|
|
223
|
+
// 127-character envelope measurement MAX_CLIENT_BUFFER_BYTES is sized against.
|
|
224
|
+
const CHARS = Symbol("ring charge");
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Empty the ring and the total measuring it, together.
|
|
228
|
+
*
|
|
229
|
+
* `/api/clear` used to be a bare `events.length = 0`, and with a running total
|
|
230
|
+
* beside the array that is a permanent debt: the total would still name events
|
|
231
|
+
* the array no longer holds, and every push after the first clear would evict
|
|
232
|
+
* against a budget already spent — a deck that answers one clear and then keeps
|
|
233
|
+
* a ring of one event for the rest of its life. The two variables move here and
|
|
234
|
+
* nowhere else.
|
|
235
|
+
*/
|
|
236
|
+
function clearEventBuffer() {
|
|
237
|
+
events.length = 0;
|
|
238
|
+
bufferedChars = 0;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* What the ring holds right now — its length, what it is charged, and the seq
|
|
243
|
+
* range it spans.
|
|
244
|
+
*
|
|
245
|
+
* Exported alongside MAX_BUFFER and MAX_BUFFER_CHARS so a test can watch the
|
|
246
|
+
* bound hold through a real server instead of watching a process die, which is
|
|
247
|
+
* the only other way this bound is observable. The seq range is here for the
|
|
248
|
+
* property eviction has to keep and nothing else checks: what leaves is always a
|
|
249
|
+
* PREFIX, so `newest - oldest + 1` equals the count. An eviction that ever took
|
|
250
|
+
* from the middle would leave a hole no resuming client could ask for again,
|
|
251
|
+
* and that is exactly what `GET /api/events` and the replay loop would then
|
|
252
|
+
* hand out without noticing.
|
|
253
|
+
*/
|
|
254
|
+
export function eventBufferStats() {
|
|
255
|
+
return {
|
|
256
|
+
events: events.length,
|
|
257
|
+
chars: bufferedChars,
|
|
258
|
+
oldestSeq: events.length > 0 ? events[0].seq : 0,
|
|
259
|
+
newestSeq: events.length > 0 ? events[events.length - 1].seq : 0,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
46
263
|
let nextSeq = 1;
|
|
47
264
|
// Identity of *this* process's seq numbering. nextSeq restarts at 1 on every
|
|
48
265
|
// boot and is re-derived by replaying events.jsonl, so it is monotonic only
|
|
@@ -104,11 +321,74 @@ export function writesLogFor(payload) {
|
|
|
104
321
|
return typeof sid !== "string" || sid === "" || !foreignSessions.has(sid);
|
|
105
322
|
}
|
|
106
323
|
|
|
324
|
+
/**
|
|
325
|
+
* Who else is holding the log this deck would empty, and whose it is to empty.
|
|
326
|
+
*
|
|
327
|
+
* `events.jsonl` is one file several decks share — that is the whole reason the
|
|
328
|
+
* election above exists — and `POST /api/clear` truncated it from whichever deck
|
|
329
|
+
* happened to be asked. So Clear on a deck scoped to one tree deleted the
|
|
330
|
+
* machine-wide deck's weeks of history, and told nobody: the other deck goes on
|
|
331
|
+
* serving what is still in its ring, so the damage only shows up the next time
|
|
332
|
+
* it boots and replays a file that is now empty (#698). Measured on macOS 15 /
|
|
333
|
+
* Node 22.14: 1407 bytes and five lines before, 134 bytes after — and the 134
|
|
334
|
+
* were the clearing deck's own `__clear` marker, appended to a log it writes
|
|
335
|
+
* nothing else to.
|
|
336
|
+
*
|
|
337
|
+
* The rule this establishes is the one the rest of the file already runs on: a
|
|
338
|
+
* deck may empty the log it WRITES. Ownership is electWriters — the same
|
|
339
|
+
* election, over the same discovery records, that decides which deck appends a
|
|
340
|
+
* line — so the process that truncates the file is the process that fills it,
|
|
341
|
+
* which is also the only way the two operations are ordered on any platform. A
|
|
342
|
+
* deck that is not the elected writer clears its own canvas and leaves the file
|
|
343
|
+
* to the deck that owns it; the confirmation says so before the user presses
|
|
344
|
+
* anything, and says how many decks share the file when the answer is "yours to
|
|
345
|
+
* empty". Nothing here decides the copy — `GET /api/clear` hands these facts to
|
|
346
|
+
* the dialog, which is the half that keeps the user from destroying history they
|
|
347
|
+
* were never told about.
|
|
348
|
+
*
|
|
349
|
+
* Deliberately NOT a scope test. Whether this deck was started with
|
|
350
|
+
* `--workspace` has nothing to do with who owns a file: two machine-wide decks
|
|
351
|
+
* share one log exactly as a scoped one shares it with a machine-wide one, and
|
|
352
|
+
* the harm is the same in both directions.
|
|
353
|
+
*
|
|
354
|
+
* Fail-safe matches writesCodexLog's, for the same reason: with no discovery
|
|
355
|
+
* record of our own — the window before the first heartbeat writes it, or a deck
|
|
356
|
+
* that cannot write one at all — nobody can elect us and we keep what a lone
|
|
357
|
+
* deck does. The COUNT is still every deck sharing the file, so even inside that
|
|
358
|
+
* window the confirmation warns rather than guessing quietly.
|
|
359
|
+
*/
|
|
360
|
+
export async function logSharing() {
|
|
361
|
+
if (!persistPath) return { path: null, decks: 1, mine: true, owner: null };
|
|
362
|
+
const fold = s => (foldsCase() ? s.toLowerCase() : s);
|
|
363
|
+
const here = fold(persistPath);
|
|
364
|
+
const sameLog = d => typeof d.persist === "string" && d.persist !== "" && fold(d.persist) === here;
|
|
365
|
+
|
|
366
|
+
let live = [];
|
|
367
|
+
try { live = await readLiveDecks(); } catch { /* unreadable dir — treated as "alone" below */ }
|
|
368
|
+
const group = live.filter(sameLog);
|
|
369
|
+
const self = group.find(d => d.pid === process.pid) ?? null;
|
|
370
|
+
if (!self) return { path: persistPath, decks: Math.max(group.length + 1, 1), mine: true, owner: null };
|
|
371
|
+
|
|
372
|
+
const writers = electWriters(group);
|
|
373
|
+
const mine = writers.has(self);
|
|
374
|
+
// One log path, so the election normally returns exactly one deck. It can
|
|
375
|
+
// return more only when two decks spell the same file differently on a
|
|
376
|
+
// case-sensitive platform, which `sameLog` folded together and electWriters
|
|
377
|
+
// did not — the lowest port of them is the one to name, and `mine` above is
|
|
378
|
+
// the election's own answer either way.
|
|
379
|
+
const owner = mine
|
|
380
|
+
? self
|
|
381
|
+
: [...writers].sort((a, b) => a.port - b.port || a.pid - b.pid)[0] ?? null;
|
|
382
|
+
return { path: persistPath, decks: group.length, mine, owner };
|
|
383
|
+
}
|
|
384
|
+
|
|
107
385
|
// ─── Persistence rotation ─────────────────────────────────────────────────
|
|
108
386
|
// 24/7 dev servers used to grow events.jsonl unbounded — saw it hit GBs
|
|
109
387
|
// across weeks. We rotate when the file passes ROTATE_AT_BYTES, archiving
|
|
110
388
|
// the previous file to .1 and starting fresh. Last-event-id replay still
|
|
111
|
-
// covers the in-memory ring buffer
|
|
389
|
+
// covers the in-memory ring buffer, which is bounded by MAX_BUFFER events AND
|
|
390
|
+
// by MAX_BUFFER_CHARS — so how far back a replay reaches depends on how large
|
|
391
|
+
// the traffic has been, not on the count alone.
|
|
112
392
|
const ROTATE_AT_BYTES = 50 * 1024 * 1024;
|
|
113
393
|
let lastRotateCheckAt = 0;
|
|
114
394
|
let rotateInProgress = false;
|
|
@@ -151,7 +431,33 @@ async function maybeRotatePersistFile() {
|
|
|
151
431
|
// the offset tailing the Codex rollout watcher already does further down.
|
|
152
432
|
const transcriptScans = new Map(); // path -> scan state
|
|
153
433
|
const transcriptScanInFlight = new Map(); // path -> in-progress scan promise
|
|
154
|
-
const
|
|
434
|
+
const transcriptScanSessions = new Map(); // session key -> Set<path>, LRU order
|
|
435
|
+
|
|
436
|
+
// What the cap protects is unbounded growth across the sessions a long-lived
|
|
437
|
+
// deck has seen and will never hear from again — so a SESSION is the unit it
|
|
438
|
+
// has to be counted in. It used to count paths, which is not the same thing:
|
|
439
|
+
// a session occupies one entry for its own JSONL plus one for every
|
|
440
|
+
// `subagents/agent-*.jsonl` beside it, and that count is set by how heavily
|
|
441
|
+
// the session delegates, not by anything the deck controls. Measured on this
|
|
442
|
+
// machine, the two live sessions that produced #611 hold 198 and 133 subagent
|
|
443
|
+
// files — 333 entries between them against a cap of 256, so each session's
|
|
444
|
+
// throttled pass evicted the other's cursors and re-read from byte 0. One of
|
|
445
|
+
// those directories alone is 130.8 MB and takes 6456 ms to fold cold against
|
|
446
|
+
// 18 ms warm, inside a 2500 ms throttle: the pass could not finish before the
|
|
447
|
+
// next one was due.
|
|
448
|
+
//
|
|
449
|
+
// So eviction drops a whole session at a time and never splits one. That is
|
|
450
|
+
// also the answer to a session whose subagent count exceeds any fixed number
|
|
451
|
+
// of entries: it is never evicted for being big, because the only thing a cap
|
|
452
|
+
// can take is some OTHER, older session. A session's cursors live and die
|
|
453
|
+
// together, which is the only grouping that makes the next pass cheap.
|
|
454
|
+
const MAX_TRANSCRIPT_SCAN_SESSIONS = 256;
|
|
455
|
+
// A session cap alone bounds identities, not bytes, so a second ceiling bounds
|
|
456
|
+
// the memory — again by dropping whole sessions, never part of one. A scan
|
|
457
|
+
// state retains 477 bytes measured, so 8192 entries is under 4 MB; it is also
|
|
458
|
+
// ten times every agent-*.jsonl that exists on this machine across its whole
|
|
459
|
+
// history (803), and 24x the 333 the two heaviest live sessions need.
|
|
460
|
+
const MAX_TRANSCRIPT_SCAN_ENTRIES = 8192;
|
|
155
461
|
|
|
156
462
|
// The transcript's `message.model`, and the only filter standing between it and
|
|
157
463
|
// every model the deck shows. Bedrock and Mantle put a provider namespace in
|
|
@@ -192,12 +498,81 @@ const USAGE_FIELD_RE = {
|
|
|
192
498
|
// `server_tool_use`, several fields earlier. Match the sub-object on the raw
|
|
193
499
|
// line instead of on the extracted blob.
|
|
194
500
|
const CACHE_CREATION_BLOCK_RE = /"cache_creation"\s*:\s*\{([^}]*)\}/g;
|
|
501
|
+
// A finished `Task`/`Agent` call is written into the PARENT's transcript as a
|
|
502
|
+
// top-level `toolUseResult`, and that object carries a `usage` block of its own
|
|
503
|
+
// — the subagent's LAST API turn, restated on the parent's line. Those same
|
|
504
|
+
// tokens are already in the subagent's own `subagents/agent-<id>.jsonl`, which
|
|
505
|
+
// #685 folds into the session's totals, so counting the restated copy here
|
|
506
|
+
// would charge them twice. Measured on a real transcript: the restated block
|
|
507
|
+
// reports 181,387 cache-read tokens and the last usage block inside that
|
|
508
|
+
// subagent's file reports 181,387 — the same tokens, written twice.
|
|
509
|
+
//
|
|
510
|
+
// Matched as a TOP-LEVEL key: a `{` or `,` and then the name unescaped. The
|
|
511
|
+
// same text quoted inside a message — an assistant writing about this very
|
|
512
|
+
// field, as this comment does — reaches the line as `\"toolUseResult\"`, whose
|
|
513
|
+
// preceding character is a backslash, so a transcript that talks about the key
|
|
514
|
+
// is not mistaken for one that carries it.
|
|
515
|
+
const TOOL_USE_RESULT_KEY_RE = /[{,]"toolUseResult"\s*:/;
|
|
516
|
+
|
|
517
|
+
/** The stretch of a transcript line whose `usage` blocks are the model's own
|
|
518
|
+
* billing records — everything before a top-level `toolUseResult`. See
|
|
519
|
+
* TOOL_USE_RESULT_KEY_RE. A line without one is billed whole, which is every
|
|
520
|
+
* assistant line and therefore every line that legitimately has usage. */
|
|
521
|
+
function billedUsageText(line) {
|
|
522
|
+
const m = TOOL_USE_RESULT_KEY_RE.exec(line);
|
|
523
|
+
return m ? line.slice(0, m.index) : line;
|
|
524
|
+
}
|
|
195
525
|
|
|
196
526
|
function grabUsageField(blob, key) {
|
|
197
527
|
const m = blob.match(USAGE_FIELD_RE[key]);
|
|
198
528
|
return m ? Number(m[1]) : 0;
|
|
199
529
|
}
|
|
200
530
|
|
|
531
|
+
// How many DISTINCT models one transcript may keep a usage bucket for.
|
|
532
|
+
//
|
|
533
|
+
// A session reaches two or three: `/model` mid-run, CC dropping to Sonnet when
|
|
534
|
+
// the weekly Opus allowance runs low, a subagent turn on a different tier. The
|
|
535
|
+
// cap is not about those. `MODEL_ID_RE` accepts anything beginning `claude-`,
|
|
536
|
+
// and every line of a transcript is bytes this process was handed rather than
|
|
537
|
+
// bytes it wrote — so a file naming a fresh `claude-<n>` on every line would
|
|
538
|
+
// otherwise grow one bucket per line, inside the same scanner #674 put a
|
|
539
|
+
// ceiling on for exactly this shape of reason. Past the cap the extra models'
|
|
540
|
+
// tokens still reach `state.usage`, which is what every token count on screen
|
|
541
|
+
// reads; they simply stop being attributed, and `usageByModelEntries` on the
|
|
542
|
+
// client prices whatever the map does not explain at the session's current
|
|
543
|
+
// model — the behaviour the whole deck had before #686.
|
|
544
|
+
const MAX_TRANSCRIPT_USAGE_MODELS = 32;
|
|
545
|
+
|
|
546
|
+
/** The per-model usage bucket for `model`, created on first sight. Null for a
|
|
547
|
+
* line whose tokens we cannot attribute — no model seen yet in this file, or
|
|
548
|
+
* the cap above already reached. */
|
|
549
|
+
function usageBucketFor(state, model) {
|
|
550
|
+
if (!model) return null;
|
|
551
|
+
const existing = state.usageByModel[model];
|
|
552
|
+
if (existing) return existing;
|
|
553
|
+
if (Object.keys(state.usageByModel).length >= MAX_TRANSCRIPT_USAGE_MODELS) return null;
|
|
554
|
+
const fresh = newUsageTotals();
|
|
555
|
+
state.usageByModel[model] = fresh;
|
|
556
|
+
return fresh;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/** Add `src`'s per-model buckets into `dst`, key for key, and return `dst`.
|
|
560
|
+
* Under the same cap as the per-file map, so a session that delegates to
|
|
561
|
+
* hundreds of files cannot assemble an unbounded one out of bounded parts. */
|
|
562
|
+
function mergeUsageByModel(dst, src) {
|
|
563
|
+
if (!src) return dst;
|
|
564
|
+
for (const [model, u] of Object.entries(src)) {
|
|
565
|
+
let bucket = dst[model];
|
|
566
|
+
if (!bucket) {
|
|
567
|
+
if (Object.keys(dst).length >= MAX_TRANSCRIPT_USAGE_MODELS) continue;
|
|
568
|
+
bucket = newUsageTotals();
|
|
569
|
+
dst[model] = bucket;
|
|
570
|
+
}
|
|
571
|
+
for (const k of Object.keys(bucket)) bucket[k] += u[k] ?? 0;
|
|
572
|
+
}
|
|
573
|
+
return dst;
|
|
574
|
+
}
|
|
575
|
+
|
|
201
576
|
function newContextBreakdown() {
|
|
202
577
|
return {
|
|
203
578
|
msgsUser: 0,
|
|
@@ -209,36 +584,180 @@ function newContextBreakdown() {
|
|
|
209
584
|
};
|
|
210
585
|
}
|
|
211
586
|
|
|
587
|
+
/** A zeroed set of the six token counters a transcript reports. One shape, so
|
|
588
|
+
* the per-file totals and the per-session sum of them add key for key. */
|
|
589
|
+
function newUsageTotals() {
|
|
590
|
+
return {
|
|
591
|
+
input_tokens: 0, output_tokens: 0,
|
|
592
|
+
cache_read_input_tokens: 0, cache_creation_input_tokens: 0,
|
|
593
|
+
ephemeral_1h_input_tokens: 0, ephemeral_5m_input_tokens: 0,
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
|
|
212
597
|
function newTranscriptState() {
|
|
213
598
|
return {
|
|
214
599
|
offset: 0, // bytes already folded in
|
|
600
|
+
midLine: false, // the cursor sits inside a line longer than MAX_SCAN_CHUNK
|
|
215
601
|
rootModel: null,
|
|
216
602
|
lastModel: null, // last claude-* model on any line, sidechain included
|
|
217
603
|
subagentModels: {},
|
|
218
604
|
aiTitle: null, // newest "ai-title" entry, the session's sentence title
|
|
219
605
|
agentName: null, // newest "agent-name" entry, the session's short name
|
|
220
|
-
usage:
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
606
|
+
usage: newUsageTotals(),
|
|
607
|
+
// The same totals, split by the model that produced them (#686). The flat
|
|
608
|
+
// bucket above stays the whole-transcript sum and is what every token count
|
|
609
|
+
// reads; this is what the DOLLARS are built from, because a session that
|
|
610
|
+
// switched model has tokens from two rate cards in one file and one rate
|
|
611
|
+
// applied to the lot is wrong by the ratio between them — measured at 60%
|
|
612
|
+
// under on a mostly-Opus session that ended on Sonnet, and 150% over on the
|
|
613
|
+
// mirror of it. Costs no extra read: `message.model` and the `"usage"` block
|
|
614
|
+
// are on the same line, and this pass already parses both.
|
|
615
|
+
usageByModel: {},
|
|
225
616
|
ctx: newContextBreakdown(),
|
|
226
617
|
};
|
|
227
618
|
}
|
|
228
619
|
|
|
229
|
-
|
|
620
|
+
// ─── Following an append-only JSONL, one bounded chunk at a time ─────────
|
|
621
|
+
// The most bytes ONE read of a transcript may allocate, and the ceiling that
|
|
622
|
+
// makes the size of the file irrelevant to the size of the allocation.
|
|
623
|
+
//
|
|
624
|
+
// #674 is what its absence cost. `readByteRange` was called with `to` set to
|
|
625
|
+
// `stat().size` on a path that arrives, unvalidated, in the body of a
|
|
626
|
+
// credential-free `POST /api/event`, and it answered by allocating the whole of
|
|
627
|
+
// it — once as a zero-filled Buffer and again as the string `toString` builds
|
|
628
|
+
// from it. Measured here on Node 22.14 / macOS against a real deck on loopback,
|
|
629
|
+
// sampling `process.memoryUsage.rss()` every 5 ms:
|
|
630
|
+
//
|
|
631
|
+
// 400 MB file, one POST: 51MB -> 848MB (2x: the Buffer and the string)
|
|
632
|
+
// 700 MB file, one POST: 52MB -> 753MB (1x: 700 MB is past V8's ~512 MB
|
|
633
|
+
// max string, so `toString` threw
|
|
634
|
+
// into the catch below and only the
|
|
635
|
+
// Buffer was paid for)
|
|
636
|
+
//
|
|
637
|
+
// Both answered 200. Neither is a one-off: see `readAppendedLines` for the
|
|
638
|
+
// second half of that defect, the cursor that could not advance.
|
|
639
|
+
//
|
|
640
|
+
// 8 MiB is picked against the only measurement that bears on it — the longest
|
|
641
|
+
// single line in a real transcript. The note above `maybeResolveSessionName`
|
|
642
|
+
// measures that at 710 KB in the 46.4 MB session on this machine, one big tool
|
|
643
|
+
// result on one line, so the ceiling is about eleven times the worst line seen
|
|
644
|
+
// and no real transcript line is ever split by it. Its worst case is 8 MiB of
|
|
645
|
+
// Buffer plus up to 8 MiB of string per in-flight scan, which is an eighth of
|
|
646
|
+
// the 128 MiB the ring buffer is already allowed to hold.
|
|
647
|
+
export const MAX_SCAN_CHUNK = 8 * 1024 * 1024;
|
|
648
|
+
|
|
649
|
+
// How far ONE pass will walk a file that is behind by more than a chunk.
|
|
650
|
+
//
|
|
651
|
+
// The cursor makes catching up cheap in steady state, but the FIRST pass over
|
|
652
|
+
// an existing transcript has the whole file to fold, and a fix that made that
|
|
653
|
+
// take one throttle window (2500 ms) per 8 MiB would have turned a 130.8 MB
|
|
654
|
+
// transcript's first attach into forty seconds of a deck showing no model, no
|
|
655
|
+
// name and no cost — the reintroduction, by a different route, of exactly the
|
|
656
|
+
// stall #611 removed. So a pass loops over chunks until it is caught up, and
|
|
657
|
+
// this bounds what one hook event can be made to walk.
|
|
658
|
+
//
|
|
659
|
+
// 256 MiB is twice the heaviest transcript this repo has ever measured (130.8
|
|
660
|
+
// MB, #611), so an honest first attach never meets it and pays nothing for it.
|
|
661
|
+
// What it stops is a caller who has got past `isClaudeTranscriptPath` below
|
|
662
|
+
// pointing one POST at something arbitrarily large: the read still terminates,
|
|
663
|
+
// the cursor keeps whatever it reached, and the next throttled pass continues
|
|
664
|
+
// from there.
|
|
665
|
+
export const MAX_SCAN_BYTES_PER_PASS = 256 * 1024 * 1024;
|
|
666
|
+
|
|
667
|
+
const NEWLINE = 0x0a;
|
|
668
|
+
|
|
669
|
+
/** Up to MAX_SCAN_CHUNK bytes of `path` from `from`, as a Buffer of exactly the
|
|
670
|
+
* bytes that were read. The cap is here rather than at the call sites so that
|
|
671
|
+
* no caller — including one added later — can name a range that allocates more
|
|
672
|
+
* than the ceiling above. */
|
|
673
|
+
async function readByteChunk(path, from, to) {
|
|
674
|
+
const len = Math.min(to - from, MAX_SCAN_CHUNK);
|
|
675
|
+
if (len <= 0) return Buffer.alloc(0);
|
|
230
676
|
const fh = await open(path, "r");
|
|
231
677
|
try {
|
|
232
|
-
const len = to - from;
|
|
233
|
-
if (len <= 0) return "";
|
|
234
678
|
const buf = Buffer.alloc(len);
|
|
235
|
-
await fh.read(buf, 0, len, from);
|
|
236
|
-
return buf.
|
|
679
|
+
const { bytesRead } = await fh.read(buf, 0, len, from);
|
|
680
|
+
return bytesRead === len ? buf : buf.subarray(0, bytesRead);
|
|
237
681
|
} finally {
|
|
238
682
|
await fh.close();
|
|
239
683
|
}
|
|
240
684
|
}
|
|
241
685
|
|
|
686
|
+
async function readByteRange(path, from, to) {
|
|
687
|
+
return (await readByteChunk(path, from, to)).toString("utf8");
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* The next batch of COMPLETE lines appended to a JSONL file, and the cursor
|
|
692
|
+
* moved past exactly the bytes they occupy.
|
|
693
|
+
*
|
|
694
|
+
* `cursor` is `{ offset, midLine }` and is written in place; `size` is the
|
|
695
|
+
* caller's `stat()` reading. Returns `{ text, advanced }` where `advanced` is
|
|
696
|
+
* the number of bytes the cursor moved — zero means "nothing complete to fold
|
|
697
|
+
* yet", which is the caller's signal to stop.
|
|
698
|
+
*
|
|
699
|
+
* THE THREE THINGS THIS HAS TO GET RIGHT, and the one it used to get wrong:
|
|
700
|
+
*
|
|
701
|
+
* 1. A partial last line is not an error. A transcript is being appended to
|
|
702
|
+
* while it is read, so the tail of any chunk that ends at EOF is routinely
|
|
703
|
+
* half a line. Those bytes stay unread and the cursor does not move —
|
|
704
|
+
* the next pass sees the whole line once its newline lands.
|
|
705
|
+
*
|
|
706
|
+
* 2. A chunk with no newline in it that DOES NOT end at EOF is a different
|
|
707
|
+
* thing entirely, and the old code could not tell the two apart: it
|
|
708
|
+
* returned without advancing in both cases. For any file with no newline in
|
|
709
|
+
* it — any binary, a sparse file a caller makes for the purpose — that made
|
|
710
|
+
* the early return permanent, so every later POST re-read the whole file
|
|
711
|
+
* from byte 0. Measured before the fix, the same 400 MB file: 797 MB on the
|
|
712
|
+
* first post, 400 MB on the second, 400 MB on the third. Here that case is
|
|
713
|
+
* what `midLine` names: a full chunk with no line boundary anywhere in it
|
|
714
|
+
* is a line longer than the ceiling, there is no line there to wait for, so
|
|
715
|
+
* the cursor walks past it and the fragment that follows is dropped when
|
|
716
|
+
* the next boundary arrives.
|
|
717
|
+
*
|
|
718
|
+
* 3. The cursor is moved by BYTES, taken from the buffer, never by
|
|
719
|
+
* `Buffer.byteLength` of the decoded text. Chunking can split a multi-byte
|
|
720
|
+
* character at the ceiling, and `toString` turns a split character into a
|
|
721
|
+
* replacement character of a different width — so measuring the advance on
|
|
722
|
+
* the string would drift the cursor on any transcript containing non-ASCII,
|
|
723
|
+
* which is most of them. Slicing at the last newline (a byte that cannot be
|
|
724
|
+
* part of a multi-byte sequence) and advancing by its index is exact.
|
|
725
|
+
*/
|
|
726
|
+
export async function readAppendedLines(path, cursor, size, chunkMax = MAX_SCAN_CHUNK) {
|
|
727
|
+
const from = cursor.offset;
|
|
728
|
+
const want = Math.min(size - from, chunkMax);
|
|
729
|
+
if (want <= 0) return { text: "", advanced: 0 };
|
|
730
|
+
const buf = await readByteChunk(path, from, from + want);
|
|
731
|
+
if (buf.length === 0) return { text: "", advanced: 0 };
|
|
732
|
+
|
|
733
|
+
const lastNl = buf.lastIndexOf(NEWLINE);
|
|
734
|
+
if (lastNl < 0) {
|
|
735
|
+
// Reaching the end of the file with no newline in hand is the ordinary
|
|
736
|
+
// partial last line: wait for its newline. (1) above. The test is where the
|
|
737
|
+
// read STOPPED, not how much was asked for, because `readByteChunk` clamps
|
|
738
|
+
// to MAX_SCAN_CHUNK and a short read is normally that clamp rather than the
|
|
739
|
+
// end of anything.
|
|
740
|
+
if (from + buf.length >= size) return { text: "", advanced: 0 };
|
|
741
|
+
// A chunk with no line boundary in it, and more file after it. (2).
|
|
742
|
+
cursor.offset = from + buf.length;
|
|
743
|
+
cursor.midLine = true;
|
|
744
|
+
return { text: "", advanced: buf.length };
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
const advanced = lastNl + 1; // (3): bytes, up to and including the \n
|
|
748
|
+
let text = buf.toString("utf8", 0, lastNl);
|
|
749
|
+
if (cursor.midLine) {
|
|
750
|
+
// This chunk opens in the middle of a line we already walked past, so
|
|
751
|
+
// everything up to the first boundary is the tail of it and folding it
|
|
752
|
+
// would fold a fragment. Only the lines after it are whole.
|
|
753
|
+
const nl = text.indexOf("\n");
|
|
754
|
+
text = nl < 0 ? "" : text.slice(nl + 1);
|
|
755
|
+
cursor.midLine = false;
|
|
756
|
+
}
|
|
757
|
+
cursor.offset = from + advanced;
|
|
758
|
+
return { text, advanced };
|
|
759
|
+
}
|
|
760
|
+
|
|
242
761
|
// ─── Session naming ──────────────────────────────────────────────────────
|
|
243
762
|
// CC writes two whole-line records that name the session, and nothing else in
|
|
244
763
|
// the transcript carries either fact:
|
|
@@ -326,17 +845,53 @@ function foldTranscriptLine(state, line) {
|
|
|
326
845
|
}
|
|
327
846
|
}
|
|
328
847
|
|
|
329
|
-
// Usage totals sum every block in the file, resets included
|
|
330
|
-
for
|
|
848
|
+
// Usage totals sum every block in the file, resets included — every block the
|
|
849
|
+
// model was actually billed for, which is why the `toolUseResult` tail is cut
|
|
850
|
+
// off first (see billedUsageText) — and are summed a second time into the
|
|
851
|
+
// bucket of the model that produced them (#686).
|
|
852
|
+
//
|
|
853
|
+
// `state.lastModel` is the attribution, and it is the LINE's model whenever
|
|
854
|
+
// the line has one: the block above has already run and assigned it. That is
|
|
855
|
+
// the whole trick — CC writes `message.model` and `message.usage` into the
|
|
856
|
+
// same JSON object, so by the time this loop reads the tokens the model that
|
|
857
|
+
// produced them is the most recent thing the scanner saw. A usage block on a
|
|
858
|
+
// line naming no model falls back to the last model seen, which is the turn it
|
|
859
|
+
// belongs to; a usage block before ANY model line gets no bucket at all and
|
|
860
|
+
// stays in the flat total alone, where the client prices it at the session's
|
|
861
|
+
// current model exactly as it did before.
|
|
862
|
+
//
|
|
863
|
+
// The bucket takes exactly what the flat total takes, off the same `billed`
|
|
864
|
+
// text: a split that read a wider stretch of the line than the total it splits
|
|
865
|
+
// would re-introduce #685's double count on one side of the arithmetic only,
|
|
866
|
+
// and the two would stop summing to each other.
|
|
867
|
+
const billed = billedUsageText(line);
|
|
868
|
+
const bucket = usageBucketFor(state, state.lastModel);
|
|
869
|
+
for (const m of billed.matchAll(USAGE_BLOCK_RE)) {
|
|
331
870
|
const blob = m[1];
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
871
|
+
const inTok = grabUsageField(blob, "input_tokens");
|
|
872
|
+
const outTok = grabUsageField(blob, "output_tokens");
|
|
873
|
+
const cacheR = grabUsageField(blob, "cache_read_input_tokens");
|
|
874
|
+
const cacheC = grabUsageField(blob, "cache_creation_input_tokens");
|
|
875
|
+
state.usage.input_tokens += inTok;
|
|
876
|
+
state.usage.output_tokens += outTok;
|
|
877
|
+
state.usage.cache_read_input_tokens += cacheR;
|
|
878
|
+
state.usage.cache_creation_input_tokens += cacheC;
|
|
879
|
+
if (bucket) {
|
|
880
|
+
bucket.input_tokens += inTok;
|
|
881
|
+
bucket.output_tokens += outTok;
|
|
882
|
+
bucket.cache_read_input_tokens += cacheR;
|
|
883
|
+
bucket.cache_creation_input_tokens += cacheC;
|
|
884
|
+
}
|
|
336
885
|
}
|
|
337
|
-
for (const m of
|
|
338
|
-
|
|
339
|
-
|
|
886
|
+
for (const m of billed.matchAll(CACHE_CREATION_BLOCK_RE)) {
|
|
887
|
+
const h1 = grabUsageField(m[1], "ephemeral_1h_input_tokens");
|
|
888
|
+
const m5 = grabUsageField(m[1], "ephemeral_5m_input_tokens");
|
|
889
|
+
state.usage.ephemeral_1h_input_tokens += h1;
|
|
890
|
+
state.usage.ephemeral_5m_input_tokens += m5;
|
|
891
|
+
if (bucket) {
|
|
892
|
+
bucket.ephemeral_1h_input_tokens += h1;
|
|
893
|
+
bucket.ephemeral_5m_input_tokens += m5;
|
|
894
|
+
}
|
|
340
895
|
}
|
|
341
896
|
|
|
342
897
|
// Context counts only what follows the most recent /clear or /compact.
|
|
@@ -365,24 +920,58 @@ function foldTranscriptLine(state, line) {
|
|
|
365
920
|
}
|
|
366
921
|
}
|
|
367
922
|
|
|
368
|
-
/**
|
|
369
|
-
*
|
|
370
|
-
*
|
|
371
|
-
*
|
|
372
|
-
*
|
|
373
|
-
*
|
|
374
|
-
* the
|
|
375
|
-
*
|
|
923
|
+
/** The session a transcript path belongs to, and the unit eviction works in.
|
|
924
|
+
* CC's two layouts collapse onto the same key: the main transcript
|
|
925
|
+
* `<slug>/<sessionId>.jsonl` loses its extension, and a subagent's
|
|
926
|
+
* `<slug>/<sessionId>/subagents/agent-<id>.jsonl` loses everything below the
|
|
927
|
+
* session directory, so both land on `<slug>/<sessionId>`. Anything else — a
|
|
928
|
+
* Codex rollout, a shape we do not recognise — is a session of its own,
|
|
929
|
+
* which is the old one-entry-per-path accounting and the safe default.
|
|
930
|
+
*
|
|
931
|
+
* `resolve` first so the two halves agree on Windows. The main path arrives
|
|
932
|
+
* from the hook payload as CC wrote it while the subagent path is built with
|
|
933
|
+
* `join`, so `C:/x/y.jsonl` and `C:\x\y\subagents\agent-1.jsonl` would
|
|
934
|
+
* otherwise be two sessions instead of one; `resolve` settles the separator
|
|
935
|
+
* on all three platforms, and leaves a backslash inside a POSIX filename
|
|
936
|
+
* alone rather than reading it as a directory break. */
|
|
937
|
+
export function transcriptSessionKey(path) {
|
|
938
|
+
const full = resolve(path);
|
|
939
|
+
const sub = /^(.*)[\\/]subagents[\\/]agent-[0-9a-f]+\.jsonl$/i.exec(full);
|
|
940
|
+
return sub ? sub[1] : full.replace(/\.jsonl$/i, "");
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/** Record a use of `path` and keep the cache under both caps. Re-inserting the
|
|
944
|
+
* session on every touch makes the Map's own insertion order the LRU order,
|
|
945
|
+
* so eviction reads one key instead of scanning for the smallest timestamp.
|
|
946
|
+
* Scanning a timestamp is also what broke the cache once: a state is created
|
|
947
|
+
* with no stamp yet, so the entry the scan had just inserted was the smallest
|
|
948
|
+
* of all and the one deleted, every time. Whatever the cache freezes on, the
|
|
949
|
+
* symptom is the same — every later transcript re-reads its whole JSONL from
|
|
950
|
+
* byte 0 on every throttled pass, the O(n)-per-pass stall the cursor exists
|
|
376
951
|
* to remove. */
|
|
377
952
|
function touchTranscriptScan(path, state) {
|
|
378
|
-
transcriptScans.delete(path);
|
|
379
953
|
transcriptScans.set(path, state);
|
|
954
|
+
const key = transcriptSessionKey(path);
|
|
955
|
+
const paths = transcriptScanSessions.get(key) ?? new Set();
|
|
956
|
+
transcriptScanSessions.delete(key); // re-insert = move to the back of the LRU
|
|
957
|
+
paths.add(path);
|
|
958
|
+
transcriptScanSessions.set(key, paths);
|
|
380
959
|
pruneTranscriptScans();
|
|
381
960
|
}
|
|
382
961
|
|
|
383
962
|
function pruneTranscriptScans() {
|
|
384
|
-
|
|
385
|
-
|
|
963
|
+
// The last session standing is never evicted, however many files it holds:
|
|
964
|
+
// dropping the cursors of the session currently being scanned is precisely
|
|
965
|
+
// the re-read this cache exists to avoid, and no cap can make its file
|
|
966
|
+
// count smaller.
|
|
967
|
+
while (
|
|
968
|
+
transcriptScanSessions.size > 1 &&
|
|
969
|
+
(transcriptScanSessions.size > MAX_TRANSCRIPT_SCAN_SESSIONS ||
|
|
970
|
+
transcriptScans.size > MAX_TRANSCRIPT_SCAN_ENTRIES)
|
|
971
|
+
) {
|
|
972
|
+
const oldest = transcriptScanSessions.keys().next().value;
|
|
973
|
+
for (const p of transcriptScanSessions.get(oldest)) transcriptScans.delete(p);
|
|
974
|
+
transcriptScanSessions.delete(oldest);
|
|
386
975
|
}
|
|
387
976
|
}
|
|
388
977
|
|
|
@@ -409,14 +998,20 @@ function scanTranscript(path) {
|
|
|
409
998
|
touchTranscriptScan(path, state);
|
|
410
999
|
}
|
|
411
1000
|
if (s.size <= state.offset) return state;
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
//
|
|
417
|
-
// the cursor where the next pass would count those lines
|
|
418
|
-
|
|
419
|
-
|
|
1001
|
+
// Chunk by chunk rather than in one allocation, and loop rather than
|
|
1002
|
+
// leave the rest for the next throttle window — see MAX_SCAN_CHUNK and
|
|
1003
|
+
// MAX_SCAN_BYTES_PER_PASS for why it is both of those and not either one
|
|
1004
|
+
// alone. `readAppendedLines` advances the cursor BEFORE this folds, which
|
|
1005
|
+
// is the same rule the single-shot read kept: a fold that throws half-way
|
|
1006
|
+
// must not leave the cursor where the next pass would count those lines
|
|
1007
|
+
// again. `advanced === 0` is "nothing complete to fold yet".
|
|
1008
|
+
let budget = MAX_SCAN_BYTES_PER_PASS;
|
|
1009
|
+
while (budget > 0 && state.offset < s.size) {
|
|
1010
|
+
const { text, advanced } = await readAppendedLines(path, state, s.size);
|
|
1011
|
+
if (advanced === 0) break;
|
|
1012
|
+
budget -= advanced;
|
|
1013
|
+
for (const line of text.split("\n")) foldTranscriptLine(state, line);
|
|
1014
|
+
}
|
|
420
1015
|
} catch { /* keep whatever we already folded */ }
|
|
421
1016
|
return state;
|
|
422
1017
|
})();
|
|
@@ -426,6 +1021,92 @@ function scanTranscript(path) {
|
|
|
426
1021
|
});
|
|
427
1022
|
}
|
|
428
1023
|
|
|
1024
|
+
// ─── Which paths the deck will follow at all ─────────────────────────────
|
|
1025
|
+
// `payload.transcript_path` is a string in the body of `POST /api/event`, and
|
|
1026
|
+
// that route is a deliberate OPEN_MUTATION: no token, no Origin, nothing. Until
|
|
1027
|
+
// #674 the deck took whatever it said and opened it. The bounds above make that
|
|
1028
|
+
// survivable; this makes it uninteresting, and the two are worth having
|
|
1029
|
+
// together for different reasons.
|
|
1030
|
+
//
|
|
1031
|
+
// WHY VALIDATE AT ALL WHEN THE READ IS ALREADY BOUNDED. Because the caller here
|
|
1032
|
+
// is not a web page — `isTrustedMutation` refuses `Sec-Fetch-Site: cross-site`
|
|
1033
|
+
// before any of this — it is a local process: the sandboxed subprocess with
|
|
1034
|
+
// loopback egress that the comment above `isAuthorizedMutation` already names,
|
|
1035
|
+
// or another UID on a shared box. Against that caller a ceiling only sets the
|
|
1036
|
+
// price per request; it does not take the lever away. What takes it away is
|
|
1037
|
+
// that there is no file it can name. Claude Code writes transcripts in exactly
|
|
1038
|
+
// one place, `<config dir>/projects/…`, and every legitimate `transcript_path`
|
|
1039
|
+
// the deck has ever seen is one of those — so the set of things worth opening
|
|
1040
|
+
// is knowable in advance, and checking membership costs one string comparison
|
|
1041
|
+
// against a syscall that used to cost the size of the file.
|
|
1042
|
+
//
|
|
1043
|
+
// WHAT IS LEFT AFTERWARDS, stated plainly: a caller who can WRITE inside that
|
|
1044
|
+
// directory can still point the deck at a file of its choosing. That caller is
|
|
1045
|
+
// this user's own processes — and this user's own processes can read
|
|
1046
|
+
// `<config dir>/agent-dag/*.json`, which is where HOOK_TOKEN lives at mode
|
|
1047
|
+
// 0600, so they hold the credential already. The gate reduces the
|
|
1048
|
+
// credential-free adversary to the one who was never credential-free. That is
|
|
1049
|
+
// the whole of what it claims, and the ceilings above are what carries the
|
|
1050
|
+
// rest.
|
|
1051
|
+
//
|
|
1052
|
+
// WHY NOT realpath. A symlink planted inside the projects directory would
|
|
1053
|
+
// defeat the containment test — but planting one needs write access to that
|
|
1054
|
+
// directory, which is the case above where the caller already holds the token.
|
|
1055
|
+
// It would also cost a syscall on every hook event, on a path that runs for
|
|
1056
|
+
// every event of every live session.
|
|
1057
|
+
//
|
|
1058
|
+
// WHY TWO ROOTS. CLAUDE_CONFIG_DIR replaces ~/.claude wholesale, and the deck
|
|
1059
|
+
// reads the variable from its OWN environment while the path is written by
|
|
1060
|
+
// whatever `claude` process the hook fired in. Those normally agree — the deck
|
|
1061
|
+
// installs its hook into the directory it resolves, so a session whose events
|
|
1062
|
+
// arrive here is a session reading that same directory — but a deck launched
|
|
1063
|
+
// from a desktop shortcut that never sourced the shell rc is a real way for
|
|
1064
|
+
// them to disagree in one direction. Accepting the default location as well
|
|
1065
|
+
// costs nothing (it is a directory only this user writes either way) and
|
|
1066
|
+
// removes half of that failure mode. The other half is why the refusal is
|
|
1067
|
+
// logged rather than silent.
|
|
1068
|
+
function claudeTranscriptRoots() {
|
|
1069
|
+
// Resolved per call, like every other claudeConfigDir() reader in this file,
|
|
1070
|
+
// so nothing captures the answer from an environment that has moved.
|
|
1071
|
+
const roots = [resolve(claudeConfigDir(), "projects")];
|
|
1072
|
+
const byDefault = resolve(homedir(), ".claude", "projects");
|
|
1073
|
+
if (!roots.includes(byDefault)) roots.push(byDefault);
|
|
1074
|
+
return roots;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
/** Is `p` a Claude Code transcript, in a directory Claude Code writes them?
|
|
1078
|
+
*
|
|
1079
|
+
* Containment is compared on the RESOLVED path with a trailing separator, so
|
|
1080
|
+
* `…/projects-of-mine/x.jsonl` is not inside `…/projects` and `..` cannot
|
|
1081
|
+
* climb out of it. The comparison is case-insensitive on Windows and macOS,
|
|
1082
|
+
* whose default filesystems are, because the two halves come from two
|
|
1083
|
+
* processes and only one of them chose the casing. */
|
|
1084
|
+
export function isClaudeTranscriptPath(p, roots = claudeTranscriptRoots()) {
|
|
1085
|
+
if (!p || typeof p !== "string") return false;
|
|
1086
|
+
if (!/\.jsonl$/i.test(p)) return false; // the only extension CC writes
|
|
1087
|
+
const fold = process.platform === "win32" || process.platform === "darwin";
|
|
1088
|
+
const full = fold ? resolve(p).toLowerCase() : resolve(p);
|
|
1089
|
+
for (const root of roots) {
|
|
1090
|
+
const prefix = (fold ? root.toLowerCase() : root) + sep;
|
|
1091
|
+
if (full.startsWith(prefix) && full.length > prefix.length) return true;
|
|
1092
|
+
}
|
|
1093
|
+
return false;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// Refusals are logged once per path and the set is capped, because the point of
|
|
1097
|
+
// the log is a misconfigured deck saying so on stderr — one line naming the
|
|
1098
|
+
// path and where transcripts are expected — and a caller posting a fresh path
|
|
1099
|
+
// per request must not turn that into a second unbounded accumulation.
|
|
1100
|
+
const refusedTranscriptPaths = new Set();
|
|
1101
|
+
const MAX_REFUSED_TRANSCRIPT_PATHS = 64;
|
|
1102
|
+
|
|
1103
|
+
function noteRefusedTranscript(p) {
|
|
1104
|
+
if (refusedTranscriptPaths.has(p)) return;
|
|
1105
|
+
if (refusedTranscriptPaths.size >= MAX_REFUSED_TRANSCRIPT_PATHS) return;
|
|
1106
|
+
refusedTranscriptPaths.add(p);
|
|
1107
|
+
console.warn(`${PRODUCT}: not reading transcript_path outside ${claudeTranscriptRoots().join(" or ")}: ${p}`);
|
|
1108
|
+
}
|
|
1109
|
+
|
|
429
1110
|
// ─── Model enrichment ────────────────────────────────────────────────────
|
|
430
1111
|
// CC's hook payloads never carry the `model` field — but every hook
|
|
431
1112
|
// references a `transcript_path` JSONL that contains lines like
|
|
@@ -455,7 +1136,7 @@ export function cachedModelId(cached) {
|
|
|
455
1136
|
* legacy-schema subagent models (older CC versions kept subagent blocks
|
|
456
1137
|
* inline with `isSidechain:true` + `parentToolUseID`). Current CC versions
|
|
457
1138
|
* store subagents in `<sessionDir>/subagents/agent-<id>.jsonl` — those are
|
|
458
|
-
* handled by `
|
|
1139
|
+
* handled by `readSubagentsFromDir` below. */
|
|
459
1140
|
export async function readModelFromTranscript(path) {
|
|
460
1141
|
const state = await scanTranscript(path);
|
|
461
1142
|
if (!state) return null;
|
|
@@ -473,8 +1154,38 @@ export async function readModelFromTranscript(path) {
|
|
|
473
1154
|
* but the reducer looks up `${sessionId}::${key}` and the subagent node id
|
|
474
1155
|
* is built from `agent_id` — identical lookup either way).
|
|
475
1156
|
*
|
|
476
|
-
* Returns { [agentId]: model }
|
|
477
|
-
|
|
1157
|
+
* Returns `{ models: { [agentId]: model }, usage: <summed totals|null> }` over
|
|
1158
|
+
* every agent-*.jsonl in the directory, or null when there is no such
|
|
1159
|
+
* directory — which is every Codex session and every legacy-schema Claude one.
|
|
1160
|
+
*
|
|
1161
|
+
* WHY IT RETURNS USAGE AND NOT ONLY MODELS (#685). `scanTranscript` folds a
|
|
1162
|
+
* file's usage totals whether or not anyone asks for them, so this walk has
|
|
1163
|
+
* been computing every subagent's spend and dropping it on the floor, while
|
|
1164
|
+
* the session's totals came from the main transcript alone — and the main
|
|
1165
|
+
* transcript does not restate a subagent's turns. Measured on one real
|
|
1166
|
+
* session on the reporter's machine: 40.4M cache-read tokens in the main
|
|
1167
|
+
* JSONL against 217.7M across its twenty subagent files, so the deck was
|
|
1168
|
+
* reporting about a sixth of what the session actually cost.
|
|
1169
|
+
*
|
|
1170
|
+
* WHY THE TOTALS ARE SUMMED HERE RATHER THAN SHIPPED PER AGENT. The obvious
|
|
1171
|
+
* shape is a `{ [agentId]: totals }` map so each subagent card can price its
|
|
1172
|
+
* own spend, and the cost of that shape is set by CC and not by us: #611
|
|
1173
|
+
* measured live sessions holding 198 and 133 of these files, and this machine
|
|
1174
|
+
* has one holding 397. At ~200 bytes an entry on a 2.5 s cadence that is
|
|
1175
|
+
* ~80 KB per pass into the SSE fan-out, the event ring and events.jsonl —
|
|
1176
|
+
* ~115 MB an hour into a log that rotates at 50 MB. One summed object is
|
|
1177
|
+
* ~150 bytes whatever the session delegates, so the session total is exact
|
|
1178
|
+
* and constant-cost, and per-card attribution stays a separate question.
|
|
1179
|
+
*
|
|
1180
|
+
* `usageByModel` is summed on the same terms and for the same reason (#686):
|
|
1181
|
+
* a subagent runs on whatever model its Task was given, which is routinely not
|
|
1182
|
+
* its parent's — the deck already reads a per-agent `models` map three lines
|
|
1183
|
+
* up precisely because of that — so folding delegated tokens into the session
|
|
1184
|
+
* total without their model would price a Haiku subagent's spend at the root's
|
|
1185
|
+
* Opus rate. It stays constant-cost the way the flat total does: the key count
|
|
1186
|
+
* is the number of MODELS a session touched, two or three, not the number of
|
|
1187
|
+
* files it delegated to. */
|
|
1188
|
+
async function readSubagentsFromDir(transcriptPath) {
|
|
478
1189
|
// Subagent dir sits next to the main jsonl: <dir>/<sessionId>/subagents/
|
|
479
1190
|
// Derive from transcript_path by stripping the .jsonl suffix.
|
|
480
1191
|
if (!transcriptPath || typeof transcriptPath !== "string") return null;
|
|
@@ -483,6 +1194,9 @@ async function readSubagentModelsFromDir(transcriptPath) {
|
|
|
483
1194
|
let entries;
|
|
484
1195
|
try { entries = await readdir(subDir); } catch { return null; }
|
|
485
1196
|
const models = {};
|
|
1197
|
+
const usage = newUsageTotals();
|
|
1198
|
+
const usageByModel = {};
|
|
1199
|
+
let spent = false;
|
|
486
1200
|
for (const f of entries) {
|
|
487
1201
|
if (!/^agent-([0-9a-f]+)\.jsonl$/i.test(f)) continue;
|
|
488
1202
|
const agentId = f.replace(/^agent-/, "").replace(/\.jsonl$/i, "");
|
|
@@ -492,11 +1206,53 @@ async function readSubagentModelsFromDir(transcriptPath) {
|
|
|
492
1206
|
// model wins — subagents may switch model mid-turn (Sonnet → Haiku for
|
|
493
1207
|
// tool-call fallback etc.).
|
|
494
1208
|
const state = await scanTranscript(full);
|
|
495
|
-
|
|
496
|
-
if (
|
|
1209
|
+
if (!state) continue;
|
|
1210
|
+
if (state.lastModel) models[agentId] = state.lastModel;
|
|
1211
|
+
// The cursor makes this cumulative and idempotent: `state.usage` is the
|
|
1212
|
+
// whole file's totals however many passes it took to fold them, so
|
|
1213
|
+
// re-summing every pass restates the same number rather than growing it.
|
|
1214
|
+
for (const k of Object.keys(usage)) usage[k] += state.usage[k];
|
|
1215
|
+
// Idempotent on the same argument as the line above: each file's split is
|
|
1216
|
+
// its own cumulative totals, so re-summing every pass restates the map
|
|
1217
|
+
// rather than growing it.
|
|
1218
|
+
mergeUsageByModel(usageByModel, state.usageByModel);
|
|
1219
|
+
if (state.usage.input_tokens || state.usage.output_tokens
|
|
1220
|
+
|| state.usage.cache_read_input_tokens || state.usage.cache_creation_input_tokens) {
|
|
1221
|
+
spent = true;
|
|
1222
|
+
}
|
|
497
1223
|
} catch { /* skip unreadable file */ }
|
|
498
1224
|
}
|
|
499
|
-
|
|
1225
|
+
const hasModels = Object.keys(models).length > 0;
|
|
1226
|
+
if (!hasModels && !spent) return null;
|
|
1227
|
+
return {
|
|
1228
|
+
models: hasModels ? models : null,
|
|
1229
|
+
usage: spent ? usage : null,
|
|
1230
|
+
usageByModel: spent ? usageByModel : null,
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
// One walk of `<sessionDir>/subagents/` serves both the model pass and the
|
|
1235
|
+
// usage pass. The two are throttled independently but fire from the same hook
|
|
1236
|
+
// events, so without this the directory — up to a few hundred files — would be
|
|
1237
|
+
// listed and stat-ed twice per window for the same answer. The TTL sits under
|
|
1238
|
+
// MODEL_READ_THROTTLE_MS so two callers inside one window share a walk and the
|
|
1239
|
+
// next window always gets a fresh one.
|
|
1240
|
+
const subagentDirScans = new Map(); // transcriptPath -> { at, promise }
|
|
1241
|
+
const SUBAGENT_DIR_TTL_MS = 2000;
|
|
1242
|
+
|
|
1243
|
+
function scanSubagentDir(transcriptPath) {
|
|
1244
|
+
if (!transcriptPath || typeof transcriptPath !== "string") return Promise.resolve(null);
|
|
1245
|
+
const now = Date.now();
|
|
1246
|
+
// Expiry is the eviction rule: an entry outlives its window by nothing, so
|
|
1247
|
+
// the map holds at most one entry per session read in the last two seconds.
|
|
1248
|
+
for (const [k, v] of subagentDirScans) {
|
|
1249
|
+
if (now - v.at >= SUBAGENT_DIR_TTL_MS) subagentDirScans.delete(k);
|
|
1250
|
+
}
|
|
1251
|
+
const memo = subagentDirScans.get(transcriptPath);
|
|
1252
|
+
if (memo) return memo.promise;
|
|
1253
|
+
const promise = readSubagentsFromDir(transcriptPath).catch(() => null);
|
|
1254
|
+
subagentDirScans.set(transcriptPath, { at: now, promise });
|
|
1255
|
+
return promise;
|
|
500
1256
|
}
|
|
501
1257
|
|
|
502
1258
|
function maybeResolveModel(payload) {
|
|
@@ -513,12 +1269,12 @@ function maybeResolveModel(payload) {
|
|
|
513
1269
|
if (now - last < MODEL_READ_THROTTLE_MS) return;
|
|
514
1270
|
modelLastReadAt.set(sid, now);
|
|
515
1271
|
pendingTranscriptReads.add(sid);
|
|
516
|
-
Promise.all([readModelFromTranscript(tp),
|
|
517
|
-
.then(([result,
|
|
1272
|
+
Promise.all([readModelFromTranscript(tp), scanSubagentDir(tp)])
|
|
1273
|
+
.then(([result, dir]) => {
|
|
518
1274
|
const rootModel = result?.rootModel ?? null;
|
|
519
1275
|
// Merge legacy (inline isSidechain) + new (subagents/ dir) maps. Dir
|
|
520
1276
|
// wins on conflict since current CC only writes to the dir.
|
|
521
|
-
const subagentModels = { ...(result?.subagentModels ?? {}), ...(
|
|
1277
|
+
const subagentModels = { ...(result?.subagentModels ?? {}), ...(dir?.models ?? {}) };
|
|
522
1278
|
if (!rootModel && Object.keys(subagentModels).length === 0) return;
|
|
523
1279
|
const prev = modelBySession.get(sid);
|
|
524
1280
|
const subsSig = JSON.stringify(subagentModels);
|
|
@@ -549,6 +1305,12 @@ const USAGE_READ_THROTTLE_MS = 2500;
|
|
|
549
1305
|
// Every entry carries its own usage object and we sum every occurrence, so
|
|
550
1306
|
// the totals are cumulative over the whole transcript — the running state
|
|
551
1307
|
// keeps them across passes and each pass only adds the newly appended blocks.
|
|
1308
|
+
//
|
|
1309
|
+
// ONE FILE, which is the whole file and no more. This reads the path it is
|
|
1310
|
+
// given; the session's delegated spend lives in the sibling `subagents/`
|
|
1311
|
+
// directory and is added by `sessionUsageTotals` below. Kept separate because
|
|
1312
|
+
// the two are read on different cadences by different callers and the tests
|
|
1313
|
+
// that pin the cursor arithmetic pin it one file at a time.
|
|
552
1314
|
export async function readUsageFromTranscript(path) {
|
|
553
1315
|
const state = await scanTranscript(path);
|
|
554
1316
|
if (!state) return null;
|
|
@@ -558,6 +1320,80 @@ export async function readUsageFromTranscript(path) {
|
|
|
558
1320
|
return totals;
|
|
559
1321
|
}
|
|
560
1322
|
|
|
1323
|
+
/** One transcript's totals broken out by the model that produced them, or null
|
|
1324
|
+
* when the scan attributed nothing.
|
|
1325
|
+
*
|
|
1326
|
+
* A SECOND EXPORT rather than one more key on the object above, and the reason
|
|
1327
|
+
* is that object's contract: `readUsageFromTranscript` is asserted with
|
|
1328
|
+
* `toEqual` on its whole shape, so a key added to it is a key every caller and
|
|
1329
|
+
* every fixture has to learn about. This costs no extra read — `scanTranscript`
|
|
1330
|
+
* hands back the state it already folded and coalesces concurrent callers onto
|
|
1331
|
+
* one pass — so the pair reads the file exactly as often as the single call
|
|
1332
|
+
* did. */
|
|
1333
|
+
export async function readUsageByModelFromTranscript(path) {
|
|
1334
|
+
const state = await scanTranscript(path);
|
|
1335
|
+
if (!state) return null;
|
|
1336
|
+
const out = {};
|
|
1337
|
+
for (const [model, u] of Object.entries(state.usageByModel)) {
|
|
1338
|
+
if (u.input_tokens === 0 && u.output_tokens === 0
|
|
1339
|
+
&& u.cache_read_input_tokens === 0 && u.cache_creation_input_tokens === 0) continue;
|
|
1340
|
+
out[model] = { ...u };
|
|
1341
|
+
}
|
|
1342
|
+
return Object.keys(out).length ? out : null;
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
/** The session's whole spend split by model — its own turns plus every
|
|
1346
|
+
* subagent's, merged (#686).
|
|
1347
|
+
*
|
|
1348
|
+
* The counterpart of `sessionUsageTotals` below and read from the same two
|
|
1349
|
+
* places, so the split and the total it splits are always a description of the
|
|
1350
|
+
* same bytes. A subagent runs on the model its Task was given, which is
|
|
1351
|
+
* routinely not its parent's, so a session sum that carried only the root's
|
|
1352
|
+
* models would price delegated tokens at whatever the root is on now — the
|
|
1353
|
+
* same mistake as the one being fixed, one level in. */
|
|
1354
|
+
export async function sessionUsageByModel(transcriptPath) {
|
|
1355
|
+
const [own, dir] = await Promise.all([
|
|
1356
|
+
readUsageByModelFromTranscript(transcriptPath),
|
|
1357
|
+
scanSubagentDir(transcriptPath),
|
|
1358
|
+
]);
|
|
1359
|
+
if (!own && !dir?.usageByModel) return null;
|
|
1360
|
+
const out = mergeUsageByModel({}, own);
|
|
1361
|
+
mergeUsageByModel(out, dir?.usageByModel);
|
|
1362
|
+
return Object.keys(out).length ? out : null;
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
/**
|
|
1366
|
+
* Everything a Claude session has spent: its own turns plus every subagent it
|
|
1367
|
+
* ran. Returns null when the session has spent nothing yet.
|
|
1368
|
+
*
|
|
1369
|
+
* WHAT THIS NUMBER MEANS, and why it is one number (#685). A session is a
|
|
1370
|
+
* bill, and delegating work does not move any of it somewhere else — the
|
|
1371
|
+
* subagent's tokens are charged to the same account on the same invoice. The
|
|
1372
|
+
* two halves live in two places on disk because CC writes them there:
|
|
1373
|
+
* `<sessionId>.jsonl` for the root's turns, `<sessionId>/subagents/agent-*.jsonl`
|
|
1374
|
+
* for each delegated one, with no overlap between them (verified against real
|
|
1375
|
+
* transcripts: a session holding 397 subagent files had zero `isSidechain`
|
|
1376
|
+
* lines in its main JSONL). Summing the two therefore counts every token
|
|
1377
|
+
* exactly once, and the reducer ASSIGNS the result rather than adding it, so a
|
|
1378
|
+
* pass that lands twice restates the same total instead of doubling it.
|
|
1379
|
+
*
|
|
1380
|
+
* The one place the two files did overlap is the `toolUseResult` block a
|
|
1381
|
+
* finished Task leaves on the parent's line, which restates the subagent's
|
|
1382
|
+
* last turn; `billedUsageText` cuts that out of the scan so this sum stays a
|
|
1383
|
+
* sum of distinct tokens.
|
|
1384
|
+
*/
|
|
1385
|
+
export async function sessionUsageTotals(transcriptPath) {
|
|
1386
|
+
const [own, dir] = await Promise.all([
|
|
1387
|
+
readUsageFromTranscript(transcriptPath),
|
|
1388
|
+
scanSubagentDir(transcriptPath),
|
|
1389
|
+
]);
|
|
1390
|
+
const delegated = dir?.usage ?? null;
|
|
1391
|
+
if (!own && !delegated) return null;
|
|
1392
|
+
const totals = own ? { ...own } : newUsageTotals();
|
|
1393
|
+
if (delegated) for (const k of Object.keys(totals)) totals[k] += delegated[k] ?? 0;
|
|
1394
|
+
return totals;
|
|
1395
|
+
}
|
|
1396
|
+
|
|
561
1397
|
function maybeResolveUsage(payload) {
|
|
562
1398
|
if (!payload || typeof payload !== "object") return;
|
|
563
1399
|
const sid = payload.session_id;
|
|
@@ -569,10 +1405,16 @@ function maybeResolveUsage(payload) {
|
|
|
569
1405
|
if (now - last < USAGE_READ_THROTTLE_MS) return;
|
|
570
1406
|
lastUsageReadAt.set(sid, now);
|
|
571
1407
|
pendingUsageReads.add(sid);
|
|
572
|
-
|
|
573
|
-
.then(usage => {
|
|
1408
|
+
Promise.all([sessionUsageTotals(tp), sessionUsageByModel(tp)])
|
|
1409
|
+
.then(([usage, usageByModel]) => {
|
|
574
1410
|
if (!usage) return;
|
|
575
|
-
|
|
1411
|
+
// `usageByModel` rides on the same event because it is the same
|
|
1412
|
+
// measurement, read from the same two places by the same walk — the main
|
|
1413
|
+
// transcript and the `subagents/` directory — so the split and the total
|
|
1414
|
+
// it splits cannot describe two different moments of the session. Sent
|
|
1415
|
+
// even when null: an absent split has to CLEAR a stale one on the client,
|
|
1416
|
+
// for the reason the flat totals are assigned rather than added.
|
|
1417
|
+
pushEvent({ hook_event_name: "UsageObserved", session_id: sid, usage, usageByModel }, "internal");
|
|
576
1418
|
})
|
|
577
1419
|
.catch(() => {})
|
|
578
1420
|
.finally(() => pendingUsageReads.delete(sid));
|
|
@@ -771,9 +1613,23 @@ async function collectMemoryFiles(paths) {
|
|
|
771
1613
|
* a test is how they would drift. */
|
|
772
1614
|
export async function scanClaudeMdFiles(cwd) {
|
|
773
1615
|
if (!cwd || typeof cwd !== "string") return [];
|
|
774
|
-
|
|
1616
|
+
// The CONFIG dir, not the home directory. CLAUDE_CONFIG_DIR relocates it
|
|
1617
|
+
// wholesale — it replaces ~/.claude rather than overlaying it — so on a
|
|
1618
|
+
// machine where it is set, `homedir()/.claude` is a directory Claude Code
|
|
1619
|
+
// does not read. Spelling it by hand here failed in both directions at once:
|
|
1620
|
+
// the user-global memory file and every auto-memory file went missing from
|
|
1621
|
+
// the modal and from the byte total beside it, while a stale ~/.claude left
|
|
1622
|
+
// over from before the variable was set got listed as if it were in context.
|
|
1623
|
+
// Resolved per call, like the two other claudeConfigDir() readers in this
|
|
1624
|
+
// file, so nothing captures the answer from an environment that has moved.
|
|
1625
|
+
const cfg = claudeConfigDir();
|
|
775
1626
|
// Walk up from cwd to filesystem root, checking the canonical CC memory
|
|
776
1627
|
// filenames plus CLAUDE.local.md (user-private) at each level.
|
|
1628
|
+
//
|
|
1629
|
+
// The `.claude/` prefix on the last two is NOT the config dir wearing a
|
|
1630
|
+
// second spelling: it is CC's per-directory project convention, one such
|
|
1631
|
+
// folder per level of the walk, and it stays literal however the config dir
|
|
1632
|
+
// moves.
|
|
777
1633
|
const paths = memoryWalkPaths(cwd, [
|
|
778
1634
|
"CLAUDE.md",
|
|
779
1635
|
"CLAUDE.local.md",
|
|
@@ -781,14 +1637,15 @@ export async function scanClaudeMdFiles(cwd) {
|
|
|
781
1637
|
join(".claude", "CLAUDE.local.md"),
|
|
782
1638
|
]);
|
|
783
1639
|
// User-global memory.
|
|
784
|
-
paths.push(join(
|
|
785
|
-
paths.push(join(
|
|
786
|
-
// Per-project auto-memory:
|
|
787
|
-
// (plus MEMORY.md index). CC injects these into context for sessions
|
|
788
|
-
//
|
|
1640
|
+
paths.push(join(cfg, "CLAUDE.md"));
|
|
1641
|
+
paths.push(join(cfg, "CLAUDE.local.md"));
|
|
1642
|
+
// Per-project auto-memory: $CLAUDE_CONFIG_DIR/projects/<slug>/memory/*.md
|
|
1643
|
+
// (plus MEMORY.md index). CC injects these into context for sessions whose
|
|
1644
|
+
// cwd matches the slug. That directory sits beside the <sessionId>.jsonl
|
|
1645
|
+
// transcripts, so it moves with the config dir by construction.
|
|
789
1646
|
const slug = ccProjectSlug(cwd);
|
|
790
1647
|
if (slug) {
|
|
791
|
-
const memDir = join(
|
|
1648
|
+
const memDir = join(cfg, "projects", slug, "memory");
|
|
792
1649
|
try {
|
|
793
1650
|
const entries = await readdir(memDir);
|
|
794
1651
|
for (const f of entries) {
|
|
@@ -1092,7 +1949,9 @@ function maybeResolveCodex(payload) {
|
|
|
1092
1949
|
// agent-dag event stream from them. Each rollout line is one append-only JSON
|
|
1093
1950
|
// object {timestamp, type, payload}; we map the relevant ones to the same
|
|
1094
1951
|
// synthetic hook payloads the reducer already understands:
|
|
1095
|
-
// session_meta → SessionStart
|
|
1952
|
+
// session_meta → SessionStart, but ONLY for a rollout
|
|
1953
|
+
// this watcher read from byte 0 (#684).
|
|
1954
|
+
// See ensureCodexRoot.
|
|
1096
1955
|
// event_msg/user_message → UserPromptSubmit (Codex ≤ 0.144)
|
|
1097
1956
|
// event_msg/item_completed/UserMessage → UserPromptSubmit (Codex ≥ 0.147)
|
|
1098
1957
|
// response_item/function_call → PreToolUse
|
|
@@ -1119,7 +1978,8 @@ function maybeResolveCodex(payload) {
|
|
|
1119
1978
|
// broadcasts them exactly like a hook event, and persists them when this deck is
|
|
1120
1979
|
// the one elected to log this rollout — see writesCodexLog. This path is
|
|
1121
1980
|
// entirely additive — the Claude hook flow is untouched.
|
|
1122
|
-
|
|
1981
|
+
// path -> { offset, sid, cwd, skip, sawBeginning, rootOpened, seenAt }
|
|
1982
|
+
const codexFileState = new Map();
|
|
1123
1983
|
const codexSessionModel = new Map(); // sid -> last model string
|
|
1124
1984
|
// sid -> the `approval_policy` of the newest `turn_context` seen on this
|
|
1125
1985
|
// session. See codexObjToPayload for why this is read and what it is NOT used
|
|
@@ -1133,20 +1993,160 @@ let codexScanRunning = false;
|
|
|
1133
1993
|
let codexWatchTimer = null;
|
|
1134
1994
|
let codexWorkspace = "";
|
|
1135
1995
|
|
|
1996
|
+
// How long one record's challenge verdict is trusted. The rollout scan runs
|
|
1997
|
+
// every 1.5s and would otherwise pay a round trip per record per tick for an
|
|
1998
|
+
// answer that changes about once per deck lifetime.
|
|
1999
|
+
//
|
|
2000
|
+
// Both answers are cached, and neither one delays a takeover, because the pid
|
|
2001
|
+
// probe runs first and is not cached at all: a deck killed with SIGTERM or
|
|
2002
|
+
// SIGKILL loses its pid immediately and its record is dropped on the very next
|
|
2003
|
+
// tick whatever this map says. What the TTL bounds is only the ghost case — a
|
|
2004
|
+
// record whose pid was recycled by an unrelated process — where five seconds of
|
|
2005
|
+
// believing a stale "yes" is at most a few unwritten lines, against a directory
|
|
2006
|
+
// listing plus an HTTP round trip on every tick of every deck forever.
|
|
2007
|
+
const DECK_PROOF_TTL_MS = 5000;
|
|
2008
|
+
// The same deadline hook.js gives a challenge, and for the same reason: a
|
|
2009
|
+
// bodyless GET to a loopback port is sub-millisecond when a deck is there and an
|
|
2010
|
+
// instant ECONNREFUSED when nothing is.
|
|
2011
|
+
const DECK_CHALLENGE_TIMEOUT_MS = 400;
|
|
2012
|
+
// `${pid}:${port}:${token}` -> { at, ok }. Keyed on the record's own identity so
|
|
2013
|
+
// a rewritten record — a deck that restarted onto the same port with a fresh
|
|
2014
|
+
// token — is a new question rather than an inherited answer.
|
|
2015
|
+
const deckProofs = new Map();
|
|
2016
|
+
|
|
1136
2017
|
/**
|
|
1137
|
-
*
|
|
2018
|
+
* Is the process listening on this record's port the deck the record describes?
|
|
2019
|
+
*
|
|
2020
|
+
* WHY THE SERVER ASKS AT ALL (#695). readLiveDecks used to trust the pid and
|
|
2021
|
+
* nothing else, and a pid is not evidence: a record left behind by a deck that
|
|
2022
|
+
* is gone — SIGKILL, an OOM kill, a power cut, a console window closed on
|
|
2023
|
+
* Windows, none of which run the shutdown that unlinks it — passes a signal-0
|
|
2024
|
+
* probe forever once the OS hands that number to some other long-lived process.
|
|
2025
|
+
* writesCodexLog then elected it, because it named a lower port than any real
|
|
2026
|
+
* deck, and every deck tailing the rollout drew the events while none of them
|
|
2027
|
+
* appended a line. The canvas looked perfectly normal and events.jsonl stopped
|
|
2028
|
+
* growing, so nothing said so until a restart replayed a log that had stopped
|
|
2029
|
+
* days earlier.
|
|
2030
|
+
*
|
|
2031
|
+
* hook.js has had the answer to this since the handshake landed and the Codex
|
|
2032
|
+
* half never got it — this is that half. The record carries the deck's own token
|
|
2033
|
+
* in plaintext (mode 0600, same user, and it is written there precisely so that
|
|
2034
|
+
* another process can challenge with it), so we can ask its port to hash that
|
|
2035
|
+
* token against a nonce it has never seen. A stranger cannot answer; the deck
|
|
2036
|
+
* that wrote the file can.
|
|
2037
|
+
*
|
|
2038
|
+
* Two records pass without being asked:
|
|
2039
|
+
*
|
|
2040
|
+
* • our own. We are the process the record describes, and challenging
|
|
2041
|
+
* ourselves over loopback inside our own scan is a question we already know
|
|
2042
|
+
* the answer to. It also must never fail: writesCodexLog looks itself up in
|
|
2043
|
+
* this list, and a deck that cannot find its own record falls back to
|
|
2044
|
+
* writing — so a self-challenge that timed out under load would turn one
|
|
2045
|
+
* unwritten line into a duplicated one.
|
|
2046
|
+
* • a record with no token, written by a deck older than the handshake. That
|
|
2047
|
+
* is exactly requiresProof's fallback in hook.js, kept in step with it on
|
|
2048
|
+
* purpose: refusing those would make every pre-1.33.71 deck invisible to the
|
|
2049
|
+
* election, which elects a writer for a log those decks are still appending
|
|
2050
|
+
* to.
|
|
2051
|
+
*
|
|
2052
|
+
* A timeout counts as a failure, like a refusal and a wrong answer. That is the
|
|
2053
|
+
* same verdict hook.js reaches — a target that misses the deadline is not posted
|
|
2054
|
+
* to either — and it errs the direction this file has always erred: the deck
|
|
2055
|
+
* writes, and "a line written twice is recoverable; a deck that quietly stops
|
|
2056
|
+
* recording anything is not" (writesCodexLog).
|
|
2057
|
+
*/
|
|
2058
|
+
async function provesDeck(d) {
|
|
2059
|
+
if (d.pid === process.pid) return true;
|
|
2060
|
+
if (typeof d.token !== "string" || d.token === "") return true;
|
|
2061
|
+
|
|
2062
|
+
const key = `${d.pid}:${d.port}:${d.token}`;
|
|
2063
|
+
const cached = deckProofs.get(key);
|
|
2064
|
+
const now = Date.now();
|
|
2065
|
+
if (cached && now - cached.at < DECK_PROOF_TTL_MS) return cached.ok;
|
|
2066
|
+
|
|
2067
|
+
const ok = await challengeDeck(d.port, d.token);
|
|
2068
|
+
deckProofs.set(key, { at: now, ok });
|
|
2069
|
+
// Records come and go; without this the map would grow by one entry per deck
|
|
2070
|
+
// that ever registered on this machine while the process is up.
|
|
2071
|
+
for (const [k, v] of deckProofs) {
|
|
2072
|
+
if (now - v.at >= DECK_PROOF_TTL_MS) deckProofs.delete(k);
|
|
2073
|
+
}
|
|
2074
|
+
return ok;
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
/**
|
|
2078
|
+
* One challenge round trip, resolving true only on a correct proof.
|
|
2079
|
+
*
|
|
2080
|
+
* The compare is constant-time for the reason hook.js's sameProof is: whatever
|
|
2081
|
+
* is on that port may not be a deck, and it must not be able to walk the
|
|
2082
|
+
* expected proof out of us one byte at a time by timing how long we take to hang
|
|
2083
|
+
* up. The nonce is fresh per call, so an answer overheard earlier is worth
|
|
2084
|
+
* nothing, and the token itself never leaves this process.
|
|
2085
|
+
*/
|
|
2086
|
+
function challengeDeck(port, token) {
|
|
2087
|
+
return new Promise(resolve => {
|
|
2088
|
+
let settled = false;
|
|
2089
|
+
const finish = ok => { if (settled) return; settled = true; resolve(ok); };
|
|
2090
|
+
const nonce = randomBytes(16).toString("hex");
|
|
2091
|
+
const want = challengeProof(token, nonce);
|
|
2092
|
+
const req = httpRequest({
|
|
2093
|
+
hostname: "127.0.0.1",
|
|
2094
|
+
port,
|
|
2095
|
+
path: `/api/hook-challenge?nonce=${nonce}`,
|
|
2096
|
+
method: "GET",
|
|
2097
|
+
timeout: DECK_CHALLENGE_TIMEOUT_MS,
|
|
2098
|
+
}, res => {
|
|
2099
|
+
if (res.statusCode !== 200) { res.resume(); return res.on("end", () => finish(false)); }
|
|
2100
|
+
let answer = "";
|
|
2101
|
+
res.setEncoding("utf8");
|
|
2102
|
+
res.on("data", c => {
|
|
2103
|
+
answer += c;
|
|
2104
|
+
// A deck answers in ~100 bytes. Anything pouring data at us is not one,
|
|
2105
|
+
// and must not be allowed to grow this buffer without bound.
|
|
2106
|
+
if (answer.length > 4096) { req.destroy(); finish(false); }
|
|
2107
|
+
});
|
|
2108
|
+
res.on("end", () => {
|
|
2109
|
+
if (settled) return;
|
|
2110
|
+
let proof;
|
|
2111
|
+
try { proof = JSON.parse(answer).proof; } catch { return finish(false); }
|
|
2112
|
+
finish(sameProof(proof, want));
|
|
2113
|
+
});
|
|
2114
|
+
});
|
|
2115
|
+
req.on("error", () => finish(false));
|
|
2116
|
+
req.on("timeout", () => req.destroy());
|
|
2117
|
+
req.end();
|
|
2118
|
+
});
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
/** hook.js's sameProof, for the same reason it is constant-time there. */
|
|
2122
|
+
function sameProof(got, want) {
|
|
2123
|
+
if (typeof got !== "string") return false;
|
|
2124
|
+
const a = Buffer.from(got, "utf8");
|
|
2125
|
+
const b = Buffer.from(want, "utf8");
|
|
2126
|
+
return a.length === b.length && timingSafeEqual(a, b);
|
|
2127
|
+
}
|
|
2128
|
+
|
|
2129
|
+
/**
|
|
2130
|
+
* Every deck registered right now, as its own discovery record spells it — and
|
|
2131
|
+
* only the ones that proved they are still the deck their record describes.
|
|
1138
2132
|
*
|
|
1139
2133
|
* These are the files hook.js enumerates; this is the server reading them for
|
|
1140
2134
|
* itself, because the rollout watcher has no hook to do it and still has to know
|
|
1141
2135
|
* which other decks are tailing the same file into the same log. Dead pids are
|
|
1142
2136
|
* ignored rather than unlinked — sweepStaleDiscovery owns that, and a poll
|
|
1143
|
-
* running every 1.5s is no place to be deleting other decks' registrations.
|
|
2137
|
+
* running every 1.5s is no place to be deleting other decks' registrations. A
|
|
2138
|
+
* record that fails the challenge is likewise left alone: see the same argument
|
|
2139
|
+
* spelled out in proveTargets in hook/hook.js.
|
|
2140
|
+
*
|
|
2141
|
+
* The pid probe stays, in front of the challenge, because it is free and it is
|
|
2142
|
+
* the one test that answers instantly when a deck is killed — which is what
|
|
2143
|
+
* keeps the takeover immediate rather than TTL-bound. See provesDeck.
|
|
1144
2144
|
*/
|
|
1145
2145
|
async function readLiveDecks() {
|
|
1146
2146
|
const dir = join(claudeConfigDir(), "agent-dag");
|
|
1147
2147
|
let files;
|
|
1148
2148
|
try { files = await readdir(dir); } catch { return []; }
|
|
1149
|
-
const
|
|
2149
|
+
const candidates = [];
|
|
1150
2150
|
for (const f of files) {
|
|
1151
2151
|
if (!f.endsWith(".json")) continue;
|
|
1152
2152
|
try {
|
|
@@ -1155,10 +2155,13 @@ async function readLiveDecks() {
|
|
|
1155
2155
|
// record cannot take part in an election either way.
|
|
1156
2156
|
if (!d || typeof d.pid !== "number" || typeof d.port !== "number") continue;
|
|
1157
2157
|
if (!isProcessAlive(d.pid)) continue;
|
|
1158
|
-
|
|
2158
|
+
candidates.push(d);
|
|
1159
2159
|
} catch { /* corrupt, or gone between listing and read */ }
|
|
1160
2160
|
}
|
|
1161
|
-
|
|
2161
|
+
// In parallel: with several decks up this is the difference between one 400ms
|
|
2162
|
+
// deadline for the scan and one per record.
|
|
2163
|
+
const proven = await Promise.all(candidates.map(provesDeck));
|
|
2164
|
+
return candidates.filter((_, i) => proven[i]);
|
|
1162
2165
|
}
|
|
1163
2166
|
|
|
1164
2167
|
// List rollout files from the newest 2 day-directories. New sessions always
|
|
@@ -1193,7 +2196,11 @@ async function readCodexHeader(path) {
|
|
|
1193
2196
|
if (nl >= 0) {
|
|
1194
2197
|
const obj = JSON.parse(text.slice(0, nl));
|
|
1195
2198
|
if (obj && obj.type === "session_meta" && obj.payload) {
|
|
1196
|
-
|
|
2199
|
+
// Canonicalised here and nowhere else: everything downstream — the
|
|
2200
|
+
// workspace test below, the log election, the cwd on every event this
|
|
2201
|
+
// rollout produces — reads state.cwd, and this is the one place it is
|
|
2202
|
+
// read off disk. See canonicalCwd.
|
|
2203
|
+
return { sid: obj.payload.id, cwd: await canonicalCwd(obj.payload.cwd) };
|
|
1197
2204
|
}
|
|
1198
2205
|
return null;
|
|
1199
2206
|
}
|
|
@@ -1549,12 +2556,71 @@ function emitCodexEvent(payload, persist) {
|
|
|
1549
2556
|
pushEvent(payload, "codex", { persist });
|
|
1550
2557
|
}
|
|
1551
2558
|
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
2559
|
+
/**
|
|
2560
|
+
* Open this rollout's session on the canvas, once, lazily — only when it
|
|
2561
|
+
* actually produces an event. Emitting eagerly for every file on disk at
|
|
2562
|
+
* startup would fill the canvas with empty roots for sessions nobody will ever
|
|
2563
|
+
* append to again.
|
|
2564
|
+
*
|
|
2565
|
+
* WHETHER A `SessionStart` IS EMITTED AT ALL IS THE WHOLE OF #684.
|
|
2566
|
+
*
|
|
2567
|
+
* This used to mint one unconditionally, and for a rollout the deck joined
|
|
2568
|
+
* partway through that event is a false statement. #683 made the falsehood
|
|
2569
|
+
* visible rather than merely wrong: a root created by anything OTHER than a
|
|
2570
|
+
* `SessionStart` is marked `synthetic` in the reducer — "the deck joined this
|
|
2571
|
+
* session after it had already begun, so the start time, the prompt history and
|
|
2572
|
+
* the early tool calls on this card are incomplete rather than empty" — and a
|
|
2573
|
+
* Codex session that carries a minted `SessionStart` clears that marker and
|
|
2574
|
+
* asserts a beginning nobody watched. The Claude side never had the problem: a
|
|
2575
|
+
* deck that starts mid-session simply never receives a `SessionStart` hook for
|
|
2576
|
+
* it, which is exactly the case the marker was built for.
|
|
2577
|
+
*
|
|
2578
|
+
* `sawBeginning` is the one fact that separates the two, and it is not a
|
|
2579
|
+
* filesystem question. It is set where the tail cursor is: TRUE when this
|
|
2580
|
+
* watcher opened the rollout at byte 0 and therefore holds every line the
|
|
2581
|
+
* session ever wrote, FALSE when codexScanOnce's `firstRun` skipped a
|
|
2582
|
+
* pre-existing file's history by seeking to its current size. That is the same
|
|
2583
|
+
* fact the event states, read off the only thing that actually knows it, and it
|
|
2584
|
+
* is the same on Linux, macOS and Windows because no platform API is consulted.
|
|
2585
|
+
*
|
|
2586
|
+
* WHY NOT `fs.watch`, AND WHY NOT `birthtime` — both measured rather than
|
|
2587
|
+
* assumed, on Node 22.14 / darwin, plus the documented behaviour elsewhere:
|
|
2588
|
+
*
|
|
2589
|
+
* • `fs.watch` on a path that does not exist throws ENOENT, and
|
|
2590
|
+
* ~/.codex/sessions/YYYY/MM/DD does not exist until the day's first
|
|
2591
|
+
* session — the same reason startCodexWatcher polls instead of watching.
|
|
2592
|
+
* • a NON-recursive watch on ~/.codex/sessions reports only `rename:2026`
|
|
2593
|
+
* when a rollout appears three levels below it; the file's creation is
|
|
2594
|
+
* invisible to it.
|
|
2595
|
+
* • `recursive: true` works here and on Windows, and on Linux only from a
|
|
2596
|
+
* Node 20.x release — while this package declares `"node": ">=18"`, so on
|
|
2597
|
+
* the OS Codex users most often run servers on it may simply not be there.
|
|
2598
|
+
* • even where it works the event does not mean "created": writing the file
|
|
2599
|
+
* and then rewriting it produced `rename:rollout.jsonl` BOTH times, and
|
|
2600
|
+
* Node documents the event type as not guaranteed and `filename` as
|
|
2601
|
+
* possibly null.
|
|
2602
|
+
* • `stat().birthtimeMs` is real on APFS, but Node documents it as falling
|
|
2603
|
+
* back to ctime or to the Unix epoch on filesystems that do not carry it —
|
|
2604
|
+
* so on some Linux hosts every rollout would look newborn.
|
|
2605
|
+
*
|
|
2606
|
+
* WHAT ANOTHER DECK SEES. Nothing new: the fix REMOVES a line from
|
|
2607
|
+
* events.jsonl, it does not add one or change a shape. A joined-late Codex
|
|
2608
|
+
* session now reaches the shared log looking exactly like a joined-late Claude
|
|
2609
|
+
* one — a root conjured by its first real event — which every deck, including
|
|
2610
|
+
* one older than #683 that has no marker to light, has always been able to
|
|
2611
|
+
* replay. The events that create the root instead (`UserPromptSubmit`,
|
|
2612
|
+
* `PreToolUse`, `ModelObserved`, …) each carry `provider: "codex"`, `cwd` and
|
|
2613
|
+
* `approval_policy` from `base` in codexObjToPayload, so nothing an older deck
|
|
2614
|
+
* read off the `SessionStart` is lost with it.
|
|
2615
|
+
*
|
|
2616
|
+
* `rootOpened` still flips in both cases, and deliberately: it gates the
|
|
2617
|
+
* per-batch AGENTS.md resolution in codexScanOnce, which asks "is this session
|
|
2618
|
+
* being drawn", not "did we announce it".
|
|
2619
|
+
*/
|
|
1555
2620
|
function ensureCodexRoot(state, persist) {
|
|
1556
|
-
if (state.
|
|
1557
|
-
state.
|
|
2621
|
+
if (state.rootOpened) return;
|
|
2622
|
+
state.rootOpened = true;
|
|
2623
|
+
if (!state.sawBeginning) return;
|
|
1558
2624
|
emitCodexEvent({ session_id: state.sid, cwd: state.cwd, provider: "codex", hook_event_name: "SessionStart" }, persist);
|
|
1559
2625
|
}
|
|
1560
2626
|
|
|
@@ -1581,16 +2647,32 @@ async function codexScanOnce(firstRun) {
|
|
|
1581
2647
|
const header = await readCodexHeader(path);
|
|
1582
2648
|
if (!header || !header.sid) continue; // not ready yet — retry next tick
|
|
1583
2649
|
if (!codexCwdInWorkspace(header.cwd, codexWorkspace)) {
|
|
1584
|
-
codexFileState.set(path, { offset: st.size, sid: header.sid, cwd: header.cwd, skip: true,
|
|
2650
|
+
codexFileState.set(path, { offset: st.size, sid: header.sid, cwd: header.cwd, skip: true, sawBeginning: false, rootOpened: false, seenAt: now });
|
|
1585
2651
|
continue;
|
|
1586
2652
|
}
|
|
1587
|
-
|
|
2653
|
+
// Opened at byte 0, so every line this session ever wrote is about to
|
|
2654
|
+
// be read: this watcher HAS its beginning, and ensureCodexRoot may say
|
|
2655
|
+
// so. The `firstRun` branch below is the one case that takes it away.
|
|
2656
|
+
state = { offset: 0, sid: header.sid, cwd: header.cwd, skip: false, sawBeginning: true, rootOpened: false, seenAt: now };
|
|
1588
2657
|
codexFileState.set(path, state);
|
|
1589
2658
|
if (firstRun) {
|
|
1590
2659
|
// On startup, skip a pre-existing session's history entirely — no
|
|
1591
|
-
//
|
|
1592
|
-
//
|
|
2660
|
+
// replay. Only future appends (a live session that keeps going) will
|
|
2661
|
+
// lazily open the root via ensureCodexRoot.
|
|
2662
|
+
//
|
|
2663
|
+
// And it opens WITHOUT a `SessionStart` (#684). Seeking to the
|
|
2664
|
+
// current size is precisely the admission that this deck did not
|
|
2665
|
+
// watch the session begin, so it must not go on to emit the event
|
|
2666
|
+
// that says it did — that is the input #683's joined-late marker is
|
|
2667
|
+
// entitled to trust. A deck RESTARTING over a session that is still
|
|
2668
|
+
// running lands here too, and correctly: the new process holds none
|
|
2669
|
+
// of the old one's history either. If the log it replays at boot
|
|
2670
|
+
// already contains a `SessionStart` this deck minted honestly in an
|
|
2671
|
+
// earlier life, the root is rebuilt unmarked from that line and stays
|
|
2672
|
+
// unmarked — the reducer only honours `synthetic` on the call that
|
|
2673
|
+
// CREATES the node, and nothing here contradicts it.
|
|
1593
2674
|
state.offset = st.size;
|
|
2675
|
+
state.sawBeginning = false;
|
|
1594
2676
|
continue;
|
|
1595
2677
|
}
|
|
1596
2678
|
}
|
|
@@ -1598,11 +2680,12 @@ async function codexScanOnce(firstRun) {
|
|
|
1598
2680
|
if (state.skip) { state.offset = st.size; continue; }
|
|
1599
2681
|
if (st.size <= state.offset) continue;
|
|
1600
2682
|
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
2683
|
+
// Same bounded reader the Claude transcripts use. A rollout is not a
|
|
2684
|
+
// caller-chosen path, so this is not the #674 exposure — but the
|
|
2685
|
+
// allocation was the size of the appended bytes here too, and one chunk
|
|
2686
|
+
// per tick is more than any real rollout appends in a tick.
|
|
2687
|
+
const { text: consume, advanced } = await readAppendedLines(path, state, st.size);
|
|
2688
|
+
if (advanced === 0) continue; // no complete line yet
|
|
1606
2689
|
|
|
1607
2690
|
// Decided per file rather than per line: which decks are up, and which of
|
|
1608
2691
|
// them tail this rollout, cannot change inside one batch of appended
|
|
@@ -1641,7 +2724,7 @@ async function codexScanOnce(firstRun) {
|
|
|
1641
2724
|
// lines, and a batch whose roots and tool calls went to one deck's log
|
|
1642
2725
|
// while its memory list went to every deck's is the split the election
|
|
1643
2726
|
// exists to prevent (#447).
|
|
1644
|
-
if (state.
|
|
2727
|
+
if (state.rootOpened) maybeResolveCodexMemory(state.sid, state.cwd, persist);
|
|
1645
2728
|
}
|
|
1646
2729
|
|
|
1647
2730
|
// Rollout files fall out of the newest-2-days listing and never come back,
|
|
@@ -1671,8 +2754,10 @@ export function startCodexWatcher(workspace) {
|
|
|
1671
2754
|
// filesystem watch is no help: fs.watch on a missing path throws, and
|
|
1672
2755
|
// watching the parent recursively is macOS/Windows-only.
|
|
1673
2756
|
//
|
|
1674
|
-
// Initial catalog:
|
|
1675
|
-
// history, then poll for new lines.
|
|
2757
|
+
// Initial catalog: park a cursor at the end of every rollout already on disk,
|
|
2758
|
+
// skipping its history, then poll for new lines. The `true` is what tells
|
|
2759
|
+
// codexScanOnce that these files pre-date this deck, and so that the roots
|
|
2760
|
+
// they eventually open must arrive without a `SessionStart` (#684).
|
|
1676
2761
|
codexScanOnce(true).catch(() => {});
|
|
1677
2762
|
codexWatchTimer = setInterval(() => { codexScanOnce(false).catch(() => {}); }, 1500);
|
|
1678
2763
|
if (codexWatchTimer.unref) codexWatchTimer.unref();
|
|
@@ -2050,9 +3135,55 @@ function pushEvent(raw, source, opts = {}) {
|
|
|
2050
3135
|
receivedAt: opts.receivedAt ?? Date.now(),
|
|
2051
3136
|
source,
|
|
2052
3137
|
payload: raw,
|
|
3138
|
+
// Charged once, here, and carried on the envelope so eviction never has to
|
|
3139
|
+
// walk a payload a second time. Symbol-keyed, so it is invisible to the two
|
|
3140
|
+
// JSON.stringify calls below and to the `{ ...e }` the replay loop makes.
|
|
3141
|
+
// Charged AFTER redactDeckToken, like everything else in this function: the
|
|
3142
|
+
// payload being measured is the one that will be stored.
|
|
3143
|
+
[CHARS]: ENVELOPE_CHARS + payloadChars(raw),
|
|
2053
3144
|
};
|
|
2054
3145
|
events.push(evt);
|
|
2055
|
-
|
|
3146
|
+
bufferedChars += evt[CHARS];
|
|
3147
|
+
|
|
3148
|
+
// Evict oldest-first until BOTH bounds hold — the count that has always been
|
|
3149
|
+
// here, and the byte budget #625 added. See MAX_BUFFER_CHARS for the numbers.
|
|
3150
|
+
//
|
|
3151
|
+
// Counted first and spliced once, rather than shifting in a loop, because the
|
|
3152
|
+
// two bounds evict at very different scales. The count bound drops exactly
|
|
3153
|
+
// one entry per push; the byte bound can drop hundreds, since twenty-seven
|
|
3154
|
+
// maximum-size events fill the whole budget on their own, and a shift per
|
|
3155
|
+
// entry would memmove the array once for each of them.
|
|
3156
|
+
//
|
|
3157
|
+
// `drop < events.length - 1` is what keeps the event just pushed, whatever it
|
|
3158
|
+
// weighs. A single event is allowed to be larger than the entire budget —
|
|
3159
|
+
// ingest admits 5,000,000 characters, and a Codex rollout line read off disk
|
|
3160
|
+
// has no length bound at all — and evicting it on arrival would leave
|
|
3161
|
+
// pushEvent returning an envelope that `GET /api/events` never shows and no
|
|
3162
|
+
// resuming client can ever be handed: a hole with no id to ask for it again,
|
|
3163
|
+
// which is the exact failure the resume path below is written to avoid. So
|
|
3164
|
+
// the true ceiling is MAX_BUFFER_CHARS plus one event, and that is stated
|
|
3165
|
+
// here rather than pretended away.
|
|
3166
|
+
//
|
|
3167
|
+
// What this does to a resuming client is what the count bound has always done
|
|
3168
|
+
// to one, only sooner: the head of the ring moves, and events that fell off
|
|
3169
|
+
// it are gone for anybody who had not been sent them yet. That is the
|
|
3170
|
+
// existing bargain for a too-old Last-Event-ID — handleSse replays whatever
|
|
3171
|
+
// is still held and the client's `lastSeq` steps forward over the gap — and
|
|
3172
|
+
// the byte bound deliberately reuses it rather than inventing a second answer.
|
|
3173
|
+
// The difference worth knowing is that the head can now move in jumps rather
|
|
3174
|
+
// than one entry at a time; resumeSse's per-pass snapshot is what makes that
|
|
3175
|
+
// safe for a replay already in flight.
|
|
3176
|
+
let drop = 0;
|
|
3177
|
+
let freed = 0;
|
|
3178
|
+
while (drop < events.length - 1
|
|
3179
|
+
&& (events.length - drop > MAX_BUFFER || bufferedChars - freed > MAX_BUFFER_CHARS)) {
|
|
3180
|
+
freed += events[drop][CHARS];
|
|
3181
|
+
drop++;
|
|
3182
|
+
}
|
|
3183
|
+
if (drop > 0) {
|
|
3184
|
+
events.splice(0, drop);
|
|
3185
|
+
bufferedChars -= freed;
|
|
3186
|
+
}
|
|
2056
3187
|
|
|
2057
3188
|
// Does this event reach the log at all? Not on a replay (it came from
|
|
2058
3189
|
// there), not when the hook told us another deck owns this session's log,
|
|
@@ -2068,7 +3199,52 @@ function pushEvent(raw, source, opts = {}) {
|
|
|
2068
3199
|
// — which runs before the listener exists and never broadcasts — paid one
|
|
2069
3200
|
// for every line of a log that rotates at 50MB. An event this deck is not
|
|
2070
3201
|
// logging is still broadcast, so a subscriber alone is reason enough.
|
|
2071
|
-
|
|
3202
|
+
//
|
|
3203
|
+
// Contained, because this line was fatal. `JSON.stringify` walks a value
|
|
3204
|
+
// recursively while `JSON.parse` does not, and the gap between the two is
|
|
3205
|
+
// enormous: measured on Node 22.14, parse accepts a body nested 4,194,303
|
|
3206
|
+
// deep and stringify gives up on the result at 4,021. So a payload in that
|
|
3207
|
+
// window parses cleanly and then throws `RangeError: Maximum call stack size
|
|
3208
|
+
// exceeded` out of here — and there is no promise on this path for the
|
|
3209
|
+
// route's `guard` to catch, because pushEvent is reached from inside a raw
|
|
3210
|
+
// `req.on("end")` listener. It was an uncaughtException, and Node's answer to
|
|
3211
|
+
// those is to exit. Measured against the real server: one POST of 24,378
|
|
3212
|
+
// bytes, nested 4,050 deep, to the credential-free `/api/event` — a
|
|
3213
|
+
// two-hundredth of the 5,000,000-character ingest cap — and the deck was
|
|
3214
|
+
// gone, with nothing on the socket to tell the poster why.
|
|
3215
|
+
//
|
|
3216
|
+
// The payload leaves the ring, not just this string, and that is the point.
|
|
3217
|
+
// `events.push` above already took the envelope, and a value nothing can
|
|
3218
|
+
// serialize is a value no reader can ever deliver: every `Last-Event-ID`
|
|
3219
|
+
// resume that replays it and every `GET /api/events` that writes it would
|
|
3220
|
+
// meet the same throw for as long as it stayed in the buffer, so one small
|
|
3221
|
+
// POST would poison both routes for the life of the entry. The envelope
|
|
3222
|
+
// therefore keeps its `seq` and loses its payload — the same replacement
|
|
3223
|
+
// envelopeJson makes for a reader, made once at the write instead of on every
|
|
3224
|
+
// read, and the same bargain about `seq`: a caller paging with `?since=`
|
|
3225
|
+
// walks past the hole rather than asking forever for what it cannot be given.
|
|
3226
|
+
//
|
|
3227
|
+
// Admitted as a stub rather than refused at ingest with a 400, deliberately.
|
|
3228
|
+
// Three of pushEvent's four callers have no HTTP peer to answer — Codex
|
|
3229
|
+
// rollout lines read off disk, the boot replay of events.jsonl, the synthetic
|
|
3230
|
+
// events the transcript scanners emit — so the containment has to live here
|
|
3231
|
+
// whatever the ingest route does, and a 400 on top would be a second
|
|
3232
|
+
// mechanism for a case this one already covers. It would also have to be paid
|
|
3233
|
+
// for: knowing a payload will not serialize means serializing it, which is
|
|
3234
|
+
// the second stringify per event on the hottest path in the process that the
|
|
3235
|
+
// paragraph above exists to have removed. The reason goes to stderr and not
|
|
3236
|
+
// to the wire, under the rule sendInternalError explains.
|
|
3237
|
+
let json = null;
|
|
3238
|
+
if (sseClients.size > 0 || persisting) {
|
|
3239
|
+
try {
|
|
3240
|
+
json = JSON.stringify(evt);
|
|
3241
|
+
} catch (err) {
|
|
3242
|
+
console.error(`${PRODUCT}: event ${seq} could not be serialized:`, err);
|
|
3243
|
+
evt.payload = null;
|
|
3244
|
+
evt.unserializable = true;
|
|
3245
|
+
json = JSON.stringify(evt);
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
2072
3248
|
|
|
2073
3249
|
if (sseClients.size > 0) {
|
|
2074
3250
|
const line = `id: ${seq}\nevent: hook\ndata: ${json}\n\n`;
|
|
@@ -2109,17 +3285,98 @@ function pushEvent(raw, source, opts = {}) {
|
|
|
2109
3285
|
if (source === "hook" && !opts.replay) {
|
|
2110
3286
|
if (raw && raw.provider === "codex") {
|
|
2111
3287
|
maybeResolveCodex(raw);
|
|
2112
|
-
} else {
|
|
3288
|
+
} else if (!raw?.transcript_path || isClaudeTranscriptPath(raw.transcript_path)) {
|
|
3289
|
+
// The gate is here, once, rather than repeated in the four scanners
|
|
3290
|
+
// below it: this is the single door a caller-chosen path comes through,
|
|
3291
|
+
// and all four read the same field off the same payload. A payload with
|
|
3292
|
+
// no transcript_path at all still goes through — every one of them
|
|
3293
|
+
// early-returns without it, and Codex hooks never send one — so the
|
|
3294
|
+
// ordinary event costs nothing but the absent-field test.
|
|
2113
3295
|
maybeResolveModel(raw);
|
|
2114
3296
|
maybeResolveUsage(raw);
|
|
2115
3297
|
maybeResolveContext(raw);
|
|
2116
3298
|
maybeResolveSessionName(raw);
|
|
3299
|
+
} else {
|
|
3300
|
+
noteRefusedTranscript(raw.transcript_path);
|
|
2117
3301
|
}
|
|
2118
3302
|
}
|
|
2119
3303
|
|
|
2120
3304
|
return evt;
|
|
2121
3305
|
}
|
|
2122
3306
|
|
|
3307
|
+
/**
|
|
3308
|
+
* Which of the log's events belong on THIS deck's canvas — the boot replay's
|
|
3309
|
+
* half of `--workspace`, and the half that did not exist (#696).
|
|
3310
|
+
*
|
|
3311
|
+
* `--workspace` filtered the two LIVE capture paths and nothing else. The log is
|
|
3312
|
+
* the machine-wide `<claude config dir>/agent-dag/events.jsonl` that every deck
|
|
3313
|
+
* on the box shares by default, and `replayLog` pushed all of it, so a deck
|
|
3314
|
+
* started with `--workspace ~/proj` came up with every session on the machine
|
|
3315
|
+
* already drawn — including the ones it had just printed it would not capture,
|
|
3316
|
+
* contradicting its own `workspace` row, README.md and the empty-state sentence
|
|
3317
|
+
* in src/web/scope.ts that exists precisely so the canvas stops asserting things
|
|
3318
|
+
* that are not true (#404).
|
|
3319
|
+
*
|
|
3320
|
+
* THE RULE IS NOT A NEW ONE. Per event it is `codexCwdInWorkspace`, the same
|
|
3321
|
+
* predicate the Codex watcher runs and the twin of `capturesSession` in
|
|
3322
|
+
* hook/hook.js — the two are pinned equal by a test walking one table of paths
|
|
3323
|
+
* through both, so `--workspace` means one thing on every path a payload can
|
|
3324
|
+
* reach the ring by, including this one. Nothing about case folding, separators
|
|
3325
|
+
* or the sibling-prefix trap (`/srv/projX` is not inside `/srv/proj`) is decided
|
|
3326
|
+
* here; it is decided there, once, per platform.
|
|
3327
|
+
*
|
|
3328
|
+
* WHAT THE MAP IS FOR. Not every line carries a cwd. The synthetic enrichment
|
|
3329
|
+
* events the transcript scanners emit — `ModelObserved`, `UsageObserved`,
|
|
3330
|
+
* `SessionNamed`, `ContextObserved` — carry `session_id` and nothing else, and
|
|
3331
|
+
* they are persisted like any other event. Judging those by the live rule alone
|
|
3332
|
+
* (no cwd, so inside no workspace) would keep an in-scope session on the canvas
|
|
3333
|
+
* while stripping its model, its token columns and its name until the next live
|
|
3334
|
+
* event arrived. So the answer for a session is learned from the events that DO
|
|
3335
|
+
* say where they run and carried forward to the ones that do not.
|
|
3336
|
+
*
|
|
3337
|
+
* That is a reconstruction of the live behaviour rather than a second notion of
|
|
3338
|
+
* scope: live, a deck only ever emits `ModelObserved` for a session whose hook
|
|
3339
|
+
* event it already accepted, so "follows its session" is what those events
|
|
3340
|
+
* already do — the map only re-derives it from a file. It costs one entry per
|
|
3341
|
+
* distinct session id in the log (a few thousand at the very most, against a log
|
|
3342
|
+
* measured in tens of megabytes) and one lookup per line.
|
|
3343
|
+
*
|
|
3344
|
+
* TWO CASES DECIDED EXPLICITLY:
|
|
3345
|
+
*
|
|
3346
|
+
* * `__clear` — the control marker `/api/clear` writes after truncating, with
|
|
3347
|
+
* `cwd: ""`. It is not a session event; it is the instruction that makes the
|
|
3348
|
+
* reducer forget everything before it. Dropping it on a scoped deck would
|
|
3349
|
+
* replay the state a user had explicitly cleared, so it is always admitted.
|
|
3350
|
+
* * a payload with no cwd and no session the map has seen — refused on a
|
|
3351
|
+
* scoped deck, which is exactly what `capturesSession` decides live for a
|
|
3352
|
+
* session that never said where it runs.
|
|
3353
|
+
*
|
|
3354
|
+
* An unscoped deck (`workspace === ""`, the default) admits everything, and
|
|
3355
|
+
* takes the cheapest possible path to saying so.
|
|
3356
|
+
*
|
|
3357
|
+
* @param {string} workspace this deck's canonical workspace; "" for machine-wide
|
|
3358
|
+
* @returns {(payload: unknown) => boolean} called once per replayed envelope, in
|
|
3359
|
+
* log order — it remembers, so the order matters and it is not reusable across
|
|
3360
|
+
* two replays.
|
|
3361
|
+
*/
|
|
3362
|
+
export function replayScope(workspace, platform = process.platform) {
|
|
3363
|
+
if (!workspace || typeof workspace !== "string") return () => true;
|
|
3364
|
+
const bySession = new Map();
|
|
3365
|
+
return function admits(payload) {
|
|
3366
|
+
if (!payload || typeof payload !== "object") return false;
|
|
3367
|
+
if (payload.hook_event_name === "__clear") return true;
|
|
3368
|
+
const sid = typeof payload.session_id === "string" ? payload.session_id : null;
|
|
3369
|
+
const cwd = typeof payload.cwd === "string" && payload.cwd !== "" ? payload.cwd : null;
|
|
3370
|
+
if (cwd) {
|
|
3371
|
+
const inside = codexCwdInWorkspace(cwd, workspace, platform);
|
|
3372
|
+
if (sid) bySession.set(sid, inside);
|
|
3373
|
+
return inside;
|
|
3374
|
+
}
|
|
3375
|
+
if (sid && bySession.has(sid)) return bySession.get(sid);
|
|
3376
|
+
return false;
|
|
3377
|
+
};
|
|
3378
|
+
}
|
|
3379
|
+
|
|
2123
3380
|
/**
|
|
2124
3381
|
* Read the log back into the ring buffer at boot.
|
|
2125
3382
|
*
|
|
@@ -2143,18 +3400,37 @@ function pushEvent(raw, source, opts = {}) {
|
|
|
2143
3400
|
* onto a terminal the deck is about to paint over (see oneLine in term.mjs for
|
|
2144
3401
|
* what a multi-line message does there), and the path was already printed at
|
|
2145
3402
|
* boot by the caller.
|
|
3403
|
+
*
|
|
3404
|
+
* `workspace` is what makes the replay agree with the two live capture paths —
|
|
3405
|
+
* see replayScope. An out-of-scope line is NOT counted into `skipped` and is not
|
|
3406
|
+
* warned about: `skipped` means "this log has bytes in it no reader can parse",
|
|
3407
|
+
* which is damage and is worth a line on the terminal, while a scoped deck
|
|
3408
|
+
* declining a session it was told not to capture is the flag doing its job. They
|
|
3409
|
+
* are two different things and only the first is ever printed.
|
|
3410
|
+
*
|
|
3411
|
+
* WHAT THE FILTER COSTS AT BOOT, honestly: nothing is saved on the read. Every
|
|
3412
|
+
* line is still streamed off disk and still JSON.parsed, because the cwd being
|
|
3413
|
+
* judged is inside the JSON — a 30 MB log is 30 MB of reading and parsing on a
|
|
3414
|
+
* scoped deck exactly as on an unscoped one. What it saves is everything after
|
|
3415
|
+
* the parse: no redaction pass, no envelope, no ring insert, no character
|
|
3416
|
+
* accounting and no eviction pressure for a line this deck should never have
|
|
3417
|
+
* held. The ring is bounded by MAX_BUFFER events AND MAX_BUFFER_CHARS, so on a
|
|
3418
|
+
* busy machine the out-of-scope traffic was not merely extra — it was evicting
|
|
3419
|
+
* the in-scope sessions the user started the deck to watch.
|
|
2146
3420
|
*/
|
|
2147
|
-
async function replayLog(filePath) {
|
|
3421
|
+
async function replayLog(filePath, workspace = "") {
|
|
2148
3422
|
if (!existsSync(filePath)) return 0;
|
|
2149
3423
|
let count = 0;
|
|
2150
3424
|
let skipped = 0;
|
|
2151
3425
|
let skippedBytes = 0;
|
|
3426
|
+
const admits = replayScope(workspace);
|
|
2152
3427
|
const rl = createInterface({ input: createReadStream(filePath, { encoding: "utf8" }) });
|
|
2153
3428
|
for await (const line of rl) {
|
|
2154
3429
|
if (!line) continue;
|
|
2155
3430
|
try {
|
|
2156
3431
|
const evt = JSON.parse(line);
|
|
2157
3432
|
if (evt && typeof evt === "object" && evt.payload) {
|
|
3433
|
+
if (!admits(evt.payload)) continue;
|
|
2158
3434
|
pushEvent(evt.payload, evt.source ?? "replay", { receivedAt: evt.receivedAt, replay: true });
|
|
2159
3435
|
count++;
|
|
2160
3436
|
}
|
|
@@ -2179,6 +3455,109 @@ function send(res, status, body, headers = {}) {
|
|
|
2179
3455
|
res.end(typeof body === "string" ? body : JSON.stringify(body));
|
|
2180
3456
|
}
|
|
2181
3457
|
|
|
3458
|
+
/**
|
|
3459
|
+
* Serialize one envelope, or a stub standing in for it.
|
|
3460
|
+
*
|
|
3461
|
+
* `JSON.stringify` throws on more than size. V8 walks a value recursively, so a
|
|
3462
|
+
* payload nested about five thousand deep overflows the C++ stack and comes
|
|
3463
|
+
* back as `RangeError: Maximum call stack size exceeded` — and a 36 KB POST to
|
|
3464
|
+
* the open ingest route is enough to put one of those in the ring, measured on
|
|
3465
|
+
* Node 22.14. Letting that throw escape would truncate the array mid-write and
|
|
3466
|
+
* leave every later read of that ring answering with invalid JSON for as long
|
|
3467
|
+
* as the event survives, which is a 36 KB way to poison a route permanently.
|
|
3468
|
+
*
|
|
3469
|
+
* So the envelope is replaced rather than dropped, and it keeps its `seq`: a
|
|
3470
|
+
* caller paging with `?since=` still walks past it, instead of asking again for
|
|
3471
|
+
* a hole it can never be given — the same bargain resumeSse makes about the
|
|
3472
|
+
* events it cannot deliver. The reason is written to stderr and not to the
|
|
3473
|
+
* wire, under the rule sendInternalError explains: this body is readable by a
|
|
3474
|
+
* DNS-rebound page and error detail is not.
|
|
3475
|
+
*/
|
|
3476
|
+
function envelopeJson(evt) {
|
|
3477
|
+
try {
|
|
3478
|
+
// `JSON.stringify(undefined)` is undefined, not a string, and inside an
|
|
3479
|
+
// array literal that would be the text "undefined" — which is not JSON.
|
|
3480
|
+
// `JSON.stringify([undefined])` says "null"; so does this.
|
|
3481
|
+
return JSON.stringify(evt) ?? "null";
|
|
3482
|
+
} catch (err) {
|
|
3483
|
+
console.error(`${PRODUCT}: event ${evt?.seq} could not be serialized:`, err);
|
|
3484
|
+
// Every field here is read defensively and typed to a primitive, because
|
|
3485
|
+
// this is the path that must not throw twice: the status line has gone out
|
|
3486
|
+
// and a second failure would leave the caller a truncated array.
|
|
3487
|
+
return JSON.stringify({
|
|
3488
|
+
seq: Number(evt?.seq) || 0,
|
|
3489
|
+
epoch: typeof evt?.epoch === "string" ? evt.epoch : SEQ_EPOCH,
|
|
3490
|
+
receivedAt: Number(evt?.receivedAt) || 0,
|
|
3491
|
+
source: typeof evt?.source === "string" ? evt.source : "unknown",
|
|
3492
|
+
payload: null,
|
|
3493
|
+
unserializable: true,
|
|
3494
|
+
});
|
|
3495
|
+
}
|
|
3496
|
+
}
|
|
3497
|
+
|
|
3498
|
+
/**
|
|
3499
|
+
* Answer with a JSON array without ever holding it as one string.
|
|
3500
|
+
*
|
|
3501
|
+
* `send` finishes a response by handing the whole body to a single
|
|
3502
|
+
* `JSON.stringify`, which for every other route is a few hundred bytes and for
|
|
3503
|
+
* `GET /api/events` is the entire ring buffer. V8 will not build a string
|
|
3504
|
+
* longer than `2^29 - 24` characters, so past 536,870,888 characters of
|
|
3505
|
+
* serialised envelopes that call throws `RangeError: Invalid string length`
|
|
3506
|
+
* straight out of the request listener — and there, as requestUrl and `guard`
|
|
3507
|
+
* both say in their own words, nothing catches it and the worker exits.
|
|
3508
|
+
* Measured on Node 22.14 / macOS: 112 posts of 4,900,061 characters, which
|
|
3509
|
+
* `POST /api/event` accepts from anyone with no credential at all, then one
|
|
3510
|
+
* plain unauthenticated GET, and the deck was gone — SSE stream, hook ingest
|
|
3511
|
+
* and event log with it. 112 events is a twentieth of MAX_BUFFER, so this is
|
|
3512
|
+
* not an exotic ring: a deck watching sessions whose Read and Bash responses
|
|
3513
|
+
* are "routinely a good fraction of" the five-million-character ingest cap
|
|
3514
|
+
* reaches it on its own at an average of 268 KB an event.
|
|
3515
|
+
*
|
|
3516
|
+
* Writing the array element by element removes the ceiling rather than raising
|
|
3517
|
+
* it. One envelope's worth of string exists at a time, so the limit applies per
|
|
3518
|
+
* envelope — and ingest caps an envelope at a hundredth of it — and the peak
|
|
3519
|
+
* cost is the ring plus one event instead of the ring plus a contiguous copy of
|
|
3520
|
+
* itself, which is the quieter half of the same bug: a 400 MB ring used to need
|
|
3521
|
+
* 800 MB and a synchronous stall on a route that feels free on a quiet deck.
|
|
3522
|
+
* This is the shape resumeSse already uses for the SSE replay, for the same
|
|
3523
|
+
* reason, and it borrows the same writeResume, so a caller that stops reading
|
|
3524
|
+
* is held at MAX_CLIENT_BUFFER_BYTES here too rather than having the whole ring
|
|
3525
|
+
* queued in userland on its behalf.
|
|
3526
|
+
*
|
|
3527
|
+
* `items` must be a snapshot the caller owns — eventsSince returns one, `filter`
|
|
3528
|
+
* always allocating — because each await lets pushEvent splice the head off the
|
|
3529
|
+
* live ring, and iterating that while it is spliced skips entries.
|
|
3530
|
+
*
|
|
3531
|
+
* No Content-Length: it is not knowable without building the string this exists
|
|
3532
|
+
* to avoid, so the answer is chunked. HTTP/1.1 requires nothing more and every
|
|
3533
|
+
* consumer reads to EOF.
|
|
3534
|
+
*
|
|
3535
|
+
* Exported so the one property that matters can be asserted directly, on a stub
|
|
3536
|
+
* rather than through a socket — the same reason queuedBytes is. "Never builds
|
|
3537
|
+
* the whole array as one string" is invisible from outside a real response,
|
|
3538
|
+
* which is how it went unnoticed here for as long as it did.
|
|
3539
|
+
*/
|
|
3540
|
+
export async function writeJsonArray(res, items) {
|
|
3541
|
+
res.writeHead(200, {
|
|
3542
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
3543
|
+
"Cache-Control": "no-store",
|
|
3544
|
+
});
|
|
3545
|
+
let frame = "[";
|
|
3546
|
+
for (const item of items) {
|
|
3547
|
+
if (res.destroyed) return;
|
|
3548
|
+
if (!await writeResume(res, frame + envelopeJson(item))) {
|
|
3549
|
+
// Stalled past REPLAY_DRAIN_MS with the cap full. Nothing useful can be
|
|
3550
|
+
// said in-band — the status line went out long ago and the array is half
|
|
3551
|
+
// written — so hang up, exactly as the replay does.
|
|
3552
|
+
try { res.destroy(); } catch {}
|
|
3553
|
+
return;
|
|
3554
|
+
}
|
|
3555
|
+
frame = ",";
|
|
3556
|
+
}
|
|
3557
|
+
if (res.destroyed) return;
|
|
3558
|
+
res.end(frame === "[" ? "[]" : "]");
|
|
3559
|
+
}
|
|
3560
|
+
|
|
2182
3561
|
async function serveStatic(req, res, url) {
|
|
2183
3562
|
// Strip leading slash, default to index.html
|
|
2184
3563
|
let rel = url.pathname.replace(/^\/+/, "");
|
|
@@ -2279,15 +3658,37 @@ function handleEventIngest(req, res, persist = true) {
|
|
|
2279
3658
|
let parsed;
|
|
2280
3659
|
try { parsed = JSON.parse(body); }
|
|
2281
3660
|
catch { return send(res, 400, { error: "invalid json" }); }
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
3661
|
+
// Everything past the parse is inside one net, because this listener is the
|
|
3662
|
+
// one place in the route table `guard` cannot reach. The route does wrap the
|
|
3663
|
+
// call — `guard(handleEventIngest(req, res, …), res)` — but this function
|
|
3664
|
+
// returns undefined and hands its work to a listener the event loop calls
|
|
3665
|
+
// later, so `guard` has nothing to attach to and a synchronous throw in here
|
|
3666
|
+
// is an uncaughtException: the whole deck, for one POST. That is exactly
|
|
3667
|
+
// what the serialization inside pushEvent was until it was contained at the
|
|
3668
|
+
// line itself, and this catch is what stops the next thing added below from
|
|
3669
|
+
// costing a process the same way. sendInternalError puts the reason on
|
|
3670
|
+
// stderr and a bare 500 on the wire, and does nothing but end the response
|
|
3671
|
+
// if the status line has already gone out.
|
|
3672
|
+
try {
|
|
3673
|
+
noteLogWriter(parsed, persist);
|
|
3674
|
+
const evt = pushEvent(parsed, "hook", { persist });
|
|
3675
|
+
send(res, 200, { ok: true, seq: evt.seq });
|
|
3676
|
+
} catch (err) {
|
|
3677
|
+
sendInternalError(res, err);
|
|
3678
|
+
}
|
|
2285
3679
|
});
|
|
2286
3680
|
// Guarded for the same reason `end` is, and more sharply: destroying the
|
|
2287
3681
|
// request above is itself what raises this, and answering a second time on a
|
|
2288
3682
|
// response already sent throws ERR_HTTP_HEADERS_SENT out of an error handler,
|
|
2289
|
-
// where nothing is waiting to catch it.
|
|
2290
|
-
|
|
3683
|
+
// where nothing is waiting to catch it. `refused` covers the 413 that
|
|
3684
|
+
// destroyed the request; `headersSent` covers the other half of that
|
|
3685
|
+
// sentence, an error arriving after `end` has already answered. That half
|
|
3686
|
+
// resisted every attempt to drive it from a socket — once `end` has fired the
|
|
3687
|
+
// message is complete and the failure goes to the socket rather than to the
|
|
3688
|
+
// request — so it carries no test, and is guarded anyway on the strength of
|
|
3689
|
+
// the hazard the sentence above already names: one condition against an
|
|
3690
|
+
// uncaughtException, if it turns out to be reachable at all.
|
|
3691
|
+
req.on("error", () => { if (!refused && !res.headersSent) send(res, 400, { error: "bad request" }); });
|
|
2291
3692
|
}
|
|
2292
3693
|
|
|
2293
3694
|
function handleSse(req, res) {
|
|
@@ -2302,6 +3703,16 @@ function handleSse(req, res) {
|
|
|
2302
3703
|
// A stale or absent id replays the whole ring, and so does a malformed one:
|
|
2303
3704
|
// Number("nonsense") is NaN, every `seq <= NaN` is false, and the catch-up
|
|
2304
3705
|
// loop below would rather compare against a number.
|
|
3706
|
+
//
|
|
3707
|
+
// An id OLDER than the ring's oldest event is a stale id and takes exactly
|
|
3708
|
+
// that path — nothing special-cases it, and #625 deliberately did not add a
|
|
3709
|
+
// second answer when it gave the ring a byte budget. Every `e.seq <=
|
|
3710
|
+
// sentThrough` test simply fails, so the client is handed everything still
|
|
3711
|
+
// held, contiguously, and the sentinel behind it; the events that were
|
|
3712
|
+
// evicted are missing from its HISTORY, never from its stream. The reducer's
|
|
3713
|
+
// guard is `env.seq <= state.lastSeq`, so the gap costs it a step forward and
|
|
3714
|
+
// nothing else. What the byte budget changed is how often and how far the
|
|
3715
|
+
// head moves, not what happens to a client that lands behind it.
|
|
2305
3716
|
const asked = Number(req.headers["last-event-id"] ?? 0);
|
|
2306
3717
|
const lastId = Number.isFinite(asked) ? asked : 0;
|
|
2307
3718
|
|
|
@@ -2339,6 +3750,16 @@ async function resumeSse(req, res, lastId) {
|
|
|
2339
3750
|
// `events` off, and iterating an array being spliced from the front skips
|
|
2340
3751
|
// entries. Events evicted that way are gone for this client, which is the
|
|
2341
3752
|
// same bargain every resume against a rotated ring already makes.
|
|
3753
|
+
//
|
|
3754
|
+
// The snapshot earns more since #625 gave the ring a byte budget as well as
|
|
3755
|
+
// a count. Under the count alone the head moved one entry per push; under
|
|
3756
|
+
// the budget a single 5 MB event can evict hundreds at once. A replay
|
|
3757
|
+
// already walking this array would have skipped every one of them — but the
|
|
3758
|
+
// snapshot holds its own references, so the pass in flight still delivers
|
|
3759
|
+
// what it was given and only a LATER pass sees the shortened ring. It also
|
|
3760
|
+
// means a slow resumer pins one ring's worth of envelopes for as long as its
|
|
3761
|
+
// pass lasts, which the budget bounds too: that pin used to be unbounded for
|
|
3762
|
+
// the same reason the ring was.
|
|
2342
3763
|
const batch = events.slice();
|
|
2343
3764
|
for (const e of batch) {
|
|
2344
3765
|
if (e.seq <= sentThrough) continue;
|
|
@@ -2351,7 +3772,19 @@ async function resumeSse(req, res, lastId) {
|
|
|
2351
3772
|
// wall-clock visibility gates and yields the "nodes appear then vanish"
|
|
2352
3773
|
// symptom on refresh.
|
|
2353
3774
|
const tagged = { ...e, replay: true };
|
|
2354
|
-
|
|
3775
|
+
// Through envelopeJson for the reason writeJsonArray is: the ring may be
|
|
3776
|
+
// holding an envelope `JSON.stringify` cannot walk. pushEvent takes the
|
|
3777
|
+
// payload out of every envelope it serializes, but it serializes nothing
|
|
3778
|
+
// when there is no subscriber and no log — which is precisely the deck a
|
|
3779
|
+
// browser is about to connect to — so the first read of such an entry can
|
|
3780
|
+
// still be this one. Uncontained it rejected into handleSse's `.catch`,
|
|
3781
|
+
// which drops the client; the browser reconnects on its own 1.5s timer,
|
|
3782
|
+
// replays the same entry, and is dropped again, so a single small POST
|
|
3783
|
+
// kept the canvas from ever loading. The stub loses the `replay` tag and
|
|
3784
|
+
// nothing turns on that: it carries no payload for the reducer to act on,
|
|
3785
|
+
// and the client's own replay gate runs until the `replay-end` sentinel
|
|
3786
|
+
// regardless of what any one envelope says.
|
|
3787
|
+
if (!await writeResume(res, `id: ${e.seq}\nevent: hook\ndata: ${envelopeJson(tagged)}\n\n`)) {
|
|
2355
3788
|
return dropSse(res);
|
|
2356
3789
|
}
|
|
2357
3790
|
sentThrough = e.seq;
|
|
@@ -2721,25 +4154,6 @@ async function handleCswapAutoAction(req, res) {
|
|
|
2721
4154
|
send(res, result.ok ? 200 : 400, result);
|
|
2722
4155
|
}
|
|
2723
4156
|
|
|
2724
|
-
async function handleSoundHook(req, res) {
|
|
2725
|
-
const { soundHookStatus } = await import(
|
|
2726
|
-
pathToFileURL(join(PKG_ROOT, "src/server/sound-hook.mjs")).href
|
|
2727
|
-
);
|
|
2728
|
-
send(res, 200, await soundHookStatus());
|
|
2729
|
-
}
|
|
2730
|
-
|
|
2731
|
-
async function handleSoundHookSet(req, res) {
|
|
2732
|
-
const { setSoundHook, restoreParkedSoundHooks } = await import(
|
|
2733
|
-
pathToFileURL(join(PKG_ROOT, "src/server/sound-hook.mjs")).href
|
|
2734
|
-
);
|
|
2735
|
-
const body = await readBody(req).catch(() => null);
|
|
2736
|
-
let parsed = null;
|
|
2737
|
-
try { parsed = JSON.parse(body ?? ""); } catch { /* handled below */ }
|
|
2738
|
-
if (!parsed || typeof parsed !== "object") return send(res, 400, { ok: false, reason: "bad_request" });
|
|
2739
|
-
if (parsed.action === "restore") return send(res, 200, await restoreParkedSoundHooks());
|
|
2740
|
-
send(res, 200, await setSoundHook(parsed.enabled === true));
|
|
2741
|
-
}
|
|
2742
|
-
|
|
2743
4157
|
// The tree this deck was told to capture, "" when it captures the whole
|
|
2744
4158
|
// machine. Set by startServer and never changed afterwards.
|
|
2745
4159
|
//
|
|
@@ -2784,17 +4198,193 @@ let _providers = { claude: true, codex: true };
|
|
|
2784
4198
|
* the events log before publishing it: what goes in the discovery file is read
|
|
2785
4199
|
* by other processes that cannot reconstruct the context it was written in.
|
|
2786
4200
|
*
|
|
2787
|
-
* Symlinks are resolved too,
|
|
2788
|
-
*
|
|
2789
|
-
*
|
|
2790
|
-
*
|
|
2791
|
-
*
|
|
2792
|
-
*
|
|
4201
|
+
* Symlinks are resolved too, so that `--workspace /tmp/proj` on a Mac is not
|
|
4202
|
+
* scoped to /tmp/proj while every session inside it reports /private/tmp/proj,
|
|
4203
|
+
* leaving the deck empty. A path that does not exist yet keeps its resolved form
|
|
4204
|
+
* rather than failing: scoping a deck to a directory you are about to create is
|
|
4205
|
+
* not an error.
|
|
4206
|
+
*
|
|
4207
|
+
* That resolution is only half the rule. Canonicalising the flag means nothing
|
|
4208
|
+
* unless the cwd it is compared against is canonicalised the same way, and that
|
|
4209
|
+
* is a separate job on each capture path: hook.js does it with normPath, and the
|
|
4210
|
+
* rollout watcher does it with canonicalCwd below. This comment used to claim
|
|
4211
|
+
* the second half came for free — that "a process's cwd comes from getcwd() and
|
|
4212
|
+
* has none left in it" — which is true on POSIX and false on Windows. See
|
|
4213
|
+
* canonicalCwd.
|
|
4214
|
+
*
|
|
4215
|
+
* WHICH REALPATH, AND WHY IT IS `.native` AT ALL THREE SITES. Node ships two
|
|
4216
|
+
* implementations with different Windows behaviour, and picking one per site is
|
|
4217
|
+
* how the flag and the two cwd paths end up meaning different things again:
|
|
4218
|
+
*
|
|
4219
|
+
* fs.realpathSync / fs.realpath a JavaScript lstat-and-readlink walk. It
|
|
4220
|
+
* resolves symlinks and junctions and NOTHING
|
|
4221
|
+
* else — a DOS 8.3 short component survives it
|
|
4222
|
+
* untouched.
|
|
4223
|
+
* …Sync.native / …native / the uv_fs_realpath, which on Windows is
|
|
4224
|
+
* fs/promises realpath GetFinalPathNameByHandleW: symlinks and
|
|
4225
|
+
* junctions, the UNC form of a mapped drive,
|
|
4226
|
+
* the LONG form of an 8.3 short name, and the
|
|
4227
|
+
* on-disk case of every component.
|
|
4228
|
+
*
|
|
4229
|
+
* The long form is the canonical one, and not by preference: it is the only
|
|
4230
|
+
* spelling that is a fixed point. Short names are an alias for the same
|
|
4231
|
+
* directory exactly as a junction is, they cannot be derived back from the long
|
|
4232
|
+
* form (8.3 generation can be disabled per volume), and they are not a corner
|
|
4233
|
+
* case a test invented — `%TEMP%` under a shortened profile directory answers
|
|
4234
|
+
* with one, which is what every GitHub Windows runner has. It is also already
|
|
4235
|
+
* this codebase's answer: persistAuth in src/server/codex-auth.mjs resolves
|
|
4236
|
+
* through the fs/promises realpath, i.e. this one.
|
|
4237
|
+
*
|
|
4238
|
+
* So all three sites name `.native` explicitly rather than one of them reaching
|
|
4239
|
+
* for whichever realpath came to hand. Being explicit is the point — the reason
|
|
4240
|
+
* this had to be said twice is that fs/promises' realpath IS the native one
|
|
4241
|
+
* while its sync namesake is not, an equivalence nothing documents and no reader
|
|
4242
|
+
* should have to know.
|
|
2793
4243
|
*/
|
|
2794
4244
|
export function canonicalWorkspace(raw) {
|
|
2795
4245
|
if (typeof raw !== "string" || raw.trim() === "") return "";
|
|
2796
4246
|
const abs = resolve(raw);
|
|
2797
|
-
try { return realpathSync(abs); } catch { return abs; }
|
|
4247
|
+
try { return realpathSync.native(abs); } catch { return abs; }
|
|
4248
|
+
}
|
|
4249
|
+
|
|
4250
|
+
// The async half of the rule canonicalWorkspace states above, named the same way
|
|
4251
|
+
// it is: fs.realpath.native is the documented callback form of uv_fs_realpath,
|
|
4252
|
+
// and promisifying it says which realpath this is at the point of use. The
|
|
4253
|
+
// fs/promises realpath is the same call and would have read as if it were the
|
|
4254
|
+
// JavaScript one.
|
|
4255
|
+
const realpathNative = promisify(realpathCb.native);
|
|
4256
|
+
|
|
4257
|
+
/**
|
|
4258
|
+
* The one spelling of the directory a Codex session says it is running in —
|
|
4259
|
+
* hook.js's normPath, for the capture path that never goes through the hook.
|
|
4260
|
+
*
|
|
4261
|
+
* `--workspace` is canonicalised above before anything compares against it, and
|
|
4262
|
+
* the Claude side canonicalises the session's cwd to match: hook.js runs every
|
|
4263
|
+
* incoming cwd through normPath (resolve + realpath) before asking
|
|
4264
|
+
* capturesSession. The Codex side compared the rollout header's `cwd` raw, and
|
|
4265
|
+
* the comment above said that was safe because a cwd comes from getcwd(), which
|
|
4266
|
+
* has already resolved every link.
|
|
4267
|
+
*
|
|
4268
|
+
* That is true of getcwd(3) and not true of Windows. There the current directory
|
|
4269
|
+
* is stored as the string it was set with, and GetCurrentDirectoryW — what
|
|
4270
|
+
* Rust's std::env::current_dir() behind Codex calls — hands that string back
|
|
4271
|
+
* without resolving a junction, a `subst` drive or a mapped network drive.
|
|
4272
|
+
* realpath does resolve them, and on a mapped drive goes further and returns the
|
|
4273
|
+
* UNC form, because libuv asks GetFinalPathNameByHandleW. So a deck started as
|
|
4274
|
+
* `--workspace Z:\proj` scoped itself to `\\server\share\proj`, the Claude
|
|
4275
|
+
* session in that tree reported `Z:\proj` and was realpath'd into the workspace
|
|
4276
|
+
* and drawn, and the Codex session beside it reported `Z:\proj` in its rollout
|
|
4277
|
+
* header, was compared raw, and silently never appeared — no error printed
|
|
4278
|
+
* anywhere, the banner still claiming the rollout watcher was running. The same
|
|
4279
|
+
* asymmetry runs in reverse for a user who passes the resolved path and works
|
|
4280
|
+
* through the junction. The log election went wrong with it: writesCodexLog
|
|
4281
|
+
* models the OTHER decks' capture with this same predicate against their
|
|
4282
|
+
* published (canonical) workspaces, so it was picking the wrong group.
|
|
4283
|
+
*
|
|
4284
|
+
* Done here, at the one read of the header, rather than inside
|
|
4285
|
+
* codexCwdInWorkspace: the result is cached in codexFileState for the life of
|
|
4286
|
+
* the file, so it costs one realpath per rollout instead of one per tick, and it
|
|
4287
|
+
* leaves the predicate the pure string function that lets it be pinned against
|
|
4288
|
+
* hook.js's copy in a test.
|
|
4289
|
+
*
|
|
4290
|
+
* `.native`, for the reason canonicalWorkspace sets out at length: three sites
|
|
4291
|
+
* canonicalise a path against each other and all three must fold the same way,
|
|
4292
|
+
* 8.3 short names included. Case IS canonicalised as a side effect — that is
|
|
4293
|
+
* what GetFinalPathNameByHandleW and realpath(3) on a case-insensitive volume
|
|
4294
|
+
* return — but nothing here DEPENDS on it: the two predicates still fold case
|
|
4295
|
+
* per-platform themselves, which is the only place that decision can stay
|
|
4296
|
+
* correct on Linux, where /srv/Proj and /srv/proj are two real directories and
|
|
4297
|
+
* realpath quite rightly keeps them apart.
|
|
4298
|
+
*
|
|
4299
|
+
* Async, unlike canonicalWorkspace, because of where each one runs.
|
|
4300
|
+
* canonicalWorkspace runs once in bin/deck.js before the server exists, so
|
|
4301
|
+
* blocking there costs nothing. This runs inside the watcher's 1.5s tick, in the
|
|
4302
|
+
* live event loop, against a path a rollout recorded some time ago — which on
|
|
4303
|
+
* the very platform this exists for is quite likely to name a mapped drive that
|
|
4304
|
+
* is no longer connected, and a synchronous realpath on one of those blocks the
|
|
4305
|
+
* whole dashboard until SMB times out.
|
|
4306
|
+
*
|
|
4307
|
+
* A cwd that no longer resolves keeps its resolved form, exactly as the flag
|
|
4308
|
+
* does: a deleted directory or a disconnected drive is not a reason to drop a
|
|
4309
|
+
* session the deck can still draw, and the resolved string is the best answer
|
|
4310
|
+
* available — for a rollout recorded in the tree the deck is scoped to and never
|
|
4311
|
+
* moved, it is also the right one. Anything that is not a non-empty string is
|
|
4312
|
+
* null, which is what readCodexHeader returned before and what both copies of
|
|
4313
|
+
* the predicate read as "this session never said where it runs".
|
|
4314
|
+
*/
|
|
4315
|
+
export async function canonicalCwd(raw) {
|
|
4316
|
+
if (typeof raw !== "string" || raw.trim() === "") return null;
|
|
4317
|
+
const abs = resolve(raw);
|
|
4318
|
+
try { return await realpathNative(abs); } catch { return abs; }
|
|
4319
|
+
}
|
|
4320
|
+
|
|
4321
|
+
/**
|
|
4322
|
+
* The deck's one irreversible action: empty the ring, and empty the log — but
|
|
4323
|
+
* only the log this deck is the one writing.
|
|
4324
|
+
*
|
|
4325
|
+
* The gate is the whole of #698. `truncate(persistPath, 0)` ran from whichever
|
|
4326
|
+
* deck was asked, and `persistPath` is one file several decks share by default,
|
|
4327
|
+
* so Clear on a deck scoped to a single tree deleted the machine-wide deck's
|
|
4328
|
+
* entire history while its canvas showed no change at all. Ownership is
|
|
4329
|
+
* electWriters, the election that already decides which of those decks appends a
|
|
4330
|
+
* line, so nothing new gets to disagree with it: the deck that fills the file is
|
|
4331
|
+
* the deck that may empty it, and a deck that writes nothing to it cannot
|
|
4332
|
+
* destroy it. See logSharing.
|
|
4333
|
+
*
|
|
4334
|
+
* A deck that does not own the log still clears its own canvas — that is what
|
|
4335
|
+
* the user pressed, and the ring is this deck's alone — and says which deck's
|
|
4336
|
+
* file it declined to touch, so the answer is a fact the UI can show rather than
|
|
4337
|
+
* a silent partial success. The dialog asked `GET /api/clear` before the press
|
|
4338
|
+
* and has already said the same thing in words.
|
|
4339
|
+
*
|
|
4340
|
+
* The `__clear` marker is broadcast and NOT persisted. It never belonged on
|
|
4341
|
+
* disk: replaying an empty log and then a marker that empties it produces the
|
|
4342
|
+
* same empty state, and appending it was how a deck that writes nothing else to
|
|
4343
|
+
* the shared file still left 134 bytes in it — the reproduction's whole
|
|
4344
|
+
* remainder. `{ persist: false }` is the same flag the hook sets on the decks it
|
|
4345
|
+
* did not elect.
|
|
4346
|
+
*/
|
|
4347
|
+
async function handleClear(res) {
|
|
4348
|
+
const sharing = await logSharing();
|
|
4349
|
+
// Not `events.length = 0`: the ring is measured by a running total now, and
|
|
4350
|
+
// emptying the array without the total leaves a debt that never clears. See
|
|
4351
|
+
// clearEventBuffer.
|
|
4352
|
+
clearEventBuffer();
|
|
4353
|
+
if (sharing.path && sharing.mine) truncate(sharing.path, 0).catch(() => {});
|
|
4354
|
+
// Drop the caches that gate an emit on "has this changed", because the
|
|
4355
|
+
// client is about to forget what they are comparing against: __clear makes
|
|
4356
|
+
// the reducer return a fresh state, so every session's name and every
|
|
4357
|
+
// subagent's model label go with it. maybeResolveSessionName then computes
|
|
4358
|
+
// the same signature, takes its early return, and emits nothing — so the
|
|
4359
|
+
// card falls back to cwd/prompt for the rest of that session while the
|
|
4360
|
+
// server is sitting on the name.
|
|
4361
|
+
//
|
|
4362
|
+
// The root model survives without help because pushEvent stamps
|
|
4363
|
+
// `raw.model` on every payload; there is no equivalent stamp for the name
|
|
4364
|
+
// or for a subagent's model, which is why those two are listed and the
|
|
4365
|
+
// rest of the per-session state is not.
|
|
4366
|
+
//
|
|
4367
|
+
// The rule, for the next cache that gates an emit: anything answering
|
|
4368
|
+
// "has this changed" has to appear in BOTH places that mean the client no
|
|
4369
|
+
// longer has it — here, and in forgetSession.
|
|
4370
|
+
nameBySession.clear();
|
|
4371
|
+
modelBySession.clear();
|
|
4372
|
+
// The read stamps go with them. Clearing only the signatures would leave
|
|
4373
|
+
// the next hook event inside MODEL_READ_THROTTLE_MS, so the transcript
|
|
4374
|
+
// would not be re-read at all and the name would stay missing until the
|
|
4375
|
+
// throttle expired — a clear followed by a keystroke is exactly when a
|
|
4376
|
+
// user is watching.
|
|
4377
|
+
lastNameReadAt.clear();
|
|
4378
|
+
modelLastReadAt.clear();
|
|
4379
|
+
pushEvent({ hook_event_name: "__clear", cwd: "" }, "internal", { persist: false });
|
|
4380
|
+
return send(res, 200, {
|
|
4381
|
+
ok: true,
|
|
4382
|
+
log: !sharing.path ? "none" : sharing.mine ? "cleared" : "kept",
|
|
4383
|
+
path: sharing.path,
|
|
4384
|
+
decks: sharing.decks,
|
|
4385
|
+
mine: sharing.mine,
|
|
4386
|
+
owner: sharing.owner ? { port: sharing.owner.port } : null,
|
|
4387
|
+
});
|
|
2798
4388
|
}
|
|
2799
4389
|
|
|
2800
4390
|
function handleHealth(_req, res) {
|
|
@@ -2916,8 +4506,8 @@ export function requestUrl(rawUrl) {
|
|
|
2916
4506
|
// secrets are. GET /api/events is the whole ring buffer: prompt text, the Bash
|
|
2917
4507
|
// command lines the agent ran, the paths and contents it wrote, the contents of
|
|
2918
4508
|
// every file it read back. /api/claude-accounts names the accounts,
|
|
2919
|
-
// /api/claude-accounts/login carries a live OAuth authorize URL, /api/
|
|
2920
|
-
//
|
|
4509
|
+
// /api/claude-accounts/login carries a live OAuth authorize URL, /api/health the
|
|
4510
|
+
// absolute workspace path.
|
|
2921
4511
|
//
|
|
2922
4512
|
// So the Host check runs for every method now. What it asks is only the
|
|
2923
4513
|
// rebinding question — did this request arrive addressed to a name that can
|
|
@@ -3044,6 +4634,36 @@ function originMatchesHost(origin, host) {
|
|
|
3044
4634
|
// worst a caller does with it is draw a session on the canvas that is not
|
|
3045
4635
|
// there. See the handshake in hook/hook.js for the authentication that does
|
|
3046
4636
|
// run on this path, which is the deck proving itself to the hook.
|
|
4637
|
+
//
|
|
4638
|
+
// That "destroys nothing" is a claim about the whole ingest path and not only
|
|
4639
|
+
// about this line, and #625 is what it cost the day it stopped being true: the
|
|
4640
|
+
// ring buffer bounded the events it kept by count and not by size, so about 430
|
|
4641
|
+
// posts of a maximum-size body — from a local process holding no credential,
|
|
4642
|
+
// which is exactly what this set permits — reached the heap limit and aborted
|
|
4643
|
+
// the process. Whatever else is added to this set, the same question has to be
|
|
4644
|
+
// asked of it: what does an unbounded number of these accumulate in? For this
|
|
4645
|
+
// one the answer is now MAX_BUFFER_CHARS.
|
|
4646
|
+
//
|
|
4647
|
+
// #674 is the same question asked a second time, and the answer was not in the
|
|
4648
|
+
// ring at all. The body of this POST carries `transcript_path`, and everything
|
|
4649
|
+
// the deck learns about a session's model, cost, name and context is read out
|
|
4650
|
+
// of the file it names — so the credential-free route does not stop at the ring
|
|
4651
|
+
// buffer, it reaches the filesystem, with a path the caller chose. It was read
|
|
4652
|
+
// by allocating the whole file: 700 MB named in one POST took a fresh deck from
|
|
4653
|
+
// 52 MB RSS to 753 MB and answered `200 {"ok":true,"seq":1}`, and because a
|
|
4654
|
+
// file with no newline in it could never advance the scan cursor, every later
|
|
4655
|
+
// POST paid it again. What bounds it now is three things, in the order they
|
|
4656
|
+
// were reached for: `isClaudeTranscriptPath` means an unrecognised path is not
|
|
4657
|
+
// opened at all, so the caller who holds no credential can name nothing;
|
|
4658
|
+
// MAX_SCAN_CHUNK means no single read allocates more than 8 MiB whatever it is
|
|
4659
|
+
// pointed at; MAX_SCAN_BYTES_PER_PASS means no single POST walks more than 256
|
|
4660
|
+
// MiB of a file.
|
|
4661
|
+
//
|
|
4662
|
+
// So the claim above holds again, with its scope written out: the worst a
|
|
4663
|
+
// caller does with this route is draw a session on the canvas that is not
|
|
4664
|
+
// there. It cannot make the deck open a file of its choosing, and it cannot
|
|
4665
|
+
// make the deck's memory a function of anything but the two constants named
|
|
4666
|
+
// here.
|
|
3047
4667
|
const OPEN_MUTATIONS = new Set(["/api/event"]);
|
|
3048
4668
|
|
|
3049
4669
|
// Constant-time comparison of two secrets, and a length test that is not.
|
|
@@ -3299,7 +4919,11 @@ export async function startServer({ port = 4317, host = "127.0.0.1", persist = n
|
|
|
3299
4919
|
if (persist) {
|
|
3300
4920
|
persistPath = resolve(persist);
|
|
3301
4921
|
try { await mkdir(pdirname(persistPath), { recursive: true }); } catch {}
|
|
3302
|
-
|
|
4922
|
+
// `_workspace`, not `workspace`: the field has just been normalised on the
|
|
4923
|
+
// line above, and the replay has to answer the same question the live paths
|
|
4924
|
+
// answer with the same string. Passed rather than read off the module scope
|
|
4925
|
+
// so replayLog states what it depends on. See replayScope.
|
|
4926
|
+
const replayed = await replayLog(persistPath, _workspace);
|
|
3303
4927
|
if (replayed > 0) {
|
|
3304
4928
|
// Don't broadcast replays as live; SSE clients catch up via Last-Event-ID
|
|
3305
4929
|
// already. Just keep the buffer + seq counter primed.
|
|
@@ -3311,7 +4935,7 @@ export async function startServer({ port = 4317, host = "127.0.0.1", persist = n
|
|
|
3311
4935
|
// background quota poll hit a network error. Answer the request instead.
|
|
3312
4936
|
const guard = (p, res) => Promise.resolve(p).catch(err => sendInternalError(res, err));
|
|
3313
4937
|
|
|
3314
|
-
const
|
|
4938
|
+
const route = (req, res) => {
|
|
3315
4939
|
const url = requestUrl(req.url);
|
|
3316
4940
|
// Unparseable request target. Nothing below can route it, and throwing here
|
|
3317
4941
|
// would be an uncaughtException inside the listener — i.e. the whole deck.
|
|
@@ -3375,50 +4999,56 @@ export async function startServer({ port = 4317, host = "127.0.0.1", persist = n
|
|
|
3375
4999
|
if (req.method === "POST" && url.pathname === "/api/claude-accounts/switch") return guard(handleClaudeAccountSwitch(req, res), res);
|
|
3376
5000
|
if (req.method === "GET" && url.pathname === "/api/claude-accounts/login") return guard(handleAccountLoginState(req, res), res);
|
|
3377
5001
|
if (req.method === "POST" && url.pathname === "/api/claude-accounts/admin") return guard(handleClaudeAccountAdmin(req, res), res);
|
|
3378
|
-
if (req.method === "GET" && url.pathname === "/api/sound-hook") return guard(handleSoundHook(req, res), res);
|
|
3379
|
-
if (req.method === "POST" && url.pathname === "/api/sound-hook") return guard(handleSoundHookSet(req, res), res);
|
|
3380
5002
|
if (req.method === "GET" && url.pathname === "/api/cswap-auto") return guard(handleCswapAuto(req, res), res);
|
|
3381
5003
|
if (req.method === "POST" && url.pathname === "/api/cswap-auto") return guard(handleCswapAutoAction(req, res), res);
|
|
3382
5004
|
|
|
5005
|
+
// Through writeJsonArray rather than `send`, and through `guard` like every
|
|
5006
|
+
// route above it: this is the one answer whose size is the ring's size, and
|
|
5007
|
+
// `send` would build all of it as a single string. See writeJsonArray.
|
|
3383
5008
|
if (req.method === "GET" && url.pathname === "/api/events") {
|
|
3384
|
-
return
|
|
5009
|
+
return guard(writeJsonArray(res, eventsSince(url.searchParams.get("since") ?? 0)), res);
|
|
3385
5010
|
}
|
|
3386
5011
|
|
|
3387
|
-
//
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
|
|
3397
|
-
// server is sitting on the name.
|
|
3398
|
-
//
|
|
3399
|
-
// The root model survives without help because pushEvent stamps
|
|
3400
|
-
// `raw.model` on every payload; there is no equivalent stamp for the name
|
|
3401
|
-
// or for a subagent's model, which is why those two are listed and the
|
|
3402
|
-
// rest of the per-session state is not.
|
|
3403
|
-
//
|
|
3404
|
-
// The rule, for the next cache that gates an emit: anything answering
|
|
3405
|
-
// "has this changed" has to appear in BOTH places that mean the client no
|
|
3406
|
-
// longer has it — here, and in forgetSession.
|
|
3407
|
-
nameBySession.clear();
|
|
3408
|
-
modelBySession.clear();
|
|
3409
|
-
// The read stamps go with them. Clearing only the signatures would leave
|
|
3410
|
-
// the next hook event inside MODEL_READ_THROTTLE_MS, so the transcript
|
|
3411
|
-
// would not be re-read at all and the name would stay missing until the
|
|
3412
|
-
// throttle expired — a clear followed by a keystroke is exactly when a
|
|
3413
|
-
// user is watching.
|
|
3414
|
-
lastNameReadAt.clear();
|
|
3415
|
-
modelLastReadAt.clear();
|
|
3416
|
-
pushEvent({ hook_event_name: "__clear", cwd: "" }, "internal");
|
|
3417
|
-
return send(res, 200, { ok: true });
|
|
5012
|
+
// GET /api/clear — what a POST to this path would do, and to whose log.
|
|
5013
|
+
// Asked by the confirmation dialog as it opens, on demand rather than on a
|
|
5014
|
+
// timer, for the reason /api/system/processes is: the answer costs a
|
|
5015
|
+
// directory read, changes only when a deck starts or stops, and matters at
|
|
5016
|
+
// exactly one moment. See logSharing.
|
|
5017
|
+
if (req.method === "GET" && url.pathname === "/api/clear") {
|
|
5018
|
+
return guard(logSharing().then(s => send(res, 200, {
|
|
5019
|
+
ok: true, path: s.path, decks: s.decks, mine: s.mine,
|
|
5020
|
+
owner: s.owner ? { port: s.owner.port } : null,
|
|
5021
|
+
})), res);
|
|
3418
5022
|
}
|
|
3419
5023
|
|
|
5024
|
+
// POST /api/clear — wipe in-memory buffer + persistence file (UI reset)
|
|
5025
|
+
if (req.method === "POST" && url.pathname === "/api/clear") return guard(handleClear(res), res);
|
|
5026
|
+
|
|
3420
5027
|
if (req.method === "GET") return serveStatic(req, res, url);
|
|
3421
5028
|
send(res, 405, { error: "method not allowed" });
|
|
5029
|
+
};
|
|
5030
|
+
|
|
5031
|
+
// `guard` covers the routes dispatched as promises. This covers the rest of
|
|
5032
|
+
// them — and, more to the point, covers a route added later by someone who
|
|
5033
|
+
// did not think to reach for `guard` because their handler looked
|
|
5034
|
+
// synchronous and cheap. That was the whole of #626: `GET /api/events` read
|
|
5035
|
+
// as a one-liner, and it was a one-liner that ended the process, because a
|
|
5036
|
+
// synchronous throw inside the listener is an uncaughtException and Node's
|
|
5037
|
+
// answer to those is to exit. `/api/system` and `/api/clear` are called the
|
|
5038
|
+
// same bare way and are covered here for free.
|
|
5039
|
+
//
|
|
5040
|
+
// Deliberately the last line of defence and not the first: a 500 with the
|
|
5041
|
+
// detail on stderr is a worse answer than a handler that knew what went
|
|
5042
|
+
// wrong, and much better than a dead deck. sendInternalError already knows
|
|
5043
|
+
// to keep the detail off the wire and to do nothing but end the response
|
|
5044
|
+
// when the headers have gone out — which is the case that matters for a
|
|
5045
|
+
// streamed answer, where the throw can arrive after the status line.
|
|
5046
|
+
const server = createServer((req, res) => {
|
|
5047
|
+
try {
|
|
5048
|
+
route(req, res);
|
|
5049
|
+
} catch (err) {
|
|
5050
|
+
sendInternalError(res, err);
|
|
5051
|
+
}
|
|
3422
5052
|
});
|
|
3423
5053
|
|
|
3424
5054
|
// Try requested port first, then up to 10 random ports from portRange.
|