ccdeck 3.29.11 → 3.31.0

package/src/server/event-pipeline.mjs

@@ -0,0 +1,235 @@
+// pushEvent, the one door every event comes through — a hook's POST, a Codex
+// rollout line, the boot replay and the synthetic events the enrichment sends
+// back — and what it does to each on the way in: the deck's token taken out,
+// the model stamped, the ring, one serialization shared by the SSE fan-out and
+// the log, the desktop notification when no page is open, the away-update's
+// activity, and the transcript scans that answer through it.
+//
+// These lived in src/server/index.mjs, after the ring. The modules this one
+// imports reach pushEvent through event-sink.mjs, connected below as this
+// module loads; index.mjs's routes import it from here. The bodies are
+// unchanged.
+import { PRODUCT } from "./brand.mjs";
+import { createBlockNotifier } from "./block-notify.mjs";
+import { notify as osNotify } from "./browser-react.mjs";
+import { notificationsOn } from "./deck-prefs.mjs";
+// The settings as this process holds them — see prefs-state.mjs.
+import { heldPrefs } from "./prefs-state.mjs";
+// The deck's own token, taken back out of every event before the ring, the
+// SSE fan-out or the log can hold it — see token-redact.mjs.
+import { redactDeckToken } from "./token-redact.mjs";
+// The ring every event is numbered and held in — see event-ring.mjs.
+import { admitEvent } from "./event-ring.mjs";
+// The SSE subscribers every event is broadcast to — see sse-clients.mjs.
+import { notifyTrays, pageCount, sseClients, trayClients, writeSse } from "./sse-clients.mjs";
+// Where the log is, which sessions this deck writes to it, and when it rolls
+// over — see event-log.mjs.
+import { eventLogPath, maybeRotatePersistFile, writesLogFor } from "./event-log.mjs";
+import { appendLogLine } from "./log-writer.mjs";
+// The away-update's word on whether a turn is running — see lifecycle.mjs.
+import { activity } from "./lifecycle.mjs";
+// The session LRU and the transcript watch — see session-tracking.mjs.
+import { outputWatch, touchSession } from "./session-tracking.mjs";
+// What the deck learns about a session that its hooks never say, read off the
+// transcript and sent back through pushEvent — see session-enrichment.mjs.
+import { knownModelId, maybeResolveContext, maybeResolveModel, maybeResolveSessionName, maybeResolveUsage } from "./session-enrichment.mjs";
+// The Codex half: a session's rollout, found by id — see codex-enrichment.mjs.
+import { maybeResolveCodex } from "./codex-enrichment.mjs";
+// The gate pushEvent asks before a transcript path is followed — see
+// transcript-gate.mjs.
+import { isClaudeTranscriptPath, noteRefusedTranscript } from "./transcript-gate.mjs";
+// How the enrichment reaches pushEvent without importing this file — see
+// event-sink.mjs. Connected below, as this module loads.
+import { connectEventSink } from "./event-sink.mjs";
+
+// The modules that emit synthetic events send them here. pushEvent is a
+// function declaration, so it already exists as this line runs, and nothing
+// emits until a request or a timer startServer arms asks it to.
+connectEventSink(pushEvent);
+
+/**
+ * The desktop notifier, built once per process.
+ *
+ * `enabled` is read here and not per event, so a switch cannot change under a
+ * running server and let two events in the same second disagree about it. The
+ * OS call is browser-react.mjs's — the same one Browser Watch has shipped on
+ * all three platforms — so this is a second caller, not a second
+ * implementation, and the platform quirks it already handles (argv rather than
+ * interpolation on macOS, the WinRT toast on Windows, a missing notify-send on
+ * Linux) are handled once.
+ *
+ * A failure goes to stderr and no further. The user cannot act on "your desktop
+ * has no notification daemon" mid-session, the deck is not broken by it, and
+ * every in-page surface still says everything it said before.
+ */
+const blockNotifier = createBlockNotifier({
+  // The desktop app, while it is connected, raises the notification itself —
+  // under its own name, icon and permission, with a click that opens its
+  // window. Only when no app is listening does the OS helper speak.
+  notify: (title, body, meta) => (trayClients.size > 0 ? notifyTrays(title, body, meta) : osNotify(title, body)),
+  product: PRODUCT,
+  // A function, not a boolean: this is a switch a person flips from the sound
+  // menu while the deck is running, and a mute that waited for a restart would
+  // not be a mute. The env var still wins inside `notificationsOn`.
+  enabled: () => notificationsOn(heldPrefs.current()),
+  onError: err => console.error(`${PRODUCT}: could not raise a desktop notification:`, err?.message ?? err),
+});
+
+export function pushEvent(raw, source, opts = {}) {
+  // First, before anything below can see it: the deck's own credential does not
+  // belong in a store that is served without one. Every entry point to the
+  // buffer, the SSE fan-out and events.jsonl passes through here, so this is
+  // the single line that keeps it out of all three. See redactDeckToken.
+  raw = redactDeckToken(raw);
+
+  // Synchronous enrichment: if we already know this session's model, stamp
+  // it on the payload so the client's recursive scanner picks it up.
+  if (raw && typeof raw === "object" && raw.session_id && !raw.model) {
+    const modelId = knownModelId(raw.session_id);
+    if (modelId) raw.model = modelId;
+  }
+
+  // Numbered, charged, held, and the ring evicted behind it — see admitEvent.
+  const evt = admitEvent(raw, source, opts.receivedAt ?? Date.now());
+  const seq = evt.seq;
+
+  // Does this event reach the log at all? Not on a replay (it came from
+  // there), not when the hook told us another deck owns this session's log,
+  // and not when we have no log. Decided before serializing because it is half
+  // of the answer to whether serializing is worth doing.
+  const persisting = eventLogPath() && !opts.replay && opts.persist !== false && writesLogFor(raw);
+
+  // One serialization, shared by both consumers — and skipped entirely when
+  // neither wants it. This used to stringify the whole envelope twice on the
+  // hottest path in the process (once for the SSE frame, once for the persist
+  // line), and built the frame even with nobody subscribed: a headless deck
+  // paid a full stringify per event for a string no one read, and boot replay
+  // — which runs before the listener exists and never broadcasts — paid one
+  // for every line of a log that rotates at 50MB. An event this deck is not
+  // logging is still broadcast, so a subscriber alone is reason enough.
+  //
+  // Contained, because this line was fatal. `JSON.stringify` walks a value
+  // recursively while `JSON.parse` does not, and the gap between the two is
+  // enormous: measured on Node 22.14, parse accepts a body nested 4,194,303
+  // deep and stringify gives up on the result at 4,021. So a payload in that
+  // window parses cleanly and then throws `RangeError: Maximum call stack size
+  // exceeded` out of here — and there is no promise on this path for the
+  // route's `guard` to catch, because pushEvent is reached from inside a raw
+  // `req.on("end")` listener. It was an uncaughtException, and Node's answer to
+  // those is to exit. Measured against the real server: one POST of 24,378
+  // bytes, nested 4,050 deep, to the credential-free `/api/event` — a
+  // two-hundredth of the 5,000,000-character ingest cap — and the deck was
+  // gone, with nothing on the socket to tell the poster why.
+  //
+  // The payload leaves the ring, not just this string, and that is the point.
+  // admitEvent above already took the envelope, and a value nothing can
+  // serialize is a value no reader can ever deliver: every `Last-Event-ID`
+  // resume that replays it and every `GET /api/events` that writes it would
+  // meet the same throw for as long as it stayed in the buffer, so one small
+  // POST would poison both routes for the life of the entry. The envelope
+  // therefore keeps its `seq` and loses its payload — the same replacement
+  // envelopeJson makes for a reader, made once at the write instead of on every
+  // read, and the same bargain about `seq`: a caller paging with `?since=`
+  // walks past the hole rather than asking forever for what it cannot be given.
+  //
+  // Admitted as a stub rather than refused at ingest with a 400, deliberately.
+  // Three of pushEvent's four callers have no HTTP peer to answer — Codex
+  // rollout lines read off disk, the boot replay of events.jsonl, the synthetic
+  // events the transcript scanners emit — so the containment has to live here
+  // whatever the ingest route does, and a 400 on top would be a second
+  // mechanism for a case this one already covers. It would also have to be paid
+  // for: knowing a payload will not serialize means serializing it, which is
+  // the second stringify per event on the hottest path in the process that the
+  // paragraph above exists to have removed. The reason goes to stderr and not
+  // to the wire, under the rule sendInternalError explains.
+  let json = null;
+  if (sseClients.size > 0 || persisting) {
+    try {
+      json = JSON.stringify(evt);
+    } catch (err) {
+      console.error(`${PRODUCT}: event ${seq} could not be serialized:`, err);
+      evt.payload = null;
+      evt.unserializable = true;
+      json = JSON.stringify(evt);
+    }
+  }
+
+  if (sseClients.size > 0) {
+    const line = `id: ${seq}\nevent: hook\ndata: ${json}\n\n`;
+    // writeSse may drop a client mid-loop; deleting from a Set while iterating
+    // it is well defined and skips only the entry removed.
+    for (const res of sseClients) writeSse(res, line);
+  }
+
+  // The desktop, when there is no page to tell.
+  //
+  // Placed here rather than in a route handler because every path that can
+  // produce a permission prompt comes through pushEvent — the hook POST, a
+  // replay, and the transcript scanners — and `sseClients.size` read at this
+  // exact point is the honest answer to "is anybody being shown this by any
+  // other means". The web notifier owns the case where a page exists, and this
+  // owns the case where none does; the two never both fire, and neither has to
+  // know the other exists. block-notify.mjs holds the gates and the cooldown.
+  blockNotifier.consider(raw, { clients: pageCount(), replay: !!opts.replay });
+
+  // Whether a turn is running, for the away-update. Not from a replay: the log
+  // is history, and a turn it shows open is one that ended in another process.
+  if (!opts.replay) activity.note(raw, evt.receivedAt);
+
+  if (persisting) {
+    // Fire-and-forget append. JSONL = newline-delimited JSON, so the whole line
+    // has to reach the file as one write — this used to be `appendFile`, which
+    // splits anything over 512 KiB into separate appends and let another
+    // event land in the middle of a large tool response. See appendLogLine.
+    //
+    // Note this runs AFTER redactDeckToken above, as every path to the log
+    // does: the string being written is the one serialization of the event the
+    // SSE frame also used, and the token was taken out of the payload before
+    // either existed.
+    const line = json + "\n";
+    appendLogLine(eventLogPath(), line);
+    // Throttled check — every 30s, or every fifth of the threshold written,
+    // whichever comes first. The byte arm is what keeps the 50 MB cap from
+    // being advisory at any real ingest rate; see maybeRotatePersistFile.
+    maybeRotatePersistFile(Buffer.byteLength(line));
+  }
+
+  // Note the session so the caches the scanners below fill can expire by
+  // least-recent use. Replays are excluded: they fill nothing, and a boot
+  // replay of a log spanning weeks would otherwise churn the whole LRU through
+  // dead session ids before the first live event even arrives.
+  if (!opts.replay && raw && typeof raw === "object") touchSession(raw.session_id);
+
+  // Kick off async transcript scans. Model and usage are both re-read
+  // periodically (throttled to 2.5s per session) so the cost columns track
+  // running totals as the session progresses and late subagent models still
+  // land; ModelObserved is only emitted when the resolved set changes. Both
+  // result in synthetic events.
+  // Provider gates the path: Claude reads transcript_path; Codex reads its
+  // rollout JSONL under ~/.codex/sessions/. The Claude scanners short-circuit
+  // when transcript_path is absent (always the case for Codex hooks).
+  if (source === "hook" && !opts.replay) {
+    if (raw && raw.provider === "codex") {
+      maybeResolveCodex(raw);
+    } else if (!raw?.transcript_path || isClaudeTranscriptPath(raw.transcript_path)) {
+      // The gate is here, once, rather than repeated in the four scanners
+      // below it: this is the single door a caller-chosen path comes through,
+      // and all four read the same field off the same payload. A payload with
+      // no transcript_path at all still goes through — every one of them
+      // early-returns without it, and Codex hooks never send one — so the
+      // ordinary event costs nothing but the absent-field test.
+      maybeResolveModel(raw);
+      maybeResolveUsage(raw);
+      maybeResolveContext(raw);
+      maybeResolveSessionName(raw);
+      // Where the transcript IS, learned from the one place it is free. The
+      // four scanners above read it on this event; the watch reads it between
+      // events, which is the whole of what it adds.
+      if (raw?.transcript_path) outputWatch.note(raw.session_id, raw.transcript_path);
+    } else {
+      noteRefusedTranscript(raw.transcript_path);
+    }
+  }
+
+  return evt;
+}

package/src/server/event-ring.mjs

@@ -0,0 +1,214 @@
+// The event ring: every envelope this deck still holds, the seq numbering they
+// are given, and the eviction that keeps the ring inside its bounds.
+//
+// These lived in src/server/index.mjs, at the head of the ring buffer section
+// and inside pushEvent. The array and the two running totals beside it have to
+// move together, so they are private to this module now and reached only
+// through the operations below: admitEvent numbers, charges and holds one
+// envelope and evicts behind it, clearEventBuffer empties all three, and the
+// readers get a copy or an answer rather than the array. The bodies are
+// unchanged.
+import { ENVELOPE_CHARS, LAST_VALUE_WINS, MAX_BUFFER, MAX_BUFFER_CHARS, MAX_RING_ENTRIES, payloadChars } from "./ring-bounds.mjs";
+
+// ─── The event ring buffer ─────────────────────────────────────────────────
+// Its two bounds, MAX_BUFFER and MAX_BUFFER_CHARS, and the charge an event is
+// counted in against the second, payloadChars, are ring-bounds.mjs's; the
+// eviction in admitEvent keeps them.
+
+const events = [];                  // ring buffer
+// The running sum of what `events` holds, in the units payloadChars charges.
+// Kept beside the array rather than recomputed, because the alternative is
+// walking every buffered payload on every push. It and `events` have to move
+// together and nothing outside this module may touch either — see
+// clearEventBuffer for the one place that empties both, and what went wrong the
+// day only one of them was emptied.
+let bufferedChars = 0;
+
+/**
+ * How many of the ring's entries are HOOK events — what MAX_BUFFER budgets.
+ *
+ * Third of the trio that has to move with `events`, for the same reason
+ * `bufferedChars` does and with the same consequence if it drifts: a count that
+ * names events the array no longer holds evicts against a budget already spent.
+ * See LAST_VALUE_WINS for what it excludes and why, and clearEventBuffer for
+ * the one place all three are emptied.
+ */
+let bufferedHookEvents = 0;
+
+/** Set on the envelopes MAX_BUFFER does not count, so eviction can decrement
+ *  the right total without re-reading a payload it already charged. */
+const ENRICHMENT = Symbol("ring enrichment");
+
+// Where the charge rides. A Symbol key rather than an ordinary field, because
+// the envelope is JSON.stringify'd on the hot path into both the SSE frame and
+// the events.jsonl line, and JSON.stringify ignores symbol-keyed properties
+// entirely. So the number stays welded to the envelope it describes — which is
+// what makes it impossible for `bufferedChars` and `events` to drift apart —
+// without reaching the wire, the log, the client's HookEnvelope type, or the
+// 127-character envelope measurement MAX_CLIENT_BUFFER_BYTES is sized against.
+const CHARS = Symbol("ring charge");
+
+/**
+ * Empty the ring and the total measuring it, together.
+ *
+ * `/api/clear` used to be a bare `events.length = 0`, and with a running total
+ * beside the array that is a permanent debt: the total would still name events
+ * the array no longer holds, and every push after the first clear would evict
+ * against a budget already spent — a deck that answers one clear and then keeps
+ * a ring of one event for the rest of its life. The two variables move here and
+ * nowhere else.
+ */
+export function clearEventBuffer() {
+  events.length = 0;
+  bufferedChars = 0;
+  bufferedHookEvents = 0;
+}
+
+/**
+ * What the ring holds right now — its length, what it is charged, and the seq
+ * range it spans.
+ *
+ * Exported from index.mjs alongside MAX_BUFFER and MAX_BUFFER_CHARS so a test
+ * can watch the bound hold through a real server instead of watching a process
+ * die, which is the only other way this bound is observable. The seq range is
+ * here for the property eviction has to keep and nothing else checks: what
+ * leaves is always a PREFIX, so `newest - oldest + 1` equals the count. An
+ * eviction that ever took from the middle would leave a hole no resuming
+ * client could ask for again, and that is exactly what `GET /api/events` and
+ * the replay loop would then hand out without noticing.
+ */
+export function eventBufferStats() {
+  return {
+    events: events.length,
+    // What MAX_BUFFER budgets since #1032, and the only number a resuming
+    // client's window is actually measured in. `events` minus this is the
+    // enrichment riding along; the ratio between them is the amplification that
+    // used to spend the window — 2.6 measured at every session count tried.
+    hookEvents: bufferedHookEvents,
+    chars: bufferedChars,
+    oldestSeq: events.length > 0 ? events[0].seq : 0,
+    newestSeq: events.length > 0 ? events[events.length - 1].seq : 0,
+  };
+}
+
+let nextSeq = 1;
+// Identity of *this* process's seq numbering. nextSeq restarts at 1 on every
+// boot and is re-derived by replaying events.jsonl, so it is monotonic only
+// within one process: rotation (50MB) or /api/clear shortens that log and the
+// next boot starts numbering well below what an already-open tab saw. Stamped
+// on every envelope so a client can tell "counter restarted" apart from "old
+// duplicate" instead of silently dropping the live stream.
+export const SEQ_EPOCH = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+/** Envelopes newer than `seq` from the ring buffer, oldest first. */
+export function eventsSince(seq) {
+  const after = Number(seq) || 0;
+  return events.filter(e => e.seq > after);
+}
+
+/** The seq the newest event was given, 0 before the first — what
+ *  `/api/health` reports as `seq`. */
+export function lastSeq() {
+  return nextSeq - 1;
+}
+
+/** Every envelope the ring holds, oldest first, in an array the caller owns —
+ *  a snapshot, so a splice of the live ring's head cannot skip an entry in it. */
+export function ringSnapshot() {
+  return events.slice();
+}
+
+/** Whether the ring still holds an envelope newer than `seq` — false for an
+ *  empty ring, whatever `seq` is. */
+export function ringHoldsNewerThan(seq) {
+  return events.length > 0 && events[events.length - 1].seq > seq;
+}
+
+/**
+ * Number one payload, charge it, hold it, and evict behind it — the envelope
+ * pushEvent goes on to serialize, broadcast and log.
+ *
+ * `raw` is the payload as it will be stored: pushEvent has already taken the
+ * deck's token out of it and stamped the model, so what is measured here is
+ * what is kept.
+ */
+export function admitEvent(raw, source, receivedAt) {
+  const seq = nextSeq++;
+  const evt = {
+    seq,
+    epoch: SEQ_EPOCH,
+    receivedAt,
+    source,
+    payload: raw,
+    // Charged once, here, and carried on the envelope so eviction never has to
+    // walk a payload a second time. Symbol-keyed, so it is invisible to the two
+    // JSON.stringify calls in pushEvent and to the `{ ...e }` the replay loop
+    // makes. Charged AFTER redactDeckToken, like everything else pushEvent does
+    // to an event: the payload being measured is the one that will be stored.
+    [CHARS]: ENVELOPE_CHARS + payloadChars(raw),
+    // Decided once, here, beside the charge and for the same reason: eviction
+    // must never have to look at a payload again to know what it is giving
+    // back. See LAST_VALUE_WINS.
+    [ENRICHMENT]: raw != null && typeof raw === "object" && LAST_VALUE_WINS.has(raw.hook_event_name),
+  };
+  events.push(evt);
+  bufferedChars += evt[CHARS];
+  if (!evt[ENRICHMENT]) bufferedHookEvents++;
+
+  // Evict oldest-first until BOTH bounds hold — the count that has always been
+  // here, and the byte budget #625 added. See MAX_BUFFER_CHARS for the numbers.
+  //
+  // Counted first and spliced once, rather than shifting in a loop, because the
+  // two bounds evict at very different scales. The count bound drops exactly
+  // one entry per push; the byte bound can drop hundreds, since twenty-seven
+  // maximum-size events fill the whole budget on their own, and a shift per
+  // entry would memmove the array once for each of them.
+  //
+  // `drop < events.length - 1` is what keeps the event just pushed, whatever it
+  // weighs. A single event is allowed to be larger than the entire budget —
+  // ingest admits 5,000,000 characters, and a Codex rollout line read off disk
+  // has no length bound at all — and evicting it on arrival would leave
+  // pushEvent returning an envelope that `GET /api/events` never shows and no
+  // resuming client can ever be handed: a hole with no id to ask for it again,
+  // which is the exact failure resumeSse is written to avoid. So the true
+  // ceiling is MAX_BUFFER_CHARS plus one event, and that is stated here rather
+  // than pretended away.
+  //
+  // What this does to a resuming client is what the count bound has always done
+  // to one, only sooner: the head of the ring moves, and events that fell off
+  // it are gone for anybody who had not been sent them yet. That is the
+  // existing bargain for a too-old Last-Event-ID — handleSse replays whatever
+  // is still held and the client's `lastSeq` steps forward over the gap — and
+  // the byte bound deliberately reuses it rather than inventing a second answer.
+  // The difference worth knowing is that the head can now move in jumps rather
+  // than one entry at a time; resumeSse's per-pass snapshot is what makes that
+  // safe for a replay already in flight.
+  //
+  // THE COUNT BOUND IS HOOK EVENTS, NOT ENTRIES (#1032). It was entries, and
+  // the deck pushes about 1.6 derived events per hook event — so the window
+  // this ring IS shrank with the session count, to 9.5 measured seconds at 200
+  // sessions. Counting only what a user actually did restores a 2,000-event
+  // window at any number of sessions, and costs the array room for the
+  // enrichment interleaved among them, which MAX_RING_ENTRIES and
+  // MAX_BUFFER_CHARS both still bound.
+  //
+  // Enrichment falls off the head exactly as it always did — no entry is
+  // skipped, held back or reordered, and the ring stays contiguous in seq.
+  // What changed is only which entries the budget is spent on.
+  let drop = 0;
+  let freed = 0;
+  let freedHookEvents = 0;
+  while (drop < events.length - 1
+    && (bufferedHookEvents - freedHookEvents > MAX_BUFFER
+      || events.length - drop > MAX_RING_ENTRIES
+      || bufferedChars - freed > MAX_BUFFER_CHARS)) {
+    freed += events[drop][CHARS];
+    if (!events[drop][ENRICHMENT]) freedHookEvents++;
+    drop++;
+  }
+  if (drop > 0) {
+    events.splice(0, drop);
+    bufferedChars -= freed;
+    bufferedHookEvents -= freedHookEvents;
+  }
+  return evt;
+}