ai-lens 0.8.119 → 0.8.120

package/.commithash

@@ -1 +1 @@
-8674f71
+bf6f7d9

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 History of changes to the `ai-lens` CLI package on npm. New entries go on top. Format: `## X.Y.Z — YYYY-MM-DD`, followed by user-facing bullets.
 
+## 0.8.120 — 2026-07-03
+- fix: session events are no longer misattributed to an unrelated repository when a tool touches a file outside the project — project refinement now only follows genuine nested sub-repos, so the `projects` filter and per-project stats stay correct
+- feat(codex): Codex sessions now report per-call token usage, matching Claude Code / Cursor granularity
+- feat(codex): Codex sessions now capture per-message assistant text for dialogue and search
+
 ## 0.8.119 — 2026-07-02
 - feat: team-memory injection now fires only on genuinely relevant prompts — the trigger matches distinctive terms of the memory title instead of an absolute score, cutting noise ~20x while keeping real matches
 - feat: the full team memory pool is delivered to the session cache (previously most team-tier memories could never surface), server-gated to clients on this version and newer

package/client/capture.js

@@ -297,45 +297,20 @@ function maybeCleanStaleTranscriptOffsets() {
 }
 
 /**
- * Incrementally read only the NEW bytes appended to a Claude Code transcript
- * since the last time this function was called for that transcript, and
- * return ONE entry per real (non-synthetic) assistant API call in the delta.
+ * Read the NEW bytes appended to a JSONL transcript since the last call for
+ * this path, returning the complete newline-terminated lines in that delta.
+ * The intricate byte-offset / rotation / partial-line handling lives HERE, in
+ * one place, shared by the Claude Code (assistant-line) and Codex (token_count)
+ * token extractors so the two can't silently diverge.
  *
- * Each entry corresponds to a single Anthropic API call: a single assistant
- * line in the JSONL with its own usage record. A single user->agent turn can
- * produce many of these (tool-use loops), and the caller is expected to emit
- * one unified TokenUsage event per entry — symmetric with Cursor's per-call
- * AgentResponse rows and Codex's per-call TokenUsage rows.
- *
- * Returns { calls, commitOffset }:
- *   - `calls` is Array<{usage, model, timestamp, uuid}>:
- *     • `usage` has Anthropic-named keys (input_tokens, output_tokens,
- *       cache_read_input_tokens, cache_creation_input_tokens) for
- *       buildTokenUsageRaw.
- *     • `model` is the model from that specific assistant line.
- *     • `timestamp` is the assistant line's own ISO timestamp if present, else
- *       null (caller falls back to the Stop event's timestamp).
- *     • `uuid` is the assistant line's `uuid` field (Claude Code transcripts
- *       always include one). Used by the caller to derive a stable event_id
- *       so concurrent Stop hook invocations can't double-emit a single API call.
- *   - `commitOffset` is a no-arg function the caller MUST invoke ONLY after
- *     successfully writing every TokenUsage event derived from `calls` into
- *     the spool. If spool writes fail (or the caller never calls it), the
- *     transcript cursor stays put and the next Stop re-reads the same delta —
- *     the server-side ON CONFLICT on the deterministic event_id then dedups
- *     whichever rows did land. This is what makes the pipeline crash-safe:
- *     we never mark transcript bytes "consumed" until we know the rows they
- *     produced are durably queued.
- *
- * Returns an empty `calls` array (and a no-op `commitOffset`) if nothing new
- * in the delta or no real assistant lines are found.
- *
- * Partial-line handling: a Stop hook can fire while Claude Code is mid-write,
- * so the delta may end before a trailing '\n'. We detect this, ignore the
- * partial tail, and persist the offset at the byte right after the LAST
- * complete '\n' so the next Stop re-reads the partial line once it's finished.
+ * Returns { lines, commitOffset }:
+ *   - `lines` — raw JSONL lines in the processable region (empty on nothing-new
+ *     or any I/O failure).
+ *   - `commitOffset` — a no-arg fn the caller MUST invoke ONLY after it has
+ *     durably consumed every line (advancing the cursor eagerly would drop rows
+ *     if a later spool write fails).
  */
-function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
+function readNewTranscriptDelta(transcriptPath) {
   // Run the rate-limited stale-offset cleanup at most once per 24h. Wrapped in
   // try/catch so a cleanup failure never prevents the token read.
   try { maybeCleanStaleTranscriptOffsets(); } catch { /* best-effort */ }
@@ -345,7 +320,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
   const noopCommit = () => {};
 
   if (!transcriptPath || typeof transcriptPath !== 'string') {
-    return { calls: [], commitOffset: noopCommit };
+    return { lines: [], commitOffset: noopCommit };
   }
 
   const offsetFile = join(TRANSCRIPT_OFFSETS_DIR, encodeURIComponent(transcriptPath));
@@ -358,7 +333,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
     currentMtime = st.mtimeMs;
   } catch (err) {
     captureLog({ msg: 'transcript-offset-error', stage: 'stat', path: transcriptPath, error: err?.message });
-    return { calls: [], commitOffset: noopCommit };
+    return { lines: [], commitOffset: noopCommit };
   }
 
   const savedState = readTranscriptOffsetState(offsetFile);
@@ -377,7 +352,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
 
   // Nothing new — just a stat, super cheap.
   if (currentSize === startOffset) {
-    return { calls: [], commitOffset: noopCommit };
+    return { lines: [], commitOffset: noopCommit };
   }
 
   let buffer = null;
@@ -395,13 +370,13 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
     // Read failed before we got any bytes — leave the cursor unchanged so the
     // next Stop retries. (Old behavior advanced past the unread bytes; that
     // silently dropped data on transient I/O errors.)
-    return { calls: [], commitOffset: noopCommit };
+    return { lines: [], commitOffset: noopCommit };
   }
 
   if (!buffer || buffer.length === 0) {
     // Defensive: shouldn't happen because of the currentSize === startOffset
     // early return above. Don't advance the cursor.
-    return { calls: [], commitOffset: noopCommit };
+    return { lines: [], commitOffset: noopCommit };
   }
 
   // Find the LAST complete newline. CRITICAL: search the BYTE buffer, not a
@@ -416,7 +391,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
   if (lastNewlineIndex === -1) {
     // The entire chunk is a partial trailing line — leave the cursor unchanged
     // so the next Stop re-reads the line once it's finished.
-    return { calls: [], commitOffset: noopCommit };
+    return { lines: [], commitOffset: noopCommit };
   }
 
   // Process only the bytes up to (and including) the last '\n'. The new
@@ -428,12 +403,36 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
   const nextState = { offset: newOffset, size: currentSize, mtime_ms: currentMtime };
   const commitOffset = () => writeTranscriptOffsetState(offsetFile, nextState);
 
+  return { lines: processable.split('\n'), commitOffset };
+}
+
+/**
+ * Incrementally read only the NEW bytes appended to a Claude Code transcript
+ * since the last time this function was called for that transcript, and
+ * return ONE entry per real (non-synthetic) assistant API call in the delta.
+ *
+ * Each entry corresponds to a single Anthropic API call: a single assistant
+ * line in the JSONL with its own usage record. A single user->agent turn can
+ * produce many of these (tool-use loops), and the caller is expected to emit
+ * one unified TokenUsage event per entry — symmetric with Cursor's per-call
+ * AgentResponse rows and Codex's per-call TokenUsage rows.
+ *
+ * Returns { calls, commitOffset } (see readNewTranscriptDelta for the
+ * commit-offset crash-safety contract): `calls` is Array<{usage, model,
+ * timestamp, uuid}> with Anthropic-named usage keys for buildTokenUsageRaw,
+ * the model + ISO timestamp from that specific assistant line, and the line's
+ * `uuid` (used by the caller to derive a stable event_id so concurrent Stop
+ * hook invocations can't double-emit a single API call).
+ */
+function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
+  const { lines, commitOffset } = readNewTranscriptDelta(transcriptPath);
+  if (!lines.length) return { calls: [], commitOffset };
+
   // Iterate lines in the processable region. Defensive parse-tolerance is
   // still useful for the rare case where a single line in the middle is
   // malformed (we skip it rather than dropping the whole delta).
   const calls = [];
   try {
-    const lines = processable.split('\n');
     for (const rawLine of lines) {
       const line = rawLine && rawLine.trim();
       if (!line) continue;
@@ -493,7 +492,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
     // to spool in this path, so it's safe to commit immediately (no risk of
     // losing events).
     commitOffset();
-    return { calls: [], commitOffset: noopCommit };
+    return { calls: [], commitOffset: () => {} };
   }
 
   // IMPORTANT: do NOT persist the offset here. The caller is responsible for
@@ -504,6 +503,173 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
   return { calls, commitOffset };
 }
 
+/**
+ * Codex analogue of extractNewClaudeApiCallsFromTranscript: read the NEW bytes
+ * appended to a Codex rollout transcript (`rollout-*.jsonl`) since the last
+ * call and, in a SINGLE delta pass, return both the per-call token usage
+ * (`calls`) and the assistant's dialogue text (`messages`).
+ *
+ * A single pass is mandatory: `readNewTranscriptDelta` advances a per-path byte
+ * cursor that is committed only after the caller spools everything, so a second
+ * independent read would re-consume the same bytes. Claude's extractor folds
+ * text into each `call` (usage + content share one assistant line); Codex keeps
+ * them separate because usage and text live in DIFFERENT interleaved records.
+ *
+ * TOKEN USAGE (`calls`) — Codex records per-call usage as `event_msg`/
+ * `token_count` records whose `payload.info.last_token_usage` is the DELTA for
+ * that single call (the sibling `total_token_usage` is the running cumulative
+ * sum). The rollout is OpenAI-shaped, so field names differ from Claude's and
+ * we normalize them to the Anthropic convention that buildTokenUsageRaw + the
+ * dashboard SQL expect:
+ *   - Codex `cached_input_tokens` ⊆ `input_tokens` (a subset), so we split them
+ *     into non-overlapping `input_tokens` (fresh) + `cache_read_input_tokens`
+ *     (cached), matching Claude where `input_tokens` excludes the cache read.
+ *   - Codex has no prompt-cache-creation notion → `cache_creation_input_tokens`
+ *     is always 0.
+ *   - `output_tokens` already includes `reasoning_output_tokens`, mirroring
+ *     Claude's output convention, so it passes through unchanged.
+ * Records whose call delta is all-zero (input+output == 0, e.g. the residual
+ * token_count emitted around a compaction) are skipped so we never spool an
+ * empty TokenUsage row.
+ *
+ * ASSISTANT TEXT (`messages`, ANL-1239) — the assistant's visible dialogue
+ * lives in `response_item`/`message` records with `role === 'assistant'`; we
+ * concatenate their `content[]` `output_text` blocks. The dedup key is
+ * `${timestamp}:${payload.id | sha(text)}`: current Codex writes a durable
+ * `msg_…` id on every assistant message (hard version cutoff ~2026-06-26; older
+ * rollouts have none), so we prefer it and fall back to a text fingerprint —
+ * mirroring how the token path synthesizes a stable key from records that lack
+ * a per-record uuid. The
+ * caller emits these as AssistantText events, metric-neutral server-side
+ * (ASSISTANT_CONTENT_TYPES). NOTE: there is deliberately no AssistantThinking
+ * analogue (unlike Claude). Current Codex emits no plaintext reasoning: the
+ * full chain-of-thought lives only in `response_item`/`reasoning`
+ * `encrypted_content`, and the short plaintext `summary` those records once
+ * carried was dropped ~2026-04 (verified on real rollouts: 91% of Feb reasoning
+ * records had a summary, 0% from April on). Since AI Lens captures live/current
+ * Codex, no recoverable thinking text exists.
+ *
+ * The model isn't on the token_count / message record; it lives on the nearest
+ * preceding `turn_context` record. We track the last-seen turn_context model
+ * within the delta and fall back to `fallbackModel` (the Stop event's model)
+ * when the governing turn_context landed in an earlier delta.
+ *
+ * @param {string} transcriptPath  Path to the Codex rollout JSONL.
+ * @param {string|null} fallbackModel  Model from the Stop event, used when no
+ *                                     turn_context precedes a record in this
+ *                                     delta.
+ * @returns {{calls: Array<{usage, model, timestamp, uuid}>,
+ *            messages: Array<{text, model, timestamp, uuid}>,
+ *            commitOffset: function}}
+ */
+function extractNewCodexRecordsFromTranscript(transcriptPath, fallbackModel) {
+  const { lines, commitOffset } = readNewTranscriptDelta(transcriptPath);
+  if (!lines.length) return { calls: [], messages: [], commitOffset };
+
+  const calls = [];
+  const messages = [];
+  let currentModel = fallbackModel || null;
+  try {
+    for (const rawLine of lines) {
+      const line = rawLine && rawLine.trim();
+      if (!line) continue;
+      let parsed;
+      try {
+        parsed = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      // A turn_context record carries the model for the records that follow it.
+      if (parsed?.type === 'turn_context' && typeof parsed.payload?.model === 'string' && parsed.payload.model) {
+        currentModel = parsed.payload.model;
+        continue;
+      }
+      const payload = parsed?.payload;
+      if (!payload) continue;
+      const ts = typeof parsed.timestamp === 'string' ? parsed.timestamp : null;
+
+      // Assistant dialogue text — response_item/message with role 'assistant'.
+      if (parsed.type === 'response_item' && payload.type === 'message' && payload.role === 'assistant') {
+        let text = '';
+        if (Array.isArray(payload.content)) {
+          for (const block of payload.content) {
+            if (!block || typeof block !== 'object') continue;
+            // Assistant content is 'output_text'; accept 'text' too, defensively.
+            if ((block.type === 'output_text' || block.type === 'text') && typeof block.text === 'string' && block.text) {
+              text += (text ? '\n' : '') + block.text;
+            }
+          }
+        }
+        if (text) {
+          // Stable per-message dedup key: `${timestamp}:${id | sha(text)}`.
+          // Current Codex writes a durable `msg_…` id on every assistant
+          // message (verified: 100% from the 2026-06-26 build onward; older
+          // rollouts, ≤ mid-May in local data, carry none — a hard version
+          // cutoff, not a gradual mix). Live capture only ever sees current
+          // rollouts, so the id is normally present; the text-hash fallback is
+          // defensive insurance for a pre-cutoff / downgraded client. Either
+          // way the key is unique per record and identical across two Stop
+          // hooks reading the same bytes (main() re-hashes source_uuid into the
+          // event_id). Prefer the durable id so the key stays human-traceable.
+          const idPart = typeof payload.id === 'string' && payload.id
+            ? payload.id
+            : createHash('sha256').update(text).digest('hex').slice(0, 16);
+          const uuid = `${ts || ''}:${idPart}`;
+          messages.push({
+            text,
+            model: currentModel,
+            timestamp: ts,
+            uuid,
+          });
+        }
+        continue;
+      }
+
+      if (payload.type !== 'token_count') continue;
+      const info = payload.info;
+      const last = info?.last_token_usage;
+      if (!last || typeof last !== 'object') continue;
+
+      const rawInput = toNumberOrNull(last.input_tokens) ?? 0;
+      const cached = toNumberOrNull(last.cached_input_tokens) ?? 0;
+      const output = toNumberOrNull(last.output_tokens) ?? 0;
+      // Skip all-zero deltas (residual token_count around a compaction).
+      if (rawInput + output <= 0) continue;
+
+      // cached_input_tokens is a SUBSET of input_tokens — split into
+      // non-overlapping fresh-input + cache-read to match Claude's convention.
+      const cacheRead = Math.min(cached, rawInput);
+      const freshInput = Math.max(0, rawInput - cacheRead);
+
+      // Stable per-call fingerprint for the dedup event_id. The token_count
+      // record has no per-record uuid, so we key on the record timestamp + the
+      // cumulative total (monotonic across calls) — unique per token_count
+      // record and identical across two concurrent Stop hooks reading the same
+      // bytes.
+      const cum = toNumberOrNull(info?.total_token_usage?.total_tokens);
+      const uuid = `${ts || ''}:${cum != null ? cum : ''}`;
+
+      calls.push({
+        usage: {
+          input_tokens: freshInput,
+          output_tokens: output,
+          cache_read_input_tokens: cacheRead,
+          cache_creation_input_tokens: 0,
+        },
+        model: currentModel,
+        timestamp: ts,
+        uuid: uuid === ':' ? null : uuid,
+      });
+    }
+  } catch (err) {
+    captureLog({ msg: 'transcript-offset-error', stage: 'parse-codex', path: transcriptPath, error: err?.message });
+    commitOffset();
+    return { calls: [], messages: [], commitOffset: () => {} };
+  }
+
+  return { calls, messages, commitOffset };
+}
+
 // =============================================================================
 // Source Detection
 // =============================================================================
@@ -687,7 +853,14 @@ function refineProjectPath(event, projectPath, sessionId) {
   const filePath = extractFilePath(toolInput);
   if (!filePath) return projectPath;
   const gitRoot = findGitRoot(filePath);
-  if (gitRoot && (!projectPath || gitRoot.length > projectPath.length)) {
+  // Refine ONLY downward — into the current project itself or a real sub-repo
+  // of it (nested repos like etl-pipelines/bk-reports-automation/). Never flip
+  // to an unrelated repo just because its path string happens to be longer:
+  // a session running in one project that touches a file in a sibling/foreign
+  // repo would otherwise get misattributed there (and mis-filtered against the
+  // `projects` config). `pathContains` handles the nesting check plus Windows
+  // backslash/drive-letter/case normalization for free.
+  if (gitRoot && (!projectPath || pathContains(projectPath, gitRoot))) {
     if (sessionId) cacheSessionPath(sessionId, gitRoot);
     return gitRoot;
   }
@@ -1369,7 +1542,7 @@ function normalizeCodex(event) {
     projectPath = refineProjectPath({ tool_input: codexToolInput(event), input: codexToolInput(event) }, projectPath, sessionId);
   }
 
-  return [{
+  const primary = {
     event_id: null,
     source: 'codex',
     session_id: sessionId,
@@ -1378,7 +1551,59 @@ function normalizeCodex(event) {
     timestamp,
     data,
     raw: event,
-  }];
+  };
+
+  // On Stop (end of a Codex agent turn), read the rollout delta once and emit
+  // one TokenUsage event per real API call PLUS one AssistantText event per
+  // assistant message (ANL-1239) — giving Codex the same row-per-call
+  // granularity as Claude Code and Cursor. The rollout path arrives as
+  // `transcript_path` (the same field detectSource keys on for `rollout-*`).
+  if (hookType === 'Stop') {
+    const transcriptPath = event.transcript_path || null;
+    const fallbackModel = typeof event.model === 'string' ? event.model : null;
+    const { calls, messages, commitOffset } = extractNewCodexRecordsFromTranscript(transcriptPath, fallbackModel);
+    const tokenEvents = calls.map(call => ({
+      event_id: null, // assigned in main() from a stable hash of call.uuid
+      source: 'codex',
+      session_id: sessionId,
+      type: 'TokenUsage',
+      project_path: projectPath,
+      timestamp: call.timestamp || timestamp,
+      data: {
+        model: call.model,
+        input_tokens: call.usage.input_tokens,
+        output_tokens: call.usage.output_tokens,
+      },
+      raw: buildTokenUsageRaw({ source_uuid: call.uuid }, call.usage, call.model),
+    }));
+    // Assistant dialogue text (ANL-1239). Stored for dialogue/search;
+    // metric-neutral server-side (ASSISTANT_CONTENT_TYPES). data.text is
+    // truncated for display; raw.text keeps the full text (redacted
+    // server-side). Codex has NO plaintext thinking, so there is no
+    // AssistantThinking counterpart (unlike the Claude Code Stop path).
+    const contentEvents = messages.map(msg => ({
+      event_id: null, // assigned in main() from a stable hash of msg.uuid
+      source: 'codex',
+      session_id: sessionId,
+      type: 'AssistantText',
+      project_path: projectPath,
+      timestamp: msg.timestamp || timestamp,
+      data: { text: truncate(msg.text, TRUNCATION_LIMITS.agentResponse), model: msg.model },
+      raw: { source_uuid: msg.uuid, model: msg.model, text: msg.text },
+    }));
+    const result = [primary, ...tokenEvents, ...contentEvents];
+    // Attach the commit callback as a non-enumerable property so it survives
+    // through normalizeEvent()/writeToSpool without leaking into iteration or
+    // length assertions — mirrors the Claude Code Stop path.
+    Object.defineProperty(result, 'commitTranscriptOffset', {
+      value: commitOffset,
+      enumerable: false,
+      writable: false,
+    });
+    return result;
+  }
+
+  return [primary];
 }
 
 // =============================================================================
@@ -2142,7 +2367,7 @@ async function main() {
       : ev.type === 'AssistantThinking' ? 'assistantthinking'
         : 'tokenusage';
     if (sourceUuid) {
-      ev.event_id = deterministicEventId(`claude_code:${kind}:${sourceUuid}`);
+      ev.event_id = deterministicEventId(`${ev.source}:${kind}:${sourceUuid}`);
     } else {
       // Fallback: stdin hash + per-event index. Should never be needed because
       // Claude Code transcripts always include uuid per record.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-lens",
-  "version": "0.8.119",
+  "version": "0.8.120",
   "type": "module",
   "description": "Centralized session analytics for AI coding tools",
   "bin": {