ai-lens 0.8.119 → 0.8.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.commithash +1 -1
- package/CHANGELOG.md +8 -0
- package/README.md +3 -1
- package/cli/hooks.js +22 -3
- package/cli/status.js +54 -7
- package/client/capture.js +316 -51
- package/client/redact.js +4 -0
- package/package.json +1 -1
package/.commithash
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
9775c12
|
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
History of changes to the `ai-lens` CLI package on npm. New entries go on top. Format: `## X.Y.Z — YYYY-MM-DD`, followed by user-facing bullets.
|
|
4
4
|
|
|
5
|
+
## 0.8.121 — 2026-07-21
|
|
6
|
+
- fix(codex): Codex token usage no longer double-counts — some Codex versions re-emit an identical usage record without advancing their own running counter, and those duplicates were inflating token totals; they are now dropped (including duplicates that land in a later session update)
|
|
7
|
+
|
|
8
|
+
## 0.8.120 — 2026-07-03
|
|
9
|
+
- fix: session events are no longer misattributed to an unrelated repository when a tool touches a file outside the project — project refinement now only follows genuine nested sub-repos, so the `projects` filter and per-project stats stay correct
|
|
10
|
+
- feat(codex): Codex sessions now report per-call token usage, matching Claude Code / Cursor granularity
|
|
11
|
+
- feat(codex): Codex sessions now capture per-message assistant text for dialogue and search
|
|
12
|
+
|
|
5
13
|
## 0.8.119 — 2026-07-02
|
|
6
14
|
- feat: team-memory injection now fires only on genuinely relevant prompts — the trigger matches distinctive terms of the memory title instead of an absolute score, cutting noise ~20x while keeping real matches
|
|
7
15
|
- feat: the full team memory pool is delivered to the session cache (previously most team-tier memories could never surface), server-gated to clients on this version and newer
|
package/README.md
CHANGED
|
@@ -345,13 +345,15 @@ Stored in `~/.ai-lens/`:
|
|
|
345
345
|
## Development
|
|
346
346
|
|
|
347
347
|
```bash
|
|
348
|
-
npm test # Run all tests (vitest
|
|
348
|
+
npm test # Run all tests (vitest)
|
|
349
349
|
npm run test:watch # Watch mode
|
|
350
350
|
npm run dev:dashboard # Dashboard dev server
|
|
351
351
|
```
|
|
352
352
|
|
|
353
353
|
Tests require PostgreSQL — set `DATABASE_URL` or use `docker compose up postgres -d` (test DB `ailens_test` is created automatically).
|
|
354
354
|
|
|
355
|
+
Test files run in parallel, each worker against its own database: `test/global-setup.js` migrates a template once and clones it into `ailens_test_w1..wN`. Set `AI_LENS_TEST_WORKERS` to change the worker count (default 6).
|
|
356
|
+
|
|
355
357
|
Architecture decisions are recorded under [`docs/adr/`](docs/adr/) — e.g. [ADR 0001 — Canonical metric predicates](docs/adr/0001-canonical-metric-predicates.md).
|
|
356
358
|
|
|
357
359
|
## Deployment
|
package/cli/hooks.js
CHANGED
|
@@ -733,6 +733,27 @@ export function getClaudeCodeToolConfig(projectRoot, label = 'Claude Code (proje
|
|
|
733
733
|
};
|
|
734
734
|
}
|
|
735
735
|
|
|
736
|
+
/**
|
|
737
|
+
* The per-machine Claude Code overlay as a tool config: same tool, but pointed at
|
|
738
|
+
* `<project>/.claude/settings.local.json`.
|
|
739
|
+
*
|
|
740
|
+
* This file is where `init` ACTUALLY writes project hooks (cli/init.js — gitignored,
|
|
741
|
+
* per-machine, HELP-1539), so anything that inspects "the project's Claude hooks" and
|
|
742
|
+
* reads only `settings.json` is looking at the wrong file. It has its own label
|
|
743
|
+
* because consumers key on `tool.name`: `status`'s capture test emits one `[<name>]`
|
|
744
|
+
* block per tested tool and the server parses those blocks into a Map — two entries
|
|
745
|
+
* sharing a name would silently overwrite each other.
|
|
746
|
+
*
|
|
747
|
+
* `Claude Code (project local overlay)` is the canonical string (already used by
|
|
748
|
+
* `cli/remove.js` and already mapped to the `claude_code` source in
|
|
749
|
+
* `server/models/client-reports.js`). Do not invent a variant.
|
|
750
|
+
*/
|
|
751
|
+
export function getClaudeCodeLocalToolConfig(projectRoot, label = 'Claude Code (project local overlay)', ctx = null) {
|
|
752
|
+
const base = getClaudeCodeToolConfig(projectRoot, label, ctx);
|
|
753
|
+
if (!base) return null;
|
|
754
|
+
return { ...base, configPath: claudeLocalSettingsPath(base), sharedConfig: true };
|
|
755
|
+
}
|
|
756
|
+
|
|
736
757
|
export function getCodexToolConfig(projectRoot, label = 'Codex (project)', ctx = null) {
|
|
737
758
|
const base = TOOL_CONFIGS.find(t => t.name === 'Codex');
|
|
738
759
|
if (!base) return null;
|
|
@@ -1226,9 +1247,7 @@ export function detectAiLensInstallScope(projectRoot) {
|
|
|
1226
1247
|
// (per-machine, gitignored); the committed `--use-repo-path` form lands in
|
|
1227
1248
|
// settings.json — check BOTH so neither install style is missed.
|
|
1228
1249
|
const claudeProject = getClaudeCodeToolConfig(projectRoot);
|
|
1229
|
-
const claudeProjectLocal =
|
|
1230
|
-
? { ...claudeProject, configPath: claudeLocalSettingsPath(claudeProject) }
|
|
1231
|
-
: null;
|
|
1250
|
+
const claudeProjectLocal = getClaudeCodeLocalToolConfig(projectRoot);
|
|
1232
1251
|
const projectTools = [
|
|
1233
1252
|
claudeProject,
|
|
1234
1253
|
claudeProjectLocal,
|
package/cli/status.js
CHANGED
|
@@ -7,7 +7,7 @@ import tls from 'node:tls';
|
|
|
7
7
|
|
|
8
8
|
import { TLS_TRUST_CODES, tlsCodeOf, tlsVerdictSummary, issuerName } from '../client/tls-trust.js';
|
|
9
9
|
|
|
10
|
-
import { getVersionInfo, readLensConfig, detectInstalledTools, getCursorToolConfig, getClaudeCodeToolConfig, getCodexToolConfig, analyzeToolHooks, checkHooksDisabled, verifyCodexHookTrust, CAPTURE_PATH, TOOL_CONFIGS, isClaudeProjectDirCommand, analyzeClaudeLocalOverlay, extractProjectDirRelPath, globalClaudeHooksActive, CONHOST_HEADLESS_PREFIX_RE } from './hooks.js';
|
|
10
|
+
import { getVersionInfo, readLensConfig, detectInstalledTools, getCursorToolConfig, getClaudeCodeToolConfig, getCodexToolConfig, getClaudeCodeLocalToolConfig, analyzeToolHooks, checkHooksDisabled, verifyCodexHookTrust, CAPTURE_PATH, TOOL_CONFIGS, isClaudeProjectDirCommand, isWrongPlatformProjectDirCommand, analyzeClaudeLocalOverlay, extractProjectDirRelPath, globalClaudeHooksActive, CONHOST_HEADLESS_PREFIX_RE } from './hooks.js';
|
|
11
11
|
import { DATA_DIR, PENDING_DIR, SENDING_DIR, SESSION_PATHS_DIR, LOG_PATH, CAPTURE_LOG_PATH, LAST_STATUS_REPORT_PATH, getGitIdentity, getMonitoredProjects } from '../client/config.js';
|
|
12
12
|
import { recoverMojibake } from '../client/mojibake-fix.js';
|
|
13
13
|
import { isProjectMonitored } from '../client/capture.js';
|
|
@@ -234,17 +234,49 @@ function validateHookCommandPaths(tool) {
|
|
|
234
234
|
}
|
|
235
235
|
|
|
236
236
|
/**
|
|
237
|
-
* Global tools (detectInstalledTools) plus project-level tools
|
|
238
|
-
*
|
|
237
|
+
* Global tools (detectInstalledTools) plus the project-level tools found under
|
|
238
|
+
* `projectRoot`. Status is assumed to be run from the project root where hooks live.
|
|
239
|
+
*
|
|
240
|
+
* Claude Code contributes up to TWO entries, and both halves matter:
|
|
241
|
+
*
|
|
242
|
+
* - `settings.local.json` (the OVERLAY) is where `init` actually writes project
|
|
243
|
+
* hooks — per-machine, gitignored (HELP-1539). Reading only `settings.json` is
|
|
244
|
+
* how this check used to report `no hook command found` for a perfectly healthy
|
|
245
|
+
* install, while the per-tool line in the SAME report said the overlay was fine.
|
|
246
|
+
*
|
|
247
|
+
* - the committed `settings.json` is tested only when its hooks can actually fire
|
|
248
|
+
* HERE. When `analyzeClaudeLocalOverlay` says an overlay is `applicable`, that
|
|
249
|
+
* means by definition "this file carries the other OS's syntax and never fires on
|
|
250
|
+
* this machine" — running such a command through the shell yields exit 127 and
|
|
251
|
+
* would report `capture failed` for someone whose capture works via the overlay.
|
|
252
|
+
* Testing a command the OS cannot execute proves nothing; skipping it makes this
|
|
253
|
+
* check agree with the per-tool line instead of contradicting it.
|
|
254
|
+
*
|
|
255
|
+
* @param {string} [projectRoot] — defaults to cwd; a parameter so this is testable
|
|
256
|
+
* without chdir (it had no coverage precisely because it took none).
|
|
257
|
+
* @param {{platform?: string, tracked?: boolean}} [opts] — overlay-analysis
|
|
258
|
+
* overrides, forwarded to analyzeClaudeLocalOverlay (tests only).
|
|
239
259
|
*/
|
|
240
|
-
function getToolsForCaptureTest() {
|
|
260
|
+
export function getToolsForCaptureTest(projectRoot = process.cwd(), opts = {}) {
|
|
241
261
|
const global = detectInstalledTools();
|
|
242
|
-
const projectRoot = process.cwd();
|
|
243
262
|
const projectTools = [];
|
|
244
263
|
const cursorProject = getCursorToolConfig(projectRoot);
|
|
245
264
|
if (cursorProject && existsSync(cursorProject.configPath)) projectTools.push(cursorProject);
|
|
265
|
+
|
|
246
266
|
const claudeProject = getClaudeCodeToolConfig(projectRoot);
|
|
247
|
-
if (claudeProject && existsSync(claudeProject.configPath))
|
|
267
|
+
if (claudeProject && existsSync(claudeProject.configPath)) {
|
|
268
|
+
const overlay = analyzeClaudeLocalOverlay(claudeProject, opts);
|
|
269
|
+
// `applicable` ⇔ wrong-platform syntax AND git-tracked. The untracked variant is
|
|
270
|
+
// just as unrunnable, so check the command form directly too.
|
|
271
|
+
const wrongForm = isWrongPlatformProjectDirCommand(
|
|
272
|
+
extractHookCommand(claudeProject) ?? '', opts.platform ?? process.platform,
|
|
273
|
+
);
|
|
274
|
+
if (!overlay.applicable && !wrongForm) projectTools.push(claudeProject);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const claudeLocal = getClaudeCodeLocalToolConfig(projectRoot);
|
|
278
|
+
if (claudeLocal && existsSync(claudeLocal.configPath)) projectTools.push(claudeLocal);
|
|
279
|
+
|
|
248
280
|
const codexProject = getCodexToolConfig(projectRoot);
|
|
249
281
|
if (codexProject && existsSync(codexProject.configPath)) projectTools.push(codexProject);
|
|
250
282
|
return [...global, ...projectTools];
|
|
@@ -260,7 +292,22 @@ function checkCaptureRun(installedTools) {
|
|
|
260
292
|
}
|
|
261
293
|
|
|
262
294
|
if (toolCommands.length === 0) {
|
|
263
|
-
|
|
295
|
+
// Say WHERE we looked. Project hooks are discovered relative to cwd, so running
|
|
296
|
+
// status from outside the project produces this result on a perfectly healthy
|
|
297
|
+
// install — and the bare old wording ("no hook command found") read as "capture is
|
|
298
|
+
// dead". Three developers currently sit in exactly this state while their own
|
|
299
|
+
// report shows capture happening seconds earlier.
|
|
300
|
+
const looked = installedTools.map(t => t.configPath);
|
|
301
|
+
return {
|
|
302
|
+
ok: null,
|
|
303
|
+
summary: looked.length
|
|
304
|
+
? `no hook command found in the configs checked here (${looked.length}) — project hooks may live in another directory`
|
|
305
|
+
: 'no tool configs found to check',
|
|
306
|
+
detail: 'Could not find an AI Lens hook command in any tool config.\n'
|
|
307
|
+
+ `Checked:\n${looked.map(p => ` ${p}`).join('\n') || ' (none)'}\n\n`
|
|
308
|
+
+ 'Project hooks are looked up relative to the current directory. If your hooks are\n'
|
|
309
|
+
+ 'installed per-project, re-run `ai-lens status` from that project root.',
|
|
310
|
+
};
|
|
264
311
|
}
|
|
265
312
|
|
|
266
313
|
const defaultCaptureScript = join(homedir(), '.ai-lens', 'client', 'capture.js');
|
package/client/capture.js
CHANGED
|
@@ -297,45 +297,20 @@ function maybeCleanStaleTranscriptOffsets() {
|
|
|
297
297
|
}
|
|
298
298
|
|
|
299
299
|
/**
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
300
|
+
* Read the NEW bytes appended to a JSONL transcript since the last call for
|
|
301
|
+
* this path, returning the complete newline-terminated lines in that delta.
|
|
302
|
+
* The intricate byte-offset / rotation / partial-line handling lives HERE, in
|
|
303
|
+
* one place, shared by the Claude Code (assistant-line) and Codex (token_count)
|
|
304
|
+
* token extractors so the two can't silently diverge.
|
|
303
305
|
*
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
* Returns { calls, commitOffset }:
|
|
311
|
-
* - `calls` is Array<{usage, model, timestamp, uuid}>:
|
|
312
|
-
* • `usage` has Anthropic-named keys (input_tokens, output_tokens,
|
|
313
|
-
* cache_read_input_tokens, cache_creation_input_tokens) for
|
|
314
|
-
* buildTokenUsageRaw.
|
|
315
|
-
* • `model` is the model from that specific assistant line.
|
|
316
|
-
* • `timestamp` is the assistant line's own ISO timestamp if present, else
|
|
317
|
-
* null (caller falls back to the Stop event's timestamp).
|
|
318
|
-
* • `uuid` is the assistant line's `uuid` field (Claude Code transcripts
|
|
319
|
-
* always include one). Used by the caller to derive a stable event_id
|
|
320
|
-
* so concurrent Stop hook invocations can't double-emit a single API call.
|
|
321
|
-
* - `commitOffset` is a no-arg function the caller MUST invoke ONLY after
|
|
322
|
-
* successfully writing every TokenUsage event derived from `calls` into
|
|
323
|
-
* the spool. If spool writes fail (or the caller never calls it), the
|
|
324
|
-
* transcript cursor stays put and the next Stop re-reads the same delta —
|
|
325
|
-
* the server-side ON CONFLICT on the deterministic event_id then dedups
|
|
326
|
-
* whichever rows did land. This is what makes the pipeline crash-safe:
|
|
327
|
-
* we never mark transcript bytes "consumed" until we know the rows they
|
|
328
|
-
* produced are durably queued.
|
|
329
|
-
*
|
|
330
|
-
* Returns an empty `calls` array (and a no-op `commitOffset`) if nothing new
|
|
331
|
-
* in the delta or no real assistant lines are found.
|
|
332
|
-
*
|
|
333
|
-
* Partial-line handling: a Stop hook can fire while Claude Code is mid-write,
|
|
334
|
-
* so the delta may end before a trailing '\n'. We detect this, ignore the
|
|
335
|
-
* partial tail, and persist the offset at the byte right after the LAST
|
|
336
|
-
* complete '\n' so the next Stop re-reads the partial line once it's finished.
|
|
306
|
+
* Returns { lines, commitOffset }:
|
|
307
|
+
* - `lines` — raw JSONL lines in the processable region (empty on nothing-new
|
|
308
|
+
* or any I/O failure).
|
|
309
|
+
* - `commitOffset` — a no-arg fn the caller MUST invoke ONLY after it has
|
|
310
|
+
* durably consumed every line (advancing the cursor eagerly would drop rows
|
|
311
|
+
* if a later spool write fails).
|
|
337
312
|
*/
|
|
338
|
-
function
|
|
313
|
+
function readNewTranscriptDelta(transcriptPath) {
|
|
339
314
|
// Run the rate-limited stale-offset cleanup at most once per 24h. Wrapped in
|
|
340
315
|
// try/catch so a cleanup failure never prevents the token read.
|
|
341
316
|
try { maybeCleanStaleTranscriptOffsets(); } catch { /* best-effort */ }
|
|
@@ -345,7 +320,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
345
320
|
const noopCommit = () => {};
|
|
346
321
|
|
|
347
322
|
if (!transcriptPath || typeof transcriptPath !== 'string') {
|
|
348
|
-
return {
|
|
323
|
+
return { lines: [], commitOffset: noopCommit };
|
|
349
324
|
}
|
|
350
325
|
|
|
351
326
|
const offsetFile = join(TRANSCRIPT_OFFSETS_DIR, encodeURIComponent(transcriptPath));
|
|
@@ -358,7 +333,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
358
333
|
currentMtime = st.mtimeMs;
|
|
359
334
|
} catch (err) {
|
|
360
335
|
captureLog({ msg: 'transcript-offset-error', stage: 'stat', path: transcriptPath, error: err?.message });
|
|
361
|
-
return {
|
|
336
|
+
return { lines: [], commitOffset: noopCommit };
|
|
362
337
|
}
|
|
363
338
|
|
|
364
339
|
const savedState = readTranscriptOffsetState(offsetFile);
|
|
@@ -377,7 +352,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
377
352
|
|
|
378
353
|
// Nothing new — just a stat, super cheap.
|
|
379
354
|
if (currentSize === startOffset) {
|
|
380
|
-
return {
|
|
355
|
+
return { lines: [], commitOffset: noopCommit };
|
|
381
356
|
}
|
|
382
357
|
|
|
383
358
|
let buffer = null;
|
|
@@ -395,13 +370,13 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
395
370
|
// Read failed before we got any bytes — leave the cursor unchanged so the
|
|
396
371
|
// next Stop retries. (Old behavior advanced past the unread bytes; that
|
|
397
372
|
// silently dropped data on transient I/O errors.)
|
|
398
|
-
return {
|
|
373
|
+
return { lines: [], commitOffset: noopCommit };
|
|
399
374
|
}
|
|
400
375
|
|
|
401
376
|
if (!buffer || buffer.length === 0) {
|
|
402
377
|
// Defensive: shouldn't happen because of the currentSize === startOffset
|
|
403
378
|
// early return above. Don't advance the cursor.
|
|
404
|
-
return {
|
|
379
|
+
return { lines: [], commitOffset: noopCommit };
|
|
405
380
|
}
|
|
406
381
|
|
|
407
382
|
// Find the LAST complete newline. CRITICAL: search the BYTE buffer, not a
|
|
@@ -416,7 +391,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
416
391
|
if (lastNewlineIndex === -1) {
|
|
417
392
|
// The entire chunk is a partial trailing line — leave the cursor unchanged
|
|
418
393
|
// so the next Stop re-reads the line once it's finished.
|
|
419
|
-
return {
|
|
394
|
+
return { lines: [], commitOffset: noopCommit };
|
|
420
395
|
}
|
|
421
396
|
|
|
422
397
|
// Process only the bytes up to (and including) the last '\n'. The new
|
|
@@ -426,14 +401,47 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
426
401
|
const processable = buffer.slice(0, lastNewlineIndex).toString('utf-8');
|
|
427
402
|
const newOffset = startOffset + lastNewlineIndex + 1;
|
|
428
403
|
const nextState = { offset: newOffset, size: currentSize, mtime_ms: currentMtime };
|
|
429
|
-
|
|
404
|
+
// `extra` lets a caller persist source-specific cursor state (e.g. Codex's
|
|
405
|
+
// running cumulative token total, ANL-1343) atomically WITH the byte offset —
|
|
406
|
+
// so on a crash-before-commit both roll back together and nothing drifts.
|
|
407
|
+
const commitOffset = (extra) => writeTranscriptOffsetState(
|
|
408
|
+
offsetFile,
|
|
409
|
+
extra ? { ...nextState, ...extra } : nextState,
|
|
410
|
+
);
|
|
411
|
+
|
|
412
|
+
// `savedState` is the offset state as it stood BEFORE this delta (may carry
|
|
413
|
+
// extra fields a prior commit persisted). Exposed so a source-specific
|
|
414
|
+
// extractor can seed cross-delta state without re-reading the offset file.
|
|
415
|
+
return { lines: processable.split('\n'), commitOffset, savedState };
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Incrementally read only the NEW bytes appended to a Claude Code transcript
|
|
420
|
+
* since the last time this function was called for that transcript, and
|
|
421
|
+
* return ONE entry per real (non-synthetic) assistant API call in the delta.
|
|
422
|
+
*
|
|
423
|
+
* Each entry corresponds to a single Anthropic API call: a single assistant
|
|
424
|
+
* line in the JSONL with its own usage record. A single user->agent turn can
|
|
425
|
+
* produce many of these (tool-use loops), and the caller is expected to emit
|
|
426
|
+
* one unified TokenUsage event per entry — symmetric with Cursor's per-call
|
|
427
|
+
* AgentResponse rows and Codex's per-call TokenUsage rows.
|
|
428
|
+
*
|
|
429
|
+
* Returns { calls, commitOffset } (see readNewTranscriptDelta for the
|
|
430
|
+
* commit-offset crash-safety contract): `calls` is Array<{usage, model,
|
|
431
|
+
* timestamp, uuid}> with Anthropic-named usage keys for buildTokenUsageRaw,
|
|
432
|
+
* the model + ISO timestamp from that specific assistant line, and the line's
|
|
433
|
+
* `uuid` (used by the caller to derive a stable event_id so concurrent Stop
|
|
434
|
+
* hook invocations can't double-emit a single API call).
|
|
435
|
+
*/
|
|
436
|
+
function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
437
|
+
const { lines, commitOffset } = readNewTranscriptDelta(transcriptPath);
|
|
438
|
+
if (!lines.length) return { calls: [], commitOffset };
|
|
430
439
|
|
|
431
440
|
// Iterate lines in the processable region. Defensive parse-tolerance is
|
|
432
441
|
// still useful for the rare case where a single line in the middle is
|
|
433
442
|
// malformed (we skip it rather than dropping the whole delta).
|
|
434
443
|
const calls = [];
|
|
435
444
|
try {
|
|
436
|
-
const lines = processable.split('\n');
|
|
437
445
|
for (const rawLine of lines) {
|
|
438
446
|
const line = rawLine && rawLine.trim();
|
|
439
447
|
if (!line) continue;
|
|
@@ -493,7 +501,7 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
493
501
|
// to spool in this path, so it's safe to commit immediately (no risk of
|
|
494
502
|
// losing events).
|
|
495
503
|
commitOffset();
|
|
496
|
-
return { calls: [], commitOffset:
|
|
504
|
+
return { calls: [], commitOffset: () => {} };
|
|
497
505
|
}
|
|
498
506
|
|
|
499
507
|
// IMPORTANT: do NOT persist the offset here. The caller is responsible for
|
|
@@ -504,6 +512,195 @@ function extractNewClaudeApiCallsFromTranscript(transcriptPath) {
|
|
|
504
512
|
return { calls, commitOffset };
|
|
505
513
|
}
|
|
506
514
|
|
|
515
|
+
/**
|
|
516
|
+
* Codex analogue of extractNewClaudeApiCallsFromTranscript: read the NEW bytes
|
|
517
|
+
* appended to a Codex rollout transcript (`rollout-*.jsonl`) since the last
|
|
518
|
+
* call and, in a SINGLE delta pass, return both the per-call token usage
|
|
519
|
+
* (`calls`) and the assistant's dialogue text (`messages`).
|
|
520
|
+
*
|
|
521
|
+
* A single pass is mandatory: `readNewTranscriptDelta` advances a per-path byte
|
|
522
|
+
* cursor that is committed only after the caller spools everything, so a second
|
|
523
|
+
* independent read would re-consume the same bytes. Claude's extractor folds
|
|
524
|
+
* text into each `call` (usage + content share one assistant line); Codex keeps
|
|
525
|
+
* them separate because usage and text live in DIFFERENT interleaved records.
|
|
526
|
+
*
|
|
527
|
+
* TOKEN USAGE (`calls`) — Codex records per-call usage as `event_msg`/
|
|
528
|
+
* `token_count` records whose `payload.info.last_token_usage` is the DELTA for
|
|
529
|
+
* that single call (the sibling `total_token_usage` is the running cumulative
|
|
530
|
+
* sum). The rollout is OpenAI-shaped, so field names differ from Claude's and
|
|
531
|
+
* we normalize them to the Anthropic convention that buildTokenUsageRaw + the
|
|
532
|
+
* dashboard SQL expect:
|
|
533
|
+
* - Codex `cached_input_tokens` ⊆ `input_tokens` (a subset), so we split them
|
|
534
|
+
* into non-overlapping `input_tokens` (fresh) + `cache_read_input_tokens`
|
|
535
|
+
* (cached), matching Claude where `input_tokens` excludes the cache read.
|
|
536
|
+
* - Codex has no prompt-cache-creation notion → `cache_creation_input_tokens`
|
|
537
|
+
* is always 0.
|
|
538
|
+
* - `output_tokens` already includes `reasoning_output_tokens`, mirroring
|
|
539
|
+
* Claude's output convention, so it passes through unchanged.
|
|
540
|
+
* Records whose call delta is all-zero (input+output == 0, e.g. the residual
|
|
541
|
+
* token_count emitted around a compaction) are skipped so we never spool an
|
|
542
|
+
* empty TokenUsage row.
|
|
543
|
+
*
|
|
544
|
+
* ASSISTANT TEXT (`messages`, ANL-1239) — the assistant's visible dialogue
|
|
545
|
+
* lives in `response_item`/`message` records with `role === 'assistant'`; we
|
|
546
|
+
* concatenate their `content[]` `output_text` blocks. The dedup key is
|
|
547
|
+
* `${timestamp}:${payload.id | sha(text)}`: current Codex writes a durable
|
|
548
|
+
* `msg_…` id on every assistant message (hard version cutoff ~2026-06-26; older
|
|
549
|
+
* rollouts have none), so we prefer it and fall back to a text fingerprint —
|
|
550
|
+
* mirroring how the token path synthesizes a stable key from records that lack
|
|
551
|
+
* a per-record uuid. The
|
|
552
|
+
* caller emits these as AssistantText events, metric-neutral server-side
|
|
553
|
+
* (ASSISTANT_CONTENT_TYPES). NOTE: there is deliberately no AssistantThinking
|
|
554
|
+
* analogue (unlike Claude). Current Codex emits no plaintext reasoning: the
|
|
555
|
+
* full chain-of-thought lives only in `response_item`/`reasoning`
|
|
556
|
+
* `encrypted_content`, and the short plaintext `summary` those records once
|
|
557
|
+
* carried was dropped ~2026-04 (verified on real rollouts: 91% of Feb reasoning
|
|
558
|
+
* records had a summary, 0% from April on). Since AI Lens captures live/current
|
|
559
|
+
* Codex, no recoverable thinking text exists.
|
|
560
|
+
*
|
|
561
|
+
* The model isn't on the token_count / message record; it lives on the nearest
|
|
562
|
+
* preceding `turn_context` record. We track the last-seen turn_context model
|
|
563
|
+
* within the delta and fall back to `fallbackModel` (the Stop event's model)
|
|
564
|
+
* when the governing turn_context landed in an earlier delta.
|
|
565
|
+
*
|
|
566
|
+
* @param {string} transcriptPath Path to the Codex rollout JSONL.
|
|
567
|
+
* @param {string|null} fallbackModel Model from the Stop event, used when no
|
|
568
|
+
* turn_context precedes a record in this
|
|
569
|
+
* delta.
|
|
570
|
+
* @returns {{calls: Array<{usage, model, timestamp, uuid}>,
|
|
571
|
+
* messages: Array<{text, model, timestamp, uuid}>,
|
|
572
|
+
* commitOffset: function}}
|
|
573
|
+
*/
|
|
574
|
+
function extractNewCodexRecordsFromTranscript(transcriptPath, fallbackModel) {
|
|
575
|
+
const { lines, commitOffset, savedState } = readNewTranscriptDelta(transcriptPath);
|
|
576
|
+
if (!lines.length) return { calls: [], messages: [], commitOffset };
|
|
577
|
+
|
|
578
|
+
const calls = [];
|
|
579
|
+
const messages = [];
|
|
580
|
+
let currentModel = fallbackModel || null;
|
|
581
|
+
// Highest cumulative token total already counted for this transcript, seeded
|
|
582
|
+
// from the persisted offset state so it survives across Stop deltas. Codex
|
|
583
|
+
// sometimes re-emits an identical token_count without advancing its
|
|
584
|
+
// cumulative counter (`total_token_usage.total_tokens`); the duplicate carries
|
|
585
|
+
// a fresh timestamp, so its event_id differs and the server dedup never
|
|
586
|
+
// collapses it — double-counting tokens (ANL-1343). We drop any token_count
|
|
587
|
+
// whose cumulative does not advance past this max. The cumulative is monotonic
|
|
588
|
+
// and never resets (verified across compactions on 22k+ real records), so a
|
|
589
|
+
// single running max is a complete, false-positive-free signal.
|
|
590
|
+
let maxTotal = toNumberOrNull(savedState?.codex_last_total) ?? 0;
|
|
591
|
+
try {
|
|
592
|
+
for (const rawLine of lines) {
|
|
593
|
+
const line = rawLine && rawLine.trim();
|
|
594
|
+
if (!line) continue;
|
|
595
|
+
let parsed;
|
|
596
|
+
try {
|
|
597
|
+
parsed = JSON.parse(line);
|
|
598
|
+
} catch {
|
|
599
|
+
continue;
|
|
600
|
+
}
|
|
601
|
+
// A turn_context record carries the model for the records that follow it.
|
|
602
|
+
if (parsed?.type === 'turn_context' && typeof parsed.payload?.model === 'string' && parsed.payload.model) {
|
|
603
|
+
currentModel = parsed.payload.model;
|
|
604
|
+
continue;
|
|
605
|
+
}
|
|
606
|
+
const payload = parsed?.payload;
|
|
607
|
+
if (!payload) continue;
|
|
608
|
+
const ts = typeof parsed.timestamp === 'string' ? parsed.timestamp : null;
|
|
609
|
+
|
|
610
|
+
// Assistant dialogue text — response_item/message with role 'assistant'.
|
|
611
|
+
if (parsed.type === 'response_item' && payload.type === 'message' && payload.role === 'assistant') {
|
|
612
|
+
let text = '';
|
|
613
|
+
if (Array.isArray(payload.content)) {
|
|
614
|
+
for (const block of payload.content) {
|
|
615
|
+
if (!block || typeof block !== 'object') continue;
|
|
616
|
+
// Assistant content is 'output_text'; accept 'text' too, defensively.
|
|
617
|
+
if ((block.type === 'output_text' || block.type === 'text') && typeof block.text === 'string' && block.text) {
|
|
618
|
+
text += (text ? '\n' : '') + block.text;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
if (text) {
|
|
623
|
+
// Stable per-message dedup key: `${timestamp}:${id | sha(text)}`.
|
|
624
|
+
// Current Codex writes a durable `msg_…` id on every assistant
|
|
625
|
+
// message (verified: 100% from the 2026-06-26 build onward; older
|
|
626
|
+
// rollouts, ≤ mid-May in local data, carry none — a hard version
|
|
627
|
+
// cutoff, not a gradual mix). Live capture only ever sees current
|
|
628
|
+
// rollouts, so the id is normally present; the text-hash fallback is
|
|
629
|
+
// defensive insurance for a pre-cutoff / downgraded client. Either
|
|
630
|
+
// way the key is unique per record and identical across two Stop
|
|
631
|
+
// hooks reading the same bytes (main() re-hashes source_uuid into the
|
|
632
|
+
// event_id). Prefer the durable id so the key stays human-traceable.
|
|
633
|
+
const idPart = typeof payload.id === 'string' && payload.id
|
|
634
|
+
? payload.id
|
|
635
|
+
: createHash('sha256').update(text).digest('hex').slice(0, 16);
|
|
636
|
+
const uuid = `${ts || ''}:${idPart}`;
|
|
637
|
+
messages.push({
|
|
638
|
+
text,
|
|
639
|
+
model: currentModel,
|
|
640
|
+
timestamp: ts,
|
|
641
|
+
uuid,
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
if (payload.type !== 'token_count') continue;
|
|
648
|
+
const info = payload.info;
|
|
649
|
+
const last = info?.last_token_usage;
|
|
650
|
+
if (!last || typeof last !== 'object') continue;
|
|
651
|
+
|
|
652
|
+
const rawInput = toNumberOrNull(last.input_tokens) ?? 0;
|
|
653
|
+
const cached = toNumberOrNull(last.cached_input_tokens) ?? 0;
|
|
654
|
+
const output = toNumberOrNull(last.output_tokens) ?? 0;
|
|
655
|
+
// Skip all-zero deltas (residual token_count around a compaction).
|
|
656
|
+
if (rawInput + output <= 0) continue;
|
|
657
|
+
|
|
658
|
+
// cached_input_tokens is a SUBSET of input_tokens — split into
|
|
659
|
+
// non-overlapping fresh-input + cache-read to match Claude's convention.
|
|
660
|
+
const cacheRead = Math.min(cached, rawInput);
|
|
661
|
+
const freshInput = Math.max(0, rawInput - cacheRead);
|
|
662
|
+
|
|
663
|
+
// Stable per-call fingerprint for the dedup event_id. The token_count
|
|
664
|
+
// record has no per-record uuid, so we key on the record timestamp + the
|
|
665
|
+
// cumulative total (monotonic across calls) — unique per token_count
|
|
666
|
+
// record and identical across two concurrent Stop hooks reading the same
|
|
667
|
+
// bytes.
|
|
668
|
+
const cum = toNumberOrNull(info?.total_token_usage?.total_tokens);
|
|
669
|
+
// Phantom-duplicate guard (ANL-1343): skip a token_count whose cumulative
|
|
670
|
+
// total does not advance past the max already counted for this transcript.
|
|
671
|
+
// Every such record is a byte-identical re-emission (verified: 100% literal
|
|
672
|
+
// duplicates, 0 false positives). cum-less records (older Codex without a
|
|
673
|
+
// total) fall through unchanged and rely on the existing dedup.
|
|
674
|
+
if (cum != null) {
|
|
675
|
+
if (cum <= maxTotal) continue;
|
|
676
|
+
maxTotal = cum;
|
|
677
|
+
}
|
|
678
|
+
const uuid = `${ts || ''}:${cum != null ? cum : ''}`;
|
|
679
|
+
|
|
680
|
+
calls.push({
|
|
681
|
+
usage: {
|
|
682
|
+
input_tokens: freshInput,
|
|
683
|
+
output_tokens: output,
|
|
684
|
+
cache_read_input_tokens: cacheRead,
|
|
685
|
+
cache_creation_input_tokens: 0,
|
|
686
|
+
},
|
|
687
|
+
model: currentModel,
|
|
688
|
+
timestamp: ts,
|
|
689
|
+
uuid: uuid === ':' ? null : uuid,
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
} catch (err) {
|
|
693
|
+
captureLog({ msg: 'transcript-offset-error', stage: 'parse-codex', path: transcriptPath, error: err?.message });
|
|
694
|
+
commitOffset({ codex_last_total: maxTotal });
|
|
695
|
+
return { calls: [], messages: [], commitOffset: () => {} };
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// Persist the advanced cumulative max together with the byte offset, so the
|
|
699
|
+
// next Stop delta seeds `maxTotal` correctly and cross-delta duplicates are
|
|
700
|
+
// still caught. Deferred: the caller invokes this only AFTER spooling.
|
|
701
|
+
return { calls, messages, commitOffset: () => commitOffset({ codex_last_total: maxTotal }) };
|
|
702
|
+
}
|
|
703
|
+
|
|
507
704
|
// =============================================================================
|
|
508
705
|
// Source Detection
|
|
509
706
|
// =============================================================================
|
|
@@ -524,10 +721,19 @@ export function detectSource(event) {
|
|
|
524
721
|
// Codex writes its rollout transcript as `.../sessions/.../rollout-<ts>-<id>.jsonl`
|
|
525
722
|
// (codex-rs session naming); Claude Code uses `~/.claude/projects/<dir>/<uuid>.jsonl`.
|
|
526
723
|
// The `rollout-*.jsonl` basename is the reliable Codex signal for SessionStart.
|
|
724
|
+
// But some Codex SessionStart events carry an EMPTY transcript_path (no rollout yet) and
|
|
725
|
+
// no turn_id, so they'd fall through to claude_code — which mislabeled the whole session
|
|
726
|
+
// downstream ([0132]). Those events carry `model: "gpt-5.x…"`; Claude Code never emits a
|
|
727
|
+
// `gpt-*` model, so it's a safe Codex tie-breaker — but ONLY for the claude-code-shaped
|
|
728
|
+
// envelope: Cursor also emits `gpt-*` models, so gate the model signal behind "no Cursor
|
|
729
|
+
// signal present" (conversation_id/cursor_version), which the Cursor branch below keys on.
|
|
527
730
|
const transcriptPath = typeof event.transcript_path === 'string' ? event.transcript_path : '';
|
|
731
|
+
const model = typeof event.model === 'string' ? event.model : '';
|
|
732
|
+
const gptModelNoCursor = /^gpt[-.]/i.test(model) && !event.conversation_id && !event.cursor_version;
|
|
528
733
|
if (event.codex_version || event.source === 'codex' || event.hook_source === 'codex'
|
|
529
734
|
|| typeof event.turn_id === 'string'
|
|
530
|
-
|| /(^|[\\/])rollout-[^\\/]*\.jsonl$/.test(transcriptPath)
|
|
735
|
+
|| /(^|[\\/])rollout-[^\\/]*\.jsonl$/.test(transcriptPath)
|
|
736
|
+
|| gptModelNoCursor) return 'codex';
|
|
531
737
|
if (event.conversation_id || event.cursor_version) return 'cursor';
|
|
532
738
|
if (event.session_id || event.transcript_path) return 'claude_code';
|
|
533
739
|
return 'unknown';
|
|
@@ -687,7 +893,14 @@ function refineProjectPath(event, projectPath, sessionId) {
|
|
|
687
893
|
const filePath = extractFilePath(toolInput);
|
|
688
894
|
if (!filePath) return projectPath;
|
|
689
895
|
const gitRoot = findGitRoot(filePath);
|
|
690
|
-
|
|
896
|
+
// Refine ONLY downward — into the current project itself or a real sub-repo
|
|
897
|
+
// of it (nested repos like etl-pipelines/bk-reports-automation/). Never flip
|
|
898
|
+
// to an unrelated repo just because its path string happens to be longer:
|
|
899
|
+
// a session running in one project that touches a file in a sibling/foreign
|
|
900
|
+
// repo would otherwise get misattributed there (and mis-filtered against the
|
|
901
|
+
// `projects` config). `pathContains` handles the nesting check plus Windows
|
|
902
|
+
// backslash/drive-letter/case normalization for free.
|
|
903
|
+
if (gitRoot && (!projectPath || pathContains(projectPath, gitRoot))) {
|
|
691
904
|
if (sessionId) cacheSessionPath(sessionId, gitRoot);
|
|
692
905
|
return gitRoot;
|
|
693
906
|
}
|
|
@@ -1369,7 +1582,7 @@ function normalizeCodex(event) {
|
|
|
1369
1582
|
projectPath = refineProjectPath({ tool_input: codexToolInput(event), input: codexToolInput(event) }, projectPath, sessionId);
|
|
1370
1583
|
}
|
|
1371
1584
|
|
|
1372
|
-
|
|
1585
|
+
const primary = {
|
|
1373
1586
|
event_id: null,
|
|
1374
1587
|
source: 'codex',
|
|
1375
1588
|
session_id: sessionId,
|
|
@@ -1378,7 +1591,59 @@ function normalizeCodex(event) {
|
|
|
1378
1591
|
timestamp,
|
|
1379
1592
|
data,
|
|
1380
1593
|
raw: event,
|
|
1381
|
-
}
|
|
1594
|
+
};
|
|
1595
|
+
|
|
1596
|
+
// On Stop (end of a Codex agent turn), read the rollout delta once and emit
|
|
1597
|
+
// one TokenUsage event per real API call PLUS one AssistantText event per
|
|
1598
|
+
// assistant message (ANL-1239) — giving Codex the same row-per-call
|
|
1599
|
+
// granularity as Claude Code and Cursor. The rollout path arrives as
|
|
1600
|
+
// `transcript_path` (the same field detectSource keys on for `rollout-*`).
|
|
1601
|
+
if (hookType === 'Stop') {
|
|
1602
|
+
const transcriptPath = event.transcript_path || null;
|
|
1603
|
+
const fallbackModel = typeof event.model === 'string' ? event.model : null;
|
|
1604
|
+
const { calls, messages, commitOffset } = extractNewCodexRecordsFromTranscript(transcriptPath, fallbackModel);
|
|
1605
|
+
const tokenEvents = calls.map(call => ({
|
|
1606
|
+
event_id: null, // assigned in main() from a stable hash of call.uuid
|
|
1607
|
+
source: 'codex',
|
|
1608
|
+
session_id: sessionId,
|
|
1609
|
+
type: 'TokenUsage',
|
|
1610
|
+
project_path: projectPath,
|
|
1611
|
+
timestamp: call.timestamp || timestamp,
|
|
1612
|
+
data: {
|
|
1613
|
+
model: call.model,
|
|
1614
|
+
input_tokens: call.usage.input_tokens,
|
|
1615
|
+
output_tokens: call.usage.output_tokens,
|
|
1616
|
+
},
|
|
1617
|
+
raw: buildTokenUsageRaw({ source_uuid: call.uuid }, call.usage, call.model),
|
|
1618
|
+
}));
|
|
1619
|
+
// Assistant dialogue text (ANL-1239). Stored for dialogue/search;
|
|
1620
|
+
// metric-neutral server-side (ASSISTANT_CONTENT_TYPES). data.text is
|
|
1621
|
+
// truncated for display; raw.text keeps the full text (redacted
|
|
1622
|
+
// server-side). Codex has NO plaintext thinking, so there is no
|
|
1623
|
+
// AssistantThinking counterpart (unlike the Claude Code Stop path).
|
|
1624
|
+
const contentEvents = messages.map(msg => ({
|
|
1625
|
+
event_id: null, // assigned in main() from a stable hash of msg.uuid
|
|
1626
|
+
source: 'codex',
|
|
1627
|
+
session_id: sessionId,
|
|
1628
|
+
type: 'AssistantText',
|
|
1629
|
+
project_path: projectPath,
|
|
1630
|
+
timestamp: msg.timestamp || timestamp,
|
|
1631
|
+
data: { text: truncate(msg.text, TRUNCATION_LIMITS.agentResponse), model: msg.model },
|
|
1632
|
+
raw: { source_uuid: msg.uuid, model: msg.model, text: msg.text },
|
|
1633
|
+
}));
|
|
1634
|
+
const result = [primary, ...tokenEvents, ...contentEvents];
|
|
1635
|
+
// Attach the commit callback as a non-enumerable property so it survives
|
|
1636
|
+
// through normalizeEvent()/writeToSpool without leaking into iteration or
|
|
1637
|
+
// length assertions — mirrors the Claude Code Stop path.
|
|
1638
|
+
Object.defineProperty(result, 'commitTranscriptOffset', {
|
|
1639
|
+
value: commitOffset,
|
|
1640
|
+
enumerable: false,
|
|
1641
|
+
writable: false,
|
|
1642
|
+
});
|
|
1643
|
+
return result;
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
return [primary];
|
|
1382
1647
|
}
|
|
1383
1648
|
|
|
1384
1649
|
// =============================================================================
|
|
@@ -2142,7 +2407,7 @@ async function main() {
|
|
|
2142
2407
|
: ev.type === 'AssistantThinking' ? 'assistantthinking'
|
|
2143
2408
|
: 'tokenusage';
|
|
2144
2409
|
if (sourceUuid) {
|
|
2145
|
-
ev.event_id = deterministicEventId(
|
|
2410
|
+
ev.event_id = deterministicEventId(`${ev.source}:${kind}:${sourceUuid}`);
|
|
2146
2411
|
} else {
|
|
2147
2412
|
// Fallback: stdin hash + per-event index. Should never be needed because
|
|
2148
2413
|
// Claude Code transcripts always include uuid per record.
|
package/client/redact.js
CHANGED
|
@@ -90,6 +90,10 @@ const PATTERNS = [
|
|
|
90
90
|
// Meilisearch master/API key (mc_ + 32+ hex chars)
|
|
91
91
|
{ type: 'MEILISEARCH_KEY', re: /\bmc_[0-9a-f]{32,}\b/g },
|
|
92
92
|
|
|
93
|
+
// Cursor Admin API key (crsr_ + 64 hex chars as issued today; 32+ to stay
|
|
94
|
+
// ahead of a longer future format)
|
|
95
|
+
{ type: 'CURSOR_ADMIN_KEY', re: /\bcrsr_[0-9a-f]{32,}\b/g },
|
|
96
|
+
|
|
93
97
|
// Connection string password (://user:password@host) — redacts password only.
|
|
94
98
|
// Tightened: exclude JSON structural chars from the user/password classes so
|
|
95
99
|
// a stringified JSON payload (e.g. MCP `tool_response` kept as a single
|