@vibe-cafe/vibe-usage 0.9.15 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -51,7 +51,7 @@ npx @vibe-cafe/vibe-usage status # Show config & detected tools
51
51
  | Tool | Data Location |
52
52
  |------|---------------|
53
53
  | Claude Code | `~/.claude/projects/` (tokens + sessions), `~/.claude/transcripts/` (sessions only); also scans `$CLAUDE_CONFIG_DIR` when set (deduped), so relocated configs and GUI/CLI env mismatches are both covered |
54
- | Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`); forked and sub-agent rollouts replay parent metadata, tasks, and `token_count` records at spawn time full or last-N-turn replay blocks are matched as a fingerprinted parent suffix plus the child's own task boundary, duplicate cumulative emissions count once, and live/archive copies of the same session are deduplicated |
54
+ | Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`); a versioned local index avoids re-reading unchanged rollouts and reads only safe append tails for ordinary sessions, while fork/sub-agent replay matching, duplicate suppression, and live/archive deduplication retain their existing semantics |
55
55
  | Grok | `$GROK_HOME/sessions/<encoded-cwd>/<session-id>/` (default `~/.grok`); token usage from `updates.jsonl` `turn_completed.usage` (per-model `modelUsage`, cache reads, reasoning); project from `summary.json` cwd; honors `GROK_HOME` |
56
56
  | GitHub Copilot CLI | `~/.copilot/session-state/*/events.jsonl` |
57
57
  | Cursor | `state.vscdb` (SQLite, reads `cursorAuth/accessToken`, fetches CSV from `cursor.com`); cloud data is stamped with a fixed `cursor-cloud` hostname so multi-machine setups don't double-count |
@@ -77,7 +77,9 @@ npx @vibe-cafe/vibe-usage status # Show config & detected tools
77
77
  - Aggregates token usage into 30-minute buckets
78
78
  - Extracts session metadata from all parsers: active time (AI generation time, excluding queue/TTFT wait), total duration, message counts
79
79
  - Uploads buckets + sessions to your vibecafe.ai dashboard (always gzip-compressed, ~94% smaller)
80
- - Incremental: parsers still compute full totals from local logs each sync (idempotent), but only buckets/sessions that are new or changed since the last successful upload are sent — a quiet machine uploads nothing. Sync state is kept in `~/.vibe-usage/state.json`; failed parsers retain their prior state, while deleted local logs are pruned. Deleting the state file triggers a one-time full re-upload, and `reset` clears it automatically after deleting cloud data
80
+ - Incremental upload: every parser emits a complete local snapshot, then only buckets/sessions that are new or changed since the last successful upload are sent — a quiet machine uploads nothing. Upload state remains in `~/.vibe-usage/state.json`; failed or still-indexing parsers retain their prior state, while deleted local logs are pruned. Deleting the state file triggers a one-time full re-upload, and `reset` clears it automatically after deleting cloud data
81
+ - Incremental Codex parsing: a versioned, disposable cache under `~/.vibe-usage/cache/codex/` stores per-rollout aggregate results and parser continuation state. Unchanged rollouts require no raw-log reads; an ordinary append reads only the new tail; forks, sub-agents, replacements, truncations, and failed safety checks fall back to the full correctness path. A bounded rolling audit occasionally re-reads one historical file. Very large first-time indexes checkpoint before the Mac app timeout and resume on the next sync instead of restarting
82
+ - The Codex parser cache contains derived aggregates and replay metadata, not raw prompt or response text. It is independent of upload state and can be deleted safely (the next sync rebuilds it). `reset` intentionally keeps it so the required full re-upload does not also require a full disk rescan. Set `VIBE_USAGE_CODEX_CACHE=0` to disable the optimization for diagnosis
81
83
  - SQLite-backed tools (Cursor, OpenCode, Kiro, Hermes) are read via Node's built-in `node:sqlite` on Node ≥ 22.5 — no `sqlite3` binary needed (works on Windows out of the box); on older Node it falls back to the system `sqlite3` CLI
82
84
  - For continuous syncing, use `npx @vibe-cafe/vibe-usage daemon` or the [Vibe Usage Mac app](https://github.com/vibe-cafe/vibe-usage-app)
83
85
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vibe-cafe/vibe-usage",
3
- "version": "0.9.15",
3
+ "version": "0.10.0",
4
4
  "description": "Track your AI coding tool token usage and sync to vibecafe.ai",
5
5
  "type": "module",
6
6
  "scripts": {
@@ -0,0 +1,138 @@
1
+ import {
2
+ existsSync,
3
+ mkdirSync,
4
+ readFileSync,
5
+ renameSync,
6
+ rmSync,
7
+ writeFileSync,
8
+ } from 'node:fs';
9
+ import { createHash, randomBytes } from 'node:crypto';
10
+ import { homedir } from 'node:os';
11
+ import { join } from 'node:path';
12
+
13
+ // Parser caches are disposable derived data. Keep their schema/version fully
14
+ // separate from ~/.vibe-usage/state.json, whose hashes are the authoritative
15
+ // record of successful uploads and must remain backward-compatible.
16
+ export const CODEX_CACHE_SCHEMA_VERSION = 1;
17
+ export const CODEX_PARSER_ALGORITHM_VERSION = 1;
18
+
19
+ function hash(value, length = 24) {
20
+ return createHash('sha256').update(value).digest('hex').slice(0, length);
21
+ }
22
+
23
+ export function codexCacheEnabled() {
24
+ return process.env.VIBE_USAGE_CODEX_CACHE !== '0';
25
+ }
26
+
27
+ export function fileSignature(stat) {
28
+ return {
29
+ size: stat.size,
30
+ mtimeMs: stat.mtimeMs,
31
+ dev: String(stat.dev),
32
+ ino: String(stat.ino),
33
+ };
34
+ }
35
+
36
+ function sameSignature(a, b) {
37
+ return a?.size === b.size
38
+ && a?.mtimeMs === b.mtimeMs
39
+ && a?.dev === b.dev
40
+ && a?.ino === b.ino;
41
+ }
42
+
43
+ export function codexCacheDir(codexHome) {
44
+ const base = process.env.VIBE_USAGE_CACHE_DIR?.trim()
45
+ || join(homedir(), '.vibe-usage', 'cache');
46
+ return join(base, 'codex', `root-${hash(codexHome)}`);
47
+ }
48
+
49
+ function entryPath(codexHome, filePath) {
50
+ return join(codexCacheDir(codexHome), `${hash(filePath, 32)}.json`);
51
+ }
52
+
53
+ function tailPath(codexHome, filePath) {
54
+ return join(codexCacheDir(codexHome), `${hash(filePath, 32)}.tail.json`);
55
+ }
56
+
57
+ export function loadCodexFileCache(codexHome, filePath, signature) {
58
+ if (!codexCacheEnabled()) return null;
59
+ const path = entryPath(codexHome, filePath);
60
+ if (!existsSync(path)) return null;
61
+ try {
62
+ const parsed = JSON.parse(readFileSync(path, 'utf8'));
63
+ if (parsed.schemaVersion !== CODEX_CACHE_SCHEMA_VERSION) return null;
64
+ if (parsed.algorithmVersion !== CODEX_PARSER_ALGORITHM_VERSION) return null;
65
+ if (parsed.filePath !== filePath) return null;
66
+ if (signature && !sameSignature(parsed.signature, signature)) return null;
67
+ return parsed;
68
+ } catch {
69
+ // Cache corruption is a performance miss, never a correctness failure.
70
+ return null;
71
+ }
72
+ }
73
+
74
+ export function saveCodexFileCache(codexHome, filePath, signature, data) {
75
+ if (!codexCacheEnabled()) return;
76
+ const dir = codexCacheDir(codexHome);
77
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
78
+ const path = entryPath(codexHome, filePath);
79
+ const tempPath = `${path}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`;
80
+ const payload = {
81
+ schemaVersion: CODEX_CACHE_SCHEMA_VERSION,
82
+ algorithmVersion: CODEX_PARSER_ALGORITHM_VERSION,
83
+ filePath,
84
+ signature,
85
+ ...data,
86
+ };
87
+ try {
88
+ writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
89
+ renameSync(tempPath, path);
90
+ } finally {
91
+ // A killed writer can leave a unique temp file; a normal failed writer
92
+ // should not. rmSync is safe here because the target is this call's exact,
93
+ // random temporary path inside the versioned cache directory.
94
+ rmSync(tempPath, { force: true });
95
+ }
96
+ }
97
+
98
+ export function loadCodexFileTail(codexHome, filePath) {
99
+ if (!codexCacheEnabled()) return null;
100
+ const path = tailPath(codexHome, filePath);
101
+ if (!existsSync(path)) return null;
102
+ try {
103
+ const parsed = JSON.parse(readFileSync(path, 'utf8'));
104
+ if (parsed.schemaVersion !== CODEX_CACHE_SCHEMA_VERSION) return null;
105
+ if (parsed.algorithmVersion !== CODEX_PARSER_ALGORITHM_VERSION) return null;
106
+ if (parsed.filePath !== filePath || !parsed.signature || !parsed.tail) return null;
107
+ return parsed;
108
+ } catch {
109
+ return null;
110
+ }
111
+ }
112
+
113
+ export function saveCodexFileTail(codexHome, filePath, signature, tail) {
114
+ if (!codexCacheEnabled() || !tail) return;
115
+ const dir = codexCacheDir(codexHome);
116
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
117
+ const path = tailPath(codexHome, filePath);
118
+ const tempPath = `${path}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`;
119
+ const payload = {
120
+ schemaVersion: CODEX_CACHE_SCHEMA_VERSION,
121
+ algorithmVersion: CODEX_PARSER_ALGORITHM_VERSION,
122
+ filePath,
123
+ signature,
124
+ tail,
125
+ };
126
+ try {
127
+ writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
128
+ renameSync(tempPath, path);
129
+ } finally {
130
+ rmSync(tempPath, { force: true });
131
+ }
132
+ }
133
+
134
+ export function removeCodexFileCache(codexHome, filePath) {
135
+ if (!codexCacheEnabled()) return;
136
+ rmSync(entryPath(codexHome, filePath), { force: true });
137
+ rmSync(tailPath(codexHome, filePath), { force: true });
138
+ }
@@ -1,9 +1,25 @@
1
- import { createReadStream, readdirSync, existsSync } from 'node:fs';
1
+ import {
2
+ closeSync,
3
+ createReadStream,
4
+ existsSync,
5
+ openSync,
6
+ readSync,
7
+ readdirSync,
8
+ statSync,
9
+ } from 'node:fs';
2
10
  import { join } from 'node:path';
3
11
  import { homedir } from 'node:os';
4
12
  import { createInterface } from 'node:readline';
5
13
  import { createHash } from 'node:crypto';
6
- import { aggregateToBuckets, extractSessions } from './index.js';
14
+ import { aggregateToBuckets } from './index.js';
15
+ import {
16
+ codexCacheEnabled,
17
+ fileSignature,
18
+ loadCodexFileCache,
19
+ loadCodexFileTail,
20
+ saveCodexFileCache,
21
+ saveCodexFileTail,
22
+ } from './codex-cache.js';
7
23
 
8
24
  // Codex stores live sessions in $CODEX_HOME/sessions (default ~/.codex) and,
9
25
  // once a session is "completed", moves its rollout file verbatim into
@@ -12,8 +28,11 @@ import { aggregateToBuckets, extractSessions } from './index.js';
12
28
  // both, index them together so fork replay-skip works across directories, and
13
29
  // select the most complete physical file when the same session briefly exists
14
30
  // in both locations during an archive move.
15
- function sessionsDirs() {
16
- const codexHome = process.env.CODEX_HOME?.trim() || join(homedir(), '.codex');
31
+ function getCodexHome() {
32
+ return process.env.CODEX_HOME?.trim() || join(homedir(), '.codex');
33
+ }
34
+
35
+ function sessionsDirs(codexHome) {
17
36
  return [
18
37
  join(codexHome, 'sessions'),
19
38
  join(codexHome, 'archived_sessions'),
@@ -42,9 +61,16 @@ function findJsonlFiles(dir) {
42
61
  return results;
43
62
  }
44
63
 
45
- function readLines(filePath) {
64
+ function readLines(filePath, snapshotSize, start = 0) {
46
65
  return createInterface({
47
- input: createReadStream(filePath, { encoding: 'utf-8' }),
66
+ input: createReadStream(filePath, {
67
+ encoding: 'utf-8',
68
+ // Rollouts are append-only while Codex is working. Bound both parser
69
+ // passes to the size captured before pass 1 so they see the same prefix
70
+ // even when the live file grows between reads.
71
+ ...(start > 0 ? { start } : {}),
72
+ ...(snapshotSize == null ? {} : { end: snapshotSize - 1 }),
73
+ }),
48
74
  crlfDelay: Infinity,
49
75
  });
50
76
  }
@@ -79,6 +105,84 @@ function extractParentThreadId(meta) {
79
105
  || null;
80
106
  }
81
107
 
108
+ /**
109
+ * Read only far enough to find the canonical (first) session_meta. This cheap
110
+ * discovery pass lets ordinary sessions skip the old full-file index pass;
111
+ * only forks, sub-agents, and parents referenced by them need replay indexes.
112
+ */
113
+ async function readSessionHeader(filePath, snapshotSize) {
114
+ for await (const line of readLines(filePath, snapshotSize)) {
115
+ if (!line.trim()) continue;
116
+ try {
117
+ const obj = JSON.parse(line);
118
+ if (obj.type !== 'session_meta' || !obj.payload) continue;
119
+ const meta = obj.payload;
120
+ return {
121
+ sessionId: meta.id || null,
122
+ forkedFromId: meta.forked_from_id || null,
123
+ parentThreadId: extractParentThreadId(meta),
124
+ sessionProject: extractProject(meta),
125
+ sessionStartedAtMs: timestampMs(meta.timestamp) ?? timestampMs(obj.timestamp),
126
+ isSubagent: isSubagentMeta(meta),
127
+ };
128
+ } catch {
129
+ continue;
130
+ }
131
+ }
132
+ return {
133
+ sessionId: null,
134
+ forkedFromId: null,
135
+ parentThreadId: null,
136
+ sessionProject: 'unknown',
137
+ sessionStartedAtMs: null,
138
+ isSubagent: false,
139
+ };
140
+ }
141
+
142
+ function cacheData(cache, changes = {}) {
143
+ return {
144
+ header: changes.header ?? cache?.header ?? null,
145
+ index: changes.index ?? cache?.index ?? null,
146
+ result: changes.result ?? cache?.result ?? null,
147
+ lastAuditedAt: changes.lastAuditedAt ?? cache?.lastAuditedAt ?? null,
148
+ };
149
+ }
150
+
151
+ const TAIL_GUARD_BYTES = 4096;
152
+
153
+ function snapshotGuard(filePath, size) {
154
+ if (size <= 0) return { hash: null, endsWithNewline: false };
155
+ const length = Math.min(size, TAIL_GUARD_BYTES);
156
+ const buffer = Buffer.allocUnsafe(length);
157
+ const fd = openSync(filePath, 'r');
158
+ try {
159
+ const read = readSync(fd, buffer, 0, length, size - length);
160
+ const slice = buffer.subarray(0, read);
161
+ return {
162
+ hash: createHash('sha256').update(slice).digest('base64url').slice(0, 20),
163
+ endsWithNewline: slice.at(-1) === 0x0a,
164
+ };
165
+ } finally {
166
+ closeSync(fd);
167
+ }
168
+ }
169
+
170
+ function tailStateFor(file) {
171
+ const prior = file.priorTail;
172
+ const tail = prior?.tail;
173
+ if (!tail || !prior.signature) return null;
174
+ if (prior.signature.size <= 0 || prior.signature.size >= file.signature.size) return null;
175
+ if (prior.signature.dev !== file.signature.dev || prior.signature.ino !== file.signature.ino) return null;
176
+ if (prior.signature.mtimeMs > file.signature.mtimeMs) return null;
177
+ if (tail.parsedBytes !== prior.signature.size || !tail.endsWithNewline || !tail.guardHash) return null;
178
+ try {
179
+ const guard = snapshotGuard(file.filePath, prior.signature.size);
180
+ return guard.hash === tail.guardHash ? tail : null;
181
+ } catch {
182
+ return null;
183
+ }
184
+ }
185
+
82
186
  function timestampMs(value) {
83
187
  if (value == null || value === '') return null;
84
188
  const n = new Date(value).getTime();
@@ -143,6 +247,34 @@ function longestReplayPrefix(child, parent) {
143
247
  return matched;
144
248
  }
145
249
 
250
+ /**
251
+ * Return the longest prefix of `child` found contiguously anywhere in
252
+ * `parent`. A live sub-agent rollout can be observed while Codex is still
253
+ * copying the parent block, before that copy reaches the parent snapshot's
254
+ * end. In that state the exact records are inherited history even though the
255
+ * stricter completed-replay suffix match above deliberately rejects them.
256
+ */
257
+ function longestPartialReplayPrefix(child, parent) {
258
+ if (child.length === 0 || parent.length === 0) return 0;
259
+
260
+ const prefix = new Array(child.length).fill(0);
261
+ for (let i = 1, matched = 0; i < child.length; i++) {
262
+ while (matched > 0 && child[i] !== child[matched]) matched = prefix[matched - 1];
263
+ if (child[i] === child[matched]) matched++;
264
+ prefix[i] = matched;
265
+ }
266
+
267
+ let matched = 0;
268
+ let longest = 0;
269
+ for (const fingerprint of parent) {
270
+ while (matched > 0 && fingerprint !== child[matched]) matched = prefix[matched - 1];
271
+ if (fingerprint === child[matched]) matched++;
272
+ longest = Math.max(longest, matched);
273
+ if (matched === child.length) matched = prefix[matched - 1];
274
+ }
275
+ return longest;
276
+ }
277
+
146
278
  // `task_started.started_at` is stored at one-second precision while the
147
279
  // canonical session timestamp has milliseconds. Real Codex Desktop rollouts
148
280
  // start the child task within a few seconds of creating the child session.
@@ -160,7 +292,7 @@ const OWN_TASK_START_WINDOW_MS = 5_000;
160
292
  * full prefix. Together they bound matching to source records that existed at
161
293
  * spawn without over-skipping child work when the parent later grows.
162
294
  */
163
- async function indexSessionFile(filePath) {
295
+ async function indexSessionFile(filePath, snapshotSize) {
164
296
  let sessionId = null;
165
297
  let forkedFromId = null;
166
298
  let parentThreadId = null;
@@ -178,7 +310,7 @@ async function indexSessionFile(filePath) {
178
310
  let firstTaskBoundary = null;
179
311
  let ownTaskBoundary = null;
180
312
 
181
- for await (const line of readLines(filePath)) {
313
+ for await (const line of readLines(filePath, snapshotSize)) {
182
314
  if (!line.trim()) continue;
183
315
  try {
184
316
  const obj = JSON.parse(line);
@@ -261,12 +393,13 @@ function replayBoundary(meta, sessionById) {
261
393
  const parentAtSpawn = parent && meta.sessionStartedAtMs != null
262
394
  ? upperBound(parent.tokenTimes, meta.sessionStartedAtMs)
263
395
  : null;
264
- const replayTokenCount = parentAtSpawn == null
265
- ? 0
266
- : longestReplayPrefix(
267
- meta.tokenFingerprints,
268
- parent.tokenFingerprints.slice(0, parentAtSpawn)
269
- );
396
+ const parentSnapshot = parentAtSpawn == null
397
+ ? []
398
+ : parent.tokenFingerprints.slice(0, parentAtSpawn);
399
+ const replayTokenCount = longestReplayPrefix(meta.tokenFingerprints, parentSnapshot);
400
+ const partialReplayTokenCount = meta.isSubagent
401
+ ? longestPartialReplayPrefix(meta.tokenFingerprints, parentSnapshot)
402
+ : 0;
270
403
 
271
404
  if (meta.isSubagent) {
272
405
  // Direct evidence inside the child wins. Legacy single-meta rollouts did
@@ -292,11 +425,25 @@ function replayBoundary(meta, sessionById) {
292
425
  : null);
293
426
  if (direct) {
294
427
  return {
295
- rawTokenCount: Math.max(replayTokenCount, direct.rawTokenCount),
428
+ rawTokenCount: Math.max(
429
+ replayTokenCount,
430
+ partialReplayTokenCount,
431
+ direct.rawTokenCount
432
+ ),
296
433
  recordIndex: direct.recordIndex,
297
434
  };
298
435
  }
299
- return { rawTokenCount: replayTokenCount, recordIndex: null };
436
+
437
+ // A recognized sub-agent can be synced while Codex is only partway
438
+ // through appending the copied parent block. The completed-replay matcher
439
+ // correctly rejects that interior slice, but counting it would create a
440
+ // temporary spike that disappears on the next sync. Exact payload overlap
441
+ // with the known parent is sufficient evidence to defer those leading
442
+ // records until the rollout reaches a stable suffix or task boundary.
443
+ return {
444
+ rawTokenCount: Math.max(replayTokenCount, partialReplayTokenCount),
445
+ recordIndex: null,
446
+ };
300
447
  }
301
448
 
302
449
  if (meta.forkedFromId) {
@@ -305,118 +452,223 @@ function replayBoundary(meta, sessionById) {
305
452
  return { rawTokenCount: 0, recordIndex: null };
306
453
  }
307
454
 
308
- export async function parse() {
309
- const dirs = sessionsDirs();
310
- if (!dirs.some(existsSync)) return { buckets: [], sessions: [] };
455
+ function boundaryKey(boundary) {
456
+ return `${boundary.rawTokenCount}:${boundary.recordIndex ?? ''}`;
457
+ }
311
458
 
312
- const entries = [];
313
- const sessionEvents = [];
314
- const files = dirs.flatMap(findJsonlFiles);
315
- if (files.length === 0) return { buckets: [], sessions: [] };
459
+ function updateFileCache(codexHome, file, changes) {
460
+ const data = cacheData(file.cache, changes);
461
+ try {
462
+ saveCodexFileCache(codexHome, file.filePath, file.signature, data);
463
+ } catch {
464
+ // A read-only home, full disk, or antivirus race must only disable the
465
+ // optimization for this run. Raw-log parsing remains the source of truth.
466
+ }
467
+ file.cache = { ...(file.cache || {}), ...data };
468
+ }
316
469
 
317
- // Pass 1: build a compact per-file index. Keep the most complete physical
318
- // copy for each logical session id so a rollout briefly present in both
319
- // sessions/ and archived_sessions/ cannot double its token buckets.
320
- const sessionById = new Map();
321
- const fileMeta = new Map();
322
- for (const filePath of files) {
323
- let meta;
324
- try {
325
- meta = await indexSessionFile(filePath);
326
- } catch {
327
- continue;
470
+ function workBudgetMs() {
471
+ const configured = Number(process.env.VIBE_USAGE_CODEX_WORK_BUDGET_MS);
472
+ if (Number.isFinite(configured) && configured > 0) return configured;
473
+ // The macOS app terminates its child after 120 seconds. Cache-building work
474
+ // in a non-interactive child therefore checkpoints before that wall so the
475
+ // next invocation resumes instead of starting from zero. Interactive users
476
+ // can let a cold build finish in one run (and can interrupt it safely).
477
+ if (codexCacheEnabled() && !process.stdout.isTTY) return 105_000;
478
+ return Number.POSITIVE_INFINITY;
479
+ }
480
+
481
+ function auditIntervalMs() {
482
+ const configured = Number(process.env.VIBE_USAGE_CODEX_AUDIT_INTERVAL_MS);
483
+ if (Number.isFinite(configured) && configured >= 0) return configured;
484
+ return 30 * 24 * 60 * 60 * 1000;
485
+ }
486
+
487
+ function auditMaxBytes() {
488
+ const configured = Number(process.env.VIBE_USAGE_CODEX_AUDIT_MAX_BYTES);
489
+ if (Number.isFinite(configured) && configured > 0) return configured;
490
+ // Keep the background audit bounded below the app's wall timeout. Larger
491
+ // active files are still invalidated immediately by their stat signature,
492
+ // and every cache generation is rebuilt after parser-algorithm changes.
493
+ return 64 * 1024 * 1024;
494
+ }
495
+
496
+ function applySessionEvent(acc, event) {
497
+ const timestampMsValue = event.timestamp.getTime();
498
+ if (acc.lastTimestampMs != null && timestampMsValue < acc.lastTimestampMs) return false;
499
+ acc.firstTimestampMs ??= timestampMsValue;
500
+ acc.lastTimestampMs = timestampMsValue;
501
+ acc.messageCount++;
502
+
503
+ if (event.role === 'user') {
504
+ if (acc.turnStartMs != null && acc.turnEndMs != null && acc.turnEndMs > acc.turnStartMs) {
505
+ acc.completedActiveSeconds += Math.round((acc.turnEndMs - acc.turnStartMs) / 1000);
328
506
  }
329
- fileMeta.set(filePath, meta);
330
- if (meta.sessionId) {
331
- const existing = sessionById.get(meta.sessionId);
332
- if (!existing || meta.parsedRecordCount > existing.parsedRecordCount) {
333
- sessionById.set(meta.sessionId, meta);
507
+ acc.turnStartMs = null;
508
+ acc.turnEndMs = null;
509
+ acc.waitingForFirstResponse = true;
510
+ acc.userMessageCount++;
511
+ acc.userPromptHours[event.timestamp.getUTCHours()]++;
512
+ } else if (acc.waitingForFirstResponse) {
513
+ acc.turnStartMs = timestampMsValue;
514
+ acc.turnEndMs = timestampMsValue;
515
+ acc.waitingForFirstResponse = false;
516
+ } else if (acc.turnStartMs != null) {
517
+ acc.turnEndMs = timestampMsValue;
518
+ }
519
+ return true;
520
+ }
521
+
522
+ function buildSessionAccumulator(events, previous = null) {
523
+ const sorted = [...events].sort((a, b) => a.timestamp - b.timestamp);
524
+ const first = sorted[0];
525
+ const acc = previous
526
+ ? {
527
+ ...previous,
528
+ userPromptHours: [...previous.userPromptHours],
334
529
  }
530
+ : {
531
+ sessionId: first?.sessionId || null,
532
+ source: first?.source || null,
533
+ project: first?.project || 'unknown',
534
+ firstTimestampMs: null,
535
+ lastTimestampMs: null,
536
+ completedActiveSeconds: 0,
537
+ turnStartMs: null,
538
+ turnEndMs: null,
539
+ waitingForFirstResponse: false,
540
+ messageCount: 0,
541
+ userMessageCount: 0,
542
+ userPromptHours: new Array(24).fill(0),
543
+ };
544
+ for (const event of sorted) {
545
+ if (!applySessionEvent(acc, event)) return null;
546
+ }
547
+ return acc.sessionId ? acc : null;
548
+ }
549
+
550
+ function sessionFromAccumulator(acc) {
551
+ if (!acc?.sessionId || acc.firstTimestampMs == null || acc.lastTimestampMs == null) return null;
552
+ let activeSeconds = acc.completedActiveSeconds;
553
+ if (acc.turnStartMs != null && acc.turnEndMs != null && acc.turnEndMs > acc.turnStartMs) {
554
+ activeSeconds += Math.round((acc.turnEndMs - acc.turnStartMs) / 1000);
555
+ }
556
+ return {
557
+ source: acc.source,
558
+ project: acc.project || 'unknown',
559
+ sessionHash: createHash('sha256').update(acc.sessionId).digest('hex').slice(0, 16),
560
+ firstMessageAt: new Date(acc.firstTimestampMs).toISOString(),
561
+ lastMessageAt: new Date(acc.lastTimestampMs).toISOString(),
562
+ durationSeconds: Math.round((acc.lastTimestampMs - acc.firstTimestampMs) / 1000),
563
+ activeSeconds,
564
+ messageCount: acc.messageCount,
565
+ userMessageCount: acc.userMessageCount,
566
+ userPromptHours: acc.userPromptHours,
567
+ };
568
+ }
569
+
570
+ function mergeBucketLists(lists) {
571
+ const entries = [];
572
+ for (const buckets of lists) {
573
+ for (const bucket of buckets || []) {
574
+ entries.push({
575
+ source: bucket.source,
576
+ model: bucket.model,
577
+ project: bucket.project,
578
+ ...(bucket.hostname ? { hostname: bucket.hostname } : {}),
579
+ timestamp: new Date(bucket.bucketStart),
580
+ inputTokens: bucket.inputTokens,
581
+ outputTokens: bucket.outputTokens,
582
+ cachedInputTokens: bucket.cachedInputTokens,
583
+ reasoningOutputTokens: bucket.reasoningOutputTokens,
584
+ });
335
585
  }
336
586
  }
587
+ return aggregateToBuckets(entries);
588
+ }
337
589
 
338
- // Pass 2: parse usage while skipping the replay prefix resolved above.
339
- for (const filePath of files) {
340
- const fm = fileMeta.get(filePath);
341
- if (!fm) continue;
342
- if (fm.sessionId && sessionById.get(fm.sessionId)?.filePath !== filePath) continue;
343
-
344
- const boundary = replayBoundary(fm, sessionById);
345
- let rawTokenSeen = 0;
346
- let parsedRecordIndex = 0;
347
- let firstSessionMetaSeen = false;
348
-
349
- const sessionProject = fm.sessionProject;
350
- // Group timing events by the real Codex session id, not the file path: the
351
- // same session can briefly exist in both sessions/ and archived_sessions/
352
- // (mid-archive, or a re-synced archive). Path-keyed grouping would emit it
353
- // as two different sessionHashes and double-count its session stats. Fall
354
- // back to the path only when the id is unknown (corrupt/missing meta).
355
- const sessionKey = fm.sessionId || filePath;
356
-
357
- let turnContextModel = 'unknown';
358
- let prevTotal = null;
359
- let prevCumulativeTotal = null;
360
- for await (const line of readLines(filePath)) {
361
- if (!line.trim()) continue;
362
- try {
363
- const obj = JSON.parse(line);
364
- parsedRecordIndex++;
590
+ async function parseSessionFile(filePath, snapshotSize, fm, boundary, {
591
+ previousTail = null,
592
+ captureTail = false,
593
+ } = {}) {
594
+ const entries = [];
595
+ const sessionEvents = [];
596
+ let rawTokenSeen = previousTail?.rawTokenSeen || 0;
597
+ let parsedRecordIndex = previousTail?.parsedRecordIndex || 0;
598
+ let firstSessionMetaSeen = previousTail?.firstSessionMetaSeen || false;
599
+
600
+ const sessionProject = fm.sessionProject;
601
+ // Group timing events by the real Codex session id, not the file path: the
602
+ // same session can briefly exist in both sessions/ and archived_sessions/
603
+ // (mid-archive, or a re-synced archive). Path-keyed grouping would emit it
604
+ // as two different sessionHashes and double-count its session stats. Fall
605
+ // back to the path only when the id is unknown (corrupt/missing meta).
606
+ const sessionKey = fm.sessionId || filePath;
607
+
608
+ let turnContextModel = previousTail?.turnContextModel || 'unknown';
609
+ let prevTotal = previousTail?.prevTotal || null;
610
+ let prevCumulativeTotal = previousTail?.prevCumulativeTotal ?? null;
611
+ const start = previousTail?.parsedBytes || 0;
612
+ for await (const line of readLines(filePath, snapshotSize, start)) {
613
+ if (!line.trim()) continue;
614
+ try {
615
+ const obj = JSON.parse(line);
616
+ parsedRecordIndex++;
365
617
 
366
618
  // A direct child task boundary covers every copied record, including
367
619
  // timing/meta events. The raw-token ordinal covers full-history and
368
620
  // last-N-turn forks whose exact payload sequence was matched in pass 1.
369
- const beforeOwnTask = boundary.recordIndex != null
370
- && parsedRecordIndex < boundary.recordIndex;
371
- const inReplayBlock = beforeOwnTask || rawTokenSeen < boundary.rawTokenCount;
372
-
373
- const isSessionMeta = obj.type === 'session_meta';
374
- const isCanonicalSessionMeta = isSessionMeta && !firstSessionMetaSeen;
375
- const isOwnSessionMeta = isSessionMeta
376
- && obj.payload?.id != null
377
- && obj.payload.id === fm.sessionId;
378
- if (isSessionMeta) firstSessionMetaSeen = true;
379
-
380
- if (obj.timestamp) {
381
- const evTs = new Date(obj.timestamp);
382
- if (!isNaN(evTs.getTime())) {
621
+ const beforeOwnTask = boundary.recordIndex != null
622
+ && parsedRecordIndex < boundary.recordIndex;
623
+ const inReplayBlock = beforeOwnTask || rawTokenSeen < boundary.rawTokenCount;
624
+
625
+ const isSessionMeta = obj.type === 'session_meta';
626
+ const isCanonicalSessionMeta = isSessionMeta && !firstSessionMetaSeen;
627
+ const isOwnSessionMeta = isSessionMeta
628
+ && obj.payload?.id != null
629
+ && obj.payload.id === fm.sessionId;
630
+ if (isSessionMeta) firstSessionMetaSeen = true;
631
+
632
+ if (obj.timestamp) {
633
+ const evTs = new Date(obj.timestamp);
634
+ if (!isNaN(evTs.getTime())) {
383
635
  // Repeated same-id metadata can be appended on resume/config
384
636
  // updates and belongs to this logical session. A different-id meta
385
637
  // is copied parent history and must not inflate timing stats.
386
- const keepSessionMeta = isCanonicalSessionMeta
387
- || (isOwnSessionMeta && !inReplayBlock);
388
- if (keepSessionMeta || (!isSessionMeta && !inReplayBlock)) {
389
- const isUserTurn = obj.type === 'turn_context' || obj.type === 'session_meta';
390
- sessionEvents.push({
391
- sessionId: sessionKey,
392
- source: 'codex',
393
- project: sessionProject,
394
- timestamp: evTs,
395
- role: isUserTurn ? 'user' : 'assistant',
396
- });
397
- }
638
+ const keepSessionMeta = isCanonicalSessionMeta
639
+ || (isOwnSessionMeta && !inReplayBlock);
640
+ if (keepSessionMeta || (!isSessionMeta && !inReplayBlock)) {
641
+ const isUserTurn = obj.type === 'turn_context' || obj.type === 'session_meta';
642
+ sessionEvents.push({
643
+ sessionId: sessionKey,
644
+ source: 'codex',
645
+ project: sessionProject,
646
+ timestamp: evTs,
647
+ role: isUserTurn ? 'user' : 'assistant',
648
+ });
398
649
  }
399
650
  }
651
+ }
400
652
 
401
- if (obj.type === 'turn_context' && obj.payload?.model) {
402
- turnContextModel = obj.payload.model;
403
- continue;
404
- }
653
+ if (obj.type === 'turn_context' && obj.payload?.model) {
654
+ turnContextModel = obj.payload.model;
655
+ continue;
656
+ }
405
657
 
406
- if (obj.type !== 'event_msg') continue;
658
+ if (obj.type !== 'event_msg') continue;
407
659
 
408
- const payload = obj.payload;
409
- if (!payload) continue;
660
+ const payload = obj.payload;
661
+ if (!payload) continue;
410
662
 
411
- if (payload.type !== 'token_count') continue;
663
+ if (payload.type !== 'token_count') continue;
412
664
 
413
665
  // Raw ordinals advance before validating usage/timestamp so pass 1 and
414
666
  // pass 2 cannot drift on a malformed copied token_count record.
415
- const isReplayedHistory = inReplayBlock;
416
- rawTokenSeen++;
667
+ const isReplayedHistory = inReplayBlock;
668
+ rawTokenSeen++;
417
669
 
418
- const info = payload.info;
419
- if (!info) continue;
670
+ const info = payload.info;
671
+ if (!info) continue;
420
672
 
421
673
  // Codex sometimes writes the same token_count twice back-to-back:
422
674
  // identical last_token_usage with an unchanged cumulative total. A
@@ -426,64 +678,334 @@ export async function parse() {
426
678
  // compaction — and must count as zero, not a second copy of
427
679
  // last_token_usage. Guarded to positive totals so builds that leave
428
680
  // total_token_usage all-zero can't suppress real usage.
429
- const cumulativeTotal = info.total_token_usage?.total_tokens;
430
- const isDuplicateEmission = typeof cumulativeTotal === 'number'
431
- && cumulativeTotal > 0
432
- && cumulativeTotal === prevCumulativeTotal;
433
- if (typeof cumulativeTotal === 'number') prevCumulativeTotal = cumulativeTotal;
681
+ const cumulativeTotal = info.total_token_usage?.total_tokens;
682
+ const isDuplicateEmission = typeof cumulativeTotal === 'number'
683
+ && cumulativeTotal > 0
684
+ && cumulativeTotal === prevCumulativeTotal;
685
+ if (typeof cumulativeTotal === 'number') prevCumulativeTotal = cumulativeTotal;
434
686
 
435
687
  // Prefer incremental per-request usage; compute delta from cumulative
436
688
  // totals as fallback. Always advance the cumulative baseline, even
437
689
  // when last_token_usage exists or the record belongs to a replay.
438
- const curr = info.total_token_usage;
439
- let usage = info.last_token_usage;
440
- if (!usage && curr) {
441
- if (prevTotal) {
442
- const delta = {
443
- input_tokens: (curr.input_tokens || 0) - (prevTotal.input_tokens || 0),
444
- output_tokens: (curr.output_tokens || 0) - (prevTotal.output_tokens || 0),
445
- cached_input_tokens: (curr.cached_input_tokens || 0) - (prevTotal.cached_input_tokens || 0),
446
- reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prevTotal.reasoning_output_tokens || 0),
447
- };
690
+ const curr = info.total_token_usage;
691
+ let usage = info.last_token_usage;
692
+ if (!usage && curr) {
693
+ if (prevTotal) {
694
+ const delta = {
695
+ input_tokens: (curr.input_tokens || 0) - (prevTotal.input_tokens || 0),
696
+ output_tokens: (curr.output_tokens || 0) - (prevTotal.output_tokens || 0),
697
+ cached_input_tokens: (curr.cached_input_tokens || 0) - (prevTotal.cached_input_tokens || 0),
698
+ reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prevTotal.reasoning_output_tokens || 0),
699
+ };
448
700
  // Cumulative counters can reset after compaction or a new usage
449
701
  // window. Treat the first post-reset total as a fresh baseline;
450
702
  // allowing a negative delta would cancel legitimate bucket usage.
451
- usage = Object.values(delta).some(value => value < 0) ? curr : delta;
452
- } else {
703
+ usage = Object.values(delta).some(value => value < 0) ? curr : delta;
704
+ } else {
453
705
  // First cumulative entry — use as-is (it's the first event's total)
454
- usage = curr;
455
- }
706
+ usage = curr;
456
707
  }
708
+ }
457
709
  // total_token_usage is session-wide, not per model. A global baseline
458
710
  // avoids counting the full cumulative total again after a model switch.
459
- if (curr) prevTotal = { ...curr };
460
- if (!usage) continue;
461
- if (isReplayedHistory || isDuplicateEmission) continue;
711
+ if (curr) prevTotal = { ...curr };
712
+ if (!usage) continue;
713
+ if (isReplayedHistory || isDuplicateEmission) continue;
462
714
 
463
- const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
464
- if (!timestamp || isNaN(timestamp.getTime())) continue;
715
+ const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
716
+ if (!timestamp || isNaN(timestamp.getTime())) continue;
465
717
 
466
- const model = info.model || payload.model || turnContextModel || 'unknown';
718
+ const model = info.model || payload.model || turnContextModel || 'unknown';
467
719
 
468
720
  // OpenAI API: input_tokens INCLUDES cached, output_tokens INCLUDES reasoning.
469
721
  // Normalize to Anthropic-style semantics where each field is non-overlapping.
470
- const cachedInput = usage.cached_input_tokens || usage.cache_read_input_tokens || 0;
471
- const reasoningOutput = usage.reasoning_output_tokens || 0;
472
- entries.push({
473
- source: 'codex',
474
- model,
475
- project: sessionProject,
476
- timestamp,
477
- inputTokens: (usage.input_tokens || 0) - cachedInput,
478
- outputTokens: (usage.output_tokens || 0) - reasoningOutput,
479
- cachedInputTokens: cachedInput,
480
- reasoningOutputTokens: reasoningOutput,
481
- });
722
+ const cachedInput = usage.cached_input_tokens || usage.cache_read_input_tokens || 0;
723
+ const reasoningOutput = usage.reasoning_output_tokens || 0;
724
+ entries.push({
725
+ source: 'codex',
726
+ model,
727
+ project: sessionProject,
728
+ timestamp,
729
+ inputTokens: (usage.input_tokens || 0) - cachedInput,
730
+ outputTokens: (usage.output_tokens || 0) - reasoningOutput,
731
+ cachedInputTokens: cachedInput,
732
+ reasoningOutputTokens: reasoningOutput,
733
+ });
734
+ } catch {
735
+ continue;
736
+ }
737
+ }
738
+
739
+ // Indexed files must match both passes exactly. Ordinary sessions take the
740
+ // single-pass fast path and have no expected counts; their byte-bounded
741
+ // snapshot is still stable, and any append invalidates the stat signature on
742
+ // the next sync.
743
+ if (fm.parsedRecordCount != null && fm.rawTokenCount != null) {
744
+ if (parsedRecordIndex !== fm.parsedRecordCount || rawTokenSeen !== fm.rawTokenCount) {
745
+ throw new Error('Codex rollout changed while syncing; retry on the next sync');
746
+ }
747
+ }
748
+
749
+ const buckets = mergeBucketLists([
750
+ previousTail?.buckets || [],
751
+ aggregateToBuckets(entries),
752
+ ]);
753
+ const sessionAccumulator = buildSessionAccumulator(
754
+ sessionEvents,
755
+ previousTail?.sessionAccumulator || null
756
+ );
757
+ // Appended records should be chronological. If an app version inserts an
758
+ // older event into the tail, the compact accumulator cannot reproduce the
759
+ // global sort exactly, so discard the optimization and rebuild this file.
760
+ if (previousTail && sessionEvents.length > 0 && !sessionAccumulator) {
761
+ return parseSessionFile(filePath, snapshotSize, fm, boundary, { captureTail });
762
+ }
763
+ const session = sessionFromAccumulator(sessionAccumulator);
764
+ const result = { buckets, sessions: session ? [session] : [] };
765
+ if (captureTail) {
766
+ const guard = snapshotGuard(filePath, snapshotSize);
767
+ result.tail = {
768
+ parsedBytes: snapshotSize,
769
+ parsedRecordIndex,
770
+ rawTokenSeen,
771
+ firstSessionMetaSeen,
772
+ turnContextModel,
773
+ prevTotal,
774
+ prevCumulativeTotal,
775
+ buckets,
776
+ sessionAccumulator,
777
+ guardHash: guard.hash,
778
+ endsWithNewline: guard.endsWithNewline,
779
+ };
780
+ }
781
+ return result;
782
+ }
783
+
784
+ function mergeFileResults(results) {
785
+ const entries = [];
786
+ const sessions = [];
787
+ for (const result of results) {
788
+ for (const bucket of result.buckets || []) {
789
+ entries.push({
790
+ source: bucket.source,
791
+ model: bucket.model,
792
+ project: bucket.project,
793
+ ...(bucket.hostname ? { hostname: bucket.hostname } : {}),
794
+ timestamp: new Date(bucket.bucketStart),
795
+ inputTokens: bucket.inputTokens,
796
+ outputTokens: bucket.outputTokens,
797
+ cachedInputTokens: bucket.cachedInputTokens,
798
+ reasoningOutputTokens: bucket.reasoningOutputTokens,
799
+ });
800
+ }
801
+ sessions.push(...(result.sessions || []));
802
+ }
803
+ return { buckets: aggregateToBuckets(entries), sessions };
804
+ }
805
+
806
+ export async function parse() {
807
+ const codexHome = getCodexHome();
808
+ const dirs = sessionsDirs(codexHome);
809
+ if (!dirs.some(existsSync)) return { buckets: [], sessions: [] };
810
+
811
+ const startedAt = Date.now();
812
+ const budget = workBudgetMs();
813
+ const overBudget = () => Date.now() - startedAt >= budget;
814
+ const cacheStats = {
815
+ headerHits: 0,
816
+ indexHits: 0,
817
+ resultHits: 0,
818
+ tailHits: 0,
819
+ filesRead: 0,
820
+ audited: 0,
821
+ };
822
+ const files = [];
823
+ for (const filePath of dirs.flatMap(findJsonlFiles)) {
824
+ try {
825
+ const stat = statSync(filePath);
826
+ if (stat.size <= 0) continue;
827
+ const signature = fileSignature(stat);
828
+ const cache = loadCodexFileCache(codexHome, filePath, signature);
829
+ const priorCache = cache || loadCodexFileCache(codexHome, filePath);
830
+ const priorTail = cache ? null : loadCodexFileTail(codexHome, filePath);
831
+ const file = {
832
+ filePath,
833
+ snapshotSize: stat.size,
834
+ signature,
835
+ cache,
836
+ priorCache,
837
+ priorTail,
838
+ header: null,
839
+ appendTail: null,
840
+ };
841
+ if (!cache && priorCache) file.appendTail = tailStateFor(file);
842
+ files.push(file);
843
+ } catch {
844
+ // The file may move to archived_sessions between discovery and stat.
845
+ // Its archived copy will be picked up on the next sync.
846
+ }
847
+ }
848
+ if (files.length === 0) return { buckets: [], sessions: [] };
849
+
850
+ // A warm cache gets a tiny rolling correctness audit. Never mix this into a
851
+ // cold/resumed build: all files must already have complete results, and at
852
+ // most one bounded file is re-read per invocation.
853
+ const auditPaths = new Set();
854
+ if (files.every(file => file.cache?.header && file.cache?.result)) {
855
+ const cutoff = Date.now() - auditIntervalMs();
856
+ const candidate = files
857
+ .filter(file => file.snapshotSize <= auditMaxBytes())
858
+ .filter(file => (file.cache.lastAuditedAt || 0) <= cutoff)
859
+ .sort((a, b) => (a.cache.lastAuditedAt || 0) - (b.cache.lastAuditedAt || 0))[0];
860
+ if (candidate) auditPaths.add(candidate.filePath);
861
+ }
862
+
863
+ // Cheap discovery: cached headers require no rollout read. On a cold build,
864
+ // read only through the first session_meta so ordinary sessions can avoid
865
+ // the former all-files replay-index pass.
866
+ const candidatesById = new Map();
867
+ for (let i = 0; i < files.length; i++) {
868
+ const file = files[i];
869
+ const reusableHeader = file.cache?.header || (file.appendTail ? file.priorCache?.header : null);
870
+ if (reusableHeader && !auditPaths.has(file.filePath)) {
871
+ file.header = reusableHeader;
872
+ if (!file.cache) file.cache = { header: reusableHeader };
873
+ cacheStats.headerHits++;
874
+ } else {
875
+ try {
876
+ file.header = await readSessionHeader(file.filePath, file.snapshotSize);
877
+ cacheStats.filesRead++;
878
+ updateFileCache(codexHome, file, { header: file.header });
482
879
  } catch {
483
880
  continue;
484
881
  }
485
882
  }
883
+ if (file.header.sessionId) {
884
+ if (!candidatesById.has(file.header.sessionId)) candidatesById.set(file.header.sessionId, []);
885
+ candidatesById.get(file.header.sessionId).push(file);
886
+ }
887
+ if (overBudget() && i < files.length - 1) {
888
+ return {
889
+ buckets: [], sessions: [], skipped: true,
890
+ indexing: { phase: 'discovery', completed: i + 1, total: files.length },
891
+ cache: cacheStats,
892
+ };
893
+ }
894
+ }
895
+
896
+ const duplicateIds = new Set(
897
+ [...candidatesById].filter(([, candidates]) => candidates.length > 1).map(([id]) => id)
898
+ );
899
+ const referencedParentIds = new Set();
900
+ for (const file of files) {
901
+ const parentId = file.header?.forkedFromId
902
+ || (file.header?.isSubagent ? file.header.parentThreadId : null);
903
+ if (parentId) referencedParentIds.add(parentId);
904
+ }
905
+
906
+ // Only replay participants, their parents, corrupt-header files, and
907
+ // duplicate physical copies need the full compact token index.
908
+ const fileMeta = new Map();
909
+ const needsIndex = new Set();
910
+ for (const file of files) {
911
+ const header = file.header;
912
+ const required = !header
913
+ || !header.sessionId
914
+ || header.isSubagent
915
+ || header.forkedFromId != null
916
+ || header.parentThreadId != null
917
+ || referencedParentIds.has(header.sessionId)
918
+ || duplicateIds.has(header.sessionId);
919
+ if (!required) {
920
+ fileMeta.set(file.filePath, { ...header, filePath: file.filePath });
921
+ continue;
922
+ }
923
+ needsIndex.add(file.filePath);
924
+ let meta = auditPaths.has(file.filePath) ? null : file.cache?.index;
925
+ if (meta) {
926
+ cacheStats.indexHits++;
927
+ } else {
928
+ try {
929
+ meta = await indexSessionFile(file.filePath, file.snapshotSize);
930
+ cacheStats.filesRead++;
931
+ updateFileCache(codexHome, file, { index: meta });
932
+ } catch {
933
+ continue;
934
+ }
935
+ }
936
+ fileMeta.set(file.filePath, meta);
937
+ if (overBudget()) {
938
+ return {
939
+ buckets: [], sessions: [], skipped: true,
940
+ indexing: { phase: 'replay-index', completed: fileMeta.size, total: files.length },
941
+ cache: cacheStats,
942
+ };
943
+ }
944
+ }
945
+
946
+ // Select the most complete physical copy exactly as before. Unique ordinary
947
+ // sessions have no full record count, but cannot compete with another copy.
948
+ const sessionById = new Map();
949
+ for (const file of files) {
950
+ const meta = fileMeta.get(file.filePath);
951
+ if (!meta?.sessionId) continue;
952
+ const existing = sessionById.get(meta.sessionId);
953
+ const count = meta.parsedRecordCount ?? 0;
954
+ const existingCount = existing?.parsedRecordCount ?? 0;
955
+ if (!existing || count > existingCount) sessionById.set(meta.sessionId, meta);
956
+ }
957
+
958
+ const results = [];
959
+ for (let i = 0; i < files.length; i++) {
960
+ const file = files[i];
961
+ const fm = fileMeta.get(file.filePath);
962
+ if (!fm) continue;
963
+ if (fm.sessionId && sessionById.get(fm.sessionId)?.filePath !== file.filePath) continue;
964
+
965
+ const boundary = needsIndex.has(file.filePath)
966
+ ? replayBoundary(fm, sessionById)
967
+ : { rawTokenCount: 0, recordIndex: null };
968
+ const key = boundaryKey(boundary);
969
+ let result = !auditPaths.has(file.filePath) && file.cache?.result?.boundaryKey === key
970
+ ? file.cache.result
971
+ : null;
972
+ if (result) {
973
+ cacheStats.resultHits++;
974
+ } else {
975
+ const previousTail = !needsIndex.has(file.filePath) && !auditPaths.has(file.filePath)
976
+ ? file.appendTail
977
+ : null;
978
+ const parsed = await parseSessionFile(file.filePath, file.snapshotSize, fm, boundary, {
979
+ previousTail,
980
+ captureTail: !needsIndex.has(file.filePath),
981
+ });
982
+ cacheStats.filesRead++;
983
+ if (previousTail) cacheStats.tailHits++;
984
+ const { tail, ...summary } = parsed;
985
+ if (tail) {
986
+ try {
987
+ saveCodexFileTail(codexHome, file.filePath, file.signature, tail);
988
+ } catch {
989
+ // Same fail-open rule as the summary cache: tail acceleration is
990
+ // optional and a write failure must not fail the parser.
991
+ }
992
+ }
993
+ result = { boundaryKey: key, ...summary };
994
+ updateFileCache(codexHome, file, { result, lastAuditedAt: Date.now() });
995
+ file.appendTail = null;
996
+ file.priorTail = null;
997
+ if (auditPaths.has(file.filePath)) cacheStats.audited++;
998
+ }
999
+ results.push(result);
1000
+
1001
+ if (overBudget() && i < files.length - 1) {
1002
+ return {
1003
+ buckets: [], sessions: [], skipped: true,
1004
+ indexing: { phase: 'usage', completed: i + 1, total: files.length },
1005
+ cache: cacheStats,
1006
+ };
1007
+ }
486
1008
  }
487
1009
 
488
- return { buckets: aggregateToBuckets(entries), sessions: extractSessions(sessionEvents) };
1010
+ return { ...mergeFileResults(results), cache: cacheStats };
489
1011
  }
package/src/sync.js CHANGED
@@ -34,6 +34,7 @@ export async function runSync({ throws = false, quiet = false } = {}) {
34
34
  const allBuckets = [];
35
35
  const allSessions = [];
36
36
  const parserResults = [];
37
+ const parserProgress = [];
37
38
  // Sources whose parser ran to completion this sync. pruneState() below is
38
39
  // scoped to these so a transient parser failure doesn't evict that tool's
39
40
  // state and force a full re-upload next run.
@@ -47,6 +48,9 @@ export async function runSync({ throws = false, quiet = false } = {}) {
47
48
  if (!Array.isArray(buckets) || !Array.isArray(sessions)) {
48
49
  throw new TypeError('Parser returned an invalid result');
49
50
  }
51
+ if (result?.indexing) {
52
+ parserProgress.push({ source, ...result.indexing });
53
+ }
50
54
  // A parser may deliberately suppress a transient error (Cursor network
51
55
  // timeout) to keep daemon logs quiet. Its empty result is not proof that
52
56
  // its prior data disappeared, so it must not be pruned this run.
@@ -71,7 +75,13 @@ export async function runSync({ throws = false, quiet = false } = {}) {
71
75
  pruneState(state, new Set(), new Set(), okSources);
72
76
  const pruned = before - (Object.keys(state.buckets).length + Object.keys(state.sessions).length);
73
77
  if (pruned > 0) saveState(state);
74
- if (!quiet) console.log(dim('暂无新数据。'));
78
+ if (!quiet && parserProgress.length > 0) {
79
+ for (const p of parserProgress) {
80
+ console.log(dim(` ${p.source}: 正在建立本地索引 ${p.completed}/${p.total}(下次同步继续)`));
81
+ }
82
+ } else if (!quiet) {
83
+ console.log(dim('暂无新数据。'));
84
+ }
75
85
  return 0;
76
86
  }
77
87
 
@@ -83,6 +93,11 @@ export async function runSync({ throws = false, quiet = false } = {}) {
83
93
  console.log(` ${dim(p.source.padEnd(14))}${parts.join(' · ')}`);
84
94
  }
85
95
  }
96
+ if (!quiet && parserProgress.length > 0) {
97
+ for (const p of parserProgress) {
98
+ console.log(dim(` ${p.source}: 正在建立本地索引 ${p.completed}/${p.total}(下次同步继续)`));
99
+ }
100
+ }
86
101
 
87
102
  let host = config.hostname;
88
103
  if (!host) {
@@ -112,10 +127,11 @@ export async function runSync({ throws = false, quiet = false } = {}) {
112
127
  for (const s of allSessions) s.project = 'unknown';
113
128
  }
114
129
 
115
- // Incremental diff: parsers above always read the full local history (cheap,
116
- // local-only). Here we drop anything whose content matches what we already
117
- // uploaded, so only new/changed items go over the network. A quiet machine
118
- // sends zero bytes; an active one sends just the current 30-min bucket.
130
+ // Incremental upload diff: parsers above emit a complete view of live local
131
+ // data (Codex may assemble that view from its disposable parser cache). Here
132
+ // we drop anything whose content matches what we already uploaded, so only
133
+ // new/changed items go over the network. A quiet machine sends zero bytes;
134
+ // an active one sends just the current 30-min bucket.
119
135
  // Missing/corrupt state.json => empty maps => one-time full upload, then
120
136
  // incremental forever after.
121
137
  const state = loadState();