@vibe-cafe/vibe-usage 0.9.16 → 0.10.0

package/README.md

@@ -51,7 +51,7 @@ npx @vibe-cafe/vibe-usage status       # Show config & detected tools
 | Tool | Data Location |
 |------|---------------|
 | Claude Code | `~/.claude/projects/` (tokens + sessions), `~/.claude/transcripts/` (sessions only); also scans `$CLAUDE_CONFIG_DIR` when set (deduped), so relocated configs and GUI/CLI env mismatches are both covered |
-| Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`); forked and sub-agent rollouts replay parent metadata, tasks, and `token_count` records at spawn time — full or last-N-turn replay blocks are matched as a fingerprinted parent suffix plus the child's own task boundary, duplicate cumulative emissions count once, and live/archive copies of the same session are deduplicated |
+| Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`); a versioned local index avoids re-reading unchanged rollouts and reads only safe append tails for ordinary sessions, while fork/sub-agent replay matching, duplicate suppression, and live/archive deduplication retain their existing semantics |
 | Grok | `$GROK_HOME/sessions/<encoded-cwd>/<session-id>/` (default `~/.grok`); token usage from `updates.jsonl` `turn_completed.usage` (per-model `modelUsage`, cache reads, reasoning); project from `summary.json` cwd; honors `GROK_HOME` |
 | GitHub Copilot CLI | `~/.copilot/session-state/*/events.jsonl` |
 | Cursor | `state.vscdb` (SQLite, reads `cursorAuth/accessToken`, fetches CSV from `cursor.com`); cloud data is stamped with a fixed `cursor-cloud` hostname so multi-machine setups don't double-count |
@@ -77,7 +77,9 @@ npx @vibe-cafe/vibe-usage status       # Show config & detected tools
 - Aggregates token usage into 30-minute buckets
 - Extracts session metadata from all parsers: active time (AI generation time, excluding queue/TTFT wait), total duration, message counts
 - Uploads buckets + sessions to your vibecafe.ai dashboard (always gzip-compressed, ~94% smaller)
-- Incremental: parsers still compute full totals from local logs each sync (idempotent), but only buckets/sessions that are new or changed since the last successful upload are sent — a quiet machine uploads nothing. Sync state is kept in `~/.vibe-usage/state.json`; failed parsers retain their prior state, while deleted local logs are pruned. Deleting the state file triggers a one-time full re-upload, and `reset` clears it automatically after deleting cloud data
+- Incremental upload: every parser emits a complete local snapshot, then only buckets/sessions that are new or changed since the last successful upload are sent — a quiet machine uploads nothing. Upload state remains in `~/.vibe-usage/state.json`; failed or still-indexing parsers retain their prior state, while deleted local logs are pruned. Deleting the state file triggers a one-time full re-upload, and `reset` clears it automatically after deleting cloud data
+- Incremental Codex parsing: a versioned, disposable cache under `~/.vibe-usage/cache/codex/` stores per-rollout aggregate results and parser continuation state. Unchanged rollouts require no raw-log reads; an ordinary append reads only the new tail; forks, sub-agents, replacements, truncations, and failed safety checks fall back to the full correctness path. A bounded rolling audit occasionally re-reads one historical file. Very large first-time indexes checkpoint before the Mac app timeout and resume on the next sync instead of restarting
+- The Codex parser cache contains derived aggregates and replay metadata, not raw prompt or response text. It is independent of upload state and can be deleted safely (the next sync rebuilds it). `reset` intentionally keeps it so the required full re-upload does not also require a full disk rescan. Set `VIBE_USAGE_CODEX_CACHE=0` to disable the optimization for diagnosis
 - SQLite-backed tools (Cursor, OpenCode, Kiro, Hermes) are read via Node's built-in `node:sqlite` on Node ≥ 22.5 — no `sqlite3` binary needed (works on Windows out of the box); on older Node it falls back to the system `sqlite3` CLI
 - For continuous syncing, use `npx @vibe-cafe/vibe-usage daemon` or the [Vibe Usage Mac app](https://github.com/vibe-cafe/vibe-usage-app)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vibe-cafe/vibe-usage",
-  "version": "0.9.16",
+  "version": "0.10.0",
   "description": "Track your AI coding tool token usage and sync to vibecafe.ai",
   "type": "module",
   "scripts": {

package/src/parsers/codex-cache.js

@@ -0,0 +1,138 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+// Parser caches are disposable derived data. Keep their schema/version fully
+// separate from ~/.vibe-usage/state.json, whose hashes are the authoritative
+// record of successful uploads and must remain backward-compatible.
+export const CODEX_CACHE_SCHEMA_VERSION = 1;
+export const CODEX_PARSER_ALGORITHM_VERSION = 1;
+
+function hash(value, length = 24) {
+  return createHash('sha256').update(value).digest('hex').slice(0, length);
+}
+
+export function codexCacheEnabled() {
+  return process.env.VIBE_USAGE_CODEX_CACHE !== '0';
+}
+
+export function fileSignature(stat) {
+  return {
+    size: stat.size,
+    mtimeMs: stat.mtimeMs,
+    dev: String(stat.dev),
+    ino: String(stat.ino),
+  };
+}
+
+function sameSignature(a, b) {
+  return a?.size === b.size
+    && a?.mtimeMs === b.mtimeMs
+    && a?.dev === b.dev
+    && a?.ino === b.ino;
+}
+
+export function codexCacheDir(codexHome) {
+  const base = process.env.VIBE_USAGE_CACHE_DIR?.trim()
+    || join(homedir(), '.vibe-usage', 'cache');
+  return join(base, 'codex', `root-${hash(codexHome)}`);
+}
+
+function entryPath(codexHome, filePath) {
+  return join(codexCacheDir(codexHome), `${hash(filePath, 32)}.json`);
+}
+
+function tailPath(codexHome, filePath) {
+  return join(codexCacheDir(codexHome), `${hash(filePath, 32)}.tail.json`);
+}
+
+export function loadCodexFileCache(codexHome, filePath, signature) {
+  if (!codexCacheEnabled()) return null;
+  const path = entryPath(codexHome, filePath);
+  if (!existsSync(path)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    if (parsed.schemaVersion !== CODEX_CACHE_SCHEMA_VERSION) return null;
+    if (parsed.algorithmVersion !== CODEX_PARSER_ALGORITHM_VERSION) return null;
+    if (parsed.filePath !== filePath) return null;
+    if (signature && !sameSignature(parsed.signature, signature)) return null;
+    return parsed;
+  } catch {
+    // Cache corruption is a performance miss, never a correctness failure.
+    return null;
+  }
+}
+
+export function saveCodexFileCache(codexHome, filePath, signature, data) {
+  if (!codexCacheEnabled()) return;
+  const dir = codexCacheDir(codexHome);
+  mkdirSync(dir, { recursive: true, mode: 0o700 });
+  const path = entryPath(codexHome, filePath);
+  const tempPath = `${path}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`;
+  const payload = {
+    schemaVersion: CODEX_CACHE_SCHEMA_VERSION,
+    algorithmVersion: CODEX_PARSER_ALGORITHM_VERSION,
+    filePath,
+    signature,
+    ...data,
+  };
+  try {
+    writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
+    renameSync(tempPath, path);
+  } finally {
+    // A killed writer can leave a unique temp file; a normal failed writer
+    // should not. rmSync is safe here because the target is this call's exact,
+    // random temporary path inside the versioned cache directory.
+    rmSync(tempPath, { force: true });
+  }
+}
+
+export function loadCodexFileTail(codexHome, filePath) {
+  if (!codexCacheEnabled()) return null;
+  const path = tailPath(codexHome, filePath);
+  if (!existsSync(path)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    if (parsed.schemaVersion !== CODEX_CACHE_SCHEMA_VERSION) return null;
+    if (parsed.algorithmVersion !== CODEX_PARSER_ALGORITHM_VERSION) return null;
+    if (parsed.filePath !== filePath || !parsed.signature || !parsed.tail) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+export function saveCodexFileTail(codexHome, filePath, signature, tail) {
+  if (!codexCacheEnabled() || !tail) return;
+  const dir = codexCacheDir(codexHome);
+  mkdirSync(dir, { recursive: true, mode: 0o700 });
+  const path = tailPath(codexHome, filePath);
+  const tempPath = `${path}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`;
+  const payload = {
+    schemaVersion: CODEX_CACHE_SCHEMA_VERSION,
+    algorithmVersion: CODEX_PARSER_ALGORITHM_VERSION,
+    filePath,
+    signature,
+    tail,
+  };
+  try {
+    writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
+    renameSync(tempPath, path);
+  } finally {
+    rmSync(tempPath, { force: true });
+  }
+}
+
+export function removeCodexFileCache(codexHome, filePath) {
+  if (!codexCacheEnabled()) return;
+  rmSync(entryPath(codexHome, filePath), { force: true });
+  rmSync(tailPath(codexHome, filePath), { force: true });
+}

package/src/parsers/codex.js

@@ -1,9 +1,25 @@
-import { createReadStream, readdirSync, existsSync, statSync } from 'node:fs';
+import {
+  closeSync,
+  createReadStream,
+  existsSync,
+  openSync,
+  readSync,
+  readdirSync,
+  statSync,
+} from 'node:fs';
 import { join } from 'node:path';
 import { homedir } from 'node:os';
 import { createInterface } from 'node:readline';
 import { createHash } from 'node:crypto';
-import { aggregateToBuckets, extractSessions } from './index.js';
+import { aggregateToBuckets } from './index.js';
+import {
+  codexCacheEnabled,
+  fileSignature,
+  loadCodexFileCache,
+  loadCodexFileTail,
+  saveCodexFileCache,
+  saveCodexFileTail,
+} from './codex-cache.js';
 
 // Codex stores live sessions in $CODEX_HOME/sessions (default ~/.codex) and,
 // once a session is "completed", moves its rollout file verbatim into
@@ -12,8 +28,11 @@ import { aggregateToBuckets, extractSessions } from './index.js';
 // both, index them together so fork replay-skip works across directories, and
 // select the most complete physical file when the same session briefly exists
 // in both locations during an archive move.
-function sessionsDirs() {
-  const codexHome = process.env.CODEX_HOME?.trim() || join(homedir(), '.codex');
+function getCodexHome() {
+  return process.env.CODEX_HOME?.trim() || join(homedir(), '.codex');
+}
+
+function sessionsDirs(codexHome) {
   return [
     join(codexHome, 'sessions'),
     join(codexHome, 'archived_sessions'),
@@ -42,13 +61,14 @@ function findJsonlFiles(dir) {
   return results;
 }
 
-function readLines(filePath, snapshotSize) {
+function readLines(filePath, snapshotSize, start = 0) {
   return createInterface({
     input: createReadStream(filePath, {
       encoding: 'utf-8',
       // Rollouts are append-only while Codex is working. Bound both parser
       // passes to the size captured before pass 1 so they see the same prefix
       // even when the live file grows between reads.
+      ...(start > 0 ? { start } : {}),
       ...(snapshotSize == null ? {} : { end: snapshotSize - 1 }),
     }),
     crlfDelay: Infinity,
@@ -85,6 +105,84 @@ function extractParentThreadId(meta) {
     || null;
 }
 
+/**
+ * Read only far enough to find the canonical (first) session_meta. This cheap
+ * discovery pass lets ordinary sessions skip the old full-file index pass;
+ * only forks, sub-agents, and parents referenced by them need replay indexes.
+ */
+async function readSessionHeader(filePath, snapshotSize) {
+  for await (const line of readLines(filePath, snapshotSize)) {
+    if (!line.trim()) continue;
+    try {
+      const obj = JSON.parse(line);
+      if (obj.type !== 'session_meta' || !obj.payload) continue;
+      const meta = obj.payload;
+      return {
+        sessionId: meta.id || null,
+        forkedFromId: meta.forked_from_id || null,
+        parentThreadId: extractParentThreadId(meta),
+        sessionProject: extractProject(meta),
+        sessionStartedAtMs: timestampMs(meta.timestamp) ?? timestampMs(obj.timestamp),
+        isSubagent: isSubagentMeta(meta),
+      };
+    } catch {
+      continue;
+    }
+  }
+  return {
+    sessionId: null,
+    forkedFromId: null,
+    parentThreadId: null,
+    sessionProject: 'unknown',
+    sessionStartedAtMs: null,
+    isSubagent: false,
+  };
+}
+
+function cacheData(cache, changes = {}) {
+  return {
+    header: changes.header ?? cache?.header ?? null,
+    index: changes.index ?? cache?.index ?? null,
+    result: changes.result ?? cache?.result ?? null,
+    lastAuditedAt: changes.lastAuditedAt ?? cache?.lastAuditedAt ?? null,
+  };
+}
+
+const TAIL_GUARD_BYTES = 4096;
+
+function snapshotGuard(filePath, size) {
+  if (size <= 0) return { hash: null, endsWithNewline: false };
+  const length = Math.min(size, TAIL_GUARD_BYTES);
+  const buffer = Buffer.allocUnsafe(length);
+  const fd = openSync(filePath, 'r');
+  try {
+    const read = readSync(fd, buffer, 0, length, size - length);
+    const slice = buffer.subarray(0, read);
+    return {
+      hash: createHash('sha256').update(slice).digest('base64url').slice(0, 20),
+      endsWithNewline: slice.at(-1) === 0x0a,
+    };
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function tailStateFor(file) {
+  const prior = file.priorTail;
+  const tail = prior?.tail;
+  if (!tail || !prior.signature) return null;
+  if (prior.signature.size <= 0 || prior.signature.size >= file.signature.size) return null;
+  if (prior.signature.dev !== file.signature.dev || prior.signature.ino !== file.signature.ino) return null;
+  if (prior.signature.mtimeMs > file.signature.mtimeMs) return null;
+  if (tail.parsedBytes !== prior.signature.size || !tail.endsWithNewline || !tail.guardHash) return null;
+  try {
+    const guard = snapshotGuard(file.filePath, prior.signature.size);
+    return guard.hash === tail.guardHash ? tail : null;
+  } catch {
+    return null;
+  }
+}
+
 function timestampMs(value) {
   if (value == null || value === '') return null;
   const n = new Date(value).getTime();
@@ -354,127 +452,223 @@ function replayBoundary(meta, sessionById) {
   return { rawTokenCount: 0, recordIndex: null };
 }
 
-export async function parse() {
-  const dirs = sessionsDirs();
-  if (!dirs.some(existsSync)) return { buckets: [], sessions: [] };
+function boundaryKey(boundary) {
+  return `${boundary.rawTokenCount}:${boundary.recordIndex ?? ''}`;
+}
 
-  const entries = [];
-  const sessionEvents = [];
-  const files = [];
-  for (const filePath of dirs.flatMap(findJsonlFiles)) {
-    try {
-      const snapshotSize = statSync(filePath).size;
-      if (snapshotSize > 0) files.push({ filePath, snapshotSize });
-    } catch {
-      // The file may move to archived_sessions between discovery and stat.
-      // Its archived copy will be picked up on the next sync.
-    }
+function updateFileCache(codexHome, file, changes) {
+  const data = cacheData(file.cache, changes);
+  try {
+    saveCodexFileCache(codexHome, file.filePath, file.signature, data);
+  } catch {
+    // A read-only home, full disk, or antivirus race must only disable the
+    // optimization for this run. Raw-log parsing remains the source of truth.
   }
-  if (files.length === 0) return { buckets: [], sessions: [] };
+  file.cache = { ...(file.cache || {}), ...data };
+}
 
-  // Pass 1: build a compact per-file index. Keep the most complete physical
-  // copy for each logical session id so a rollout briefly present in both
-  // sessions/ and archived_sessions/ cannot double its token buckets.
-  const sessionById = new Map();
-  const fileMeta = new Map();
-  for (const { filePath, snapshotSize } of files) {
-    let meta;
-    try {
-      meta = await indexSessionFile(filePath, snapshotSize);
-    } catch {
-      continue;
+function workBudgetMs() {
+  const configured = Number(process.env.VIBE_USAGE_CODEX_WORK_BUDGET_MS);
+  if (Number.isFinite(configured) && configured > 0) return configured;
+  // The macOS app terminates its child after 120 seconds. Cache-building work
+  // in a non-interactive child therefore checkpoints before that wall so the
+  // next invocation resumes instead of starting from zero. Interactive users
+  // can let a cold build finish in one run (and can interrupt it safely).
+  if (codexCacheEnabled() && !process.stdout.isTTY) return 105_000;
+  return Number.POSITIVE_INFINITY;
+}
+
+function auditIntervalMs() {
+  const configured = Number(process.env.VIBE_USAGE_CODEX_AUDIT_INTERVAL_MS);
+  if (Number.isFinite(configured) && configured >= 0) return configured;
+  return 30 * 24 * 60 * 60 * 1000;
+}
+
+function auditMaxBytes() {
+  const configured = Number(process.env.VIBE_USAGE_CODEX_AUDIT_MAX_BYTES);
+  if (Number.isFinite(configured) && configured > 0) return configured;
+  // Keep the background audit bounded below the app's wall timeout. Larger
+  // active files are still invalidated immediately by their stat signature,
+  // and every cache generation is rebuilt after parser-algorithm changes.
+  return 64 * 1024 * 1024;
+}
+
+function applySessionEvent(acc, event) {
+  const timestampMsValue = event.timestamp.getTime();
+  if (acc.lastTimestampMs != null && timestampMsValue < acc.lastTimestampMs) return false;
+  acc.firstTimestampMs ??= timestampMsValue;
+  acc.lastTimestampMs = timestampMsValue;
+  acc.messageCount++;
+
+  if (event.role === 'user') {
+    if (acc.turnStartMs != null && acc.turnEndMs != null && acc.turnEndMs > acc.turnStartMs) {
+      acc.completedActiveSeconds += Math.round((acc.turnEndMs - acc.turnStartMs) / 1000);
     }
-    fileMeta.set(filePath, meta);
-    if (meta.sessionId) {
-      const existing = sessionById.get(meta.sessionId);
-      if (!existing || meta.parsedRecordCount > existing.parsedRecordCount) {
-        sessionById.set(meta.sessionId, meta);
+    acc.turnStartMs = null;
+    acc.turnEndMs = null;
+    acc.waitingForFirstResponse = true;
+    acc.userMessageCount++;
+    acc.userPromptHours[event.timestamp.getUTCHours()]++;
+  } else if (acc.waitingForFirstResponse) {
+    acc.turnStartMs = timestampMsValue;
+    acc.turnEndMs = timestampMsValue;
+    acc.waitingForFirstResponse = false;
+  } else if (acc.turnStartMs != null) {
+    acc.turnEndMs = timestampMsValue;
+  }
+  return true;
+}
+
+function buildSessionAccumulator(events, previous = null) {
+  const sorted = [...events].sort((a, b) => a.timestamp - b.timestamp);
+  const first = sorted[0];
+  const acc = previous
+    ? {
+        ...previous,
+        userPromptHours: [...previous.userPromptHours],
       }
+    : {
+        sessionId: first?.sessionId || null,
+        source: first?.source || null,
+        project: first?.project || 'unknown',
+        firstTimestampMs: null,
+        lastTimestampMs: null,
+        completedActiveSeconds: 0,
+        turnStartMs: null,
+        turnEndMs: null,
+        waitingForFirstResponse: false,
+        messageCount: 0,
+        userMessageCount: 0,
+        userPromptHours: new Array(24).fill(0),
+      };
+  for (const event of sorted) {
+    if (!applySessionEvent(acc, event)) return null;
+  }
+  return acc.sessionId ? acc : null;
+}
+
+function sessionFromAccumulator(acc) {
+  if (!acc?.sessionId || acc.firstTimestampMs == null || acc.lastTimestampMs == null) return null;
+  let activeSeconds = acc.completedActiveSeconds;
+  if (acc.turnStartMs != null && acc.turnEndMs != null && acc.turnEndMs > acc.turnStartMs) {
+    activeSeconds += Math.round((acc.turnEndMs - acc.turnStartMs) / 1000);
+  }
+  return {
+    source: acc.source,
+    project: acc.project || 'unknown',
+    sessionHash: createHash('sha256').update(acc.sessionId).digest('hex').slice(0, 16),
+    firstMessageAt: new Date(acc.firstTimestampMs).toISOString(),
+    lastMessageAt: new Date(acc.lastTimestampMs).toISOString(),
+    durationSeconds: Math.round((acc.lastTimestampMs - acc.firstTimestampMs) / 1000),
+    activeSeconds,
+    messageCount: acc.messageCount,
+    userMessageCount: acc.userMessageCount,
+    userPromptHours: acc.userPromptHours,
+  };
+}
+
+function mergeBucketLists(lists) {
+  const entries = [];
+  for (const buckets of lists) {
+    for (const bucket of buckets || []) {
+      entries.push({
+        source: bucket.source,
+        model: bucket.model,
+        project: bucket.project,
+        ...(bucket.hostname ? { hostname: bucket.hostname } : {}),
+        timestamp: new Date(bucket.bucketStart),
+        inputTokens: bucket.inputTokens,
+        outputTokens: bucket.outputTokens,
+        cachedInputTokens: bucket.cachedInputTokens,
+        reasoningOutputTokens: bucket.reasoningOutputTokens,
+      });
     }
   }
+  return aggregateToBuckets(entries);
+}
 
-  // Pass 2: parse usage while skipping the replay prefix resolved above.
-  for (const { filePath, snapshotSize } of files) {
-    const fm = fileMeta.get(filePath);
-    if (!fm) continue;
-    if (fm.sessionId && sessionById.get(fm.sessionId)?.filePath !== filePath) continue;
-
-    const boundary = replayBoundary(fm, sessionById);
-    let rawTokenSeen = 0;
-    let parsedRecordIndex = 0;
-    let firstSessionMetaSeen = false;
-
-    const sessionProject = fm.sessionProject;
-    // Group timing events by the real Codex session id, not the file path: the
-    // same session can briefly exist in both sessions/ and archived_sessions/
-    // (mid-archive, or a re-synced archive). Path-keyed grouping would emit it
-    // as two different sessionHashes and double-count its session stats. Fall
-    // back to the path only when the id is unknown (corrupt/missing meta).
-    const sessionKey = fm.sessionId || filePath;
-
-    let turnContextModel = 'unknown';
-    let prevTotal = null;
-    let prevCumulativeTotal = null;
-    for await (const line of readLines(filePath, snapshotSize)) {
-      if (!line.trim()) continue;
-      try {
-        const obj = JSON.parse(line);
-        parsedRecordIndex++;
+async function parseSessionFile(filePath, snapshotSize, fm, boundary, {
+  previousTail = null,
+  captureTail = false,
+} = {}) {
+  const entries = [];
+  const sessionEvents = [];
+  let rawTokenSeen = previousTail?.rawTokenSeen || 0;
+  let parsedRecordIndex = previousTail?.parsedRecordIndex || 0;
+  let firstSessionMetaSeen = previousTail?.firstSessionMetaSeen || false;
+
+  const sessionProject = fm.sessionProject;
+  // Group timing events by the real Codex session id, not the file path: the
+  // same session can briefly exist in both sessions/ and archived_sessions/
+  // (mid-archive, or a re-synced archive). Path-keyed grouping would emit it
+  // as two different sessionHashes and double-count its session stats. Fall
+  // back to the path only when the id is unknown (corrupt/missing meta).
+  const sessionKey = fm.sessionId || filePath;
+
+  let turnContextModel = previousTail?.turnContextModel || 'unknown';
+  let prevTotal = previousTail?.prevTotal || null;
+  let prevCumulativeTotal = previousTail?.prevCumulativeTotal ?? null;
+  const start = previousTail?.parsedBytes || 0;
+  for await (const line of readLines(filePath, snapshotSize, start)) {
+    if (!line.trim()) continue;
+    try {
+      const obj = JSON.parse(line);
+      parsedRecordIndex++;
 
         // A direct child task boundary covers every copied record, including
         // timing/meta events. The raw-token ordinal covers full-history and
         // last-N-turn forks whose exact payload sequence was matched in pass 1.
-        const beforeOwnTask = boundary.recordIndex != null
-          && parsedRecordIndex < boundary.recordIndex;
-        const inReplayBlock = beforeOwnTask || rawTokenSeen < boundary.rawTokenCount;
-
-        const isSessionMeta = obj.type === 'session_meta';
-        const isCanonicalSessionMeta = isSessionMeta && !firstSessionMetaSeen;
-        const isOwnSessionMeta = isSessionMeta
-          && obj.payload?.id != null
-          && obj.payload.id === fm.sessionId;
-        if (isSessionMeta) firstSessionMetaSeen = true;
-
-        if (obj.timestamp) {
-          const evTs = new Date(obj.timestamp);
-          if (!isNaN(evTs.getTime())) {
+      const beforeOwnTask = boundary.recordIndex != null
+        && parsedRecordIndex < boundary.recordIndex;
+      const inReplayBlock = beforeOwnTask || rawTokenSeen < boundary.rawTokenCount;
+
+      const isSessionMeta = obj.type === 'session_meta';
+      const isCanonicalSessionMeta = isSessionMeta && !firstSessionMetaSeen;
+      const isOwnSessionMeta = isSessionMeta
+        && obj.payload?.id != null
+        && obj.payload.id === fm.sessionId;
+      if (isSessionMeta) firstSessionMetaSeen = true;
+
+      if (obj.timestamp) {
+        const evTs = new Date(obj.timestamp);
+        if (!isNaN(evTs.getTime())) {
             // Repeated same-id metadata can be appended on resume/config
             // updates and belongs to this logical session. A different-id meta
             // is copied parent history and must not inflate timing stats.
-            const keepSessionMeta = isCanonicalSessionMeta
-              || (isOwnSessionMeta && !inReplayBlock);
-            if (keepSessionMeta || (!isSessionMeta && !inReplayBlock)) {
-              const isUserTurn = obj.type === 'turn_context' || obj.type === 'session_meta';
-              sessionEvents.push({
-                sessionId: sessionKey,
-                source: 'codex',
-                project: sessionProject,
-                timestamp: evTs,
-                role: isUserTurn ? 'user' : 'assistant',
-              });
-            }
+          const keepSessionMeta = isCanonicalSessionMeta
+            || (isOwnSessionMeta && !inReplayBlock);
+          if (keepSessionMeta || (!isSessionMeta && !inReplayBlock)) {
+            const isUserTurn = obj.type === 'turn_context' || obj.type === 'session_meta';
+            sessionEvents.push({
+              sessionId: sessionKey,
+              source: 'codex',
+              project: sessionProject,
+              timestamp: evTs,
+              role: isUserTurn ? 'user' : 'assistant',
+            });
           }
         }
+      }
 
-        if (obj.type === 'turn_context' && obj.payload?.model) {
-          turnContextModel = obj.payload.model;
-          continue;
-        }
+      if (obj.type === 'turn_context' && obj.payload?.model) {
+        turnContextModel = obj.payload.model;
+        continue;
+      }
 
-        if (obj.type !== 'event_msg') continue;
+      if (obj.type !== 'event_msg') continue;
 
-        const payload = obj.payload;
-        if (!payload) continue;
+      const payload = obj.payload;
+      if (!payload) continue;
 
-        if (payload.type !== 'token_count') continue;
+      if (payload.type !== 'token_count') continue;
 
         // Raw ordinals advance before validating usage/timestamp so pass 1 and
         // pass 2 cannot drift on a malformed copied token_count record.
-        const isReplayedHistory = inReplayBlock;
-        rawTokenSeen++;
+      const isReplayedHistory = inReplayBlock;
+      rawTokenSeen++;
 
-        const info = payload.info;
-        if (!info) continue;
+      const info = payload.info;
+      if (!info) continue;
 
         // Codex sometimes writes the same token_count twice back-to-back:
         // identical last_token_usage with an unchanged cumulative total. A
@@ -484,71 +678,334 @@ export async function parse() {
         // compaction — and must count as zero, not a second copy of
         // last_token_usage. Guarded to positive totals so builds that leave
         // total_token_usage all-zero can't suppress real usage.
-        const cumulativeTotal = info.total_token_usage?.total_tokens;
-        const isDuplicateEmission = typeof cumulativeTotal === 'number'
-          && cumulativeTotal > 0
-          && cumulativeTotal === prevCumulativeTotal;
-        if (typeof cumulativeTotal === 'number') prevCumulativeTotal = cumulativeTotal;
+      const cumulativeTotal = info.total_token_usage?.total_tokens;
+      const isDuplicateEmission = typeof cumulativeTotal === 'number'
+        && cumulativeTotal > 0
+        && cumulativeTotal === prevCumulativeTotal;
+      if (typeof cumulativeTotal === 'number') prevCumulativeTotal = cumulativeTotal;
 
         // Prefer incremental per-request usage; compute delta from cumulative
         // totals as fallback. Always advance the cumulative baseline, even
         // when last_token_usage exists or the record belongs to a replay.
-        const curr = info.total_token_usage;
-        let usage = info.last_token_usage;
-        if (!usage && curr) {
-          if (prevTotal) {
-            const delta = {
-              input_tokens: (curr.input_tokens || 0) - (prevTotal.input_tokens || 0),
-              output_tokens: (curr.output_tokens || 0) - (prevTotal.output_tokens || 0),
-              cached_input_tokens: (curr.cached_input_tokens || 0) - (prevTotal.cached_input_tokens || 0),
-              reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prevTotal.reasoning_output_tokens || 0),
-            };
+      const curr = info.total_token_usage;
+      let usage = info.last_token_usage;
+      if (!usage && curr) {
+        if (prevTotal) {
+          const delta = {
+            input_tokens: (curr.input_tokens || 0) - (prevTotal.input_tokens || 0),
+            output_tokens: (curr.output_tokens || 0) - (prevTotal.output_tokens || 0),
+            cached_input_tokens: (curr.cached_input_tokens || 0) - (prevTotal.cached_input_tokens || 0),
+            reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prevTotal.reasoning_output_tokens || 0),
+          };
             // Cumulative counters can reset after compaction or a new usage
             // window. Treat the first post-reset total as a fresh baseline;
             // allowing a negative delta would cancel legitimate bucket usage.
-            usage = Object.values(delta).some(value => value < 0) ? curr : delta;
-          } else {
+          usage = Object.values(delta).some(value => value < 0) ? curr : delta;
+        } else {
             // First cumulative entry — use as-is (it's the first event's total)
-            usage = curr;
-          }
+          usage = curr;
         }
+      }
         // total_token_usage is session-wide, not per model. A global baseline
         // avoids counting the full cumulative total again after a model switch.
-        if (curr) prevTotal = { ...curr };
-        if (!usage) continue;
-        if (isReplayedHistory || isDuplicateEmission) continue;
+      if (curr) prevTotal = { ...curr };
+      if (!usage) continue;
+      if (isReplayedHistory || isDuplicateEmission) continue;
 
-        const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
-        if (!timestamp || isNaN(timestamp.getTime())) continue;
+      const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
+      if (!timestamp || isNaN(timestamp.getTime())) continue;
 
-        const model = info.model || payload.model || turnContextModel || 'unknown';
+      const model = info.model || payload.model || turnContextModel || 'unknown';
 
         // OpenAI API: input_tokens INCLUDES cached, output_tokens INCLUDES reasoning.
         // Normalize to Anthropic-style semantics where each field is non-overlapping.
-        const cachedInput = usage.cached_input_tokens || usage.cache_read_input_tokens || 0;
-        const reasoningOutput = usage.reasoning_output_tokens || 0;
-        entries.push({
-          source: 'codex',
-          model,
-          project: sessionProject,
-          timestamp,
-          inputTokens: (usage.input_tokens || 0) - cachedInput,
-          outputTokens: (usage.output_tokens || 0) - reasoningOutput,
-          cachedInputTokens: cachedInput,
-          reasoningOutputTokens: reasoningOutput,
-        });
+      const cachedInput = usage.cached_input_tokens || usage.cache_read_input_tokens || 0;
+      const reasoningOutput = usage.reasoning_output_tokens || 0;
+      entries.push({
+        source: 'codex',
+        model,
+        project: sessionProject,
+        timestamp,
+        inputTokens: (usage.input_tokens || 0) - cachedInput,
+        outputTokens: (usage.output_tokens || 0) - reasoningOutput,
+        cachedInputTokens: cachedInput,
+        reasoningOutputTokens: reasoningOutput,
+      });
+    } catch {
+      continue;
+    }
+  }
+
+  // Indexed files must match both passes exactly. Ordinary sessions take the
+  // single-pass fast path and have no expected counts; their byte-bounded
+  // snapshot is still stable, and any append invalidates the stat signature on
+  // the next sync.
+  if (fm.parsedRecordCount != null && fm.rawTokenCount != null) {
+    if (parsedRecordIndex !== fm.parsedRecordCount || rawTokenSeen !== fm.rawTokenCount) {
+      throw new Error('Codex rollout changed while syncing; retry on the next sync');
+    }
+  }
+
+  const buckets = mergeBucketLists([
+    previousTail?.buckets || [],
+    aggregateToBuckets(entries),
+  ]);
+  const sessionAccumulator = buildSessionAccumulator(
+    sessionEvents,
+    previousTail?.sessionAccumulator || null
+  );
+  // Appended records should be chronological. If an app version inserts an
+  // older event into the tail, the compact accumulator cannot reproduce the
+  // global sort exactly, so discard the optimization and rebuild this file.
+  if (previousTail && sessionEvents.length > 0 && !sessionAccumulator) {
+    return parseSessionFile(filePath, snapshotSize, fm, boundary, { captureTail });
+  }
+  const session = sessionFromAccumulator(sessionAccumulator);
+  const result = { buckets, sessions: session ? [session] : [] };
+  if (captureTail) {
+    const guard = snapshotGuard(filePath, snapshotSize);
+    result.tail = {
+      parsedBytes: snapshotSize,
+      parsedRecordIndex,
+      rawTokenSeen,
+      firstSessionMetaSeen,
+      turnContextModel,
+      prevTotal,
+      prevCumulativeTotal,
+      buckets,
+      sessionAccumulator,
+      guardHash: guard.hash,
+      endsWithNewline: guard.endsWithNewline,
+    };
+  }
+  return result;
+}
+
+function mergeFileResults(results) {
+  const entries = [];
+  const sessions = [];
+  for (const result of results) {
+    for (const bucket of result.buckets || []) {
+      entries.push({
+        source: bucket.source,
+        model: bucket.model,
+        project: bucket.project,
+        ...(bucket.hostname ? { hostname: bucket.hostname } : {}),
+        timestamp: new Date(bucket.bucketStart),
+        inputTokens: bucket.inputTokens,
+        outputTokens: bucket.outputTokens,
+        cachedInputTokens: bucket.cachedInputTokens,
+        reasoningOutputTokens: bucket.reasoningOutputTokens,
+      });
+    }
+    sessions.push(...(result.sessions || []));
+  }
+  return { buckets: aggregateToBuckets(entries), sessions };
+}
+
+export async function parse() {
+  const codexHome = getCodexHome();
+  const dirs = sessionsDirs(codexHome);
+  if (!dirs.some(existsSync)) return { buckets: [], sessions: [] };
+
+  const startedAt = Date.now();
+  const budget = workBudgetMs();
+  const overBudget = () => Date.now() - startedAt >= budget;
+  const cacheStats = {
+    headerHits: 0,
+    indexHits: 0,
+    resultHits: 0,
+    tailHits: 0,
+    filesRead: 0,
+    audited: 0,
+  };
+  const files = [];
+  for (const filePath of dirs.flatMap(findJsonlFiles)) {
+    try {
+      const stat = statSync(filePath);
+      if (stat.size <= 0) continue;
+      const signature = fileSignature(stat);
+      const cache = loadCodexFileCache(codexHome, filePath, signature);
+      const priorCache = cache || loadCodexFileCache(codexHome, filePath);
+      const priorTail = cache ? null : loadCodexFileTail(codexHome, filePath);
+      const file = {
+        filePath,
+        snapshotSize: stat.size,
+        signature,
+        cache,
+        priorCache,
+        priorTail,
+        header: null,
+        appendTail: null,
+      };
+      if (!cache && priorCache) file.appendTail = tailStateFor(file);
+      files.push(file);
+    } catch {
+      // The file may move to archived_sessions between discovery and stat.
+      // Its archived copy will be picked up on the next sync.
+    }
+  }
+  if (files.length === 0) return { buckets: [], sessions: [] };
+
+  // A warm cache gets a tiny rolling correctness audit. Never mix this into a
+  // cold/resumed build: all files must already have complete results, and at
+  // most one bounded file is re-read per invocation.
+  const auditPaths = new Set();
+  if (files.every(file => file.cache?.header && file.cache?.result)) {
+    const cutoff = Date.now() - auditIntervalMs();
+    const candidate = files
+      .filter(file => file.snapshotSize <= auditMaxBytes())
+      .filter(file => (file.cache.lastAuditedAt || 0) <= cutoff)
+      .sort((a, b) => (a.cache.lastAuditedAt || 0) - (b.cache.lastAuditedAt || 0))[0];
+    if (candidate) auditPaths.add(candidate.filePath);
+  }
+
+  // Cheap discovery: cached headers require no rollout read. On a cold build,
+  // read only through the first session_meta so ordinary sessions can avoid
+  // the former all-files replay-index pass.
+  const candidatesById = new Map();
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const reusableHeader = file.cache?.header || (file.appendTail ? file.priorCache?.header : null);
+    if (reusableHeader && !auditPaths.has(file.filePath)) {
+      file.header = reusableHeader;
+      if (!file.cache) file.cache = { header: reusableHeader };
+      cacheStats.headerHits++;
+    } else {
+      try {
+        file.header = await readSessionHeader(file.filePath, file.snapshotSize);
+        cacheStats.filesRead++;
+        updateFileCache(codexHome, file, { header: file.header });
       } catch {
         continue;
       }
     }
+    if (file.header.sessionId) {
+      if (!candidatesById.has(file.header.sessionId)) candidatesById.set(file.header.sessionId, []);
+      candidatesById.get(file.header.sessionId).push(file);
+    }
+    if (overBudget() && i < files.length - 1) {
+      return {
+        buckets: [], sessions: [], skipped: true,
+        indexing: { phase: 'discovery', completed: i + 1, total: files.length },
+        cache: cacheStats,
+      };
+    }
+  }
 
-    // The byte bound makes append-only growth invisible to both passes. If a
-    // rollout was instead truncated or replaced in place, fail the Codex
-    // parser rather than upload totals produced from two different snapshots.
-    if (parsedRecordIndex !== fm.parsedRecordCount || rawTokenSeen !== fm.rawTokenCount) {
-      throw new Error('Codex rollout changed while syncing; retry on the next sync');
+  const duplicateIds = new Set(
+    [...candidatesById].filter(([, candidates]) => candidates.length > 1).map(([id]) => id)
+  );
+  const referencedParentIds = new Set();
+  for (const file of files) {
+    const parentId = file.header?.forkedFromId
+      || (file.header?.isSubagent ? file.header.parentThreadId : null);
+    if (parentId) referencedParentIds.add(parentId);
+  }
+
+  // Only replay participants, their parents, corrupt-header files, and
+  // duplicate physical copies need the full compact token index.
+  const fileMeta = new Map();
+  const needsIndex = new Set();
+  for (const file of files) {
+    const header = file.header;
+    const required = !header
+      || !header.sessionId
+      || header.isSubagent
+      || header.forkedFromId != null
+      || header.parentThreadId != null
+      || referencedParentIds.has(header.sessionId)
+      || duplicateIds.has(header.sessionId);
+    if (!required) {
+      fileMeta.set(file.filePath, { ...header, filePath: file.filePath });
+      continue;
+    }
+    needsIndex.add(file.filePath);
+    let meta = auditPaths.has(file.filePath) ? null : file.cache?.index;
+    if (meta) {
+      cacheStats.indexHits++;
+    } else {
+      try {
+        meta = await indexSessionFile(file.filePath, file.snapshotSize);
+        cacheStats.filesRead++;
+        updateFileCache(codexHome, file, { index: meta });
+      } catch {
+        continue;
+      }
+    }
+    fileMeta.set(file.filePath, meta);
+    if (overBudget()) {
+      return {
+        buckets: [], sessions: [], skipped: true,
+        indexing: { phase: 'replay-index', completed: fileMeta.size, total: files.length },
+        cache: cacheStats,
+      };
+    }
+  }
+
+  // Select the most complete physical copy exactly as before. Unique ordinary
+  // sessions have no full record count, but cannot compete with another copy.
+  const sessionById = new Map();
+  for (const file of files) {
+    const meta = fileMeta.get(file.filePath);
+    if (!meta?.sessionId) continue;
+    const existing = sessionById.get(meta.sessionId);
+    const count = meta.parsedRecordCount ?? 0;
+    const existingCount = existing?.parsedRecordCount ?? 0;
+    if (!existing || count > existingCount) sessionById.set(meta.sessionId, meta);
+  }
+
+  const results = [];
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const fm = fileMeta.get(file.filePath);
+    if (!fm) continue;
+    if (fm.sessionId && sessionById.get(fm.sessionId)?.filePath !== file.filePath) continue;
+
+    const boundary = needsIndex.has(file.filePath)
+      ? replayBoundary(fm, sessionById)
+      : { rawTokenCount: 0, recordIndex: null };
+    const key = boundaryKey(boundary);
+    let result = !auditPaths.has(file.filePath) && file.cache?.result?.boundaryKey === key
+      ? file.cache.result
+      : null;
+    if (result) {
+      cacheStats.resultHits++;
+    } else {
+      const previousTail = !needsIndex.has(file.filePath) && !auditPaths.has(file.filePath)
+        ? file.appendTail
+        : null;
+      const parsed = await parseSessionFile(file.filePath, file.snapshotSize, fm, boundary, {
+        previousTail,
+        captureTail: !needsIndex.has(file.filePath),
+      });
+      cacheStats.filesRead++;
+      if (previousTail) cacheStats.tailHits++;
+      const { tail, ...summary } = parsed;
+      if (tail) {
+        try {
+          saveCodexFileTail(codexHome, file.filePath, file.signature, tail);
+        } catch {
+          // Same fail-open rule as the summary cache: tail acceleration is
+          // optional and a write failure must not fail the parser.
+        }
+      }
+      result = { boundaryKey: key, ...summary };
+      updateFileCache(codexHome, file, { result, lastAuditedAt: Date.now() });
+      file.appendTail = null;
+      file.priorTail = null;
+      if (auditPaths.has(file.filePath)) cacheStats.audited++;
+    }
+    results.push(result);
+
+    if (overBudget() && i < files.length - 1) {
+      return {
+        buckets: [], sessions: [], skipped: true,
+        indexing: { phase: 'usage', completed: i + 1, total: files.length },
+        cache: cacheStats,
+      };
     }
   }
 
-  return { buckets: aggregateToBuckets(entries), sessions: extractSessions(sessionEvents) };
+  return { ...mergeFileResults(results), cache: cacheStats };
 }

package/src/sync.js

@@ -34,6 +34,7 @@ export async function runSync({ throws = false, quiet = false } = {}) {
   const allBuckets = [];
   const allSessions = [];
   const parserResults = [];
+  const parserProgress = [];
   // Sources whose parser ran to completion this sync. pruneState() below is
   // scoped to these so a transient parser failure doesn't evict that tool's
   // state and force a full re-upload next run.
@@ -47,6 +48,9 @@ export async function runSync({ throws = false, quiet = false } = {}) {
       if (!Array.isArray(buckets) || !Array.isArray(sessions)) {
         throw new TypeError('Parser returned an invalid result');
       }
+      if (result?.indexing) {
+        parserProgress.push({ source, ...result.indexing });
+      }
       // A parser may deliberately suppress a transient error (Cursor network
       // timeout) to keep daemon logs quiet. Its empty result is not proof that
       // its prior data disappeared, so it must not be pruned this run.
@@ -71,7 +75,13 @@ export async function runSync({ throws = false, quiet = false } = {}) {
     pruneState(state, new Set(), new Set(), okSources);
     const pruned = before - (Object.keys(state.buckets).length + Object.keys(state.sessions).length);
     if (pruned > 0) saveState(state);
-    if (!quiet) console.log(dim('暂无新数据。'));
+    if (!quiet && parserProgress.length > 0) {
+      for (const p of parserProgress) {
+        console.log(dim(`  ${p.source}: 正在建立本地索引 ${p.completed}/${p.total}（下次同步继续）`));
+      }
+    } else if (!quiet) {
+      console.log(dim('暂无新数据。'));
+    }
     return 0;
   }
 
@@ -83,6 +93,11 @@ export async function runSync({ throws = false, quiet = false } = {}) {
       console.log(`  ${dim(p.source.padEnd(14))}${parts.join(' · ')}`);
     }
   }
+  if (!quiet && parserProgress.length > 0) {
+    for (const p of parserProgress) {
+      console.log(dim(`  ${p.source}: 正在建立本地索引 ${p.completed}/${p.total}（下次同步继续）`));
+    }
+  }
 
   let host = config.hostname;
   if (!host) {
@@ -112,10 +127,11 @@ export async function runSync({ throws = false, quiet = false } = {}) {
     for (const s of allSessions) s.project = 'unknown';
   }
 
-  // Incremental diff: parsers above always read the full local history (cheap,
-  // local-only). Here we drop anything whose content matches what we already
-  // uploaded, so only new/changed items go over the network. A quiet machine
-  // sends zero bytes; an active one sends just the current 30-min bucket.
+  // Incremental upload diff: parsers above emit a complete view of live local
+  // data (Codex may assemble that view from its disposable parser cache). Here
+  // we drop anything whose content matches what we already uploaded, so only
+  // new/changed items go over the network. A quiet machine sends zero bytes;
+  // an active one sends just the current 30-min bucket.
   // Missing/corrupt state.json => empty maps => one-time full upload, then
   // incremental forever after.
   const state = loadState();