@vibe-cafe/vibe-usage 0.9.11 → 0.9.13

package/README.md

@@ -51,7 +51,7 @@ npx @vibe-cafe/vibe-usage status       # Show config & detected tools
 | Tool | Data Location |
 |------|---------------|
 | Claude Code | `~/.claude/projects/` (tokens + sessions), `~/.claude/transcripts/` (sessions only); also scans `$CLAUDE_CONFIG_DIR` when set (deduped), so relocated configs and GUI/CLI env mismatches are both covered |
-| Codex CLI | `~/.codex/sessions/` and `~/.codex/archived_sessions/` |
+| Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`); forked and sub-agent rollouts replay parent metadata, tasks, and `token_count` records at spawn time — full or last-N-turn replay blocks are matched as a fingerprinted parent suffix plus the child's own task boundary, duplicate cumulative emissions count once, and live/archive copies of the same session are deduplicated |
 | GitHub Copilot CLI | `~/.copilot/session-state/*/events.jsonl` |
 | Cursor | `state.vscdb` (SQLite, reads `cursorAuth/accessToken`, fetches CSV from `cursor.com`); cloud data is stamped with a fixed `cursor-cloud` hostname so multi-machine setups don't double-count |
 | Gemini CLI | `~/.gemini/tmp/<project_hash>/chats/session-*.jsonl` (current line-delimited format) and legacy `session-*.json`; recurses into nested subagent sessions |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vibe-cafe/vibe-usage",
-  "version": "0.9.11",
+  "version": "0.9.13",
   "description": "Track your AI coding tool token usage and sync to vibecafe.ai",
   "type": "module",
   "scripts": {

package/src/parsers/codex.js

@@ -2,20 +2,23 @@ import { createReadStream, readdirSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { homedir } from 'node:os';
 import { createInterface } from 'node:readline';
+import { createHash } from 'node:crypto';
 import { aggregateToBuckets, extractSessions } from './index.js';
 
-// Codex stores live sessions in ~/.codex/sessions and, once a session is
-// "completed", moves its rollout file verbatim into ~/.codex/archived_sessions.
-// A session can be archived between two syncs, so scanning only the live dir
-// loses that session's usage forever. We scan both: the parser is stateless
-// and the server dedups on (source, sessionHash/bucket), so re-reading an
-// archived file that was already synced from sessions/ is idempotent. Indexing
-// both together also keeps fork replay-skip correct when a fork and its parent
-// end up split across the two directories.
-const SESSIONS_DIRS = [
-  join(homedir(), '.codex', 'sessions'),
-  join(homedir(), '.codex', 'archived_sessions'),
-];
+// Codex stores live sessions in $CODEX_HOME/sessions (default ~/.codex) and,
+// once a session is "completed", moves its rollout file verbatim into
+// $CODEX_HOME/archived_sessions. A session can be archived between two syncs,
+// so scanning only the live dir loses that session's usage forever. We scan
+// both, index them together so fork replay-skip works across directories, and
+// select the most complete physical file when the same session briefly exists
+// in both locations during an archive move.
+function sessionsDirs() {
+  const codexHome = process.env.CODEX_HOME?.trim() || join(homedir(), '.codex');
+  return [
+    join(codexHome, 'sessions'),
+    join(codexHome, 'archived_sessions'),
+  ];
+}
 
 /**
  * Recursively find all .jsonl files under a directory.
@@ -57,59 +60,265 @@ function extractProject(meta) {
 }
 
 /**
- * Stream a session file once and extract its index metadata: the session
- * id, the forked-from id, the project name, and the total count of
- * `event_msg/token_count` records. The token_count total is used to size
- * the replayed-history block of a forked session — a fork copies the
- * original conversation verbatim, so it begins with exactly as many
- * token_count records as the source session has in total.
+ * A sub-agent rollout (spawned thread / guardian / collab agent). Depending
+ * on the Codex version the marker is `thread_source: "subagent"`, a
+ * `source: { subagent: {...} }` object, or just a `parent_thread_id` — check
+ * all three so no version's sub-agents slip through as normal sessions.
+ */
+function isSubagentMeta(meta) {
+  if (meta.thread_source === 'subagent') return true;
+  const src = meta.source;
+  if (src === 'subagent') return true;
+  if (src && typeof src === 'object' && 'subagent' in src) return true;
+  return meta.parent_thread_id != null;
+}
+
+function extractParentThreadId(meta) {
+  return meta.parent_thread_id
+    || meta.source?.subagent?.thread_spawn?.parent_thread_id
+    || null;
+}
+
+function timestampMs(value) {
+  if (value == null || value === '') return null;
+  const n = new Date(value).getTime();
+  return Number.isFinite(n) ? n : null;
+}
+
+function epochMs(value) {
+  if (typeof value === 'string' && value.trim() !== '') value = Number(value);
+  if (typeof value !== 'number' || !Number.isFinite(value)) return null;
+  return value < 1e12 ? value * 1000 : value;
+}
+
+function isTaskStarted(payload) {
+  return payload?.type === 'task_started' || payload?.type === 'turn_started';
+}
+
+function upperBound(sorted, target) {
+  let lo = 0;
+  let hi = sorted.length;
+  while (lo < hi) {
+    const mid = lo + ((hi - lo) >> 1);
+    if (sorted[mid] <= target) lo = mid + 1;
+    else hi = mid;
+  }
+  return lo;
+}
+
+function tokenFingerprint(payload) {
+  // Copied rollout items are re-serialized with a fresh outer timestamp, but
+  // their token_count payload is unchanged. A compact payload hash therefore
+  // identifies replayed records without retaining raw usage objects in memory.
+  return createHash('sha256')
+    .update(JSON.stringify(payload))
+    .digest('base64url')
+    .slice(0, 16);
+}
+
+/**
+ * Return the longest prefix of `child` that is also a suffix of `parent`.
+ * Codex can fork full history or the last N turns, but the copied block always
+ * reaches the source snapshot's end. Requiring the suffix prevents a child's
+ * coincidentally repeated payload from matching an unrelated interior turn.
+ * KMP keeps this linear even when many token payloads are identical.
+ */
+function longestReplayPrefix(child, parent) {
+  if (child.length === 0 || parent.length === 0) return 0;
+
+  const prefix = new Array(child.length).fill(0);
+  for (let i = 1, matched = 0; i < child.length; i++) {
+    while (matched > 0 && child[i] !== child[matched]) matched = prefix[matched - 1];
+    if (child[i] === child[matched]) matched++;
+    prefix[i] = matched;
+  }
+
+  let matched = 0;
+  for (let i = 0; i < parent.length; i++) {
+    const fingerprint = parent[i];
+    while (matched > 0 && fingerprint !== child[matched]) matched = prefix[matched - 1];
+    if (fingerprint === child[matched]) matched++;
+    if (matched === child.length && i < parent.length - 1) matched = prefix[matched - 1];
+  }
+  return matched;
+}
+
+// `task_started.started_at` is stored at one-second precision while the
+// canonical session timestamp has milliseconds. Real Codex Desktop rollouts
+// start the child task within a few seconds of creating the child session.
+const OWN_TASK_START_WINDOW_MS = 5_000;
+
+/**
+ * Stream a rollout once and build a compact replay index. A fork/sub-agent
+ * file starts with its own session_meta and can then contain the source
+ * session's complete metadata and history. Only the first session_meta is
+ * canonical; later ones are replayed records and must never overwrite it.
+ *
+ * tokenTimes preserves raw token_count ordinals (including malformed usage
+ * records) on a monotonic timeline. tokenFingerprints identifies an exact
+ * copied sequence even when Codex forks only the last N turns instead of a
+ * full prefix. Together they bound matching to source records that existed at
+ * spawn without over-skipping child work when the parent later grows.
  */
 async function indexSessionFile(filePath) {
   let sessionId = null;
   let forkedFromId = null;
+  let parentThreadId = null;
   let sessionProject = 'unknown';
-  let tokenCountRecords = 0;
+  let sessionStartedAtMs = null;
+  let isSubagent = false;
+  let sessionMetaCount = 0;
+  let parsedRecordCount = 0;
+  let rawTokenCount = 0;
+  let logicalTimestamp = Number.NEGATIVE_INFINITY;
+  const tokenTimes = [];
+  const tokenFingerprints = [];
+  let pendingTokenTimeIndexes = [];
+  const taskBoundaries = [];
+  let firstTaskBoundary = null;
+  let ownTaskBoundary = null;
 
   for await (const line of readLines(filePath)) {
     if (!line.trim()) continue;
     try {
       const obj = JSON.parse(line);
+      parsedRecordCount++;
+
+      const recordTimestamp = timestampMs(obj.timestamp);
+      if (recordTimestamp != null) {
+        logicalTimestamp = Math.max(logicalTimestamp, recordTimestamp);
+        // An invalid token_count timestamp is placed at the next valid record
+        // time. If there is no next valid record it remains +Infinity, which
+        // deliberately biases a parent-at-spawn boundary toward under-skip.
+        for (const idx of pendingTokenTimeIndexes) tokenTimes[idx] = logicalTimestamp;
+        pendingTokenTimeIndexes = [];
+      }
+
       if (obj.type === 'session_meta' && obj.payload) {
-        const meta = obj.payload;
-        sessionId = meta.id || sessionId;
-        forkedFromId = meta.forked_from_id || null;
-        sessionProject = extractProject(meta);
+        sessionMetaCount++;
+        if (sessionMetaCount === 1) {
+          const meta = obj.payload;
+          sessionId = meta.id || null;
+          forkedFromId = meta.forked_from_id || null;
+          parentThreadId = extractParentThreadId(meta);
+          isSubagent = isSubagentMeta(meta);
+          sessionProject = extractProject(meta);
+          sessionStartedAtMs = timestampMs(meta.timestamp) ?? recordTimestamp;
+        }
       } else if (obj.type === 'event_msg' && obj.payload?.type === 'token_count') {
-        tokenCountRecords++;
+        rawTokenCount++;
+        tokenFingerprints.push(tokenFingerprint(obj.payload));
+        if (recordTimestamp == null) {
+          tokenTimes.push(Number.POSITIVE_INFINITY);
+          pendingTokenTimeIndexes.push(tokenTimes.length - 1);
+        } else {
+          tokenTimes.push(logicalTimestamp);
+        }
+      } else if (obj.type === 'event_msg' && isTaskStarted(obj.payload)) {
+        const boundary = {
+          recordIndex: parsedRecordCount,
+          rawTokenCount,
+          startedAtMs: epochMs(obj.payload.started_at),
+        };
+        taskBoundaries.push(boundary);
+        firstTaskBoundary ??= boundary;
+
+        const startedAtMs = boundary.startedAtMs;
+        if (sessionStartedAtMs != null && startedAtMs != null
+            && Math.abs(startedAtMs - sessionStartedAtMs) <= OWN_TASK_START_WINDOW_MS) {
+          // Keep the last match so a copied parent task that happened to start
+          // in the same second cannot win over the child's later own boundary.
+          ownTaskBoundary = boundary;
+        }
       }
     } catch {
       continue;
     }
   }
 
-  return { sessionId, forkedFromId, sessionProject, tokenCountRecords };
+  return {
+    filePath,
+    sessionId,
+    forkedFromId,
+    parentThreadId,
+    sessionProject,
+    sessionStartedAtMs,
+    isSubagent,
+    sessionMetaCount,
+    parsedRecordCount,
+    rawTokenCount,
+    tokenTimes,
+    tokenFingerprints,
+    taskBoundaries,
+    firstTaskBoundary,
+    ownTaskBoundary,
+  };
+}
+
+function replayBoundary(meta, sessionById) {
+  const parentId = meta.forkedFromId || (meta.isSubagent ? meta.parentThreadId : null);
+  const parent = parentId ? sessionById.get(parentId) : null;
+  const parentAtSpawn = parent && meta.sessionStartedAtMs != null
+    ? upperBound(parent.tokenTimes, meta.sessionStartedAtMs)
+    : null;
+  const replayTokenCount = parentAtSpawn == null
+    ? 0
+    : longestReplayPrefix(
+      meta.tokenFingerprints,
+      parent.tokenFingerprints.slice(0, parentAtSpawn)
+    );
+
+  if (meta.isSubagent) {
+    // Direct evidence inside the child wins. Legacy single-meta rollouts did
+    // not replay task_started records, so their first task remains a safe
+    // fallback. Double-meta files must not use their copied parent's first
+    // task_started as the boundary.
+    // Exact token matching also handles LastNTurns forks. When it identifies
+    // the copied token suffix, the last task_started at that same raw ordinal
+    // is the child's own task boundary (copied history is written first).
+    const matchedTaskBoundaries = replayTokenCount > 0
+      ? meta.taskBoundaries.filter(boundary => (
+        boundary.rawTokenCount === replayTokenCount
+        && boundary.startedAtMs != null
+        && meta.sessionStartedAtMs != null
+        && boundary.startedAtMs >= Math.floor(meta.sessionStartedAtMs / 1000) * 1000
+      ))
+      : [];
+    const matchedTaskBoundary = matchedTaskBoundaries.at(-1) || null;
+    const direct = matchedTaskBoundary
+      || meta.ownTaskBoundary
+      || (meta.sessionMetaCount === 1 && !meta.forkedFromId
+        ? meta.firstTaskBoundary
+        : null);
+    if (direct) {
+      return {
+        rawTokenCount: Math.max(replayTokenCount, direct.rawTokenCount),
+        recordIndex: direct.recordIndex,
+      };
+    }
+    return { rawTokenCount: replayTokenCount, recordIndex: null };
+  }
+
+  if (meta.forkedFromId) {
+    return { rawTokenCount: replayTokenCount, recordIndex: null };
+  }
+  return { rawTokenCount: 0, recordIndex: null };
 }
 
 export async function parse() {
-  if (!SESSIONS_DIRS.some(existsSync)) return { buckets: [], sessions: [] };
+  const dirs = sessionsDirs();
+  if (!dirs.some(existsSync)) return { buckets: [], sessions: [] };
 
   const entries = [];
   const sessionEvents = [];
-  const files = SESSIONS_DIRS.flatMap(findJsonlFiles);
+  const files = dirs.flatMap(findJsonlFiles);
   if (files.length === 0) return { buckets: [], sessions: [] };
 
-  // Pass 1: index every session by its UUID and count its token_count
-  // records. A forked session (session_meta.payload.forked_from_id) starts
-  // with the original conversation replayed verbatim — including every
-  // token_count, all timestamped in a burst at the fork instant. Those
-  // tokens are already counted from the original session's own file, so
-  // re-counting them here double-counts usage and produces a spurious
-  // token/cost spike at the fork time. Timestamps cannot distinguish the
-  // replay from new activity (the replay burst is stamped at/after the fork
-  // instant, within the same 1–3s window), so we instead skip exactly the
-  // original session's token_count count from the start of each fork.
-  const tokenCountById = new Map(); // sessionId → number of token_count records
-  const fileMeta = new Map(); // filePath -> { forkedFromId, sessionProject }
+  // Pass 1: build a compact per-file index. Keep the most complete physical
+  // copy for each logical session id so a rollout briefly present in both
+  // sessions/ and archived_sessions/ cannot double its token buckets.
+  const sessionById = new Map();
+  const fileMeta = new Map();
   for (const filePath of files) {
     let meta;
     try {
@@ -119,30 +328,23 @@ export async function parse() {
     }
     fileMeta.set(filePath, meta);
     if (meta.sessionId) {
-      tokenCountById.set(meta.sessionId, meta.tokenCountRecords);
+      const existing = sessionById.get(meta.sessionId);
+      if (!existing || meta.parsedRecordCount > existing.parsedRecordCount) {
+        sessionById.set(meta.sessionId, meta);
+      }
     }
   }
 
-  // Pass 2: parse usage, skipping each fork's replayed-history token_counts.
+  // Pass 2: parse usage while skipping the replay prefix resolved above.
   for (const filePath of files) {
     const fm = fileMeta.get(filePath);
     if (!fm) continue;
-    const { forkedFromId } = fm;
-
-    // How many leading token_count records are copied history. A fork's file
-    // begins with the *entire* source file replayed verbatim, so the count
-    // to skip is the source's total token_count count. This is correct even
-    // for chained forks: a fork-of-a-fork replays the parent fork's whole
-    // file (which itself already contains the grandparent's replay), so
-    // skipping the parent's full count skips exactly the duplicated region.
-    // If the source file is missing (rotated/deleted) we cannot locate the
-    // boundary; skip nothing so incomplete data over-counts rather than
-    // silently dropping real usage.
-    let replayTokenCountToSkip = 0;
-    if (forkedFromId != null) {
-      replayTokenCountToSkip = tokenCountById.get(forkedFromId) ?? 0;
-    }
-    let tokenCountSeen = 0;
+    if (fm.sessionId && sessionById.get(fm.sessionId)?.filePath !== filePath) continue;
+
+    const boundary = replayBoundary(fm, sessionById);
+    let rawTokenSeen = 0;
+    let parsedRecordIndex = 0;
+    let firstSessionMetaSeen = false;
 
     const sessionProject = fm.sessionProject;
     // Group timing events by the real Codex session id, not the file path: the
@@ -153,29 +355,37 @@ export async function parse() {
     const sessionKey = fm.sessionId || filePath;
 
     let turnContextModel = 'unknown';
-    const prevTotal = new Map();
+    let prevTotal = null;
+    let prevCumulativeTotal = null;
     for await (const line of readLines(filePath)) {
       if (!line.trim()) continue;
       try {
         const obj = JSON.parse(line);
+        parsedRecordIndex++;
+
+        // A direct child task boundary covers every copied record, including
+        // timing/meta events. The raw-token ordinal covers full-history and
+        // last-N-turn forks whose exact payload sequence was matched in pass 1.
+        const beforeOwnTask = boundary.recordIndex != null
+          && parsedRecordIndex < boundary.recordIndex;
+        const inReplayBlock = beforeOwnTask || rawTokenSeen < boundary.rawTokenCount;
 
-        // A fork's replayed-history block is the run from the start of the
-        // file up to and including the Nth token_count, where N is the source
-        // session's total token_count count. We are still inside that block
-        // until we have *passed* the Nth token_count. (token_count is the
-        // last event of each turn, so the boundary lands cleanly at a turn
-        // edge — the new conversation's events come strictly after it.)
-        const inReplayBlock = tokenCountSeen < replayTokenCountToSkip;
+        const isSessionMeta = obj.type === 'session_meta';
+        const isCanonicalSessionMeta = isSessionMeta && !firstSessionMetaSeen;
+        const isOwnSessionMeta = isSessionMeta
+          && obj.payload?.id != null
+          && obj.payload.id === fm.sessionId;
+        if (isSessionMeta) firstSessionMetaSeen = true;
 
         if (obj.timestamp) {
           const evTs = new Date(obj.timestamp);
           if (!isNaN(evTs.getTime())) {
-            // Skip replayed history events so a forked session's
-            // duration/active-time/message counts reflect only the new
-            // conversation, not the copied original. session_meta itself is
-            // kept: it marks when the fork actually started.
-            const isReplay = inReplayBlock && obj.type !== 'session_meta';
-            if (!isReplay) {
+            // Repeated same-id metadata can be appended on resume/config
+            // updates and belongs to this logical session. A different-id meta
+            // is copied parent history and must not inflate timing stats.
+            const keepSessionMeta = isCanonicalSessionMeta
+              || (isOwnSessionMeta && !inReplayBlock);
+            if (keepSessionMeta || (!isSessionMeta && !inReplayBlock)) {
               const isUserTurn = obj.type === 'turn_context' || obj.type === 'session_meta';
               sessionEvents.push({
                 sessionId: sessionKey,
@@ -200,44 +410,58 @@ export async function parse() {
 
         if (payload.type !== 'token_count') continue;
 
+        // Raw ordinals advance before validating usage/timestamp so pass 1 and
+        // pass 2 cannot drift on a malformed copied token_count record.
+        const isReplayedHistory = inReplayBlock;
+        rawTokenSeen++;
+
         const info = payload.info;
         if (!info) continue;
 
-        const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
-        if (!timestamp || isNaN(timestamp.getTime())) continue;
-
-        // This is the (tokenCountSeen+1)-th token_count in the file. If it
-        // falls inside the fork's replay block it's an exact copy of a record
-        // already counted from the source session's own file — skip it (but
-        // still advance the cumulative-total baseline below so the first real
-        // post-fork delta is measured correctly).
-        const isReplayedHistory = tokenCountSeen < replayTokenCountToSkip;
-        tokenCountSeen++;
+        // Codex sometimes writes the same token_count twice back-to-back:
+        // identical last_token_usage with an unchanged cumulative total. A
+        // real API call always advances the cumulative counter (its input
+        // tokens alone are non-zero), so an unchanged positive total marks a
+        // duplicate emission — or a zero-usage bookkeeping event such as
+        // compaction — and must count as zero, not a second copy of
+        // last_token_usage. Guarded to positive totals so builds that leave
+        // total_token_usage all-zero can't suppress real usage.
+        const cumulativeTotal = info.total_token_usage?.total_tokens;
+        const isDuplicateEmission = typeof cumulativeTotal === 'number'
+          && cumulativeTotal > 0
+          && cumulativeTotal === prevCumulativeTotal;
+        if (typeof cumulativeTotal === 'number') prevCumulativeTotal = cumulativeTotal;
 
-        // Prefer incremental per-request usage; compute delta from cumulative total as fallback
+        // Prefer incremental per-request usage; compute delta from cumulative
+        // totals as fallback. Always advance the cumulative baseline, even
+        // when last_token_usage exists or the record belongs to a replay.
+        const curr = info.total_token_usage;
         let usage = info.last_token_usage;
-        if (!usage && info.total_token_usage) {
-          const totalKey = `${info.model || payload.model || turnContextModel || ''}`;
-          const prev = prevTotal.get(totalKey);
-          const curr = info.total_token_usage;
-          if (prev) {
-            usage = {
-              input_tokens: (curr.input_tokens || 0) - (prev.input_tokens || 0),
-              output_tokens: (curr.output_tokens || 0) - (prev.output_tokens || 0),
-              cached_input_tokens: (curr.cached_input_tokens || 0) - (prev.cached_input_tokens || 0),
-              reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prev.reasoning_output_tokens || 0),
+        if (!usage && curr) {
+          if (prevTotal) {
+            const delta = {
+              input_tokens: (curr.input_tokens || 0) - (prevTotal.input_tokens || 0),
+              output_tokens: (curr.output_tokens || 0) - (prevTotal.output_tokens || 0),
+              cached_input_tokens: (curr.cached_input_tokens || 0) - (prevTotal.cached_input_tokens || 0),
+              reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prevTotal.reasoning_output_tokens || 0),
             };
+            // Cumulative counters can reset after compaction or a new usage
+            // window. Treat the first post-reset total as a fresh baseline;
+            // allowing a negative delta would cancel legitimate bucket usage.
+            usage = Object.values(delta).some(value => value < 0) ? curr : delta;
           } else {
             // First cumulative entry — use as-is (it's the first event's total)
             usage = curr;
           }
-          // Always advance the cumulative baseline, even for replayed history,
-          // so the first real post-fork delta is measured against the last
-          // replayed total instead of being mistaken for a fresh "first entry".
-          prevTotal.set(totalKey, { ...curr });
         }
+        // total_token_usage is session-wide, not per model. A global baseline
+        // avoids counting the full cumulative total again after a model switch.
+        if (curr) prevTotal = { ...curr };
         if (!usage) continue;
-        if (isReplayedHistory) continue;
+        if (isReplayedHistory || isDuplicateEmission) continue;
+
+        const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
+        if (!timestamp || isNaN(timestamp.getTime())) continue;
 
         const model = info.model || payload.model || turnContextModel || 'unknown';