@vimoxshah/tokenflow 1.1.0

package/src/index.js

@@ -0,0 +1,18 @@
+/**
+ * Library entry point. Everything the CLI and dashboard use is exported here,
+ * so the platform can be embedded in another tool without shelling out.
+ */
+export * from './core/schema.js';
+export * from './core/validate.js';
+export * from './core/registry.js';
+export * from './core/config.js';
+export * from './core/pricing.js';
+export * from './core/model-map.js';
+export * from './core/interface-map.js';
+export { refresh, enrich, walk } from './core/ingest.js';
+export { Store, encodeRecord, decodeRecord } from './core/store.js';
+export { buildBundle, queryRecords } from './core/bundle.js';
+export * from './analytics/index.js';
+export * from './export/csv.js';
+export { buildSnapshot } from './export/html-snapshot.js';
+export { startServer } from './server/server.js';

package/src/providers/anthropic/index.js

@@ -0,0 +1,294 @@
+/**
+ * Anthropic — Claude Code / Claude Agent SDK session transcripts.
+ *
+ * Source: <claude-home>/projects/<slugified-cwd>/<sessionId>.jsonl
+ * Homes are auto-discovered: $CLAUDE_CONFIG_DIR, ~/.claude, ~/.config/claude,
+ * and any ~/.claude-* sibling that contains a `projects/` directory (people
+ * who run several accounts keep them side by side).
+ *
+ * ## Two correctness traps this adapter exists to handle
+ *
+ * 1. **Streaming snapshots.** A single assistant message is written to the
+ *    transcript several times as it streams. Every line carries the SAME
+ *    `requestId` + `message.id` and a *growing* `output_tokens`, with input and
+ *    cache counts constant. Summing them inflates output by 3-5x. The correct
+ *    value is the final snapshot, so we group by (requestId, message.id) and
+ *    keep the per-field maximum.
+ *
+ *    A group can straddle the end of the file, because a live session is still
+ *    being written. Rather than defer it (which would silently drop the last
+ *    record of every settled file, forever), the group is emitted at EOF and
+ *    its emitted totals are remembered as that file's `tails`. If the file
+ *    later grows and the same (requestId, message.id) reappears with larger
+ *    counts, only the DELTA is emitted. So the record is complete, never
+ *    duplicated, and never double counted.
+ *
+ * 2. **Synthetic messages.** Locally generated assistant entries have
+ *    `requestId: null` and an all-zero usage block. They are not API calls;
+ *    counting them would add fake zero-token requests and skew "requests" and
+ *    "tokens per request". They are skipped and counted separately.
+ *
+ * Token semantics (already the exclusive convention the schema wants):
+ *   input_tokens                 fresh prompt tokens, EXCLUDING cache
+ *   cache_read_input_tokens      -> cache_read_tokens
+ *   cache_creation_input_tokens  -> cache_write_tokens
+ *   cache_creation.ephemeral_1h  -> cache_refresh_tokens (subset of writes)
+ *   output_tokens_details.thinking_tokens -> reasoning_tokens (subset of output)
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { readLines } from '../../core/jsonl.js';
+import { walk } from '../../core/ingest.js';
+import { MEASUREMENT } from '../../core/schema.js';
+
+const USAGE_MARK = '"usage"';
+/** Groups this far behind the read head cannot still be open. */
+const GROUP_FLUSH_LAG = 2 << 20;
+
+export function candidateHomes(ctx) {
+  const configured = ctx?.config?.sources?.anthropic?.paths;
+  if (Array.isArray(configured) && configured.length) return configured.map(expand);
+  const home = ctx?.home || os.homedir();
+  const out = [];
+  if (process.env.CLAUDE_CONFIG_DIR) out.push(...process.env.CLAUDE_CONFIG_DIR.split(path.delimiter));
+  out.push(path.join(home, '.claude'), path.join(home, '.config', 'claude'));
+  // Sibling homes: ~/.claude-work, ~/.claude-personal, ...
+  try {
+    for (const e of fs.readdirSync(home, { withFileTypes: true })) {
+      if (!e.isDirectory() && !e.isSymbolicLink()) continue;
+      if (!/^\.claude([-.].+)?$/.test(e.name)) continue;
+      out.push(path.join(home, e.name));
+    }
+  } catch { /* home unreadable: fall through to the explicit list */ }
+  return [...new Set(out)].filter((d) => {
+    try {
+      return fs.statSync(path.join(d, 'projects')).isDirectory();
+    } catch {
+      return false;
+    }
+  });
+}
+
+function expand(p) {
+  return p.startsWith('~') ? path.join(os.homedir(), p.slice(1)) : p;
+}
+
+/** "-Users-me-projects-checkout-api" -> "checkout-api" */
+export function projectFromSlug(slug, cwd) {
+  const base = cwd || slug.replace(/^-/, '/').replace(/-/g, '/');
+  const parts = String(cwd || base).split('/').filter(Boolean);
+  if (!parts.length) return null;
+  const last = parts[parts.length - 1];
+  // A temp dir tells us nothing useful about the project.
+  if (/^(T|tmp|temp)$/i.test(last) && parts.length > 1) return parts[parts.length - 2];
+  return last;
+}
+
+export default createProvider({
+  id: 'anthropic',
+  name: 'Anthropic (Claude Code / Agent SDK)',
+  description: 'Per-request token usage from Claude Code and Claude Agent SDK session transcripts.',
+  measurement: MEASUREMENT.PRIMARY,
+  requires: ['A Claude Code home directory with a projects/ folder'],
+
+  async detect(ctx) {
+    const homes = candidateHomes(ctx);
+    if (!homes.length) {
+      return { available: false, detail: 'No Claude Code home found (looked for ~/.claude*/projects)' };
+    }
+    return {
+      available: true,
+      detail: homes.map((h) => shortHome(h, ctx)).join(', '),
+      paths: homes,
+    };
+  },
+
+  async discover(ctx) {
+    const files = [];
+    for (const home of candidateHomes(ctx)) {
+      const label = path.basename(home);
+      const root = path.join(home, 'projects');
+      for (const f of walk(root, (n) => n.endsWith('.jsonl'))) {
+        let stat;
+        try { stat = fs.statSync(f); } catch { continue; }
+        if (!stat.size) continue;
+        files.push({ key: `${label}:${path.relative(root, f)}`, path: f, stat, home, label });
+      }
+    }
+    return files;
+  },
+
+  async ingestFile(ref, ctx, emit) {
+    /** @type {Map<string, {rec:object, start:number}>} */
+    const open = new Map();
+    /** Totals already emitted for a group that straddled a previous EOF. */
+    const tails = ref.state.tails || {};
+    let records = 0;
+    let malformed = 0;
+    let synthetic = 0;
+
+    const FIELDS = ['input_tokens', 'output_tokens', 'cache_read_tokens', 'cache_write_tokens', 'cache_refresh_tokens', 'reasoning_tokens'];
+
+    /**
+     * Emit a group.
+     *
+     * One logical message can be split across two flushes: once because it fell
+     * far behind the read head, once at EOF. `carried` holds the absolute totals
+     * already emitted for a key, so a reappearance emits only the delta and the
+     * two halves sum to the true maximum rather than to their sum.
+     */
+    const carried = { ...tails };
+    const flush = (key, remember = false) => {
+      const g = open.get(key);
+      if (!g) return;
+      open.delete(key);
+      const rec = g.rec;
+      const base = carried[key];
+
+      const absolute = {};
+      for (const f of FIELDS) {
+        const prev = base ? (base[f] ?? 0) : 0;
+        absolute[f] = rec[f] === null ? prev : Math.max(prev, rec[f]);
+      }
+      if (remember) carried[key] = absolute;
+
+      if (base) {
+        let any = false;
+        for (const f of FIELDS) {
+          if (rec[f] === null) continue;
+          rec[f] = Math.max(0, rec[f] - (base[f] ?? 0));
+          if (rec[f] > 0) any = true;
+        }
+        rec.metadata = { ...rec.metadata, continuation_of: key };
+        if (!any) return; // nothing new in this group
+      }
+      emit(rec);
+      records++;
+    };
+
+    const res = readLines(
+      ref.path,
+      (line, offsetAfter) => {
+        const lineStart = offsetAfter - Buffer.byteLength(line) - 1;
+        let d;
+        try {
+          d = JSON.parse(line);
+        } catch {
+          malformed++;
+          return;
+        }
+        if (d.type !== 'assistant') return;
+        const msg = d.message;
+        const u = msg && msg.usage;
+        if (!u) return;
+
+        const inTok = n(u.input_tokens);
+        const outTok = n(u.output_tokens);
+        const crTok = n(u.cache_read_input_tokens);
+        const cwTok = n(u.cache_creation_input_tokens);
+
+        // Synthetic / locally generated entry: no API call happened. Claude Code
+        // marks these either by omitting requestId with an all-zero usage block,
+        // or by naming the model `<synthetic>` outright.
+        if (msg.model === '<synthetic>' || (!d.requestId && !inTok && !outTok && !crTok && !cwTok)) {
+          synthetic++;
+          return;
+        }
+
+        const key = `${d.requestId || 'na'}|${msg.id || d.uuid}`;
+        const cwd = d.cwd || null;
+        const partial = {
+          timestamp: d.timestamp,
+          model: msg.model || null,
+          input_tokens: inTok,
+          output_tokens: outTok,
+          cache_read_tokens: crTok,
+          cache_write_tokens: cwTok,
+          cache_refresh_tokens: u.cache_creation ? n(u.cache_creation.ephemeral_1h_input_tokens) : null,
+          reasoning_tokens: u.output_tokens_details ? n(u.output_tokens_details.thinking_tokens) : null,
+          session_id: d.sessionId || null,
+          conversation_id: d.sessionId || null,
+          request_id: d.requestId || null,
+          project: projectFromSlug(ref.key, cwd),
+          repository: projectFromSlug(ref.key, cwd),
+          git_branch: d.gitBranch || null,
+          category: d.isSidechain ? 'subagent' : 'main',
+          client: 'claude-code',
+          application: 'Claude Code',
+          interfaceSignals: [d.entrypoint, d.userType === 'external' ? null : d.userType],
+          metadata: {
+            cwd,
+            version: d.version || null,
+            entrypoint: d.entrypoint || null,
+            service_tier: u.service_tier ?? null,
+            speed: u.speed ?? null,
+            stop_reason: msg.stop_reason ?? null,
+            sidechain: !!d.isSidechain,
+            home: ref.label,
+            cache_write_5m: u.cache_creation ? n(u.cache_creation.ephemeral_5m_input_tokens) : null,
+            web_search_requests: u.server_tool_use ? n(u.server_tool_use.web_search_requests) : null,
+            web_fetch_requests: u.server_tool_use ? n(u.server_tool_use.web_fetch_requests) : null,
+          },
+        };
+
+        const g = open.get(key);
+        if (!g) {
+          open.set(key, { rec: partial, start: lineStart });
+        } else {
+          // Keep the largest snapshot of every field; output grows as it streams.
+          const r = g.rec;
+          r.output_tokens = maxN(r.output_tokens, partial.output_tokens);
+          r.input_tokens = maxN(r.input_tokens, partial.input_tokens);
+          r.cache_read_tokens = maxN(r.cache_read_tokens, partial.cache_read_tokens);
+          r.cache_write_tokens = maxN(r.cache_write_tokens, partial.cache_write_tokens);
+          r.cache_refresh_tokens = maxN(r.cache_refresh_tokens, partial.cache_refresh_tokens);
+          r.reasoning_tokens = maxN(r.reasoning_tokens, partial.reasoning_tokens);
+          r.timestamp = partial.timestamp || r.timestamp;
+          r.metadata.stop_reason = partial.metadata.stop_reason ?? r.metadata.stop_reason;
+        }
+
+        // Anything far enough behind the read head is provably finished.
+        for (const [k, v] of open) {
+          // Far enough behind the read head that it is almost certainly done —
+          // but remember it, in case a sidechain interleaves it back in.
+          if (k !== key && offsetAfter - v.start > GROUP_FLUSH_LAG) flush(k, true);
+        }
+      },
+      { start: ref.start, must: [USAGE_MARK] },
+    );
+
+    // Emit whatever is still open, and remember what we emitted for each of
+    // those groups. If the file grows and one of them continues, only the
+    // delta will be emitted next time (see `flush`).
+    // Only groups still open at EOF can continue in a later append, so only
+    // those are worth persisting. Groups flushed mid-read are finished.
+    const openKeys = [...open.keys()];
+    for (const key of openKeys) flush(key, true);
+    const nextTails = {};
+    for (const key of openKeys) if (carried[key]) nextTails[key] = carried[key];
+    ref.state.tails = nextTails;
+
+    return {
+      offset: res.offset,
+      records,
+      malformed,
+      openAtEof: openKeys.length,
+      synthetic,
+    };
+  },
+});
+
+function shortHome(h, ctx) {
+  const home = ctx?.home || os.homedir();
+  return h.startsWith(home) ? '~' + h.slice(home.length) : h;
+}
+function n(v) {
+  return v === undefined || v === null ? null : Number(v);
+}
+function maxN(a, b) {
+  if (a === null) return b;
+  if (b === null) return a;
+  return Math.max(a, b);
+}

package/src/providers/cline/index.js

@@ -0,0 +1,120 @@
+/**
+ * Cline CLI — session records.
+ *
+ * Cline's session files record which provider/model was used and when, but
+ * they contain **no token accounting at all**. That is exactly the case the
+ * schema's missing-value contract exists for: every token field is `null`
+ * ("the source does not report this"), the record is marked
+ * `measurement: activity`, and the dashboard counts these sessions in activity
+ * metrics while explicitly excluding them from token totals — rather than
+ * quietly adding a session's worth of zeros and dragging every average down.
+ *
+ * Source: ~/.cline/data/sessions/<id>/<id>.json
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { MEASUREMENT } from '../../core/schema.js';
+
+function root(ctx) {
+  const configured = ctx?.config?.sources?.cline?.path;
+  if (configured) return configured.startsWith('~') ? path.join(os.homedir(), configured.slice(1)) : configured;
+  return path.join(ctx?.home || os.homedir(), '.cline', 'data', 'sessions');
+}
+
+export default createProvider({
+  id: 'cline',
+  name: 'Cline CLI',
+  description: 'Session-level AI activity (model, provider, duration). Cline does not log token counts.',
+  measurement: MEASUREMENT.ACTIVITY,
+  requires: ['~/.cline/data/sessions'],
+
+  async detect(ctx) {
+    const d = root(ctx);
+    try {
+      const n = fs.readdirSync(d).length;
+      return { available: n > 0, detail: `${n} session directories · no token fields reported by this source`, paths: [d] };
+    } catch {
+      return { available: false, detail: 'No ~/.cline/data/sessions directory found' };
+    }
+  },
+
+  async discover(ctx) {
+    const d = root(ctx);
+    const out = [];
+    let dirs = [];
+    try {
+      dirs = fs.readdirSync(d, { withFileTypes: true });
+    } catch {
+      return out;
+    }
+    for (const e of dirs) {
+      if (!e.isDirectory()) continue;
+      const f = path.join(d, e.name, `${e.name}.json`);
+      let stat;
+      try { stat = fs.statSync(f); } catch { continue; }
+      out.push({ key: e.name, path: f, stat });
+    }
+    return out;
+  },
+
+  async ingestFile(ref, ctx, emit) {
+    let d;
+    try {
+      d = JSON.parse(fs.readFileSync(ref.path, 'utf8'));
+    } catch (err) {
+      return { offset: ref.stat.size, records: 0, malformed: 1 };
+    }
+    const ts = d.started_at || d.ended_at;
+    if (!ts) return { offset: ref.stat.size, records: 0 };
+
+    const project = d.workspace_root || d.cwd ? path.basename(d.workspace_root || d.cwd) : null;
+    const git = d.metadata?.git || {};
+    const messages = countMessages(ref.path.replace(/\.json$/, '.messages.json'));
+
+    emit({
+      timestamp: ts,
+      model: d.model || null,
+      // `provider: "cline"` in the file names the *client*, not the model
+      // vendor — the vendor is derived from the model string.
+      client: 'cline',
+      application: 'Cline CLI',
+      interfaceSignals: [d.source, d.interactive ? 'cli' : null],
+      // Every token field stays null: not available, not zero.
+      input_tokens: null,
+      output_tokens: null,
+      cache_read_tokens: null,
+      cache_write_tokens: null,
+      cache_refresh_tokens: null,
+      reasoning_tokens: null,
+      session_id: d.session_id || ref.key,
+      project,
+      repository: git.repo || project,
+      git_branch: git.branch || null,
+      category: d.status || null,
+      duration_ms: d.started_at && d.ended_at ? new Date(d.ended_at).getTime() - new Date(d.started_at).getTime() : null,
+      measurement: MEASUREMENT.ACTIVITY,
+      metadata: {
+        cwd: d.workspace_root || d.cwd || null,
+        status: d.status ?? null,
+        exit_code: d.exit_code ?? null,
+        team: d.team_name ?? null,
+        agent_messages: messages.assistant,
+        total_messages: messages.total,
+        tokens_reported: false,
+      },
+    });
+    return { offset: ref.stat.size, records: 1 };
+  },
+});
+
+function countMessages(f) {
+  try {
+    const d = JSON.parse(fs.readFileSync(f, 'utf8'));
+    const msgs = Array.isArray(d.messages) ? d.messages : [];
+    return { total: msgs.length, assistant: msgs.filter((m) => m.role === 'assistant').length };
+  } catch {
+    return { total: null, assistant: null };
+  }
+}

package/src/providers/cursor/index.js

@@ -0,0 +1,143 @@
+/**
+ * Cursor — AI code-authorship tracking database.
+ *
+ * Cursor does not expose token counts locally, but it does record what the AI
+ * actually *did*: every AI-authored code hash (with model, file, and
+ * conversation) and per-commit authorship attribution (AI vs human lines).
+ *
+ * That makes it the productivity-correlation source: it measures work output,
+ * which is the thing token counts are so often wrongly assumed to prove. These
+ * records are `measurement: activity` with all token fields null, so they can
+ * never leak into a token total.
+ *
+ * Source: ~/.cursor/ai-tracking/ai-code-tracking.db  (read-only, via node:sqlite)
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { MEASUREMENT } from '../../core/schema.js';
+import { openReadOnly } from '../../core/sqlite.js';
+
+function dbPath(ctx) {
+  const c = ctx?.config?.sources?.cursor?.db;
+  if (c) return c.startsWith('~') ? path.join(os.homedir(), c.slice(1)) : c;
+  return path.join(ctx?.home || os.homedir(), '.cursor', 'ai-tracking', 'ai-code-tracking.db');
+}
+
+export default createProvider({
+  id: 'cursor',
+  name: 'Cursor (AI code activity)',
+  description: 'AI-authored edits and per-commit AI/human line attribution. Activity only — Cursor does not log tokens locally.',
+  measurement: MEASUREMENT.ACTIVITY,
+  requires: ['~/.cursor/ai-tracking/ai-code-tracking.db', 'Node 22.5+ (node:sqlite)'],
+
+  async detect(ctx) {
+    const f = dbPath(ctx);
+    if (!fs.existsSync(f)) return { available: false, detail: 'No Cursor ai-code-tracking.db found' };
+    return { available: true, detail: 'activity source — AI-authored edits + commit attribution', paths: [f] };
+  },
+
+  async fetchUsage(ctx, emit, sourceState) {
+    const f = dbPath(ctx);
+    const db = openReadOnly(f);
+    let records = 0;
+    const notes = [];
+    const cursor = sourceState?.cursor || { edits: 0, commits: 0 };
+
+    try {
+      const edits = db.prepare(
+        `SELECT hash, source, fileExtension, fileName, requestId, conversationId, timestamp, createdAt, model
+           FROM ai_code_hashes WHERE createdAt > ? ORDER BY createdAt ASC LIMIT 200000`,
+      ).all(cursor.edits || 0);
+
+      for (const r of edits) {
+        const ts = r.timestamp || r.createdAt;
+        if (!ts) continue;
+        const project = projectOf(r.fileName);
+        emit({
+          id: `cursor-edit-${r.hash}-${r.createdAt}`,
+          timestamp: new Date(Number(ts)).toISOString(),
+          // `default` / null are Cursor's own placeholders; do not guess a
+          // model. The PROVIDER however is certain — this row came from
+          // Cursor's own tracking DB — so providerHint attributes unmatched
+          // models to cursor instead of an anonymous "unknown" bucket.
+          model: r.model && r.model !== 'default' ? r.model : null,
+          providerHint: 'cursor',
+          client: 'cursor',
+          application: 'Cursor',
+          interfaceSignals: ['cursor'],
+          input_tokens: null, output_tokens: null, cache_read_tokens: null,
+          cache_write_tokens: null, cache_refresh_tokens: null, reasoning_tokens: null,
+          conversation_id: r.conversationId || null,
+          session_id: r.conversationId || null,
+          request_id: r.requestId || null,
+          project,
+          repository: project,
+          category: `ai-edit:${r.source || 'unknown'}`,
+          measurement: MEASUREMENT.ACTIVITY,
+          metadata: {
+            file: r.fileName || null,
+            ext: r.fileExtension || null,
+            attribution_source: r.source || null,
+            tokens_reported: false,
+          },
+        });
+        records++;
+        cursor.edits = Math.max(cursor.edits || 0, Number(r.createdAt));
+      }
+
+      const commits = db.prepare(
+        `SELECT commitHash, branchName, scoredAt, commitDate, commitMessage, linesAdded, linesDeleted,
+                composerLinesAdded, tabLinesAdded, humanLinesAdded, v2AiPercentage
+           FROM scored_commits WHERE scoredAt > ? ORDER BY scoredAt ASC LIMIT 100000`,
+      ).all(cursor.commits || 0);
+
+      for (const r of commits) {
+        const ts = r.commitDate ? Date.parse(r.commitDate) : Number(r.scoredAt);
+        if (!ts || Number.isNaN(ts)) continue;
+        emit({
+          id: `cursor-commit-${r.commitHash}-${r.branchName}`,
+          timestamp: new Date(ts).toISOString(),
+          model: null,
+          client: 'cursor',
+          application: 'Cursor',
+          interfaceSignals: ['cursor'],
+          input_tokens: null, output_tokens: null, cache_read_tokens: null,
+          cache_write_tokens: null, cache_refresh_tokens: null, reasoning_tokens: null,
+          git_branch: r.branchName || null,
+          category: 'commit',
+          measurement: MEASUREMENT.ACTIVITY,
+          metadata: {
+            commit: r.commitHash,
+            message: r.commitMessage || null,
+            lines_added: nOrNull(r.linesAdded),
+            lines_deleted: nOrNull(r.linesDeleted),
+            ai_lines_added: nOrNull(r.composerLinesAdded),
+            tab_lines_added: nOrNull(r.tabLinesAdded),
+            human_lines_added: nOrNull(r.humanLinesAdded),
+            ai_percentage: r.v2AiPercentage === null || r.v2AiPercentage === undefined ? null : Number(r.v2AiPercentage),
+            tokens_reported: false,
+          },
+        });
+        records++;
+        cursor.commits = Math.max(cursor.commits || 0, Number(r.scoredAt));
+      }
+      if (edits.length === 200000) notes.push('edit batch hit the 200k row cap — run refresh again to continue');
+    } finally {
+      db.close();
+    }
+    return { records, cursor, notes };
+  },
+});
+
+function projectOf(file) {
+  if (!file) return null;
+  const parts = String(file).split('/').filter(Boolean);
+  const i = parts.findIndex((p) => p === 'projects' || p === 'repos' || p === 'src' || p === 'work');
+  if (i >= 0 && parts[i + 1]) return parts[i + 1];
+  return parts.length > 1 ? parts[parts.length - 2] : null;
+}
+function nOrNull(v) {
+  return v === null || v === undefined ? null : Number(v);
+}