@vimoxshah/tokenflow 1.1.0

package/src/providers/generic/index.js

@@ -0,0 +1,268 @@
+/**
+ * Generic importer — the escape hatch that keeps this project useful for
+ * providers nobody has written an adapter for yet.
+ *
+ * Accepts CSV / TSV / JSON / JSONL / SQLite, driven by a saved field mapping
+ * so the second import of the same export needs no configuration at all.
+ *
+ * A mapping lives at $TOKENFLOW_HOME/mappings/<name>.json:
+ *
+ * {
+ *   "name": "openrouter-export",
+ *   "format": "csv",                  // csv | tsv | json | jsonl | sqlite
+ *   "files": ["~/Downloads/openrouter-*.csv"],
+ *   "table": "usage",                 // sqlite only
+ *   "timestampFormat": "iso",         // iso | epoch_ms | epoch_s
+ *   "defaults": { "client": "openrouter", "interface": "API" },
+ *   "fields": {
+ *     "timestamp":          "created_at",
+ *     "model":              "model",
+ *     "input_tokens":       "prompt_tokens",
+ *     "output_tokens":      "completion_tokens",
+ *     "cache_read_tokens":  "cached_tokens",
+ *     "estimated_cost":     "cost_usd",
+ *     "session_id":         "generation_id"
+ *   }
+ * }
+ *
+ * Unmapped token fields stay `null` — the importer never fills a gap with 0.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { readLines } from '../../core/jsonl.js';
+import { paths } from '../../core/config.js';
+import { MEASUREMENT } from '../../core/schema.js';
+import { openReadOnly } from '../../core/sqlite.js';
+
+export const MAPPABLE_FIELDS = [
+  'timestamp', 'model', 'provider', 'client', 'application', 'interface',
+  'input_tokens', 'output_tokens', 'cache_read_tokens', 'cache_write_tokens',
+  'cache_refresh_tokens', 'reasoning_tokens', 'total_tokens',
+  'session_id', 'conversation_id', 'request_id',
+  'project', 'repository', 'git_branch', 'category', 'estimated_cost', 'user',
+];
+
+export function loadMappings(ctx) {
+  const dir = paths().mappings;
+  const out = [];
+  let files = [];
+  try {
+    files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
+  } catch {
+    files = [];
+  }
+  for (const f of files) {
+    try {
+      const m = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
+      m.name = m.name || f.replace(/\.json$/, '');
+      out.push(m);
+    } catch { /* skip unreadable mapping */ }
+  }
+  for (const m of ctx?.config?.sources?.generic?.imports || []) out.push(m);
+  return out;
+}
+
+function expandGlobs(patterns) {
+  const out = [];
+  for (const raw of patterns || []) {
+    const p = raw.startsWith('~') ? path.join(os.homedir(), raw.slice(1)) : raw;
+    if (!p.includes('*')) {
+      if (fs.existsSync(p)) out.push(p);
+      continue;
+    }
+    const dir = path.dirname(p);
+    const rx = new RegExp('^' + path.basename(p).replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$');
+    try {
+      for (const f of fs.readdirSync(dir)) if (rx.test(f)) out.push(path.join(dir, f));
+    } catch { /* directory missing */ }
+  }
+  return out;
+}
+
+export default createProvider({
+  id: 'generic',
+  name: 'Generic import (CSV / JSON / JSONL / SQLite)',
+  description: 'Import any usage export with a saved field mapping. Use this for providers without a dedicated adapter.',
+  measurement: MEASUREMENT.PRIMARY,
+  requires: ['a mapping in $TOKENFLOW_HOME/mappings/*.json'],
+
+  async detect(ctx) {
+    const maps = loadMappings(ctx);
+    if (!maps.length) {
+      return { available: false, detail: 'no import mappings defined (see `tokenflow import --help`)' };
+    }
+    const files = maps.flatMap((m) => expandGlobs(m.files));
+    return {
+      available: files.length > 0,
+      detail: `${maps.length} mapping(s), ${files.length} file(s)`,
+      paths: files,
+    };
+  },
+
+  async discover(ctx) {
+    const out = [];
+    for (const m of loadMappings(ctx)) {
+      for (const f of expandGlobs(m.files)) {
+        let stat;
+        try { stat = fs.statSync(f); } catch { continue; }
+        out.push({ key: `${m.name}:${path.basename(f)}`, path: f, stat, mapping: m });
+      }
+    }
+    return out;
+  },
+
+  async ingestFile(ref, ctx, emit) {
+    const m = ref.mapping;
+    const fmt = (m.format || inferFormat(ref.path)).toLowerCase();
+    let records = 0;
+    let malformed = 0;
+    let offset = ref.stat.size;
+    let seq = 0;
+
+    const push = (row) => {
+      const rec = mapRow(row, m, seq++);
+      if (!rec) { malformed++; return; }
+      emit(rec);
+      records++;
+    };
+
+    if (fmt === 'jsonl') {
+      const res = readLines(ref.path, (line) => {
+        try { push(JSON.parse(line)); } catch { malformed++; }
+      }, { start: ref.start });
+      offset = res.offset;
+    } else if (fmt === 'json') {
+      const d = JSON.parse(fs.readFileSync(ref.path, 'utf8'));
+      const rows = Array.isArray(d) ? d : (d.records || d.data || d.usage || []);
+      for (const r of rows) push(r);
+    } else if (fmt === 'csv' || fmt === 'tsv') {
+      const rows = parseDelimited(fs.readFileSync(ref.path, 'utf8'), fmt === 'tsv' ? '\t' : ',');
+      for (const r of rows) push(r);
+    } else if (fmt === 'sqlite') {
+      const db = openReadOnly(ref.path);
+      try {
+        const sql = m.query || `SELECT * FROM "${String(m.table).replace(/"/g, '""')}"`;
+        for (const r of db.prepare(sql).all()) push(r);
+      } finally {
+        db.close();
+      }
+    } else {
+      throw new Error(`unsupported format "${fmt}" for ${ref.path}`);
+    }
+    return { offset, records, malformed };
+  },
+
+  /** Exposed for tests and for `tokenflow import --dry-run`. */
+  normalize(row, mapping) {
+    return mapRow(row, mapping, 0);
+  },
+});
+
+export function mapRow(row, m, seq) {
+  const f = m.fields || {};
+  const get = (name) => {
+    const src = f[name];
+    if (!src) return undefined;
+    if (typeof src === 'string') return row[src];
+    if (src && typeof src === 'object' && src.const !== undefined) return src.const;
+    return undefined;
+  };
+  const ts = normalizeTs(get('timestamp'), m.timestampFormat);
+  if (!ts) return null;
+
+  const out = {
+    id: m.idField && row[m.idField] ? `generic-${m.name}-${row[m.idField]}` : undefined,
+    timestamp: ts,
+    model: str(get('model')) ?? m.defaults?.model ?? null,
+    provider: str(get('provider')) ?? m.defaults?.provider ?? undefined,
+    client: str(get('client')) ?? m.defaults?.client ?? 'generic',
+    application: str(get('application')) ?? m.defaults?.application ?? m.name,
+    interface: str(get('interface')) ?? m.defaults?.interface ?? undefined,
+    interfaceSignals: [str(get('interface')), m.defaults?.interface, str(get('client')), m.defaults?.client],
+    session_id: str(get('session_id')),
+    conversation_id: str(get('conversation_id')),
+    request_id: str(get('request_id')),
+    project: str(get('project')) ?? m.defaults?.project ?? null,
+    repository: str(get('repository')),
+    git_branch: str(get('git_branch')),
+    category: str(get('category')),
+    user: str(get('user')) ?? m.defaults?.user ?? null,
+    measurement: m.measurement || MEASUREMENT.PRIMARY,
+    metadata: { import: m.name, ...(m.metadata || {}) },
+  };
+  for (const tf of ['input_tokens', 'output_tokens', 'cache_read_tokens', 'cache_write_tokens', 'cache_refresh_tokens', 'reasoning_tokens']) {
+    out[tf] = numOrNull(get(tf));
+  }
+  const cost = numOrNull(get('estimated_cost'));
+  if (cost !== null) out.measured_cost = cost;
+  return out;
+}
+
+function inferFormat(p) {
+  const e = path.extname(p).toLowerCase();
+  if (e === '.jsonl' || e === '.ndjson') return 'jsonl';
+  if (e === '.json') return 'json';
+  if (e === '.tsv') return 'tsv';
+  if (e === '.db' || e === '.sqlite' || e === '.sqlite3') return 'sqlite';
+  return 'csv';
+}
+
+export function normalizeTs(v, format) {
+  if (v === undefined || v === null || v === '') return null;
+  if (format === 'epoch_ms' || (typeof v === 'number' && v > 1e11)) {
+    const d = new Date(Number(v));
+    return Number.isNaN(d.getTime()) ? null : d.toISOString();
+  }
+  if (format === 'epoch_s' || (typeof v === 'number' && v > 1e8)) {
+    const d = new Date(Number(v) * 1000);
+    return Number.isNaN(d.getTime()) ? null : d.toISOString();
+  }
+  const d = new Date(v);
+  return Number.isNaN(d.getTime()) ? null : d.toISOString();
+}
+
+/** RFC4180-ish delimited parser: quotes, escaped quotes, embedded newlines. */
+export function parseDelimited(text, delim = ',') {
+  const rows = [];
+  let row = [];
+  let cell = '';
+  let q = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (q) {
+      if (c === '"') {
+        if (text[i + 1] === '"') { cell += '"'; i++; } else q = false;
+      } else cell += c;
+      continue;
+    }
+    if (c === '"') { q = true; continue; }
+    if (c === delim) { row.push(cell); cell = ''; continue; }
+    if (c === '\n' || c === '\r') {
+      if (c === '\r' && text[i + 1] === '\n') i++;
+      row.push(cell); cell = '';
+      if (row.length > 1 || row[0] !== '') rows.push(row);
+      row = [];
+      continue;
+    }
+    cell += c;
+  }
+  if (cell !== '' || row.length) { row.push(cell); rows.push(row); }
+  if (!rows.length) return [];
+  const header = rows[0].map((h) => h.trim().replace(/^/, ''));
+  return rows.slice(1).map((r) => {
+    const o = {};
+    header.forEach((h, i) => { o[h] = r[i]; });
+    return o;
+  });
+}
+
+function str(v) {
+  return v === undefined || v === null || v === '' ? null : String(v);
+}
+function numOrNull(v) {
+  if (v === undefined || v === null || v === '') return null;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+}

package/src/providers/git/index.js

@@ -0,0 +1,188 @@
+/**
+ * Git activity — the correlation source.
+ *
+ * Emits one `activity` record per commit (files changed, insertions,
+ * deletions) so the Productivity section can put AI usage next to actual
+ * shipped work. It never claims causation: the analytics layer labels these
+ * relationships as correlations and refuses to compute one at all when the
+ * two series don't overlap in time.
+ *
+ * Repositories come from, in order:
+ *   1. config `sources.git.repos: [ ... ]`
+ *   2. config `sources.git.scanRoots: [ ... ]` (searched for .git, depth 3)
+ *   3. `sources.git.autoFromUsage: true` — the working directories already
+ *      observed in ingested usage records, which means zero configuration for
+ *      the common case where you code where you prompt
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { execFileSync } from 'node:child_process';
+import { createProvider } from '../../core/registry.js';
+import { MEASUREMENT } from '../../core/schema.js';
+import { paths } from '../../core/config.js';
+import { readJson } from '../../core/store.js';
+
+const SEP = '';
+
+function expand(p) {
+  return p.startsWith('~') ? path.join(os.homedir(), p.slice(1)) : p;
+}
+
+export function resolveRepos(ctx) {
+  const cfg = ctx?.config?.sources?.git || {};
+  const out = new Set();
+  for (const r of cfg.repos || []) if (isRepo(expand(r))) out.add(expand(r));
+  for (const root of cfg.scanRoots || []) {
+    for (const r of scan(expand(root), 3)) out.add(r);
+  }
+  if (cfg.autoFromUsage !== false) {
+    const known = readJson(path.join(paths().data, 'project-paths.json'), { paths: [] });
+    for (const p of known.paths || []) {
+      const r = repoRoot(p);
+      if (r) out.add(r);
+    }
+  }
+  return [...out];
+}
+
+function isRepo(d) {
+  try {
+    return fs.statSync(path.join(d, '.git')).isDirectory() || fs.statSync(path.join(d, '.git')).isFile();
+  } catch {
+    return false;
+  }
+}
+
+function repoRoot(p) {
+  let d = p;
+  for (let i = 0; i < 8 && d && d !== '/'; i++) {
+    if (isRepo(d)) return d;
+    d = path.dirname(d);
+  }
+  return null;
+}
+
+function scan(root, depth) {
+  const out = [];
+  const stack = [[root, 0]];
+  while (stack.length) {
+    const [d, lvl] = stack.pop();
+    if (lvl > depth) continue;
+    if (isRepo(d)) { out.push(d); continue; }
+    let entries = [];
+    try { entries = fs.readdirSync(d, { withFileTypes: true }); } catch { continue; }
+    for (const e of entries) {
+      if (e.isDirectory() && !e.name.startsWith('.') && e.name !== 'node_modules') stack.push([path.join(d, e.name), lvl + 1]);
+    }
+  }
+  return out;
+}
+
+export default createProvider({
+  id: 'git',
+  name: 'Git activity (correlation)',
+  description: 'Commits, files changed and line churn per repository. Activity only — used to correlate AI usage with shipped work.',
+  measurement: MEASUREMENT.ACTIVITY,
+  requires: ['git on PATH', 'at least one repository configured or discoverable'],
+
+  async detect(ctx) {
+    try {
+      execFileSync('git', ['--version'], { stdio: 'ignore' });
+    } catch {
+      return { available: false, detail: 'git is not on PATH' };
+    }
+    const repos = resolveRepos(ctx);
+    if (!repos.length) {
+      return {
+        available: false,
+        detail: 'no repositories found — set sources.git.repos or sources.git.scanRoots in config.yaml',
+      };
+    }
+    return { available: true, detail: `${repos.length} repositor${repos.length === 1 ? 'y' : 'ies'}`, paths: repos };
+  },
+
+  async fetchUsage(ctx, emit, sourceState) {
+    const repos = resolveRepos(ctx);
+    const cursor = sourceState?.cursor || {};
+    const author = ctx.config?.sources?.git?.author || null;
+    let records = 0;
+    const notes = [];
+
+    for (const repo of repos) {
+      const name = path.basename(repo);
+      const since = cursor[repo] || ctx.config?.sources?.git?.since || null;
+      const args = [
+        '-C', repo, 'log', '--no-merges', '--numstat', '--date=iso-strict',
+        `--pretty=format:C${SEP}%H${SEP}%aI${SEP}%an${SEP}%ae${SEP}%D${SEP}%s`,
+      ];
+      if (since) args.push(`--since=${since}`);
+      if (author) args.push(`--author=${author}`);
+      let out;
+      try {
+        out = execFileSync('git', args, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
+      } catch (err) {
+        notes.push(`${name}: ${String(err.message).split('\n')[0]}`);
+        continue;
+      }
+      let cur = null;
+      let newest = since;
+      const flush = () => {
+        if (!cur) return;
+        emit({
+          id: `git-${cur.hash}`,
+          timestamp: cur.date,
+          model: null,
+          client: 'git',
+          application: 'Git',
+          interface: 'CLI',
+          input_tokens: null, output_tokens: null, cache_read_tokens: null,
+          cache_write_tokens: null, cache_refresh_tokens: null, reasoning_tokens: null,
+          project: name,
+          repository: name,
+          git_branch: cur.branch || null,
+          category: 'commit',
+          measurement: MEASUREMENT.ACTIVITY,
+          metadata: {
+            commit: cur.hash,
+            message: cur.subject,
+            author: cur.author,
+            files_changed: cur.files,
+            insertions: cur.ins,
+            deletions: cur.del,
+            repo_path: repo,
+            tokens_reported: false,
+          },
+        });
+        records++;
+        cur = null;
+      };
+      for (const line of out.split('\n')) {
+        if (line.startsWith('C' + SEP)) {
+          flush();
+          const [, hash, date, an, ae, refs, subject] = line.split(SEP);
+          cur = { hash, date, author: an, email: ae, branch: parseRefs(refs), subject: subject || '', files: 0, ins: 0, del: 0 };
+          if (!newest || date > newest) newest = date;
+          continue;
+        }
+        if (!cur || !line.trim()) continue;
+        const m = line.split('\t');
+        if (m.length < 3) continue;
+        cur.files++;
+        if (m[0] !== '-') cur.ins += Number(m[0]) || 0;
+        if (m[1] !== '-') cur.del += Number(m[1]) || 0;
+      }
+      flush();
+      if (newest) cursor[repo] = newest;
+    }
+    return { records, cursor, notes };
+  },
+});
+
+function parseRefs(refs) {
+  if (!refs) return null;
+  const m = refs.match(/HEAD -> ([^,]+)/);
+  if (m) return m[1].trim();
+  const first = refs.split(',')[0].trim();
+  return first && !first.startsWith('tag:') ? first : null;
+}

package/src/providers/headroom/index.js

@@ -0,0 +1,114 @@
+/**
+ * Headroom — local LLM gateway / prompt-compression proxy.
+ *
+ * This is an **overlay** source, and the distinction matters. The proxy sees
+ * the same requests that the Codex adapter already recorded from the client
+ * side. Adding both into one total would double-count every routed token, so
+ * overlay records are excluded from token totals by default
+ * (`analytics.includeOverlaySources: false`) and used for what only the
+ * gateway knows:
+ *
+ *   - a **measured** dollar cost per request (`cost_usd`), as opposed to our
+ *     own price-table estimate — which gives the Cost section an independent
+ *     cross-check instead of a single unverifiable number
+ *   - the compression delta (`before` -> `after` prompt tokens), i.e. tokens
+ *     the client believed it sent versus tokens actually billed
+ *
+ * `input_tokens` is the post-compression count that was really sent. Output
+ * and cache are `null`: the savings log does not carry them.
+ *
+ * Source: ~/.headroom/savings_events.jsonl
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { readLines } from '../../core/jsonl.js';
+import { MEASUREMENT } from '../../core/schema.js';
+
+function dir(ctx) {
+  const c = ctx?.config?.sources?.headroom?.path;
+  if (c) return c.startsWith('~') ? path.join(os.homedir(), c.slice(1)) : c;
+  return path.join(ctx?.home || os.homedir(), '.headroom');
+}
+
+export default createProvider({
+  id: 'headroom',
+  name: 'Headroom gateway (overlay)',
+  description: 'Measured per-request cost and prompt-compression savings from a local LLM proxy. Overlay: excluded from token totals to avoid double counting.',
+  measurement: MEASUREMENT.OVERLAY,
+  requires: ['~/.headroom/savings_events.jsonl'],
+
+  async detect(ctx) {
+    const f = path.join(dir(ctx), 'savings_events.jsonl');
+    if (!fs.existsSync(f)) return { available: false, detail: 'No ~/.headroom/savings_events.jsonl found' };
+    return { available: true, detail: 'overlay source — measured cost, excluded from token totals', paths: [f] };
+  },
+
+  async discover(ctx) {
+    const d = dir(ctx);
+    const out = [];
+    for (const name of ['savings_events.jsonl']) {
+      const f = path.join(d, name);
+      try {
+        out.push({ key: name, path: f, stat: fs.statSync(f) });
+      } catch { /* absent */ }
+    }
+    return out;
+  },
+
+  async ingestFile(ref, ctx, emit) {
+    let records = 0;
+    let malformed = 0;
+    let seq = 0;
+    const res = readLines(
+      ref.path,
+      (line) => {
+        let d;
+        try {
+          d = JSON.parse(line);
+        } catch {
+          malformed++;
+          return;
+        }
+        if (!d.ts && !d.timestamp) return;
+        const after = num(d.after);
+        emit({
+          id: `headroom-${d.ts || d.timestamp}-${seq++}`,
+          timestamp: d.ts || d.timestamp,
+          model: d.model || null,
+          gateway: 'headroom',
+          client: d.client || 'unknown',
+          application: d.client ? `${d.client} via Headroom` : 'Headroom gateway',
+          // The proxy log has no surface field of its own; the client name is
+          // the only surface evidence available.
+          interfaceSignals: [d.client],
+          input_tokens: after,
+          output_tokens: null,
+          cache_read_tokens: null,
+          cache_write_tokens: null,
+          cache_refresh_tokens: null,
+          reasoning_tokens: null,
+          measured_cost: d.cost_usd === undefined || d.cost_usd === null ? null : Number(d.cost_usd),
+          session_id: null,
+          request_id: null,
+          measurement: MEASUREMENT.OVERLAY,
+          metadata: {
+            prompt_tokens_before_compression: num(d.before),
+            prompt_tokens_after_compression: after,
+            tokens_saved: num(d.saved),
+            proxy_source: d.source ?? null,
+            pid: d.pid ?? null,
+          },
+        });
+        records++;
+      },
+      { start: ref.start },
+    );
+    return { offset: res.offset, records, malformed };
+  },
+});
+
+function num(v) {
+  return v === undefined || v === null ? null : Number(v);
+}