@vimoxshah/tokenflow 1.1.0

package/src/providers/openai/index.js

@@ -0,0 +1,370 @@
+/**
+ * OpenAI — Codex CLI / Codex IDE / Codex Desktop rollout transcripts.
+ *
+ * Source: ~/.codex/sessions/YYYY/MM/DD/rollout-<ts>-<id>.jsonl
+ *         (+ ~/.codex/archived_sessions)
+ *
+ * ## Token semantics — the important difference from Anthropic
+ *
+ * Codex reports OpenAI's convention, where `input_tokens` is INCLUSIVE of
+ * `cached_input_tokens`. Copying it straight into the schema would count every
+ * cached prompt token twice: once as fresh input and once as cache read. So:
+ *
+ *   input_tokens      = input_tokens - cached_input_tokens   (fresh only)
+ *   cache_read_tokens = cached_input_tokens
+ *   cache_write_tokens = cache_write_input_tokens, or **null** on older
+ *                        sessions that never emitted the field — not 0, because
+ *                        "the CLI didn't report it" and "no cache was written"
+ *                        are different facts
+ *   reasoning_tokens  = reasoning_output_tokens  (a subset of output_tokens)
+ *
+ * ## One record per turn, and why the obvious reading is wrong
+ *
+ * Codex emits a `token_count` event repeatedly while a turn runs, and
+ * `last_token_usage` is **re-reported as the turn's context grows** — 27k, then
+ * 29k, then 30k, ... all describing the same growing conversation. Its
+ * `total_token_usage` is simply a running SUM of those re-reports, so it is not
+ * a usable cumulative counter: on this corpus it over-reports a heavy agent day
+ * by ~45x (82.8B claimed vs 1.8B real), and one 2.5-minute subagent session
+ * "spends" 7.5B tokens.
+ *
+ * The reliable structure is the same streaming-snapshot pattern Anthropic's
+ * transcripts have. Within a turn, the `last_token_usage` series is split into
+ * monotonically non-decreasing RUNS. A run is one growing context; a DROP means
+ * a new one (a context compaction, or a fresh call). The turn's usage is the
+ * sum of each run's maximum.
+ *
+ * That reduces to the identity for simple sessions (one event per turn -> the
+ * event itself) and was cross-checked against an independent gateway billing
+ * log, which agreed to within the same order of magnitude where the naive
+ * reading was off by 45x.
+ *
+ * `metadata.token_count_events` and `metadata.usage_segments` are recorded on
+ * every turn so this reconstruction is auditable rather than a hidden fudge.
+ *
+ * A `model_provider` that is not a known model vendor is recorded as a
+ * **gateway** (a proxy/router) rather than as the vendor, so "who served this"
+ * and "who made this model" stay separate dimensions.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { readLines } from '../../core/jsonl.js';
+import { walk } from '../../core/ingest.js';
+import { MEASUREMENT } from '../../core/schema.js';
+
+const MARKS = ['"token_count"', '"session_meta"', '"thread_settings_applied"', '"turn_context"', '"task_started"', '"task_complete"'];
+const VENDORS = new Set(['openai', 'anthropic', 'google', 'gemini', 'deepseek', 'zai', 'z.ai', 'xai', 'mistral', 'azure', 'bedrock', 'vertex', 'ollama']);
+
+const USAGE_FIELDS = ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', 'output_tokens', 'reasoning_output_tokens'];
+
+export function codexHomes(ctx) {
+  const configured = ctx?.config?.sources?.openai?.paths;
+  if (Array.isArray(configured) && configured.length) return configured.map(expand);
+  const home = ctx?.home || os.homedir();
+  const out = [];
+  if (process.env.CODEX_HOME) out.push(process.env.CODEX_HOME);
+  out.push(path.join(home, '.codex'));
+  return [...new Set(out)].filter((d) => {
+    try {
+      return fs.statSync(path.join(d, 'sessions')).isDirectory();
+    } catch {
+      return false;
+    }
+  });
+}
+
+function expand(p) {
+  return p.startsWith('~') ? path.join(os.homedir(), p.slice(1)) : p;
+}
+
+/**
+ * Accumulate a turn's `last_token_usage` series into disjoint totals.
+ *
+ * Fields are tracked independently but segmented by `total_tokens`, because
+ * that is the value whose monotonicity tells us whether we are still looking at
+ * the same growing context.
+ *
+ * A field that never appears in ANY event stays `null` — "the CLI didn't report
+ * cache writes" is a different fact from "no cache was written".
+ */
+export class TurnUsage {
+  constructor() {
+    this.acc = Object.fromEntries(USAGE_FIELDS.map((f) => [f, 0]));
+    this.seen = Object.fromEntries(USAGE_FIELDS.map((f) => [f, false]));
+    this.run = null;
+    this.prevTotal = -1;
+    this.events = 0;
+    this.segments = 0;
+  }
+
+  add(last) {
+    if (!last) return;
+    const total = num(last.total_tokens);
+    this.events++;
+    if (this.run === null || total >= this.prevTotal) {
+      if (this.run === null) {
+        this.run = Object.fromEntries(USAGE_FIELDS.map((f) => [f, 0]));
+        this.segments++;
+      }
+      for (const f of USAGE_FIELDS) {
+        const v = last[f];
+        if (v === undefined || v === null) continue;
+        this.seen[f] = true;
+        this.run[f] = Math.max(this.run[f], Number(v));
+      }
+    } else {
+      // The context shrank: the previous run is finished.
+      this.closeRun();
+      this.segments++;
+      this.run = Object.fromEntries(USAGE_FIELDS.map((f) => [f, 0]));
+      for (const f of USAGE_FIELDS) {
+        const v = last[f];
+        if (v === undefined || v === null) continue;
+        this.seen[f] = true;
+        this.run[f] = Number(v);
+      }
+    }
+    this.prevTotal = total;
+  }
+
+  closeRun() {
+    if (!this.run) return;
+    for (const f of USAGE_FIELDS) this.acc[f] += this.run[f];
+    this.run = null;
+  }
+
+  /** @returns {Record<string, number|null>} */
+  finish() {
+    this.closeRun();
+    /** @type {Record<string, number|null>} */
+    const out = {};
+    for (const f of USAGE_FIELDS) out[f] = this.seen[f] ? this.acc[f] : null;
+    return out;
+  }
+
+  toJSON() {
+    return { acc: this.acc, seen: this.seen, run: this.run, prevTotal: this.prevTotal, events: this.events, segments: this.segments };
+  }
+
+  static from(o) {
+    const t = new TurnUsage();
+    if (!o) return t;
+    Object.assign(t, { acc: o.acc || t.acc, seen: o.seen || t.seen, run: o.run ?? null, prevTotal: o.prevTotal ?? -1, events: o.events || 0, segments: o.segments || 0 });
+    return t;
+  }
+}
+
+export default createProvider({
+  id: 'openai',
+  name: 'OpenAI (Codex CLI / IDE / Desktop)',
+  description: 'Per-turn token usage from Codex rollout transcripts, including gateway-routed models.',
+  measurement: MEASUREMENT.PRIMARY,
+  requires: ['~/.codex/sessions (Codex CLI 0.1x+)'],
+
+  async detect(ctx) {
+    const homes = codexHomes(ctx);
+    if (!homes.length) return { available: false, detail: 'No ~/.codex/sessions directory found' };
+    return { available: true, detail: homes.join(', '), paths: homes };
+  },
+
+  async discover(ctx) {
+    const files = [];
+    for (const home of codexHomes(ctx)) {
+      for (const sub of ['sessions', 'archived_sessions']) {
+        const root = path.join(home, sub);
+        if (!fs.existsSync(root)) continue;
+        for (const f of walk(root, (n) => n.endsWith('.jsonl'))) {
+          let stat;
+          try { stat = fs.statSync(f); } catch { continue; }
+          if (!stat.size) continue;
+          files.push({ key: `${sub}:${path.relative(root, f)}`, path: f, stat });
+        }
+      }
+    }
+    return files;
+  },
+
+  async ingestFile(ref, ctx, emit) {
+    const s = ref.state;
+    let records = 0;
+    let malformed = 0;
+
+    /** The turn currently being accumulated. */
+    let turn = s.openTurn
+      ? { id: s.openTurn.id, firstTs: s.openTurn.firstTs, lastTs: s.openTurn.lastTs, usage: TurnUsage.from(s.openTurn.usage), continued: true }
+      : null;
+
+    const newTurn = (id, ts) => ({ id: id || null, firstTs: ts, lastTs: ts, usage: new TurnUsage(), continued: false });
+
+    const flushTurn = () => {
+      if (!turn || !turn.usage.events) { turn = null; return; }
+      const u = turn.usage.finish();
+      const routeProvider = (s.routeProvider || '').toLowerCase();
+      const isVendor = VENDORS.has(routeProvider);
+      const project = s.cwd ? path.basename(s.cwd) : null;
+      const inTotal = u.input_tokens;
+      const cached = u.cached_input_tokens;
+      // OpenAI's input_tokens INCLUDES cached_input_tokens; the schema wants
+      // them mutually exclusive.
+      const fresh = inTotal === null ? null : cached === null ? inTotal : Math.max(0, inTotal - cached);
+      if (!fresh && !cached && !u.output_tokens && !u.cache_write_input_tokens) { turn = null; return; }
+      emit({
+        timestamp: turn.lastTs,
+        model: s.model || null,
+        providerHint: isVendor ? routeProvider : null,
+        gateway: routeProvider && !isVendor ? routeProvider : null,
+        input_tokens: fresh,
+        cache_read_tokens: cached,
+        cache_write_tokens: u.cache_write_input_tokens,
+        output_tokens: u.output_tokens,
+        reasoning_tokens: u.reasoning_output_tokens,
+        cache_refresh_tokens: null,
+        session_id: s.sessionId || ref.key,
+        conversation_id: s.threadId || null,
+        request_id: turn.id,
+        project,
+        repository: project,
+        category: s.threadSource === 'subagent' ? 'subagent' : 'main',
+        client: 'codex',
+        application: 'Codex',
+        interfaceSignals: [s.source, s.originator, s.threadSource],
+        duration_ms: s.lastDuration ?? null,
+        metadata: {
+          cwd: s.cwd || null,
+          cli_version: s.cliVersion || null,
+          originator: s.originator || null,
+          source: s.source || null,
+          agent_role: s.agentRole || null,
+          reasoning_effort: s.effort || null,
+          service_tier: s.tier || null,
+          context_window: s.contextWindow ?? null,
+          time_to_first_token_ms: s.lastTtft ?? null,
+          plan_type: s.planType ?? null,
+          token_count_events: turn.usage.events,
+          usage_segments: turn.usage.segments,
+          turn_continued: turn.continued || undefined,
+        },
+      });
+      records++;
+      s.lastDuration = null;
+      s.lastTtft = null;
+      turn = null;
+    };
+
+    const res = readLines(
+      ref.path,
+      (line) => {
+        let d;
+        try {
+          d = JSON.parse(line);
+        } catch {
+          malformed++;
+          return;
+        }
+        const p = d.payload || {};
+
+        if (d.type === 'session_meta') {
+          s.sessionId = p.session_id || p.id || null;
+          s.threadId = p.id || null;
+          s.cwd = p.cwd || s.cwd || null;
+          s.originator = p.originator || s.originator || null;
+          s.source = flattenSource(p.source) || s.source || null;
+          s.agentRole = agentRole(p.source) || s.agentRole || null;
+          s.threadSource = p.thread_source || s.threadSource || null;
+          s.cliVersion = p.cli_version || s.cliVersion || null;
+          s.routeProvider = p.model_provider || s.routeProvider || null;
+          if (p.model) s.model = p.model;
+          return;
+        }
+        if (d.type === 'turn_context') {
+          if (p.cwd) s.cwd = p.cwd;
+          if (p.model) s.model = p.model;
+          if (p.turn_id && (!turn || turn.id !== p.turn_id)) {
+            flushTurn();
+            turn = newTurn(p.turn_id, d.timestamp);
+          }
+          return;
+        }
+        if (d.type === 'event_msg' && p.type === 'thread_settings_applied') {
+          const t = p.thread_settings || {};
+          if (t.model) s.model = t.model;
+          if (t.model_provider_id) s.routeProvider = t.model_provider_id;
+          if (t.cwd) s.cwd = t.cwd;
+          if (t.reasoning_effort) s.effort = t.reasoning_effort;
+          if (t.service_tier) s.tier = t.service_tier;
+          return;
+        }
+        if (d.type === 'event_msg' && p.type === 'task_started') {
+          if (!turn || turn.id !== p.turn_id) {
+            flushTurn();
+            turn = newTurn(p.turn_id, d.timestamp);
+          }
+          if (p.model_context_window) s.contextWindow = p.model_context_window;
+          return;
+        }
+        if (d.type === 'event_msg' && p.type === 'task_complete') {
+          s.lastDuration = p.duration_ms ?? null;
+          s.lastTtft = p.time_to_first_token_ms ?? null;
+          if (turn && (turn.id === null || !p.turn_id || turn.id === p.turn_id)) flushTurn();
+          return;
+        }
+        if (!(d.type === 'event_msg' && p.type === 'token_count')) return;
+
+        const info = p.info || {};
+        const last = info.last_token_usage || info.total_token_usage;
+        if (!last) return;
+        if (info.model_context_window) s.contextWindow = info.model_context_window;
+        if (p.rate_limits?.plan_type) s.planType = p.rate_limits.plan_type;
+
+        // Sessions from older builds have no turn markers; treat each event as
+        // its own turn rather than dropping the usage.
+        if (!turn) turn = newTurn(null, d.timestamp);
+        turn.usage.add(last);
+        turn.lastTs = d.timestamp;
+        if (turn.id === null) flushTurn();
+      },
+      { start: ref.start, must: MARKS },
+    );
+
+    // A turn still open at EOF keeps accumulating on the next refresh, so no
+    // partial record is written for it and none is written twice.
+    s.openTurn = turn && turn.usage.events
+      ? { id: turn.id, firstTs: turn.firstTs, lastTs: turn.lastTs, usage: turn.usage.toJSON() }
+      : null;
+    if (turn && !turn.id) flushTurn();
+
+    return { offset: res.offset, records, malformed, openTurn: s.openTurn ? 1 : 0 };
+  },
+});
+
+function flattenSource(src) {
+  if (!src) return null;
+  if (typeof src === 'string') return src;
+  if (typeof src === 'object') {
+    const k = Object.keys(src)[0];
+    if (!k) return null;
+    const v = src[k];
+    if (typeof v === 'string') return `${k}:${v}`;
+    if (v && typeof v === 'object') {
+      const inner = Object.values(v)[0];
+      return typeof inner === 'string' ? `${k}:${inner}` : k;
+    }
+    return k;
+  }
+  return null;
+}
+
+/** "luna_worker" from a subagent thread_spawn descriptor, when present. */
+function agentRole(src) {
+  try {
+    return src?.subagent?.thread_spawn?.agent_role ?? src?.subagent?.other ?? null;
+  } catch {
+    return null;
+  }
+}
+
+function num(v) {
+  return v === undefined || v === null ? 0 : Number(v);
+}

package/src/providers/opencode/index.js

@@ -0,0 +1,245 @@
+/**
+ * OpenCode — opencode session/message database.
+ *
+ * Source: <data-home>/opencode/opencode.db  (SQLite; $XDG_DATA_HOME-aware,
+ * so ~/.local/share/opencode/opencode.db by default). Read-only, via
+ * node:sqlite, read from a snapshot like every SQLite adapter here.
+ *
+ * ## What this source is
+ *
+ * One row per message in `message`, with a JSON `data` column. Assistant rows
+ * carry the usage block the provider returned:
+ *
+ *     data.tokens = { input, output, reasoning, cache: { read, write }, total }
+ *     data.modelID / data.providerID / data.cost / data.time.{created,completed}
+ *
+ * The `session` table adds directory, agent and parent (subagent) links; the
+ * `project` table adds the worktree.
+ *
+ * ## Token semantics (verified against a live corpus)
+ *
+ *   input        fresh prompt tokens, EXCLUSIVE of cache reads. Confirmed
+ *                empirically: cache_read regularly exceeds input on long
+ *                sessions, which is impossible under an inclusive convention,
+ *                and data.tokens.total == input+output+reasoning+read+write
+ *                exactly, i.e. the fields are disjoint addends.
+ *   output       generated tokens. `reasoning` is USUALLY a subset of output
+ *                (OpenAI/Anthropic convention) but some providers behind the
+ *                gateway report it additively — 5% of a real corpus had
+ *                reasoning > output. Where that happens the only reading that
+ *                satisfies the schema's subset invariant without losing tokens
+ *                is output += reasoning, with reasoning kept as the subset.
+ *   cache.read   -> cache_read_tokens
+ *   cache.write  -> cache_write_tokens
+ *
+ * ## The incremental trap: rows are UPDATED in place
+ *
+ * An assistant row is inserted when the request starts and its tokens are
+ * revised as the response finalises — on a real corpus every single
+ * token-bearing row had time_updated != time_created. A naive
+ * "time_created > cursor" cursor would freeze the first, partial snapshot
+ * forever. So this adapter uses the same tails mechanism as the Anthropic
+ * adapter: re-read a recent window ordered by time_updated, remember per
+ * message what has already been emitted, and emit only the DELTA when a
+ * message's totals grew. Each emission carries the message's own
+ * time.created, so usage stays attributed to the request that produced it.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { createProvider } from '../../core/registry.js';
+import { MEASUREMENT, hashId } from '../../core/schema.js';
+import { openReadOnly, sqliteAvailable } from '../../core/sqlite.js';
+
+const ROW_CAP = 500000;
+/** How far behind the watermark to re-read, to catch in-place updates. */
+const OVERLAP_MS = 7 * 24 * 60 * 60 * 1000;
+/** Emitted-tail entries older than this, relative to the watermark, are pruned. */
+const TAIL_TTL_MS = 30 * 24 * 60 * 60 * 1000;
+
+const FIELDS = ['input_tokens', 'output_tokens', 'cache_read_tokens', 'cache_write_tokens', 'reasoning_tokens'];
+
+/** providerID values that ARE the model vendor; anything else routes through a gateway. */
+const VENDORS = new Set([
+  'anthropic', 'openai', 'google', 'gemini', 'deepseek', 'xai', 'groq',
+  'mistral', 'azure', 'bedrock', 'vertex', 'ollama', 'amazon', 'moonshot',
+  'alibaba', 'qwen', 'zai', 'github-copilot',
+]);
+
+export function dbPath(ctx) {
+  const c = ctx?.config?.sources?.opencode;
+  const p = c?.db || (Array.isArray(c?.paths) ? c.paths[0] : null);
+  if (p) return expand(p);
+  const dataHome = process.env.XDG_DATA_HOME || path.join(ctx?.home || os.homedir(), '.local', 'share');
+  return path.join(dataHome, 'opencode', 'opencode.db');
+}
+
+function expand(p) {
+  return p.startsWith('~') ? path.join(os.homedir(), p.slice(1)) : p;
+}
+
+export default createProvider({
+  id: 'opencode',
+  name: 'OpenCode',
+  description: 'Per-request token usage from the opencode message database.',
+  measurement: MEASUREMENT.PRIMARY,
+  requires: ['~/.local/share/opencode/opencode.db', 'Node 22.5+ (node:sqlite)'],
+
+  async detect(ctx) {
+    if (!sqliteAvailable()) return { available: false, detail: 'node:sqlite unavailable — needs Node 22.5+' };
+    const f = dbPath(ctx);
+    if (!fs.existsSync(f)) {
+      return { available: false, detail: 'No opencode.db found (looked in $XDG_DATA_HOME/opencode)' };
+    }
+    return { available: true, detail: shortPath(f, ctx), paths: [f] };
+  },
+
+  async fetchUsage(ctx, emit, sourceState) {
+    const f = dbPath(ctx);
+    const db = openReadOnly(f);
+    let records = 0;
+    let skipped = 0;
+    let updated = 0;
+    const notes = [];
+
+    try {
+      const cursor = sourceState.cursor || { updated: 0 };
+      // Tails: per-message totals already emitted. Keyed by message id, they
+      // make re-reading the overlap window idempotent (delta 0 => no record).
+      const tails = sourceState.tails || (sourceState.tails = {});
+
+      const rows = db.prepare(
+        `SELECT m.id AS msg_id, m.session_id AS session_id, m.time_created AS time_created,
+                m.time_updated AS time_updated, m.data AS data,
+                s.directory AS directory, s.parent_id AS parent_id, s.agent AS agent,
+                s.version AS cli_version, p.worktree AS worktree
+           FROM message m
+           LEFT JOIN session s ON s.id = m.session_id
+           LEFT JOIN project p ON p.id = s.project_id
+          WHERE json_extract(m.data,'$.role') = 'assistant'
+            AND json_extract(m.data,'$.tokens') IS NOT NULL
+            AND m.time_updated > ?
+          ORDER BY m.time_updated ASC
+          LIMIT ?`,
+      ).all(Math.max(0, (cursor.updated || 0) - OVERLAP_MS), ROW_CAP);
+      if (rows.length === ROW_CAP) notes.push('row cap hit — run refresh again to continue');
+
+      let watermark = cursor.updated || 0;
+
+      for (const r of rows) {
+        watermark = Math.max(watermark, Number(r.time_updated) || 0);
+        let d;
+        try { d = JSON.parse(r.data); } catch { continue; }
+        const t = d.tokens || {};
+        const created = Number(r.time_created);
+        if (!created) continue;
+
+        const base = {
+          input_tokens: n(t.input),
+          output_tokens: n(t.output),
+          cache_read_tokens: n(t.cache?.read),
+          cache_write_tokens: n(t.cache?.write),
+          reasoning_tokens: n(t.reasoning),
+        };
+        // Some providers report reasoning additively rather than as a subset
+        // of output. Folding keeps the subset invariant without losing tokens.
+        if (base.reasoning_tokens !== null && base.output_tokens !== null && base.reasoning_tokens > base.output_tokens) {
+          base.output_tokens += base.reasoning_tokens;
+        }
+
+        // An assistant row with no counts at all is an aborted request, not a
+        // zero-cost API call. Skip it; if tokens arrive later the row's
+        // time_updated advances and the next pass picks it up fresh.
+        if (FIELDS.every((k) => !base[k])) { skipped++; continue; }
+
+        const key = String(r.msg_id || `${r.session_id}:${created}`);
+        const prev = tails[key];
+        const delta = {};
+        let any = false;
+        for (const k of FIELDS) {
+          const was = prev ? (prev.f[k] ?? 0) : 0;
+          const now = base[k] ?? 0;
+          delta[k] = Math.max(0, now - was);
+          if (delta[k] > 0) any = true;
+        }
+        // Remember what this message accounts for, whether or not it grew —
+        // an unchanged re-read must stay a no-op forever. The emission index
+        // keeps each delta's record id distinct and stable across runs.
+        const emissionIndex = prev ? (prev.n || 0) : 0;
+        tails[key] = { u: Number(r.time_updated) || created, f: { ...base }, n: emissionIndex + 1 };
+        if (prev && !any) continue;
+        updated++;
+
+        const providerID = str(d.providerID)?.toLowerCase() || null;
+        const isVendor = providerID && VENDORS.has(providerID);
+        const project = projectOf(r.worktree || r.directory);
+        const completed = n(d.time?.completed);
+
+        emit({
+          id: hashId('opencode', key, '#', emissionIndex),
+          timestamp: new Date(created).toISOString(),
+          model: str(d.modelID),
+          gateway: providerID && !isVendor ? providerID : null,
+          ...pick(delta),
+          session_id: r.session_id || null,
+          conversation_id: r.session_id || null,
+          request_id: r.msg_id || null,
+          project,
+          repository: project,
+          category: r.parent_id ? 'subagent' : (str(r.agent) || 'main'),
+          client: 'opencode',
+          application: 'OpenCode',
+          interfaceSignals: [],
+          duration_ms: completed !== null && completed >= created ? completed - created : null,
+          metadata: {
+            cwd: str(r.directory),
+            provider_id: providerID,
+            mode: str(d.mode),
+            agent: str(r.agent),
+            finish: str(d.finish),
+            cli_version: str(r.cli_version),
+            reported_total: n(t.total),
+            ...(prev ? { continuation_of: key } : {}),
+          },
+        });
+        records++;
+      }
+
+      // Bound the tail state: anything this far behind the watermark cannot
+      // reappear inside the overlap window, so its emitted totals are final.
+      const floor = watermark - TAIL_TTL_MS;
+      for (const k of Object.keys(tails)) if ((tails[k].u || 0) < floor) delete tails[k];
+
+      cursor.updated = watermark;
+      sourceState.cursor = cursor;
+    } finally {
+      db.close();
+    }
+    return { records, cursor: sourceState.cursor, notes };
+  },
+});
+
+function pick(delta) {
+  const out = {};
+  for (const k of FIELDS) out[k] = delta[k];
+  return out;
+}
+
+function projectOf(dir) {
+  if (!dir) return null;
+  const parts = String(dir).split('/').filter(Boolean);
+  return parts.length ? parts[parts.length - 1] : null;
+}
+
+function shortPath(p, ctx) {
+  const home = ctx?.home || os.homedir();
+  return p.startsWith(home) ? '~' + p.slice(home.length) : p;
+}
+
+function n(v) {
+  return v === undefined || v === null ? null : Number(v);
+}
+
+function str(v) {
+  return v === undefined || v === null || v === '' ? null : String(v);
+}

package/src/sdk.js

@@ -0,0 +1,46 @@
+/**
+ * The provider SDK — the minimum surface a community adapter needs.
+ *
+ *   import { createProvider, registerProvider, normalizeUsage, validateUsage }
+ *     from 'tokenflow/sdk';
+ *
+ * See docs/creating-provider.md for the full contract.
+ */
+export { createProvider, registerProvider, getProvider, listProviders } from './core/registry.js';
+export { validateUsage, validateProvider } from './core/validate.js';
+export {
+  createRecord, computeTotal, dateParts, hashId,
+  MEASUREMENT, INTERFACE, INTERFACE_ORDER, interfaceClass,
+  BILLABLE_TOKEN_FIELDS, BREAKDOWN_TOKEN_FIELDS, TOKEN_FIELDS,
+} from './core/schema.js';
+export { classifyModel, BUILTIN_MODEL_RULES } from './core/model-map.js';
+export { classifyInterface } from './core/interface-map.js';
+export { readLines, readJsonLines } from './core/jsonl.js';
+export { walk, enrich } from './core/ingest.js';
+export { openReadOnly, tables, columns, sqliteAvailable } from './core/sqlite.js';
+export { registerAnalytics, listAnalytics } from './analytics/index.js';
+import { enrich } from './core/ingest.js';
+
+/**
+ * Normalize a partial record without running a full ingest — handy in adapter
+ * unit tests and in `--dry-run` importers.
+ * @param {object} partial
+ * @param {{provider:{id:string,name:string,measurement?:string}, tz?:string|null, priceBook?:object,
+ *   rules?:object[], config?:object, user?:string|null, machine?:string|null,
+ *   seq?:number, fileRef?:object|null}} opt
+ */
+export function normalizeUsage(partial, opt) {
+  return enrich(partial, {
+    ctx: {
+      tz: opt.tz ?? null,
+      priceBook: opt.priceBook ?? { lookup: () => null },
+      rules: opt.rules,
+      config: opt.config ?? {},
+      user: opt.user ?? null,
+      machine: opt.machine ?? null,
+    },
+    provider: opt.provider,
+    seq: opt.seq ?? 0,
+    fileRef: opt.fileRef ?? null,
+  });
+}