@vimoxshah/tokenflow 1.1.0

package/src/core/units.js

@@ -0,0 +1,147 @@
+/**
+ * Number / date formatting shared by the CLI, the API and the browser bundle.
+ * Kept dependency-free and side-effect-free so it can be inlined into the
+ * static HTML snapshot verbatim.
+ */
+
+const UNITS = [
+  { v: 1e12, s: 'T' },
+  { v: 1e9, s: 'B' },
+  { v: 1e6, s: 'M' },
+  { v: 1e3, s: 'K' },
+];
+
+/**
+ * Compact token formatting. Handles billions/trillions without losing
+ * readability and never renders a raw 12-digit integer.
+ * 1_742_000_000 -> "1.74B"
+ * @param {number|null|undefined} n
+ * @param {{digits?:number, na?:string}} [opt]
+ */
+export function compact(n, opt = {}) {
+  const na = opt.na ?? '—';
+  if (n === null || n === undefined || Number.isNaN(n)) return na;
+  const neg = n < 0;
+  const a = Math.abs(n);
+  if (a < 1000) {
+    const s = Number.isInteger(a) ? String(a) : a.toFixed(opt.digits ?? 1);
+    return (neg ? '-' : '') + s;
+  }
+  for (const u of UNITS) {
+    if (a >= u.v) {
+      const q = a / u.v;
+      // 3 significant figures reads best across K/M/B/T
+      const d = opt.digits !== undefined ? opt.digits : q >= 100 ? 0 : q >= 10 ? 1 : 2;
+      // Trim trailing zeros so 2.50T reads as 2.5T and 1.00B as 1B.
+      const q2 = q.toFixed(d).replace(/(\.\d*?)0+$/, '$1').replace(/\.$/, '');
+      return (neg ? '-' : '') + q2 + u.s;
+    }
+  }
+  return String(n);
+}
+
+/** Thousands-separated integer, for table cells and tooltips. */
+export function int(n, na = '—') {
+  if (n === null || n === undefined || Number.isNaN(n)) return na;
+  return Math.round(n).toLocaleString('en-US');
+}
+
+/** @param {number|null|undefined} n */
+export function usd(n, na = '—') {
+  if (n === null || n === undefined || Number.isNaN(n)) return na;
+  const a = Math.abs(n);
+  if (a === 0) return '$0.00';
+  if (a < 0.01) return '$' + n.toFixed(4);
+  if (a < 1000) return '$' + n.toFixed(2);
+  if (a < 1e6) return '$' + (n / 1e3).toFixed(1) + 'K';
+  return '$' + (n / 1e6).toFixed(2) + 'M';
+}
+
+/** @param {number|null|undefined} f fraction 0..1 */
+export function pct(f, digits = 1, na = '—') {
+  if (f === null || f === undefined || Number.isNaN(f) || !Number.isFinite(f)) return na;
+  return (f * 100).toFixed(digits) + '%';
+}
+
+/** Signed percentage change, for comparison mode and trend chips. */
+export function delta(cur, prev, digits = 0) {
+  if (prev === null || prev === undefined || cur === null || cur === undefined) return null;
+  if (prev === 0) return cur === 0 ? 0 : null; // undefined growth from zero base
+  return (cur - prev) / prev;
+}
+
+export function signedPct(f, digits = 0) {
+  if (f === null || f === undefined || !Number.isFinite(f)) return '—';
+  const s = f > 0 ? '+' : '';
+  return s + (f * 100).toFixed(digits) + '%';
+}
+
+const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+export const DOW = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'];
+
+/** "2026-08-12" -> "Aug 12" */
+export function shortDate(iso) {
+  if (!iso) return '—';
+  const [y, m, d] = iso.split('-').map(Number);
+  return `${MONTHS[m - 1]} ${d}`;
+}
+
+/** "2026-08-12" -> "Aug 12, 2026" */
+export function longDate(iso) {
+  if (!iso) return '—';
+  const [y, m, d] = iso.split('-').map(Number);
+  return `${MONTHS[m - 1]} ${d}, ${y}`;
+}
+
+/** Hour label for axis ticks: 0 -> "00", 14 -> "14". */
+export function hourLabel(h) {
+  return String(h).padStart(2, '0');
+}
+
+/** "10:00 – 13:00" from an inclusive hour window. */
+export function hourWindow(a, b) {
+  return `${hourLabel(a)}:00 – ${hourLabel((b + 1) % 24)}:00`;
+}
+
+export function humanDuration(ms) {
+  if (ms === null || ms === undefined) return '—';
+  const s = Math.round(ms / 1000);
+  if (s < 60) return s + 's';
+  const m = Math.floor(s / 60);
+  if (m < 60) return m + 'm ' + (s % 60) + 's';
+  const h = Math.floor(m / 60);
+  if (h < 24) return h + 'h ' + (m % 60) + 'm';
+  return Math.floor(h / 24) + 'd ' + (h % 24) + 'h';
+}
+
+export function relativeTime(iso, now = Date.now()) {
+  if (!iso) return 'never';
+  const diff = now - new Date(iso).getTime();
+  if (diff < 0) return 'just now';
+  const s = Math.floor(diff / 1000);
+  if (s < 45) return 'just now';
+  if (s < 90) return '1 min ago';
+  const m = Math.floor(s / 60);
+  if (m < 60) return `${m} min ago`;
+  const h = Math.floor(m / 60);
+  if (h < 24) return `${h} hour${h === 1 ? '' : 's'} ago`;
+  const d = Math.floor(h / 24);
+  if (d < 30) return `${d} day${d === 1 ? '' : 's'} ago`;
+  return longDate(iso.slice(0, 10));
+}
+
+/**
+ * "2h 13m" / "3d 4h" / "45s" from milliseconds — empty unit tails are dropped
+ * ("5h", not "5h 0m"). Null-safe: unknown stays null rather than pretending.
+ */
+export function countdown(ms) {
+  if (ms === null || ms === undefined || !Number.isFinite(ms)) return null;
+  const s = Math.max(0, Math.round(ms / 1000));
+  if (s < 60) return `${s}s`;
+  const m = Math.floor(s / 60);
+  if (m < 60) return `${m}m`;
+  const h = Math.floor(m / 60);
+  if (h < 48) return m % 60 ? `${h}h ${m % 60}m` : `${h}h`;
+  const d = Math.floor(h / 24);
+  return h % 24 ? `${d}d ${h % 24}h` : `${d}d`;
+}

package/src/core/validate.js

@@ -0,0 +1,123 @@
+/**
+ * Validation for normalized records and for provider definitions.
+ * Used by the adapter tests, by `tokenflow validate`, and by the generic
+ * importer before it accepts a mapped file.
+ */
+import { BILLABLE_TOKEN_FIELDS, BREAKDOWN_TOKEN_FIELDS, INTERFACE, MEASUREMENT, computeTotal } from './schema.js';
+
+const IFACES = new Set(Object.values(INTERFACE));
+const MEAS = new Set(Object.values(MEASUREMENT));
+
+/**
+ * @param {object} r
+ * @returns {{ok:boolean, errors:string[], warnings:string[]}}
+ */
+export function validateUsage(r) {
+  const errors = [];
+  const warnings = [];
+
+  if (!r || typeof r !== 'object') return { ok: false, errors: ['record is not an object'], warnings };
+
+  if (!r.timestamp || Number.isNaN(new Date(r.timestamp).getTime())) errors.push('timestamp missing or unparseable');
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(r.date || '')) errors.push('date must be YYYY-MM-DD');
+  if (!(Number.isInteger(r.hour) && r.hour >= 0 && r.hour <= 23)) errors.push('hour must be 0-23');
+  if (!(Number.isInteger(r.dow) && r.dow >= 0 && r.dow <= 6)) errors.push('dow must be 0-6 (0=Mon)');
+  if (!r.provider) errors.push('provider missing');
+  if (!r.model) errors.push('model missing');
+  if (!IFACES.has(r.interface)) errors.push(`interface "${r.interface}" is not one of ${[...IFACES].join('|')}`);
+  if (!MEAS.has(r.measurement)) errors.push(`measurement "${r.measurement}" is invalid`);
+  if (!r.source) errors.push('source missing');
+  if (!r.id) errors.push('id missing');
+
+  for (const f of [...BILLABLE_TOKEN_FIELDS, ...BREAKDOWN_TOKEN_FIELDS]) {
+    const v = r[f];
+    if (v === null) continue;
+    if (typeof v !== 'number' || !Number.isFinite(v)) errors.push(`${f} must be a finite number or null`);
+    else if (v < 0) errors.push(`${f} must not be negative`);
+  }
+
+  // Subset invariants — the whole point of separating breakdown fields.
+  if (r.reasoning_tokens !== null && r.output_tokens !== null && r.reasoning_tokens > r.output_tokens) {
+    errors.push('reasoning_tokens exceeds output_tokens (it must be a subset)');
+  }
+  if (r.cache_refresh_tokens !== null && r.cache_write_tokens !== null && r.cache_refresh_tokens > r.cache_write_tokens) {
+    errors.push('cache_refresh_tokens exceeds cache_write_tokens (it must be a subset)');
+  }
+
+  const t = computeTotal(r);
+  if (r.total_tokens !== t.total) errors.push(`total_tokens (${r.total_tokens}) != sum of billable fields (${t.total})`);
+  if (!!r.total_is_partial !== t.partial) errors.push('total_is_partial disagrees with the field availability');
+
+  if (r.measurement === MEASUREMENT.PRIMARY && t.total === null) {
+    warnings.push('primary record with no token data at all — consider measurement "activity"');
+  }
+  if (r.estimated_cost !== null && !r.cost_basis) errors.push('estimated_cost present but cost_basis is null');
+  if (r.cost_basis && r.estimated_cost === null) errors.push('cost_basis present but estimated_cost is null');
+
+  return { ok: errors.length === 0, errors, warnings };
+}
+
+/** @param {object} p provider definition */
+export function validateProvider(p) {
+  const errors = [];
+  if (!p || typeof p !== 'object') return { ok: false, errors: ['provider is not an object'] };
+  if (!p.id || !/^[a-z0-9][a-z0-9-]*$/.test(p.id)) errors.push('id must be a lowercase slug');
+  if (!p.name) errors.push('name is required');
+  if (typeof p.detect !== 'function') errors.push('detect() is required');
+  const hasIngest = typeof p.ingestFile === 'function' || typeof p.fetchUsage === 'function';
+  if (!hasIngest) errors.push('either ingestFile() or fetchUsage() is required');
+  if (typeof p.ingestFile === 'function' && typeof p.discover !== 'function') {
+    errors.push('ingestFile() requires discover() to enumerate source files');
+  }
+  if (p.measurement && !MEAS.has(p.measurement)) errors.push(`measurement "${p.measurement}" is invalid`);
+  return { ok: errors.length === 0, errors };
+}
+
+/** Aggregate data-quality report used by the Data Health indicator. */
+export function summarizeQuality(cube, sessions, state) {
+  const D = cube.dims.length;
+  const mi = (n) => D + cube.measures.indexOf(n);
+  let req = 0, naIn = 0, naOut = 0, naCr = 0, naCw = 0, tokens = 0;
+  const providers = new Set(), models = new Set(), clients = new Set(), ifaces = new Set();
+  let minD = null, maxD = null;
+  for (const r of cube.rows) {
+    req += r[mi('req')];
+    naIn += r[mi('naIn')]; naOut += r[mi('naOut')]; naCr += r[mi('naCr')]; naCw += r[mi('naCw')];
+    tokens += r[mi('in')] + r[mi('out')] + r[mi('cr')] + r[mi('cw')];
+    providers.add(r[3]); models.add(r[4]); clients.add(r[6]); ifaces.add(r[7]);
+    if (minD === null || r[0] < minD) minD = r[0];
+    if (maxD === null || r[0] > maxD) maxD = r[0];
+  }
+  const cells = req * 4;
+  const missing = cells ? (naIn + naOut + naCr + naCw) / cells : 0;
+  const files = Object.values(state.sources || {}).reduce((a, s) => a + Object.keys(s.files || {}).length, 0);
+  let grade = 'Excellent';
+  if (missing > 0.4) grade = 'Fair';
+  else if (missing > 0.12) grade = 'Good';
+  if (req === 0) grade = 'No data';
+  return {
+    grade,
+    records: req,
+    tokens,
+    sessions: Object.keys(sessions.rows || {}).length,
+    sourceFiles: files,
+    coverage: { from: minD, to: maxD },
+    providers: providers.size,
+    models: models.size,
+    clients: clients.size,
+    interfaces: ifaces.size,
+    missingTokenFieldRate: missing,
+    missingByField: {
+      input_tokens: req ? naIn / req : 0,
+      output_tokens: req ? naOut / req : 0,
+      cache_read_tokens: req ? naCr / req : 0,
+      cache_write_tokens: req ? naCw / req : 0,
+    },
+    // Structural dedup: identical bytes are never read twice, so the duplicate
+    // count is a fact about the ingest model, not an estimate.
+    duplicateRecords: 0,
+    malformedLines: state.counters?.malformed ?? 0,
+    staleGenerations: (state.stale || []).length,
+    lastRefresh: state.lastRefresh,
+  };
+}

package/src/core/watch.js

@@ -0,0 +1,287 @@
+/**
+ * The watch daemon — `tokenflow watch`.
+ *
+ * A single background process that keeps TokenFlow live: incremental refresh
+ * on an interval, a live status snapshot after every cycle, and (opt-in) OS
+ * notifications when a limit crosses its threshold or a high-severity anomaly
+ * appears.
+ *
+ * Design constraints this file takes seriously:
+ *
+ * Single instance.  A pidfile guards against two watchers racing on the same
+ *   store; a stale pidfile from a crashed run is detected and replaced.
+ * Failure isolation. One bad provider cannot stop the loop: refresh errors
+ *   are recorded into the status file and back off exponentially instead.
+ * Sleep/wake. The loop reschedules from wall-clock reality every tick, so a
+ *   laptop that slept for three hours simply refreshes once on wake rather
+ *   than trying to "catch up" with a burst of cycles.
+ * Nothing leaves the machine. Refresh reads local logs; notifications go to
+ *   the local OS; the status file stays in $TOKENFLOW_HOME.
+ */
+import fs from 'node:fs';
+import { loadConfig, paths, ensureDirs } from './config.js';
+import { refresh } from './ingest.js';
+import { listProviders } from './registry.js';
+import { buildLiveStatus, writeLiveStatus, readLiveStatus } from './live-status.js';
+import { notify as osNotify } from './notify.js';
+
+const MAX_BACKOFF_MS = 15 * 60 * 1000;
+
+/** Exponential backoff between cycles: base × 2^failures, capped. */
+export function computeDelayMs(baseMs, consecutiveFailures) {
+  if (!Number.isFinite(baseMs) || baseMs <= 0) return MAX_BACKOFF_MS;
+  const f = Math.max(0, Math.floor(consecutiveFailures || 0));
+  return Math.min(baseMs * 2 ** f, MAX_BACKOFF_MS);
+}
+
+/**
+ * Compare two live-status snapshots and return the alert-worthy transitions.
+ *
+ * Pure function — the daemon's notification policy lives here so it can be
+ * tested without spawning anything:
+ *   - a configured limit moving ok→warn / warn→exceeded / ok→exceeded
+ *   - a limit recovering back to ok
+ *   - high-severity anomalies dated TODAY that were not in the previous
+ *     snapshot. Historical highs are data, not news: a first cycle must not
+ *     replay last week's spikes as notifications.
+ */
+export function detectTransitions(prev, next) {
+  const out = [];
+  if (!next) return out;
+  const today = String(next.generatedAt || '').slice(0, 10);
+  const prevLimits = new Map((prev?.capacity?.states || []).map((s) => [s.id, s.status]));
+  for (const s of next.capacity?.states || []) {
+    const before = prevLimits.get(s.id);
+    if (before === s.status) continue;
+    if (s.status === 'exceeded') {
+      out.push({
+        kind: 'limit', id: s.id,
+        title: `Limit exceeded: ${s.label}`,
+        body: `${Math.round((s.pctUsed ?? 1) * 100)}% of ${s.scope} ${s.metric}. Resets in ${humanize(s.resetsInMs)}.`,
+      });
+    } else if (s.status === 'warn' && before !== undefined && before !== 'warn') {
+      out.push({
+        kind: 'limit', id: s.id,
+        title: `Approaching limit: ${s.label}`,
+        body: `${Math.round(s.pctUsed * 100)}% of ${s.scope} ${s.metric} used.`,
+      });
+    } else if (s.status === 'ok' && (before === 'warn' || before === 'exceeded')) {
+      out.push({ kind: 'recovered', id: s.id, title: `Limit recovered: ${s.label}`, body: 'Window reset or usage revised.' });
+    }
+  }
+  // Anomalies alert only when they happened today AND we have not announced
+  // them before. A rebuilt status file therefore stays quiet about history.
+  const prevAnomalies = new Set((prev?.anomalies || []).map((a) => a.id));
+  for (const a of next.anomalies || []) {
+    if (a.severity !== 'high') continue;
+    if (!today || a.date !== today) continue;
+    if (prevAnomalies.has(a.id)) continue;
+    out.push({ kind: 'anomaly', id: a.id, title: `Unusual ${(a.type || 'event').replace(/_/g, ' ')}`, body: a.detail });
+  }
+  // Milestones celebrate only on their first sighting too — the same record
+  // broken twice is one party, not two.
+  const prevMilestones = new Set((prev?.milestones || []).map((m) => m.id));
+  for (const m of next.milestones || []) {
+    if (!today || !m.id.endsWith(`:${today}`)) continue;
+    if (prevMilestones.has(m.id)) continue;
+    out.push({ kind: 'milestone', id: m.id, title: `${m.icon} ${m.title}`, body: m.detail });
+  }
+  return out;
+}
+
+function humanize(ms) {
+  if (!Number.isFinite(ms)) return '?';
+  const m = Math.round(ms / 60000);
+  if (m < 60) return `${m}m`;
+  const h = Math.floor(m / 60);
+  if (h < 48) return `${h}h ${m % 60}m`;
+  return `${Math.floor(h / 24)}d ${h % 24}h`;
+}
+
+// ------------------------------------------------------------- instance ----
+
+function readLockPid() {
+  try {
+    const pid = Number(fs.readFileSync(paths().watchPid, 'utf8').trim());
+    return Number.isFinite(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+function processAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // ESRCH = no such process; EPERM = alive but owned by someone else.
+    return err.code !== 'ESRCH';
+  }
+}
+export { processAlive };
+
+/** Set while THIS process runs a watcher loop — guards same-process double start. */
+let ownedHere = false;
+
+/**
+ * Guard the single-instance rule. Throws with a readable hint when another
+ * watcher already owns this store (including an earlier startWatch in this
+ * very process); silently replaces a dead one left by a crash.
+ */
+export function acquireWatchLock() {
+  if (ownedHere) {
+    throw new Error('a watcher is already running in this process');
+  }
+  ensureDirs();
+  const existing = readLockPid();
+  if (existing && processAlive(existing)) {
+    throw Object.assign(new Error(`a watcher is already running (pid ${existing})`), {
+      hint: '`tokenflow watch --status` shows it; `tokenflow watch --stop` stops it.',
+    });
+  }
+  fs.writeFileSync(paths().watchPid, String(process.pid));
+  ownedHere = true;
+}
+
+export function releaseWatchLock() {
+  const p = paths().watchPid;
+  const pid = readLockPid();
+  if (pid === process.pid) {
+    try { fs.unlinkSync(p); } catch { /* already gone — that is fine */ }
+  }
+  ownedHere = false;
+}
+
+/** Is a watcher running right now (and does it own the lock)? */
+export function watchIsRunning() {
+  const pid = readLockPid();
+  return !!(pid && processAlive(pid));
+}
+
+export function stopWatch() {
+  const pid = readLockPid();
+  if (!pid || !processAlive(pid)) {
+    try { fs.unlinkSync(paths().watchPid); } catch { /* nothing to clean */ }
+    return { stopped: false, reason: 'not running' };
+  }
+  try {
+    process.kill(pid, 'SIGTERM');
+    return { stopped: true, pid };
+  } catch (err) {
+    return { stopped: false, reason: err.message };
+  }
+}
+
+// ---------------------------------------------------------------- cycle ----
+
+let inCycle = false;
+
+/**
+ * One pass: refresh → build status → notify transitions → persist.
+ *
+ * `daemon` identifies the caller: a long-running watcher (startWatch) passes
+ * `{ intervalSeconds }` so the snapshot can say "running"; one-shot callers
+ * (`watch --once`, tests, CI) pass nothing and the snapshot honestly reports
+ * that no watcher is running.
+ */
+export async function runCycle(opt = {}) {
+  if (inCycle) return { skipped: true }; // coalesce overlapping triggers
+  inCycle = true;
+  const t0 = Date.now();
+  try {
+    const prev = readLiveStatus();
+    const report = await refresh({
+      // An explicit registry/providers override keeps tests and embedded uses
+      // hermetic: `registry: [], providers: []` touches no source at all.
+      registry: opt.registry || listProviders(),
+      providers: opt.providers,
+      deadlineMs: opt.refreshBudgetMs ?? 45000,
+      onProgress: opt.onProgress,
+    });
+
+    const status = buildLiveStatus({ config: opt.config });
+    status.watcher = opt.daemon
+      ? {
+        ...(prev?.watcher || {}),
+        pid: process.pid,
+        mode: 'daemon',
+        intervalSeconds: opt.daemon.intervalSeconds ?? null,
+        startedAt: prev?.watcher?.startedAt || new Date().toISOString(),
+        cycles: (prev?.watcher?.cycles || 0) + 1,
+        lastCycleAt: new Date().toISOString(),
+        lastCycleMs: Date.now() - t0,
+        lastRefreshReport: {
+          done: report.done,
+          newRecords: report.newRecords,
+          providers: (report.providers || []).map((p) => ({ id: p.id, status: p.status, records: p.records })),
+        },
+      }
+      : null;
+
+    const transitions = detectTransitions(prev, status);
+    status.lastCycle = { at: status.generatedAt, durationMs: Date.now() - t0, newRecords: report.newRecords };
+    writeLiveStatus(status);
+
+    if (opt.notifications && transitions.length) {
+      for (const tr of transitions.slice(0, 4)) await osNotify(tr);
+    }
+    return { skipped: false, transitions, report, durationMs: Date.now() - t0 };
+  } finally {
+    inCycle = false;
+  }
+}
+
+/** Record a failed cycle into the status file without losing the last good data. */
+export function recordCycleError(err) {
+  const cur = readLiveStatus() || {};
+  cur.lastError = { message: String(err.message || err).slice(0, 300), at: new Date().toISOString() };
+  try {
+    writeLiveStatus(cur);
+  } catch {
+    /* disk-level failure: nothing more a watcher can do here */
+  }
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Run the daemon until SIGINT/SIGTERM.
+ *
+ * @param {{intervalSeconds?:number, notifications?:boolean, config?:object,
+ *          onCycle?:Function, maxCycles?:number}} opt
+ *   `maxCycles` exists for tests/CI only — a production watcher never stops
+ *   on its own.
+ */
+export async function startWatch(opt = {}) {
+  const config = opt.config || loadConfig();
+  const intervalSeconds = Number(
+    opt.intervalSeconds ?? config.watch?.intervalSeconds ?? 120,
+  );
+  const notifications = opt.notifications ?? !!config.watch?.notifications;
+  acquireWatchLock();
+
+  let failures = 0;
+  let cycles = 0;
+  const baseMs = Math.max(10, intervalSeconds) * 1000;
+
+  while (true) {
+    try {
+      const r = await runCycle({
+        config,
+        daemon: { intervalSeconds },
+        notifications,
+      });
+      failures = 0;
+      cycles++;
+      opt.onCycle?.(r);
+    } catch (err) {
+      failures++;
+      recordCycleError(err);
+      opt.onCycle?.({ error: err.message });
+    }
+    if (opt.maxCycles && cycles >= opt.maxCycles) break;
+    // Reschedule from now, never from a schedule built at startup: a machine
+    // that slept through six intervals wakes to exactly one refresh.
+    await sleep(computeDelayMs(baseMs, failures));
+  }
+}