@vimoxshah/tokenflow 1.1.0

package/src/analytics/aggregate.js

@@ -0,0 +1,247 @@
+/**
+ * Filtering + aggregation primitives.
+ *
+ * Everything downstream operates on a "slice": the cube rows that survive the
+ * global filter bar, plus the sessions that survive the same filter. Slices are
+ * computed in-process (in the browser, from a bundle loaded once) so changing a
+ * filter costs no network round trip.
+ *
+ * This module is deliberately free of Node imports so the exact same code runs
+ * in the CLI, the API and the browser.
+ */
+
+export const MEASURES = ['in', 'out', 'cr', 'cw', 'cf', 'rs', 'req', 'cost', 'costMeasured', 'costReq', 'naIn', 'naOut', 'naCr', 'naCw'];
+
+/** Column indices for a cube built by src/core/store.js. */
+export function indexCube(cube) {
+  const d = {};
+  cube.dims.forEach((k, i) => { d[k] = i; });
+  const m = {};
+  cube.measures.forEach((k, i) => { m[k] = cube.dims.length + i; });
+  return { d, m, rows: cube.rows, dims: cube.dims, measures: cube.measures };
+}
+
+export const EMPTY_FILTERS = {
+  from: null, to: null,
+  hourFrom: null, hourTo: null,
+  dows: null,
+  provider: null, model: null, model_family: null, client: null,
+  interface: null, gateway: null, project: null, repository: null, service_tier: null,
+  includeOverlay: false,
+  includeActivity: true,
+};
+
+/**
+ * @param {ReturnType<typeof indexCube>} ix
+ * @param {Partial<typeof EMPTY_FILTERS>} f
+ * @returns {any[][]} cube rows
+ */
+export function filterCube(ix, f = {}) {
+  const { d } = ix;
+  const set = (v) => (v && v.length ? new Set(v) : null);
+  const P = set(f.provider), M = set(f.model), MF = set(f.model_family), C = set(f.client),
+    I = set(f.interface), G = set(f.gateway), PJ = set(f.project), RP = set(f.repository),
+    ST = set(f.service_tier), W = f.dows && f.dows.length ? new Set(f.dows) : null;
+  const from = f.from || null, to = f.to || null;
+  const hf = f.hourFrom === null || f.hourFrom === undefined ? null : Number(f.hourFrom);
+  const ht = f.hourTo === null || f.hourTo === undefined ? null : Number(f.hourTo);
+  const overlay = !!f.includeOverlay;
+  const activity = f.includeActivity !== false;
+
+  const out = [];
+  for (const r of ix.rows) {
+    const ms = r[d.ms];
+    if (ms === 'overlay' && !overlay) continue;
+    if (ms === 'activity' && !activity) continue;
+    if (from && r[d.d] < from) continue;
+    if (to && r[d.d] > to) continue;
+    if (hf !== null || ht !== null) {
+      const h = r[d.h];
+      if (hf !== null && ht !== null) {
+        // A window may wrap past midnight (22 -> 03).
+        const ok = hf <= ht ? h >= hf && h <= ht : h >= hf || h <= ht;
+        if (!ok) continue;
+      } else if (hf !== null && h < hf) continue;
+      else if (ht !== null && h > ht) continue;
+    }
+    if (W && !W.has(r[d.w])) continue;
+    if (P && !P.has(r[d.p])) continue;
+    if (M && !M.has(r[d.m])) continue;
+    if (MF && !MF.has(r[d.mf])) continue;
+    if (C && !C.has(r[d.c])) continue;
+    if (I && !I.has(r[d.i])) continue;
+    if (G && !G.has(r[d.g])) continue;
+    if (PJ && !PJ.has(r[d.pj])) continue;
+    if (RP && !RP.has(r[d.rp])) continue;
+    if (ST && !ST.has(r[d.st])) continue;
+    out.push(r);
+  }
+  return out;
+}
+
+/** Sessions filtered by the same predicate set (dimension keys match the cube). */
+export function filterSessions(sessions, f = {}) {
+  const set = (v) => (v && v.length ? new Set(v) : null);
+  const P = set(f.provider), M = set(f.model), MF = set(f.model_family), C = set(f.client),
+    I = set(f.interface), G = set(f.gateway), PJ = set(f.project), RP = set(f.repository),
+    ST = set(f.service_tier), W = f.dows && f.dows.length ? new Set(f.dows) : null;
+  const overlay = !!f.includeOverlay;
+  const activity = f.includeActivity !== false;
+  return sessions.filter((s) => {
+    if (s.ms === 'overlay' && !overlay) return false;
+    if (s.ms === 'activity' && !activity) return false;
+    if (f.from && s.d < f.from) return false;
+    if (f.to && s.d > f.to) return false;
+    if (W && !W.has(s.w)) return false;
+    if (P && !P.has(s.p)) return false;
+    if (M && !M.has(s.m)) return false;
+    if (MF && !MF.has(s.mf)) return false;
+    if (C && !C.has(s.c)) return false;
+    if (I && !I.has(s.i)) return false;
+    if (G && !G.has(s.g)) return false;
+    if (PJ && !PJ.has(s.pj)) return false;
+    if (RP && !RP.has(s.rp)) return false;
+    if (ST && !ST.has(s.st)) return false;
+    return true;
+  });
+}
+
+export function zeroMeasures() {
+  const o = {};
+  for (const k of MEASURES) o[k] = 0;
+  return o;
+}
+
+/** Sum measures across rows. Adds a derived `total` and `naAny`. */
+export function sumRows(rows, ix) {
+  const { m } = ix;
+  const o = zeroMeasures();
+  for (const r of rows) for (const k of MEASURES) o[k] += r[m[k]];
+  return finalize(o);
+}
+
+export function addInto(acc, r, ix) {
+  const { m } = ix;
+  for (const k of MEASURES) acc[k] += r[m[k]];
+  return acc;
+}
+
+export function finalize(o) {
+  o.total = o.in + o.out + o.cr + o.cw;
+  o.cache = o.cr + o.cw;
+  o.naAny = o.naIn + o.naOut + o.naCr + o.naCw;
+  return o;
+}
+
+/**
+ * Group rows by a key derived from dimension columns.
+ * @returns {Map<string, {key:string, m:ReturnType<typeof zeroMeasures>, rows:any[][]}>}
+ */
+export function groupRows(rows, ix, keyFn, { keepRows = false } = {}) {
+  const out = new Map();
+  for (const r of rows) {
+    const k = keyFn(r, ix.d);
+    if (k === null || k === undefined) continue;
+    let g = out.get(k);
+    if (!g) {
+      g = { key: k, m: zeroMeasures(), rows: keepRows ? [] : null };
+      out.set(k, g);
+    }
+    addInto(g.m, r, ix);
+    if (keepRows) g.rows.push(r);
+  }
+  for (const g of out.values()) finalize(g.m);
+  return out;
+}
+
+/** Sorted, finalized array form of groupRows, largest total first. */
+export function rank(rows, ix, keyFn, { limit = null, keepRows = false } = {}) {
+  const arr = [...groupRows(rows, ix, keyFn, { keepRows }).values()];
+  arr.sort((a, b) => b.m.total - a.m.total || (a.key < b.key ? -1 : 1));
+  return limit ? arr.slice(0, limit) : arr;
+}
+
+/** All distinct values of a dimension, with request counts — for the filter bar. */
+export function facet(ix, dim, rows = ix.rows) {
+  const i = ix.d[dim];
+  const out = new Map();
+  for (const r of rows) {
+    const v = r[i];
+    const e = out.get(v) || { value: v, req: 0, total: 0 };
+    e.req += r[ix.m.req];
+    e.total += r[ix.m.in] + r[ix.m.out] + r[ix.m.cr] + r[ix.m.cw];
+    out.set(v, e);
+  }
+  return [...out.values()].sort((a, b) => b.total - a.total || (a.value < b.value ? -1 : 1));
+}
+
+// --------------------------------------------------------------- calendar ---
+
+export function toISODate(d) {
+  return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}-${String(d.getUTCDate()).padStart(2, '0')}`;
+}
+
+export function parseISODate(s) {
+  const [y, m, d] = s.split('-').map(Number);
+  return new Date(Date.UTC(y, m - 1, d));
+}
+
+export function addDays(iso, n) {
+  const d = parseISODate(iso);
+  d.setUTCDate(d.getUTCDate() + n);
+  return toISODate(d);
+}
+
+export function daysBetween(a, b) {
+  return Math.round((parseISODate(b).getTime() - parseISODate(a).getTime()) / 86400000);
+}
+
+/** Inclusive list of every calendar date in a range — including empty ones. */
+export function dateRange(from, to) {
+  const out = [];
+  if (!from || !to || from > to) return out;
+  let cur = from;
+  let guard = 0;
+  while (cur <= to && guard++ < 20000) {
+    out.push(cur);
+    cur = addDays(cur, 1);
+  }
+  return out;
+}
+
+/** ISO week key, e.g. "2026-W33". */
+export function weekKey(iso) {
+  const d = parseISODate(iso);
+  const day = (d.getUTCDay() + 6) % 7;
+  d.setUTCDate(d.getUTCDate() - day + 3);
+  const firstThursday = new Date(Date.UTC(d.getUTCFullYear(), 0, 4));
+  const fday = (firstThursday.getUTCDay() + 6) % 7;
+  firstThursday.setUTCDate(firstThursday.getUTCDate() - fday + 3);
+  const week = 1 + Math.round((d.getTime() - firstThursday.getTime()) / (7 * 86400000));
+  return `${d.getUTCFullYear()}-W${String(week).padStart(2, '0')}`;
+}
+
+export function monthKey(iso) {
+  return iso.slice(0, 7);
+}
+
+/** Monday of the week containing `iso` — the label for weekly buckets. */
+export function weekStart(iso) {
+  const d = parseISODate(iso);
+  const day = (d.getUTCDay() + 6) % 7;
+  d.setUTCDate(d.getUTCDate() - day);
+  return toISODate(d);
+}
+
+export function percentile(sortedAsc, p) {
+  if (!sortedAsc.length) return null;
+  if (sortedAsc.length === 1) return sortedAsc[0];
+  const idx = (sortedAsc.length - 1) * p;
+  const lo = Math.floor(idx);
+  const hi = Math.ceil(idx);
+  return sortedAsc[lo] + (sortedAsc[hi] - sortedAsc[lo]) * (idx - lo);
+}
+
+export function mean(xs) {
+  return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null;
+}

package/src/analytics/anomalies.js

@@ -0,0 +1,222 @@
+/**
+ * Anomaly detection.
+ *
+ * Robust (median / MAD, Iglewicz–Hoaglin modified z-score) detection over the
+ * daily series, plus structural events the statistical layer cannot see:
+ * calendar gaps in an otherwise-active pattern and entities that appeared for
+ * the first time recently.
+ *
+ * Every anomaly carries its own arithmetic — observed value, baseline median,
+ * modified z — so a reader can check the call instead of trusting it. A quiet
+ * dataset produces an empty list; that is the honest answer, not a failure.
+ *
+ * Pure module: no Node imports, deterministic output.
+ */
+
+const MAD_SCALE = 1.4826;
+const SPIKE_Z = 3.5;      // Iglewicz–Hoaglin threshold for a modified z-score
+const HIGH_Z = 6;         // well past that: call it high severity
+const BASELINE_WINDOW = 60;
+const MIN_BASELINE = 10;
+
+/**
+ * @typedef {Object} Anomaly
+ * @property {string} id
+ * @property {string} type
+ * @property {string} date
+ * @property {'high'|'warn'|'info'} severity
+ * @property {number|null} observed
+ * @property {number|null} [expectedMedian]
+ * @property {number|null} [ratio]
+ * @property {number|null} [z]
+ * @property {string} detail
+ * @property {string} [metric]
+ */
+
+function median(sortedAsc) {
+  if (!sortedAsc.length) return null;
+  const mid = Math.floor(sortedAsc.length / 2);
+  return sortedAsc.length % 2 ? sortedAsc[mid] : (sortedAsc[mid - 1] + sortedAsc[mid]) / 2;
+}
+
+/**
+ * Baseline stats for index i computed over the preceding window, excluding i
+ * itself so an event can never drag its own baseline toward it.
+ */
+function baseline(values, i, window = BASELINE_WINDOW) {
+  const lo = Math.max(0, i - window);
+  const hist = values.slice(lo, i).filter((v) => Number.isFinite(v));
+  if (hist.length < MIN_BASELINE) return null;
+  const med = median([...hist].sort((a, b) => a - b));
+  const mad = median(hist.map((v) => Math.abs(v - med)).sort((a, b) => a - b));
+  return { n: hist.length, med, mad };
+}
+
+/** Modified z-score; null when the MAD is degenerate (e.g. flat history). */
+export function modifiedZ(x, { med, mad }) {
+  if (!Number.isFinite(x)) return null;
+  if (mad === 0) {
+    // Perfectly flat history: any deviation is infinitely surprising
+    // statistically, but capping keeps severity honest — this surfaces as a
+    // warning, never as "high".
+    if (x === med) return 0;
+    return x > med ? 4.99 : -4.99;
+  }
+  return (0.6745 * (x - med)) / mad;
+}
+
+const SEV_ORDER = { high: 0, warn: 1, info: 2 };
+
+/** @param {Anomaly[]} out */
+function addSpikeEvents(out, series, values, metric, kind, label, fmt) {
+  for (let i = 0; i < series.length; i++) {
+    const x = values[i];
+    if (!Number.isFinite(x) || x <= 0) continue;
+    const base = baseline(values, i);
+    if (!base || base.med <= 0) continue;
+    const z = modifiedZ(x, base);
+    if (z === null || z < SPIKE_Z) continue;
+    const ratio = x / base.med;
+    out.push({
+      id: `${kind}:${series[i].key}`,
+      type: kind,
+      date: series[i].key,
+      severity: z >= HIGH_Z ? 'high' : 'warn',
+      observed: x,
+      expectedMedian: base.med,
+      ratio,
+      z,
+      detail: `${label} of ${fmt(x)} is ${ratio.toFixed(1)}× the trailing ${base.n}-day median (${fmt(base.med)}) — a robust z-score of ${z.toFixed(1)}.`,
+      metric,
+    });
+  }
+}
+
+function valuesOf(series, metric) {
+  return series.map((d) => Number(d[metric]));
+}
+
+function isWeekend(iso) {
+  const [y, m, d] = iso.split('-').map(Number);
+  const dow = new Date(Date.UTC(y, m - 1, d)).getUTCDay();
+  return dow === 0 || dow === 6;
+}
+
+/**
+ * Detect anomalies over a calendar-complete daily series.
+ *
+ * @param {{key:string,total:number,cost?:number|null,req:number,active:boolean,
+ *           tokenActive:boolean,[k:string]:any}[]} daily
+ * @returns {Anomaly[]}
+ */
+export function detectAnomalies(daily, opt = {}) {
+  /** @type {Anomaly[]} */
+  const out = [];
+  if (!Array.isArray(daily) || daily.length < MIN_BASELINE + 1) return out;
+
+  const totals = valuesOf(daily, 'total');
+  addSpikeEvents(out, daily, totals, 'total', 'token_spike', 'Token usage', (n) => compact(n));
+  const hasCost = daily.some((d) => Number.isFinite(Number(d.cost)) && Number(d.cost) > 0);
+  if (hasCost) addSpikeEvents(out, daily, valuesOf(daily, 'cost'), 'cost', 'cost_spike', 'Estimated cost', (n) => `$${Number(n).toFixed(2)}`);
+
+  // ---- request-rate spikes -----------------------------------------------
+  addSpikeEvents(out, daily, valuesOf(daily, 'req'), 'req', 'request_spike', 'Request volume', (n) => String(Math.round(n)));
+
+  // ---- ingestion-gap candidates ------------------------------------------
+  // A zero day is normal (weekends, holidays). It is only interesting when it
+  // sits inside a stretch where this dataset was otherwise active every
+  // weekday — hence "possible gap", never "gap".
+  for (let i = 1; i < daily.length - 1; i++) {
+    const d = daily[i];
+    if (d.total > 0 || d.active || isWeekend(d.key)) continue;
+    const around = [daily[i - 1], daily[i + 1]];
+    if (!around.every((x) => x.tokenActive)) continue;
+    out.push({
+      id: `gap:${d.key}`,
+      type: 'possible_gap',
+      date: d.key,
+      severity: 'info',
+      observed: 0,
+      expectedMedian: null,
+      z: null,
+      detail: `No activity at all on ${d.key}, a weekday between two active days — possible ingestion gap or a genuine day off.`,
+    });
+  }
+
+  // ---- sudden drops ------------------------------------------------------
+  // The mirror of a spike, but only meaningful on weekdays with real history:
+  // usage halving on a Saturday is a pattern, not an anomaly.
+  for (let i = 0; i < daily.length; i++) {
+    const x = totals[i];
+    if (!(x > 0) || isWeekend(daily[i].key)) continue;
+    const base = baseline(totals, i);
+    if (!base || base.med <= 0) continue;
+    const z = modifiedZ(x, base);
+    if (z === null || z > -SPIKE_Z) continue;
+    out.push({
+      id: `drop:${daily[i].key}`,
+      type: 'usage_drop',
+      date: daily[i].key,
+      severity: z <= -HIGH_Z ? 'high' : 'warn',
+      observed: x,
+      expectedMedian: base.med,
+      ratio: x / base.med,
+      z,
+      detail: `Weekday usage fell to ${compact(x)} — ${(x / base.med).toFixed(2)}× the trailing ${base.n}-day median (${compact(base.med)}), a robust z-score of ${z.toFixed(1)}.`,
+    });
+  }
+
+  const cap = opt.limit ?? 12;
+  return out
+    .sort((a, b) =>
+      SEV_ORDER[a.severity] - SEV_ORDER[b.severity]
+      || (a.date < b.date ? 1 : a.date > b.date ? -1 : 0)
+      || (b.z ?? 0) - (a.z ?? 0))
+    .slice(0, cap);
+}
+
+/**
+ * Entities (models or providers) seen for the first time within `withinDays`
+ * of `today`. First-seen is derived from the full cube rows, independent of
+ * the filtered range, because "new" means new to the whole dataset.
+ *
+ * @param {ReturnType<import('./aggregate.js').indexCube>} ix
+ * @param {{today:string, withinDays?:number, dim?:'m'|'p'}} opt
+ */
+export function firstSeenEntities(ix, opt) {
+  const dimKey = opt.dim === 'p' ? ix.d.p : ix.d.m;
+  const noun = opt.dim === 'p' ? 'provider' : 'model';
+  const cutoff = isoMinusDays(opt.today, opt.withinDays ?? 7);
+  /** @type {Map<string,{first:string,last:string,tokens:number}>} */
+  const seen = new Map();
+  for (const r of ix.rows) {
+    const date = r[ix.d.d];
+    if (date > opt.today) continue;
+    const key = r[dimKey];
+    let e = seen.get(key);
+    if (!e) { e = { first: date, last: date, tokens: 0 }; seen.set(key, e); }
+    if (date < e.first) e.first = date;
+    if (date > e.last) e.last = date;
+    e.tokens += r[ix.m.in] + r[ix.m.out] + r[ix.m.cr] + r[ix.m.cw];
+  }
+  const out = [];
+  for (const [key, e] of seen) {
+    if (e.first < cutoff) continue;
+    out.push({ entity: key, noun, firstSeen: e.first, lastUsed: e.last, tokens: e.tokens });
+  }
+  return out.sort((a, b) => (a.firstSeen < b.firstSeen ? 1 : -1));
+}
+
+function isoMinusDays(iso, n) {
+  const [y, m, d] = iso.split('-').map(Number);
+  const dt = new Date(Date.UTC(y, m - 1, d - n));
+  return `${dt.getUTCFullYear()}-${String(dt.getUTCMonth() + 1).padStart(2, '0')}-${String(dt.getUTCDate()).padStart(2, '0')}`;
+}
+
+function compact(n) {
+  if (!Number.isFinite(n)) return '—';
+  if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`;
+  if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`;
+  if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`;
+  return String(Math.round(n));
+}