@vimoxshah/tokenflow 1.1.2 → 1.2.0

package/src/server/routes/annotations.js

@@ -0,0 +1,42 @@
+/**
+ * Day annotations API.
+ *
+ * GET returns the whole file; POST adds or removes one entry. Validation and
+ * storage live in src/core/annotations.js — this file only shapes the HTTP
+ * request/response and turns a validation Error into a 400.
+ */
+import { readAnnotations, addAnnotation, removeAnnotation } from '../../core/annotations.js';
+
+/** @type {import('./index.js').Route[]} */
+export const ANNOTATIONS_ROUTES = [
+  {
+    method: 'GET',
+    path: '/api/annotations',
+    handler: (req, res, url, ctx) => ctx.json(readAnnotations()),
+  },
+  {
+    method: 'POST',
+    path: '/api/annotations',
+    handler: async (req, res, url, ctx) => {
+      let body;
+      try {
+        body = JSON.parse((await ctx.readBody()) || '{}');
+      } catch {
+        return ctx.json({ error: 'invalid JSON body' }, 400);
+      }
+      try {
+        if (body.op === 'add') {
+          const item = addAnnotation({ date: body.date, text: body.text });
+          return ctx.json({ ok: true, item, ...readAnnotations() });
+        }
+        if (body.op === 'remove') {
+          const removed = removeAnnotation(body.id);
+          return ctx.json({ ok: true, removed, ...readAnnotations() });
+        }
+        return ctx.json({ error: `unknown op "${body.op}"` }, 400);
+      } catch (err) {
+        return ctx.json({ error: err.message }, 400);
+      }
+    },
+  },
+];

package/src/server/routes/cache-health.js

@@ -0,0 +1,95 @@
+/**
+ * GET /api/cache-health?from=YYYY-MM-DD&to=YYYY-MM-DD
+ *
+ * Churn detection: scans primary records in the given window, groups them by
+ * session in timestamp order, and reports every turn that invalidated and
+ * rewrote the prompt cache at a premium (src/analytics/cache-health.js has
+ * the exact rule). This needs request-level records, which never leave the
+ * machine and never ship in a snapshot, so it is a live-dashboard-only route.
+ *
+ * A record with no session_id cannot be placed in a turn sequence, so it is
+ * left out of every group rather than guessed at.
+ */
+import { Store, decodeRecord, readJson } from '../../core/store.js';
+import { buildPriceBook } from '../../core/pricing.js';
+import { MEASUREMENT } from '../../core/schema.js';
+import { detectChurn, summarize } from '../../analytics/cache-health.js';
+
+/** Hard ceiling on how many records one request scans. */
+const MAX_SCAN_RECORDS = 200000;
+
+/**
+ * `YYYY-MM` for every month between `from` and `to`, inclusive. Returns
+ * `undefined` (no restriction) when either bound is missing, matching how
+ * `Store#scanRecords`'s `months` option treats a falsy value.
+ * @param {string|null} from
+ * @param {string|null} to
+ */
+function monthsBetween(from, to) {
+  if (!from || !to) return undefined;
+  const out = [];
+  let y = Number(from.slice(0, 4));
+  let m = Number(from.slice(5, 7));
+  const ey = Number(to.slice(0, 4));
+  const em = Number(to.slice(5, 7));
+  let guard = 0;
+  while ((y < ey || (y === ey && m <= em)) && guard++ < 600) {
+    out.push(`${y}-${String(m).padStart(2, '0')}`);
+    m++;
+    if (m > 12) { m = 1; y++; }
+  }
+  return out;
+}
+
+function handleCacheHealth(req, res, url, ctx) {
+  const from = url.searchParams.get('from') || null;
+  const to = url.searchParams.get('to') || null;
+
+  const store = new Store();
+  const book = buildPriceBook(readJson(ctx.paths.pricing, {}));
+  const months = monthsBetween(from, to);
+
+  const bySession = new Map();
+  let scanned = 0;
+  let truncated = false;
+  store.scanRecords((o) => {
+    scanned++;
+    if (scanned > MAX_SCAN_RECORDS) { truncated = true; return false; }
+    if (from && o.d < from) return;
+    if (to && o.d > to) return;
+    if (o.ms !== MEASUREMENT.PRIMARY) return;
+    if (!o.s) return; // no session id: cannot be placed in a turn sequence
+    const r = decodeRecord(o);
+    let group = bySession.get(r.session_id);
+    if (!group) { group = []; bySession.set(r.session_id, group); }
+    group.push(r);
+  }, { months });
+
+  const events = [];
+  for (const group of bySession.values()) events.push(...detectChurn(group, book));
+  // Highest estimated premium first; unpriced events (premiumUsd: null) sink
+  // to the bottom rather than sorting arbitrarily.
+  events.sort((a, b) => {
+    if (a.premiumUsd === null && b.premiumUsd === null) return 0;
+    if (a.premiumUsd === null) return 1;
+    if (b.premiumUsd === null) return -1;
+    return b.premiumUsd - a.premiumUsd;
+  });
+
+  ctx.json({
+    from,
+    to,
+    scanned,
+    truncated,
+    events,
+    summary: summarize(events),
+  });
+}
+
+export const CACHE_HEALTH_ROUTES = [
+  {
+    method: 'GET',
+    path: '/api/cache-health',
+    handler: handleCacheHealth,
+  },
+];

package/src/server/routes/index.js

@@ -0,0 +1,54 @@
+/**
+ * The dashboard server's route registry.
+ *
+ * A registered route is an API endpoint that lives outside server.js. The
+ * server dispatches to a matching entry before its own inline chain, so a new
+ * tab can ship its own endpoint without anyone editing server.js — the same
+ * reason src/ui/views/ exists for the UI side.
+ *
+ * A handler is called as `handler(req, res, url, ctx)` and owns the response
+ * from that point on: nothing after it runs. Registered GET routes get exactly
+ * the protection inline GET routes get today, which is loopback binding and
+ * nothing else; a route that changes state must be POST, so it is covered by
+ * the same-origin/token check server.js applies to every non-GET request.
+ *
+ * The full contract is in docs/ui-views.md under "Server routes".
+ */
+
+/**
+ * The helpers a route handler is given. `config` and `paths` are lazy: they are
+ * only read from disk if the handler actually asks for them.
+ *
+ * @typedef {object} RouteContext
+ * @property {(body:any, code?:number)=>void} json respond with JSON.
+ * @property {(code:number, type:string, body:any)=>void} send respond with anything else.
+ * @property {(file:string)=>void} sendFile respond with a file from disk.
+ * @property {()=>Promise<string>} readBody read the request body as text.
+ * @property {object} config the loaded config (lazy, cached per request).
+ * @property {object} paths the resolved store paths (lazy, cached per request).
+ * @property {(opt?:{config?:object, receipts?:boolean})=>any} buildBundle build the aggregate bundle the UI loads. `receipts: false` skips the whole-store receipt scan, which costs seconds a route that only needs sessions should not pay.
+ * @property {(query:object)=>object} queryRecords the record query behind /api/records.
+ * @property {string} root the repository root.
+ * @property {string} host the bound host.
+ * @property {number} port the bound port (correct even when the server bound port 0).
+ */
+
+/**
+ * One endpoint. `path` is matched exactly; there is no pattern syntax, because
+ * a query string is a better fit for everything this dashboard needs.
+ *
+ * @typedef {object} Route
+ * @property {string} method 'GET', 'POST', … matched case-insensitively.
+ * @property {string} path e.g. '/api/receipts'.
+ * @property {(req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse, url: URL, ctx: RouteContext) => any} handler
+ */
+
+import { ANNOTATIONS_ROUTES } from './annotations.js';
+import { CACHE_HEALTH_ROUTES } from './cache-health.js';
+import { SESSION_ROUTE } from './session.js';
+
+/**
+ * Every registered route. Order does not matter: paths are matched exactly.
+ * @type {Route[]}
+ */
+export const ROUTES = [...ANNOTATIONS_ROUTES, ...CACHE_HEALTH_ROUTES, SESSION_ROUTE];

package/src/server/routes/session.js

@@ -0,0 +1,157 @@
+/**
+ * GET /api/session?id=<sessionId>
+ *
+ * One session's primary records, in timestamp order, for the Session anatomy
+ * tab. The cube has no notion of a turn, so this is the only way to see where
+ * a single session's money went — and request-level records never ship in the
+ * bundle or the snapshot, which is why this is a live-dashboard-only route.
+ *
+ * Two things this route deliberately does not do:
+ *
+ *   - It never returns prompt or code content. The stored records hold none,
+ *     and the projection below is an allow-list rather than a delete-list so
+ *     that a future metadata field cannot leak in by being added upstream.
+ *   - It never scans the whole store. The session's own date range comes from
+ *     the bundle's session rows, and only the months that range touches are
+ *     read (widened by a day at each end, because a session row's date is a
+ *     local calendar date and a record's timestamp is UTC).
+ *
+ * One field is served beyond the set this route was specified with:
+ * `service_tier`. The browser prices each turn itself from the price book, and
+ * the tier is a multiplier on that price (OpenAI priority and fast are 4x,
+ * batch is 0.5x). Without it the anatomy tab would quietly disagree with the
+ * Cost tab on exactly the sessions that cost the most. It is a billing
+ * dimension the cube already carries, not content.
+ */
+import { Store } from '../../core/store.js';
+import { MEASUREMENT } from '../../core/schema.js';
+import { addDays } from '../../analytics/aggregate.js';
+
+/** Most turns one response will carry. Reported in the body, never silent. */
+export const RECORD_CAP = 5000;
+
+/**
+ * The key `Store#upsertSession` filed a record under. A source with no session
+ * id gets a synthetic one, so matching on `o.s` alone would never find those
+ * sessions.
+ * @param {object} o an encoded record
+ */
+function sessionKeyOf(o) {
+  return o.s || `${o.so}:${o.d}:${o.pj || 'unknown'}`;
+}
+
+/** `YYYY-MM` for every month between two dates, inclusive. */
+function monthsBetween(from, to) {
+  const out = [];
+  let y = Number(from.slice(0, 4));
+  let m = Number(from.slice(5, 7));
+  const ey = Number(to.slice(0, 4));
+  const em = Number(to.slice(5, 7));
+  let guard = 0;
+  while ((y < ey || (y === ey && m <= em)) && guard++ < 600) {
+    out.push(`${y}-${String(m).padStart(2, '0')}`);
+    m++;
+    if (m > 12) { m = 1; y++; }
+  }
+  return out;
+}
+
+/** The calendar date part of an ISO timestamp, or null. */
+function dateOf(iso) {
+  return typeof iso === 'string' && iso.length >= 10 ? iso.slice(0, 10) : null;
+}
+
+/**
+ * The fields the anatomy view is allowed to see. Everything else — metadata,
+ * cwd, titles, ids that identify a machine or a person — stays on disk.
+ * @param {object} o an encoded record
+ */
+function slim(o) {
+  const meta = o.x || {};
+  return {
+    ts: o.ts ?? null,
+    model: o.m ?? null,
+    provider: o.p ?? null,
+    source: o.so ?? null,
+    input_tokens: o.in ?? null,
+    cache_read_tokens: o.cr ?? null,
+    cache_write_tokens: o.cw ?? null,
+    cache_refresh_tokens: o.cf ?? null,
+    output_tokens: o.ou ?? null,
+    reasoning_tokens: o.rs ?? null,
+    // The billing tier changes what an identical turn costs (OpenAI priority
+    // is 4x), so pricing in the browser needs it to agree with the Cost tab.
+    service_tier: o.tr ?? null,
+    category: o.k ?? null,
+    request_id: o.rq ?? null,
+    // No adapter records a parent link today: Claude Code marks a sidechain and
+    // OpenCode/Hermes fold their parent id into `category`. The pair is carried
+    // anyway so the fan-out becomes a real tree the day one does.
+    agent: meta.agent ?? meta.agent_role ?? null,
+    parent: meta.parent_id ?? meta.parent_session_id ?? null,
+  };
+}
+
+function handleSession(req, res, url, ctx) {
+  const id = url.searchParams.get('id');
+  if (!id) return ctx.json({ error: 'id is required' }, 400);
+
+  // Session rows only; the receipt scan behind the full bundle costs seconds
+  // and this route never looks at it.
+  const sessions = ctx.buildBundle({ receipts: false }).sessions || [];
+  const row = sessions.find((s) => s.id === id);
+  if (!row) return ctx.json({ error: `no session "${id}"` }, 404);
+
+  const first = row.d || dateOf(row.start);
+  const last = dateOf(row.end) || first;
+  if (!first || !last) return ctx.json({ error: `session "${id}" has no date range` }, 404);
+  const from = addDays(first < last ? first : last, -1);
+  const to = addDays(first > last ? first : last, 1);
+  const months = monthsBetween(from, to);
+
+  const store = new Store();
+  const matched = [];
+  let scanned = 0;
+  store.scanRecords((o) => {
+    scanned++;
+    if (o.ms !== MEASUREMENT.PRIMARY) return;
+    if (sessionKeyOf(o) !== id) return;
+    matched.push(o);
+  }, { months });
+
+  matched.sort((a, b) => (a.ts === b.ts ? 0 : (a.ts ?? '') < (b.ts ?? '') ? -1 : 1));
+  const records = matched.slice(0, RECORD_CAP).map(slim);
+
+  ctx.json({
+    id,
+    session: {
+      id: row.id,
+      source: row.so ?? null,
+      project: row.pj ?? null,
+      repository: row.rp ?? null,
+      branch: row.br ?? null,
+      model: row.m ?? null,
+      provider: row.p ?? null,
+      client: row.c ?? null,
+      interface: row.i ?? null,
+      start: row.start ?? null,
+      end: row.end ?? null,
+      date: row.d ?? null,
+      requests: row.req ?? null,
+    },
+    months,
+    scanned,
+    total: matched.length,
+    returned: records.length,
+    cap: RECORD_CAP,
+    truncated: matched.length > RECORD_CAP,
+    records,
+  });
+}
+
+/** @type {import('./index.js').Route} */
+export const SESSION_ROUTE = {
+  method: 'GET',
+  path: '/api/session',
+  handler: handleSession,
+};

package/src/server/server.js

@@ -22,6 +22,7 @@ import { PROVIDER_REGIONS, resolveMyLocation } from '../core/geo.js';
 import { normalizeLimits } from '../analytics/capacity.js';
 import { streamRecordsCsv, exportFilename } from '../export/csv.js';
 import { writeJson, readJson } from '../core/store.js';
+import { ROUTES } from './routes/index.js';
 
 const MIME = {
   '.html': 'text/html; charset=utf-8',
@@ -41,6 +42,44 @@ export async function startServer({ port = 7799, host = '127.0.0.1', open = fals
   const authToken = token === false ? null : token || randomBytes(12).toString('base64url');
 
   let refreshing = false;
+  // The real port is only known after listen() when the caller asked for 0.
+  let boundPort = port;
+
+  // Paths the inline chain below owns. A registered route may not take one:
+  // registered routes are dispatched first, so the shadowing would be silent
+  // and would look nothing like "my new route is wrong".
+  const RESERVED = new Set([
+    '/', '/index.html', '/api/bundle', '/api/health', '/api/ping', '/api/providers',
+    '/api/records', '/api/live', '/api/geo', '/api/config', '/api/export.csv',
+    '/api/refresh', '/api/pricing', '/api/prefs',
+  ]);
+  for (const r of ROUTES) {
+    if (r && (RESERVED.has(r.path) || String(r.path || '').startsWith('/src/'))) {
+      console.warn(`routes: ${r.method} ${r.path} is a built-in path and will shadow it`);
+    }
+  }
+
+  /**
+   * The helpers a registered route handler gets. Built per request; config and
+   * paths are read lazily so a handler that never asks never pays.
+   */
+  function routeContext(req, res) {
+    let cfg = null;
+    let pth = null;
+    return {
+      json: (body, code) => json(res, body, code),
+      send: (code, type, body) => send(res, code, type, body),
+      sendFile: (file) => sendFile(res, file),
+      readBody: () => readBody(req),
+      get config() { return (cfg ||= loadConfig()); },
+      get paths() { return (pth ||= paths()); },
+      buildBundle,
+      queryRecords,
+      root: ROOT,
+      host,
+      get port() { return boundPort; },
+    };
+  }
 
   const server = http.createServer(async (req, res) => {
     const url = new URL(req.url, `http://${req.headers.host || 'localhost'}`);
@@ -54,6 +93,13 @@ export async function startServer({ port = 7799, host = '127.0.0.1', open = fals
         if (!sameSite && given !== authToken) return send(res, 403, 'text/plain', 'forbidden');
       }
 
+      // Registered routes, after the token check and before the inline chain,
+      // so a tab added under src/ui/views/ can ship its endpoint without this
+      // file growing a branch for it. See src/server/routes/index.js.
+      const route = ROUTES.find((r) => r && r.path === p
+        && String(r.method || 'GET').toUpperCase() === req.method);
+      if (route) return await route.handler(req, res, url, routeContext(req, res));
+
       if (p === '/' || p === '/index.html') {
         return sendFile(res, path.join(UI, 'index.html'));
       }
@@ -227,7 +273,7 @@ export async function startServer({ port = 7799, host = '127.0.0.1', open = fals
   });
   // address() is `string` for a pipe/socket and AddressInfo for TCP.
   const bound = server.address();
-  const boundPort = typeof bound === 'object' && bound !== null ? bound.port : port;
+  boundPort = typeof bound === 'object' && bound !== null ? bound.port : port;
   const addr = `http://${host}:${boundPort}`;
   return { server, url: addr, token: authToken, close: () => new Promise((r) => server.close(r)) };
 }