@ipv9/tokentracker-cli 0.39.43 → 0.39.44

package/src/lib/doctor.js

@@ -1,4 +1,5 @@
 const fs = require("node:fs/promises");
+const { findRowViolations } = require("./queue-compact");
 const { constants } = require("node:fs");
 const path = require("node:path");
 
@@ -28,6 +29,9 @@ async function buildDoctorReport({
   if (paths.cliPath) {
     checks.push(await checkCliEntrypoint(paths.cliPath));
   }
+  if (paths.queuePath) {
+    checks.push(await checkQueueRows(paths.queuePath));
+  }
 
   // No cloud reachability check: TokenTracker is local-only, so there is no
   // remote endpoint whose availability could affect anything here.
@@ -358,8 +362,76 @@ function summarizeChecks(checks = []) {
   return summary;
 }
 
+// CLAUDE.md states the column invariant in prose:
+//
+//   total = input + output + cache_creation + cache_read + reasoning
+//
+// Nothing enforced it at runtime, so a miswritten or corrupt row was aggregated
+// and rendered rather than flagged — and a parser bug of exactly that shape is
+// the class CLAUDE.md records at 1.6-7x magnitude. Same conversion as the
+// curated-expiry and version-lockstep checks: a rule that lived in a document
+// starts running.
+//
+// A warn rather than a fail: the rows are already on disk and already being
+// rendered, so failing the whole health check would be reporting a crisis the
+// user cannot act on in the moment. What they can act on is knowing which rows,
+// and how many.
+const QUEUE_VIOLATIONS_SHOWN = 5;
+
+function queueCheck(status, detail, meta) {
+  return { id: "queue.row_invariant", status, detail, critical: false, meta };
+}
+
+async function readQueueRowsForDoctor(queuePath) {
+  const raw = await fs.readFile(queuePath, "utf8");
+  const rows = [];
+  let malformed = 0;
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      rows.push(JSON.parse(line));
+    } catch {
+      malformed += 1;
+    }
+  }
+  return { rows, malformed };
+}
+
+async function checkQueueRows(queuePath) {
+  let rows;
+  let malformed;
+  try {
+    ({ rows, malformed } = await readQueueRowsForDoctor(queuePath));
+  } catch (err) {
+    if (err && err.code === "ENOENT") {
+      return queueCheck("ok", "no queue yet", { path: queuePath });
+    }
+    return queueCheck("warn", `queue unreadable: ${err?.message || err}`, { path: queuePath });
+  }
+
+  const violations = findRowViolations(rows);
+  if (violations.length === 0 && malformed === 0) {
+    return queueCheck("ok", `${rows.length} rows satisfy the column invariant`, {
+      path: queuePath,
+      rows: rows.length,
+    });
+  }
+
+  const parts = [];
+  if (violations.length > 0) parts.push(`${violations.length} row problem(s)`);
+  if (malformed > 0) parts.push(`${malformed} unparseable line(s)`);
+  return queueCheck("warn", `${parts.join(", ")} in ${rows.length + malformed} line(s)`, {
+    path: queuePath,
+    rows: rows.length,
+    malformed,
+    violations: violations.length,
+    examples: violations.slice(0, QUEUE_VIOLATIONS_SHOWN),
+  });
+}
+
 module.exports = {
   buildDoctorReport,
+  checkQueueRows,
   buildBrowserOpenerCheck,
   buildNodeVersionCheck,
   isHeadlessEnvironment,

package/src/lib/local-api.js

@@ -20,6 +20,83 @@ const AVATAR_PROXY_MAX_BYTES = 512 * 1024; // 512 KiB per image
 const AVATAR_PROXY_MAX_ENTRIES = 64;
 const avatarProxyCache = new Map();
 
+// Sentinel for "a redirect left the allowlist", kept distinct from a network
+// failure so the caller can answer 403 rather than a generic upstream error.
+const AVATAR_REDIRECT_BLOCKED = Symbol("avatar-redirect-blocked");
+const AVATAR_MAX_REDIRECTS = 3;
+
+// Reads at most `maxBytes`, and STOPS READING at the limit. Returns null when the
+// response is over it.
+//
+// `AVATAR_PROXY_MAX_BYTES` read like a download cap and was not one: the whole
+// upstream body was buffered with `arrayBuffer()` first, and the constant then
+// only decided whether the result entered the cache. An oversized response was
+// still read into memory in full and still written to the client, so an
+// allowlisted host serving a very large image/* drove loopback-server memory
+// with no ceiling.
+//
+// Two gates, because either alone is soft: `content-length` is a claim the peer
+// makes and can lie about or omit, and counting bytes without it means the
+// transfer has already started. Check the claim, then count anyway.
+async function readCappedAvatarBody(response, maxBytes) {
+  const declared = Number(response.headers.get("content-length"));
+  if (Number.isFinite(declared) && declared > maxBytes) return null;
+  // A HEAD response, or a stub without a stream. `arrayBuffer()` is bounded here
+  // by the content-length check above and by the length check below.
+  if (!response.body) {
+    const buffered = Buffer.from(await response.arrayBuffer());
+    return buffered.length > maxBytes ? null : buffered;
+  }
+  const chunks = [];
+  let total = 0;
+  for await (const chunk of response.body) {
+    total += chunk.length;
+    // Returning from a for-await calls the iterator's return(), which cancels
+    // the stream — the download stops here rather than running to completion.
+    if (total > maxBytes) return null;
+    chunks.push(Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+
+// One check for the URL the caller asked for AND for every redirect hop, so the
+// two can't drift apart. Port is part of the address: `https://gravatar.com:8443/`
+// shares the hostname but is a different service, so leaving the port free turns
+// this proxy into a port prober against every allowlisted host. An empty `port`
+// is the scheme default (443 / 80) — the only one permitted.
+function isAllowedAvatarTarget(url, allowlist) {
+  if (url.protocol !== "https:" && url.protocol !== "http:") return false;
+  if (url.port !== "") return false;
+  return allowlist.some(
+    (host) => url.hostname === host || url.hostname.endsWith(`.${host}`),
+  );
+}
+
+// The avatar allowlist is checked once, against the URL the caller asked for.
+// `fetch(..., { redirect: "follow" })` then goes wherever the response points and
+// validates nothing further — so an allowlisted CDN issuing a redirect, or anyone
+// able to place one there, turns this loopback server into a way to reach
+// 169.254.169.254, another service on 127.0.0.1, or any internal address.
+// Follow by hand and re-check the whole address at every hop.
+async function fetchAvatarFollowingAllowlist(url, options, allowlist, fetchImpl = fetch) {
+  let current = url;
+  for (let hop = 0; hop <= AVATAR_MAX_REDIRECTS; hop += 1) {
+    const response = await fetchImpl(current, { ...options, redirect: "manual" });
+    if (response.status < 300 || response.status >= 400) return response;
+    const location = response.headers.get("location");
+    if (!location) return response;
+    let next;
+    try {
+      next = new URL(location, current);
+    } catch {
+      return AVATAR_REDIRECT_BLOCKED;
+    }
+    if (!isAllowedAvatarTarget(next, allowlist)) return AVATAR_REDIRECT_BLOCKED;
+    current = next.toString();
+  }
+  return AVATAR_REDIRECT_BLOCKED;
+}
+
 // ---------------------------------------------------------------------------
 // Per-model pricing — delegated to src/lib/pricing/
 //   - CURATED overrides (kiro-*, hy3-*, composer-*, kimi-for-coding, etc.)
@@ -67,7 +144,12 @@ function readProjectQueueData(projectQueuePath) {
   for (const line of lines) {
     try {
       const row = JSON.parse(line);
-      const key = `${row.project_key || ""}|${row.source || ""}|${row.hour_start || ""}`;
+      // Model is part of the identity now. A legacy row has no `model` and gets
+      // the same "unknown" slot the state migration gives its stranded bucket,
+      // so the two share one key instead of both surviving and double-counting.
+      const key =
+        `${row.project_key || ""}|${row.source || ""}` +
+        `|${row.model || "unknown"}|${row.hour_start || ""}`;
       seen.set(key, row);
     } catch {
       // skip malformed
@@ -687,6 +769,182 @@ const HOP_BY_HOP_HEADERS = new Set([
 // Main handler factory
 // ---------------------------------------------------------------------------
 
+// Project usage, filtered the way every other card is filtered.
+//
+// Extracted from the request handler so the aggregation can be tested without
+// standing up a server — and because the handler block reached 129 lines once it
+// stopped ignoring the query string.
+// The client has always sent from/to/source/limit/timeZone; the project handler
+// read none of them and aggregated the whole file unconditionally. Pick "24h"
+// and every other card narrowed while Projects kept showing all-time totals,
+// with nothing on screen saying so — someone comparing "this week's spend by
+// repo" was reading lifetime numbers.
+//
+// Pure surfacing: hour_start is already on every row. Same shape as the other
+// range-filtered handlers (getTimeZoneContext + rowDayKey), so a day boundary
+// means the same thing on every card.
+function buildProjectFilters(url) {
+  const from = url.searchParams.get("from") || "";
+  const to = url.searchParams.get("to") || "";
+  const timeZoneContext = getTimeZoneContext(url);
+  const requestedSource = (url.searchParams.get("source") || "").trim().toLowerCase();
+  const rawLimit = Number(url.searchParams.get("limit"));
+
+  return {
+    limit: Number.isFinite(rawLimit) && rawLimit > 0 ? Math.floor(rawLimit) : null,
+    // An absent bound means "no bound". The other handlers compare against ""
+    // directly, which is safe there because the client always sends both; here
+    // it does not, and `day <= ""` would silently return nothing.
+    inWindow(row) {
+      if (!from && !to) return true;
+      if (!row.hour_start) return false;
+      const day = rowDayKey(row, timeZoneContext);
+      return (!from || day >= from) && (!to || day <= to);
+    },
+    matchesSource(row) {
+      return !requestedSource || String(row.source || "").toLowerCase() === requestedSource;
+    },
+  };
+}
+
+// Which sources contributed usage in this window but carry no project
+// attribution at all. Without naming them, their absence reads as "that tool
+// cost nothing here" — the panel under-reports and looks complete while doing
+// it. `projectBucketsQueued` exists in 7 of the parsers; Cursor, Copilot, Zed,
+// Goose and Kiro have no per-repo story at all.
+function findUnattributedSources({ queuePath, allProjectRows, inWindow, matchesSource }) {
+  const attributed = new Set(
+    allProjectRows.filter(inWindow).map((row) => String(row.source || "").toLowerCase()),
+  );
+  return [
+    ...new Set(
+      readQueueData(queuePath)
+        .filter((row) => inWindow(row) && matchesSource(row))
+        .map((row) => String(row.source || "").toLowerCase())
+        .filter((src) => src && !attributed.has(src)),
+    ),
+  ].sort();
+}
+
+// No project-attributed rows at all (project attribution never synced, or only
+// non-project-capable CLIs used). Falls back to per-source totals so the panel
+// is not simply empty. This path once ALSO ran for the non-empty case and
+// produced fiction — "session-file count x total tokens" gave every short-and-
+// hot project the same weight as every long-and-cold one. Empty case only.
+function aggregateBySource(queuePath, inWindow, matchesSource) {
+  const bySrc = new Map();
+  for (const row of readQueueData(queuePath)) {
+    if (!inWindow(row) || !matchesSource(row)) continue;
+    const src = row.source || "unknown";
+    if (!bySrc.has(src)) {
+      bySrc.set(src, {
+        project_key: src,
+        // Synthetic source-only row: project_ref stays empty rather than
+        // fabricating `https://${src}.ai`, which resolves to unrelated domains
+        // (codex.ai, cursor.ai) and was sent to the dashboard as a clickable
+        // href before v0.11.1.
+        project_ref: "",
+        total_tokens: 0,
+        billable_total_tokens: 0,
+      });
+    }
+    bySrc.get(src).total_tokens += row.total_tokens || 0;
+    bySrc.get(src).billable_total_tokens += row.total_tokens || 0;
+  }
+  return bySrc;
+}
+
+// Per-repo cost, which the data model could not express until project rows
+// carried a model. computeRowCost is the same function the rest of the product
+// prices with, so a repo total and a model total cannot disagree.
+//
+// A row with no model prices at 0 and is counted as UNATTRIBUTED rather than
+// folded silently into the total — that is the #94 tier doing exactly what it
+// was added for. "Recorded before we recorded models" renders honestly instead
+// of as a confident $0.
+function aggregateByProject(rows) {
+  const byProject = new Map();
+  for (const row of rows) {
+    const key = row.project_key || "unknown";
+    if (!byProject.has(key)) {
+      byProject.set(key, {
+        project_key: key,
+        project_ref: row.project_ref || key,
+        total_tokens: 0,
+        billable_total_tokens: 0,
+        total_cost_usd: 0,
+        unattributed_tokens: 0,
+      });
+    }
+    const agg = byProject.get(key);
+    const tokens = Number(row.total_tokens || 0);
+    agg.total_tokens += tokens;
+    agg.billable_total_tokens += tokens;
+    const model = typeof row.model === "string" ? row.model.trim() : "";
+    if (!model || model === "unknown") {
+      agg.unattributed_tokens += tokens;
+    } else {
+      agg.total_cost_usd += computeRowCost(row);
+    }
+    if (!agg.project_ref && row.project_ref) agg.project_ref = row.project_ref;
+  }
+  return byProject;
+}
+
+// Rows -> ranked entries, in the shape the panel already renders (token counts
+// as strings). Split out so buildProjectUsageSummary stays under the size rule.
+function rankProjectEntries(byKey) {
+  return Array.from(byKey.values())
+    .sort((a, b) => b.billable_total_tokens - a.billable_total_tokens)
+    .map((e) => ({
+      ...e,
+      total_tokens: String(e.total_tokens),
+      billable_total_tokens: String(e.billable_total_tokens),
+      // Strings for the same reason the token counts are: the panel formats
+      // them, and a float in JSON invites a rounding difference between the two
+      // sides of the wire.
+      total_cost_usd: (e.total_cost_usd ?? 0).toFixed(6),
+      unattributed_tokens: String(e.unattributed_tokens ?? 0),
+    }));
+}
+
+function buildProjectUsageSummary({ url, queuePath, projectQueuePath }) {
+    // Use the per-project bucket log that rollout.js emits — it already
+    // carries the actual tokens attributed to each (project_key, source,
+    // hour_start). Falling back to "session-file count × total tokens"
+    // (the old behavior) produced pure fiction: every short-and-hot
+    // project got the same weight as every long-and-cold one.
+    const allProjectRows = readProjectQueueData(projectQueuePath);
+
+    const { inWindow, matchesSource, limit } = buildProjectFilters(url);
+
+    const projectRows = allProjectRows.filter((row) => inWindow(row) && matchesSource(row));
+
+    const unattributedSources = findUnattributedSources({
+      queuePath,
+      allProjectRows,
+      inWindow,
+      matchesSource,
+    });
+
+    const byProject = aggregateByProject(projectRows);
+
+    let entries =
+      byProject.size === 0
+        ? rankProjectEntries(aggregateBySource(queuePath, inWindow, matchesSource))
+        : rankProjectEntries(byProject);
+
+    if (limit != null) entries = entries.slice(0, limit);
+
+    return {
+      generated_at: new Date().toISOString(),
+      entries,
+      // Named so the panel can say what it cannot account for, rather than
+      // letting a missing tool read as a tool that cost nothing.
+      unattributed_sources: unattributedSources,
+    };
+}
+
 function createLocalApiHandler({ queuePath }) {
   const qp = queuePath || resolveQueuePath();
 
@@ -833,11 +1091,8 @@ function createLocalApiHandler({ queuePath }) {
         "abs.twimg.com",
         "api.dicebear.com",
       ];
-      const hostOk = AVATAR_HOST_ALLOWLIST.some(
-        (h) => parsed.hostname === h || parsed.hostname.endsWith(`.${h}`),
-      );
-      if (!hostOk) {
-        json(res, { error: "Host not allowed" }, 403);
+      if (!isAllowedAvatarTarget(parsed, AVATAR_HOST_ALLOWLIST)) {
+        json(res, { error: "Address not allowed" }, 403);
         return true;
       }
 
@@ -855,15 +1110,22 @@ function createLocalApiHandler({ queuePath }) {
       }
 
       try {
-        const upstream = await fetch(cacheKey, {
-          method,
-          redirect: "follow",
-          headers: {
-            accept: req.headers["accept"] || "image/*",
-            "accept-language": req.headers["accept-language"] || "en",
-            "user-agent": "TokenTracker/AvatarProxy",
+        const upstream = await fetchAvatarFollowingAllowlist(
+          cacheKey,
+          {
+            method,
+            headers: {
+              accept: req.headers["accept"] || "image/*",
+              "accept-language": req.headers["accept-language"] || "en",
+              "user-agent": "TokenTracker/AvatarProxy",
+            },
           },
-        });
+          AVATAR_HOST_ALLOWLIST,
+        );
+        if (upstream === AVATAR_REDIRECT_BLOCKED) {
+          json(res, { error: "Redirect target not allowed" }, 403);
+          return true;
+        }
         if (!upstream.ok) {
           json(res, { error: `Upstream ${upstream.status}` }, upstream.status);
           return true;
@@ -873,15 +1135,21 @@ function createLocalApiHandler({ queuePath }) {
           json(res, { error: "Not an image" }, 415);
           return true;
         }
-        const body = Buffer.from(await upstream.arrayBuffer());
-        if (body.length <= AVATAR_PROXY_MAX_BYTES) {
-          // Simple LRU: drop oldest if over capacity.
-          if (avatarProxyCache.size >= AVATAR_PROXY_MAX_ENTRIES) {
-            const oldestKey = avatarProxyCache.keys().next().value;
-            if (oldestKey) avatarProxyCache.delete(oldestKey);
-          }
-          avatarProxyCache.set(cacheKey, { body, contentType, fetchedAt: now });
+        const body = await readCappedAvatarBody(upstream, AVATAR_PROXY_MAX_BYTES);
+        if (body === null) {
+          // Refused rather than served-but-not-cached, which is what the old
+          // length check amounted to. An avatar this size is not an avatar, and
+          // the dashboard falls back to its local icon on a failed image load.
+          json(res, { error: "Avatar too large" }, 413);
+          return true;
         }
+        // Within the cap by construction, so caching is unconditional.
+        // Simple LRU: drop oldest if over capacity.
+        if (avatarProxyCache.size >= AVATAR_PROXY_MAX_ENTRIES) {
+          const oldestKey = avatarProxyCache.keys().next().value;
+          if (oldestKey) avatarProxyCache.delete(oldestKey);
+        }
+        avatarProxyCache.set(cacheKey, { body, contentType, fetchedAt: now });
         res.writeHead(200, {
           "Content-Type": contentType,
           "Cache-Control": "public, max-age=3600",
@@ -1230,78 +1498,14 @@ function createLocalApiHandler({ queuePath }) {
 
     // --- project-usage-summary ---
     if (p === "/functions/tokentracker-project-usage-summary") {
-      // Use the per-project bucket log that rollout.js emits — it already
-      // carries the actual tokens attributed to each (project_key, source,
-      // hour_start). Falling back to "session-file count × total tokens"
-      // (the old behavior) produced pure fiction: every short-and-hot
-      // project got the same weight as every long-and-cold one.
-      const projectQueuePath = path.join(
-        path.dirname(qp),
-        "project.queue.jsonl",
+      json(
+        res,
+        buildProjectUsageSummary({
+          url,
+          queuePath: qp,
+          projectQueuePath: path.join(path.dirname(qp), "project.queue.jsonl"),
+        }),
       );
-      const projectRows = readProjectQueueData(projectQueuePath);
-
-      const byProject = new Map();
-      for (const row of projectRows) {
-        const key = row.project_key || "unknown";
-        if (!byProject.has(key)) {
-          byProject.set(key, {
-            project_key: key,
-            project_ref: row.project_ref || key,
-            total_tokens: 0,
-            billable_total_tokens: 0,
-          });
-        }
-        const agg = byProject.get(key);
-        agg.total_tokens += Number(row.total_tokens || 0);
-        agg.billable_total_tokens += Number(row.total_tokens || 0);
-        if (!agg.project_ref && row.project_ref) agg.project_ref = row.project_ref;
-      }
-
-      // If no project-attributed rows exist yet (user hasn't synced project
-      // attribution, or never used a project-capable CLI), fall back to
-      // per-source aggregation over the main queue so the panel isn't
-      // totally empty. This path used to also exist for the non-empty case
-      // and produce wrong numbers; keep it only as the empty fallback.
-      let entries;
-      if (byProject.size === 0) {
-        const rows = readQueueData(qp);
-        const bySrc = new Map();
-        for (const row of rows) {
-          const src = row.source || "unknown";
-          if (!bySrc.has(src)) {
-            bySrc.set(src, {
-              project_key: src,
-              // Synthetic source-only row: leave project_ref empty rather than
-              // fabricating `https://${src}.ai`, which resolves to unrelated
-              // domains (e.g. codex.ai, cursor.ai) and was sent to the
-              // dashboard as a clickable href before v0.11.1 / this commit.
-              project_ref: "",
-              total_tokens: 0,
-              billable_total_tokens: 0,
-            });
-          }
-          bySrc.get(src).total_tokens += row.total_tokens || 0;
-          bySrc.get(src).billable_total_tokens += row.total_tokens || 0;
-        }
-        entries = Array.from(bySrc.values())
-          .sort((a, b) => b.billable_total_tokens - a.billable_total_tokens)
-          .map((e) => ({
-            ...e,
-            total_tokens: String(e.total_tokens),
-            billable_total_tokens: String(e.billable_total_tokens),
-          }));
-      } else {
-        entries = Array.from(byProject.values())
-          .sort((a, b) => b.billable_total_tokens - a.billable_total_tokens)
-          .map((e) => ({
-            ...e,
-            total_tokens: String(e.total_tokens),
-            billable_total_tokens: String(e.billable_total_tokens),
-          }));
-      }
-
-      json(res, { generated_at: new Date().toISOString(), entries });
       return true;
     }
 
@@ -1534,7 +1738,16 @@ function createLocalApiHandler({ queuePath }) {
 }
 
 module.exports = {
+  // Exported for tests: the aggregation runs without a server.
+  buildProjectUsageSummary,
   createLocalApiHandler,
+  // Exported so the redirect allowlist can be tested directly: the SSRF this
+  // closes is only observable across a redirect hop, which no handler-level
+  // test reaches.
+  fetchAvatarFollowingAllowlist,
+  readCappedAvatarBody,
+  isAllowedAvatarTarget,
+  AVATAR_REDIRECT_BLOCKED,
   resolveQueuePath,
   // Exported for cross-consumer tests (pricing + native contract lock).
   MODEL_PRICING,