@ipv9/tokentracker-cli 0.39.43 → 0.39.44

package/src/lib/queue-compact.js

@@ -0,0 +1,220 @@
+"use strict";
+
+// Compaction for the append-only queue, and the row invariant that lived in
+// prose.
+//
+// Measured on a real install: 34,492 lines, 5,595 unique keys, 28,897
+// superseded (83.8%), 11 MB. Five sixths of the file is dead weight, and
+// `readQueueData` in local-api.js re-reads and re-dedups ALL of it on every
+// endpoint call — a dashboard refresh hits 6-8 endpoints, auto-refresh defaults
+// to 30s. Nothing ever reclaims it; the only rewrite in the codebase is a
+// one-off migration.
+//
+// The design risk here is close to zero for one reason: THE READERS ALREADY
+// DEFINE THE OUTPUT. `readQueueData` keeps the last row per
+// `source|model|hour_start`. Compaction only has to produce what every reader
+// already computes, so it keeps the last RAW LINE per key — not a re-serialised
+// row. Byte-identical API responses then follow by construction rather than by
+// luck, because the surviving bytes are the exact bytes the reader would have
+// kept.
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+// Must match readQueueData in src/lib/local-api.js. If that key ever changes,
+// this one has to change with it — a test asserts they agree on real rows.
+function queueRowKey(row) {
+  return `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
+}
+
+const TOKEN_COLUMNS = [
+  "input_tokens",
+  "output_tokens",
+  "cache_creation_input_tokens",
+  "cached_input_tokens",
+  "reasoning_output_tokens",
+];
+
+const THIRTY_MINUTES_MS = 30 * 60 * 1000;
+
+// Codex and every-code report reasoning tokens that are ALREADY COUNTED inside
+// output_tokens, so their total_tokens correctly excludes the reasoning column —
+// adding it would count those tokens twice.
+//
+// This is not a special case invented here. `computeRowCost` in
+// src/lib/pricing/index.js:309 makes exactly the same distinction, and charges
+// reasoning at zero for these two sources for the same reason.
+//
+// Found by running this check against a real 34,922-row queue: 8,236 rows
+// "violated" the invariant, every one of them source=codex, and in every case
+// the difference was exactly reasoning_output_tokens. The rows were right, the
+// check was wrong, and so was the prose it came from — CLAUDE.md said "sum of
+// all columns" with no exception. Corrected there too.
+const REASONING_FOLDED_INTO_OUTPUT = new Set(["codex", "every-code"]);
+
+function expectedTotal(row) {
+  const folded = REASONING_FOLDED_INTO_OUTPUT.has(String(row.source || "").toLowerCase());
+  return TOKEN_COLUMNS.reduce((acc, column) => {
+    if (folded && column === "reasoning_output_tokens") return acc;
+    const value = Number(row[column] ?? 0);
+    return acc + (Number.isFinite(value) ? value : 0);
+  }, 0);
+}
+
+// Decides which lines survive, without touching the disk. Split out so the
+// decision is testable on its own and so `analyze` and `compact` cannot drift.
+//
+// Malformed lines are KEPT. They are invisible to every reader already, so
+// dropping them would not change a single API response — but it would destroy
+// bytes nobody has looked at, and a partial write worth investigating is
+// exactly the kind of thing that should survive a routine maintenance command.
+function planCompaction(raw) {
+  const lines = raw.split("\n");
+  const keep = new Set();
+  const lastForKey = new Map();
+  let parseable = 0;
+  let malformed = 0;
+
+  lines.forEach((line, index) => {
+    if (!line.trim()) return;
+    let row;
+    try {
+      row = JSON.parse(line);
+    } catch {
+      malformed += 1;
+      keep.add(index);
+      return;
+    }
+    parseable += 1;
+    lastForKey.set(queueRowKey(row), index);
+  });
+
+  for (const index of lastForKey.values()) keep.add(index);
+
+  const kept = [...keep].sort((a, b) => a - b);
+  return {
+    lines: kept.map((index) => lines[index]),
+    stats: {
+      totalLines: parseable + malformed,
+      parseable,
+      malformed,
+      uniqueKeys: lastForKey.size,
+      superseded: parseable - lastForKey.size,
+      keptLines: kept.length,
+    },
+  };
+}
+
+function analyzeQueue(queuePath) {
+  let raw;
+  try {
+    raw = fs.readFileSync(queuePath, "utf8");
+  } catch (e) {
+    if (e?.code === "ENOENT") {
+      return { totalLines: 0, parseable: 0, malformed: 0, uniqueKeys: 0, superseded: 0, keptLines: 0, ratio: 0, bytes: 0 };
+    }
+    throw e;
+  }
+  const { stats } = planCompaction(raw);
+  return {
+    ...stats,
+    ratio: stats.parseable > 0 ? stats.superseded / stats.parseable : 0,
+    bytes: Buffer.byteLength(raw, "utf8"),
+  };
+}
+
+// Writes to a temp file in the same directory and renames over the original.
+// Same atomic-replace pattern the project-queue rewrite already uses. An
+// interrupt between write and rename leaves the original untouched — the temp
+// file is the only casualty.
+//
+// The CALLER holds the sync lock. This does not take it, because the lock is
+// per-invocation state owned by the sync command, and a second acquisition
+// inside would deadlock against the first.
+function compactQueue(queuePath) {
+  let raw;
+  try {
+    raw = fs.readFileSync(queuePath, "utf8");
+  } catch (e) {
+    if (e?.code === "ENOENT") return { changed: false, reason: "no queue file" };
+    throw e;
+  }
+
+  const before = Buffer.byteLength(raw, "utf8");
+  const { lines, stats } = planCompaction(raw);
+  if (stats.superseded === 0) {
+    return { changed: false, reason: "nothing superseded", ...stats, bytesBefore: before, bytesAfter: before };
+  }
+
+  const out = lines.join("\n") + "\n";
+  const tmp = path.join(
+    path.dirname(queuePath),
+    `${path.basename(queuePath)}.compact.${process.pid}.tmp`,
+  );
+  fs.writeFileSync(tmp, out, "utf8");
+  try {
+    fs.renameSync(tmp, queuePath);
+  } catch (e) {
+    fs.unlinkSync(tmp);
+    throw e;
+  }
+
+  return {
+    changed: true,
+    ...stats,
+    bytesBefore: before,
+    bytesAfter: Buffer.byteLength(out, "utf8"),
+  };
+}
+
+// CLAUDE.md states the column invariant in prose:
+//
+//   total = input + output + cache_creation + cache_read + reasoning
+//
+// Nothing enforced it at runtime. A miswritten or corrupt row was aggregated and
+// rendered, not flagged — and a parser bug of exactly this shape is the class
+// CLAUDE.md records at 1.6-7x magnitude. Same conversion as the curated-expiry
+// and version-lockstep checks: a rule that lived in a document starts running.
+//
+// Returns one finding per violating row, capped by the caller.
+function findRowViolations(rows) {
+  const findings = [];
+  rows.forEach((row, index) => {
+    const where = `row ${index + 1} (${row.source || "?"}|${row.model || "?"}|${row.hour_start || "?"})`;
+
+    for (const column of TOKEN_COLUMNS) {
+      const value = Number(row[column] ?? 0);
+      if (!Number.isFinite(value)) {
+        findings.push(`${where}: ${column} is not a number (${JSON.stringify(row[column])})`);
+      } else if (value < 0) {
+        findings.push(`${where}: ${column} is negative (${value})`);
+      }
+    }
+
+    const sum = expectedTotal(row);
+    const total = Number(row.total_tokens ?? 0);
+    if (Number.isFinite(total) && total !== sum) {
+      findings.push(`${where}: total_tokens ${total} != expected ${sum}`);
+    }
+
+    const bucket = Date.parse(row.hour_start);
+    if (!Number.isFinite(bucket)) {
+      findings.push(`${where}: hour_start is not a timestamp`);
+    } else if (bucket % THIRTY_MINUTES_MS !== 0) {
+      findings.push(`${where}: hour_start is not on a 30-minute UTC boundary`);
+    }
+  });
+  return findings;
+}
+
+module.exports = {
+  queueRowKey,
+  expectedTotal,
+  REASONING_FOLDED_INTO_OUTPUT,
+  planCompaction,
+  analyzeQueue,
+  compactQueue,
+  findRowViolations,
+  TOKEN_COLUMNS,
+  THIRTY_MINUTES_MS,
+};

package/src/lib/rollout.js

@@ -937,11 +937,12 @@ async function parseRolloutFile({
         projectState,
         currentProjectKey,
         source,
+        model,
         bucketStart,
         currentProjectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(currentProjectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(currentProjectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
   }
@@ -1061,11 +1062,12 @@ async function parseClaudeFile({
         projectState,
         projectKey,
         source,
+        model,
         bucketStart,
         projectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
   }
@@ -1145,11 +1147,12 @@ async function parseGeminiFile({
         projectState,
         projectKey,
         source,
+        model,
         bucketStart,
         projectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
     totals = currentTotals;
@@ -1256,11 +1259,12 @@ async function parseOpencodeMessageFile({
       projectState,
       projectKey,
       source,
+      model,
       bucketStart,
       projectRef,
     );
     addTotals(projectBucket.totals, delta);
-    projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+    projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
   }
   return { messageKey, lastTotals: currentTotals, eventsAggregated: 1, shouldUpdate: true };
 }
@@ -1598,6 +1602,7 @@ async function enqueueTouchedProjectBuckets({
         project_ref: projectRef,
         project_key: projectKey,
         source: bucket.source,
+        model: bucket.model || PROJECT_MODEL_UNATTRIBUTED,
         hour_start: bucket.hour_start,
         input_tokens: totals.input_tokens,
         cached_input_tokens: totals.cached_input_tokens,
@@ -1767,12 +1772,16 @@ function normalizeProjectState(raw) {
     projects[key] = { ...value };
   }
 
-  return {
-    version: 2,
+  const normalized = {
+    version: 3,
     buckets,
     projects,
     updatedAt: typeof state.updatedAt === "string" ? state.updatedAt : null,
   };
+  // Every parser reaches project state through here, so one call covers all of
+  // them. Idempotent: a v3 key already has four segments and is skipped.
+  migrateProjectBucketsToModelKey(normalized);
+  return normalized;
 }
 
 function normalizeOpencodeState(raw) {
@@ -1819,10 +1828,11 @@ function getHourlyBucket(state, source, model, hourStart) {
   return bucket;
 }
 
-function getProjectBucket(state, projectKey, source, hourStart, projectRef) {
+function getProjectBucket(state, projectKey, source, model, hourStart, projectRef) {
   const buckets = state.buckets;
   const normalizedSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
-  const key = projectBucketKey(projectKey, normalizedSource, hourStart);
+  const normalizedModel = normalizeModelInput(model) || PROJECT_MODEL_UNATTRIBUTED;
+  const key = projectBucketKey(projectKey, normalizedSource, normalizedModel, hourStart);
   let bucket = buckets[key];
   if (!bucket || typeof bucket !== "object") {
     bucket = {
@@ -1831,6 +1841,7 @@ function getProjectBucket(state, projectKey, source, hourStart, projectRef) {
       project_key: projectKey,
       project_ref: projectRef,
       source: normalizedSource,
+      model: normalizedModel,
       hour_start: hourStart,
     };
     buckets[key] = bucket;
@@ -1921,9 +1932,53 @@ function bucketKey(source, model, hourStart) {
   return `${safeSource}${BUCKET_SEPARATOR}${safeModel}${BUCKET_SEPARATOR}${hourStart}`;
 }
 
-function projectBucketKey(projectKey, source, hourStart) {
+// Per-repo COST is per-model, and the key had no model in it, so the repo x model
+// join this product's README promises ("which repo, which model, and which
+// hour") had never existed. Adding model is what makes computeRowCost usable
+// per project.
+//
+// The transition hazard is the whole job. A legacy row keyed
+// project|source|hour and a new row keyed project|source|model|hour describe the
+// SAME bucket at different granularity, so a reader that keeps both
+// double-counts. Rows written before this change are keyed with
+// PROJECT_MODEL_UNATTRIBUTED, so they occupy one slot alongside the model-keyed
+// rows rather than shadowing them — see migrateProjectBucketsToModelKey.
+const PROJECT_MODEL_UNATTRIBUTED = "unknown";
+
+function projectBucketKey(projectKey, source, model, hourStart) {
   const safeSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
-  return `${projectKey}${BUCKET_SEPARATOR}${safeSource}${BUCKET_SEPARATOR}${hourStart}`;
+  const safeModel = normalizeModelInput(model) || PROJECT_MODEL_UNATTRIBUTED;
+  return (
+    `${projectKey}${BUCKET_SEPARATOR}${safeSource}` +
+    `${BUCKET_SEPARATOR}${safeModel}${BUCKET_SEPARATOR}${hourStart}`
+  );
+}
+
+// Re-keys buckets carried in cursors.json from the pre-model key to the new one,
+// tagging them PROJECT_MODEL_UNATTRIBUTED.
+//
+// Without this the old bucket keeps its old key, never matches again, and its
+// running total is stranded: new usage in the same hour starts from zero under
+// the model key while the reader still sees the old row. Sum = double count.
+//
+// After it, the stranded bucket becomes the `unknown` slot for that hour. It
+// stops receiving new usage (that goes to the model-specific bucket), so it is
+// frozen at its pre-upgrade total and the two sum to the right number — no
+// double count, and nothing lost.
+function migrateProjectBucketsToModelKey(projectState) {
+  const buckets = projectState?.buckets;
+  if (!buckets || typeof buckets !== "object") return 0;
+  let migrated = 0;
+  for (const [key, bucket] of Object.entries(buckets)) {
+    if (key.split(BUCKET_SEPARATOR).length !== 3) continue;
+    const [projectKey, source, hourStart] = key.split(BUCKET_SEPARATOR);
+    const nextKey = projectBucketKey(projectKey, source, PROJECT_MODEL_UNATTRIBUTED, hourStart);
+    if (buckets[nextKey]) continue;
+    buckets[nextKey] = { ...bucket, model: PROJECT_MODEL_UNATTRIBUTED };
+    delete buckets[key];
+    migrated += 1;
+  }
+  return migrated;
 }
 
 function groupBucketKey(source, hourStart) {
@@ -2663,11 +2718,14 @@ async function parseOpencodeDbIncremental({
           projectState,
           projectKey,
           defaultSource,
+          model,
           bucketStart,
           projectRef,
         );
         addTotals(projectBucket.totals, delta);
-        projectTouchedBuckets.add(projectBucketKey(projectKey, defaultSource, bucketStart));
+        projectTouchedBuckets.add(
+          projectBucketKey(projectKey, defaultSource, model, bucketStart),
+        );
       }
     }
 
@@ -8608,11 +8666,12 @@ async function parseAntigravityFile({
         projectState,
         projectKey,
         source,
+        model,
         bucketStart,
         projectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
     // Snapshot the pre-planner context first. The planner's own content+tool_calls
@@ -8707,6 +8766,11 @@ function isCjkCodePoint(code) {
 }
 
 module.exports = {
+  // Exported for tests: the mixed-era rule is the risky part of this change.
+  projectBucketKey,
+  migrateProjectBucketsToModelKey,
+  normalizeProjectState,
+  PROJECT_MODEL_UNATTRIBUTED,
   listRolloutFiles,
   listClaudeProjectFiles,
   listGeminiSessionFiles,

package/src/lib/skills-manager.js

@@ -287,13 +287,13 @@ function hashDirectory(dir) {
           continue;
         }
         const execBit = process.platform === "win32" ? 0 : stat.mode & 0o111 ? 1 : 0;
-        hash.update(`${rel}${execBit}`);
+        hash.update(`${rel}\0${execBit}\0`);
         try {
           hash.update(fs.readFileSync(abs));
         } catch (_e) {
           // unreadable file — fold its absence in deterministically
         }
-        hash.update("");
+        hash.update("\0");
       }
     }
   };

package/src/lib/usage-limits.js

@@ -218,10 +218,23 @@ async function fetchCodexUsageLimits(
     method: "GET",
     headers,
   });
-  // 401/403/404 from wham means "no usage data available for this auth state" — render
-  // a neutral empty state instead of a red "Fetch failed" error.
+  // 401/403/404 from wham means "no usage data available for this auth state" —
+  // a free or multi-account user, or a token that went stale. #52 asked for a
+  // neutral state here rather than a red "Fetch failed".
+  //
+  // It got NO state. With no error and no windows, LimitChips falls through to
+  // `windows.length === 0 -> return null` and the chip disappears entirely,
+  // which is the outcome #105 calls worse than never having had a chip: the user
+  // has been trained to look there and now reads absence as "plenty left".
+  //
+  // `notice` is the third state the UI was missing. Visible and subtle, so #52's
+  // intent survives without #105's failure mode.
   if (res.status === 401 || res.status === 403 || res.status === 404) {
-    return { primary_window: null, secondary_window: null };
+    return {
+      primary_window: null,
+      secondary_window: null,
+      notice: "No usage data for this sign-in. Run `codex` to sign in again.",
+    };
   }
   if (!res.ok) {
     throw new Error(`Codex API returned ${res.status}`);
@@ -2013,6 +2026,10 @@ async function getUsageLimits({
     codex = {
       configured: true,
       error: null,
+      // A 4xx from wham produces no windows and no error. Carrying its `notice`
+      // through is what keeps the chip on screen instead of falling through to
+      // LimitChips' `windows.length === 0 -> return null`.
+      notice: codexResult.value.notice || null,
       plan_type: codexPlanType || null,
       primary_window: codexResult.value.primary_window,
       secondary_window: codexResult.value.secondary_window,