@ipv9/tokentracker-cli 0.39.43 → 0.39.45

package/src/lib/process-list.js

@@ -0,0 +1,91 @@
+const cp = require("node:child_process");
+
+// Shared process-listing primitives. Extracted from usage-limits.js so that a
+// second caller (transcript-suppression.js) can reuse the parser instead of
+// keeping a second copy of the same regex in the tree.
+//
+// PRIVACY: callers get raw command lines here, and a command line can contain a
+// user's file paths. Nothing in this module writes, logs, caches, or returns a
+// command line beyond the synchronous call — that constraint belongs to every
+// caller, and CONTRIBUTING.md's rule ("never log, store, transmit, or print ...
+// file paths from user code") is what it exists to satisfy.
+
+const PS_BINARY = "/bin/ps";
+// `-x` (own user, including processes with no controlling terminal) rather than
+// `-ax` (every user on the box). The suppression check reports its findings over
+// an unauthenticated loopback endpoint, and on a multi-user host `-a` would make
+// that endpoint answer questions about other people's sessions. Scoping the scan
+// itself is the narrow fix: a session TokenTracker could not have recorded
+// anyway is one this user is not running.
+//
+// Verified on both supported platforms rather than assumed from documented
+// semantics, because Linux `ps` is procps and parses dash-prefixed options as
+// UNIX-style, where `-x` is not an option at all. It does accept this as the BSD
+// `x`: on Debian 12 / procps-ng 4.0.2, `ps -x -o pid=,command=` exits 0 and
+// lists one user, while `-ax` on the same box lists seven. macOS/BSD `ps` is the
+// native case. Had procps rejected it, every Linux host would have fallen into
+// `process_list_failed` — a permanent non-advisory warn, which would pin
+// `degraded` for a whole platform.
+//
+// Frozen because two modules now share this array. Importing one constant stops
+// the two scans from drifting apart editorially; freezing is what stops a caller
+// pushing `-a` onto it at runtime.
+const PS_ARGS = Object.freeze(["-x", "-o", "pid=,command="]);
+const PS_TIMEOUT_MS = 4000;
+const PS_MAX_BUFFER = 10 * 1024 * 1024;
+
+function parseProcessLine(line) {
+  const match = String(line || "")
+    .trim()
+    .match(/^(\d+)\s+(.*)$/);
+  if (!match) return null;
+  return {
+    pid: Number(match[1]),
+    command: match[2],
+  };
+}
+
+// `/bin/ps` with these flags is a POSIX assumption. Windows has no equivalent
+// at this path, and guessing at `tasklist` output would be an untested code
+// path, so the honest answer there is "not supported" rather than "no problems
+// found" — the caller must not turn this into a passing check.
+function isProcessListSupported(platform = process.platform) {
+  return platform !== "win32";
+}
+
+// Returns { supported, ok, lines, reason }. `ok: false` never throws: a machine
+// that refuses `ps` (sandbox, hardened runtime) is a normal condition here, and
+// the caller reports it as "could not check" rather than as "nothing found".
+function listProcessLines({ commandRunner, platform = process.platform } = {}) {
+  if (!isProcessListSupported(platform)) {
+    return { supported: false, ok: false, lines: [], reason: "unsupported_platform" };
+  }
+
+  const runner = typeof commandRunner === "function" ? commandRunner : cp.spawnSync;
+  let result;
+  try {
+    result = runner(PS_BINARY, PS_ARGS, {
+      encoding: "utf8",
+      maxBuffer: PS_MAX_BUFFER,
+      timeout: PS_TIMEOUT_MS,
+    });
+  } catch {
+    return { supported: true, ok: false, lines: [], reason: "process_list_failed" };
+  }
+
+  if (result?.error || result?.status !== 0) {
+    return { supported: true, ok: false, lines: [], reason: "process_list_failed" };
+  }
+
+  const stdout = typeof result?.stdout === "string" ? result.stdout : "";
+  return { supported: true, ok: true, lines: stdout.split("\n"), reason: null };
+}
+
+module.exports = {
+  PS_ARGS,
+  PS_BINARY,
+  PS_TIMEOUT_MS,
+  isProcessListSupported,
+  listProcessLines,
+  parseProcessLine,
+};

package/src/lib/queue-compact.js

@@ -0,0 +1,220 @@
+"use strict";
+
+// Compaction for the append-only queue, and the row invariant that lived in
+// prose.
+//
+// Measured on a real install: 34,492 lines, 5,595 unique keys, 28,897
+// superseded (83.8%), 11 MB. Five sixths of the file is dead weight, and
+// `readQueueData` in local-api.js re-reads and re-dedups ALL of it on every
+// endpoint call — a dashboard refresh hits 6-8 endpoints, auto-refresh defaults
+// to 30s. Nothing ever reclaims it; the only rewrite in the codebase is a
+// one-off migration.
+//
+// The design risk here is close to zero for one reason: THE READERS ALREADY
+// DEFINE THE OUTPUT. `readQueueData` keeps the last row per
+// `source|model|hour_start`. Compaction only has to produce what every reader
+// already computes, so it keeps the last RAW LINE per key — not a re-serialised
+// row. Byte-identical API responses then follow by construction rather than by
+// luck, because the surviving bytes are the exact bytes the reader would have
+// kept.
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+// Must match readQueueData in src/lib/local-api.js. If that key ever changes,
+// this one has to change with it — a test asserts they agree on real rows.
+function queueRowKey(row) {
+  return `${row.source || ""}|${row.model || ""}|${row.hour_start || ""}`;
+}
+
+const TOKEN_COLUMNS = [
+  "input_tokens",
+  "output_tokens",
+  "cache_creation_input_tokens",
+  "cached_input_tokens",
+  "reasoning_output_tokens",
+];
+
+const THIRTY_MINUTES_MS = 30 * 60 * 1000;
+
+// Codex and every-code report reasoning tokens that are ALREADY COUNTED inside
+// output_tokens, so their total_tokens correctly excludes the reasoning column —
+// adding it would count those tokens twice.
+//
+// This is not a special case invented here. `computeRowCost` in
+// src/lib/pricing/index.js:309 makes exactly the same distinction, and charges
+// reasoning at zero for these two sources for the same reason.
+//
+// Found by running this check against a real 34,922-row queue: 8,236 rows
+// "violated" the invariant, every one of them source=codex, and in every case
+// the difference was exactly reasoning_output_tokens. The rows were right, the
+// check was wrong, and so was the prose it came from — CLAUDE.md said "sum of
+// all columns" with no exception. Corrected there too.
+const REASONING_FOLDED_INTO_OUTPUT = new Set(["codex", "every-code"]);
+
+function expectedTotal(row) {
+  const folded = REASONING_FOLDED_INTO_OUTPUT.has(String(row.source || "").toLowerCase());
+  return TOKEN_COLUMNS.reduce((acc, column) => {
+    if (folded && column === "reasoning_output_tokens") return acc;
+    const value = Number(row[column] ?? 0);
+    return acc + (Number.isFinite(value) ? value : 0);
+  }, 0);
+}
+
+// Decides which lines survive, without touching the disk. Split out so the
+// decision is testable on its own and so `analyze` and `compact` cannot drift.
+//
+// Malformed lines are KEPT. They are invisible to every reader already, so
+// dropping them would not change a single API response — but it would destroy
+// bytes nobody has looked at, and a partial write worth investigating is
+// exactly the kind of thing that should survive a routine maintenance command.
+function planCompaction(raw) {
+  const lines = raw.split("\n");
+  const keep = new Set();
+  const lastForKey = new Map();
+  let parseable = 0;
+  let malformed = 0;
+
+  lines.forEach((line, index) => {
+    if (!line.trim()) return;
+    let row;
+    try {
+      row = JSON.parse(line);
+    } catch {
+      malformed += 1;
+      keep.add(index);
+      return;
+    }
+    parseable += 1;
+    lastForKey.set(queueRowKey(row), index);
+  });
+
+  for (const index of lastForKey.values()) keep.add(index);
+
+  const kept = [...keep].sort((a, b) => a - b);
+  return {
+    lines: kept.map((index) => lines[index]),
+    stats: {
+      totalLines: parseable + malformed,
+      parseable,
+      malformed,
+      uniqueKeys: lastForKey.size,
+      superseded: parseable - lastForKey.size,
+      keptLines: kept.length,
+    },
+  };
+}
+
+function analyzeQueue(queuePath) {
+  let raw;
+  try {
+    raw = fs.readFileSync(queuePath, "utf8");
+  } catch (e) {
+    if (e?.code === "ENOENT") {
+      return { totalLines: 0, parseable: 0, malformed: 0, uniqueKeys: 0, superseded: 0, keptLines: 0, ratio: 0, bytes: 0 };
+    }
+    throw e;
+  }
+  const { stats } = planCompaction(raw);
+  return {
+    ...stats,
+    ratio: stats.parseable > 0 ? stats.superseded / stats.parseable : 0,
+    bytes: Buffer.byteLength(raw, "utf8"),
+  };
+}
+
+// Writes to a temp file in the same directory and renames over the original.
+// Same atomic-replace pattern the project-queue rewrite already uses. An
+// interrupt between write and rename leaves the original untouched — the temp
+// file is the only casualty.
+//
+// The CALLER holds the sync lock. This does not take it, because the lock is
+// per-invocation state owned by the sync command, and a second acquisition
+// inside would deadlock against the first.
+function compactQueue(queuePath) {
+  let raw;
+  try {
+    raw = fs.readFileSync(queuePath, "utf8");
+  } catch (e) {
+    if (e?.code === "ENOENT") return { changed: false, reason: "no queue file" };
+    throw e;
+  }
+
+  const before = Buffer.byteLength(raw, "utf8");
+  const { lines, stats } = planCompaction(raw);
+  if (stats.superseded === 0) {
+    return { changed: false, reason: "nothing superseded", ...stats, bytesBefore: before, bytesAfter: before };
+  }
+
+  const out = lines.join("\n") + "\n";
+  const tmp = path.join(
+    path.dirname(queuePath),
+    `${path.basename(queuePath)}.compact.${process.pid}.tmp`,
+  );
+  fs.writeFileSync(tmp, out, "utf8");
+  try {
+    fs.renameSync(tmp, queuePath);
+  } catch (e) {
+    fs.unlinkSync(tmp);
+    throw e;
+  }
+
+  return {
+    changed: true,
+    ...stats,
+    bytesBefore: before,
+    bytesAfter: Buffer.byteLength(out, "utf8"),
+  };
+}
+
+// CLAUDE.md states the column invariant in prose:
+//
+//   total = input + output + cache_creation + cache_read + reasoning
+//
+// Nothing enforced it at runtime. A miswritten or corrupt row was aggregated and
+// rendered, not flagged — and a parser bug of exactly this shape is the class
+// CLAUDE.md records at 1.6-7x magnitude. Same conversion as the curated-expiry
+// and version-lockstep checks: a rule that lived in a document starts running.
+//
+// Returns one finding per violating row, capped by the caller.
+function findRowViolations(rows) {
+  const findings = [];
+  rows.forEach((row, index) => {
+    const where = `row ${index + 1} (${row.source || "?"}|${row.model || "?"}|${row.hour_start || "?"})`;
+
+    for (const column of TOKEN_COLUMNS) {
+      const value = Number(row[column] ?? 0);
+      if (!Number.isFinite(value)) {
+        findings.push(`${where}: ${column} is not a number (${JSON.stringify(row[column])})`);
+      } else if (value < 0) {
+        findings.push(`${where}: ${column} is negative (${value})`);
+      }
+    }
+
+    const sum = expectedTotal(row);
+    const total = Number(row.total_tokens ?? 0);
+    if (Number.isFinite(total) && total !== sum) {
+      findings.push(`${where}: total_tokens ${total} != expected ${sum}`);
+    }
+
+    const bucket = Date.parse(row.hour_start);
+    if (!Number.isFinite(bucket)) {
+      findings.push(`${where}: hour_start is not a timestamp`);
+    } else if (bucket % THIRTY_MINUTES_MS !== 0) {
+      findings.push(`${where}: hour_start is not on a 30-minute UTC boundary`);
+    }
+  });
+  return findings;
+}
+
+module.exports = {
+  queueRowKey,
+  expectedTotal,
+  REASONING_FOLDED_INTO_OUTPUT,
+  planCompaction,
+  analyzeQueue,
+  compactQueue,
+  findRowViolations,
+  TOKEN_COLUMNS,
+  THIRTY_MINUTES_MS,
+};

package/src/lib/rollout.js

@@ -937,11 +937,12 @@ async function parseRolloutFile({
         projectState,
         currentProjectKey,
         source,
+        model,
         bucketStart,
         currentProjectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(currentProjectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(currentProjectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
   }
@@ -1061,11 +1062,12 @@ async function parseClaudeFile({
         projectState,
         projectKey,
         source,
+        model,
         bucketStart,
         projectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
   }
@@ -1145,11 +1147,12 @@ async function parseGeminiFile({
         projectState,
         projectKey,
         source,
+        model,
         bucketStart,
         projectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
     totals = currentTotals;
@@ -1256,11 +1259,12 @@ async function parseOpencodeMessageFile({
       projectState,
       projectKey,
       source,
+      model,
       bucketStart,
       projectRef,
     );
     addTotals(projectBucket.totals, delta);
-    projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+    projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
   }
   return { messageKey, lastTotals: currentTotals, eventsAggregated: 1, shouldUpdate: true };
 }
@@ -1598,6 +1602,7 @@ async function enqueueTouchedProjectBuckets({
         project_ref: projectRef,
         project_key: projectKey,
         source: bucket.source,
+        model: bucket.model || PROJECT_MODEL_UNATTRIBUTED,
         hour_start: bucket.hour_start,
         input_tokens: totals.input_tokens,
         cached_input_tokens: totals.cached_input_tokens,
@@ -1767,12 +1772,16 @@ function normalizeProjectState(raw) {
     projects[key] = { ...value };
   }
 
-  return {
-    version: 2,
+  const normalized = {
+    version: 3,
     buckets,
     projects,
     updatedAt: typeof state.updatedAt === "string" ? state.updatedAt : null,
   };
+  // Every parser reaches project state through here, so one call covers all of
+  // them. Idempotent: a v3 key already has four segments and is skipped.
+  migrateProjectBucketsToModelKey(normalized);
+  return normalized;
 }
 
 function normalizeOpencodeState(raw) {
@@ -1819,10 +1828,11 @@ function getHourlyBucket(state, source, model, hourStart) {
   return bucket;
 }
 
-function getProjectBucket(state, projectKey, source, hourStart, projectRef) {
+function getProjectBucket(state, projectKey, source, model, hourStart, projectRef) {
   const buckets = state.buckets;
   const normalizedSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
-  const key = projectBucketKey(projectKey, normalizedSource, hourStart);
+  const normalizedModel = normalizeModelInput(model) || PROJECT_MODEL_UNATTRIBUTED;
+  const key = projectBucketKey(projectKey, normalizedSource, normalizedModel, hourStart);
   let bucket = buckets[key];
   if (!bucket || typeof bucket !== "object") {
     bucket = {
@@ -1831,6 +1841,7 @@ function getProjectBucket(state, projectKey, source, hourStart, projectRef) {
       project_key: projectKey,
       project_ref: projectRef,
       source: normalizedSource,
+      model: normalizedModel,
       hour_start: hourStart,
     };
     buckets[key] = bucket;
@@ -1921,9 +1932,53 @@ function bucketKey(source, model, hourStart) {
   return `${safeSource}${BUCKET_SEPARATOR}${safeModel}${BUCKET_SEPARATOR}${hourStart}`;
 }
 
-function projectBucketKey(projectKey, source, hourStart) {
+// Per-repo COST is per-model, and the key had no model in it, so the repo x model
+// join this product's README promises ("which repo, which model, and which
+// hour") had never existed. Adding model is what makes computeRowCost usable
+// per project.
+//
+// The transition hazard is the whole job. A legacy row keyed
+// project|source|hour and a new row keyed project|source|model|hour describe the
+// SAME bucket at different granularity, so a reader that keeps both
+// double-counts. Rows written before this change are keyed with
+// PROJECT_MODEL_UNATTRIBUTED, so they occupy one slot alongside the model-keyed
+// rows rather than shadowing them — see migrateProjectBucketsToModelKey.
+const PROJECT_MODEL_UNATTRIBUTED = "unknown";
+
+function projectBucketKey(projectKey, source, model, hourStart) {
   const safeSource = normalizeSourceInput(source) || DEFAULT_SOURCE;
-  return `${projectKey}${BUCKET_SEPARATOR}${safeSource}${BUCKET_SEPARATOR}${hourStart}`;
+  const safeModel = normalizeModelInput(model) || PROJECT_MODEL_UNATTRIBUTED;
+  return (
+    `${projectKey}${BUCKET_SEPARATOR}${safeSource}` +
+    `${BUCKET_SEPARATOR}${safeModel}${BUCKET_SEPARATOR}${hourStart}`
+  );
+}
+
+// Re-keys buckets carried in cursors.json from the pre-model key to the new one,
+// tagging them PROJECT_MODEL_UNATTRIBUTED.
+//
+// Without this the old bucket keeps its old key, never matches again, and its
+// running total is stranded: new usage in the same hour starts from zero under
+// the model key while the reader still sees the old row. Sum = double count.
+//
+// After it, the stranded bucket becomes the `unknown` slot for that hour. It
+// stops receiving new usage (that goes to the model-specific bucket), so it is
+// frozen at its pre-upgrade total and the two sum to the right number — no
+// double count, and nothing lost.
+function migrateProjectBucketsToModelKey(projectState) {
+  const buckets = projectState?.buckets;
+  if (!buckets || typeof buckets !== "object") return 0;
+  let migrated = 0;
+  for (const [key, bucket] of Object.entries(buckets)) {
+    if (key.split(BUCKET_SEPARATOR).length !== 3) continue;
+    const [projectKey, source, hourStart] = key.split(BUCKET_SEPARATOR);
+    const nextKey = projectBucketKey(projectKey, source, PROJECT_MODEL_UNATTRIBUTED, hourStart);
+    if (buckets[nextKey]) continue;
+    buckets[nextKey] = { ...bucket, model: PROJECT_MODEL_UNATTRIBUTED };
+    delete buckets[key];
+    migrated += 1;
+  }
+  return migrated;
 }
 
 function groupBucketKey(source, hourStart) {
@@ -2663,11 +2718,14 @@ async function parseOpencodeDbIncremental({
           projectState,
           projectKey,
           defaultSource,
+          model,
           bucketStart,
           projectRef,
         );
         addTotals(projectBucket.totals, delta);
-        projectTouchedBuckets.add(projectBucketKey(projectKey, defaultSource, bucketStart));
+        projectTouchedBuckets.add(
+          projectBucketKey(projectKey, defaultSource, model, bucketStart),
+        );
       }
     }
 
@@ -3340,8 +3398,11 @@ async function parseHermesIncremental({ hermesPath, dbPath, cursors, queuePath,
       // Skip if delta is zero (session unchanged since last sync)
       if (dInput === 0 && dOutput === 0 && dCacheRead === 0 && dCacheWrite === 0 && dReasoning === 0) continue;
 
-      // Prefer ended_at for bucket placement; fall back to started_at
-      const epochSec = endedAt ?? startedAt;
+      // A first observation has only the session start as a usable timestamp.
+      // Attribute later active-session deltas to this sync so cross-day usage
+      // does not keep growing the day on which the session originally started.
+      // Once Hermes records completion, ended_at is authoritative for the final delta.
+      const epochSec = endedAt ?? (prev ? Date.parse(updatedAt) / 1000 : startedAt);
       if (!epochSec || !Number.isFinite(epochSec)) continue;
       const tsIso = new Date(epochSec * 1000).toISOString();
       const bucketStart = toUtcHalfHourStart(tsIso);
@@ -8608,11 +8669,12 @@ async function parseAntigravityFile({
         projectState,
         projectKey,
         source,
+        model,
         bucketStart,
         projectRef,
       );
       addTotals(projectBucket.totals, delta);
-      projectTouchedBuckets.add(projectBucketKey(projectKey, source, bucketStart));
+      projectTouchedBuckets.add(projectBucketKey(projectKey, source, model, bucketStart));
     }
     eventsAggregated += 1;
     // Snapshot the pre-planner context first. The planner's own content+tool_calls
@@ -8707,6 +8769,11 @@ function isCjkCodePoint(code) {
 }
 
 module.exports = {
+  // Exported for tests: the mixed-era rule is the risky part of this change.
+  projectBucketKey,
+  migrateProjectBucketsToModelKey,
+  normalizeProjectState,
+  PROJECT_MODEL_UNATTRIBUTED,
   listRolloutFiles,
   listClaudeProjectFiles,
   listGeminiSessionFiles,

package/src/lib/single-flight.js

@@ -0,0 +1,59 @@
+"use strict";
+
+/**
+ * Coalesce concurrent calls to one expensive async producer.
+ *
+ * The problem it solves is not "the same work runs twice" — it is that the work
+ * being duplicated fans out to every configured provider's private endpoint. Two
+ * dashboard tabs, a route mount, and a scheduled revalidation can all arrive
+ * inside the same second on a cold cache; without this, each one launches its own
+ * full sweep.
+ *
+ * `run(fn)` returns the in-flight promise when one exists, so every caller in a
+ * window shares a single execution and a single result object. The slot is
+ * released as soon as the work settles — success or failure — so the next call
+ * starts fresh work rather than replaying a stale outcome.
+ *
+ * Deliberately NOT keyed by argument: the single consumer here has one shared
+ * result for the whole process. A joining caller therefore receives the FIRST
+ * caller's work, arguments included. That is the intended trade for the fan-out,
+ * and it is safe only because every production call site passes the same inputs.
+ *
+ * @returns {(fn: () => Promise<any>) => Promise<any>}
+ */
+function createSingleFlight() {
+  let inFlight = null;
+
+  return function run(fn) {
+    if (inFlight) return inFlight;
+
+    // `Promise.resolve().then(fn)` rather than `fn()` so a synchronous throw
+    // inside fn becomes a rejection on this path too, instead of escaping past
+    // the slot cleanup and wedging `inFlight` forever.
+    //
+    // Release carries no `inFlight === pending` identity guard, deliberately. A
+    // later run can only claim the slot after this one released it — while
+    // `pending` is unsettled every arrival joins instead of replacing it — so a
+    // stale callback clearing a successor's slot is unreachable, and a mutation
+    // test found the guard dead. Adding a way to clear the slot from outside
+    // would make it reachable again; there is none, and #141 requires that a
+    // cache reset specifically must not do it.
+    const pending = Promise.resolve()
+      .then(fn)
+      .finally(() => {
+        inFlight = null;
+      });
+
+    // Module state now holds a promise that real callers may all walk away from.
+    // Without a handler of its own, a rejection reaching only this reference is
+    // an unhandledRejection raised from state nobody is watching — a failure mode
+    // that did not exist while every caller owned its own promise. Callers still
+    // receive the rejection; this only marks the stored reference as handled.
+    pending.catch(() => {});
+
+    inFlight = pending;
+    return pending;
+  };
+}
+
+module.exports = { createSingleFlight };

package/src/lib/skills-manager.js

@@ -287,13 +287,13 @@ function hashDirectory(dir) {
           continue;
         }
         const execBit = process.platform === "win32" ? 0 : stat.mode & 0o111 ? 1 : 0;
-        hash.update(`${rel}${execBit}`);
+        hash.update(`${rel}\0${execBit}\0`);
         try {
           hash.update(fs.readFileSync(abs));
         } catch (_e) {
           // unreadable file — fold its absence in deterministically
         }
-        hash.update("");
+        hash.update("\0");
       }
     }
   };