@mauricode/token-derby 2.12.2 → 2.12.3

package/dist/bin.js

@@ -782,13 +782,14 @@ function apiBase() {
   return process.env.TOKEN_DERBY_API_BASE ?? ENVIRONMENTS[selectedEnv()].apiBase;
 }
 var HEARTBEAT_INTERVAL_MS = 6e4;
+var SCAN_TIMEOUT_MS = HEARTBEAT_INTERVAL_MS * 0.75;
 var HEARTBEAT_RETRY_DELAYS_MS = [1e3, 2e3, 4e3, 8e3, 15e3];
 
 // src/version.ts
 import { createRequire } from "module";
 function readVersion() {
-  if ("2.12.2".length > 0) {
-    return "2.12.2";
+  if ("2.12.3".length > 0) {
+    return "2.12.3";
   }
   try {
     const req = createRequire(import.meta.url);
@@ -885,8 +886,8 @@ function getIdentity() {
 function _resetIdentityCacheForTests() {
   identityCache = null;
 }
-async function request(method, path8, body, horseAuthToken, fetchImpl = fetch) {
-  const url = path8.startsWith("http") ? path8 : `${apiBase()}${path8}`;
+async function request(method, path9, body, horseAuthToken, fetchImpl = fetch) {
+  const url = path9.startsWith("http") ? path9 : `${apiBase()}${path9}`;
   const headers = {};
   headers[CLI_VERSION_HEADER] = CLI_VERSION;
   headers["user-agent"] = `token-derby/${CLI_VERSION}`;
@@ -1580,27 +1581,168 @@ function runHeartbeatLoop(opts) {
 }
 
 // src/tokens/transcripts.ts
+import * as fs4 from "fs/promises";
+import * as path6 from "path";
+
+// src/tokens/pool.ts
+var SCAN_CONCURRENCY = 12;
+async function mapWithConcurrency(items, limit, fn) {
+  const out = new Array(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (true) {
+      const i = next++;
+      if (i >= items.length) return;
+      out[i] = await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+  return out;
+}
+
+// src/tokens/scan-cache.ts
 import * as fs3 from "fs/promises";
 import * as path5 from "path";
+var CACHE_VERSION = 1;
+function isEntry(v) {
+  const e = v;
+  return !!e && typeof e.mtimeMs === "number" && typeof e.size === "number" && typeof e.offset === "number";
+}
+var ScanCache = class _ScanCache {
+  constructor(source, entries) {
+    this.source = source;
+    this.entries = entries;
+  }
+  source;
+  entries;
+  touched = /* @__PURE__ */ new Set();
+  static async open(source) {
+    return new _ScanCache(source, await loadEntries(source));
+  }
+  /**
+   * Total bytes this source was known to hold at its last completed scan, for
+   * diagnosing which source is blowing the beat budget. 0 = never scanned.
+   */
+  static async knownBytes(source) {
+    let total = 0;
+    for (const entry of (await loadEntries(source)).values()) total += entry.size;
+    return total;
+  }
+  has(file) {
+    return this.entries.has(file);
+  }
+  /**
+   * Fold an append-only file, consuming only the bytes added since last time.
+   *
+   * A trailing line with no newline yet is folded into the RETURNED value but
+   * not committed to the cache — matching a plain whole-file read, while still
+   * re-reading that line once the writer completes it.
+   */
+  async readIncremental(file, fold) {
+    const st = await fs3.stat(file);
+    const prev = this.entries.get(file);
+    this.touched.add(file);
+    if (prev && prev.mtimeMs === st.mtimeMs && prev.size === st.size) return prev.value;
+    const grew = prev !== void 0 && st.size > prev.size;
+    const start = grew ? prev.offset : 0;
+    const acc = grew ? prev.value : fold.empty();
+    const { lines, tail, consumedTo } = await readCompleteLines(file, start, st.size);
+    const committed = lines.length > 0 ? fold.append(acc, lines) : acc;
+    this.entries.set(file, { mtimeMs: st.mtimeMs, size: st.size, offset: consumedTo, value: committed });
+    return tail === null ? committed : fold.append(committed, [tail]);
+  }
+  /**
+   * For files that are rewritten whole rather than appended to (Gemini's .json
+   * chats). Gated on mtime+size, recomputed in full whenever either moves.
+   */
+  async readWhenChanged(file, compute) {
+    const st = await fs3.stat(file);
+    const prev = this.entries.get(file);
+    this.touched.add(file);
+    if (prev && prev.mtimeMs === st.mtimeMs && prev.size === st.size) return prev.value;
+    const value = await compute(await fs3.readFile(file, "utf8"));
+    this.entries.set(file, { mtimeMs: st.mtimeMs, size: st.size, offset: st.size, value });
+    return value;
+  }
+  /** Persist, dropping any entry not read since `open` so the file can't grow forever. */
+  async save() {
+    for (const key of [...this.entries.keys()]) {
+      if (!this.touched.has(key)) this.entries.delete(key);
+    }
+    const target = cacheFile(this.source);
+    const tmp = `${target}.tmp`;
+    try {
+      await fs3.mkdir(path5.dirname(target), { recursive: true });
+      await fs3.writeFile(tmp, JSON.stringify({ version: CACHE_VERSION, files: Object.fromEntries(this.entries) }));
+      await fs3.rename(tmp, target);
+    } catch {
+    }
+  }
+};
+function cacheFile(source) {
+  return path5.join(homeDir(), "scan-cache", `${source}.json`);
+}
+async function loadEntries(source) {
+  let parsed;
+  try {
+    parsed = JSON.parse(await fs3.readFile(cacheFile(source), "utf8"));
+  } catch {
+    return /* @__PURE__ */ new Map();
+  }
+  if (parsed?.version !== CACHE_VERSION || typeof parsed.files !== "object" || parsed.files === null) {
+    return /* @__PURE__ */ new Map();
+  }
+  const out = /* @__PURE__ */ new Map();
+  for (const [file, entry] of Object.entries(parsed.files)) {
+    if (isEntry(entry)) out.set(file, entry);
+  }
+  return out;
+}
+async function readCompleteLines(file, start, end) {
+  if (end <= start) return { lines: [], tail: null, consumedTo: start };
+  const fh = await fs3.open(file, "r");
+  try {
+    const buf = Buffer.allocUnsafe(end - start);
+    const { bytesRead } = await fh.read(buf, 0, end - start, start);
+    const chunk = buf.subarray(0, bytesRead);
+    const lastNl = chunk.lastIndexOf(10);
+    if (lastNl === -1) {
+      return { lines: [], tail: chunk.toString("utf8") || null, consumedTo: start };
+    }
+    const tail = chunk.subarray(lastNl + 1).toString("utf8");
+    return {
+      lines: chunk.subarray(0, lastNl).toString("utf8").split("\n"),
+      tail: tail === "" ? null : tail,
+      consumedTo: start + lastNl + 1
+    };
+  } finally {
+    await fh.close();
+  }
+}
+
+// src/tokens/transcripts.ts
 var MAX_PROJECT_DEPTH = 8;
 function conversationId(file, root) {
-  const rel = path5.relative(root, file);
-  const [project, session] = rel.split(path5.sep);
+  const rel = path6.relative(root, file);
+  const [project, session] = rel.split(path6.sep);
   if (project === void 0 || session === void 0) return rel.replace(/\.jsonl$/, "");
   return `${project}/${session.replace(/\.jsonl$/, "")}`;
 }
 async function sumTokensByConversation() {
   const root = claudeProjectsDir();
   const files = await listJsonlFiles(root);
+  const cache = await ScanCache.open("claude");
+  const totals = await mapWithConcurrency(files, SCAN_CONCURRENCY, (f) => cache.readIncremental(f, CLAUDE_FOLD));
+  await cache.save();
   const byConv = /* @__PURE__ */ new Map();
-  for (const file of files) {
-    const t = await sumFile(file);
+  files.forEach((file, i) => {
+    const t = totals[i];
     const id = conversationId(file, root);
     const acc = byConv.get(id) ?? { input: 0, output: 0 };
     acc.input += t.input;
     acc.output += t.output;
     byConv.set(id, acc);
-  }
+  });
   return byConv;
 }
 async function sumTokens() {
@@ -1614,25 +1756,25 @@ async function sumTokens() {
   return { input, output };
 }
 async function listJsonlFiles(root) {
-  const projects = await fs3.readdir(root);
+  const projects = await fs4.readdir(root);
   const out = [];
   for (const project of projects) {
-    const projectDir = path5.join(root, project);
-    const stat3 = await fs3.stat(projectDir);
-    if (!stat3.isDirectory()) continue;
+    const projectDir = path6.join(root, project);
+    const stat4 = await fs4.stat(projectDir);
+    if (!stat4.isDirectory()) continue;
     await collectJsonl(projectDir, MAX_PROJECT_DEPTH, out);
   }
   return out;
 }
 async function collectJsonl(dir, depth, out) {
   if (depth <= 0) return;
-  const entries = await fs3.readdir(dir);
+  const entries = await fs4.readdir(dir);
   for (const entry of entries) {
     if (entry.endsWith(".jsonl")) {
-      out.push(path5.join(dir, entry));
+      out.push(path6.join(dir, entry));
     } else if (depth > 1) {
-      const child = path5.join(dir, entry);
-      const st = await fs3.stat(child);
+      const child = path6.join(dir, entry);
+      const st = await fs4.stat(child);
       if (st.isDirectory()) await collectJsonl(child, depth - 1, out);
     }
   }
@@ -1640,43 +1782,49 @@ async function collectJsonl(dir, depth, out) {
 function addNum(value) {
   return typeof value === "number" && Number.isFinite(value) ? value : 0;
 }
-async function sumFile(file) {
-  const raw = await fs3.readFile(file, "utf8");
-  let input = 0;
-  let output = 0;
-  for (const line of raw.split("\n")) {
-    if (!line.trim()) continue;
-    let parsed;
-    try {
-      parsed = JSON.parse(line);
-    } catch {
-      continue;
+var CLAUDE_FOLD = {
+  empty: () => ({ input: 0, output: 0 }),
+  append: (acc, lines) => {
+    let { input, output } = acc;
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      let parsed;
+      try {
+        parsed = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      const usage = parsed?.message?.usage;
+      if (!usage) continue;
+      input += addNum(usage.input_tokens) + addNum(usage.cache_creation_input_tokens);
+      output += addNum(usage.output_tokens);
     }
-    const usage = parsed?.message?.usage;
-    if (!usage) continue;
-    input += addNum(usage.input_tokens) + addNum(usage.cache_creation_input_tokens);
-    output += addNum(usage.output_tokens);
+    return { input, output };
   }
-  return { input, output };
-}
+};
 
 // src/tokens/codex.ts
-import * as fs4 from "fs/promises";
-import * as path6 from "path";
+import * as fs5 from "fs/promises";
+import * as path7 from "path";
 function num(v) {
   return typeof v === "number" && Number.isFinite(v) ? v : 0;
 }
 async function sumCodexByConversation() {
   const root = codexSessionsDir();
-  await fs4.stat(root);
+  await fs5.stat(root);
   const files = [
-    ...await collectRollouts(path6.join(root, "sessions")),
-    ...await collectRollouts(path6.join(root, "archived_sessions"))
+    ...await collectRollouts(path7.join(root, "sessions")),
+    ...await collectRollouts(path7.join(root, "archived_sessions"))
   ];
+  const cache = await ScanCache.open("codex");
+  const totals = await mapWithConcurrency(
+    files,
+    SCAN_CONCURRENCY,
+    (f) => cache.readIncremental(f, CODEX_FOLD).catch(() => ({ input: 0, output: 0 }))
+  );
+  await cache.save();
   const byConv = /* @__PURE__ */ new Map();
-  for (const file of files) {
-    byConv.set(file, await lastTokenCount(file));
-  }
+  files.forEach((file, i) => byConv.set(file, totals[i]));
   return byConv;
 }
 async function sumCodexTokens() {
@@ -1692,58 +1840,60 @@ async function sumCodexTokens() {
 async function collectRollouts(dir) {
   let entries;
   try {
-    entries = await fs4.readdir(dir, { withFileTypes: true });
+    entries = await fs5.readdir(dir, { withFileTypes: true });
   } catch (e) {
     if (e?.code === "ENOENT") return [];
     throw e;
   }
   const out = [];
   for (const entry of entries) {
-    const full = path6.join(dir, entry.name);
+    const full = path7.join(dir, entry.name);
     if (entry.isDirectory()) out.push(...await collectRollouts(full));
     else if (entry.name.startsWith("rollout-") && entry.name.endsWith(".jsonl")) out.push(full);
   }
   return out;
 }
-async function lastTokenCount(file) {
-  let raw;
-  try {
-    raw = await fs4.readFile(file, "utf8");
-  } catch {
-    return { input: 0, output: 0 };
-  }
-  let usage = null;
-  for (const line of raw.split("\n")) {
-    if (!line.trim()) continue;
-    let parsed;
-    try {
-      parsed = JSON.parse(line);
-    } catch {
-      continue;
-    }
-    if (parsed?.payload?.type === "token_count" && parsed.payload.info?.total_token_usage) {
-      usage = parsed.payload.info.total_token_usage;
+var CODEX_FOLD = {
+  empty: () => ({ input: 0, output: 0 }),
+  append: (acc, lines) => {
+    let usage = null;
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      let parsed;
+      try {
+        parsed = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (parsed?.payload?.type === "token_count" && parsed.payload.info?.total_token_usage) {
+        usage = parsed.payload.info.total_token_usage;
+      }
     }
+    if (!usage) return acc;
+    return {
+      input: Math.max(0, num(usage.input_tokens) - num(usage.cached_input_tokens)),
+      output: num(usage.output_tokens)
+    };
   }
-  if (!usage) return { input: 0, output: 0 };
-  return {
-    input: Math.max(0, num(usage.input_tokens) - num(usage.cached_input_tokens)),
-    output: num(usage.output_tokens)
-  };
-}
+};
 
 // src/tokens/gemini.ts
-import * as fs5 from "fs/promises";
-import * as path7 from "path";
+import * as fs6 from "fs/promises";
+import * as path8 from "path";
 function num2(v) {
   return typeof v === "number" && Number.isFinite(v) ? v : 0;
 }
 async function sumGeminiByConversation() {
   const files = await listChatFiles(geminiTmpDir());
+  const cache = await ScanCache.open("gemini");
+  const totals = await mapWithConcurrency(
+    files,
+    SCAN_CONCURRENCY,
+    (f) => cache.readWhenChanged(f, async (raw) => sumGeminiRaw(f, raw)).catch(() => ({ input: 0, output: 0 }))
+  );
+  await cache.save();
   const byConv = /* @__PURE__ */ new Map();
-  for (const file of files) {
-    byConv.set(file, await sumGeminiFile(file));
-  }
+  files.forEach((file, i) => byConv.set(file, totals[i]));
   return byConv;
 }
 async function sumGeminiTokens() {
@@ -1757,29 +1907,23 @@ async function sumGeminiTokens() {
   return { input, output };
 }
 async function listChatFiles(root) {
-  const entries = await fs5.readdir(root);
+  const entries = await fs6.readdir(root);
   const out = [];
   for (const entry of entries) {
-    const chatsDir = path7.join(root, entry, "chats");
+    const chatsDir = path8.join(root, entry, "chats");
     let files;
     try {
-      files = await fs5.readdir(chatsDir);
+      files = await fs6.readdir(chatsDir);
     } catch {
       continue;
     }
     for (const f of files) {
-      if (f.endsWith(".json") || f.endsWith(".jsonl")) out.push(path7.join(chatsDir, f));
+      if (f.endsWith(".json") || f.endsWith(".jsonl")) out.push(path8.join(chatsDir, f));
     }
   }
   return out;
 }
-async function sumGeminiFile(file) {
-  let raw;
-  try {
-    raw = await fs5.readFile(file, "utf8");
-  } catch {
-    return { input: 0, output: 0 };
-  }
+function sumGeminiRaw(file, raw) {
   const messages = file.endsWith(".jsonl") ? parseJsonl(raw) : parseJson(raw);
   let input = 0;
   let output = 0;
@@ -1815,6 +1959,21 @@ function parseJsonl(raw) {
 function isStall(r) {
   return "stall" in r;
 }
+var TIMED_OUT = /* @__PURE__ */ Symbol("scan-timeout");
+async function scanWithTimeout(scan, timeoutMs, describeTimeout) {
+  let timer;
+  const budget = new Promise((resolve) => {
+    timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
+  });
+  try {
+    const result = await Promise.race([scan(), budget]);
+    if (result !== TIMED_OUT) return result;
+    const detail = describeTimeout ? await describeTimeout() : null;
+    return { stall: detail ?? `Token scan timed out after ${Math.round(timeoutMs / 1e3)}s` };
+  } finally {
+    clearTimeout(timer);
+  }
+}
 var SCALAR_READERS = {
   claude: sumTokens,
   codex: sumCodexTokens,
@@ -1828,25 +1987,71 @@ var BY_CONVERSATION_READERS = {
 function scoreFor(race, t) {
   return race.counts_input ? t.input + t.output : t.output;
 }
-async function readAllSources(race, primary) {
+async function readAllSources(race, primary, progress) {
+  progress?.begin(primary);
+  const primaryScan = BY_CONVERSATION_READERS[primary]().then(
+    (map) => ({ ok: true, map }),
+    (err) => ({ ok: false, err })
+  ).finally(() => progress?.end(primary));
+  const secondaryKeys = MODEL_KEYS.filter((k) => k !== primary);
+  const secondaryScans = secondaryKeys.map((k) => {
+    progress?.begin(k);
+    return SCALAR_READERS[k]().then((t) => scoreFor(race, t)).catch(() => 0).finally(() => progress?.end(k));
+  });
+  const [primaryResult, secondaryValues] = await Promise.all([
+    primaryScan,
+    Promise.all(secondaryScans)
+  ]);
   const primaryByConv = /* @__PURE__ */ new Map();
-  try {
-    const primaryMap = await BY_CONVERSATION_READERS[primary]();
-    for (const [id, totals] of primaryMap) primaryByConv.set(id, scoreFor(race, totals));
-  } catch (e) {
-    if (e?.code !== "ENOENT") {
-      return { stall: `Can't read ${primary} token usage: ${e?.message ?? String(e)}` };
-    }
+  if (primaryResult.ok) {
+    for (const [id, totals] of primaryResult.map) primaryByConv.set(id, scoreFor(race, totals));
+  } else if (primaryResult.err?.code !== "ENOENT") {
+    const err = primaryResult.err;
+    return { stall: `Can't read ${primary} token usage: ${err?.message ?? String(err)}` };
   }
   const secondary = { claude: 0, codex: 0, gemini: 0 };
-  await Promise.all(
-    MODEL_KEYS.filter((k) => k !== primary).map(async (k) => {
-      secondary[k] = await SCALAR_READERS[k]().then((t) => scoreFor(race, t)).catch(() => 0);
-    })
-  );
+  secondaryKeys.forEach((k, i) => {
+    secondary[k] = secondaryValues[i] ?? 0;
+  });
   return { secondary, primaryByConv };
 }
 
+// src/tokens/scan-progress.ts
+function skipVar(key) {
+  return `TOKEN_DERBY_${key.toUpperCase()}_DIR`;
+}
+function formatBytes(bytes) {
+  if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
+  return `${Math.round(bytes / 1e6)} MB`;
+}
+var ScanProgress = class {
+  started = /* @__PURE__ */ new Map();
+  finished = /* @__PURE__ */ new Set();
+  begin(key) {
+    this.started.set(key, Date.now());
+  }
+  end(key) {
+    this.finished.add(key);
+  }
+  /** Sources begun but never finished, longest-running first. */
+  outstanding() {
+    return [...this.started.entries()].filter(([key]) => !this.finished.has(key)).sort((a, b) => a[1] - b[1]).map(([key]) => key);
+  }
+};
+function describeScanTimeout(timeoutMs, outstanding) {
+  const budget = `Token scan timed out after ${Math.round(timeoutMs / 1e3)}s`;
+  if (outstanding.length === 0) return budget;
+  const named = outstanding.map((s) => s.bytes > 0 ? `${s.key} (${formatBytes(s.bytes)})` : s.key).join(", ");
+  const biggest = [...outstanding].sort((a, b) => b.bytes - a.bytes)[0];
+  return `${budget} \u2014 ${named} still scanning; point ${skipVar(biggest.key)} at an empty dir to skip it`;
+}
+async function diagnoseScanTimeout(timeoutMs, progress) {
+  const outstanding = await Promise.all(
+    progress.outstanding().map(async (key) => ({ key, bytes: await ScanCache.knownBytes(key) }))
+  );
+  return describeScanTimeout(timeoutMs, outstanding);
+}
+
 // src/tokens/primary-cap.ts
 var PRIMARY_TOP_CONVERSATIONS = 5;
 function primaryConversationCap(enabled) {
@@ -2006,19 +2211,21 @@ function RunRace({ active, initialState, pendingMode, ownUserName }) {
   }, [race?.status]);
   useEffect2(() => {
     const tracker = trackerRef.current;
-    const scanWithTimeout = async () => {
+    const scanBeat = async () => {
+      const progress = new ScanProgress();
       try {
-        return await Promise.race([
-          readAllSources(active, active.primary_model),
-          new Promise((_, reject) => setTimeout(() => reject(new Error("scan timeout")), 1e4))
-        ]);
-      } catch {
-        return { stall: "Token scan timed out" };
+        return await scanWithTimeout(
+          () => readAllSources(active, active.primary_model, progress),
+          SCAN_TIMEOUT_MS,
+          () => diagnoseScanTimeout(SCAN_TIMEOUT_MS, progress)
+        );
+      } catch (e) {
+        return { stall: `Token scan failed: ${e?.message ?? String(e)}` };
       }
     };
     runHeartbeatLoop({
       prepareBeat: async () => {
-        const reading = await scanWithTimeout();
+        const reading = await scanBeat();
         tracker.recordReading(reading);
         if (pendingRef.current && !isStall(reading)) tracker.reprime();
         setStalled(tracker.stalled);