@pi-unipi/unipi 2.2.7 → 2.3.0

package/packages/info-screen/usage-parser.ts

@@ -5,8 +5,8 @@
  * Reference: tmustier/pi-extensions/usage-extension
  */
 
-import { readdirSync, readFileSync, statSync, existsSync } from "node:fs";
-import { join, basename } from "node:path";
+import { readdirSync, readFileSync, statSync, existsSync, mkdirSync, writeFileSync, renameSync, openSync, readSync, closeSync } from "node:fs";
+import { join, basename, dirname } from "node:path";
 import { homedir } from "node:os";
 
 /** Usage data for a single message */
@@ -54,6 +54,75 @@ interface PeriodBounds {
   end: Date;
 }
 
+/**
+ * A single usage record, stored in the cache in compact tuple form.
+ *
+ * [timestamp, hashTokens, countedTokens, cost, modelIndex, counted]
+ *
+ * `hashTokens` is input+output+cacheRead+cacheWrite and exists ONLY to rebuild
+ * the dedup key. `countedTokens` is input+output+cacheWrite, which is what the
+ * totals actually sum (cacheRead is deliberately excluded).
+ *
+ * `counted` (1/0) mirrors the original `input > 0 || output > 0 || cost > 0`
+ * check. It must be stored separately because the original claims the dedup
+ * hash BEFORE applying that filter — so a zero-usage message still suppresses
+ * a later duplicate. Collapsing the two would change the totals.
+ */
+type UsageRecord = [number, number, number, number, number, number];
+
+/** Per-file cache entry, invalidated on mtime or size change. */
+interface CachedFile {
+  mtimeMs: number;
+  size: number;
+  records: UsageRecord[];
+}
+
+interface UsageCacheFile {
+  version: number;
+  /** Interned model names; records store an index into this array. */
+  models: string[];
+  files: Record<string, CachedFile>;
+}
+
+/**
+ * Bump when the record layout or parsing semantics change, so stale caches
+ * from an older build are discarded rather than silently reused.
+ */
+const CACHE_VERSION = 1;
+
+function getCachePath(): string {
+  const base = process.env.UNIPI_DIR || join(homedir(), ".unipi");
+  return join(base, "cache", "usage-stats.json");
+}
+
+function readCache(): UsageCacheFile {
+  const empty: UsageCacheFile = { version: CACHE_VERSION, models: [], files: {} };
+  try {
+    const path = getCachePath();
+    if (!existsSync(path)) return empty;
+    const parsed = JSON.parse(readFileSync(path, "utf-8")) as UsageCacheFile;
+    if (!parsed || parsed.version !== CACHE_VERSION) return empty;
+    if (!Array.isArray(parsed.models) || typeof parsed.files !== "object") return empty;
+    return parsed;
+  } catch {
+    // Corrupt or unreadable cache: rebuild from scratch.
+    return empty;
+  }
+}
+
+function writeCache(cache: UsageCacheFile): void {
+  try {
+    const path = getCachePath();
+    mkdirSync(dirname(path), { recursive: true });
+    // Write-then-rename so a crash mid-write cannot leave a torn cache behind.
+    const tmp = `${path}.${process.pid}.tmp`;
+    writeFileSync(tmp, JSON.stringify(cache), "utf-8");
+    renameSync(tmp, path);
+  } catch {
+    // A cache we cannot persist is a performance loss, not a correctness one.
+  }
+}
+
 /**
  * Get the sessions directory path.
  */
@@ -95,71 +164,95 @@ function getPeriodBounds(): { today: PeriodBounds; week: PeriodBounds; month: Pe
 
 
 /**
- * Parse a JSONL session file and extract usage data.
- * Matches tmustier's parsing logic.
+ * Read a file line-by-line without materializing it in memory.
+ *
+ * Session files reach 200MB+, so readFileSync would allocate the whole file
+ * (and its split() array) just to scan it once. Reads in 1MB chunks and keeps
+ * only the trailing partial line between chunks.
  */
-function parseSessionFile(
-  filePath: string,
-  seenHashes: Set<string>
-): Array<{ usage: MessageUsage; model: string; timestamp: number }> {
-  const results: Array<{ usage: MessageUsage; model: string; timestamp: number }> = [];
-
+function forEachLine(filePath: string, onLine: (line: string) => void): void {
+  const CHUNK = 1024 * 1024;
+  let fd: number | undefined;
   try {
-    const content = readFileSync(filePath, "utf-8");
-    const lines = content.trim().split("\n");
-
-    for (let i = 0; i < lines.length; i++) {
-      const line = lines[i];
-      if (!line.trim()) continue;
-
-      try {
-        const entry = JSON.parse(line);
-
-        // Match tmustier's parsing: check entry.type === "message" and entry.message?.role === "assistant"
-        if (entry.type === "message" && entry.message?.role === "assistant") {
-          const msg = entry.message;
-          if (msg.usage && msg.provider && msg.model) {
-            const input = msg.usage.input || 0;
-            const output = msg.usage.output || 0;
-            const cacheRead = msg.usage.cacheRead || 0;
-            const cacheWrite = msg.usage.cacheWrite || 0;
-            const cost = msg.usage.cost?.total || 0;
-
-            // Get timestamp
-            const fallbackTs = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
-            const timestamp = msg.timestamp || (Number.isNaN(fallbackTs) ? 0 : fallbackTs);
-
-            // Deduplicate copied history across branched session files
-            const totalTokens = input + output + cacheRead + cacheWrite;
-            const hash = `${timestamp}:${totalTokens}`;
-            if (seenHashes.has(hash)) continue;
-            seenHashes.add(hash);
-
-            // Only include if we have valid data
-            if (input > 0 || output > 0 || cost > 0) {
-              results.push({
-                usage: {
-                  input,
-                  output,
-                  cacheRead,
-                  cacheWrite,
-                  cost: { total: cost },
-                },
-                model: msg.model,
-                timestamp,
-              });
-            }
-          }
-        }
-      } catch {
-        // Skip malformed lines
+    fd = openSync(filePath, "r");
+    const buf = Buffer.allocUnsafe(CHUNK);
+    let carry = "";
+    for (;;) {
+      const bytes = readSync(fd, buf, 0, CHUNK, null);
+      if (bytes <= 0) break;
+      // latin1 would corrupt multi-byte UTF-8 split across a chunk boundary;
+      // toString("utf8") on a Buffer slice handles the common case, and any
+      // partial sequence lands in `carry` and is completed by the next chunk.
+      const text = carry + buf.toString("utf8", 0, bytes);
+      let start = 0;
+      for (;;) {
+        const nl = text.indexOf("\n", start);
+        if (nl === -1) break;
+        onLine(text.slice(start, nl));
+        start = nl + 1;
       }
+      carry = text.slice(start);
     }
+    if (carry.length > 0) onLine(carry);
   } catch {
     // Skip unreadable files
+  } finally {
+    if (fd !== undefined) {
+      try { closeSync(fd); } catch { /* already closed */ }
+    }
   }
+}
+
+/**
+ * Extract the compact usage records from one session file.
+ *
+ * Deliberately does NOT deduplicate: dedup is cross-file and order-dependent,
+ * so it must happen at aggregation time. Caching raw per-file records keeps
+ * each entry independent and lets any single file be re-parsed in isolation.
+ */
+function extractRecords(filePath: string, modelIndex: Map<string, number>, models: string[]): UsageRecord[] {
+  const records: UsageRecord[] = [];
+
+  forEachLine(filePath, (line) => {
+    if (!line || !line.trim()) return;
+    let entry: any;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      return; // Skip malformed lines
+    }
+
+    if (entry.type !== "message" || entry.message?.role !== "assistant") return;
+    const msg = entry.message;
+    if (!msg.usage || !msg.provider || !msg.model) return;
 
-  return results;
+    const input = msg.usage.input || 0;
+    const output = msg.usage.output || 0;
+    const cacheRead = msg.usage.cacheRead || 0;
+    const cacheWrite = msg.usage.cacheWrite || 0;
+    const cost = msg.usage.cost?.total || 0;
+
+    const fallbackTs = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
+    const timestamp = msg.timestamp || (Number.isNaN(fallbackTs) ? 0 : fallbackTs);
+
+    let idx = modelIndex.get(msg.model);
+    if (idx === undefined) {
+      idx = models.length;
+      models.push(msg.model);
+      modelIndex.set(msg.model, idx);
+    }
+
+    records.push([
+      timestamp,
+      input + output + cacheRead + cacheWrite, // dedup key component
+      input + output + cacheWrite,             // counted tokens (excludes cacheRead)
+      cost,
+      idx,
+      input > 0 || output > 0 || cost > 0 ? 1 : 0,
+    ]);
+  });
+
+  return records;
 }
 
 /**
@@ -186,8 +279,30 @@ function collectSessionFiles(dir: string, files: string[]): void {
  * Matches tmustier's parsing logic.
  */
 export function parseUsageStats(): UsageStats {
-  const sessionsDir = getSessionsDir();
-  const stats: UsageStats = {
+  const { stats } = collectStats(null);
+  return stats;
+}
+
+/**
+ * Async variant that yields to the event loop while parsing.
+ *
+ * A cold parse is several seconds of pure CPU. Running it synchronously starves
+ * the event loop, so keystrokes queue up and the UI cannot repaint. Deferring
+ * the *start* (setTimeout) does not help — the block must be broken up.
+ * `yieldEvery` files, control returns to the loop.
+ */
+export async function parseUsageStatsAsync(): Promise<UsageStats> {
+  const yielder = async (): Promise<void> => {
+    await new Promise<void>((resolve) => setImmediate(resolve));
+  };
+  const { stats, pending } = collectStats(yielder);
+  if (pending) await pending;
+  return stats;
+}
+
+/** Empty stats accumulator. */
+function emptyStats(): UsageStats {
+  return {
     tokens: { today: 0, week: 0, month: 0, allTime: 0 },
     cost: { today: 0, week: 0, month: 0, allTime: 0 },
     byModel: {},
@@ -197,93 +312,168 @@ export function parseUsageStats(): UsageStats {
     sessionCount: 0,
     messageCount: 0,
   };
+}
 
-  if (!existsSync(sessionsDir)) return stats;
-
-  const periods = getPeriodBounds();
-  const seenHashes = new Set<string>();
+/**
+ * Shared implementation for the sync and async entry points.
+ *
+ * When `yielder` is null the whole scan runs synchronously and `stats` is fully
+ * populated on return. When provided, the returned `stats` object is filled in
+ * as `pending` progresses, and the caller must await it.
+ */
+function collectStats(
+  yielder: (() => Promise<void>) | null,
+): { stats: UsageStats; pending: Promise<void> | null } {
+  const stats = emptyStats();
+  const sessionsDir = getSessionsDir();
+  if (!existsSync(sessionsDir)) return { stats, pending: null };
 
-  // Collect all session files recursively
   const sessionFiles: string[] = [];
   collectSessionFiles(sessionsDir, sessionFiles);
   sessionFiles.sort();
 
-  for (const filePath of sessionFiles) {
-    const messages = parseSessionFile(filePath, seenHashes);
+  const cache = readCache();
+  const models = cache.models.slice();
+  const modelIndex = new Map<string, number>();
+  models.forEach((name, i) => modelIndex.set(name, i));
 
-    if (messages.length === 0) continue;
+  const nextFiles: Record<string, CachedFile> = {};
+  let cacheDirty = false;
 
-    stats.sessionCount++;
-    stats.messageCount += messages.length;
+  // Statting 600 files costs ~1ms, so the mtime+size gate is essentially free
+  // compared to re-reading gigabytes of immutable history.
+  const work: Array<{ path: string; cached: CachedFile | null }> = [];
+  for (const filePath of sessionFiles) {
+    let mtimeMs = 0;
+    let size = 0;
+    try {
+      const st = statSync(filePath);
+      mtimeMs = st.mtimeMs;
+      size = st.size;
+    } catch {
+      continue; // Vanished between listing and statting.
+    }
 
-    for (const msg of messages) {
-      // Match tmustier's token calculation: input + output + cacheWrite (not cacheRead)
-      const totalTokens = msg.usage.input + msg.usage.output + msg.usage.cacheWrite;
+    const hit = cache.files[filePath];
+    if (hit && hit.mtimeMs === mtimeMs && hit.size === size) {
+      nextFiles[filePath] = hit;
+      work.push({ path: filePath, cached: hit });
+    } else {
+      cacheDirty = true;
+      work.push({ path: filePath, cached: null });
+      nextFiles[filePath] = { mtimeMs, size, records: [] };
+    }
+  }
 
-      // All time
-      stats.tokens.allTime += totalTokens;
-      stats.cost.allTime += msg.usage.cost.total;
+  // A file disappearing means the old cache had entries we must drop.
+  if (Object.keys(nextFiles).length !== Object.keys(cache.files).length) cacheDirty = true;
 
-      // Today
-      if (msg.timestamp >= periods.today.start.getTime()) {
-        stats.tokens.today += totalTokens;
-        stats.cost.today += msg.usage.cost.total;
-      }
+  const seenHashes = new Set<string>();
+  const periods = getPeriodBounds();
+  const todayStart = periods.today.start.getTime();
+  const weekStart = periods.week.start.getTime();
+  const monthStart = periods.month.start.getTime();
+
+  const bump = (
+    bucket: Record<string, { tokens: number; cost: number; sessions: number }>,
+    model: string,
+    tokens: number,
+    cost: number,
+  ) => {
+    let entry = bucket[model];
+    if (!entry) {
+      entry = { tokens: 0, cost: 0, sessions: 0 };
+      bucket[model] = entry;
+    }
+    entry.tokens += tokens;
+    entry.cost += cost;
+    entry.sessions++;
+  };
 
-      // This week
-      if (msg.timestamp >= periods.week.start.getTime()) {
-        stats.tokens.week += totalTokens;
-        stats.cost.week += msg.usage.cost.total;
+  const aggregate = (records: UsageRecord[]): void => {
+    let counted = 0;
+    for (const rec of records) {
+      const [timestamp, hashTokens, countedTokens, cost, modelIdx, isCounted] = rec;
+
+      // Dedup key is claimed even for records that are not counted, matching
+      // the original ordering (hash added before the validity check).
+      const hash = `${timestamp}:${hashTokens}`;
+      if (seenHashes.has(hash)) continue;
+      seenHashes.add(hash);
+      if (!isCounted) continue;
+
+      counted++;
+      const model = models[modelIdx] ?? "unknown";
+
+      stats.tokens.allTime += countedTokens;
+      stats.cost.allTime += cost;
+      bump(stats.byModel, model, countedTokens, cost);
+
+      if (timestamp >= todayStart) {
+        stats.tokens.today += countedTokens;
+        stats.cost.today += cost;
+        bump(stats.byModelToday, model, countedTokens, cost);
       }
-
-      // This month
-      if (msg.timestamp >= periods.month.start.getTime()) {
-        stats.tokens.month += totalTokens;
-        stats.cost.month += msg.usage.cost.total;
+      if (timestamp >= weekStart) {
+        stats.tokens.week += countedTokens;
+        stats.cost.week += cost;
+        bump(stats.byModelWeek, model, countedTokens, cost);
       }
-
-      // By model (all time)
-      const model = msg.model;
-      if (!stats.byModel[model]) {
-        stats.byModel[model] = { tokens: 0, cost: 0, sessions: 0 };
-      }
-      stats.byModel[model].tokens += totalTokens;
-      stats.byModel[model].cost += msg.usage.cost.total;
-      stats.byModel[model].sessions++;
-
-      // By model (today)
-      if (msg.timestamp >= periods.today.start.getTime()) {
-        if (!stats.byModelToday[model]) {
-          stats.byModelToday[model] = { tokens: 0, cost: 0, sessions: 0 };
-        }
-        stats.byModelToday[model].tokens += totalTokens;
-        stats.byModelToday[model].cost += msg.usage.cost.total;
-        stats.byModelToday[model].sessions++;
+      if (timestamp >= monthStart) {
+        stats.tokens.month += countedTokens;
+        stats.cost.month += cost;
+        bump(stats.byModelMonth, model, countedTokens, cost);
       }
+    }
 
-      // By model (this week)
-      if (msg.timestamp >= periods.week.start.getTime()) {
-        if (!stats.byModelWeek[model]) {
-          stats.byModelWeek[model] = { tokens: 0, cost: 0, sessions: 0 };
-        }
-        stats.byModelWeek[model].tokens += totalTokens;
-        stats.byModelWeek[model].cost += msg.usage.cost.total;
-        stats.byModelWeek[model].sessions++;
-      }
+    if (counted > 0) {
+      stats.sessionCount++;
+      stats.messageCount += counted;
+    }
+  };
 
-      // By model (this month)
-      if (msg.timestamp >= periods.month.start.getTime()) {
-        if (!stats.byModelMonth[model]) {
-          stats.byModelMonth[model] = { tokens: 0, cost: 0, sessions: 0 };
-        }
-        stats.byModelMonth[model].tokens += totalTokens;
-        stats.byModelMonth[model].cost += msg.usage.cost.total;
-        stats.byModelMonth[model].sessions++;
-      }
+  const finish = (): void => {
+    if (cacheDirty) {
+      writeCache({ version: CACHE_VERSION, models, files: nextFiles });
+    }
+  };
+
+  const YIELD_EVERY = 25;
+
+  if (!yielder) {
+    for (const item of work) {
+      const records = item.cached
+        ? item.cached.records
+        : (nextFiles[item.path].records = extractRecords(item.path, modelIndex, models));
+      aggregate(records);
     }
+    finish();
+    return { stats, pending: null };
   }
 
-  return stats;
+  const pending = (async () => {
+    let sinceYield = 0;
+    for (const item of work) {
+      let records: UsageRecord[];
+      if (item.cached) {
+        records = item.cached.records;
+      } else {
+        records = extractRecords(item.path, modelIndex, models);
+        nextFiles[item.path].records = records;
+        // Only re-parsed files are expensive; cache hits are near-free, so
+        // yielding is gated on real work to avoid pointless loop turns.
+        sinceYield++;
+      }
+      aggregate(records);
+      if (sinceYield >= YIELD_EVERY) {
+        sinceYield = 0;
+        await yielder();
+      }
+    }
+    finish();
+  })();
+
+  return { stats, pending };
 }
 
 /**

package/packages/input-shortcuts/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/input-shortcuts",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Keyboard shortcuts for stash/restore, undo/redo, clipboard, and thinking toggle — chord-based overlay system",
   "type": "module",
   "main": "src/index.ts",
@@ -33,7 +33,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@pi-unipi/core": "2.2.0"
+    "@pi-unipi/core": "2.3.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",

package/packages/kanboard/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/kanboard",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Visualization layer for unipi workflow — HTTP server with htmx/Alpine.js UI, modular parsers, TUI overlay, and kanban board",
   "type": "module",
   "main": "index.ts",
@@ -39,7 +39,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@pi-unipi/core": "2.2.0"
+    "@pi-unipi/core": "2.3.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",

package/packages/mcp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/mcp",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "MCP server management extension for Pi coding agent — browse, add, configure, and use MCP servers",
   "type": "module",
   "main": "src/index.ts",
@@ -27,7 +27,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@pi-unipi/core": "2.2.0"
+    "@pi-unipi/core": "2.3.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",