claude-code-session-manager 0.35.18 → 0.37.0

package/src/main/historyAggregator.cjs

@@ -5,6 +5,7 @@ const fsp = require('node:fs/promises');
 const path = require('node:path');
 const os = require('node:os');
 const { schemas } = require('./ipcSchemas.cjs');
+const historyRollup = require('./lib/historyRollup.cjs');
 
 const PROJECTS_DIR = path.join(os.homedir(), '.claude', 'projects');
 const PARSE_BUDGET_MS = 2_000;
@@ -115,19 +116,23 @@ async function readSlice(filePath, from, to) {
 
 /**
  * Scan JSONL lines into an aggregate accumulator (mutates acc).
- * Returns the first timestamp seen when captureFirst=true, else null.
+ * Returns { firstTs, lastTs } — firstTs is only captured when
+ * captureFirst=true (else null); lastTs is the most recent event timestamp
+ * seen in this batch of lines (file order, so simply "last one wins").
  */
 function scanAggrLines(lines, acc, captureFirst) {
   let firstTs = null;
+  let lastTs = null;
   for (const raw of lines) {
     const line = raw.trim();
     if (!line) continue;
     let obj;
     try { obj = JSON.parse(line); } catch { continue; }
 
-    if (captureFirst && firstTs === null) {
-      const ts = obj.ts ?? obj.timestamp;
-      if (ts) firstTs = ts;
+    const ts = obj.ts ?? obj.timestamp;
+    if (ts) {
+      if (captureFirst && firstTs === null) firstTs = ts;
+      lastTs = ts;
     }
 
     const role = obj.role ?? obj.message?.role;
@@ -178,7 +183,21 @@ function scanAggrLines(lines, acc, captureFirst) {
       acc.errorCount++;
     }
   }
-  return firstTs;
+  return { firstTs, lastTs };
+}
+
+/**
+ * Per-file activeMinutes = elapsed wall-clock time between the file's first
+ * and last event, clamped to [0, 24h] so a corrupt/out-of-order timestamp
+ * pair can't blow up a day's total. O(1).
+ */
+function computeActiveMinutes(firstTs, lastTs) {
+  if (!firstTs || !lastTs) return 0;
+  const a = Date.parse(firstTs);
+  const b = Date.parse(lastTs);
+  if (!Number.isFinite(a) || !Number.isFinite(b)) return 0;
+  const minutes = (b - a) / 60000;
+  return Math.max(0, Math.min(24 * 60, minutes));
 }
 
 // ── cached file parsers ───────────────────────────────────────────────────────
@@ -204,6 +223,8 @@ async function parseJSONL(filePath, stat) {
     byModel: {},
     errorCount: 0,
     sessionDate: null,
+    firstEventTs: null,
+    lastEventTs: null,
     skipped: false,
   });
 
@@ -213,7 +234,11 @@ async function parseJSONL(filePath, stat) {
 
   const cached = aggrCache.get(filePath);
   if (cached) {
-    if (cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
+    // cached.result.lastEventTs !== undefined guards against pre-activeMinutes
+    // cache entries (shape: no firstEventTs/lastEventTs fields) — those must
+    // fall through to a full reparse so activeMinutes gets backfilled, rather
+    // than being served as a stale exact hit.
+    if (cached.size === stat.size && cached.mtimeMs === stat.mtimeMs && cached.result.lastEventTs !== undefined) {
       return { result: cached.result, cacheHit: true };
     }
 
@@ -230,7 +255,7 @@ async function parseJSONL(filePath, stat) {
           const readFrom = cached.readOffset ?? cached.size;
           const tail = await readSlice(filePath, readFrom, stat.size);
           const delta = emptyAcc();
-          scanAggrLines(tail.split('\n'), delta, false);
+          const { lastTs: deltaLastTs } = scanAggrLines(tail.split('\n'), delta, false);
           const prev = cached.result;
           const merged = {
             promptCount: prev.promptCount + delta.promptCount,
@@ -243,6 +268,8 @@ async function parseJSONL(filePath, stat) {
             byModel: {},
             errorCount: prev.errorCount + delta.errorCount,
             sessionDate: prev.sessionDate, // firstTs doesn't change on appends
+            firstEventTs: prev.firstEventTs ?? null,
+            lastEventTs: deltaLastTs ?? (prev.lastEventTs ?? null),
             skipped: false,
           };
           for (const [k, v] of Object.entries(delta.toolBreakdown)) {
@@ -282,7 +309,9 @@ async function parseJSONL(filePath, stat) {
   }
 
   const acc = emptyAcc();
-  const firstTs = scanAggrLines(text.split('\n'), acc, true);
+  const { firstTs, lastTs } = scanAggrLines(text.split('\n'), acc, true);
+  acc.firstEventTs = firstTs;
+  acc.lastEventTs = lastTs;
 
   try {
     acc.sessionDate = firstTs
@@ -298,6 +327,296 @@ async function parseJSONL(filePath, stat) {
   return { result: acc, cacheHit: false };
 }
 
+// ── rollup bridge ─────────────────────────────────────────────────────────────
+// Converts between the in-memory per-(date,project) accumulator bucket shape
+// used by aggregate()/finalizeClosedDays() and the on-disk rollup line shape
+// (one line per (date, projectDir, modelId) — see historyRollup.cjs header).
+
+function emptyBucket(sessionDate, encodedCwd) {
+  return {
+    date: sessionDate,
+    projectCwd: decodeCwd(encodedCwd),
+    encodedCwd,
+    promptCount: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheCreationTokens: 0,
+    toolCallCount: 0,
+    toolBreakdown: {},
+    byModel: {},
+    sessionCount: 0,
+    errorCount: 0,
+    activeMinutes: 0,
+  };
+}
+
+/** Fold one parsed-file result into an accumulator bucket (mutates b). */
+function foldParsedIntoBucket(b, parsed) {
+  b.promptCount += parsed.promptCount;
+  b.inputTokens += parsed.inputTokens;
+  b.outputTokens += parsed.outputTokens;
+  b.cacheReadTokens += parsed.cacheReadTokens;
+  b.cacheCreationTokens += parsed.cacheCreationTokens;
+  b.toolCallCount += parsed.toolCallCount;
+  for (const [tool, cnt] of Object.entries(parsed.toolBreakdown)) {
+    b.toolBreakdown[tool] = (b.toolBreakdown[tool] ?? 0) + cnt;
+  }
+  for (const [modelId, srcBucket] of Object.entries(parsed.byModel ?? {})) {
+    const dst = b.byModel[modelId] ?? (b.byModel[modelId] = {
+      inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
+    });
+    dst.inputTokens += srcBucket.inputTokens;
+    dst.outputTokens += srcBucket.outputTokens;
+    dst.cacheReadTokens += srcBucket.cacheReadTokens;
+    dst.cacheCreationTokens += srcBucket.cacheCreationTokens;
+  }
+  b.sessionCount++;
+  b.errorCount += parsed.errorCount;
+  b.activeMinutes += computeActiveMinutes(parsed.firstEventTs, parsed.lastEventTs);
+}
+
+/** One accumulator bucket → its rollup lines (one totals line + one per model). */
+function bucketToRollupLines(b) {
+  const lines = [{
+    date: b.date,
+    projectDir: b.encodedCwd,
+    modelId: historyRollup.TOTALS_MODEL_ID,
+    promptCount: b.promptCount,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheCreationTokens: 0,
+    toolCallCount: b.toolCallCount,
+    toolBreakdown: b.toolBreakdown,
+    errorCount: b.errorCount,
+    sessionCount: b.sessionCount,
+    activeMinutes: b.activeMinutes,
+    v: historyRollup.ROLLUP_VERSION,
+  }];
+  for (const [modelId, mb] of Object.entries(b.byModel)) {
+    lines.push({
+      date: b.date,
+      projectDir: b.encodedCwd,
+      modelId,
+      promptCount: 0,
+      inputTokens: mb.inputTokens,
+      outputTokens: mb.outputTokens,
+      cacheReadTokens: mb.cacheReadTokens,
+      cacheCreationTokens: mb.cacheCreationTokens,
+      toolCallCount: 0,
+      toolBreakdown: {},
+      errorCount: 0,
+      sessionCount: 0,
+      activeMinutes: 0,
+      v: historyRollup.ROLLUP_VERSION,
+    });
+  }
+  return lines;
+}
+
+function finalizedMarkerLine(date) {
+  return {
+    date,
+    projectDir: historyRollup.FINALIZED_PROJECT_ID,
+    modelId: historyRollup.FINALIZED_MODEL_ID,
+    finalizedAt: Date.now(),
+    v: historyRollup.ROLLUP_VERSION,
+  };
+}
+
+/**
+ * Reconstruct accumulator-shaped rows from rollup lines, restricted to dates
+ * in `finalizedDates` — only finalized days are trusted as complete enough
+ * to serve without a live re-scan.
+ */
+function rollupMapToRows(map, finalizedDates) {
+  const grouped = new Map();
+  for (const bucket of map.values()) {
+    if (bucket.projectDir === historyRollup.FINALIZED_PROJECT_ID && bucket.modelId === historyRollup.FINALIZED_MODEL_ID) continue;
+    if (!finalizedDates.has(bucket.date)) continue;
+    const groupKey = `${bucket.date}|${bucket.projectDir}`;
+    let row = grouped.get(groupKey);
+    if (!row) {
+      row = emptyBucket(bucket.date, bucket.projectDir);
+      grouped.set(groupKey, row);
+    }
+    if (bucket.modelId === historyRollup.TOTALS_MODEL_ID) {
+      row.promptCount += bucket.promptCount || 0;
+      row.toolCallCount += bucket.toolCallCount || 0;
+      row.errorCount += bucket.errorCount || 0;
+      row.sessionCount += bucket.sessionCount || 0;
+      row.activeMinutes += bucket.activeMinutes || 0;
+      for (const [tool, cnt] of Object.entries(bucket.toolBreakdown || {})) {
+        row.toolBreakdown[tool] = (row.toolBreakdown[tool] ?? 0) + cnt;
+      }
+    } else {
+      const dst = row.byModel[bucket.modelId] ?? (row.byModel[bucket.modelId] = {
+        inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
+      });
+      dst.inputTokens += bucket.inputTokens || 0;
+      dst.outputTokens += bucket.outputTokens || 0;
+      dst.cacheReadTokens += bucket.cacheReadTokens || 0;
+      dst.cacheCreationTokens += bucket.cacheCreationTokens || 0;
+      row.inputTokens += bucket.inputTokens || 0;
+      row.outputTokens += bucket.outputTokens || 0;
+      row.cacheReadTokens += bucket.cacheReadTokens || 0;
+      row.cacheCreationTokens += bucket.cacheCreationTokens || 0;
+    }
+  }
+  return grouped;
+}
+
+/**
+ * Boot-time (and self-healing) finalize pass: walks ALL transcripts, buckets
+ * every day strictly before today, and persists them to the rollup with a
+ * `finalizedAt` stamp. Once a day is finalized, aggregate() trusts the rollup
+ * for it — never re-touching the underlying .jsonl files — so it survives
+ * both process restarts and transcript deletion/retention cleanup.
+ *
+ * O(total transcript bytes) — bounded by the shared parseJSONL LRU cache.
+ *
+ * opts.budgetMs (optional): caps the wall-clock time spent walking project
+ * directories. Project directories are visited in a stable sorted order and
+ * are each read to completion once started (never split mid-directory) so a
+ * date's bucket is only ever built from directories that were fully walked.
+ * If the budget runs out before every project directory has been visited,
+ * NO date is stamped `finalizedAt` this call (a date isn't safe to trust as
+ * complete until every directory has contributed) — the caller can invoke
+ * finalizeClosedDays() again later and it will redo the (idempotent) walk.
+ * Partial results are still persisted as plain (non-finalized) rollup lines
+ * so aggregate() benefits from them immediately.
+ *
+ * opts.dryRun (optional): computes what would be finalized without writing
+ * anything to the rollup file.
+ */
+async function finalizeClosedDays({ budgetMs, dryRun = false } = {}) {
+  const deadline = typeof budgetMs === 'number' ? Date.now() + budgetMs : null;
+  const today = localDate(new Date());
+
+  let projectDirs;
+  try {
+    projectDirs = await fsp.readdir(PROJECTS_DIR, { withFileTypes: true });
+  } catch {
+    return { finalizedDates: [], partial: false };
+  }
+
+  projectDirs = projectDirs
+    .filter((e) => e.isDirectory())
+    .sort((a, b) => a.name.localeCompare(b.name));
+
+  const buckets = new Map();
+  let partial = false;
+
+  for (const projEntry of projectDirs) {
+    if (deadline !== null && Date.now() >= deadline) {
+      partial = true;
+      break;
+    }
+
+    const encodedCwd = projEntry.name;
+    const projectDir = path.join(PROJECTS_DIR, encodedCwd);
+
+    let files;
+    try { files = await fsp.readdir(projectDir, { withFileTypes: true }); } catch { continue; }
+
+    for (const fileEntry of files) {
+      if (!fileEntry.name.endsWith('.jsonl')) continue;
+      const filePath = path.join(projectDir, fileEntry.name);
+
+      let stat;
+      try { stat = await fsp.stat(filePath); } catch { continue; }
+
+      const { result: parsed } = await parseJSONL(filePath, stat);
+      if (parsed.skipped) continue;
+
+      const { sessionDate } = parsed;
+      if (!sessionDate || sessionDate >= today) continue;
+
+      const key = `${sessionDate}|${encodedCwd}`;
+      if (!buckets.has(key)) buckets.set(key, emptyBucket(sessionDate, encodedCwd));
+      foldParsedIntoBucket(buckets.get(key), parsed);
+    }
+  }
+
+  const dates = new Set();
+  const entries = [];
+  for (const b of buckets.values()) {
+    dates.add(b.date);
+    entries.push(...bucketToRollupLines(b));
+  }
+  // Only stamp dates as finalized (safe to trust with no live re-scan) when
+  // the walk visited every project directory within budget — a partial walk
+  // may be missing contributions from directories not yet visited.
+  if (!partial) {
+    for (const date of dates) entries.push(finalizedMarkerLine(date));
+  }
+
+  if (!dryRun) {
+    await historyRollup.appendRollupDays(entries);
+  }
+  return { finalizedDates: partial ? [] : Array.from(dates), partial };
+}
+
+/**
+ * Refresh TODAY's rollup buckets from a live (LRU-warm, cheap) parse of every
+ * transcript touched today, and upsert them as provisional lines (v2, no
+ * finalizedAt) so the History dashboard's "today" row stays current without
+ * the renderer ever triggering a transcript scan itself. Meant to be called
+ * on a timer (HISTORY_INTRADAY_REFRESH_MS) plus once at app start.
+ *
+ * readRollup/appendRollupDays last-write-wins per (date, projectDir, modelId)
+ * key, so re-running this simply replaces today's previous provisional lines
+ * with fresher ones — no separate "clear provisional" step needed.
+ */
+async function refreshIntradayToday() {
+  const today = localDate(new Date());
+
+  let projectDirs;
+  try {
+    projectDirs = await fsp.readdir(PROJECTS_DIR, { withFileTypes: true });
+  } catch {
+    return { date: today, projectsUpdated: 0 };
+  }
+
+  const buckets = new Map();
+  for (const projEntry of projectDirs) {
+    if (!projEntry.isDirectory()) continue;
+    const encodedCwd = projEntry.name;
+    const projectDir = path.join(PROJECTS_DIR, encodedCwd);
+
+    let files;
+    try { files = await fsp.readdir(projectDir, { withFileTypes: true }); } catch { continue; }
+
+    for (const fileEntry of files) {
+      if (!fileEntry.name.endsWith('.jsonl')) continue;
+      const filePath = path.join(projectDir, fileEntry.name);
+
+      let stat;
+      try { stat = await fsp.stat(filePath); } catch { continue; }
+
+      // A file whose mtime is before today can't have been touched today —
+      // skip it without even reading it (matches aggregate()'s fast-skip).
+      const mtimeDate = localDate(new Date(stat.mtimeMs));
+      if (mtimeDate < today) continue;
+
+      const { result: parsed } = await parseJSONL(filePath, stat);
+      if (parsed.skipped) continue;
+      if (parsed.sessionDate !== today) continue;
+
+      const key = `${today}|${encodedCwd}`;
+      if (!buckets.has(key)) buckets.set(key, emptyBucket(today, encodedCwd));
+      foldParsedIntoBucket(buckets.get(key), parsed);
+    }
+  }
+
+  const entries = [];
+  for (const b of buckets.values()) entries.push(...bucketToRollupLines(b));
+  if (entries.length) await historyRollup.appendRollupDays(entries);
+
+  return { date: today, projectsUpdated: buckets.size };
+}
+
 // ── aggregate ─────────────────────────────────────────────────────────────────
 
 async function aggregate(req) {
@@ -307,6 +626,28 @@ async function aggregate(req) {
   if (effectiveTo > today) effectiveTo = today;
   const effectiveFrom = req?.fromDate ? req.fromDate : subtractDays(today, 30);
 
+  // Closed (pre-today) days: consult the durable rollup first. A day only
+  // short-circuits the live file walk once it's finalized — see
+  // finalizeClosedDays(). Unfinalized past days still get live-parsed below
+  // (and folded back into the rollup as a self-healing backfill) so results
+  // stay correct even before the boot finalize pass has run.
+  const yesterday = subtractDays(today, 1);
+  const rollupToDate = effectiveTo < today ? effectiveTo : yesterday;
+  let rollupMap = new Map();
+  if (effectiveFrom <= rollupToDate) {
+    rollupMap = await historyRollup.readRollup(effectiveFrom, rollupToDate);
+  }
+  const finalizedDates = new Set();
+  for (const bucket of rollupMap.values()) {
+    if (
+      bucket.projectDir === historyRollup.FINALIZED_PROJECT_ID &&
+      bucket.modelId === historyRollup.FINALIZED_MODEL_ID &&
+      bucket.finalizedAt
+    ) {
+      finalizedDates.add(bucket.date);
+    }
+  }
+
   const buckets = new Map();
   let truncated = false;
   let skippedLargeFiles = 0;
@@ -317,7 +658,7 @@ async function aggregate(req) {
   try {
     projectDirs = await fsp.readdir(PROJECTS_DIR, { withFileTypes: true });
   } catch {
-    return { rows: [], partial: false, truncated: false, scannedMs: Date.now() - t0 };
+    return { rows: [], partial: false, truncated: false, scannedMs: Date.now() - t0, cacheSavingsUsd: 0, rollupDays: 0, liveDays: 0 };
   }
 
   outer:
@@ -340,6 +681,14 @@ async function aggregate(req) {
       let stat;
       try { stat = await fsp.stat(filePath); } catch { continue; }
 
+      // Fast skip: a file last modified on an already-finalized past day is
+      // trusted from the rollup and never touched. A file whose true content
+      // belongs to an OLDER, not-yet-finalized day gets caught by
+      // finalizeClosedDays() the next time it runs (that pass reads full
+      // content, not mtime, and finalizes every day it discovers).
+      const mtimeDate = localDate(new Date(stat.mtimeMs));
+      if (mtimeDate < today && finalizedDates.has(mtimeDate)) continue;
+
       const t1 = Date.now();
       const { result: parsed, cacheHit } = await parseJSONL(filePath, stat);
       if (!cacheHit) parseBudgetSpentMs += Date.now() - t1;
@@ -352,46 +701,13 @@ async function aggregate(req) {
       // hide a Pacific-time user's most recent activity entirely.
       if (!sessionDate || sessionDate < effectiveFrom || sessionDate > effectiveTo) continue;
 
-      const key = `${sessionDate}|${encodedCwd}`;
-      if (!buckets.has(key)) {
-        buckets.set(key, {
-          date: sessionDate,
-          projectCwd: decodeCwd(encodedCwd),
-          encodedCwd,
-          promptCount: 0,
-          inputTokens: 0,
-          outputTokens: 0,
-          cacheReadTokens: 0,
-          cacheCreationTokens: 0,
-          toolCallCount: 0,
-          toolBreakdown: {},
-          byModel: {},
-          sessionCount: 0,
-          errorCount: 0,
-        });
-      }
+      // Content-confirmed finalized day (mtime heuristic above missed it) —
+      // the rollup is authoritative; don't double count.
+      if (sessionDate < today && finalizedDates.has(sessionDate)) continue;
 
-      const b = buckets.get(key);
-      b.promptCount += parsed.promptCount;
-      b.inputTokens += parsed.inputTokens;
-      b.outputTokens += parsed.outputTokens;
-      b.cacheReadTokens += parsed.cacheReadTokens;
-      b.cacheCreationTokens += parsed.cacheCreationTokens;
-      b.toolCallCount += parsed.toolCallCount;
-      for (const [tool, cnt] of Object.entries(parsed.toolBreakdown)) {
-        b.toolBreakdown[tool] = (b.toolBreakdown[tool] ?? 0) + cnt;
-      }
-      for (const [modelId, srcBucket] of Object.entries(parsed.byModel ?? {})) {
-        const dst = b.byModel[modelId] ?? (b.byModel[modelId] = {
-          inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
-        });
-        dst.inputTokens += srcBucket.inputTokens;
-        dst.outputTokens += srcBucket.outputTokens;
-        dst.cacheReadTokens += srcBucket.cacheReadTokens;
-        dst.cacheCreationTokens += srcBucket.cacheCreationTokens;
-      }
-      b.sessionCount++;
-      b.errorCount += parsed.errorCount;
+      const key = `${sessionDate}|${encodedCwd}`;
+      if (!buckets.has(key)) buckets.set(key, emptyBucket(sessionDate, encodedCwd));
+      foldParsedIntoBucket(buckets.get(key), parsed);
 
       if (!cacheHit && parseBudgetSpentMs > PARSE_BUDGET_MS) {
         skippedBudgetFiles++;
@@ -405,8 +721,23 @@ async function aggregate(req) {
     }
   }
 
+  // Self-healing backfill: any past (pre-today) day we just live-parsed was
+  // walked in full above (the file walk isn't restricted to a single day), so
+  // fold it into the rollup now — it'll be trusted outright once the next
+  // finalize pass stamps it `finalizedAt`.
+  const backfillEntries = [];
+  for (const b of buckets.values()) {
+    if (b.date < today) backfillEntries.push(...bucketToRollupLines(b));
+  }
+  if (backfillEntries.length) {
+    await historyRollup.appendRollupDays(backfillEntries);
+  }
+
+  const rollupRows = rollupMapToRows(rollupMap, finalizedDates);
+  const allBucketEntries = [...rollupRows.values(), ...buckets.values()];
+
   let cacheSavingsUsd = 0;
-  const rows = Array.from(buckets.values()).map((b) => {
+  const rows = allBucketEntries.map((b) => {
     const byModel = {};
     let estimatedCostUsd = 0;
     for (const [modelId, bucket] of Object.entries(b.byModel)) {
@@ -427,8 +758,11 @@ async function aggregate(req) {
 
   rows.sort((a, b) => a.date.localeCompare(b.date) || a.projectCwd.localeCompare(b.projectCwd));
 
+  const rollupDays = new Set(Array.from(rollupRows.values()).map((r) => r.date)).size;
+  const liveDays = new Set(Array.from(buckets.values()).map((r) => r.date)).size;
+
   const scannedMs = Date.now() - t0;
-  return { rows, partial: truncated, truncated, scannedMs, skippedLargeFiles, cacheSavingsUsd };
+  return { rows, partial: truncated, truncated, scannedMs, skippedLargeFiles, cacheSavingsUsd, rollupDays, liveDays };
 }
 
 // ── IPC registration ──────────────────────────────────────────────────────────
@@ -485,10 +819,17 @@ module.exports = {
   registerHistoryAggregatorHandlers,
   remote,
   MODEL_PRICING,
+  finalizeClosedDays,
+  refreshIntradayToday,
   // exported for tests
   scanAggrLines,
   parseJSONL,
   resolvePricingKey,
   CACHE_MAX,
   LRUCache,
+  computeActiveMinutes,
+  // exported for historyDashboard.cjs (pure, no I/O at require-time; reused
+  // rather than re-implemented so date-range math stays in one place)
+  localDate,
+  subtractDays,
 };