codexmeter 1.0.18 → 1.0.20

package/server/rollout-reader.js

@@ -1,8 +1,18 @@
-import { existsSync } from 'fs';
+import { existsSync, statSync } from 'fs';
 import { readFile } from 'fs/promises';
+import { spawn } from 'child_process';
 import { createDayKeyFormatter } from './day-key.js';
 
 const ACTIVE_GAP_CAP_MS = 15 * 60 * 1000;
+const LINE_HEADER_SCAN_CHARS = 512;
+const TIMESTAMP_RE = /"timestamp"\s*:\s*"([^"]+)"/;
+const DEFAULT_RG_MIN_BYTES = 10 * 1024 * 1024;
+const RG_RELEVANT_PATTERN =
+  '"type"\\s*:\\s*"session_meta"|' +
+  '"type"\\s*:\\s*"turn_context"|' +
+  '"type"\\s*:\\s*"token_count"';
+const RG_TOKEN_COUNT_PATTERN = '"type"\\s*:\\s*"token_count"';
+const RG_TIMESTAMP_PATTERN = '^\\{[^{}]*"timestamp"\\s*:\\s*"[^"]+"';
 
 export async function enrichFromRollout(rolloutPath, opts = {}) {
   if (!rolloutPath || !existsSync(rolloutPath)) {
@@ -11,6 +21,9 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
 
   const tz = opts.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone;
   const toDayKey = opts.toDayKey || createDayKeyFormatter(tz);
+  const fastScan = opts.fastScan === true;
+  const rgScan = opts.rgScan === true;
+  const rgMinBytes = Math.max(0, Number(opts.rgMinBytes ?? DEFAULT_RG_MIN_BYTES) || 0);
   const result = {
     model_name: null,
     reasoning_effort: null,
@@ -27,7 +40,20 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
   };
 
   try {
-    const content = await readFile(rolloutPath, 'utf8');
+    let lines = null;
+    let activeFromRgTimestamps = false;
+    if (rgScan && shouldUseRipgrep(rolloutPath, rgMinBytes)) {
+      const rgResult = await readRolloutLinesWithRipgrep(rolloutPath);
+      if (rgResult) {
+        lines = rgResult.relevantLines;
+        applyTimestampMatches(result, rgResult.timestampMatches, toDayKey);
+        activeFromRgTimestamps = true;
+      }
+    }
+    if (!lines) {
+      const content = await readFile(rolloutPath, 'utf8');
+      lines = content.split(/\r?\n/);
+    }
 
     let prevTimestamp = null;
     let activeMs = 0;
@@ -39,21 +65,17 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
     let lastReasoningOutputTokens = 0;
     let lastTotalTokens = 0;
     let hasSeenUsage = false;
-    for (const line of content.split(/\r?\n/)) {
+    for (const line of lines) {
       if (!line.trim()) continue;
 
       try {
-        const obj = JSON.parse(line);
+        let obj = null;
+        let ts = null;
 
-        if (obj.timestamp) {
-          const ts = new Date(obj.timestamp).getTime();
-          if (!isNaN(ts)) {
-            if (!result.first_timestamp || ts < result.first_timestamp) {
-              result.first_timestamp = ts;
-            }
-            if (!result.last_timestamp || ts > result.last_timestamp) {
-              result.last_timestamp = ts;
-            }
+        if (fastScan || activeFromRgTimestamps) {
+          ts = extractTimestampMs(line);
+          if (ts !== null && !activeFromRgTimestamps) {
+            updateTimestampBounds(result, ts);
             if (prevTimestamp !== null && ts >= prevTimestamp) {
               const deltaMs = Math.min(ts - prevTimestamp, ACTIVE_GAP_CAP_MS);
               activeMs += deltaMs;
@@ -64,6 +86,25 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
             }
             prevTimestamp = ts;
           }
+          if (!activeFromRgTimestamps && !isRolloutLineWorthParsing(line)) continue;
+          obj = JSON.parse(line);
+        } else {
+          obj = JSON.parse(line);
+          if (obj.timestamp) {
+            ts = new Date(obj.timestamp).getTime();
+            if (!isNaN(ts)) {
+              updateTimestampBounds(result, ts);
+              if (prevTimestamp !== null && ts >= prevTimestamp) {
+                const deltaMs = Math.min(ts - prevTimestamp, ACTIVE_GAP_CAP_MS);
+                activeMs += deltaMs;
+                if (deltaMs > 0) {
+                  const dayKey = toDayKey(prevTimestamp);
+                  activeByDay.set(dayKey, (activeByDay.get(dayKey) || 0) + deltaMs);
+                }
+              }
+              prevTimestamp = ts;
+            }
+          }
         }
 
         if (obj.type === 'event_msg' && obj.payload?.type === 'token_count') {
@@ -121,16 +162,13 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
             lastOutputTokens = outputTokens;
             lastReasoningOutputTokens = reasoningOutputTokens;
             lastTotalTokens = totalTokens;
-            if (obj.timestamp && hasUsageBoundarySignal(usageDelta)) {
-              const ts = new Date(obj.timestamp).getTime();
-              if (!isNaN(ts)) {
-                if (!result.first_usage_timestamp || ts < result.first_usage_timestamp) {
-                  result.first_usage_timestamp = ts;
-                }
-                if (hasUsage(usageDelta)) {
-                  const dayKey = toDayKey(ts);
-                  mergeUsageTotals(usageByDay, dayKey, usageDelta);
-                }
+            if (ts !== null && hasUsageBoundarySignal(usageDelta)) {
+              if (!result.first_usage_timestamp || ts < result.first_usage_timestamp) {
+                result.first_usage_timestamp = ts;
+              }
+              if (hasUsage(usageDelta)) {
+                const dayKey = toDayKey(ts);
+                mergeUsageTotals(usageByDay, dayKey, usageDelta);
               }
             }
           }
@@ -179,19 +217,30 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
   return result;
 }
 
-export async function readUsageTimeline(rolloutPath) {
+export async function readUsageTimeline(rolloutPath, opts = {}) {
   if (!rolloutPath || !existsSync(rolloutPath)) {
     return [];
   }
 
   try {
-    const content = await readFile(rolloutPath, 'utf8');
+    const fastScan = opts.fastScan === true;
+    const rgScan = opts.rgScan === true;
+    const rgMinBytes = Math.max(0, Number(opts.rgMinBytes ?? DEFAULT_RG_MIN_BYTES) || 0);
+    let lines = null;
+    if (rgScan && shouldUseRipgrep(rolloutPath, rgMinBytes)) {
+      lines = await readMatchingLinesWithRipgrep(RG_TOKEN_COUNT_PATTERN, rolloutPath);
+    }
+    if (!lines) {
+      const content = await readFile(rolloutPath, 'utf8');
+      lines = content.split(/\r?\n/);
+    }
     const timeline = [];
     let segmentId = 0;
     let lastTotalTokens = 0;
     let hasSeenUsage = false;
-    for (const line of content.split(/\r?\n/)) {
+    for (const line of lines) {
       if (!line.trim()) continue;
+      if (fastScan && !isTokenCountEventLine(line)) continue;
       try {
         const obj = JSON.parse(line);
         if (obj.type !== 'event_msg' || obj.payload?.type !== 'token_count') continue;
@@ -241,6 +290,116 @@ export function findUsageAtOrBefore(timeline, timestampMs) {
   return findUsageEntryAtOrBefore(timeline, timestampMs)?.usage || null;
 }
 
+function shouldUseRipgrep(rolloutPath, minBytes) {
+  try {
+    return statSync(rolloutPath).size >= minBytes;
+  } catch {
+    return false;
+  }
+}
+
+async function readRolloutLinesWithRipgrep(rolloutPath) {
+  const [relevantLines, timestampMatches] = await Promise.all([
+    readMatchingLinesWithRipgrep(RG_RELEVANT_PATTERN, rolloutPath),
+    readMatchingLinesWithRipgrep(RG_TIMESTAMP_PATTERN, rolloutPath, ['--only-matching']),
+  ]);
+  if (!relevantLines || !timestampMatches) return null;
+  return { relevantLines, timestampMatches };
+}
+
+function readMatchingLinesWithRipgrep(pattern, rolloutPath, extraArgs = []) {
+  return new Promise((resolve) => {
+    const args = [
+      '--no-heading',
+      '--no-line-number',
+      '--color',
+      'never',
+      ...extraArgs,
+      pattern,
+      rolloutPath,
+    ];
+    const child = spawn('rg', args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+
+    child.stdout.setEncoding('utf8');
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.on('error', () => resolve(null));
+    child.on('close', (code) => {
+      if (code !== 0 && code !== 1) {
+        resolve(null);
+        return;
+      }
+      resolve(stdout ? stdout.split(/\r?\n/).filter(Boolean) : []);
+    });
+  });
+}
+
+function applyTimestampMatches(result, timestampMatches, toDayKey) {
+  let prevTimestamp = null;
+  let activeMs = 0;
+  const activeByDay = new Map();
+
+  for (const line of timestampMatches || []) {
+    const ts = extractTimestampMs(line);
+    if (ts === null) continue;
+    updateTimestampBounds(result, ts);
+    if (prevTimestamp !== null && ts >= prevTimestamp) {
+      const deltaMs = Math.min(ts - prevTimestamp, ACTIVE_GAP_CAP_MS);
+      activeMs += deltaMs;
+      if (deltaMs > 0) {
+        const dayKey = toDayKey(prevTimestamp);
+        activeByDay.set(dayKey, (activeByDay.get(dayKey) || 0) + deltaMs);
+      }
+    }
+    prevTimestamp = ts;
+  }
+
+  if (activeMs > 0) {
+    const activeByDaySeconds = Object.fromEntries(
+      [...activeByDay.entries()].map(([dayKey, ms]) => [dayKey, Math.round(ms / 1000)])
+    );
+    result.active_by_day = activeByDaySeconds;
+    result.active_seconds = Object.values(activeByDaySeconds).reduce((sum, seconds) => sum + seconds, 0);
+  }
+}
+
+function updateTimestampBounds(result, ts) {
+  if (!result.first_timestamp || ts < result.first_timestamp) {
+    result.first_timestamp = ts;
+  }
+  if (!result.last_timestamp || ts > result.last_timestamp) {
+    result.last_timestamp = ts;
+  }
+}
+
+function extractTimestampMs(line) {
+  const head = line.length > LINE_HEADER_SCAN_CHARS
+    ? line.slice(0, LINE_HEADER_SCAN_CHARS)
+    : line;
+  const match = TIMESTAMP_RE.exec(head);
+  if (!match) return null;
+  const ts = Date.parse(match[1]);
+  return Number.isNaN(ts) ? null : ts;
+}
+
+function isRolloutLineWorthParsing(line) {
+  const head = line.length > LINE_HEADER_SCAN_CHARS
+    ? line.slice(0, LINE_HEADER_SCAN_CHARS)
+    : line;
+  return head.includes('"type":"token_count"') ||
+    head.includes('"type":"turn_context"') ||
+    head.includes('"type":"session_meta"');
+}
+
+function isTokenCountEventLine(line) {
+  const head = line.length > LINE_HEADER_SCAN_CHARS
+    ? line.slice(0, LINE_HEADER_SCAN_CHARS)
+    : line;
+  return head.includes('"type":"token_count"');
+}
+
 function normalizeUsageTotals(usage) {
   return {
     input_tokens: usage.input_tokens || 0,

package/server/rollout-worker-pool.js

@@ -3,8 +3,9 @@ import { Worker } from 'worker_threads';
 
 export function createRolloutWorkerPool(opts = {}) {
   const size = normalizePoolSize(opts.size);
+  const readerOptions = opts.readerOptions || {};
   if (size <= 1) {
-    return createInlinePool();
+    return createInlinePool({ readerOptions });
   }
 
   const workers = new Set();
@@ -89,6 +90,7 @@ export function createRolloutWorkerPool(opts = {}) {
         id,
         rolloutPath: task.rolloutPath,
         timezone: task.timezone,
+        readerOptions,
       });
     }
   }
@@ -107,6 +109,70 @@ export function createRolloutWorkerPool(opts = {}) {
     return Promise.all(rolloutPaths.map((rolloutPath) => runTask(rolloutPath, timezone)));
   }
 
+  async function mapRolloutsInChunks(rolloutPaths, timezone, { chunkSize = 100, onChunk } = {}) {
+    const results = new Array(rolloutPaths.length);
+    const completed = new Array(rolloutPaths.length).fill(false);
+    const safeChunkSize = Math.max(1, Number(chunkSize) || 1);
+    const maxInFlight = Math.max(safeChunkSize, size * 4);
+    let nextStartIndex = 0;
+    let nextFlushIndex = 0;
+    let activeCount = 0;
+    let completedCount = 0;
+    let flushChain = Promise.resolve();
+    let rejected = false;
+
+    const scheduleFlush = (force = false) => {
+      if (typeof onChunk !== 'function') return;
+      const chunk = [];
+      while (nextFlushIndex < results.length && completed[nextFlushIndex]) {
+        chunk.push({ index: nextFlushIndex, result: results[nextFlushIndex] });
+        nextFlushIndex += 1;
+        if (!force && chunk.length >= safeChunkSize) break;
+      }
+      if (!chunk.length) return;
+      flushChain = flushChain.then(() => onChunk(chunk));
+    };
+
+    await new Promise((resolve, reject) => {
+      const launchNext = () => {
+        if (rejected) return;
+        while (activeCount < maxInFlight && nextStartIndex < rolloutPaths.length) {
+          const index = nextStartIndex;
+          const rolloutPath = rolloutPaths[index];
+          nextStartIndex += 1;
+          activeCount += 1;
+          runTask(rolloutPath, timezone)
+            .then((result) => {
+              results[index] = result;
+              completed[index] = true;
+              completedCount += 1;
+              activeCount -= 1;
+              scheduleFlush(false);
+              if (completedCount >= rolloutPaths.length && activeCount === 0) {
+                resolve();
+                return;
+              }
+              launchNext();
+            })
+            .catch((error) => {
+              rejected = true;
+              reject(error);
+            });
+        }
+
+        if (completedCount >= rolloutPaths.length && activeCount === 0) {
+          resolve();
+        }
+      };
+
+      launchNext();
+    });
+
+    scheduleFlush(true);
+    await flushChain;
+    return results;
+  }
+
   async function close() {
     closed = true;
     while (queuedTasks.length > 0) {
@@ -122,10 +188,10 @@ export function createRolloutWorkerPool(opts = {}) {
     idleWorkers.length = 0;
   }
 
-  return { mapRollouts, close, size };
+  return { mapRollouts, mapRolloutsInChunks, close, size };
 }
 
-function createInlinePool() {
+function createInlinePool({ readerOptions = {} } = {}) {
   return {
     size: 1,
     async mapRollouts(rolloutPaths, timezone) {
@@ -133,7 +199,7 @@ function createInlinePool() {
       return Promise.all(
         rolloutPaths.map(async (rolloutPath) => {
           try {
-            const data = await enrichFromRollout(rolloutPath, { timezone });
+            const data = await enrichFromRollout(rolloutPath, { timezone, ...readerOptions });
             return { ok: true, data, error: null };
           } catch (error) {
             return {
@@ -145,6 +211,39 @@ function createInlinePool() {
         })
       );
     },
+    async mapRolloutsInChunks(rolloutPaths, timezone, { chunkSize = 100, onChunk } = {}) {
+      const { enrichFromRollout } = await import('./rollout-reader.js');
+      const results = [];
+      const completed = [];
+      const safeChunkSize = Math.max(1, Number(chunkSize) || 1);
+
+      for (let index = 0; index < rolloutPaths.length; index += 1) {
+        try {
+          const data = await enrichFromRollout(rolloutPaths[index], { timezone, ...readerOptions });
+          const result = { ok: true, data, error: null };
+          results[index] = result;
+          completed.push({ index, result });
+        } catch (error) {
+          const result = {
+            ok: false,
+            data: null,
+            error: error instanceof Error ? error.message : String(error),
+          };
+          results[index] = result;
+          completed.push({ index, result });
+        }
+
+        if (typeof onChunk === 'function' && completed.length >= safeChunkSize) {
+          await onChunk(completed.splice(0, completed.length));
+        }
+      }
+
+      if (typeof onChunk === 'function' && completed.length) {
+        await onChunk(completed.splice(0, completed.length));
+      }
+
+      return results;
+    },
     async close() {},
   };
 }

package/server/rollout-worker.js

@@ -6,10 +6,10 @@ if (!parentPort) {
 }
 
 parentPort.on('message', async (message) => {
-  const { id, rolloutPath, timezone } = message || {};
+  const { id, rolloutPath, timezone, readerOptions } = message || {};
 
   try {
-    const result = await enrichFromRollout(rolloutPath, { timezone });
+    const result = await enrichFromRollout(rolloutPath, { timezone, ...(readerOptions || {}) });
     parentPort.postMessage({ id, ok: true, data: result });
   } catch (error) {
     parentPort.postMessage({

package/src/utils/animationsDefault.js

@@ -6,7 +6,7 @@ const AUTO = 'auto';
  * This is the single source of truth for:
  * - Overview client-side presentation timing
  * - Overview ECharts timing defaults
- * - Overview server-side tail pacing
+ * - Overview live snapshot transport cadence
  *
  * `speed` scales the main timings:
  * - `1` = baseline
@@ -21,8 +21,7 @@ export const OVERVIEW_INGEST_ANIMATION = {
   speed: 1,
   live: {
     frameIntervalMs: 33,
-    overviewHz: 20,
-    dayKeysPerEmit: 1,
+    snapshotHz: 12,
   },
   main: {
     presentationDurationMs: 220,
@@ -35,15 +34,13 @@ export const OVERVIEW_INGEST_ANIMATION = {
     // Master switch for the end-of-ingest slowdown behavior.
     enabled: true,
     // Progress threshold where the tail mode starts, expressed from 0..1.
-    startPercent: 0.5,
+    startPercent: 0.95,
     // Fixed presentation duration in ms during the tail; use AUTO to derive it from main.durationScale.
     durationMs: AUTO,
     // Multiplier applied to the normal presentation duration when durationMs is AUTO.
     durationScale: 10,
     // Shared easing applied by the Overview presentation animator during the tail.
     easing: 'cubicOut',
-    // Backend Overview live-update cadence during the tail, in patches per second.
-    overviewHz: 10,
   },
   daily: {
     chartAppearDurationMs: AUTO,
@@ -106,6 +103,7 @@ export const OVERVIEW_INGEST_ANIMATION = {
 
     // Final Flash Settings
     finalFlashDurationMs: 1000,
+    finalFlashDelayMs: 200,
     finalFlashMaxOpacity: 0.35,
 
     // Display Tween Settings