codexmeter 1.0.1 → 1.0.3

package/server/index.js

@@ -1,13 +1,16 @@
 import express from 'express';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { attachLiveSubscriber, createIngestState, detachLiveSubscriber, restartIngest, runIngest } from './ingest.js';
+import { attachLiveSubscriber, createIngestState, detachLiveSubscriber, getLatestReplay, restartIngest, runIngest } from './ingest.js';
+import { createJobSummary, createVideoExportManager, getActiveVideoExportJob, getVideoExportJob, getVideoExportSupport, startOverviewVideoExport } from './export-video.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 export function createServer(codexHome, opts = {}) {
   const app = express();
+  app.use(express.json());
   const state = createIngestState();
+  const exportManager = createVideoExportManager();
   const distDir = path.join(__dirname, '..', 'dist');
   const apiOnly = opts.devApiOnly === true;
   const ingestOpts = { ...opts };
@@ -77,6 +80,85 @@ export function createServer(codexHome, opts = {}) {
   app.get('/api/heatmap', wrap('heatmap'));
   app.get('/api/families', wrap('families'));
 
+  app.post('/api/export/overview-video', async (req, res) => {
+    const activeJob = getActiveVideoExportJob(exportManager);
+    if (activeJob) {
+      res.status(409).json({ error: 'Another export job is already running.' });
+      return;
+    }
+    const replay = getLatestReplay(state);
+    if (!replay) {
+      res.status(409).json({ error: 'No completed ingest replay is available yet.' });
+      return;
+    }
+
+    const appBaseUrl = req.get('x-codexmeter-client-base') || opts.frontendBaseUrl || `${req.protocol}://${req.get('host')}`;
+    const settledEnvelope = state.aggregates ? {
+      overview: { data: state.aggregates.overview },
+      repos: { data: state.aggregates.repos },
+      models: { data: state.aggregates.models },
+      families: { data: state.aggregates.families },
+      daily: { data: state.aggregates.daily },
+      heatmap: { data: state.aggregates.heatmap },
+    } : null;
+    try {
+      const job = await startOverviewVideoExport(exportManager, {
+        replay,
+        settledEnvelope,
+        appBaseUrl,
+        installPortableBrowser: Boolean(req.body?.install_portable_browser),
+      });
+      res.status(202).json(createJobSummary(job, `${req.protocol}://${req.get('host')}`));
+    } catch (err) {
+      res.status(err.statusCode || 500).json({ error: err.message || String(err) });
+    }
+  });
+
+  app.get('/api/export/active', (req, res) => {
+    const job = getActiveVideoExportJob(exportManager);
+    if (!job) {
+      res.json({ job: null });
+      return;
+    }
+    res.json({ job: createJobSummary(job, `${req.protocol}://${req.get('host')}`) });
+  });
+
+  app.get('/api/export/support', async (_req, res) => {
+    res.json(await getVideoExportSupport());
+  });
+
+  app.get('/api/export/:jobId/status', (req, res) => {
+    const job = getVideoExportJob(exportManager, req.params.jobId);
+    if (!job) {
+      res.status(404).json({ error: 'Export job not found.' });
+      return;
+    }
+    res.json(createJobSummary(job, `${req.protocol}://${req.get('host')}`));
+  });
+
+  app.get('/api/export/:jobId/render-data', (req, res) => {
+    const job = getVideoExportJob(exportManager, req.params.jobId);
+    if (!job) {
+      res.status(404).json({ error: 'Export job not found.' });
+      return;
+    }
+    const payload = exportManager.getRenderPayload(req.params.jobId);
+    if (!payload) {
+      res.status(404).json({ error: 'Export render data not found.' });
+      return;
+    }
+    res.json(payload);
+  });
+
+  app.get('/api/export/:jobId/file', (req, res) => {
+    const job = getVideoExportJob(exportManager, req.params.jobId);
+    if (!job || job.status !== 'complete' || !job.output_path) {
+      res.status(404).json({ error: 'Export file not ready.' });
+      return;
+    }
+    res.download(job.output_path, job.file_name || `codexmeter-overview-${job.id}.mp4`);
+  });
+
   app.get('/api/sessions', (req, res) => {
     const q = (req.query.q || '').toLowerCase();
     let sessions = state.sessions || [];

package/server/ingest.js

@@ -9,17 +9,14 @@ import { buildAggregates, buildSessionView } from './aggregator.js';
 import { createDayKeyFormatter } from './day-key.js';
 import { createLiveAggregateState, createEmptyLivePatch, applySessionToLiveState, buildLiveBootstrap, buildLivePatch } from './live-state.js';
 import { createRolloutWorkerPool } from './rollout-worker-pool.js';
+import { beginReplayCapture, createReplayCaptureState, failReplayCapture, getReplaySnapshot, recordReplayEvent, resetReplayCapture } from './export-replay.js';
+import { OVERVIEW_INGEST_ANIMATION } from '../src/utils/animationsDefault.js';
 
 const LIVE_FRAME_INTERVAL_MS = 50;
 const LIVE_DAYS_PER_SECOND = 6;
-const LIVE_DAY_CADENCE_MS = Math.round(1000 / LIVE_DAYS_PER_SECOND);
+const LIVE_OVERVIEW_CADENCE_MS = Math.round(1000 / 10);
 const LIVE_DAY_KEYS_PER_EMIT = 1;
-const LIVE_SURFACE_CADENCE_MS = {
-  overview: LIVE_FRAME_INTERVAL_MS * 2,
-  rankings: LIVE_FRAME_INTERVAL_MS * 3,
-  daily: LIVE_DAY_CADENCE_MS,
-  heatmap: LIVE_DAY_CADENCE_MS,
-};
+const LIVE_SESSION_REORDER_BUFFER = 160;
 
 export function createIngestState() {
   return {
@@ -43,9 +40,11 @@ export function createIngestState() {
     live_subscribers: new Set(),
     live_pump_timer: null,
     live_pending_patch: createEmptyLivePatch(),
+    live_session_buffer: [],
     live_progress_dirty: false,
     live_last_emit_at: 0,
-    live_last_surface_emit_at: { overview: 0, rankings: 0, daily: 0, heatmap: 0 },
+    live_last_overview_emit_at: 0,
+    replay_capture: createReplayCaptureState(),
   };
 }
 
@@ -60,6 +59,12 @@ export async function runIngest(codexHome, state, opts = {}) {
     state.live_state = createLiveAggregateState(tz);
     state.phase = 'inventory';
     state.percent = 0;
+    beginReplayCapture(state.replay_capture, state.ingest_id, {
+      ingest_id: state.ingest_id,
+      seq: 0,
+      progress: progressPayload(state),
+      data: buildLiveBootstrap(state.live_state),
+    });
     queueLiveProgress(state);
     broadcastBootstrap(state);
 
@@ -100,6 +105,7 @@ export async function runIngest(codexHome, state, opts = {}) {
         usage_total: null,
         usage_by_day: null,
         has_usage_by_day: false,
+        live_sort_ts: null,
         active_by_day: null,
         agent_role: t.agent_role,
         agent_nickname: t.agent_nickname,
@@ -134,7 +140,7 @@ export async function runIngest(codexHome, state, opts = {}) {
 
     state.needs_enrichment = candidates.length;
     state.percent = candidates.length > 0 ? 0.08 : 0.90;
-    const BATCH_SIZE = opts.batchSize || 100;
+    const BATCH_SIZE = opts.batchSize || 40;
     const ROOT_REFRESH_EVERY = opts.rootRefreshEvery || 1000;
     let lastRootRefreshCount = 0;
 
@@ -166,12 +172,16 @@ export async function runIngest(codexHome, state, opts = {}) {
             s.usage_by_day = buildUsageByDayMetrics(s.model_name, data.usage_by_day);
             s.has_usage_by_day = s.usage_by_day.length > 0;
           }
+          if (data.first_usage_timestamp) {
+            s.live_sort_ts = Math.floor(data.first_usage_timestamp / 1000);
+          }
           if (data.active_seconds && data.active_seconds > 0) {
             s.elapsed_seconds = data.active_seconds;
             s.active_by_day = data.active_by_day || null;
           }
         }
         finalizeSessionMetrics(s, toDayKey);
+        s.live_sort_day = deriveLiveSortDay(s, toDayKey);
         s.materialized = true;
       }
 
@@ -183,8 +193,14 @@ export async function runIngest(codexHome, state, opts = {}) {
         assignRootThreadIds(sessions);
         lastRootRefreshCount = state.enriched;
       }
+      const liveReady = bufferLiveSessionsForPresentation(state, batch);
+      if (state.live_session_buffer[0]?.live_sort_day) {
+        state.current_date_bucket = state.live_session_buffer[0].live_sort_day;
+      } else if (liveReady[0]?.live_sort_day) {
+        state.current_date_bucket = liveReady[0].live_sort_day;
+      }
       const livePatch = createEmptyLivePatch();
-      for (const session of batch) {
+      for (const session of liveReady) {
         applySessionToLiveState(state.live_state, session, livePatch);
       }
       queueLivePatch(state, livePatch);
@@ -203,6 +219,11 @@ export async function runIngest(codexHome, state, opts = {}) {
     }
 
     assignRootThreadIds(sessions);
+    const finalLivePatch = createEmptyLivePatch();
+    for (const session of flushBufferedLiveSessions(state)) {
+      applySessionToLiveState(state.live_state, session, finalLivePatch);
+    }
+    queueLivePatch(state, finalLivePatch);
 
     state.phase = 'aggregation';
     state.percent = state.needs_enrichment > 0 ? 0.95 : 0.90;
@@ -222,6 +243,7 @@ export async function runIngest(codexHome, state, opts = {}) {
     if (!isCurrentRun()) return;
     state.error = err.message;
     state.phase = 'error';
+    failReplayCapture(state.replay_capture);
     console.error('Ingest error:', err);
     flushLive(state, 'ingest-error');
   } finally {
@@ -253,9 +275,11 @@ export function restartIngest(codexHome, state, opts = {}) {
   state.live_state = null;
   state.live_seq = 0;
   state.live_pending_patch = createEmptyLivePatch();
+  state.live_session_buffer = [];
   state.live_progress_dirty = false;
   state.live_last_emit_at = 0;
-  state.live_last_surface_emit_at = { overview: 0, rankings: 0, daily: 0, heatmap: 0 };
+  state.live_last_overview_emit_at = 0;
+  resetReplayCapture(state.replay_capture);
 
   return runIngest(codexHome, state, opts);
 }
@@ -373,6 +397,49 @@ function buildUsageByDayMetrics(modelName, usageByDay) {
   return entries;
 }
 
+function deriveLiveSortDay(session, toDayKey) {
+  if (session.has_usage_by_day && session.usage_by_day?.length) {
+    let best = session.usage_by_day[0];
+    for (const entry of session.usage_by_day) {
+      if ((entry.tokens || 0) > (best.tokens || 0)) {
+        best = entry;
+        continue;
+      }
+      if ((entry.tokens || 0) === (best.tokens || 0) && String(entry.day) > String(best.day)) {
+        best = entry;
+      }
+    }
+    return best?.day || null;
+  }
+  if (session.started_at) {
+    return toDayKey(session.started_at * 1000);
+  }
+  return null;
+}
+
+function compareLiveSessionOrder(a, b) {
+  const dayCompare = String(a.live_sort_day || '9999-12-31').localeCompare(String(b.live_sort_day || '9999-12-31'));
+  if (dayCompare !== 0) return dayCompare;
+  const liveTsCompare = (a.live_sort_ts || a.started_at || 0) - (b.live_sort_ts || b.started_at || 0);
+  if (liveTsCompare !== 0) return liveTsCompare;
+  return String(a.thread_id || '').localeCompare(String(b.thread_id || ''));
+}
+
+function bufferLiveSessionsForPresentation(state, sessions) {
+  if (!sessions.length) return [];
+  state.live_session_buffer.push(...sessions);
+  state.live_session_buffer.sort(compareLiveSessionOrder);
+  const flushCount = Math.max(0, state.live_session_buffer.length - LIVE_SESSION_REORDER_BUFFER);
+  if (flushCount <= 0) return [];
+  return state.live_session_buffer.splice(0, flushCount);
+}
+
+function flushBufferedLiveSessions(state) {
+  if (!state.live_session_buffer.length) return [];
+  state.live_session_buffer.sort(compareLiveSessionOrder);
+  return state.live_session_buffer.splice(0, state.live_session_buffer.length);
+}
+
 function queueLiveProgress(state) {
   state.live_progress_dirty = true;
   ensureLivePump(state);
@@ -385,13 +452,14 @@ function queueLivePatch(state, patch) {
 }
 
 function ensureLivePump(state) {
-  if (state.live_pump_timer || !state.live_subscribers.size) return;
+  if (state.live_pump_timer) return;
+  if (!state.live_subscribers.size && !state.replay_capture.active) return;
   state.live_pump_timer = setInterval(() => flushLive(state), LIVE_FRAME_INTERVAL_MS);
 }
 
 function stopLivePumpIfIdle(state) {
   if (!state.live_pump_timer) return;
-  if (state.live_subscribers.size) return;
+  if (state.live_subscribers.size || state.replay_capture.active) return;
   clearInterval(state.live_pump_timer);
   state.live_pump_timer = null;
   finalizeWithoutSubscribers(state);
@@ -399,6 +467,7 @@ function stopLivePumpIfIdle(state) {
 
 function finalizeWithoutSubscribers(state) {
   if (state.live_subscribers.size) return;
+  if (state.replay_capture.active) return;
   if (!state.presentation_complete_pending) return;
   state.phase = 'complete';
   state.percent = 1;
@@ -409,7 +478,7 @@ function finalizeWithoutSubscribers(state) {
 }
 
 function flushLive(state, forcedEvent = null) {
-  if (!state.live_subscribers.size) {
+  if (!state.live_subscribers.size && !state.replay_capture.active) {
     state.live_pending_patch = createEmptyLivePatch();
     state.live_progress_dirty = false;
     stopLivePumpIfIdle(state);
@@ -449,12 +518,16 @@ function flushLive(state, forcedEvent = null) {
     payload.data = buildLiveBootstrap(state.live_state);
   }
 
-  broadcastLive(state, event, payload);
+  recordReplayEvent(state.replay_capture, event, payload);
+  if (state.live_subscribers.size) {
+    broadcastLive(state, event, payload);
+  }
   state.live_last_emit_at = Date.now();
   state.live_progress_dirty = shouldEmitComplete ? false : (event === 'progress' ? false : state.live_progress_dirty);
   if (forcedEvent) {
     state.live_pending_patch = createEmptyLivePatch();
   }
+  stopLivePumpIfIdle(state);
 }
 
 function broadcastLive(state, event, payload) {
@@ -496,41 +569,48 @@ function takeFlushablePatch(state) {
   const now = Date.now();
   const sent = createEmptyLivePatch();
 
-  if (state.live_pending_patch.overview.size > 0 && readyForSurface(state, 'overview', now)) {
-    moveSet(state.live_pending_patch.overview, sent.overview);
-    state.live_last_surface_emit_at.overview = now;
-  }
-
-  const rankingsDirty =
+  const overviewDirty =
+    state.live_pending_patch.overview.size > 0 ||
     state.live_pending_patch.repos.total.size > 0 || state.live_pending_patch.repos.d7.size > 0 || state.live_pending_patch.repos.d30.size > 0 ||
     state.live_pending_patch.models.total.size > 0 || state.live_pending_patch.models.d7.size > 0 || state.live_pending_patch.models.d30.size > 0 ||
-    state.live_pending_patch.families.total.size > 0 || state.live_pending_patch.families.d7.size > 0 || state.live_pending_patch.families.d30.size > 0;
+    state.live_pending_patch.families.total.size > 0 || state.live_pending_patch.families.d7.size > 0 || state.live_pending_patch.families.d30.size > 0 ||
+    state.live_pending_patch.daily.size > 0 || state.live_pending_patch.heatmap.size > 0;
 
-  if (rankingsDirty && readyForSurface(state, 'rankings', now)) {
-    moveRangeSets(state.live_pending_patch.repos, sent.repos);
-    moveRangeSets(state.live_pending_patch.models, sent.models);
-    moveRangeSets(state.live_pending_patch.families, sent.families);
-    state.live_last_surface_emit_at.rankings = now;
+  if (!overviewDirty || !readyForOverview(state, now)) {
+    return sent;
   }
 
-  const dayDirty = state.live_pending_patch.daily.size > 0 || state.live_pending_patch.heatmap.size > 0;
-  if (dayDirty && readyForSurface(state, 'daily', now)) {
-    const nextDayKeys = takeNextChronologicalDayKeys(
-      state.live_pending_patch.daily,
-      state.live_pending_patch.heatmap,
-      LIVE_DAY_KEYS_PER_EMIT
-    );
-    moveSpecificKeys(state.live_pending_patch.daily, sent.daily, nextDayKeys);
-    moveSpecificKeys(state.live_pending_patch.heatmap, sent.heatmap, nextDayKeys);
-    state.live_last_surface_emit_at.daily = now;
-    state.live_last_surface_emit_at.heatmap = now;
-  }
+  moveSet(state.live_pending_patch.overview, sent.overview);
+  moveRangeSets(state.live_pending_patch.repos, sent.repos);
+  moveRangeSets(state.live_pending_patch.models, sent.models);
+  moveRangeSets(state.live_pending_patch.families, sent.families);
+
+  const nextDayKeys = takeNextChronologicalDayKeys(
+    state.live_pending_patch.daily,
+    state.live_pending_patch.heatmap,
+    LIVE_DAY_KEYS_PER_EMIT
+  );
+  moveSpecificKeys(state.live_pending_patch.daily, sent.daily, nextDayKeys);
+  moveSpecificKeys(state.live_pending_patch.heatmap, sent.heatmap, nextDayKeys);
+  state.live_last_overview_emit_at = now;
 
   return sent;
 }
 
-function readyForSurface(state, surfaceKey, now) {
-  return (now - state.live_last_surface_emit_at[surfaceKey]) >= LIVE_SURFACE_CADENCE_MS[surfaceKey];
+function readyForOverview(state, now) {
+  return (now - state.live_last_overview_emit_at) >= getOverviewCadenceMs(state);
+}
+
+function getOverviewCadenceMs(state) {
+  const tail = OVERVIEW_INGEST_ANIMATION.tail;
+  const tailStartPercent = Math.min(Math.max(tail?.startPercent ?? 0.95, 0), 0.999);
+  const tailHz = Number.isFinite(tail?.overviewHz) && tail.overviewHz > 0 ? tail.overviewHz : 5;
+  const tailCadenceMs = Math.round(1000 / tailHz);
+
+  if (tail?.enabled && (state.presentation_complete_pending || (state.percent || 0) >= tailStartPercent)) {
+    return tailCadenceMs;
+  }
+  return LIVE_OVERVIEW_CADENCE_MS;
 }
 
 function moveSet(from, to) {
@@ -602,6 +682,10 @@ export function detachLiveSubscriber(state, res) {
   stopLivePumpIfIdle(state);
 }
 
+export function getLatestReplay(state) {
+  return getReplaySnapshot(state.replay_capture);
+}
+
 function broadcastBootstrap(state) {
   if (!state.live_subscribers.size) return;
   const payload = {

package/server/live-state.js

@@ -82,7 +82,7 @@ export function buildLiveBootstrap(live) {
 export function buildLivePatch(live, patch) {
   return {
     overview: Object.fromEntries(
-      [...patch.overview].map((rangeKey) => [rangeKey, serializeOverviewBucket(live.overview[rangeKey])])
+      [...patch.overview].map((rangeKey) => [rangeKey, serializeOverviewBucket(live.overview[rangeKey], live.lowerBounds[rangeKey])])
     ),
     repos: serializePatchedTopRanges(live.repos, live.repoTopKeys, patch.repos, serializeRepoSummary),
     models: serializePatchedTopRanges(live.models, live.modelTopKeys, patch.models, serializeModelSummary),
@@ -352,14 +352,17 @@ function ensureHeatmapDay(dayMap, dayKey) {
 
 function serializeOverview(live) {
   return {
-    total: serializeOverviewBucket(live.overview.total),
-    d7: serializeOverviewBucket(live.overview.d7),
-    d30: serializeOverviewBucket(live.overview.d30),
+    total: serializeOverviewBucket(live.overview.total, live.lowerBounds.total),
+    d7: serializeOverviewBucket(live.overview.d7, live.lowerBounds.d7),
+    d30: serializeOverviewBucket(live.overview.d30, live.lowerBounds.d30),
     cost_assumptions: CACHE_ASSUMPTIONS,
   };
 }
 
-function serializeOverviewBucket(bucket) {
+function serializeOverviewBucket(bucket, lowerBound = 0) {
+  const boundedFrom = bucket.earliest === Infinity
+    ? null
+    : Math.max(bucket.earliest, lowerBound || 0);
   return {
     total_tokens: bucket.total_tokens,
     total_cost: bucket.total_cost,
@@ -368,7 +371,7 @@ function serializeOverviewBucket(bucket) {
     active_models: bucket.modelSet.size,
     total_elapsed_seconds: bucket.total_elapsed_seconds,
     date_range: {
-      from: bucket.earliest === Infinity ? null : bucket.earliest,
+      from: boundedFrom,
       to: bucket.latest === -Infinity ? null : bucket.latest,
     },
     coverage: {

package/server/rollout-reader.js

@@ -15,6 +15,7 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
     model_name: null,
     reasoning_effort: null,
     first_timestamp: null,
+    first_usage_timestamp: null,
     last_timestamp: null,
     active_seconds: null,
     active_by_day: null,
@@ -97,6 +98,9 @@ export async function enrichFromRollout(rolloutPath, opts = {}) {
             if (obj.timestamp && hasUsage(usageDelta)) {
               const ts = new Date(obj.timestamp).getTime();
               if (!isNaN(ts)) {
+                if (!result.first_usage_timestamp || ts < result.first_usage_timestamp) {
+                  result.first_usage_timestamp = ts;
+                }
                 const dayKey = toDayKey(ts);
                 mergeUsageTotals(usageByDay, dayKey, usageDelta);
               }