@kbediako/codex-orchestrator 0.1.3 → 0.1.4

package/dist/orchestrator/src/cli/metrics/metricsAggregator.js

@@ -1,6 +1,10 @@
-import { appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { appendFile, mkdir, open, readFile, readdir, rename, rm, stat } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
+import { createInterface } from 'node:readline';
 import { acquireLockWithRetry } from '../../persistence/lockFile.js';
+import { EnvUtils } from '../../../../packages/shared/config/index.js';
+import { writeJsonAtomic } from '../utils/fs.js';
 const REQUIRED_COMPLETENESS_FIELDS = [
     'instance_stats',
     'privacy_events',
@@ -9,6 +13,8 @@ const REQUIRED_COMPLETENESS_FIELDS = [
 const METRICS_LOCK_FILENAME = 'metrics.lock';
 const METRICS_PENDING_DIRNAME = 'metrics.pending';
 const DEFAULT_LOCK_STALE_MS = 5 * 60 * 1000;
+const DEFAULT_PENDING_BATCH_MAX_LINES = 500;
+const DEFAULT_PENDING_BATCH_MAX_BYTES = 1024 * 1024;
 const DEFAULT_LOCK_RETRY = {
     maxAttempts: 4,
     initialDelayMs: 50,
@@ -55,10 +61,20 @@ class MetricsLockError extends Error {
         this.name = 'MetricsLockError';
     }
 }
-async function drainMetricsEntryFile(env, path) {
-    let raw = '';
+async function streamMetricsEntryLines(path, onLine) {
+    let count = 0;
+    let reader;
+    let stream;
     try {
-        raw = await readFile(path, 'utf8');
+        stream = createReadStream(path, { encoding: 'utf8' });
+        reader = createInterface({ input: stream, crlfDelay: Infinity });
+        for await (const line of reader) {
+            if (line.trim().length === 0) {
+                continue;
+            }
+            count += 1;
+            await onLine(line);
+        }
     }
     catch (error) {
         if (error.code === 'ENOENT') {
@@ -66,21 +82,104 @@ async function drainMetricsEntryFile(env, path) {
         }
         throw error;
     }
-    const lines = raw.trim().split('\n').filter(Boolean);
-    if (lines.length === 0) {
-        await rm(path, { force: true });
-        return 0;
+    finally {
+        reader?.close();
+        stream?.destroy();
+    }
+    return count;
+}
+function isMetricsEntryCandidate(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return false;
+    }
+    const candidate = value;
+    if (typeof candidate.run_id !== 'string' || candidate.run_id.trim().length === 0) {
+        return false;
+    }
+    if (typeof candidate.status !== 'string' || candidate.status.trim().length === 0) {
+        return false;
+    }
+    if (typeof candidate.recorded_at !== 'string' || candidate.recorded_at.trim().length === 0) {
+        return false;
+    }
+    return true;
+}
+function parseMetricsEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(trimmed);
+        return isMetricsEntryCandidate(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+async function promotePendingTmpFile(tmpPath, pendingDir) {
+    let lineCount = 0;
+    let invalid = false;
+    try {
+        await streamMetricsEntryLines(tmpPath, async (line) => {
+            lineCount += 1;
+            if (!parseMetricsEntry(line)) {
+                invalid = true;
+                throw new Error('invalid metrics entry');
+            }
+        });
+    }
+    catch (error) {
+        if (invalid) {
+            return false;
+        }
+        if (error.code === 'ENOENT') {
+            return false;
+        }
+        throw error;
+    }
+    if (lineCount === 0) {
+        return false;
+    }
+    const targetPath = tmpPath.replace(/\.tmp$/, '');
+    try {
+        await rename(tmpPath, targetPath);
+        return true;
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === 'ENOENT') {
+            return false;
+        }
+        if (code === 'EEXIST') {
+            const recoveryPath = join(pendingDir, `recovered-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`);
+            await rename(tmpPath, recoveryPath);
+            return true;
+        }
+        throw error;
+    }
+}
+function normalizeBatchLimit(value) {
+    if (!Number.isFinite(value) || value <= 0) {
+        return Number.POSITIVE_INFINITY;
     }
-    await mkdir(getMetricsRoot(env), { recursive: true });
-    const payload = `${lines.join('\n')}\n`;
-    await appendFile(getMetricsPath(env), payload, 'utf8');
-    await rm(path, { force: true });
-    return lines.length;
+    return value;
+}
+function getPendingBatchLimits() {
+    const maxLines = EnvUtils.getInt('CODEX_METRICS_PENDING_BATCH_MAX_LINES', DEFAULT_PENDING_BATCH_MAX_LINES);
+    const maxBytes = EnvUtils.getInt('CODEX_METRICS_PENDING_BATCH_MAX_BYTES', DEFAULT_PENDING_BATCH_MAX_BYTES);
+    return {
+        maxLines: normalizeBatchLimit(maxLines),
+        maxBytes: normalizeBatchLimit(maxBytes)
+    };
 }
 export async function mergePendingMetricsEntries(env) {
     const pendingDir = getMetricsPendingDir(env);
+    const metricsRoot = getMetricsRoot(env);
+    const metricsPath = getMetricsPath(env);
     let merged = 0;
     const staleTmpMs = DEFAULT_LOCK_STALE_MS;
+    const { maxLines: maxBatchLines, maxBytes: maxBatchBytes } = getPendingBatchLimits();
     for (let pass = 0; pass < 2; pass += 1) {
         let entries = [];
         try {
@@ -93,6 +192,7 @@ export async function mergePendingMetricsEntries(env) {
             throw error;
         }
         const now = Date.now();
+        let promotedTmp = false;
         for (const entry of entries) {
             if (!entry.isFile() || !entry.name.endsWith('.tmp')) {
                 continue;
@@ -101,7 +201,13 @@ export async function mergePendingMetricsEntries(env) {
             try {
                 const stats = await stat(tmpPath);
                 if (now - stats.mtimeMs > staleTmpMs) {
-                    await rm(tmpPath, { force: true });
+                    const promoted = await promotePendingTmpFile(tmpPath, pendingDir);
+                    if (promoted) {
+                        promotedTmp = true;
+                    }
+                    if (!promoted) {
+                        await rm(tmpPath, { force: true });
+                    }
                 }
             }
             catch (error) {
@@ -115,14 +221,79 @@ export async function mergePendingMetricsEntries(env) {
             .map((entry) => entry.name)
             .sort();
         if (files.length === 0) {
-            break;
+            if (!promotedTmp) {
+                break;
+            }
+            continue;
         }
+        await mkdir(metricsRoot, { recursive: true });
+        let payloadLines = [];
+        let filesToRemove = [];
+        let payloadLineCount = 0;
+        let payloadBytes = 0;
+        const flushBatch = async () => {
+            if (payloadLines.length > 0) {
+                const payload = `${payloadLines.join('\n')}\n`;
+                await ensureMetricsTrailingNewline(metricsPath);
+                await appendFile(metricsPath, payload, 'utf8');
+            }
+            if (filesToRemove.length > 0) {
+                await Promise.all(filesToRemove.map((filePath) => rm(filePath, { force: true })));
+            }
+            payloadLines = [];
+            filesToRemove = [];
+            payloadLineCount = 0;
+            payloadBytes = 0;
+        };
         for (const file of files) {
-            merged += await drainMetricsEntryFile(env, join(pendingDir, file));
+            const filePath = join(pendingDir, file);
+            const fileLineCount = await streamMetricsEntryLines(filePath, async (line) => {
+                const lineBytes = Buffer.byteLength(line, 'utf8') + 1;
+                const wouldExceedLines = payloadLineCount + 1 > maxBatchLines;
+                const wouldExceedBytes = payloadBytes + lineBytes > maxBatchBytes;
+                if (payloadLines.length > 0 && (wouldExceedLines || wouldExceedBytes)) {
+                    await flushBatch();
+                }
+                payloadLines.push(line);
+                payloadLineCount += 1;
+                payloadBytes += lineBytes;
+                merged += 1;
+            });
+            if (fileLineCount === 0) {
+                await rm(filePath, { force: true });
+                continue;
+            }
+            filesToRemove.push(filePath);
         }
+        await flushBatch();
     }
     return merged;
 }
+export async function ensureMetricsTrailingNewline(path) {
+    try {
+        const handle = await open(path, 'r');
+        try {
+            const stats = await handle.stat();
+            if (stats.size === 0) {
+                return;
+            }
+            const buffer = Buffer.alloc(1);
+            await handle.read(buffer, 0, 1, stats.size - 1);
+            if (buffer[0] !== 0x0a) {
+                await appendFile(path, '\n', 'utf8');
+            }
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return;
+        }
+        throw error;
+    }
+}
 export async function withMetricsLock(env, action, options = {}) {
     const overrides = options.retry ?? {};
     const sanitizedOverrides = Object.fromEntries(Object.entries(overrides).filter(([, value]) => value !== undefined));
@@ -158,19 +329,30 @@ export async function withMetricsLock(env, action, options = {}) {
 export async function updateMetricsAggregates(env) {
     const metricsRoot = getMetricsRoot(env);
     const metricsPath = getMetricsPath(env);
-    const entries = await loadMetricsEntries(metricsPath);
-    if (entries.length === 0) {
+    const state = createAggregationState();
+    await streamMetricsEntryLines(metricsPath, async (line) => {
+        const entry = parseMetricsEntry(line);
+        if (!entry) {
+            return;
+        }
+        if (state.seenRunIds.has(entry.run_id)) {
+            return;
+        }
+        state.seenRunIds.add(entry.run_id);
+        accumulateMetricsEntry(state, entry);
+    });
+    if (state.totalRuns === 0 || !state.baselineEntry) {
         return;
     }
     const metricsDir = join(metricsRoot, 'metrics');
     await mkdir(metricsDir, { recursive: true });
+    await ensureBaseline(metricsDir, state.baselineEntry);
     await Promise.all([
-        ensureBaseline(metricsDir, entries[0]),
-        writePostRollout(metricsDir, entries),
-        writeCompleteness(metricsDir, entries),
-        writeMttrDelta(env, entries),
-        writeTfgrpoEpochAggregates(metricsDir, entries),
-        writeLearningState(env, entries)
+        writePostRollout(metricsDir, state),
+        writeCompleteness(metricsDir, state),
+        writeMttrDelta(env, state),
+        writeTfgrpoEpochAggregates(metricsDir, state),
+        writeLearningState(env, state)
     ]);
 }
 async function ensureBaseline(dir, entry) {
@@ -191,11 +373,11 @@ async function ensureBaseline(dir, entry) {
         duration_seconds: entry.duration_seconds,
         completion_rate: entry.status === 'succeeded' ? 1 : 0
     };
-    await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, 'utf8');
+    await writeJsonAtomic(baselinePath, baseline);
 }
-async function writePostRollout(dir, entries) {
-    const totalRuns = entries.length;
-    const succeededRuns = entries.filter((entry) => entry.status === 'succeeded').length;
+async function writePostRollout(dir, state) {
+    const totalRuns = state.totalRuns;
+    const succeededRuns = state.succeededRuns;
     const completionRate = totalRuns > 0 ? succeededRuns / totalRuns : 0;
     const payload = {
         total_runs: totalRuns,
@@ -204,44 +386,29 @@ async function writePostRollout(dir, entries) {
         meets_threshold: completionRate >= 0.95,
         updated_at: new Date().toISOString()
     };
-    await writeFile(join(dir, 'post-rollout.json'), `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    await writeJsonAtomic(join(dir, 'post-rollout.json'), payload);
 }
-async function writeCompleteness(dir, entries) {
-    const fieldChecks = REQUIRED_COMPLETENESS_FIELDS.length * entries.length;
+async function writeCompleteness(dir, state) {
+    const fieldChecks = REQUIRED_COMPLETENESS_FIELDS.length * state.totalRuns;
     if (fieldChecks === 0) {
         return;
     }
-    const missingCounts = Object.fromEntries(REQUIRED_COMPLETENESS_FIELDS.map((field) => [field, 0]));
-    for (const entry of entries) {
-        if (!Array.isArray(entry.instance_stats) || entry.instance_stats.length === 0) {
-            missingCounts.instance_stats += 1;
-        }
-        if (!Array.isArray(entry.privacy_events) || entry.privacy_events.length === 0) {
-            missingCounts.privacy_events += 1;
-        }
-        if (!entry.control_plane_status || entry.control_plane_status === 'unknown') {
-            missingCounts.control_plane_status += 1;
-        }
-    }
-    const totalMissing = Object.values(missingCounts).reduce((sum, value) => sum + value, 0);
+    const totalMissing = Object.values(state.missingCounts).reduce((sum, value) => sum + value, 0);
     const ratio = totalMissing / fieldChecks;
     const payload = {
         checked_fields: REQUIRED_COMPLETENESS_FIELDS,
-        missing_counts: missingCounts,
+        missing_counts: state.missingCounts,
         missing_field_ratio: ratio,
         meets_threshold: ratio < 0.05,
         updated_at: new Date().toISOString()
     };
-    await writeFile(join(dir, 'completeness.json'), `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    await writeJsonAtomic(join(dir, 'completeness.json'), payload);
 }
-async function writeMttrDelta(env, entries) {
-    const durations = entries
-        .map((entry) => entry.duration_seconds)
-        .filter((value) => typeof value === 'number' && Number.isFinite(value));
-    if (durations.length === 0) {
+async function writeMttrDelta(env, state) {
+    if (state.durationCount === 0) {
         return;
     }
-    const currentMttr = average(durations);
+    const currentMttr = state.durationSum / state.durationCount;
     const metricsDir = join(env.runsRoot, env.taskId, 'metrics');
     const baselinePath = join(metricsDir, 'baseline.json');
     let baselineMttr = currentMttr;
@@ -265,139 +432,202 @@ async function writeMttrDelta(env, entries) {
     };
     const outDir = join(env.outRoot, env.taskId, 'metrics');
     await mkdir(outDir, { recursive: true });
-    await writeFile(join(outDir, 'mttr-delta.json'), `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    await writeJsonAtomic(join(outDir, 'mttr-delta.json'), payload);
 }
-async function writeTfgrpoEpochAggregates(dir, entries) {
-    const grouped = new Map();
-    for (const entry of entries) {
-        if (typeof entry.tfgrpo_epoch !== 'number') {
-            continue;
-        }
-        const bucket = grouped.get(entry.tfgrpo_epoch) ?? [];
-        bucket.push(entry);
-        grouped.set(entry.tfgrpo_epoch, bucket);
-    }
-    if (grouped.size === 0) {
+async function writeTfgrpoEpochAggregates(dir, state) {
+    if (state.epochs.size === 0) {
         return;
     }
-    const epochs = Array.from(grouped.entries())
+    const epochs = Array.from(state.epochs.entries())
         .sort(([a], [b]) => a - b)
-        .map(([epoch, group]) => summarizeEpoch(epoch, group));
+        .map(([epoch, aggregate]) => ({
+        epoch,
+        runs: aggregate.runs,
+        tool_calls: aggregate.tool_calls,
+        token_total: aggregate.token_total,
+        cost_usd: roundCurrency(aggregate.cost_usd),
+        latency_ms: aggregate.latency_ms,
+        group_size_avg: aggregate.group_size_count > 0
+            ? aggregate.group_size_sum / aggregate.group_size_count
+            : null,
+        tools: Array.from(aggregate.tool_stats.entries()).map(([tool, toolAggregate]) => ({
+            tool,
+            runs: toolAggregate.runs,
+            tokens: toolAggregate.tokens,
+            cost_usd: roundCurrency(toolAggregate.costUsd),
+            latency_ms: toolAggregate.latencyMs
+        }))
+    }));
     const payload = {
         epochs,
         updated_at: new Date().toISOString()
     };
-    await writeFile(join(dir, 'per-epoch.json'), `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    await writeJsonAtomic(join(dir, 'per-epoch.json'), payload);
 }
-async function writeLearningState(env, entries) {
-    const validationStatuses = entries
-        .map((entry) => entry.learning_validation_status)
-        .filter((value) => typeof value === 'string');
-    const validationSummary = {
-        passed: validationStatuses.filter((status) => status === 'validated').length,
-        failed: validationStatuses.filter((status) => status === 'snapshot_failed').length,
-        stalled: validationStatuses.filter((status) => status === 'stalled_snapshot').length,
-        manual: validationStatuses.filter((status) => status === 'needs_manual_scenario').length
-    };
-    const reviewerRejections = entries.reduce((sum, entry) => sum + (entry.learning_review_rejections ?? 0), 0);
-    const reviewerLatencies = entries
-        .map((entry) => entry.learning_review_latency_ms)
-        .filter((value) => typeof value === 'number' && Number.isFinite(value));
-    const reviewerLatencyMs = reviewerLatencies.length > 0
-        ? reviewerLatencies.reduce((sum, value) => sum + value, 0) / reviewerLatencies.length
+async function writeLearningState(env, state) {
+    const reviewerLatencyMs = state.reviewerLatencyCount > 0
+        ? state.reviewerLatencySum / state.reviewerLatencyCount
         : null;
-    const regressions = entries.reduce((sum, entry) => sum + (entry.learning_regressions_detected ?? 0), 0);
-    const patternPromotions = entries.reduce((sum, entry) => sum + (entry.learning_pattern_promoted ?? 0), 0);
-    const patternDeprecations = entries.reduce((sum, entry) => sum + (entry.learning_pattern_deprecated ?? 0), 0);
-    const throughputCandidates = entries.reduce((sum, entry) => sum + (entry.learning_throughput_candidates ?? 0), 0);
     const alerts = {
-        total: entries.reduce((sum, entry) => sum + (entry.learning_alerts ?? 0), 0),
-        snapshot_failed: entries.filter((entry) => entry.learning_snapshot_status === 'snapshot_failed').length,
-        stalled_snapshot: entries.filter((entry) => entry.learning_snapshot_status === 'stalled_snapshot').length,
-        needs_manual_scenario: validationSummary.manual
+        total: state.alertsTotal,
+        snapshot_failed: state.alertsSnapshotFailed,
+        stalled_snapshot: state.alertsStalledSnapshot,
+        needs_manual_scenario: state.validationSummary.manual
     };
     const payload = {
         updated_at: new Date().toISOString(),
         safety: {
-            validation: validationSummary,
-            reviewer: { rejections: reviewerRejections, average_latency_ms: reviewerLatencyMs },
-            regression_detection: { detected: regressions },
-            pattern_hygiene: { promoted: patternPromotions, deprecated: patternDeprecations }
+            validation: state.validationSummary,
+            reviewer: { rejections: state.reviewerRejections, average_latency_ms: reviewerLatencyMs },
+            regression_detection: { detected: state.regressions },
+            pattern_hygiene: { promoted: state.patternPromotions, deprecated: state.patternDeprecations }
         },
-        throughput: { candidates: throughputCandidates },
+        throughput: { candidates: state.throughputCandidates },
         alerts
     };
     const outDir = join(env.outRoot, env.taskId);
     await mkdir(outDir, { recursive: true });
-    await writeFile(join(outDir, 'state.json'), `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    await writeJsonAtomic(join(outDir, 'state.json'), payload);
 }
-function summarizeEpoch(epoch, entries) {
-    const runs = entries.length;
-    const toolCalls = entries.reduce((sum, entry) => sum + (entry.tool_calls ?? 0), 0);
-    const tokenTotal = entries.reduce((sum, entry) => sum + (entry.token_total ?? 0), 0);
-    const costUsd = roundCurrency(entries.reduce((sum, entry) => sum + (entry.cost_usd ?? 0), 0));
-    const latencyMs = entries.reduce((sum, entry) => sum + (entry.latency_ms ?? 0), 0);
-    const groupSizes = entries
-        .map((entry) => entry.tfgrpo_group_size)
-        .filter((value) => typeof value === 'number');
-    const groupSizeAvg = groupSizes.length > 0 ? groupSizes.reduce((sum, value) => sum + value, 0) / groupSizes.length : null;
+function createAggregationState() {
+    const missingCounts = Object.fromEntries(REQUIRED_COMPLETENESS_FIELDS.map((field) => [field, 0]));
     return {
-        epoch,
-        runs,
-        tool_calls: toolCalls,
-        token_total: tokenTotal,
-        cost_usd: costUsd,
-        latency_ms: latencyMs,
-        group_size_avg: groupSizeAvg,
-        tools: aggregateToolStats(entries)
+        totalRuns: 0,
+        succeededRuns: 0,
+        baselineEntry: null,
+        seenRunIds: new Set(),
+        missingCounts,
+        durationSum: 0,
+        durationCount: 0,
+        epochs: new Map(),
+        validationSummary: {
+            passed: 0,
+            failed: 0,
+            stalled: 0,
+            manual: 0
+        },
+        reviewerRejections: 0,
+        reviewerLatencySum: 0,
+        reviewerLatencyCount: 0,
+        regressions: 0,
+        patternPromotions: 0,
+        patternDeprecations: 0,
+        throughputCandidates: 0,
+        alertsTotal: 0,
+        alertsSnapshotFailed: 0,
+        alertsStalledSnapshot: 0
     };
 }
-function aggregateToolStats(entries) {
-    const aggregates = new Map();
-    for (const entry of entries) {
-        const stats = entry.tool_stats ?? [];
+function accumulateMetricsEntry(state, entry) {
+    state.totalRuns += 1;
+    if (!state.baselineEntry) {
+        state.baselineEntry = entry;
+    }
+    if (entry.status === 'succeeded') {
+        state.succeededRuns += 1;
+    }
+    if (!Array.isArray(entry.instance_stats) || entry.instance_stats.length === 0) {
+        state.missingCounts.instance_stats += 1;
+    }
+    if (!Array.isArray(entry.privacy_events) || entry.privacy_events.length === 0) {
+        state.missingCounts.privacy_events += 1;
+    }
+    if (!entry.control_plane_status || entry.control_plane_status === 'unknown') {
+        state.missingCounts.control_plane_status += 1;
+    }
+    if (typeof entry.duration_seconds === 'number' && Number.isFinite(entry.duration_seconds)) {
+        state.durationSum += entry.duration_seconds;
+        state.durationCount += 1;
+    }
+    if (typeof entry.tfgrpo_epoch === 'number') {
+        const aggregate = getEpochAggregate(state.epochs, entry.tfgrpo_epoch);
+        aggregate.runs += 1;
+        aggregate.tool_calls += typeof entry.tool_calls === 'number' ? entry.tool_calls : 0;
+        aggregate.token_total += typeof entry.token_total === 'number' ? entry.token_total : 0;
+        aggregate.cost_usd += typeof entry.cost_usd === 'number' ? entry.cost_usd : 0;
+        aggregate.latency_ms += typeof entry.latency_ms === 'number' ? entry.latency_ms : 0;
+        if (typeof entry.tfgrpo_group_size === 'number' && Number.isFinite(entry.tfgrpo_group_size)) {
+            aggregate.group_size_sum += entry.tfgrpo_group_size;
+            aggregate.group_size_count += 1;
+        }
+        const stats = Array.isArray(entry.tool_stats) ? entry.tool_stats : [];
         for (const stat of stats) {
             if (typeof stat.tool !== 'string' || !stat.tool) {
                 continue;
             }
-            const current = aggregates.get(stat.tool) ?? { runs: 0, tokens: 0, costUsd: 0, latencyMs: 0 };
+            const current = aggregate.tool_stats.get(stat.tool) ?? { runs: 0, tokens: 0, costUsd: 0, latencyMs: 0 };
             current.runs += 1;
             current.tokens += typeof stat.tokens === 'number' ? stat.tokens : 0;
             current.costUsd += typeof stat.cost_usd === 'number' ? stat.cost_usd : 0;
             current.latencyMs += typeof stat.latency_ms === 'number' ? stat.latency_ms : 0;
-            aggregates.set(stat.tool, current);
+            aggregate.tool_stats.set(stat.tool, current);
         }
     }
-    return Array.from(aggregates.entries()).map(([tool, aggregate]) => ({
-        tool,
-        runs: aggregate.runs,
-        tokens: aggregate.tokens,
-        cost_usd: roundCurrency(aggregate.costUsd),
-        latency_ms: aggregate.latencyMs
-    }));
-}
-async function loadMetricsEntries(path) {
-    try {
-        const raw = await readFile(path, 'utf8');
-        return raw
-            .trim()
-            .split('\n')
-            .filter(Boolean)
-            .map((line) => JSON.parse(line));
-    }
-    catch (error) {
-        if (error.code === 'ENOENT') {
-            return [];
+    if (typeof entry.learning_validation_status === 'string') {
+        switch (entry.learning_validation_status) {
+            case 'validated':
+                state.validationSummary.passed += 1;
+                break;
+            case 'snapshot_failed':
+                state.validationSummary.failed += 1;
+                break;
+            case 'stalled_snapshot':
+                state.validationSummary.stalled += 1;
+                break;
+            case 'needs_manual_scenario':
+                state.validationSummary.manual += 1;
+                break;
+            default:
+                break;
         }
-        throw error;
+    }
+    if (typeof entry.learning_review_rejections === 'number') {
+        state.reviewerRejections += entry.learning_review_rejections;
+    }
+    if (typeof entry.learning_review_latency_ms === 'number' &&
+        Number.isFinite(entry.learning_review_latency_ms)) {
+        state.reviewerLatencySum += entry.learning_review_latency_ms;
+        state.reviewerLatencyCount += 1;
+    }
+    if (typeof entry.learning_regressions_detected === 'number') {
+        state.regressions += entry.learning_regressions_detected;
+    }
+    if (typeof entry.learning_pattern_promoted === 'number') {
+        state.patternPromotions += entry.learning_pattern_promoted;
+    }
+    if (typeof entry.learning_pattern_deprecated === 'number') {
+        state.patternDeprecations += entry.learning_pattern_deprecated;
+    }
+    if (typeof entry.learning_throughput_candidates === 'number') {
+        state.throughputCandidates += entry.learning_throughput_candidates;
+    }
+    if (typeof entry.learning_alerts === 'number') {
+        state.alertsTotal += entry.learning_alerts;
+    }
+    if (entry.learning_snapshot_status === 'snapshot_failed') {
+        state.alertsSnapshotFailed += 1;
+    }
+    if (entry.learning_snapshot_status === 'stalled_snapshot') {
+        state.alertsStalledSnapshot += 1;
     }
 }
-function average(values) {
-    if (values.length === 0) {
-        return 0;
+function getEpochAggregate(epochs, epoch) {
+    const existing = epochs.get(epoch);
+    if (existing) {
+        return existing;
     }
-    const sum = values.reduce((total, value) => total + value, 0);
-    return sum / values.length;
+    const created = {
+        runs: 0,
+        tool_calls: 0,
+        token_total: 0,
+        cost_usd: 0,
+        latency_ms: 0,
+        group_size_sum: 0,
+        group_size_count: 0,
+        tool_stats: new Map()
+    };
+    epochs.set(epoch, created);
+    return created;
 }
 function roundCurrency(value) {
     return Math.round(value * 1_000_000) / 1_000_000;