kiku-agent 1.3.1 → 1.3.3

package/server/agent/eval/modules/evaluation/runner.js

@@ -0,0 +1,288 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { applyGates } from '../analysis/gates.js';
+import { judgeReport } from '../analysis/judge.js';
+import { DEFAULTS, REPO_ROOT } from '../core/config.js';
+import {
+  acquireRunLock,
+  assertResumeCompatible,
+  buildMetadata,
+  createSaver,
+  finalizeMetadata,
+  loadReport,
+  loadReportWithBackups,
+  rotateBackups,
+  verifyCorpusFreshness,
+  writeBlockedReport,
+} from '../core/store.js';
+import { writeDashboardReport } from '../reporting/dashboard.js';
+import {
+  allCases,
+  cases,
+  expandedCases,
+  legacyCases,
+  securityCases,
+  validateCases,
+} from './cases.js';
+
+const cliEntry = fileURLToPath(new URL('../core/cli.js', import.meta.url));
+
+export function selectCases({ suite = DEFAULTS.suite, ids, limit } = {}) {
+  const full = [...new Set([...allCases, ...(expandedCases || [])])];
+  const suiteMap = {
+    catalog: cases,
+    expanded: expandedCases || [],
+    security: securityCases,
+    legacy: legacyCases,
+    all: full,
+    full,
+  };
+  const idList = Array.isArray(ids)
+    ? ids.map(s => String(s).trim()).filter(Boolean)
+    : (typeof ids === 'string' ? ids.split(',').map(s => s.trim()).filter(Boolean) : null);
+  const corpus = (idList && idList.length > 0) ? full : (suiteMap[suite] || full);
+  if (idList?.some(id => !corpus.some(c => c.id === id))) throw new Error('Unknown case id');
+  const count = limit === undefined ? corpus.length : Number(limit);
+  if (!Number.isInteger(count) || count < 1) throw new Error('limit must be a positive integer');
+  return corpus.filter(c => !idList || idList.includes(c.id)).slice(0, count);
+}
+
+export async function probeModel({ baseUrl, model }) {
+  const isLocal = baseUrl.includes('127.0.0.1') || baseUrl.includes('localhost');
+  if (isLocal) {
+    const { wakeUpLmStudio } = await import('../../../lmStudioWakeup.js');
+    await wakeUpLmStudio({ baseUrl, model });
+  }
+  const response = await fetch(`${baseUrl.replace(/\/$/, '')}/models`, { signal: AbortSignal.timeout(8000) });
+  if (!response.ok) throw new Error(`HTTP ${response.status}`);
+  const models = await response.json();
+  if (!models.data?.some(m => m.id === model)) throw new Error(`Model not advertised: ${model}`);
+}
+
+function spawnWorker(caseId, { model, baseUrl, timeoutMs, workerGraceMs, signal, onStep }) {
+  return new Promise(resolve => {
+    const template = allCases.find(c => c.id === caseId) || expandedCases?.find(c => c.id === caseId);
+    if (signal?.aborted) {
+      resolve({ id: template.id, category: template.category, language: template.language, question: template.question, status: 'error', error: 'Aborted' });
+      return;
+    }
+    const child = spawn(
+      process.execPath,
+      [cliEntry, '__worker', '--id', caseId, '--model', model, '--base-url', baseUrl, '--timeout-ms', String(timeoutMs)],
+      { cwd: REPO_ROOT, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }
+    );
+    let stdout = '', stderr = '';
+    let timedOut = false;
+    const timer = setTimeout(() => { timedOut = true; child.kill(); }, timeoutMs + workerGraceMs);
+    const onAbort = () => {
+      try { child.kill(); } catch {}
+    };
+    if (signal) signal.addEventListener('abort', onAbort, { once: true });
+    child.stdout.on('data', b => {
+      const text = b.toString();
+      stdout += text;
+      if (onStep) {
+        const lines = text.split(/\r?\n/);
+        for (const l of lines) {
+          if (l.startsWith('EVAL_STEP=')) {
+            try {
+              onStep({ caseId, ...JSON.parse(l.slice('EVAL_STEP='.length)) });
+            } catch {}
+          }
+        }
+      }
+    });
+    child.stderr.on('data', b => { stderr = (stderr + b).slice(-5000); });
+    child.on('error', error => {
+      clearTimeout(timer);
+      if (signal) signal.removeEventListener('abort', onAbort);
+      resolve({ id: template.id, category: template.category, language: template.language, question: template.question, status: 'error', error: error.message });
+    });
+    child.on('close', () => {
+      clearTimeout(timer);
+      if (signal) signal.removeEventListener('abort', onAbort);
+      const line = stdout.split(/\r?\n/).find(l => l.startsWith('EVAL_RESULT='));
+      try {
+        if (!line) throw new Error(timedOut ? 'Worker hard timeout' : (signal?.aborted ? 'Aborted' : `Worker failed: ${stderr}`));
+        resolve(JSON.parse(line.slice('EVAL_RESULT='.length)));
+      } catch (error) { resolve({ id: template.id, category: template.category, language: template.language, question: template.question, status: 'error', error: error.message }); }
+    });
+  });
+}
+
+export async function runSuite(options = {}) {
+  const {
+    suite = DEFAULTS.suite,
+    ids,
+    limit,
+    model = DEFAULTS.model,
+    baseUrl = DEFAULTS.baseUrl,
+    timeoutMs = DEFAULTS.timeoutMs,
+    workerGraceMs = DEFAULTS.workerGraceMs,
+    out = DEFAULTS.out,
+    from,
+    resume = false,
+    retryFailed = false,
+    retryErrors = false,
+    preservePrior = false,
+    judgeModel,
+    judgeOnly = false,
+    gate = false,
+    gatesFile = DEFAULTS.gatesFile,
+    saveAs,
+    dashboard = false,
+    dashboardOut = DEFAULTS.dashboardOut,
+    signal,
+    onStart,
+    onCaseStart,
+    onCaseComplete,
+    onStep,
+  } = options;
+
+  validateCases();
+
+  const outPath = path.isAbsolute(out) ? out : path.resolve(REPO_ROOT, out);
+  await fs.mkdir(path.dirname(outPath), { recursive: true });
+
+  let report;
+  let save = null;
+  if (judgeOnly) {
+    if (!judgeModel) throw new Error('judge-only requires judge-model');
+    report = await loadReport(outPath);
+    await verifyCorpusFreshness(report);
+    save = createSaver(outPath, report);
+  } else {
+    let selected = selectCases({ suite, ids, limit });
+    let prior = null;
+    const fromPath = from ? (path.isAbsolute(from) ? from : path.resolve(REPO_ROOT, from)) : outPath;
+    prior = await loadReportWithBackups(fromPath);
+    if (!prior && fromPath !== path.resolve(REPO_ROOT, DEFAULTS.out)) {
+      prior = await loadReportWithBackups(path.resolve(REPO_ROOT, DEFAULTS.out));
+    }
+    if (fromPath === outPath && prior) {
+      await rotateBackups(outPath, 5);
+    }
+    const doRetry = retryFailed || retryErrors || preservePrior;
+    if (doRetry && prior) {
+      if (!ids) {
+        const targetIds = new Set(
+          prior.rows
+            .filter(r => {
+              if (r.status === 'error' || r.error || r.timedOut) return true;
+              if (retryErrors) return false;
+              return (r.coverage && r.coverage.matched < r.coverage.total) || r.pass === false;
+            })
+            .map(r => r.id)
+        );
+        selected = selected.filter(c => targetIds.has(c.id));
+      }
+    }
+    try {
+      await probeModel({ baseUrl, model });
+    } catch (error) {
+      await writeBlockedReport(outPath, { reason: error.message, planned: selected.length });
+      throw error;
+    }
+    const metadata = buildMetadata({ model, baseUrl, timeoutMs, caseIds: selected.map(c => c.id) });
+    await finalizeMetadata(metadata);
+    report = { metadata, rows: [] };
+    if (doRetry && prior) {
+      report.metadata = {
+        ...prior.metadata,
+        resumedAt: new Date().toISOString(),
+        model: model || prior.metadata.model,
+        baseUrl: baseUrl || prior.metadata.baseUrl
+      };
+      await finalizeMetadata(report.metadata);
+      delete report.metadata.archivedAt;
+      report.rows = [...prior.rows];
+    } else if (resume && prior) {
+      assertResumeCompatible(prior, report);
+      report.rows = prior.rows.filter(r => selected.some(c => c.id === r.id));
+      report.metadata.startedAt = prior.metadata.startedAt;
+      report.metadata.resumedAt = new Date().toISOString();
+    }
+    save = createSaver(outPath, report);
+    await save();
+    const casesToRun = resume ? selected.filter(c => !report.rows.some(r => r.id === c.id)) : selected;
+    if (onStart) onStart({ planned: casesToRun.length, total: selected.length, model, suite, caseIds: casesToRun.map(c => c.id) });
+    const lock = await acquireRunLock(outPath);
+    try {
+      let doneCases = 0;
+      let consecutiveErrors = 0;
+      for (const c of casesToRun) {
+        if (signal?.aborted) break;
+        doneCases++;
+        if (onCaseStart) onCaseStart(c, { current: doneCases, total: casesToRun.length });
+        const result = await spawnWorker(c.id, { model, baseUrl, timeoutMs, workerGraceMs, signal, onStep });
+        result.model = model;
+        const existingIdx = report.rows.findIndex(r => r.id === c.id);
+        if (existingIdx >= 0) {
+          report.rows[existingIdx] = result;
+        } else {
+          report.rows.push(result);
+        }
+        await save(result);
+        await lock.touch();
+        if (onCaseComplete) onCaseComplete(c, result, { current: doneCases, total: casesToRun.length, report });
+        console.log(`${doneCases}/${casesToRun.length} ${c.id}: ${result.status}; ${result.durationMs ?? '?'}ms; coverage ${result.coverage?.matched ?? '-'}/${result.coverage?.total ?? '-'}`);
+        if (result.status === 'error') {
+          consecutiveErrors++;
+          if (consecutiveErrors >= 3) {
+            const abortMsg = `Evaluation aborted: 3 consecutive errors encountered (${result.error || 'Connection failure'}). Please check LM Studio manually. (เกิดข้อผิดพลาดติดต่อกัน 3 ครั้ง กรุณาตรวจสอบสถานะของ LM Studio หรือการตั้งค่าด้วยตนเอง)`;
+            console.error(abortMsg);
+            report.metadata.abortReason = abortMsg;
+            break;
+          }
+        } else {
+          consecutiveErrors = 0;
+        }
+        if (signal?.aborted) break;
+      }
+      report.metadata.finishedAt = new Date().toISOString();
+      await save();
+    } finally {
+      await lock.release();
+    }
+    if (report.metadata.abortReason) {
+      throw new Error(report.metadata.abortReason);
+    }
+  }
+
+  if (!save) save = createSaver(outPath, report);
+  const postRunLock = await acquireRunLock(outPath);
+  try {
+    if (judgeModel) {
+      await judgeReport(report, { judgeModel, baseUrl, timeoutMs }, save);
+    }
+    await save();
+
+    if (gate) {
+      const gateEval = await applyGates(report, gatesFile, save);
+      if (!gateEval.passed) process.exitCode = 2;
+    }
+  } finally {
+    await postRunLock.release();
+  }
+
+  if (saveAs) {
+    const snapshotPath = path.resolve(REPO_ROOT, DEFAULTS.snapshotsDir, `${saveAs}.json`);
+    await fs.mkdir(path.dirname(snapshotPath), { recursive: true });
+    await fs.writeFile(snapshotPath, JSON.stringify(report, null, 2));
+    console.log(`Snapshot saved: ${snapshotPath}`);
+  }
+
+  if (dashboard) {
+    const dashPath = path.isAbsolute(dashboardOut) ? dashboardOut : path.resolve(REPO_ROOT, dashboardOut);
+    await writeDashboardReport(report, dashPath);
+    console.log(`Performance dashboard generated: ${dashPath}`);
+  }
+
+  console.log(JSON.stringify(report.summary, null, 2));
+  if (report.summary.errors || report.summary.judgeErrors) {
+    process.exitCode = 1;
+  }
+  return report;
+}

package/server/agent/eval/modules/evaluation/scoring.js

@@ -0,0 +1,363 @@
+import { computeBenchmarkSummary } from '../reporting/benchmark.js';
+const normalize = s => String(s).normalize('NFKC').toLowerCase().replace(/\s+/g, ' ');
+
+function aliasMatches(text, alias) {
+  const normalizedText = normalize(text);
+  const normalizedAlias = normalize(alias);
+  if (!normalizedAlias) return false;
+  if (!/[\p{N}]/u.test(normalizedAlias)) return normalizedText.includes(normalizedAlias);
+  const timeMatch = normalizedAlias.match(/^(\d{1,2}):(\d{2})$/);
+  const escaped = timeMatch
+    ? `0?${timeMatch[1]}:${timeMatch[2]}`
+    : normalizedAlias.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  return new RegExp(`(?<![\\p{N}])${escaped}(?![\\p{N}])`, 'u').test(normalizedText);
+}
+
+function expandAnswerEquivalences(s) {
+  let text = String(s);
+  text = text.replace(/(?:พ\.ศ\.|พุทธศักราช)\s*(\d{4})/g, (match, beYear) => {
+    const ce = parseInt(beYear, 10) - 543;
+    return `${match} (${ce}) ${ce}`;
+  });
+  text = text.replace(/(\d+)\s*(?:minutes?|mins?|นาที)\s*(?:and|กับ|และ)?\s*(\d+)\s*(?:seconds?|secs?|วินาที)/gi, (match, m, s) => {
+    const secPadded = String(s).padStart(2, '0');
+    return `${match} ${m}:${secPadded}`;
+  });
+  return text;
+}
+
+export function scoreCoverage(c, answer) {
+  if (!c.groups.length) return { matched: 0, total: 0, allMatched: null, missing: [] };
+  const text = normalize(expandAnswerEquivalences(answer));
+  const missing = c.groups.filter(group => !group.some(alias => aliasMatches(text, alias)));
+  return { matched: c.groups.length - missing.length, total: c.groups.length, allMatched: missing.length === 0, missing };
+}
+
+export function parseJudgment(content) {
+  const clean = String(content).trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
+  const result = JSON.parse(clean);
+  for (const field of ['correctness', 'grounding', 'instructionFollowing']) {
+    if (!Number.isInteger(result[field]) || result[field] < 0 || result[field] > 4) throw new Error(`Invalid judge ${field}`);
+  }
+  if (typeof result.reason !== 'string' || !result.reason.trim() || !Array.isArray(result.unsupportedClaims) || result.unsupportedClaims.some(x => typeof x !== 'string')) throw new Error('Invalid judge explanation');
+  return result;
+}
+
+export function evaluateToolCorrectness(c, trace = []) {
+  const actualTools = [...new Set(trace.filter(t => t.event === 'start' && t.name).map(t => t.name))];
+  const expectedTools = Array.isArray(c?.expectedTools) ? [...new Set(c.expectedTools)] : [];
+
+  if (!expectedTools.length) {
+    return {
+      expected: [],
+      actual: actualTools,
+      matched: [],
+      missing: [],
+      unexpected: actualTools,
+      precision: null,
+      recall: null,
+      f1: null,
+      isCorrect: null,
+      notApplicable: true,
+      toolsUsed: actualTools
+    };
+  }
+
+  const matched = actualTools.filter(t => expectedTools.includes(t));
+  const missing = expectedTools.filter(t => !actualTools.includes(t));
+  const unexpected = actualTools.filter(t => !expectedTools.includes(t));
+
+  const precision = actualTools.length ? matched.length / actualTools.length : 0.0;
+  const recall = expectedTools.length ? matched.length / expectedTools.length : 0.0;
+  const f1 = (precision + recall) > 0 ? (2 * precision * recall) / (precision + recall) : 0.0;
+
+  const isCorrect = matched.length > 0 && recall >= 0.5 && precision >= 0.5;
+
+  return {
+    expected: expectedTools,
+    actual: actualTools,
+    matched,
+    missing,
+    unexpected,
+    precision: Number(precision.toFixed(3)),
+    recall: Number(recall.toFixed(3)),
+    f1: Number(f1.toFixed(3)),
+    isCorrect,
+    notApplicable: false,
+    toolsUsed: actualTools
+  };
+}
+
+export function detectTrajectoryLoops(trace = []) {
+  const toolStarts = trace.filter(t => t.event === 'start' && t.name);
+  if (toolStarts.length < 2) {
+    return { hasLoop: false, loopCount: 0, duplicateCalls: [], isThrashing: false };
+  }
+
+  const calls = toolStarts.map(t => {
+    let paramKey = '';
+    try {
+      paramKey = JSON.stringify(t.parameters || {});
+    } catch {
+      paramKey = String(t.parameters);
+    }
+    return `${t.name}:${paramKey}`;
+  });
+
+  const duplicateCalls = [];
+  let loopCount = 0;
+
+  for (let i = 1; i < calls.length; i++) {
+    if (calls[i] === calls[i - 1]) {
+      loopCount++;
+      duplicateCalls.push(calls[i]);
+    }
+  }
+
+  if (calls.length >= 4) {
+    for (let i = 3; i < calls.length; i++) {
+      if (calls[i] === calls[i - 2] && calls[i - 1] === calls[i - 3] && calls[i] !== calls[i - 1]) {
+        loopCount++;
+        duplicateCalls.push(`oscillating:${calls[i]}<->${calls[i - 1]}`);
+      }
+    }
+  }
+
+  const isThrashing = (loopCount >= 1 && duplicateCalls.some(d => d.startsWith('oscillating'))) || loopCount >= 2 || toolStarts.length > 8;
+
+  return {
+    hasLoop: loopCount > 0,
+    loopCount,
+    duplicateCalls: [...new Set(duplicateCalls)],
+    isThrashing
+  };
+}
+
+export function computeLatencyBreakdown(row) {
+  const trace = Array.isArray(row.trace) ? row.trace : [];
+  const totalDurationMs = Number.isFinite(row.durationMs) ? Math.max(0, row.durationMs) : 0;
+
+  const toolExecutionMs = trace
+    .filter(t => t.event === 'result' && Number.isFinite(t.durationMs))
+    .reduce((acc, t) => acc + Math.max(0, t.durationMs), 0);
+
+  const llmInferenceMs = Math.max(0, totalDurationMs - toolExecutionMs);
+  const toolPercentage = totalDurationMs > 0 ? Number(((toolExecutionMs / totalDurationMs) * 100).toFixed(1)) : 0;
+  const llmPercentage = totalDurationMs > 0 ? Number(((llmInferenceMs / totalDurationMs) * 100).toFixed(1)) : 0;
+
+  const ttftMs = row.metrics?.timing_ms?.ttft ?? row.telemetry?.timeToFirstTokenMs ?? row.metrics?.ttftMs ?? null;
+  const decodeTps = row.metrics?.rates?.decode_tps ?? row.telemetry?.generationTokensPerSecond ?? row.metrics?.decodeTps ?? null;
+  const prefillTps = row.metrics?.rates?.prefill_tps ?? row.telemetry?.promptTokensPerSecond ?? row.metrics?.prefillTps ?? null;
+  const outputTokens = row.metrics?.counts?.completion_tokens ?? row.telemetry?.outputTokens ?? row.metrics?.outputTokens ?? null;
+
+  return {
+    totalDurationMs,
+    toolExecutionMs,
+    llmInferenceMs,
+    toolPercentage,
+    llmPercentage,
+    ttftMs,
+    decodeTps,
+    prefillTps,
+    outputTokens
+  };
+}
+
+export function checkGroundedness(c, answer, trace = []) {
+  if (!answer || typeof answer !== 'string') return { isGrounded: false, supportRatio: 0, checkedFacts: 0, notApplicable: false };
+  const text = normalize(answer);
+
+  const expectedTools = Array.isArray(c?.expectedTools) ? c.expectedTools : [];
+  if (!expectedTools.length) {
+    return {
+      isGrounded: true,
+      supportRatio: 1.0,
+      notApplicable: true,
+      evidenceFound: false,
+      note: 'Tool observation not required for this case'
+    };
+  }
+
+  const toolOutputs = trace
+    .filter(t => t.event === 'result' && t.value)
+    .map(t => typeof t.value === 'object' ? JSON.stringify(t.value) : String(t.value))
+    .join(' ')
+    .toLowerCase();
+
+  if (!toolOutputs.trim()) {
+    return {
+      isGrounded: false,
+      supportRatio: 0.0,
+      notApplicable: false,
+      evidenceFound: false,
+      note: 'No tool observations found despite expected tools'
+    };
+  }
+
+  const groups = Array.isArray(c?.groups) ? c.groups : [];
+  if (!groups.length) {
+    return { isGrounded: true, supportRatio: 1.0, notApplicable: false, evidenceFound: true };
+  }
+
+  let matchedInAnswer = 0;
+  let groundedInTools = 0;
+
+  for (const group of groups) {
+    const presentInAnswer = group.some(alias => aliasMatches(text, alias));
+    if (presentInAnswer) {
+      matchedInAnswer++;
+      const presentInTools = group.some(alias => aliasMatches(toolOutputs, alias));
+      if (presentInTools) {
+        groundedInTools++;
+      }
+    }
+  }
+
+  if (matchedInAnswer === 0) {
+    return {
+      isGrounded: false,
+      supportRatio: 0.0,
+      notApplicable: false,
+      matchedInAnswer: 0,
+      groundedInTools: 0,
+      evidenceFound: true,
+      note: 'No extractable facts in answer to ground'
+    };
+  }
+
+  const supportRatio = Number((groundedInTools / matchedInAnswer).toFixed(2));
+  return {
+    isGrounded: supportRatio >= 0.8,
+    supportRatio,
+    notApplicable: false,
+    matchedInAnswer,
+    groundedInTools,
+    evidenceFound: true
+  };
+}
+
+export function summarize(rows, planned) {
+  const completed = rows.filter(r => r.status === 'completed');
+  const coverage = completed.filter(r => r.coverage && r.coverage.total > 0);
+  const judged = completed.filter(r => (r.judgeResult && !r.judgeResult.judgeError) || r.judgment);
+  const times = completed.map(r => r.durationMs).filter(Number.isFinite).sort((a, b) => a - b);
+  const fraction = (a, b) => b ? Number((a / b).toFixed(4)) : null;
+
+  const toolEligible = completed.filter(r => r.toolCorrectness && !r.toolCorrectness.notApplicable);
+  const toolCorrectRows = toolEligible.filter(r => r.toolCorrectness.isCorrect === true);
+
+  const loopFreeRows = completed.filter(r => !r.trajectoryHealth?.hasLoop);
+
+  const groundedEligible = completed.filter(r => r.groundedness && !r.groundedness.notApplicable);
+  const groundedRows = groundedEligible.filter(r => r.groundedness.isGrounded === true);
+
+  const llmTimes = completed.map(r => r.latencyBreakdown?.llmInferenceMs).filter(Number.isFinite);
+  const toolTimes = completed.map(r => r.latencyBreakdown?.toolExecutionMs).filter(Number.isFinite);
+  const decodeTpsList = completed.map(r => r.latencyBreakdown?.decodeTps).filter(Number.isFinite);
+
+  const mean = list => list.length ? Number((list.reduce((a, b) => a + b, 0) / list.length).toFixed(1)) : null;
+
+  const categoryStats = {};
+  for (const r of rows) {
+    const cat = r.category || 'unknown';
+    if (!categoryStats[cat]) {
+      categoryStats[cat] = { total: 0, completed: 0, factsMatched: 0, toolsCorrect: 0, toolEligible: 0, grounded: 0, groundedEligible: 0 };
+    }
+    categoryStats[cat].total++;
+    if (r.status === 'completed') {
+      categoryStats[cat].completed++;
+      if (r.coverage?.allMatched) categoryStats[cat].factsMatched++;
+      if (r.toolCorrectness && !r.toolCorrectness.notApplicable) {
+        categoryStats[cat].toolEligible++;
+        if (r.toolCorrectness.isCorrect) categoryStats[cat].toolsCorrect++;
+      }
+      if (r.groundedness && !r.groundedness.notApplicable) {
+        categoryStats[cat].groundedEligible++;
+        if (r.groundedness.isGrounded) categoryStats[cat].grounded++;
+      }
+    }
+  }
+
+  const languageStats = {};
+  for (const r of rows) {
+    const lang = r.language || 'unknown';
+    if (!languageStats[lang]) {
+      languageStats[lang] = { total: 0, completed: 0, factsMatched: 0, toolsCorrect: 0, toolEligible: 0, grounded: 0, groundedEligible: 0, meanMs: null };
+    }
+    languageStats[lang].total++;
+    if (r.status === 'completed') {
+      languageStats[lang].completed++;
+      if (r.coverage?.allMatched) languageStats[lang].factsMatched++;
+      if (r.toolCorrectness && !r.toolCorrectness.notApplicable) {
+        languageStats[lang].toolEligible++;
+        if (r.toolCorrectness.isCorrect) languageStats[lang].toolsCorrect++;
+      }
+      if (r.groundedness && !r.groundedness.notApplicable) {
+        languageStats[lang].groundedEligible++;
+        if (r.groundedness.isGrounded) languageStats[lang].grounded++;
+      }
+    }
+  }
+  for (const lang of Object.keys(languageStats)) {
+    const langTimes = completed.filter(r => (r.language || 'unknown') === lang).map(r => r.durationMs).filter(Number.isFinite);
+    languageStats[lang].meanMs = mean(langTimes);
+  }
+
+  const isRowJudgePass = r => {
+    if (r.judgeResult && !r.judgeResult.judgeError) {
+      return r.judgeResult.score === 1.0 || r.judgeResult.verdict === 'PASS';
+    }
+    if (r.judgment) {
+      return r.judgment.correctness >= 3 && r.judgment.grounding >= 3 && r.judgment.instructionFollowing >= 3 && !r.judgment.unsupportedClaims?.length;
+    }
+    return false;
+  };
+
+  const judgedWithCoverage = judged.filter(r => r.coverage && r.coverage.total > 0);
+  const judgeAgreed = judgedWithCoverage.filter(r => {
+    const judgePass = isRowJudgePass(r);
+    return Boolean(judgePass) === Boolean(r.coverage.allMatched);
+  });
+  const judgeAgreementRate = judgedWithCoverage.length ? fraction(judgeAgreed.length, judgedWithCoverage.length) : null;
+  const benchmark = computeBenchmarkSummary(rows);
+
+  return {
+    planned,
+    attempted: rows.length,
+    completed: completed.length,
+    errors: rows.length - completed.length,
+    completionRate: fraction(completed.length, planned),
+    allFactsMatched: coverage.filter(r => r.coverage.allMatched).length,
+    coverageEligibleCompleted: coverage.length,
+    lexicalCoverageRate: fraction(coverage.filter(r => r.coverage.allMatched).length, coverage.length),
+
+    toolAccuracyRate: toolEligible.length ? fraction(toolCorrectRows.length, toolEligible.length) : null,
+    toolEligibleCompleted: toolEligible.length,
+
+    loopFreeRate: completed.length ? fraction(loopFreeRows.length, completed.length) : null,
+
+    groundednessRate: groundedEligible.length ? fraction(groundedRows.length, groundedEligible.length) : null,
+    groundednessEligibleCompleted: groundedEligible.length,
+
+    meanLlmMs: mean(llmTimes),
+    meanToolMs: mean(toolTimes),
+    meanDecodeTps: mean(decodeTpsList),
+    categoryStats,
+    languageStats,
+
+    judged: judged.length,
+    judgeErrors: rows.filter(r => r.judgeError || r.judgeResult?.judgeError).length,
+    judgePasses: judged.filter(r => isRowJudgePass(r)).length,
+    judgeFlaggedUnsupported: judged.filter(r => r.judgment?.unsupportedClaims?.length).length,
+    judgeAgreementRate,
+
+    medianMs: times.length ? times[Math.floor((times.length - 1) / 2)] : null,
+    p95Ms: times.length ? times[Math.ceil(times.length * .95) - 1] : null,
+    passAt1: benchmark.values.passAt1,
+    trajectoryOptimality: benchmark.values.trajectoryOptimality,
+    toolF1: benchmark.values.toolF1,
+    hallucinationRate: benchmark.values.hallucinationRate,
+    costPerTask: benchmark.values.costPerTask,
+    benchmark,
+  };
+}