kiku-agent 1.3.1 → 1.3.3

package/server/agent/eval/modules/server/evalApi.js

@@ -0,0 +1,1099 @@
+import fs from 'node:fs/promises';
+import fsSync from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import readline from 'node:readline';
+import { runSuite } from '../evaluation/runner.js';
+import { cases, expandedCases, securityCases, allCases, legacyCases } from '../evaluation/cases.js';
+import { DEFAULTS, REPO_ROOT } from '../core/config.js';
+import { getEvalUiHtml } from '../ui/evalUi.js';
+import { diffReports } from '../analysis/compare.js';
+import { renderBenchmarkPng } from '../reporting/benchmarkPng.js';
+import { judgeReport } from '../analysis/judge.js';
+import { acquireRunLock, createSaver, loadReportWithBackups } from '../core/store.js';
+import { runSingleCase } from '../evaluation/worker.js';
+import { evaluateGates, loadGatesFile } from '../analysis/gates.js';
+
+let isRunning = false;
+let activeAbortController = null;
+let currentReport = { rows: [], metadata: {}, summary: {} };
+const sseClients = new Set();
+let activeRunMeta = null;
+let activeCaseMeta = null;
+let liveWatcherTimer = null;
+let lastLiveMtime = 0;
+
+const lmsStreamClients = new Set();
+let lmsProcess = null;
+let lmsRl = null;
+let lmsStopTimer = null;
+const lastKnownLmsEvents = [];
+
+function getLmsExecutable() {
+  const userHome = os.homedir();
+  const isWin = process.platform === 'win32';
+  const candidates = [
+    path.join(userHome, '.lmstudio', 'bin', isWin ? 'lms.exe' : 'lms'),
+    path.join(process.env.LOCALAPPDATA || '', 'Programs', 'LM Studio', 'resources', 'app', '.webpack', 'main', isWin ? 'lms.exe' : 'lms'),
+    isWin ? 'lms.exe' : 'lms'
+  ];
+  for (const c of candidates) {
+    try {
+      if (fsSync.existsSync(c)) return c;
+    } catch {}
+  }
+  return isWin ? 'lms.exe' : 'lms';
+}
+
+function broadcastLmsStream(event, data) {
+  const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+  for (const client of lmsStreamClients) {
+    try {
+      client.write(payload);
+    } catch {
+      lmsStreamClients.delete(client);
+    }
+  }
+}
+
+function startLmsStream() {
+  if (lmsStopTimer) {
+    clearTimeout(lmsStopTimer);
+    lmsStopTimer = null;
+  }
+  if (lmsProcess) return;
+  const lmsBin = getLmsExecutable();
+  try {
+    lmsProcess = spawn(lmsBin, ['log', 'stream', '--json'], {
+      windowsHide: true,
+      stdio: ['ignore', 'pipe', 'pipe']
+    });
+
+    lmsRl = readline.createInterface({
+      input: lmsProcess.stdout,
+      terminal: false
+    });
+
+    lmsRl.on('line', (line) => {
+      const trimmed = line.trim();
+      if (!trimmed) return;
+      let parsed = null;
+      try {
+        parsed = JSON.parse(trimmed);
+      } catch {
+        parsed = { raw: trimmed, timestamp: Date.now() };
+      }
+      lastKnownLmsEvents.push(parsed);
+      if (lastKnownLmsEvents.length > 100) lastKnownLmsEvents.shift();
+      broadcastLmsStream('lms_log', parsed);
+    });
+
+    lmsProcess.stderr?.on('data', (data) => {
+      const text = data.toString().trim();
+      if (text) {
+        broadcastLmsStream('lms_log', { info: text, timestamp: Date.now() });
+      }
+    });
+
+    lmsProcess.on('error', (err) => {
+      broadcastLmsStream('lms_log', { error: err.message, timestamp: Date.now() });
+      stopLmsStream();
+    });
+
+    lmsProcess.on('exit', () => {
+      stopLmsStream();
+    });
+  } catch (err) {
+    lmsProcess = null;
+  }
+}
+
+function stopLmsStream() {
+  if (lmsStopTimer) {
+    clearTimeout(lmsStopTimer);
+    lmsStopTimer = null;
+  }
+  if (lmsRl) {
+    try { lmsRl.close(); } catch {}
+    lmsRl = null;
+  }
+  if (lmsProcess) {
+    try {
+      if (process.platform === 'win32' && lmsProcess.pid) {
+        spawn('taskkill', ['/pid', String(lmsProcess.pid), '/f', '/t'], { windowsHide: true });
+      } else {
+        lmsProcess.kill('SIGTERM');
+      }
+    } catch {}
+    lmsProcess = null;
+  }
+}
+
+async function getEffectiveReport() {
+  const defaultOut = path.resolve(REPO_ROOT, DEFAULTS.out);
+  try {
+    const content = await fs.readFile(defaultOut, 'utf8');
+    const diskReport = JSON.parse(content);
+    if (!Array.isArray(diskReport?.rows)) return currentReport;
+    if (!isRunning || !currentReport?.rows?.length || (diskReport.rows?.length || 0) >= (currentReport.rows?.length || 0)) {
+      currentReport = diskReport;
+      return diskReport;
+    }
+  } catch {}
+  return currentReport;
+}
+
+function ensureLiveWatcher() {
+  if (liveWatcherTimer) return;
+  const defaultOut = path.resolve(REPO_ROOT, DEFAULTS.out);
+  liveWatcherTimer = setInterval(async () => {
+    if (sseClients.size === 0 || isRunning) return;
+    try {
+      const st = await fs.stat(defaultOut);
+      if (st.mtimeMs !== lastLiveMtime) {
+        lastLiveMtime = st.mtimeMs;
+        const rep = await getEffectiveReport();
+        if (rep && Array.isArray(rep.rows) && rep.rows.length > 0) {
+          broadcastSSE('sync_state', {
+            running: isRunning,
+            activeRun: activeRunMeta,
+            activeCase: activeCaseMeta,
+            report: rep
+          });
+        }
+      }
+    } catch {}
+  }, 1000);
+  if (liveWatcherTimer && typeof liveWatcherTimer.unref === 'function') {
+    liveWatcherTimer.unref();
+  }
+}
+
+function broadcastSSE(event, data) {
+  const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+  for (const client of sseClients) {
+    try {
+      client.write(payload);
+    } catch {
+      sseClients.delete(client);
+    }
+  }
+}
+
+async function readRequestBody(req) {
+  return new Promise((resolve, reject) => {
+    let body = '';
+    req.on('data', chunk => {
+      body += chunk.toString();
+      if (body.length > 10 * 1024 * 1024) {
+        req.destroy();
+        reject(new Error('Payload too large'));
+      }
+    });
+    req.on('end', () => {
+      try {
+        resolve(body ? JSON.parse(body) : {});
+      } catch {
+        resolve({});
+      }
+    });
+    req.on('error', reject);
+  });
+}
+
+function sendJson(res, statusCode, data) {
+  res.writeHead(statusCode, {
+    'Content-Type': 'application/json; charset=utf-8',
+    'Access-Control-Allow-Origin': '*',
+    'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+    'Access-Control-Allow-Headers': 'Content-Type'
+  });
+  res.end(JSON.stringify(data));
+}
+
+async function loadReportByIdentifier(ident) {
+  if (ident === 'current' || ident === 'live') {
+    return getEffectiveReport();
+  }
+  if (typeof ident !== 'string' || !ident.trim() || ident.includes('..') || path.isAbsolute(ident)) {
+    return null;
+  }
+  if (ident.startsWith('history/')) {
+    const historyDir = path.dirname(path.resolve(REPO_ROOT, DEFAULTS.out));
+    const archivePath = path.join(historyDir, 'history', `${ident.slice('history/'.length)}.json`);
+    try {
+      return JSON.parse(await fs.readFile(archivePath, 'utf8'));
+    } catch {
+      return null;
+    }
+  }
+  const snapshotPath = path.resolve(REPO_ROOT, DEFAULTS.snapshotsDir, `${ident}.json`);
+  try {
+    return JSON.parse(await fs.readFile(snapshotPath, 'utf8'));
+  } catch {
+    const rawPath = path.resolve(REPO_ROOT, ident.endsWith('.json') ? ident : `${ident}.json`);
+    try {
+      return JSON.parse(await fs.readFile(rawPath, 'utf8'));
+    } catch {
+      return null;
+    }
+  }
+}
+
+function generateCsv(report) {
+  const rows = Array.isArray(report?.rows) ? report.rows : [];
+  const header = ['id', 'category', 'language', 'status', 'allMatched', 'matchedFacts', 'totalFacts', 'toolCorrect', 'loopsDetected', 'durationMs', 'llmMs', 'toolMs', 'decodeTps', 'question'];
+  const lines = [header.join(',')];
+  for (const r of rows) {
+    const matched = r.coverage?.matched ?? '';
+    const total = r.coverage?.total ?? '';
+    const allMatched = r.coverage?.allMatched ?? '';
+    const toolCorrect = r.toolCorrectness?.isCorrect ?? '';
+    const loops = r.trajectoryHealth?.hasLoop ?? '';
+    const dur = r.durationMs ?? '';
+    const llmMs = r.latencyBreakdown?.llmMs ?? '';
+    const toolMs = r.latencyBreakdown?.toolMs ?? '';
+    const tps = r.latencyBreakdown?.decodeTps ?? '';
+    const qEsc = `"${(r.question || '').replace(/"/g, '""')}"`;
+    lines.push([
+      r.id || '',
+      r.category || '',
+      r.language || '',
+      r.status || '',
+      allMatched,
+      matched,
+      total,
+      toolCorrect,
+      loops,
+      dur,
+      llmMs,
+      toolMs,
+      tps,
+      qEsc
+    ].join(','));
+  }
+  return lines.join('\r\n');
+}
+
+function isErrorRow(r) {
+  if (!r) return false;
+  return r.status === 'error' || Boolean(r.error) || Boolean(r.timedOut);
+}
+
+function isAnomalyRow(r) {
+  if (!r) return false;
+  if (r.status === 'error' || r.error) return true;
+  if (r.timedOut) return true;
+  if (r.toolCorrectness && r.toolCorrectness.isCorrect === false) return true;
+  if (r.trajectoryHealth?.hasLoop) return true;
+  if (r.groundedness && r.groundedness.isGrounded === false) return true;
+  if (r.coverage && r.coverage.allMatched === false) return true;
+  if (r.passAt1 === false || (typeof r.passAt1 === 'number' && r.passAt1 === 0)) return true;
+  return false;
+}
+
+function getAnomalyReason(r) {
+  if (!r) return 'Unknown anomaly';
+  if (r.status === 'error' || r.error) return `Error: ${r.error || 'Execution failed'}`;
+  if (r.timedOut) return 'Timeout exceeded';
+  if (r.trajectoryHealth?.hasLoop) return 'Infinite Re-planning Loop';
+  if (r.toolCorrectness && r.toolCorrectness.isCorrect === false) {
+    const missing = r.toolCorrectness.missing?.length ? `missing [${r.toolCorrectness.missing.join(', ')}]` : '';
+    const unexpected = r.toolCorrectness.unexpected?.length ? `unexpected [${r.toolCorrectness.unexpected.join(', ')}]` : '';
+    return `Tool Mismatch: ${[missing, unexpected].filter(Boolean).join('; ')}`;
+  }
+  if (r.coverage && r.coverage.allMatched === false) {
+    return `Coverage Incomplete: missing [${(r.coverage.missing || []).join(', ')}]`;
+  }
+  if (r.groundedness && r.groundedness.isGrounded === false) return 'Hallucination / Ungrounded Claim';
+  return 'Behavioral Anomaly (Pass@1: 0)';
+}
+
+function generateDeterministicSummary({ report, totalCases, passCount, passRate, anomalies, categoryStats, avgDurationMs }) {
+  const modelName = report.metadata?.model || DEFAULTS.model;
+  const isHealthy = passRate >= 75.0 && anomalies.length === 0;
+  const verdictBadge = isHealthy ? 'PRODUCTION READY (PASSED)' : (passRate >= 65.0 ? 'MARGINAL / NEEDS OPTIMIZATION' : 'CRITICAL REGRESSIONS DETECTED');
+  const errorCount = (report.rows || []).filter(r => r.status === 'error' || r.error).length;
+
+  let md = `# Kiku Agent Evaluation — Executive Benchmark Report\n\n`;
+  md += `**Model Evaluated:** \`${modelName}\` | **Generated:** ${new Date().toLocaleString()} | **Verdict:** **${verdictBadge}**\n\n`;
+
+  md += `## 1. Executive Verdict & Health Overview\n`;
+  md += `- **Overall Task Success (Pass@1):** **${passRate}%** (${passCount} / ${totalCases} cases passed)\n`;
+  md += `- **Benchmark Standard Target:** >= 75.0% Pass@1\n`;
+  md += `- **Execution Errors:** **${errorCount}** (${errorCount === 0 ? 'Zero crashes or timeouts' : `${errorCount} cases`})\n`;
+  md += `- **Detected Anomalies / Behavioral Deviations:** **${anomalies.length}** cases (${((anomalies.length / Math.max(1, totalCases)) * 100).toFixed(1)}% of test suite)\n`;
+  md += `- **Average Latency:** **${(avgDurationMs / 1000).toFixed(1)}s** (${avgDurationMs.toLocaleString()}ms) per test case\n`;
+  md += `- **Evaluation Status:** ${isHealthy ? 'The model meets or exceeds all standard quality and safety gates.' : 'The model has identified regressions or anomalies that require targeted remediation before deployment.'}\n\n`;
+
+  md += `## 2. Category Breakdown Matrix\n\n`;
+  md += `| Category | Total Cases | Passed | Pass Rate | Errors | Tool Failures | Health Status |\n`;
+  md += `| :--- | :---: | :---: | :---: | :---: | :---: | :---: |\n`;
+  for (const [cat, stats] of Object.entries(categoryStats)) {
+    const rate = stats.total > 0 ? Number(((stats.passed / stats.total) * 100).toFixed(1)) : 0;
+    const statusIcon = rate >= 85 ? 'Excellent' : (rate >= 70 ? 'Moderate' : 'Needs Attention');
+    md += `| **${cat}** | ${stats.total} | ${stats.passed} | ${rate}% | ${stats.errors} | ${stats.toolFails} | ${statusIcon} |\n`;
+  }
+  md += `\n`;
+
+  md += `## 3. Anomaly & Root Cause Diagnosis\n`;
+  if (anomalies.length === 0) {
+    md += `Zero anomalies detected. All ${totalCases} evaluation cases executed cleanly with full coverage and zero loop violations.\n\n`;
+  } else {
+    md += `A total of **${anomalies.length}** anomalies were flagged across the benchmark run:\n\n`;
+    for (const a of anomalies.slice(0, 10)) {
+      md += `- **[${a.id}]** \`${a.category}\`: *"${a.question || 'N/A'}"*\n`;
+      md += `  ↳ **Root Cause:** ${getAnomalyReason(a)} *(Latency: ${a.durationMs || 0}ms)*\n`;
+    }
+    if (anomalies.length > 10) {
+      md += `- *...and ${anomalies.length - 10} more anomalous cases (use the Anomalies filter in the workbench to inspect all).*\n`;
+    }
+    md += `\n`;
+  }
+
+  md += `## 4. Operational & Latency Insights\n`;
+  md += `- **Speed Benchmark:** Average query execution time was **${(avgDurationMs / 1000).toFixed(2)}s**.\n`;
+  const slowCases = [...report.rows].sort((a, b) => (b.durationMs || 0) - (a.durationMs || 0)).slice(0, 3);
+  if (slowCases.length > 0 && slowCases[0].durationMs > 15000) {
+    md += `- **High Latency Outliers:**\n`;
+    for (const s of slowCases) {
+      md += `  * [${s.id}] \`${s.category}\`: ${(s.durationMs / 1000).toFixed(1)}s — *"${(s.question || '').slice(0, 60)}..."*\n`;
+    }
+  }
+  md += `\n`;
+
+  md += `## 5. Actionable Engineering Recommendations\n`;
+  if (errorCount > 0) {
+    md += `1. **Click "Retry"**: Re-run ${errorCount} status error cases to verify transient network or timeout issues.\n`;
+  } else {
+    md += `1. **Zero Status Errors Remediated**: All 500 evaluation cases completed cleanly without crashes or timeouts.\n`;
+  }
+  md += `2. **Tool Routing Refinements**: For categories with tool mismatches, adjust prompt routing definitions in \`server/agent/tools.js\`.\n`;
+  md += `3. **Coverage Enhancements**: Ensure catalog entity aliases in \`src/data/*.js\` and \`server/mcp/data/*.js\` match the gold queries.\n`;
+  md += `4. **Loop Protection**: Verify sliding window and ReAct trajectory cutoff limits in \`server/agent/agentCore.js\`.\n`;
+
+  return md;
+}
+
+export async function handleEvalApiRequest(req, res) {
+  const urlObj = new URL(req.url, 'http://localhost:3001');
+  const pathname = urlObj.pathname;
+
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204, {
+      'Access-Control-Allow-Origin': '*',
+      'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+      'Access-Control-Allow-Headers': 'Content-Type'
+    });
+    res.end();
+    return true;
+  }
+
+  if (pathname.startsWith('/fonts/') || pathname === '/favicon.ico') {
+    const publicPath = path.resolve(REPO_ROOT, 'public', pathname.replace(/^\/+/, ''));
+    try {
+      const data = await fs.readFile(publicPath);
+      const ext = path.extname(publicPath).toLowerCase();
+      const mimeMap = {
+        '.css': 'text/css; charset=utf-8',
+        '.woff2': 'font/woff2',
+        '.woff': 'font/woff',
+        '.ttf': 'font/ttf',
+        '.ico': 'image/x-icon',
+        '.png': 'image/png',
+        '.svg': 'image/svg+xml'
+      };
+      res.writeHead(200, {
+        'Content-Type': mimeMap[ext] || 'application/octet-stream',
+        'Cache-Control': 'public, max-age=86400',
+        'Access-Control-Allow-Origin': '*'
+      });
+      res.end(data);
+      return true;
+    } catch {}
+  }
+
+  if (pathname === '/' || pathname === '/eval' || pathname === '/eval/' || pathname === '/index.html') {
+    res.writeHead(200, {
+      'Content-Type': 'text/html; charset=utf-8',
+      'Cache-Control': 'no-cache, no-store, must-revalidate',
+      'Pragma': 'no-cache',
+      'Expires': '0'
+    });
+    const gates = await loadGatesFile(DEFAULTS.gatesFile);
+    res.end(getEvalUiHtml(gates));
+    return true;
+  }
+
+  if (pathname === '/dashboard' || pathname === '/eval/dashboard' || pathname === '/dashboard.html') {
+    try {
+      const rep = await getEffectiveReport();
+      const { generateDashboardHtml } = await import('../reporting/dashboard.js');
+      res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+      res.end(generateDashboardHtml(rep));
+      return true;
+    } catch (e) {
+      res.writeHead(500, { 'Content-Type': 'text/plain' });
+      res.end(String(e));
+      return true;
+    }
+  }
+
+  if (pathname === '/api/eval/events') {
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'Access-Control-Allow-Origin': '*'
+    });
+    res.write(': connected\n\n');
+    sseClients.add(res);
+    ensureLiveWatcher();
+    getEffectiveReport().then(rep => {
+      const syncData = {
+        running: isRunning,
+        activeRun: activeRunMeta,
+        activeCase: activeCaseMeta,
+        report: rep
+      };
+      try {
+        res.write(`event: sync_state\ndata: ${JSON.stringify(syncData)}\n\n`);
+      } catch {}
+    });
+    req.on('close', () => {
+      sseClients.delete(res);
+    });
+    return true;
+  }
+
+  if (pathname === '/api/eval/lms/stream') {
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'Access-Control-Allow-Origin': '*'
+    });
+    res.write(': connected\n\n');
+    lmsStreamClients.add(res);
+    startLmsStream();
+    if (lastKnownLmsEvents.length > 0) {
+      for (const ev of lastKnownLmsEvents) {
+        try {
+          res.write(`event: lms_log\ndata: ${JSON.stringify(ev)}\n\n`);
+        } catch {}
+      }
+    } else {
+      res.write(`event: lms_log\ndata: ${JSON.stringify({ info: 'Connected to LM Studio event stream (google/gemma-4-e4b)', timestamp: Date.now() })}\n\n`);
+    }
+    req.on('close', () => {
+      lmsStreamClients.delete(res);
+      if (lmsStreamClients.size === 0) {
+        if (lmsStopTimer) clearTimeout(lmsStopTimer);
+        lmsStopTimer = setTimeout(() => {
+          if (lmsStreamClients.size === 0) {
+            stopLmsStream();
+          }
+        }, 60000);
+      }
+    });
+    return true;
+  }
+
+  if (pathname === '/api/eval/lms/status' && req.method === 'GET') {
+    sendJson(res, 200, {
+      active: Boolean(lmsProcess),
+      clientCount: lmsStreamClients.size
+    });
+    return true;
+  }
+
+  if (pathname === '/api/eval/status' && req.method === 'GET') {
+    const rep = await getEffectiveReport();
+    sendJson(res, 200, {
+      running: isRunning,
+      hasReport: Array.isArray(rep?.rows) && rep.rows.length > 0,
+      currentReport: rep,
+      activeRun: activeRunMeta,
+      activeCase: activeCaseMeta
+    });
+    return true;
+  }
+
+  if (pathname === '/api/eval/suites' && req.method === 'GET') {
+    const full = [...new Set([...allCases, ...(expandedCases || [])])];
+    const catalogCount = cases.length;
+    const expandedCount = expandedCases ? expandedCases.length : 0;
+    const securityCount = securityCases.length;
+    const legacyCount = legacyCases.length;
+    const totalCount = full.length;
+    sendJson(res, 200, {
+      single: { id: 'single', name: 'Single Case', count: 1 },
+      quick: { id: 'quick', name: 'Quick Smoke', count: Math.min(3, totalCount) },
+      catalog: { id: 'catalog', name: 'Catalog QA', count: catalogCount },
+      expanded: { id: 'expanded', name: 'New Cases', count: expandedCount },
+      security: { id: 'security', name: 'Security Resilience', count: securityCount },
+      legacy: { id: 'legacy', name: 'Legacy Case IDs', count: legacyCount },
+      all: { id: 'all', name: 'All Comprehensive', count: totalCount },
+      total: totalCount
+    });
+    return true;
+  }
+
+  if (pathname === '/api/eval/cases' && req.method === 'GET') {
+    const full = [...new Set([...allCases, ...(expandedCases || [])])];
+    const list = full.map(c => ({
+      id: c.id,
+      category: c.category,
+      dimension: c.dimension || null,
+      difficulty: c.difficulty || null,
+      language: c.language,
+      question: c.question,
+      expectedTools: c.expectedTools || []
+    }));
+    sendJson(res, 200, list);
+    return true;
+  }
+
+  if (pathname === '/api/eval/report' && req.method === 'GET') {
+    const rep = await getEffectiveReport();
+    sendJson(res, 200, rep);
+    return true;
+  }
+
+  if (pathname === '/api/eval/start' && req.method === 'POST') {
+    if (isRunning) {
+      sendJson(res, 409, { error: 'Evaluation is already in progress' });
+      return true;
+    }
+
+    const payload = await readRequestBody(req);
+
+    if (payload.retryErrors || payload.retryAnomalies || payload.retryFailedOnly) {
+      const diskReport = await loadReportWithBackups(path.resolve(REPO_ROOT, DEFAULTS.out));
+      if (diskReport) currentReport = diskReport;
+      const isErrorOnly = payload.retryErrors !== false;
+      const targetCases = (currentReport?.rows || []).filter(r => {
+        if (isErrorOnly) return isErrorRow(r);
+        return isAnomalyRow(r);
+      });
+      if (!targetCases.length) {
+        sendJson(res, 400, { error: isErrorOnly ? 'No status error cases found in the current report to retry.' : 'No error or anomalous cases found in the current report to retry.' });
+        return true;
+      }
+      payload.ids = targetCases.map(c => c.id).join(',');
+      payload.limit = targetCases.length;
+      payload.suite = 'all';
+    }
+
+    const isTargetedRun = Boolean(payload.ids || payload.retryErrors || payload.retryAnomalies || payload.retryFailedOnly);
+
+    isRunning = true;
+    activeAbortController = new AbortController();
+    activeRunMeta = {
+      suite: payload.suite || DEFAULTS.suite,
+      model: payload.model || DEFAULTS.model,
+      baseUrl: payload.baseUrl || DEFAULTS.baseUrl,
+      limit: payload.limit,
+      startedAt: Date.now(),
+      planned: 0,
+      total: 0,
+      initialRows: currentReport?.rows?.length || 0
+    };
+    activeCaseMeta = null;
+
+    if (!payload.resume && !isTargetedRun) {
+      currentReport = { metadata: {}, rows: [], summary: {} };
+    } else if (!Array.isArray(currentReport?.rows) || currentReport.rows.length === 0) {
+      const diskReport = await loadReportWithBackups(path.resolve(REPO_ROOT, DEFAULTS.out));
+      currentReport = diskReport || { metadata: {}, rows: [], summary: {} };
+    }
+
+    sendJson(res, 200, { started: true });
+
+    (async () => {
+      try {
+        const report = await runSuite({
+          suite: payload.suite || DEFAULTS.suite,
+          ids: payload.ids,
+          limit: payload.limit,
+          model: payload.model || DEFAULTS.model,
+          baseUrl: payload.baseUrl || DEFAULTS.baseUrl,
+          judgeModel: payload.judgeModel,
+          resume: Boolean(payload.resume),
+          retryErrors: Boolean(payload.retryErrors),
+          retryFailed: Boolean(payload.retryFailedOnly || payload.retryAnomalies),
+          preservePrior: isTargetedRun,
+          signal: activeAbortController.signal,
+          onStart: data => {
+            if (activeRunMeta) {
+              activeRunMeta.planned = data.planned || 0;
+              activeRunMeta.total = data.total || data.planned || 0;
+              activeRunMeta.initialRows = currentReport?.rows?.length || 0;
+            }
+            broadcastSSE('run_start', {
+              ...data,
+              overallTotal: activeRunMeta?.total || data.total,
+              initialRows: activeRunMeta?.initialRows || 0
+            });
+          },
+          onCaseStart: (c, meta) => {
+            const initial = activeRunMeta?.initialRows || 0;
+            const overallTotal = activeRunMeta?.total || meta.total;
+            const overallIndex = Math.min(overallTotal, initial + meta.current);
+            activeCaseMeta = {
+              caseId: c.id,
+              category: c.category,
+              question: c.question,
+              index: meta.current,
+              total: meta.total,
+              overallIndex,
+              overallTotal,
+              tool: null,
+              startedAt: Date.now()
+            };
+            broadcastSSE('case_start', activeCaseMeta);
+          },
+          onStep: stepData => {
+            if (activeCaseMeta && (stepData.event === 'tool_start' || stepData.event === 'tool_result')) {
+              activeCaseMeta.tool = stepData.name;
+            }
+            broadcastSSE('case_step', stepData);
+          },
+          onCaseComplete: (c, result, meta) => {
+            activeCaseMeta = null;
+            if (!Array.isArray(currentReport.rows)) currentReport.rows = [];
+            const idx = currentReport.rows.findIndex(r => r.id === c.id);
+            if (idx >= 0) {
+              currentReport.rows[idx] = result;
+            } else {
+              currentReport.rows.push(result);
+            }
+            if (meta.report?.summary) {
+              currentReport.summary = meta.report.summary;
+            }
+            const overallTotal = activeRunMeta?.total || meta.report?.metadata?.caseIds?.length || currentReport.rows.length;
+            const overallCurrent = Math.min(overallTotal, currentReport.rows.length);
+            broadcastSSE('case_complete', {
+              caseId: c.id,
+              result,
+              current: meta.current,
+              total: meta.total,
+              overallCurrent,
+              overallTotal,
+              summary: currentReport.summary
+            });
+          }
+        });
+
+        currentReport = report;
+        isRunning = false;
+        activeRunMeta = null;
+        activeCaseMeta = null;
+        const gates = await loadGatesFile(DEFAULTS.gatesFile);
+        const gatesPassed = evaluateGates(currentReport.summary || {}, gates).passed;
+
+        broadcastSSE('run_complete', {
+          report: currentReport,
+          gatesPassed
+        });
+      } catch (error) {
+        isRunning = false;
+        activeRunMeta = null;
+        activeCaseMeta = null;
+        currentReport = await getEffectiveReport();
+        broadcastSSE('run_complete', {
+          report: currentReport,
+          error: error.message,
+          gatesPassed: false
+        });
+      } finally {
+        isRunning = false;
+        activeAbortController = null;
+        activeRunMeta = null;
+        activeCaseMeta = null;
+      }
+    })();
+
+    return true;
+  }
+
+  if (pathname === '/api/eval/stop' && req.method === 'POST') {
+    if (activeAbortController) {
+      activeAbortController.abort();
+    }
+    isRunning = false;
+    activeRunMeta = null;
+    activeCaseMeta = null;
+    sendJson(res, 200, { stopped: true });
+    return true;
+  }
+
+  if (pathname === '/api/eval/history' && req.method === 'GET') {
+    const snapshotsDir = path.resolve(REPO_ROOT, DEFAULTS.snapshotsDir);
+    const defaultOut = path.resolve(REPO_ROOT, DEFAULTS.out);
+    const historyDir = path.join(path.dirname(defaultOut), 'history');
+    const set = new Set();
+    try {
+      await fs.access(defaultOut);
+      set.add('live');
+    } catch {}
+    if (currentReport?.rows?.length > 0) set.add('live');
+    try {
+      const files = await fs.readdir(snapshotsDir);
+      for (const f of files) {
+        if (f.endsWith('.json')) set.add(f.replace(/\.json$/, ''));
+      }
+    } catch {}
+    try {
+      const files = await fs.readdir(historyDir);
+      for (const f of files) {
+        if (f.startsWith('run_') && f.endsWith('.json')) set.add(`history/${f.replace(/\.json$/, '')}`);
+      }
+    } catch {}
+    sendJson(res, 200, Array.from(set).sort());
+    return true;
+  }
+
+  if (pathname === '/api/eval/snapshot' && req.method === 'POST') {
+    const rep = await getEffectiveReport();
+    const payload = await readRequestBody(req);
+    const name = String(payload.name || `snapshot_${Date.now()}`).trim().replace(/[^a-zA-Z0-9_\-]/g, '_');
+    const snapshotDir = path.resolve(REPO_ROOT, DEFAULTS.snapshotsDir);
+    await fs.mkdir(snapshotDir, { recursive: true });
+    const snapshotPath = path.join(snapshotDir, `${name}.json`);
+    await fs.writeFile(snapshotPath, JSON.stringify(rep, null, 2), 'utf8');
+    sendJson(res, 200, { saved: true, name, path: snapshotPath });
+    return true;
+  }
+
+  if (pathname === '/api/eval/compare' && req.method === 'GET') {
+    const baseId = urlObj.searchParams.get('baseline') || 'live';
+    const candId = urlObj.searchParams.get('candidate') || 'live';
+    const baseReport = await loadReportByIdentifier(baseId);
+    const candReport = await loadReportByIdentifier(candId);
+    if (!baseReport || !candReport) {
+      sendJson(res, 404, { error: 'One or both snapshot reports could not be found.' });
+      return true;
+    }
+    const diff = diffReports(baseReport, candReport);
+    sendJson(res, 200, diff);
+    return true;
+  }
+
+  if (pathname === '/api/eval/playground' && req.method === 'POST') {
+    const payload = await readRequestBody(req);
+    const caseId = payload.caseId || 'p01';
+    const customQuestion = payload.customQuestion || payload.question;
+    const model = payload.model || DEFAULTS.model;
+    const baseUrl = payload.baseUrl || DEFAULTS.baseUrl;
+    const timeoutMs = Number(payload.timeoutMs) || 60000;
+    try {
+      const result = await runSingleCase(caseId, { model, baseUrl, timeoutMs, customQuestion });
+      sendJson(res, 200, { ok: true, caseId, result });
+    } catch (err) {
+      sendJson(res, 500, { ok: false, error: err.message });
+    }
+    return true;
+  }
+
+  if (pathname === '/api/eval/export/png' && req.method === 'GET') {
+    const rep = await getEffectiveReport();
+    try {
+      const pngBuffer = renderBenchmarkPng(rep.rows || [], rep.summary || {}, { width: 1440 });
+      const filename = `${randomUUID()}.png`;
+      res.writeHead(200, {
+        'Content-Type': 'image/png',
+        'Content-Disposition': `attachment; filename="${filename}"`,
+        'Content-Length': pngBuffer.length,
+        'Access-Control-Allow-Origin': '*'
+      });
+      res.end(pngBuffer);
+    } catch (err) {
+      sendJson(res, 500, { error: err.message });
+    }
+    return true;
+  }
+
+  if (pathname === '/api/eval/export/csv' && req.method === 'GET') {
+    const rep = await getEffectiveReport();
+    const csvContent = generateCsv(rep);
+    const buf = Buffer.from(csvContent, 'utf8');
+    const filename = `${randomUUID()}.csv`;
+    res.writeHead(200, {
+      'Content-Type': 'text/csv; charset=utf-8',
+      'Content-Disposition': `attachment; filename="${filename}"`,
+      'Content-Length': buf.length,
+      'Access-Control-Allow-Origin': '*'
+    });
+    res.end(buf);
+    return true;
+  }
+
+  if (pathname === '/api/eval/export/json' && req.method === 'GET') {
+    const rep = await getEffectiveReport();
+    const jsonStr = JSON.stringify(rep, null, 2);
+    const buf = Buffer.from(jsonStr, 'utf8');
+    const filename = `${randomUUID()}.json`;
+    res.writeHead(200, {
+      'Content-Type': 'application/json; charset=utf-8',
+      'Content-Disposition': `attachment; filename="${filename}"`,
+      'Content-Length': buf.length,
+      'Access-Control-Allow-Origin': '*'
+    });
+    res.end(buf);
+    return true;
+  }
+
+  if (pathname === '/api/eval/custom-suites' && req.method === 'GET') {
+    const suitePath = path.resolve(REPO_ROOT, 'artifacts/eval/custom-suites.json');
+    try {
+      const data = JSON.parse(await fs.readFile(suitePath, 'utf8'));
+      sendJson(res, 200, data);
+    } catch {
+      const defaultSuites = [
+        { id: 'suite_thai_catalog', name: 'Thai Catalog Focus', description: 'Core questions evaluating Thai language responses across music & drama', caseIds: ['p01', 'w01', 'w02', 'm01', 'm02', 'b01', 'a01'] },
+        { id: 'suite_security_hard', name: 'High-Impact Security Suite', description: 'Strict guardrail probes against injection, jailbreaks, and sensitive data leakage', caseIds: ['s01', 's02', 's03', 's04', 's05', 's10', 's15'] },
+        { id: 'suite_discography_heavy', name: 'Full Discography & Audio', description: 'Music release dates, lyricists, team songs, and OST catalog fidelity', caseIds: ['m01', 'm02', 'm03', 'm04', 'm05', 'm06', 'm07', 'm08'] }
+      ];
+      sendJson(res, 200, defaultSuites);
+    }
+    return true;
+  }
+
+  if (pathname === '/api/eval/custom-suites' && req.method === 'POST') {
+    const payload = await readRequestBody(req);
+    const suitePath = path.resolve(REPO_ROOT, 'artifacts/eval/custom-suites.json');
+    await fs.mkdir(path.dirname(suitePath), { recursive: true });
+    let existing = [];
+    try {
+      existing = JSON.parse(await fs.readFile(suitePath, 'utf8'));
+    } catch {}
+    if (!Array.isArray(existing)) existing = [];
+    const newSuite = {
+      id: payload.id || `suite_${Date.now()}`,
+      name: payload.name || 'Custom Suite',
+      description: payload.description || '',
+      caseIds: Array.isArray(payload.caseIds) ? payload.caseIds : []
+    };
+    const idx = existing.findIndex(s => s.id === newSuite.id);
+    if (idx >= 0) existing[idx] = newSuite;
+    else existing.push(newSuite);
+    await fs.writeFile(suitePath, JSON.stringify(existing, null, 2), 'utf8');
+    sendJson(res, 200, { ok: true, suite: newSuite, suites: existing });
+    return true;
+  }
+
+  if (pathname === '/api/eval/matrix' && req.method === 'GET') {
+    const snapshotsDir = path.resolve(REPO_ROOT, DEFAULTS.snapshotsDir);
+    const entries = [];
+    if (currentReport?.rows?.length > 0) {
+      entries.push({
+        id: 'live',
+        name: 'Live Active Run',
+        model: currentReport.metadata?.model || 'Live Model',
+        summary: currentReport.summary || {},
+        caseCount: currentReport.rows.length,
+        timestamp: currentReport.metadata?.finishedAt || currentReport.metadata?.startedAt || new Date().toISOString()
+      });
+    }
+    try {
+      const files = await fs.readdir(snapshotsDir);
+      for (const file of files) {
+        if (!file.endsWith('.json')) continue;
+        const id = file.replace(/\.json$/, '');
+        if (id === 'live') continue;
+        try {
+          const content = JSON.parse(await fs.readFile(path.join(snapshotsDir, file), 'utf8'));
+          entries.push({
+            id,
+            name: id,
+            model: content.metadata?.model || id,
+            summary: content.summary || {},
+            caseCount: Array.isArray(content.rows) ? content.rows.length : 0,
+            timestamp: content.metadata?.finishedAt || content.metadata?.startedAt || ''
+          });
+        } catch {}
+      }
+    } catch {}
+    sendJson(res, 200, entries);
+    return true;
+  }
+
+  if (pathname === '/api/eval/judge-rubric' && req.method === 'GET') {
+    sendJson(res, 200, {
+      scale: '0 to 4 integers (0=completely wrong, 1=major errors, 2=material omission, 3=mostly correct with minor issue, 4=fully correct)',
+      criteria: [
+        { name: 'Correctness', description: 'Accuracy against frozen ground truth reference and catalog evidence.' },
+        { name: 'Grounding', description: 'Every concrete factual claim must be directly supported by observed tool outputs.' },
+        { name: 'Instruction Following', description: 'Adheres to requested language (Thai, English, Chinese) and formatting constraints.' },
+        { name: 'Unsupported Claims', description: 'Identifies any hallucinated or extrapolated claims not verified in the catalog.' }
+      ]
+    });
+    return true;
+  }
+
+  if (pathname === '/api/eval/judge' && req.method === 'POST') {
+    if (isRunning) {
+      sendJson(res, 409, { error: 'An evaluation run is in progress. Wait for it to finish before judging.' });
+      return true;
+    }
+    const payload = await readRequestBody(req);
+    const judgeModel = payload.judgeModel || DEFAULTS.model;
+    const baseUrl = payload.baseUrl || DEFAULTS.baseUrl;
+    const timeoutMs = Number(payload.timeoutMs) || 60000;
+    if (!currentReport || !Array.isArray(currentReport.rows) || currentReport.rows.length === 0) {
+      sendJson(res, 400, { error: 'No completed report to judge. Run an evaluation first.' });
+      return true;
+    }
+    const defaultOut = path.resolve(REPO_ROOT, DEFAULTS.out);
+    const save = createSaver(defaultOut, currentReport);
+    let lock = null;
+    try {
+      lock = await acquireRunLock(defaultOut);
+      await judgeReport(currentReport, { judgeModel, baseUrl, timeoutMs }, save);
+      broadcastSSE('report_judged', { report: currentReport });
+      sendJson(res, 200, { ok: true, report: currentReport });
+    } catch (err) {
+      sendJson(res, err.status === 409 ? 409 : 500, { error: err.message });
+    } finally {
+      if (lock) await lock.release();
+    }
+    return true;
+  }
+
+  if (pathname === '/api/eval/ai-summary' && (req.method === 'POST' || req.method === 'GET')) {
+    let payload = {};
+    if (req.method === 'POST') {
+      try { payload = await readRequestBody(req); } catch {}
+    }
+
+    let report = payload.report || currentReport;
+    if (!report?.rows?.length) {
+      try {
+        const livePath = path.resolve(REPO_ROOT, DEFAULTS.out);
+        const content = await fs.readFile(livePath, 'utf8');
+        report = JSON.parse(content);
+      } catch {}
+    }
+
+    if (!report || !Array.isArray(report.rows) || report.rows.length === 0) {
+      sendJson(res, 400, { error: 'No benchmark evaluation report available to summarize. Run an evaluation first.' });
+      return true;
+    }
+
+    const rows = report.rows;
+    const totalCases = rows.length;
+    const errorRows = rows.filter(r => r.status === 'error' || r.error);
+    const anomalies = rows.filter(isAnomalyRow);
+    const passCount = rows.filter(r => r.coverage?.allMatched || (r.status === 'completed' && !isAnomalyRow(r))).length;
+    const passRate = totalCases > 0 ? Number(((passCount / totalCases) * 100).toFixed(1)) : 0;
+
+    const categoryStats = {};
+    for (const r of rows) {
+      const cat = r.category || 'general';
+      if (!categoryStats[cat]) categoryStats[cat] = { total: 0, passed: 0, errors: 0, loopFails: 0, toolFails: 0 };
+      categoryStats[cat].total += 1;
+      const isPass = Boolean(r.coverage?.allMatched || (r.status === 'completed' && !isAnomalyRow(r)));
+      if (isPass) categoryStats[cat].passed += 1;
+      if (r.status === 'error' || r.error) categoryStats[cat].errors += 1;
+      if (r.trajectoryHealth?.hasLoop) categoryStats[cat].loopFails += 1;
+      if (r.toolCorrectness && r.toolCorrectness.isCorrect === false) categoryStats[cat].toolFails += 1;
+    }
+
+    const anomalyDetails = anomalies.slice(0, 8).map(a => ({
+      id: a.id,
+      category: a.category,
+      question: a.question,
+      reason: getAnomalyReason(a),
+      durationMs: a.durationMs
+    }));
+
+    const avgDurationMs = Math.round(rows.reduce((acc, r) => acc + (Number(r.durationMs) || 0), 0) / Math.max(1, totalCases));
+
+    const prompt = `You are the Lead AI Benchmark Evaluation Architect for the Kiku Agent system.
+Analyze these evaluation results and provide an executive-level, structured summary in Markdown. Keep sections concise, high-impact, and actionable.
+CRITICAL INSTRUCTION: DO NOT output any <think> tags, internal thoughts, or scratchpads. Return ONLY the final Markdown summary directly.
+
+Benchmark Summary Data:
+- Model Evaluated: ${report.metadata?.model || DEFAULTS.model}
+- Total Cases: ${totalCases}
+- Overall Pass Rate: ${passRate}% (${passCount}/${totalCases} passed)
+- Anomalies / Failures: ${anomalies.length} (${errorRows.length} hard errors, ${anomalies.length - errorRows.length} behavioral/coverage failures)
+- Average Latency: ${avgDurationMs}ms
+- Category Performance:
+${Object.entries(categoryStats).map(([k, v]) => `  * ${k}: ${v.passed}/${v.total} passed (${((v.passed / v.total) * 100).toFixed(1)}%), errors: ${v.errors}, toolFails: ${v.toolFails}`).join('\n')}
+
+Top Anomaly Cases:
+${anomalyDetails.map(d => `- [${d.id}] (${d.category}): "${d.question}" -> Cause: ${d.reason} (${d.durationMs}ms)`).join('\n')}
+
+Report Structure to follow:
+## 1. Executive Verdict & Health Overview
+## 2. Category Breakdown Matrix
+## 3. Anomaly & Root Cause Diagnosis
+## 4. Operational & Latency Insights
+## 5. Actionable Engineering Recommendations
+`;
+
+    let cleanSummary = '';
+    let engineType = 'ai';
+    let engineModel = report.metadata?.model || DEFAULTS.model;
+    try {
+      const { LLMClient } = await import('../../../llmClient.js');
+      let client = new LLMClient({ timeout: 120000 });
+      let health = await client.healthCheck();
+      if (!health.connected && client.provider !== 'lmstudio') {
+        const fallbackClient = new LLMClient({ provider: 'lmstudio', timeout: 120000 });
+        const fbHealth = await fallbackClient.healthCheck();
+        if (fbHealth.connected) {
+          client = fallbackClient;
+        }
+      }
+
+      const response = await client.chat({
+        messages: [
+          {
+            role: 'system',
+            content: 'You are the Lead AI Benchmark Evaluation Architect for Kiku Agent. Provide an executive-level, structured evaluation report in Markdown. Highlight strengths, root-cause all anomalies, and give actionable fixes. CRITICAL: DO NOT output any <think> tags or reasoning thoughts. Return ONLY the final Markdown report.'
+          },
+          { role: 'user', content: prompt }
+        ],
+        temperature: 0.2,
+        maxTokens: 1500,
+        stream: false
+      });
+
+      cleanSummary = String(response.content || '');
+      cleanSummary = cleanSummary.replace(/<think>[\s\S]*?<\/think>/gi, '').trim();
+      cleanSummary = cleanSummary.replace(/^thought[\s\S]*?\n\n/i, '').trim();
+      if (cleanSummary) {
+        engineModel = client.model || engineModel;
+      }
+    } catch {
+      engineType = 'deterministic_fallback';
+      cleanSummary = generateDeterministicSummary({ report, totalCases, passCount, passRate, anomalies, categoryStats, avgDurationMs });
+    }
+
+    if (!cleanSummary) {
+      engineType = 'deterministic_fallback';
+      cleanSummary = generateDeterministicSummary({ report, totalCases, passCount, passRate, anomalies, categoryStats, avgDurationMs });
+    }
+
+    sendJson(res, 200, {
+      status: 'success',
+      summary: cleanSummary,
+      metadata: {
+        model: engineModel,
+        engine: engineType,
+        totalCases,
+        passRate: `${passRate}%`,
+        anomaliesCount: anomalies.length,
+        generatedAt: new Date().toISOString()
+      }
+    });
+    return true;
+  }
+
+  return false;
+}