localarena 0.1.0 → 0.2.0

package/javascript/src/report.js

@@ -0,0 +1,355 @@
+import { writeFile } from "node:fs/promises";
+
+import { EvaluationRun } from "./evaluation.js";
+
+function escapeHTML(value) {
+  const text = value === null || value === undefined ? "" : String(value);
+  return text
+    .replaceAll("&", "&")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#x27;");
+}
+
+function formatNumber(value, digits = 3) {
+  if (value === null || value === undefined) {
+    return "—";
+  }
+  if (typeof value !== "number") {
+    return String(value);
+  }
+  return value.toFixed(digits).replace(/\.?0+$/u, "");
+}
+
+export function renderHTMLReport(run, options = {}) {
+  if (!(run instanceof EvaluationRun)) {
+    throw new TypeError("run must be an EvaluationRun");
+  }
+  if (options === null || typeof options !== "object" || Array.isArray(options)) {
+    throw new TypeError("options must be an object");
+  }
+  if (options.title !== undefined && typeof options.title !== "string") {
+    throw new TypeError("title must be a string");
+  }
+
+  const reportTitle = options.title || run.name;
+  const summary = run.summary();
+  const taskIds = run.tasks.map(({ id }) => String(id));
+  const errors = run.records.filter(({ error }) => error !== null).length;
+  const scored = run.records.filter(({ score }) => score !== null).length;
+  const totalSeconds = run.records.reduce(
+    (total, record) => total + record.durationSeconds,
+    0,
+  );
+  const judgeRows = run.tasks
+    .filter(
+      (task) =>
+        task.evaluator?.type === "model_judge" &&
+        task.evaluator.target !== null &&
+        typeof task.evaluator.target === "object",
+    )
+    .map((task) => {
+      const target = task.evaluator.target;
+      const parameters =
+        target.parameters !== null &&
+        typeof target.parameters === "object"
+          ? target.parameters
+          : {};
+      return `
+            <tr>
+              <td>${escapeHTML(task.id)}</td>
+              <td><strong>${escapeHTML(target.name)}</strong></td>
+              <td>${escapeHTML(target.provider)}</td>
+              <td>${escapeHTML(target.model)}</td>
+              <td class="metric">${escapeHTML(parameters.max_tokens)}</td>
+              <td class="metric">${formatNumber(parameters.temperature)}</td>
+            </tr>`;
+    })
+    .join("");
+  const judgeSection =
+    judgeRows.length === 0
+      ? ""
+      : `
+    <section>
+      <h2>Live judges</h2>
+      <div class="table-wrap">
+        <table>
+          <thead><tr><th>Task</th><th>Target</th><th>Provider</th><th>Model</th><th>Max tokens</th><th>Temperature</th></tr></thead>
+          <tbody>${judgeRows}</tbody>
+        </table>
+      </div>
+    </section>`;
+
+  const scoreRows = summary
+    .map((row) => {
+      const score = row.average_score;
+      const width =
+        score === null ? 0 : Math.max(0, Math.min(100, score * 100));
+      return `
+            <tr>
+              <td><strong>${escapeHTML(row.name)}</strong><br>
+                <span class="muted">${escapeHTML(row.provider)} · ${escapeHTML(row.model)}</span>
+              </td>
+              <td class="metric">${formatNumber(score)}</td>
+              <td class="chart-cell"><div class="bar-track"><div class="bar" style="width:${width.toFixed(2)}%"></div></div></td>
+              <td class="metric">${formatNumber(row.pass_rate)}</td>
+              <td class="metric">${formatNumber(row.elo, 1)}</td>
+              <td class="metric">${formatNumber(row.average_latency_seconds)}</td>
+              <td class="metric">${escapeHTML(row.input_tokens)} / ${escapeHTML(row.output_tokens)}</td>
+              <td class="metric">${formatNumber(row.judge_latency_seconds)}</td>
+              <td class="metric">${escapeHTML(row.judge_input_tokens)} / ${escapeHTML(row.judge_output_tokens)}</td>
+              <td class="metric">${escapeHTML(row.successful)} / ${escapeHTML(row.runs)}</td>
+              <td class="metric error-count">${escapeHTML(row.errors)}</td>
+            </tr>`;
+    })
+    .join("");
+
+  const grouped = new Map();
+  const failures = new Set();
+  for (const record of run.records) {
+    const key = JSON.stringify([record.target, record.taskId]);
+    if (record.score !== null) {
+      if (!grouped.has(key)) {
+        grouped.set(key, []);
+      }
+      grouped.get(key).push(record.score.value);
+    }
+    if (record.error !== null) {
+      failures.add(key);
+    }
+  }
+
+  const matrixHeader = taskIds
+    .map((taskId) => `<th>${escapeHTML(taskId)}</th>`)
+    .join("");
+  const matrixRows = run.models
+    .map((model) => {
+      const name = String(model.name);
+      const cells = taskIds
+        .map((taskId) => {
+          const key = JSON.stringify([name, taskId]);
+          const values = grouped.get(key) ?? [];
+          if (values.length > 0) {
+            const value =
+              values.reduce((total, item) => total + item, 0) /
+              values.length;
+            const hue = value * 120;
+            return `<td class="heat" style="--heat:${hue.toFixed(1)}">${formatNumber(value)}</td>`;
+          }
+          if (failures.has(key)) {
+            return '<td class="heat failed">error</td>';
+          }
+          return '<td class="heat missing">—</td>';
+        })
+        .join("");
+      return `<tr><th>${escapeHTML(name)}</th>${cells}</tr>`;
+    })
+    .join("");
+
+  const detailRows = run.records
+    .map((record) => {
+      const scoreText =
+        record.score === null ? "—" : formatNumber(record.score.value);
+      const answer =
+        run.includeContent && record.generation !== null
+          ? record.generation.text
+          : "";
+      const reason =
+        run.includeContent && record.score !== null
+          ? record.score.reason
+          : record.score === null
+            ? "No score"
+            : "Not retained";
+      const error =
+        run.includeContent && record.error !== null
+          ? `<h4>Error</h4><pre>${escapeHTML(record.error)}</pre>`
+          : "";
+      return `
+            <details class="record ${record.error ? "record-error" : ""}">
+              <summary>
+                <span class="status status-${escapeHTML(record.status)}">${escapeHTML(record.status)}</span>
+                <strong>${escapeHTML(record.target)}</strong>
+                <span>× ${escapeHTML(record.taskId)}</span>
+                <span class="grow"></span>
+                <span>score ${scoreText}</span>
+                <span>${formatNumber(record.durationSeconds)}s</span>
+              </summary>
+              <div class="record-grid">
+                <div><h4>Answer</h4><pre>${answer ? escapeHTML(answer) : "Not retained"}</pre></div>
+                <div>
+                  <h4>Evaluation</h4>
+                  <p>${escapeHTML(reason)}</p>
+                  <h4>Provider result</h4>
+                  <dl>
+                    <dt>Provider</dt><dd>${escapeHTML(record.provider)}</dd>
+                    <dt>Model</dt><dd>${escapeHTML(record.model)}</dd>
+                    <dt>Repetition</dt><dd>${record.repetition}</dd>
+                    <dt>Finish reason</dt><dd>${record.generation ? escapeHTML(record.generation.finishReason ?? "—") : "—"}</dd>
+                  </dl>
+                  ${error}
+                </div>
+              </div>
+            </details>`;
+    })
+    .join("");
+
+  return `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="color-scheme" content="dark light">
+  <link rel="icon" href="data:,">
+  <title>${escapeHTML(reportTitle)} · LocalArena</title>
+  <style>
+    :root {
+      --bg: #0b1020;
+      --panel: #121a2d;
+      --panel-2: #18233a;
+      --text: #e8edf7;
+      --muted: #9aa8bf;
+      --line: #2a3853;
+      --accent: #5eead4;
+      --accent-2: #60a5fa;
+      --danger: #fb7185;
+      --shadow: 0 18px 50px rgba(0,0,0,.22);
+    }
+    * { box-sizing: border-box; }
+    body {
+      margin: 0;
+      background: radial-gradient(circle at 12% 0%, #172554 0, transparent 30rem), var(--bg);
+      color: var(--text);
+      font: 15px/1.5 ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+    }
+    main { width: min(1380px, calc(100% - 32px)); margin: 42px auto 80px; }
+    h1 { font-size: clamp(30px, 5vw, 54px); line-height: 1.05; margin: 0 0 10px; letter-spacing: -.04em; }
+    h2 { margin: 0 0 20px; font-size: 22px; }
+    h4 { margin: 0 0 8px; }
+    .eyebrow { color: var(--accent); font-size: 12px; font-weight: 800; letter-spacing: .16em; text-transform: uppercase; }
+    .subtitle { color: var(--muted); margin: 0; }
+    .cards { display: grid; grid-template-columns: repeat(5, minmax(120px, 1fr)); gap: 12px; margin: 28px 0; }
+    .card, section {
+      border: 1px solid var(--line);
+      border-radius: 16px;
+      background: color-mix(in srgb, var(--panel) 94%, transparent);
+      box-shadow: var(--shadow);
+    }
+    .card { padding: 18px; }
+    .card .value { display: block; font-size: 27px; font-weight: 800; }
+    .card .label, .muted { color: var(--muted); }
+    section { padding: 22px; margin-top: 18px; overflow: hidden; }
+    .table-wrap { overflow-x: auto; }
+    table { width: 100%; border-collapse: collapse; }
+    th, td { border-bottom: 1px solid var(--line); padding: 12px 10px; text-align: left; vertical-align: middle; }
+    th { color: var(--muted); font-size: 12px; letter-spacing: .05em; text-transform: uppercase; }
+    tbody tr:last-child td, tbody tr:last-child th { border-bottom: 0; }
+    .metric { font-variant-numeric: tabular-nums; white-space: nowrap; }
+    .chart-cell { min-width: 180px; width: 28%; }
+    .bar-track { width: 100%; height: 11px; border-radius: 999px; background: var(--panel-2); overflow: hidden; }
+    .bar { height: 100%; border-radius: inherit; background: linear-gradient(90deg, var(--accent-2), var(--accent)); }
+    .error-count { color: var(--danger); }
+    .matrix th:first-child { position: sticky; left: 0; background: var(--panel); z-index: 1; }
+    .heat { text-align: center; font-weight: 800; color: white; background: hsl(var(--heat) 48% 33%); min-width: 90px; }
+    .heat.failed { background: #7f1d1d; }
+    .heat.missing { background: var(--panel-2); color: var(--muted); }
+    details.record { border-top: 1px solid var(--line); }
+    details.record:first-of-type { border-top: 0; }
+    summary { cursor: pointer; display: flex; align-items: center; gap: 13px; padding: 14px 4px; list-style: none; }
+    summary::-webkit-details-marker { display: none; }
+    summary::before { content: "›"; color: var(--muted); font-size: 22px; transition: transform .15s ease; }
+    details[open] summary::before { transform: rotate(90deg); }
+    .grow { flex: 1; }
+    .status { padding: 3px 8px; border-radius: 999px; font-size: 11px; font-weight: 800; text-transform: uppercase; background: #164e63; }
+    .status-generation_error, .status-score_error { background: #881337; }
+    .record-grid { display: grid; grid-template-columns: 1.5fr 1fr; gap: 18px; padding: 4px 4px 20px 42px; }
+    pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 0; padding: 13px; border-radius: 10px; background: #070b15; color: #dbeafe; max-height: 420px; overflow: auto; }
+    dl { display: grid; grid-template-columns: max-content 1fr; gap: 5px 12px; margin: 0; }
+    dt { color: var(--muted); }
+    dd { margin: 0; overflow-wrap: anywhere; }
+    footer { color: var(--muted); text-align: center; margin-top: 26px; font-size: 13px; }
+    @media (max-width: 820px) {
+      .cards { grid-template-columns: repeat(2, 1fr); }
+      .record-grid { grid-template-columns: 1fr; padding-left: 4px; }
+      summary span:not(.status):not(.grow) { display: none; }
+    }
+    @media (prefers-color-scheme: light) {
+      :root {
+        --bg: #f5f7fb;
+        --panel: #ffffff;
+        --panel-2: #edf2f7;
+        --text: #172033;
+        --muted: #60708a;
+        --line: #d8e0eb;
+        --shadow: 0 16px 45px rgba(35,50,80,.09);
+      }
+      body { background: radial-gradient(circle at 12% 0%, #dbeafe 0, transparent 30rem), var(--bg); }
+      pre { background: #111827; color: #e5edff; }
+      .matrix th:first-child { background: var(--panel); }
+    }
+  </style>
+</head>
+<body>
+  <main>
+    <header>
+      <div class="eyebrow">LocalArena live evaluation</div>
+      <h1>${escapeHTML(reportTitle)}</h1>
+      <p class="subtitle">Run ${escapeHTML(run.id)} · ${escapeHTML(run.startedAt)} → ${escapeHTML(run.finishedAt)}</p>
+    </header>
+
+    <div class="cards">
+      <div class="card"><span class="value">${run.models.length}</span><span class="label">models</span></div>
+      <div class="card"><span class="value">${run.tasks.length}</span><span class="label">tasks</span></div>
+      <div class="card"><span class="value">${run.records.length}</span><span class="label">evaluation rows</span></div>
+      <div class="card"><span class="value">${scored}</span><span class="label">scored</span></div>
+      <div class="card"><span class="value">${errors}</span><span class="label">errors · ${formatNumber(totalSeconds, 1)}s total</span></div>
+    </div>
+
+    <section>
+      <h2>Leaderboard</h2>
+      <div class="table-wrap">
+        <table>
+          <thead><tr><th>Model</th><th>Score</th><th>Relative score</th><th>Pass rate</th><th>Elo</th><th>Candidate latency (s)</th><th>Candidate tokens in / out</th><th>Judge latency (s)</th><th>Judge tokens in / out</th><th>Successful / runs</th><th>Errors</th></tr></thead>
+          <tbody>${scoreRows}</tbody>
+        </table>
+      </div>
+    </section>
+
+    ${judgeSection}
+
+    <section>
+      <h2>Task matrix</h2>
+      <div class="table-wrap">
+        <table class="matrix">
+          <thead><tr><th>Model</th>${matrixHeader}</tr></thead>
+          <tbody>${matrixRows}</tbody>
+        </table>
+      </div>
+    </section>
+
+    <section>
+      <h2>Outputs and diagnostics</h2>
+      ${detailRows}
+    </section>
+
+    <footer>Generated locally by LocalArena. Provider credentials and endpoint URLs are not included.</footer>
+  </main>
+</body>
+</html>
+`;
+}
+
+export async function writeHTMLReport(run, path, options = {}) {
+  if (
+    typeof path !== "string" &&
+    !(path instanceof URL) &&
+    !Buffer.isBuffer(path)
+  ) {
+    throw new TypeError("path must be a string, URL, or Buffer");
+  }
+  await writeFile(path, renderHTMLReport(run, options), "utf8");
+  return path;
+}
+
+export const renderHtmlReport = renderHTMLReport;
+export const writeHtmlReport = writeHTMLReport;