audrey 0.17.0 → 0.20.0

package/benchmarks/report.js

@@ -1,255 +0,0 @@
-import { mkdirSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-
-const PALETTE = {
-  audrey: '#0f766e',
-  vector: '#0369a1',
-  keyword: '#6d28d9',
-  recent: '#b45309',
-  external: '#1d4ed8',
-  accent: '#111827',
-  muted: '#6b7280',
-  surface: '#f8fafc',
-  border: '#cbd5e1',
-};
-
-function escapeHtml(text) {
-  return String(text)
-    .replaceAll('&', '&')
-    .replaceAll('<', '&lt;')
-    .replaceAll('>', '&gt;')
-    .replaceAll('"', '&quot;');
-}
-
-function chartBarColor(label) {
-  if (label === 'Audrey') return PALETTE.audrey;
-  if (label.includes('Vector')) return PALETTE.vector;
-  if (label.includes('Keyword')) return PALETTE.keyword;
-  if (label.includes('Recent')) return PALETTE.recent;
-  return PALETTE.external;
-}
-
-function renderBarChart({ title, rows, valueSuffix = '%', maxValue = 100 }) {
-  const width = 960;
-  const height = 420;
-  const margin = { top: 56, right: 32, bottom: 88, left: 64 };
-  const plotWidth = width - margin.left - margin.right;
-  const plotHeight = height - margin.top - margin.bottom;
-  const barWidth = Math.max(32, Math.floor(plotWidth / Math.max(rows.length, 1)) - 18);
-  const gap = rows.length > 1 ? (plotWidth - barWidth * rows.length) / (rows.length - 1) : 0;
-
-  const bars = rows.map((row, index) => {
-    const value = Math.max(0, Math.min(maxValue, row.value));
-    const barHeight = (value / maxValue) * plotHeight;
-    const x = margin.left + index * (barWidth + gap);
-    const y = margin.top + plotHeight - barHeight;
-    return `
-      <rect x="${x}" y="${y}" width="${barWidth}" height="${barHeight}" rx="8" fill="${chartBarColor(row.label)}" />
-      <text x="${x + barWidth / 2}" y="${y - 10}" text-anchor="middle" font-size="15" fill="${PALETTE.accent}">${value.toFixed(1)}${valueSuffix}</text>
-      <text x="${x + barWidth / 2}" y="${height - 42}" text-anchor="middle" font-size="14" fill="${PALETTE.muted}">${escapeHtml(row.label)}</text>
-    `;
-  }).join('\n');
-
-  const grid = [0, 25, 50, 75, 100].map(tick => {
-    const y = margin.top + plotHeight - (tick / maxValue) * plotHeight;
-    return `
-      <line x1="${margin.left}" y1="${y}" x2="${width - margin.right}" y2="${y}" stroke="${PALETTE.border}" stroke-dasharray="4 4" />
-      <text x="${margin.left - 10}" y="${y + 5}" text-anchor="end" font-size="13" fill="${PALETTE.muted}">${tick}${valueSuffix}</text>
-    `;
-  }).join('\n');
-
-  return `<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" role="img" aria-label="${escapeHtml(title)}">
-  <rect width="100%" height="100%" fill="white" />
-  <text x="${margin.left}" y="34" font-size="24" font-weight="700" fill="${PALETTE.accent}">${escapeHtml(title)}</text>
-  ${grid}
-  ${bars}
-</svg>`;
-}
-
-function renderTrendList(trends) {
-  return trends.map(trend => `
-    <li>
-      <strong>${escapeHtml(trend.title)}</strong><br />
-      ${escapeHtml(trend.summary)}<br />
-      <a href="${trend.source}">${escapeHtml(trend.source)}</a>
-    </li>
-  `).join('\n');
-}
-
-function renderCaseRows(localCases) {
-  return localCases.map(caseResult => `
-    <tr>
-      <td>${escapeHtml(caseResult.title)}</td>
-      <td>${escapeHtml(caseResult.suite)}</td>
-      <td>${escapeHtml(caseResult.family)}</td>
-      ${caseResult.results.map(result => {
-        const bg = result.passed ? '#ecfdf5' : result.score >= 0.5 ? '#fff7ed' : '#fef2f2';
-        const fg = result.passed ? '#065f46' : result.score >= 0.5 ? '#9a3412' : '#991b1b';
-        return `<td style="background:${bg};color:${fg}">${result.score.toFixed(2)}<br /><span style="font-size:12px">${escapeHtml(result.summary)}</span></td>`;
-      }).join('')}
-    </tr>
-  `).join('\n');
-}
-
-function renderSuiteSections(suiteCharts) {
-  if (suiteCharts.length === 0) return '';
-  return suiteCharts.map(chart => `
-    <section class="callout">
-      <h2>${escapeHtml(chart.title)}</h2>
-      <p>${escapeHtml(chart.description)}</p>
-      <img src="./${escapeHtml(chart.fileName)}" alt="${escapeHtml(chart.title)} chart" />
-    </section>
-  `).join('\n');
-}
-
-export function writeBenchmarkArtifacts({
-  outputDir,
-  summary,
-  localOverall,
-  localSuites,
-  externalOverall,
-  trends,
-  readmeAssetsDir,
-}) {
-  mkdirSync(outputDir, { recursive: true });
-
-  const localChart = renderBarChart({
-    title: 'Audrey vs Local Memory Baselines',
-    rows: localOverall.map(row => ({ label: row.system, value: row.scorePercent })),
-  });
-  const externalChart = renderBarChart({
-    title: 'Published LLM Memory Standards (LoCoMo)',
-    rows: externalOverall.map(row => ({ label: row.system, value: row.score })),
-  });
-
-  writeFileSync(join(outputDir, 'local-overall.svg'), localChart, 'utf8');
-  writeFileSync(join(outputDir, 'published-locomo.svg'), externalChart, 'utf8');
-  writeFileSync(join(outputDir, 'summary.json'), JSON.stringify(summary, null, 2), 'utf8');
-
-  const suiteCharts = localSuites.map(suite => {
-    const fileName = `${suite.id}-overall.svg`;
-    const chart = renderBarChart({
-      title: `${suite.title} Benchmark`,
-      rows: suite.overall.map(row => ({ label: row.system, value: row.scorePercent })),
-    });
-    writeFileSync(join(outputDir, fileName), chart, 'utf8');
-    return {
-      id: suite.id,
-      title: `${suite.title} Benchmark`,
-      description: suite.description,
-      fileName,
-      path: join(outputDir, fileName),
-    };
-  });
-
-  let readmeAssets = null;
-  if (readmeAssetsDir) {
-    mkdirSync(readmeAssetsDir, { recursive: true });
-    const localReadmeChart = join(readmeAssetsDir, 'local-benchmark.svg');
-    const externalReadmeChart = join(readmeAssetsDir, 'published-memory-standards.svg');
-    writeFileSync(localReadmeChart, localChart, 'utf8');
-    writeFileSync(externalReadmeChart, externalChart, 'utf8');
-
-    const operationsSuite = suiteCharts.find(chart => chart.id === 'operations');
-    let operationsReadmeChart = null;
-    if (operationsSuite) {
-      operationsReadmeChart = join(readmeAssetsDir, 'operations-benchmark.svg');
-      writeFileSync(
-        operationsReadmeChart,
-        renderBarChart({
-          title: 'Audrey Memory Operations Benchmark',
-          rows: (localSuites.find(suite => suite.id === 'operations')?.overall || [])
-            .map(row => ({ label: row.system, value: row.scorePercent })),
-        }),
-        'utf8',
-      );
-    }
-
-    readmeAssets = {
-      localChart: localReadmeChart,
-      operationsChart: operationsReadmeChart,
-      externalChart: externalReadmeChart,
-    };
-  }
-
-  const html = `<!doctype html>
-<html lang="en">
-<head>
-  <meta charset="utf-8" />
-  <title>Audrey Memory Benchmark</title>
-  <style>
-    body { font-family: "Segoe UI", Arial, sans-serif; margin: 32px; color: ${PALETTE.accent}; background: ${PALETTE.surface}; }
-    main { max-width: 1120px; margin: 0 auto; }
-    h1, h2 { margin-bottom: 12px; }
-    p, li { line-height: 1.5; }
-    .callout { background: white; border: 1px solid ${PALETTE.border}; border-radius: 16px; padding: 20px; margin-bottom: 24px; }
-    .grid { display: grid; gap: 24px; grid-template-columns: 1fr; }
-    img { width: 100%; border: 1px solid ${PALETTE.border}; border-radius: 16px; background: white; }
-    table { width: 100%; border-collapse: collapse; background: white; border-radius: 16px; overflow: hidden; }
-    th, td { border: 1px solid ${PALETTE.border}; padding: 12px; vertical-align: top; text-align: left; }
-    th { background: #e2e8f0; }
-    code { background: #e2e8f0; padding: 2px 6px; border-radius: 6px; }
-  </style>
-</head>
-<body>
-  <main>
-    <h1>Audrey Memory Benchmark</h1>
-    <div class="callout">
-      <p><strong>Method:</strong> Audrey is scored on a LongMemEval-inspired retrieval benchmark plus an operation-level lifecycle benchmark. The report still separates local Audrey-versus-baseline results from published external LoCoMo numbers so the comparison stays honest.</p>
-      <p><strong>Run:</strong> <code>${escapeHtml(summary.command)}</code></p>
-      <p><strong>Generated:</strong> ${escapeHtml(summary.generatedAt)}</p>
-    </div>
-
-    <div class="grid">
-      <section class="callout">
-        <h2>Combined Local Benchmark</h2>
-        <img src="./local-overall.svg" alt="Combined local benchmark bar chart" />
-      </section>
-
-      ${renderSuiteSections(suiteCharts)}
-
-      <section class="callout">
-        <h2>Published Leaderboard</h2>
-        <img src="./published-locomo.svg" alt="Published LoCoMo leaderboard bar chart" />
-      </section>
-    </div>
-
-    <section class="callout">
-      <h2>Case Matrix</h2>
-      <table>
-        <thead>
-          <tr>
-            <th>Case</th>
-            <th>Suite</th>
-            <th>Family</th>
-            ${summary.local.overall.map(row => `<th>${escapeHtml(row.system)}</th>`).join('')}
-          </tr>
-        </thead>
-        <tbody>
-          ${renderCaseRows(summary.local.cases)}
-        </tbody>
-      </table>
-    </section>
-
-    <section class="callout">
-      <h2>March 23, 2026 Memory Trends</h2>
-      <ul>
-        ${renderTrendList(trends)}
-      </ul>
-    </section>
-  </main>
-</body>
-</html>`;
-
-  writeFileSync(join(outputDir, 'report.html'), html, 'utf8');
-
-  return {
-    json: join(outputDir, 'summary.json'),
-    html: join(outputDir, 'report.html'),
-    localChart: join(outputDir, 'local-overall.svg'),
-    suiteCharts,
-    externalChart: join(outputDir, 'published-locomo.svg'),
-    readmeAssets,
-  };
-}