audrey 0.21.0 → 1.0.0

package/benchmarks/report.js

@@ -0,0 +1,259 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const PALETTE = {
+  audrey: '#0f766e',
+  vector: '#0369a1',
+  keyword: '#6d28d9',
+  recent: '#b45309',
+  external: '#1d4ed8',
+  accent: '#111827',
+  muted: '#6b7280',
+  surface: '#f8fafc',
+  border: '#cbd5e1',
+};
+
+function escapeHtml(text) {
+  return String(text)
+    .replaceAll('&', '&')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;');
+}
+
+function chartBarColor(label) {
+  if (label === 'Audrey') return PALETTE.audrey;
+  if (label.includes('Vector')) return PALETTE.vector;
+  if (label.includes('Keyword')) return PALETTE.keyword;
+  if (label.includes('Recent')) return PALETTE.recent;
+  return PALETTE.external;
+}
+
+function renderBarChart({ title, rows, valueSuffix = '%', maxValue = 100 }) {
+  const width = 960;
+  const height = 420;
+  const margin = { top: 56, right: 32, bottom: 88, left: 64 };
+  const plotWidth = width - margin.left - margin.right;
+  const plotHeight = height - margin.top - margin.bottom;
+  const barWidth = Math.max(32, Math.floor(plotWidth / Math.max(rows.length, 1)) - 18);
+  const gap = rows.length > 1 ? (plotWidth - barWidth * rows.length) / (rows.length - 1) : 0;
+
+  const bars = rows.map((row, index) => {
+    const value = Math.max(0, Math.min(maxValue, row.value));
+    const barHeight = (value / maxValue) * plotHeight;
+    const x = margin.left + index * (barWidth + gap);
+    const y = margin.top + plotHeight - barHeight;
+    return `
+      <rect x="${x}" y="${y}" width="${barWidth}" height="${barHeight}" rx="8" fill="${chartBarColor(row.label)}" />
+      <text x="${x + barWidth / 2}" y="${y - 10}" text-anchor="middle" font-size="15" fill="${PALETTE.accent}">${value.toFixed(1)}${valueSuffix}</text>
+      <text x="${x + barWidth / 2}" y="${height - 42}" text-anchor="middle" font-size="14" fill="${PALETTE.muted}">${escapeHtml(row.label)}</text>
+    `;
+  }).join('\n');
+
+  const grid = [0, 25, 50, 75, 100].map(tick => {
+    const y = margin.top + plotHeight - (tick / maxValue) * plotHeight;
+    return `
+      <line x1="${margin.left}" y1="${y}" x2="${width - margin.right}" y2="${y}" stroke="${PALETTE.border}" stroke-dasharray="4 4" />
+      <text x="${margin.left - 10}" y="${y + 5}" text-anchor="end" font-size="13" fill="${PALETTE.muted}">${tick}${valueSuffix}</text>
+    `;
+  }).join('\n');
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" role="img" aria-label="${escapeHtml(title)}">
+  <rect width="100%" height="100%" fill="white" />
+  <text x="${margin.left}" y="34" font-size="24" font-weight="700" fill="${PALETTE.accent}">${escapeHtml(title)}</text>
+  ${grid}
+  ${bars}
+</svg>`;
+}
+
+function renderTrendList(trends) {
+  return trends.map(trend => `
+    <li>
+      <strong>${escapeHtml(trend.title)}</strong><br />
+      ${escapeHtml(trend.summary)}<br />
+      <a href="${trend.source}">${escapeHtml(trend.source)}</a>
+    </li>
+  `).join('\n');
+}
+
+function renderCaseRows(localCases) {
+  return localCases.map(caseResult => `
+    <tr>
+      <td>${escapeHtml(caseResult.title)}</td>
+      <td>${escapeHtml(caseResult.suite)}</td>
+      <td>${escapeHtml(caseResult.family)}</td>
+      ${caseResult.results.map(result => {
+        const bg = result.passed ? '#ecfdf5' : result.score >= 0.5 ? '#fff7ed' : '#fef2f2';
+        const fg = result.passed ? '#065f46' : result.score >= 0.5 ? '#9a3412' : '#991b1b';
+        return `<td style="background:${bg};color:${fg}">${result.score.toFixed(2)}<br /><span style="font-size:12px">${escapeHtml(result.summary)}</span></td>`;
+      }).join('')}
+    </tr>
+  `).join('\n');
+}
+
+function renderSuiteSections(suiteCharts) {
+  if (suiteCharts.length === 0) return '';
+  return suiteCharts.map(chart => `
+    <section class="callout">
+      <h2>${escapeHtml(chart.title)}</h2>
+      <p>${escapeHtml(chart.description)}</p>
+      <img src="./${escapeHtml(chart.fileName)}" alt="${escapeHtml(chart.title)} chart" />
+    </section>
+  `).join('\n');
+}
+
+export function writeBenchmarkArtifacts({
+  outputDir,
+  summary,
+  localOverall,
+  localSuites,
+  externalOverall,
+  trends,
+  readmeAssetsDir,
+}) {
+  mkdirSync(outputDir, { recursive: true });
+
+  const localChartTitle = summary.local?.overall_scope === 'comparable_suites'
+    ? 'Audrey vs Comparable Local Memory Baselines'
+    : 'Selected Audrey Regression Suite';
+  const localChart = renderBarChart({
+    title: localChartTitle,
+    rows: localOverall.map(row => ({ label: row.system, value: row.scorePercent })),
+  });
+  const externalChart = renderBarChart({
+    title: 'Published LLM Memory Standards (LoCoMo)',
+    rows: externalOverall.map(row => ({ label: row.system, value: row.score })),
+  });
+
+  writeFileSync(join(outputDir, 'local-overall.svg'), localChart, 'utf8');
+  writeFileSync(join(outputDir, 'published-locomo.svg'), externalChart, 'utf8');
+  writeFileSync(join(outputDir, 'summary.json'), JSON.stringify(summary, null, 2), 'utf8');
+
+  const suiteCharts = localSuites.map(suite => {
+    const fileName = `${suite.id}-overall.svg`;
+    const chart = renderBarChart({
+      title: `${suite.title} Benchmark`,
+      rows: suite.overall.map(row => ({ label: row.system, value: row.scorePercent })),
+    });
+    writeFileSync(join(outputDir, fileName), chart, 'utf8');
+    return {
+      id: suite.id,
+      title: `${suite.title} Benchmark`,
+      description: suite.description,
+      fileName,
+      path: join(outputDir, fileName),
+    };
+  });
+
+  let readmeAssets = null;
+  if (readmeAssetsDir) {
+    mkdirSync(readmeAssetsDir, { recursive: true });
+    const localReadmeChart = join(readmeAssetsDir, 'local-benchmark.svg');
+    const externalReadmeChart = join(readmeAssetsDir, 'published-memory-standards.svg');
+    writeFileSync(localReadmeChart, localChart, 'utf8');
+    writeFileSync(externalReadmeChart, externalChart, 'utf8');
+
+    const operationsSuite = suiteCharts.find(chart => chart.id === 'operations');
+    let operationsReadmeChart = null;
+    if (operationsSuite) {
+      operationsReadmeChart = join(readmeAssetsDir, 'operations-benchmark.svg');
+      writeFileSync(
+        operationsReadmeChart,
+        renderBarChart({
+          title: 'Audrey Memory Operations Benchmark',
+          rows: (localSuites.find(suite => suite.id === 'operations')?.overall || [])
+            .map(row => ({ label: row.system, value: row.scorePercent })),
+        }),
+        'utf8',
+      );
+    }
+
+    readmeAssets = {
+      localChart: localReadmeChart,
+      operationsChart: operationsReadmeChart,
+      externalChart: externalReadmeChart,
+    };
+  }
+
+  const html = `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Audrey Memory Benchmark</title>
+  <style>
+    body { font-family: "Segoe UI", Arial, sans-serif; margin: 32px; color: ${PALETTE.accent}; background: ${PALETTE.surface}; }
+    main { max-width: 1120px; margin: 0 auto; }
+    h1, h2 { margin-bottom: 12px; }
+    p, li { line-height: 1.5; }
+    .callout { background: white; border: 1px solid ${PALETTE.border}; border-radius: 16px; padding: 20px; margin-bottom: 24px; }
+    .grid { display: grid; gap: 24px; grid-template-columns: 1fr; }
+    img { width: 100%; border: 1px solid ${PALETTE.border}; border-radius: 16px; background: white; }
+    table { width: 100%; border-collapse: collapse; background: white; border-radius: 16px; overflow: hidden; }
+    th, td { border: 1px solid ${PALETTE.border}; padding: 12px; vertical-align: top; text-align: left; }
+    th { background: #e2e8f0; }
+    code { background: #e2e8f0; padding: 2px 6px; border-radius: 6px; }
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Audrey Memory Benchmark</h1>
+    <div class="callout">
+      <p><strong>Method:</strong> Audrey is scored on a local regression suite inspired by LongMemEval-style retrieval, operation-level lifecycle behavior, and agent guard-loop benchmarks. The combined local chart uses comparable retrieval/lifecycle suites when available; the guard loop is reported as its own controller regression suite. Published external LoCoMo numbers stay separate so the comparison remains honest.</p>
+      <p><strong>Scope:</strong> ${escapeHtml(summary.local?.overall_scope ?? 'unknown')} across ${escapeHtml((summary.local?.overall_suite_ids ?? []).join(', '))}; ${summary.local?.cases?.length ?? 0} total cases.</p>
+      <p><strong>Run:</strong> <code>${escapeHtml(summary.command)}</code></p>
+      <p><strong>Generated:</strong> ${escapeHtml(summary.generatedAt)}</p>
+    </div>
+
+    <div class="grid">
+      <section class="callout">
+        <h2>Combined Local Benchmark</h2>
+        <img src="./local-overall.svg" alt="Combined local benchmark bar chart" />
+      </section>
+
+      ${renderSuiteSections(suiteCharts)}
+
+      <section class="callout">
+        <h2>Published Leaderboard</h2>
+        <img src="./published-locomo.svg" alt="Published LoCoMo leaderboard bar chart" />
+      </section>
+    </div>
+
+    <section class="callout">
+      <h2>Case Matrix</h2>
+      <table>
+        <thead>
+          <tr>
+            <th>Case</th>
+            <th>Suite</th>
+            <th>Family</th>
+            ${summary.local.overall.map(row => `<th>${escapeHtml(row.system)}</th>`).join('')}
+          </tr>
+        </thead>
+        <tbody>
+          ${renderCaseRows(summary.local.cases)}
+        </tbody>
+      </table>
+    </section>
+
+    <section class="callout">
+      <h2>March 23, 2026 Memory Trends</h2>
+      <ul>
+        ${renderTrendList(trends)}
+      </ul>
+    </section>
+  </main>
+</body>
+</html>`;
+
+  writeFileSync(join(outputDir, 'report.html'), html, 'utf8');
+
+  return {
+    json: join(outputDir, 'summary.json'),
+    html: join(outputDir, 'report.html'),
+    localChart: join(outputDir, 'local-overall.svg'),
+    suiteCharts,
+    externalChart: join(outputDir, 'published-locomo.svg'),
+    readmeAssets,
+  };
+}

package/benchmarks/run-external-guardbench.mjs

@@ -0,0 +1,281 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { basename, dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { writeGuardBenchConformanceCard } from './create-conformance-card.mjs';
+import { computeGuardBenchArtifactHashes, validateGuardBenchArtifacts } from './validate-guardbench-artifacts.mjs';
+import { publicArtifactValue } from './public-paths.mjs';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const KNOWN_ADAPTERS = new Map([
+  ['mem0', {
+    name: 'mem0-platform',
+    path: 'benchmarks/adapters/mem0-platform.mjs',
+    requiredEnv: ['MEM0_API_KEY'],
+  }],
+  ['mem0-platform', {
+    name: 'mem0-platform',
+    path: 'benchmarks/adapters/mem0-platform.mjs',
+    requiredEnv: ['MEM0_API_KEY'],
+  }],
+  ['zep', {
+    name: 'zep-cloud',
+    path: 'benchmarks/adapters/zep-cloud.mjs',
+    requiredEnv: ['ZEP_API_KEY'],
+  }],
+  ['zep-cloud', {
+    name: 'zep-cloud',
+    path: 'benchmarks/adapters/zep-cloud.mjs',
+    requiredEnv: ['ZEP_API_KEY'],
+  }],
+]);
+
+export function parseExternalArgs(argv = process.argv.slice(2)) {
+  const args = {
+    adapter: 'mem0-platform',
+    outDir: null,
+    check: false,
+    dryRun: false,
+    json: false,
+    minPassRate: null,
+    allowMissingEnv: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (token === '--adapter' && argv[i + 1]) args.adapter = argv[++i];
+    else if (token === '--out-dir' && argv[i + 1]) args.outDir = argv[++i];
+    else if (token === '--check') args.check = true;
+    else if (token === '--dry-run') args.dryRun = true;
+    else if (token === '--json') args.json = true;
+    else if (token === '--min-pass-rate' && argv[i + 1]) args.minPassRate = argv[++i];
+    else if (token === '--allow-missing-env') args.allowMissingEnv = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return args;
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+function adapterSpec(adapter) {
+  const known = KNOWN_ADAPTERS.get(adapter);
+  if (known) return known;
+
+  const adapterPath = resolve(ROOT, adapter);
+  return {
+    name: basename(adapter).replace(/\.[cm]?js$/i, ''),
+    path: adapterPath,
+    requiredEnv: [],
+  };
+}
+
+export function buildExternalGuardBenchRun(args = {}, env = process.env) {
+  const spec = adapterSpec(args.adapter ?? 'mem0-platform');
+  const adapterPath = resolve(ROOT, spec.path);
+  const outDir = resolve(ROOT, args.outDir ?? `benchmarks/output/external/${spec.name}`);
+  const missingEnv = spec.requiredEnv.filter(name => !env[name]);
+  const command = [
+    process.execPath,
+    resolve(ROOT, 'benchmarks/guardbench.js'),
+    '--adapter',
+    adapterPath,
+    '--out-dir',
+    outDir,
+  ];
+
+  if (args.check) command.push('--check');
+  if (args.json) command.push('--json');
+  if (args.minPassRate != null) command.push('--min-pass-rate', String(args.minPassRate));
+  const validationCommand = [
+    process.execPath,
+    resolve(ROOT, 'benchmarks/validate-guardbench-artifacts.mjs'),
+    '--dir',
+    outDir,
+  ];
+
+  return {
+    adapter: spec.name,
+    adapterPath,
+    outDir,
+    requiredEnv: spec.requiredEnv,
+    missingEnv,
+    command,
+    validationCommand,
+  };
+}
+
+export function evaluateAdapterConformance(summary, adapterName) {
+  const failures = [];
+  const externalSubjects = (summary.manifest?.subjects ?? []).filter(subject => subject.external);
+  const resolvedAdapterName = summary.systemSummaries?.some(row => row.system === adapterName)
+    ? adapterName
+    : externalSubjects.length === 1
+      ? externalSubjects[0].name
+      : adapterName;
+  const systemSummary = summary.systemSummaries?.find(row => row.system === resolvedAdapterName);
+  if (!systemSummary) {
+    failures.push(`Missing system summary for adapter ${resolvedAdapterName}`);
+  }
+
+  const expectedScenarios = summary.scenarios ?? summary.manifest?.scenarios?.length ?? 0;
+  const adapterRows = (summary.cases ?? [])
+    .flatMap(entry => entry.results ?? [])
+    .filter(row => row.system === resolvedAdapterName);
+
+  if (adapterRows.length !== expectedScenarios) {
+    failures.push(`Adapter ${resolvedAdapterName} returned ${adapterRows.length}/${expectedScenarios} scenario rows`);
+  }
+  if (systemSummary && systemSummary.scenarios !== expectedScenarios) {
+    failures.push(`Adapter ${resolvedAdapterName} system summary has ${systemSummary.scenarios}/${expectedScenarios} scenarios`);
+  }
+  if (systemSummary && systemSummary.redactionLeaks !== 0) {
+    failures.push(`Adapter ${resolvedAdapterName} leaked ${systemSummary.redactionLeaks} seeded secret(s) in decision output`);
+  }
+  if (adapterRows.some(row => row.external !== true)) {
+    failures.push(`Adapter ${resolvedAdapterName} rows are not marked external`);
+  }
+
+  return {
+    ok: failures.length === 0,
+    adapter: resolvedAdapterName,
+    requestedAdapter: adapterName,
+    scenarios: systemSummary?.scenarios ?? adapterRows.length,
+    expectedScenarios,
+    fullContractPassRate: systemSummary?.passRate ?? null,
+    decisionAccuracy: systemSummary?.decisionAccuracy ?? null,
+    redactionLeaks: systemSummary?.redactionLeaks ?? null,
+    failures,
+  };
+}
+
+function usage() {
+  return `Usage: node benchmarks/run-external-guardbench.mjs [options]
+
+Options:
+  --adapter <name|path>       Adapter alias or ESM adapter path. Default: mem0-platform.
+  --out-dir <path>            Output directory. Default: benchmarks/output/external/<adapter>.
+  --check                     Fail if Audrey Guard pass rate is below the threshold.
+  --min-pass-rate <percent>   GuardBench pass-rate threshold for --check.
+  --json                      Forward JSON output from GuardBench.
+  --dry-run                   Print the resolved command and metadata without running.
+  --allow-missing-env         Permit running even when known runtime env vars are absent.
+`;
+}
+
+export function writeExternalRunMetadata(path, metadata) {
+  mkdirSync(path, { recursive: true });
+  const file = resolve(path, 'external-run-metadata.json');
+  writeFileSync(file, `${JSON.stringify(publicArtifactValue(metadata), null, 2)}\n`, 'utf-8');
+  return file;
+}
+
+async function main() {
+  const args = parseExternalArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+
+  const run = buildExternalGuardBenchRun(args);
+  const startedAt = new Date().toISOString();
+  const metadata = {
+    suite: 'GuardBench external adapter run',
+    startedAt,
+    adapter: run.adapter,
+    adapterPath: run.adapterPath,
+    outDir: run.outDir,
+    requiredEnv: run.requiredEnv,
+    missingEnv: run.missingEnv,
+    command: run.command,
+    validationCommand: run.validationCommand,
+    dryRun: args.dryRun,
+  };
+
+  if (!existsSync(run.adapterPath)) {
+    throw new Error(`Adapter not found: ${run.adapterPath}`);
+  }
+
+  if (run.missingEnv.length && !args.allowMissingEnv && !args.dryRun) {
+    metadata.status = 'blocked';
+    metadata.blockReason = `Missing runtime environment: ${run.missingEnv.join(', ')}`;
+    const metadataPath = writeExternalRunMetadata(run.outDir, metadata);
+    throw new Error(`${metadata.blockReason}. Metadata written to ${metadataPath}`);
+  }
+
+  if (args.dryRun) {
+    metadata.status = run.missingEnv.length ? 'dry-run-missing-env' : 'dry-run-ready';
+    const metadataPath = writeExternalRunMetadata(run.outDir, metadata);
+    if (args.json) {
+      console.log(JSON.stringify({ ...metadata, metadataPath }, null, 2));
+    } else {
+      console.log(`External GuardBench dry run: ${run.adapter}`);
+      console.log(`Command: ${run.command.map(part => JSON.stringify(part)).join(' ')}`);
+      console.log(`Metadata: ${metadataPath}`);
+      if (run.missingEnv.length) console.log(`Missing runtime env: ${run.missingEnv.join(', ')}`);
+    }
+    return;
+  }
+
+  writeExternalRunMetadata(run.outDir, { ...metadata, status: 'running' });
+  const child = spawnSync(run.command[0], run.command.slice(1), {
+    cwd: ROOT,
+    env: process.env,
+    stdio: 'inherit',
+  });
+  const validation = validateGuardBenchArtifacts({ dir: run.outDir });
+  let adapterConformance = {
+    ok: false,
+    adapter: run.adapter,
+    failures: ['GuardBench summary was not available for adapter conformance evaluation'],
+  };
+  if (child.status === 0) {
+    try {
+      const summary = readJson(resolve(run.outDir, 'guardbench-summary.json'));
+      adapterConformance = evaluateAdapterConformance(summary, run.adapter);
+    } catch (error) {
+      adapterConformance = {
+        ok: false,
+        adapter: run.adapter,
+        failures: [error.message],
+      };
+    }
+  }
+  if (validation.ok) {
+    console.log(`External GuardBench artifact validation passed: ${run.outDir}`);
+  } else {
+    console.error('External GuardBench artifact validation failed:');
+    for (const failure of validation.failures) console.error(`- ${failure}`);
+  }
+  if (adapterConformance.ok) {
+    console.log(`External GuardBench adapter conformance passed: ${adapterConformance.adapter}`);
+  } else {
+    console.error('External GuardBench adapter conformance failed:');
+    for (const failure of adapterConformance.failures) console.error(`- ${failure}`);
+  }
+  const completed = {
+    ...metadata,
+    completedAt: new Date().toISOString(),
+    status: child.status === 0 && validation.ok && adapterConformance.ok ? 'passed' : 'failed',
+    exitCode: child.status,
+    signal: child.signal,
+    artifactHashes: child.status === 0 ? computeGuardBenchArtifactHashes(run.outDir) : undefined,
+    artifactValidation: validation,
+    adapterConformance,
+  };
+  const metadataPath = writeExternalRunMetadata(run.outDir, completed);
+  const card = child.status === 0 ? writeGuardBenchConformanceCard({ dir: run.outDir }) : null;
+  console.log(`External GuardBench metadata: ${metadataPath}`);
+  if (card) console.log(`External GuardBench conformance card: ${card.path}`);
+  process.exitCode = child.status === 0 && validation.ok && adapterConformance.ok ? 0 : (child.status ?? 1);
+}
+
+if (process.argv[1] && process.argv[1].endsWith('run-external-guardbench.mjs')) {
+  main().catch(err => {
+    console.error(err.message);
+    process.exit(1);
+  });
+}