audrey 1.1.3 → 1.2.1

package/benchmarks/report.js

@@ -1,280 +0,0 @@
-import { mkdirSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-
-const PALETTE = {
-  audrey: '#0f766e',
-  vector: '#0369a1',
-  keyword: '#6d28d9',
-  recent: '#b45309',
-  external: '#1d4ed8',
-  accent: '#111827',
-  muted: '#6b7280',
-  surface: '#f8fafc',
-  border: '#cbd5e1',
-};
-
-function escapeHtml(text) {
-  return String(text)
-    .replaceAll('&', '&')
-    .replaceAll('<', '&lt;')
-    .replaceAll('>', '&gt;')
-    .replaceAll('"', '&quot;');
-}
-
-function chartBarColor(label) {
-  if (label === 'Audrey') return PALETTE.audrey;
-  if (label.includes('Vector')) return PALETTE.vector;
-  if (label.includes('Keyword')) return PALETTE.keyword;
-  if (label.includes('Recent')) return PALETTE.recent;
-  return PALETTE.external;
-}
-
-function renderBarChart({ title, rows, valueSuffix = '%', maxValue = 100 }) {
-  const width = 960;
-  const height = 420;
-  const margin = { top: 56, right: 32, bottom: 88, left: 64 };
-  const plotWidth = width - margin.left - margin.right;
-  const plotHeight = height - margin.top - margin.bottom;
-  const barWidth = Math.max(32, Math.floor(plotWidth / Math.max(rows.length, 1)) - 18);
-  const gap = rows.length > 1 ? (plotWidth - barWidth * rows.length) / (rows.length - 1) : 0;
-
-  const bars = rows
-    .map((row, index) => {
-      const value = Math.max(0, Math.min(maxValue, row.value));
-      const barHeight = (value / maxValue) * plotHeight;
-      const x = margin.left + index * (barWidth + gap);
-      const y = margin.top + plotHeight - barHeight;
-      return `
-      <rect x="${x}" y="${y}" width="${barWidth}" height="${barHeight}" rx="8" fill="${chartBarColor(row.label)}" />
-      <text x="${x + barWidth / 2}" y="${y - 10}" text-anchor="middle" font-size="15" fill="${PALETTE.accent}">${value.toFixed(1)}${valueSuffix}</text>
-      <text x="${x + barWidth / 2}" y="${height - 42}" text-anchor="middle" font-size="14" fill="${PALETTE.muted}">${escapeHtml(row.label)}</text>
-    `;
-    })
-    .join('\n');
-
-  const grid = [0, 25, 50, 75, 100]
-    .map(tick => {
-      const y = margin.top + plotHeight - (tick / maxValue) * plotHeight;
-      return `
-      <line x1="${margin.left}" y1="${y}" x2="${width - margin.right}" y2="${y}" stroke="${PALETTE.border}" stroke-dasharray="4 4" />
-      <text x="${margin.left - 10}" y="${y + 5}" text-anchor="end" font-size="13" fill="${PALETTE.muted}">${tick}${valueSuffix}</text>
-    `;
-    })
-    .join('\n');
-
-  return `<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" role="img" aria-label="${escapeHtml(title)}">
-  <rect width="100%" height="100%" fill="white" />
-  <text x="${margin.left}" y="34" font-size="24" font-weight="700" fill="${PALETTE.accent}">${escapeHtml(title)}</text>
-  ${grid}
-  ${bars}
-</svg>`;
-}
-
-function renderTrendList(trends) {
-  return trends
-    .map(
-      trend => `
-    <li>
-      <strong>${escapeHtml(trend.title)}</strong><br />
-      ${escapeHtml(trend.summary)}<br />
-      <a href="${trend.source}">${escapeHtml(trend.source)}</a>
-    </li>
-  `,
-    )
-    .join('\n');
-}
-
-function renderCaseRows(localCases) {
-  return localCases
-    .map(
-      caseResult => `
-    <tr>
-      <td>${escapeHtml(caseResult.title)}</td>
-      <td>${escapeHtml(caseResult.suite)}</td>
-      <td>${escapeHtml(caseResult.family)}</td>
-      ${caseResult.results
-        .map(result => {
-          const bg = result.passed ? '#ecfdf5' : result.score >= 0.5 ? '#fff7ed' : '#fef2f2';
-          const fg = result.passed ? '#065f46' : result.score >= 0.5 ? '#9a3412' : '#991b1b';
-          return `<td style="background:${bg};color:${fg}">${result.score.toFixed(2)}<br /><span style="font-size:12px">${escapeHtml(result.summary)}</span></td>`;
-        })
-        .join('')}
-    </tr>
-  `,
-    )
-    .join('\n');
-}
-
-function renderSuiteSections(suiteCharts) {
-  if (suiteCharts.length === 0) return '';
-  return suiteCharts
-    .map(
-      chart => `
-    <section class="callout">
-      <h2>${escapeHtml(chart.title)}</h2>
-      <p>${escapeHtml(chart.description)}</p>
-      <img src="./${escapeHtml(chart.fileName)}" alt="${escapeHtml(chart.title)} chart" />
-    </section>
-  `,
-    )
-    .join('\n');
-}
-
-export function writeBenchmarkArtifacts({
-  outputDir,
-  summary,
-  localOverall,
-  localSuites,
-  externalOverall,
-  trends,
-  readmeAssetsDir,
-}) {
-  mkdirSync(outputDir, { recursive: true });
-
-  const localChartTitle =
-    summary.local?.overall_scope === 'comparable_suites'
-      ? 'Audrey vs Comparable Local Memory Baselines'
-      : 'Selected Audrey Regression Suite';
-  const localChart = renderBarChart({
-    title: localChartTitle,
-    rows: localOverall.map(row => ({ label: row.system, value: row.scorePercent })),
-  });
-  const externalChart = renderBarChart({
-    title: 'Published LLM Memory Standards (LoCoMo)',
-    rows: externalOverall.map(row => ({ label: row.system, value: row.score })),
-  });
-
-  writeFileSync(join(outputDir, 'local-overall.svg'), localChart, 'utf8');
-  writeFileSync(join(outputDir, 'published-locomo.svg'), externalChart, 'utf8');
-  writeFileSync(join(outputDir, 'summary.json'), JSON.stringify(summary, null, 2), 'utf8');
-
-  const suiteCharts = localSuites.map(suite => {
-    const fileName = `${suite.id}-overall.svg`;
-    const chart = renderBarChart({
-      title: `${suite.title} Benchmark`,
-      rows: suite.overall.map(row => ({ label: row.system, value: row.scorePercent })),
-    });
-    writeFileSync(join(outputDir, fileName), chart, 'utf8');
-    return {
-      id: suite.id,
-      title: `${suite.title} Benchmark`,
-      description: suite.description,
-      fileName,
-      path: join(outputDir, fileName),
-    };
-  });
-
-  let readmeAssets = null;
-  if (readmeAssetsDir) {
-    mkdirSync(readmeAssetsDir, { recursive: true });
-    const localReadmeChart = join(readmeAssetsDir, 'local-benchmark.svg');
-    const externalReadmeChart = join(readmeAssetsDir, 'published-memory-standards.svg');
-    writeFileSync(localReadmeChart, localChart, 'utf8');
-    writeFileSync(externalReadmeChart, externalChart, 'utf8');
-
-    const operationsSuite = suiteCharts.find(chart => chart.id === 'operations');
-    let operationsReadmeChart = null;
-    if (operationsSuite) {
-      operationsReadmeChart = join(readmeAssetsDir, 'operations-benchmark.svg');
-      writeFileSync(
-        operationsReadmeChart,
-        renderBarChart({
-          title: 'Audrey Memory Operations Benchmark',
-          rows: (localSuites.find(suite => suite.id === 'operations')?.overall || []).map(row => ({
-            label: row.system,
-            value: row.scorePercent,
-          })),
-        }),
-        'utf8',
-      );
-    }
-
-    readmeAssets = {
-      localChart: localReadmeChart,
-      operationsChart: operationsReadmeChart,
-      externalChart: externalReadmeChart,
-    };
-  }
-
-  const html = `<!doctype html>
-<html lang="en">
-<head>
-  <meta charset="utf-8" />
-  <title>Audrey Memory Benchmark</title>
-  <style>
-    body { font-family: "Segoe UI", Arial, sans-serif; margin: 32px; color: ${PALETTE.accent}; background: ${PALETTE.surface}; }
-    main { max-width: 1120px; margin: 0 auto; }
-    h1, h2 { margin-bottom: 12px; }
-    p, li { line-height: 1.5; }
-    .callout { background: white; border: 1px solid ${PALETTE.border}; border-radius: 16px; padding: 20px; margin-bottom: 24px; }
-    .grid { display: grid; gap: 24px; grid-template-columns: 1fr; }
-    img { width: 100%; border: 1px solid ${PALETTE.border}; border-radius: 16px; background: white; }
-    table { width: 100%; border-collapse: collapse; background: white; border-radius: 16px; overflow: hidden; }
-    th, td { border: 1px solid ${PALETTE.border}; padding: 12px; vertical-align: top; text-align: left; }
-    th { background: #e2e8f0; }
-    code { background: #e2e8f0; padding: 2px 6px; border-radius: 6px; }
-  </style>
-</head>
-<body>
-  <main>
-    <h1>Audrey Memory Benchmark</h1>
-    <div class="callout">
-      <p><strong>Method:</strong> Audrey is scored on a local regression suite inspired by LongMemEval-style retrieval, operation-level lifecycle behavior, and agent guard-loop benchmarks. The combined local chart uses comparable retrieval/lifecycle suites when available; the guard loop is reported as its own controller regression suite. Published external LoCoMo numbers stay separate so the comparison remains honest.</p>
-      <p><strong>Scope:</strong> ${escapeHtml(summary.local?.overall_scope ?? 'unknown')} across ${escapeHtml((summary.local?.overall_suite_ids ?? []).join(', '))}; ${summary.local?.cases?.length ?? 0} total cases.</p>
-      <p><strong>Run:</strong> <code>${escapeHtml(summary.command)}</code></p>
-      <p><strong>Generated:</strong> ${escapeHtml(summary.generatedAt)}</p>
-    </div>
-
-    <div class="grid">
-      <section class="callout">
-        <h2>Combined Local Benchmark</h2>
-        <img src="./local-overall.svg" alt="Combined local benchmark bar chart" />
-      </section>
-
-      ${renderSuiteSections(suiteCharts)}
-
-      <section class="callout">
-        <h2>Published Leaderboard</h2>
-        <img src="./published-locomo.svg" alt="Published LoCoMo leaderboard bar chart" />
-      </section>
-    </div>
-
-    <section class="callout">
-      <h2>Case Matrix</h2>
-      <table>
-        <thead>
-          <tr>
-            <th>Case</th>
-            <th>Suite</th>
-            <th>Family</th>
-            ${summary.local.overall.map(row => `<th>${escapeHtml(row.system)}</th>`).join('')}
-          </tr>
-        </thead>
-        <tbody>
-          ${renderCaseRows(summary.local.cases)}
-        </tbody>
-      </table>
-    </section>
-
-    <section class="callout">
-      <h2>March 23, 2026 Memory Trends</h2>
-      <ul>
-        ${renderTrendList(trends)}
-      </ul>
-    </section>
-  </main>
-</body>
-</html>`;
-
-  writeFileSync(join(outputDir, 'report.html'), html, 'utf8');
-
-  return {
-    json: join(outputDir, 'summary.json'),
-    html: join(outputDir, 'report.html'),
-    localChart: join(outputDir, 'local-overall.svg'),
-    suiteCharts,
-    externalChart: join(outputDir, 'published-locomo.svg'),
-    readmeAssets,
-  };
-}

package/benchmarks/run-external-guardbench.mjs

@@ -1,303 +0,0 @@
-import { spawnSync } from 'node:child_process';
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
-import { basename, dirname, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { writeGuardBenchConformanceCard } from './create-conformance-card.mjs';
-import {
-  computeGuardBenchArtifactHashes,
-  validateGuardBenchArtifacts,
-} from './validate-guardbench-artifacts.mjs';
-import { publicArtifactValue } from './public-paths.mjs';
-
-const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
-const KNOWN_ADAPTERS = new Map([
-  [
-    'mem0',
-    {
-      name: 'mem0-platform',
-      path: 'benchmarks/adapters/mem0-platform.mjs',
-      requiredEnv: ['MEM0_API_KEY'],
-    },
-  ],
-  [
-    'mem0-platform',
-    {
-      name: 'mem0-platform',
-      path: 'benchmarks/adapters/mem0-platform.mjs',
-      requiredEnv: ['MEM0_API_KEY'],
-    },
-  ],
-  [
-    'zep',
-    {
-      name: 'zep-cloud',
-      path: 'benchmarks/adapters/zep-cloud.mjs',
-      requiredEnv: ['ZEP_API_KEY'],
-    },
-  ],
-  [
-    'zep-cloud',
-    {
-      name: 'zep-cloud',
-      path: 'benchmarks/adapters/zep-cloud.mjs',
-      requiredEnv: ['ZEP_API_KEY'],
-    },
-  ],
-]);
-
-export function parseExternalArgs(argv = process.argv.slice(2)) {
-  const args = {
-    adapter: 'mem0-platform',
-    outDir: null,
-    check: false,
-    dryRun: false,
-    json: false,
-    minPassRate: null,
-    allowMissingEnv: false,
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if (token === '--adapter' && argv[i + 1]) args.adapter = argv[++i];
-    else if (token === '--out-dir' && argv[i + 1]) args.outDir = argv[++i];
-    else if (token === '--check') args.check = true;
-    else if (token === '--dry-run') args.dryRun = true;
-    else if (token === '--json') args.json = true;
-    else if (token === '--min-pass-rate' && argv[i + 1]) args.minPassRate = argv[++i];
-    else if (token === '--allow-missing-env') args.allowMissingEnv = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-
-  return args;
-}
-
-function readJson(path) {
-  return JSON.parse(readFileSync(path, 'utf-8'));
-}
-
-function adapterSpec(adapter) {
-  const known = KNOWN_ADAPTERS.get(adapter);
-  if (known) return known;
-
-  const adapterPath = resolve(ROOT, adapter);
-  return {
-    name: basename(adapter).replace(/\.[cm]?js$/i, ''),
-    path: adapterPath,
-    requiredEnv: [],
-  };
-}
-
-export function buildExternalGuardBenchRun(args = {}, env = process.env) {
-  const spec = adapterSpec(args.adapter ?? 'mem0-platform');
-  const adapterPath = resolve(ROOT, spec.path);
-  const outDir = resolve(ROOT, args.outDir ?? `benchmarks/output/external/${spec.name}`);
-  const missingEnv = spec.requiredEnv.filter(name => !env[name]);
-  const command = [
-    process.execPath,
-    resolve(ROOT, 'benchmarks/guardbench.js'),
-    '--adapter',
-    adapterPath,
-    '--out-dir',
-    outDir,
-  ];
-
-  if (args.check) command.push('--check');
-  if (args.json) command.push('--json');
-  if (args.minPassRate != null) command.push('--min-pass-rate', String(args.minPassRate));
-  const validationCommand = [
-    process.execPath,
-    resolve(ROOT, 'benchmarks/validate-guardbench-artifacts.mjs'),
-    '--dir',
-    outDir,
-  ];
-
-  return {
-    adapter: spec.name,
-    adapterPath,
-    outDir,
-    requiredEnv: spec.requiredEnv,
-    missingEnv,
-    command,
-    validationCommand,
-  };
-}
-
-export function evaluateAdapterConformance(summary, adapterName) {
-  const failures = [];
-  const externalSubjects = (summary.manifest?.subjects ?? []).filter(subject => subject.external);
-  const resolvedAdapterName = summary.systemSummaries?.some(row => row.system === adapterName)
-    ? adapterName
-    : externalSubjects.length === 1
-      ? externalSubjects[0].name
-      : adapterName;
-  const systemSummary = summary.systemSummaries?.find(row => row.system === resolvedAdapterName);
-  if (!systemSummary) {
-    failures.push(`Missing system summary for adapter ${resolvedAdapterName}`);
-  }
-
-  const expectedScenarios = summary.scenarios ?? summary.manifest?.scenarios?.length ?? 0;
-  const adapterRows = (summary.cases ?? [])
-    .flatMap(entry => entry.results ?? [])
-    .filter(row => row.system === resolvedAdapterName);
-
-  if (adapterRows.length !== expectedScenarios) {
-    failures.push(
-      `Adapter ${resolvedAdapterName} returned ${adapterRows.length}/${expectedScenarios} scenario rows`,
-    );
-  }
-  if (systemSummary && systemSummary.scenarios !== expectedScenarios) {
-    failures.push(
-      `Adapter ${resolvedAdapterName} system summary has ${systemSummary.scenarios}/${expectedScenarios} scenarios`,
-    );
-  }
-  if (systemSummary && systemSummary.redactionLeaks !== 0) {
-    failures.push(
-      `Adapter ${resolvedAdapterName} leaked ${systemSummary.redactionLeaks} seeded secret(s) in decision output`,
-    );
-  }
-  if (adapterRows.some(row => row.external !== true)) {
-    failures.push(`Adapter ${resolvedAdapterName} rows are not marked external`);
-  }
-
-  return {
-    ok: failures.length === 0,
-    adapter: resolvedAdapterName,
-    requestedAdapter: adapterName,
-    scenarios: systemSummary?.scenarios ?? adapterRows.length,
-    expectedScenarios,
-    fullContractPassRate: systemSummary?.passRate ?? null,
-    decisionAccuracy: systemSummary?.decisionAccuracy ?? null,
-    redactionLeaks: systemSummary?.redactionLeaks ?? null,
-    failures,
-  };
-}
-
-function usage() {
-  return `Usage: node benchmarks/run-external-guardbench.mjs [options]
-
-Options:
-  --adapter <name|path>       Adapter alias or ESM adapter path. Default: mem0-platform.
-  --out-dir <path>            Output directory. Default: benchmarks/output/external/<adapter>.
-  --check                     Fail if Audrey Guard pass rate is below the threshold.
-  --min-pass-rate <percent>   GuardBench pass-rate threshold for --check.
-  --json                      Forward JSON output from GuardBench.
-  --dry-run                   Print the resolved command and metadata without running.
-  --allow-missing-env         Permit running even when known runtime env vars are absent.
-`;
-}
-
-export function writeExternalRunMetadata(path, metadata) {
-  mkdirSync(path, { recursive: true });
-  const file = resolve(path, 'external-run-metadata.json');
-  writeFileSync(file, `${JSON.stringify(publicArtifactValue(metadata), null, 2)}\n`, 'utf-8');
-  return file;
-}
-
-async function main() {
-  const args = parseExternalArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-
-  const run = buildExternalGuardBenchRun(args);
-  const startedAt = new Date().toISOString();
-  const metadata = {
-    suite: 'GuardBench external adapter run',
-    startedAt,
-    adapter: run.adapter,
-    adapterPath: run.adapterPath,
-    outDir: run.outDir,
-    requiredEnv: run.requiredEnv,
-    missingEnv: run.missingEnv,
-    command: run.command,
-    validationCommand: run.validationCommand,
-    dryRun: args.dryRun,
-  };
-
-  if (!existsSync(run.adapterPath)) {
-    throw new Error(`Adapter not found: ${run.adapterPath}`);
-  }
-
-  if (run.missingEnv.length && !args.allowMissingEnv && !args.dryRun) {
-    metadata.status = 'blocked';
-    metadata.blockReason = `Missing runtime environment: ${run.missingEnv.join(', ')}`;
-    const metadataPath = writeExternalRunMetadata(run.outDir, metadata);
-    throw new Error(`${metadata.blockReason}. Metadata written to ${metadataPath}`);
-  }
-
-  if (args.dryRun) {
-    metadata.status = run.missingEnv.length ? 'dry-run-missing-env' : 'dry-run-ready';
-    const metadataPath = writeExternalRunMetadata(run.outDir, metadata);
-    if (args.json) {
-      console.log(JSON.stringify({ ...metadata, metadataPath }, null, 2));
-    } else {
-      console.log(`External GuardBench dry run: ${run.adapter}`);
-      console.log(`Command: ${run.command.map(part => JSON.stringify(part)).join(' ')}`);
-      console.log(`Metadata: ${metadataPath}`);
-      if (run.missingEnv.length) console.log(`Missing runtime env: ${run.missingEnv.join(', ')}`);
-    }
-    return;
-  }
-
-  writeExternalRunMetadata(run.outDir, { ...metadata, status: 'running' });
-  const child = spawnSync(run.command[0], run.command.slice(1), {
-    cwd: ROOT,
-    env: process.env,
-    stdio: 'inherit',
-  });
-  const validation = validateGuardBenchArtifacts({ dir: run.outDir });
-  let adapterConformance = {
-    ok: false,
-    adapter: run.adapter,
-    failures: ['GuardBench summary was not available for adapter conformance evaluation'],
-  };
-  if (child.status === 0) {
-    try {
-      const summary = readJson(resolve(run.outDir, 'guardbench-summary.json'));
-      adapterConformance = evaluateAdapterConformance(summary, run.adapter);
-    } catch (error) {
-      adapterConformance = {
-        ok: false,
-        adapter: run.adapter,
-        failures: [error.message],
-      };
-    }
-  }
-  if (validation.ok) {
-    console.log(`External GuardBench artifact validation passed: ${run.outDir}`);
-  } else {
-    console.error('External GuardBench artifact validation failed:');
-    for (const failure of validation.failures) console.error(`- ${failure}`);
-  }
-  if (adapterConformance.ok) {
-    console.log(`External GuardBench adapter conformance passed: ${adapterConformance.adapter}`);
-  } else {
-    console.error('External GuardBench adapter conformance failed:');
-    for (const failure of adapterConformance.failures) console.error(`- ${failure}`);
-  }
-  const completed = {
-    ...metadata,
-    completedAt: new Date().toISOString(),
-    status: child.status === 0 && validation.ok && adapterConformance.ok ? 'passed' : 'failed',
-    exitCode: child.status,
-    signal: child.signal,
-    artifactHashes: child.status === 0 ? computeGuardBenchArtifactHashes(run.outDir) : undefined,
-    artifactValidation: validation,
-    adapterConformance,
-  };
-  const metadataPath = writeExternalRunMetadata(run.outDir, completed);
-  const card = child.status === 0 ? writeGuardBenchConformanceCard({ dir: run.outDir }) : null;
-  console.log(`External GuardBench metadata: ${metadataPath}`);
-  if (card) console.log(`External GuardBench conformance card: ${card.path}`);
-  process.exitCode =
-    child.status === 0 && validation.ok && adapterConformance.ok ? 0 : (child.status ?? 1);
-}
-
-if (process.argv[1] && process.argv[1].endsWith('run-external-guardbench.mjs')) {
-  main().catch(err => {
-    console.error(err.message);
-    process.exit(1);
-  });
-}