audrey 0.23.1 → 1.0.0

package/benchmarks/baselines.js

@@ -0,0 +1,169 @@
+import { createEmbeddingProvider } from '../dist/src/embedding.js';
+import { cosineSimilarity } from '../dist/src/utils.js';
+
+function normalize(text) {
+  return String(text || '').toLowerCase();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+function keywordScore(queryTokens, content) {
+  const contentTokens = new Set(tokenize(content));
+  if (queryTokens.length === 0) return 0;
+  let matches = 0;
+  for (const token of queryTokens) {
+    if (contentTokens.has(token)) matches++;
+  }
+  return matches / queryTokens.length;
+}
+
+function sortByScore(rows) {
+  return rows
+    .filter(row => Number.isFinite(row.score))
+    .sort((a, b) => b.score - a.score || String(b.createdAt || '').localeCompare(String(a.createdAt || '')));
+}
+
+function flattenMemories(benchmarkCase, ids = []) {
+  return benchmarkCase.memory.map((memory, index) => ({
+    id: ids[index] || `memory-${index + 1}`,
+    content: memory.content,
+    source: memory.source,
+    createdAt: memory.createdAt || new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+    private: Boolean(memory.private),
+  }));
+}
+
+function buildSyntheticCase(query, memories, options = {}) {
+  return {
+    query,
+    memory: memories.map(memory => ({
+      content: memory.content,
+      source: memory.source,
+      createdAt: memory.createdAt,
+      private: memory.private,
+    })),
+    options,
+  };
+}
+
+async function runBaselineRetrieval(system, syntheticCase, providerConfig, limit = 5) {
+  switch (system) {
+    case 'Vector Only':
+      return runVectorOnlyBaseline(syntheticCase, providerConfig, limit);
+    case 'Keyword + Recency':
+      return runKeywordRecencyBaseline(syntheticCase, limit);
+    case 'Recent Window':
+      return runRecentWindowBaseline(syntheticCase, limit);
+    default:
+      throw new Error(`Unknown baseline system: ${system}`);
+  }
+}
+
+function createOperationMemory(state, step) {
+  const index = state.counter++;
+  return {
+    id: `memory-${index + 1}`,
+    content: step.memory.content,
+    source: step.memory.source,
+    createdAt: step.memory.createdAt || new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+    private: Boolean(step.memory.private),
+  };
+}
+
+async function applyBaselineStep(system, state, step, providerConfig) {
+  if (step.type === 'encode') {
+    const memory = createOperationMemory(state, step);
+    state.memories.push(memory);
+    if (step.saveAs) {
+      state.aliases.set(step.saveAs, memory.id);
+    }
+    return;
+  }
+
+  if (step.type === 'forgetByQuery') {
+    const syntheticCase = buildSyntheticCase(step.query, state.memories, step.options);
+    const [match] = await runBaselineRetrieval(system, syntheticCase, providerConfig, 1);
+    if (match && Number.isFinite(match.score) && match.score > 0) {
+      state.memories = state.memories.filter(memory => memory.id !== match.id);
+    }
+    return;
+  }
+
+  if (step.type === 'consolidate') {
+    return;
+  }
+
+  throw new Error(`Unsupported baseline step: ${step.type}`);
+}
+
+export async function runBaselineScenario(system, benchmarkCase, providerConfig, limit = 5) {
+  if (benchmarkCase.kind !== 'operations') {
+    return runBaselineRetrieval(system, benchmarkCase, providerConfig, limit);
+  }
+
+  const state = {
+    counter: 0,
+    memories: [],
+    aliases: new Map(),
+  };
+
+  for (const step of benchmarkCase.steps || []) {
+    await applyBaselineStep(system, state, step, providerConfig);
+  }
+
+  return runBaselineRetrieval(
+    system,
+    buildSyntheticCase(benchmarkCase.query, state.memories, benchmarkCase.options),
+    providerConfig,
+    limit,
+  );
+}
+
+export function runKeywordRecencyBaseline(benchmarkCase, limit = 5) {
+  const queryTokens = tokenize(benchmarkCase.query);
+  return sortByScore(flattenMemories(benchmarkCase).map(memory => ({
+    ...memory,
+    type: 'episodic',
+    score: keywordScore(queryTokens, memory.content),
+  }))).slice(0, limit);
+}
+
+export function runRecentWindowBaseline(benchmarkCase, limit = 3) {
+  return flattenMemories(benchmarkCase)
+    .sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
+    .slice(0, limit)
+    .map((memory, index) => ({
+      ...memory,
+      type: 'episodic',
+      score: 1 - index * 0.1,
+    }));
+}
+
+export async function runVectorOnlyBaseline(benchmarkCase, providerConfig, limit = 5) {
+  const provider = createEmbeddingProvider(providerConfig);
+  if (typeof provider.ready === 'function') {
+    await provider.ready();
+  }
+
+  const queryVector = await provider.embed(benchmarkCase.query);
+  const queryBuffer = provider.vectorToBuffer(queryVector);
+
+  const rows = [];
+  for (const memory of flattenMemories(benchmarkCase)) {
+    const vector = await provider.embed(memory.content);
+    const score = cosineSimilarity(queryBuffer, provider.vectorToBuffer(vector), provider);
+    rows.push({
+      ...memory,
+      type: 'episodic',
+      score,
+    });
+  }
+
+  return sortByScore(rows).slice(0, limit);
+}

package/benchmarks/build-leaderboard.mjs

@@ -0,0 +1,170 @@
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { verifyGuardBenchSubmissionBundle } from './verify-submission-bundle.mjs';
+import { validateSchema } from './validate-guardbench-artifacts.mjs';
+import { publicPath } from './public-paths.mjs';
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+function percent(value) {
+  return value == null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
+}
+
+function number(value) {
+  return value == null ? 'n/a' : String(value);
+}
+
+function rowFromBundle(dir) {
+  const verification = verifyGuardBenchSubmissionBundle({ dir });
+  const manifest = readJson(join(resolve(dir), 'submission-manifest.json'));
+  return {
+    subject: manifest.subject,
+    score: manifest.score,
+    conformance: manifest.conformance,
+    source: {
+      dir: publicPath(resolve(dir)),
+      manifestGeneratedAt: manifest.generatedAt,
+      fileCount: manifest.files?.length ?? 0,
+    },
+    verification,
+  };
+}
+
+function compareRows(a, b) {
+  return (
+    Number(b.verification.ok) - Number(a.verification.ok)
+    || Number(b.conformance.ok) - Number(a.conformance.ok)
+    || (b.score.fullContractPassRate ?? -1) - (a.score.fullContractPassRate ?? -1)
+    || (b.score.decisionAccuracy ?? -1) - (a.score.decisionAccuracy ?? -1)
+    || (b.score.evidenceRecall ?? -1) - (a.score.evidenceRecall ?? -1)
+    || (a.score.redactionLeaks ?? Number.MAX_SAFE_INTEGER) - (b.score.redactionLeaks ?? Number.MAX_SAFE_INTEGER)
+    || (a.score.latency?.p95Ms ?? Number.MAX_SAFE_INTEGER) - (b.score.latency?.p95Ms ?? Number.MAX_SAFE_INTEGER)
+    || a.subject.name.localeCompare(b.subject.name)
+  );
+}
+
+export function buildGuardBenchLeaderboard(options = {}) {
+  const bundleDirs = options.bundleDirs?.length
+    ? options.bundleDirs
+    : ['benchmarks/output/submission-bundle'];
+  const rows = bundleDirs.map(rowFromBundle).sort(compareRows)
+    .map((row, index) => ({ rank: index + 1, ...row }));
+  return {
+    schemaVersion: '1.0.0',
+    suite: 'GuardBench leaderboard',
+    generatedAt: new Date().toISOString(),
+    ranking: [
+      'verified bundle',
+      'adapter conformance',
+      'fullContractPassRate',
+      'decisionAccuracy',
+      'evidenceRecall',
+      'redactionLeaks ascending',
+      'latency.p95Ms ascending',
+      'subject.name',
+    ],
+    rows,
+    failures: rows.flatMap(row => row.verification.failures.map(failure => `${row.subject.name}: ${failure}`)),
+  };
+}
+
+export function writeGuardBenchLeaderboard(options = {}) {
+  const outJson = resolve(options.outJson ?? 'benchmarks/output/leaderboard/guardbench-leaderboard.json');
+  const outMd = resolve(options.outMd ?? 'benchmarks/output/leaderboard/guardbench-leaderboard.md');
+  const schemasDir = resolve(options.schemasDir ?? 'benchmarks/schemas');
+  const leaderboard = buildGuardBenchLeaderboard(options);
+  const schema = readJson(join(schemasDir, 'guardbench-leaderboard.schema.json'));
+  const schemaErrors = validateSchema(leaderboard, schema, 'guardbench-leaderboard');
+  if (schemaErrors.length) {
+    throw new Error(`GuardBench leaderboard schema validation failed: ${schemaErrors.join('; ')}`);
+  }
+  mkdirSync(dirname(outJson), { recursive: true });
+  mkdirSync(dirname(outMd), { recursive: true });
+  writeFileSync(outJson, `${JSON.stringify(leaderboard, null, 2)}\n`, 'utf-8');
+  writeFileSync(outMd, renderMarkdown(leaderboard), 'utf-8');
+  return { leaderboard, outJson, outMd };
+}
+
+export function renderMarkdown(leaderboard) {
+  const lines = [
+    '# GuardBench Leaderboard',
+    '',
+    `Generated: ${leaderboard.generatedAt}`,
+    '',
+    '| Rank | Subject | Verified | Conformant | Full Contract | Decision Accuracy | Evidence Recall | Redaction Leaks | p95 Latency | Bundle |',
+    '|---:|---|---:|---:|---:|---:|---:|---:|---:|---|',
+  ];
+  for (const row of leaderboard.rows) {
+    lines.push([
+      row.rank,
+      row.subject.name,
+      row.verification.ok ? 'yes' : 'no',
+      row.conformance.ok ? 'yes' : 'no',
+      percent(row.score.fullContractPassRate),
+      percent(row.score.decisionAccuracy),
+      percent(row.score.evidenceRecall),
+      number(row.score.redactionLeaks),
+      row.score.latency?.p95Ms == null ? 'n/a' : `${row.score.latency.p95Ms}ms`,
+      row.source.dir,
+    ].join(' | ').replace(/^/, '| ').replace(/$/, ' |'));
+  }
+  if (leaderboard.failures.length) {
+    lines.push('', '## Verification Failures', '');
+    for (const failure of leaderboard.failures) lines.push(`- ${failure}`);
+  }
+  lines.push('');
+  return `${lines.join('\n')}`;
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    bundleDirs: [],
+    outJson: 'benchmarks/output/leaderboard/guardbench-leaderboard.json',
+    outMd: 'benchmarks/output/leaderboard/guardbench-leaderboard.md',
+    json: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if ((token === '--bundle' || token === '--dir') && argv[i + 1]) args.bundleDirs.push(argv[++i]);
+    else if (token === '--out-json' && argv[i + 1]) args.outJson = argv[++i];
+    else if (token === '--out-md' && argv[i + 1]) args.outMd = argv[++i];
+    else if (token === '--schemas-dir' && argv[i + 1]) args.schemasDir = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+  return args;
+}
+
+function usage() {
+  return [
+    'Usage: node benchmarks/build-leaderboard.mjs [--bundle <submission-bundle>] [--json]',
+    '',
+    'Builds ranked JSON and Markdown GuardBench leaderboard artifacts from verified',
+    'submission bundles. Repeat --bundle for multiple systems.',
+  ].join('\n');
+}
+
+async function main() {
+  const args = parseArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+  const result = writeGuardBenchLeaderboard(args);
+  if (args.json) console.log(JSON.stringify(result.leaderboard, null, 2));
+  else {
+    console.log(`GuardBench leaderboard JSON: ${result.outJson}`);
+    console.log(`GuardBench leaderboard Markdown: ${result.outMd}`);
+  }
+  if (result.leaderboard.failures.length) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]).endsWith('build-leaderboard.mjs')) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}