audrey 0.23.1 → 1.0.1

package/benchmarks/adapters/zep-cloud.mjs

@@ -0,0 +1,280 @@
+import { randomBytes } from 'node:crypto';
+
+const DEFAULT_BASE_URL = 'https://api.getzep.com';
+const DEFAULT_INGEST_DELAY_MS = 0;
+const BATCH_SIZE = 50;
+
+function requireEnv(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(`${name} is required for the Zep Cloud GuardBench adapter.`);
+  }
+  return value;
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function actionText(action) {
+  return [action.action, action.command, action.tool, action.cwd, ...(action.files ?? [])]
+    .filter(Boolean)
+    .join('\n');
+}
+
+function normalize(text) {
+  return String(text || '').toLowerCase();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(token => token.length > 2);
+}
+
+function tokenOverlap(a, b) {
+  const tokens = tokenize(a);
+  if (tokens.length === 0) return 0;
+  const other = new Set(tokenize(b));
+  let matches = 0;
+  for (const token of tokens) {
+    if (other.has(token)) matches++;
+  }
+  return matches / tokens.length;
+}
+
+function resultText(result) {
+  return [
+    result?.fact,
+    result?.content,
+    result?.summary,
+    result?.name,
+    result?.context,
+  ].filter(Boolean).join('\n');
+}
+
+function collectSearchResults(response) {
+  const rows = [];
+  for (const key of ['edges', 'episodes', 'nodes', 'observations', 'thread_summaries']) {
+    if (Array.isArray(response?.[key])) rows.push(...response[key]);
+  }
+  if (response?.context) rows.push({ uuid: 'zep-context', context: response.context });
+  return rows;
+}
+
+function evidenceIds(results) {
+  return results.map((result, index) =>
+    result?.uuid ?? result?.id ?? result?.task_id ?? result?.thread_id ?? `zep-result-${index + 1}`);
+}
+
+function decisionFromSearchResults(results, action, unsupportedFault = null) {
+  if (unsupportedFault) {
+    return {
+      decision: 'warn',
+      riskScore: 0.55,
+      evidenceIds: evidenceIds(results),
+      recommendedActions: ['External adapter cannot inject storage faults into Zep Cloud; verify memory health separately.'],
+      summary: `Zep Cloud adapter cannot emulate fault injection: ${unsupportedFault}.`,
+    };
+  }
+
+  const query = actionText(action);
+  const texts = results.map(resultText);
+  const joined = texts.join('\n').toLowerCase();
+  if (/\b(must-follow|never|do not|high-risk|conflicting)\b/i.test(joined)) {
+    const relevant = texts.some(text => tokenOverlap(query, text) >= 0.18);
+    return {
+      decision: relevant ? 'block' : 'warn',
+      riskScore: relevant ? 0.85 : 0.55,
+      evidenceIds: evidenceIds(results),
+      recommendedActions: ['Review retrieved Zep graph memory before acting.'],
+      summary: relevant
+        ? 'Zep returned policy-like graph memory with lexical overlap; adapter escalated from retrieval to a block.'
+        : 'Zep returned policy-like graph memory without enough lexical overlap for a block.',
+    };
+  }
+
+  if (results.length > 0) {
+    return {
+      decision: 'warn',
+      riskScore: 0.35,
+      evidenceIds: evidenceIds(results),
+      recommendedActions: ['Treat retrieved Zep graph memory as advisory context.'],
+      summary: 'Zep returned related graph memory, but no policy-like control evidence was found.',
+    };
+  }
+
+  return {
+    decision: 'allow',
+    riskScore: 0,
+    evidenceIds: [],
+    recommendedActions: [],
+    summary: 'Zep returned no relevant graph memory for this action.',
+  };
+}
+
+class ZepCloudClient {
+  constructor({
+    apiKey = requireEnv('ZEP_API_KEY'),
+    baseUrl = process.env.ZEP_BASE_URL ?? DEFAULT_BASE_URL,
+    authScheme = process.env.ZEP_AUTH_SCHEME ?? 'Api-Key',
+    fetchImpl = globalThis.fetch,
+  } = {}) {
+    this.apiKey = apiKey;
+    this.baseUrl = baseUrl.replace(/\/+$/, '');
+    this.authScheme = authScheme;
+    this.fetch = fetchImpl;
+  }
+
+  get authorization() {
+    return this.authScheme ? `${this.authScheme} ${this.apiKey}` : this.apiKey;
+  }
+
+  async request(path, { method = 'GET', body, okStatuses = [200, 201, 204], ignoreNotFound = false } = {}) {
+    const response = await this.fetch(`${this.baseUrl}${path}`, {
+      method,
+      headers: {
+        Authorization: this.authorization,
+        'Content-Type': 'application/json',
+      },
+      body: body == null ? undefined : JSON.stringify(body),
+    });
+
+    if (ignoreNotFound && response.status === 404) return null;
+    if (!okStatuses.includes(response.status)) {
+      const text = await response.text();
+      throw new Error(`Zep ${method} ${path} failed ${response.status}: ${text.slice(0, 500)}`);
+    }
+
+    if (response.status === 204) return null;
+    const text = await response.text();
+    return text ? JSON.parse(text) : null;
+  }
+
+  async createUser(userId) {
+    return this.request('/api/v2/users', {
+      method: 'POST',
+      body: { user_id: userId },
+    });
+  }
+
+  async createSession({ sessionId, userId }) {
+    return this.request('/api/v2/sessions', {
+      method: 'POST',
+      body: { session_id: sessionId, user_id: userId },
+    });
+  }
+
+  async addMessages({ sessionId, messages }) {
+    if (messages.length === 0) return null;
+    return this.request(`/api/v2/sessions/${encodeURIComponent(sessionId)}/memory`, {
+      method: 'POST',
+      body: { messages, return_context: false },
+    });
+  }
+
+  async searchGraph({ userId, query }) {
+    return this.request('/api/v2/graph/search', {
+      method: 'POST',
+      body: {
+        user_id: userId,
+        query: query.slice(0, 400),
+        scope: 'edges',
+        limit: 10,
+      },
+    });
+  }
+
+  async deleteUser(userId) {
+    return this.request(`/api/v2/users/${encodeURIComponent(userId)}`, {
+      method: 'DELETE',
+      ignoreNotFound: true,
+    });
+  }
+}
+
+function message(content) {
+  return {
+    role: 'guardbench',
+    role_type: 'norole',
+    content,
+  };
+}
+
+function memoryMessagesFromScenario(scenario) {
+  const messages = [];
+  for (const memory of scenario.seed.seededMemories ?? []) {
+    messages.push(message(memory.content));
+  }
+  for (const event of scenario.seed.seededToolEvents ?? []) {
+    const seededSecret = event.errorSummaryPattern && scenario.privateSeed?.seededSecrets?.[0]
+      ? `${'x'.repeat(1990)} ${scenario.privateSeed.seededSecrets[0]}`
+      : '';
+    messages.push(message([
+      `Tool event: ${event.tool ?? 'tool'}`,
+      event.action ? `Action: ${event.action}` : '',
+      event.outcome ? `Outcome: ${event.outcome}` : '',
+      event.errorSummary ? `Error: ${event.errorSummary}` : '',
+      event.errorSummaryPattern ? `Error pattern: ${event.errorSummaryPattern}` : '',
+      seededSecret ? `Error: ${seededSecret}` : '',
+      event.output ? `Output: ${event.output}` : '',
+    ].filter(Boolean).join('\n')));
+  }
+  if (scenario.seed.seededNoise?.count) {
+    for (let i = 0; i < scenario.seed.seededNoise.count; i++) {
+      messages.push(message(`Irrelevant background memory ${i}: UI color preference, lunch note, or unrelated calendar detail.`));
+    }
+  }
+  return messages;
+}
+
+async function addInBatches(client, { sessionId, messages }) {
+  for (let i = 0; i < messages.length; i += BATCH_SIZE) {
+    await client.addMessages({
+      sessionId,
+      messages: messages.slice(i, i + BATCH_SIZE),
+    });
+  }
+}
+
+function idForScenario(kind, scenario) {
+  const prefix = process.env.ZEP_GUARDBENCH_USER_PREFIX ?? 'audrey-guardbench';
+  const runId = process.env.ZEP_GUARDBENCH_RUN_ID ?? `${Date.now()}-${randomBytes(8).toString('hex')}`;
+  return `${prefix}-${runId}-${kind}-${scenario.id}`.toLowerCase();
+}
+
+export function createGuardBenchAdapter(options = {}) {
+  return {
+    name: 'Zep Cloud',
+    description: 'Zep Cloud REST adapter using v2 users, sessions, memory.add, graph.search, and user cleanup.',
+    async setup({ scenario }) {
+      const client = new ZepCloudClient(options);
+      const userId = idForScenario('user', scenario);
+      const sessionId = idForScenario('session', scenario);
+      const messages = memoryMessagesFromScenario(scenario);
+      await client.createUser(userId);
+      await client.createSession({ sessionId, userId });
+      await addInBatches(client, { sessionId, messages });
+      const ingestDelayMs = Number(options.ingestDelayMs ?? process.env.ZEP_GUARDBENCH_INGEST_DELAY_MS ?? DEFAULT_INGEST_DELAY_MS);
+      if (ingestDelayMs > 0) await sleep(ingestDelayMs);
+      return { client, userId, sessionId };
+    },
+    async decide({ scenario, action, state }) {
+      const search = await state.client.searchGraph({
+        userId: state.userId,
+        query: actionText(action),
+      });
+      const results = collectSearchResults(search);
+      return decisionFromSearchResults(results, action, scenario.seed.faultInjection);
+    },
+    async cleanup({ state }) {
+      if (state?.client && state?.userId && process.env.ZEP_GUARDBENCH_SKIP_CLEANUP !== '1') {
+        await state.client.deleteUser(state.userId);
+      }
+    },
+  };
+}
+
+export default createGuardBenchAdapter();

package/benchmarks/baselines.js

@@ -0,0 +1,169 @@
+import { createEmbeddingProvider } from '../dist/src/embedding.js';
+import { cosineSimilarity } from '../dist/src/utils.js';
+
+function normalize(text) {
+  return String(text || '').toLowerCase();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+function keywordScore(queryTokens, content) {
+  const contentTokens = new Set(tokenize(content));
+  if (queryTokens.length === 0) return 0;
+  let matches = 0;
+  for (const token of queryTokens) {
+    if (contentTokens.has(token)) matches++;
+  }
+  return matches / queryTokens.length;
+}
+
+function sortByScore(rows) {
+  return rows
+    .filter(row => Number.isFinite(row.score))
+    .sort((a, b) => b.score - a.score || String(b.createdAt || '').localeCompare(String(a.createdAt || '')));
+}
+
+function flattenMemories(benchmarkCase, ids = []) {
+  return benchmarkCase.memory.map((memory, index) => ({
+    id: ids[index] || `memory-${index + 1}`,
+    content: memory.content,
+    source: memory.source,
+    createdAt: memory.createdAt || new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+    private: Boolean(memory.private),
+  }));
+}
+
+function buildSyntheticCase(query, memories, options = {}) {
+  return {
+    query,
+    memory: memories.map(memory => ({
+      content: memory.content,
+      source: memory.source,
+      createdAt: memory.createdAt,
+      private: memory.private,
+    })),
+    options,
+  };
+}
+
+async function runBaselineRetrieval(system, syntheticCase, providerConfig, limit = 5) {
+  switch (system) {
+    case 'Vector Only':
+      return runVectorOnlyBaseline(syntheticCase, providerConfig, limit);
+    case 'Keyword + Recency':
+      return runKeywordRecencyBaseline(syntheticCase, limit);
+    case 'Recent Window':
+      return runRecentWindowBaseline(syntheticCase, limit);
+    default:
+      throw new Error(`Unknown baseline system: ${system}`);
+  }
+}
+
+function createOperationMemory(state, step) {
+  const index = state.counter++;
+  return {
+    id: `memory-${index + 1}`,
+    content: step.memory.content,
+    source: step.memory.source,
+    createdAt: step.memory.createdAt || new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+    private: Boolean(step.memory.private),
+  };
+}
+
+async function applyBaselineStep(system, state, step, providerConfig) {
+  if (step.type === 'encode') {
+    const memory = createOperationMemory(state, step);
+    state.memories.push(memory);
+    if (step.saveAs) {
+      state.aliases.set(step.saveAs, memory.id);
+    }
+    return;
+  }
+
+  if (step.type === 'forgetByQuery') {
+    const syntheticCase = buildSyntheticCase(step.query, state.memories, step.options);
+    const [match] = await runBaselineRetrieval(system, syntheticCase, providerConfig, 1);
+    if (match && Number.isFinite(match.score) && match.score > 0) {
+      state.memories = state.memories.filter(memory => memory.id !== match.id);
+    }
+    return;
+  }
+
+  if (step.type === 'consolidate') {
+    return;
+  }
+
+  throw new Error(`Unsupported baseline step: ${step.type}`);
+}
+
+export async function runBaselineScenario(system, benchmarkCase, providerConfig, limit = 5) {
+  if (benchmarkCase.kind !== 'operations') {
+    return runBaselineRetrieval(system, benchmarkCase, providerConfig, limit);
+  }
+
+  const state = {
+    counter: 0,
+    memories: [],
+    aliases: new Map(),
+  };
+
+  for (const step of benchmarkCase.steps || []) {
+    await applyBaselineStep(system, state, step, providerConfig);
+  }
+
+  return runBaselineRetrieval(
+    system,
+    buildSyntheticCase(benchmarkCase.query, state.memories, benchmarkCase.options),
+    providerConfig,
+    limit,
+  );
+}
+
+export function runKeywordRecencyBaseline(benchmarkCase, limit = 5) {
+  const queryTokens = tokenize(benchmarkCase.query);
+  return sortByScore(flattenMemories(benchmarkCase).map(memory => ({
+    ...memory,
+    type: 'episodic',
+    score: keywordScore(queryTokens, memory.content),
+  }))).slice(0, limit);
+}
+
+export function runRecentWindowBaseline(benchmarkCase, limit = 3) {
+  return flattenMemories(benchmarkCase)
+    .sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
+    .slice(0, limit)
+    .map((memory, index) => ({
+      ...memory,
+      type: 'episodic',
+      score: 1 - index * 0.1,
+    }));
+}
+
+export async function runVectorOnlyBaseline(benchmarkCase, providerConfig, limit = 5) {
+  const provider = createEmbeddingProvider(providerConfig);
+  if (typeof provider.ready === 'function') {
+    await provider.ready();
+  }
+
+  const queryVector = await provider.embed(benchmarkCase.query);
+  const queryBuffer = provider.vectorToBuffer(queryVector);
+
+  const rows = [];
+  for (const memory of flattenMemories(benchmarkCase)) {
+    const vector = await provider.embed(memory.content);
+    const score = cosineSimilarity(queryBuffer, provider.vectorToBuffer(vector), provider);
+    rows.push({
+      ...memory,
+      type: 'episodic',
+      score,
+    });
+  }
+
+  return sortByScore(rows).slice(0, limit);
+}

package/benchmarks/build-leaderboard.mjs

@@ -0,0 +1,170 @@
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { verifyGuardBenchSubmissionBundle } from './verify-submission-bundle.mjs';
+import { validateSchema } from './validate-guardbench-artifacts.mjs';
+import { publicPath } from './public-paths.mjs';
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+function percent(value) {
+  return value == null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
+}
+
+function number(value) {
+  return value == null ? 'n/a' : String(value);
+}
+
+function rowFromBundle(dir) {
+  const verification = verifyGuardBenchSubmissionBundle({ dir });
+  const manifest = readJson(join(resolve(dir), 'submission-manifest.json'));
+  return {
+    subject: manifest.subject,
+    score: manifest.score,
+    conformance: manifest.conformance,
+    source: {
+      dir: publicPath(resolve(dir)),
+      manifestGeneratedAt: manifest.generatedAt,
+      fileCount: manifest.files?.length ?? 0,
+    },
+    verification,
+  };
+}
+
+function compareRows(a, b) {
+  return (
+    Number(b.verification.ok) - Number(a.verification.ok)
+    || Number(b.conformance.ok) - Number(a.conformance.ok)
+    || (b.score.fullContractPassRate ?? -1) - (a.score.fullContractPassRate ?? -1)
+    || (b.score.decisionAccuracy ?? -1) - (a.score.decisionAccuracy ?? -1)
+    || (b.score.evidenceRecall ?? -1) - (a.score.evidenceRecall ?? -1)
+    || (a.score.redactionLeaks ?? Number.MAX_SAFE_INTEGER) - (b.score.redactionLeaks ?? Number.MAX_SAFE_INTEGER)
+    || (a.score.latency?.p95Ms ?? Number.MAX_SAFE_INTEGER) - (b.score.latency?.p95Ms ?? Number.MAX_SAFE_INTEGER)
+    || a.subject.name.localeCompare(b.subject.name)
+  );
+}
+
+export function buildGuardBenchLeaderboard(options = {}) {
+  const bundleDirs = options.bundleDirs?.length
+    ? options.bundleDirs
+    : ['benchmarks/output/submission-bundle'];
+  const rows = bundleDirs.map(rowFromBundle).sort(compareRows)
+    .map((row, index) => ({ rank: index + 1, ...row }));
+  return {
+    schemaVersion: '1.0.0',
+    suite: 'GuardBench leaderboard',
+    generatedAt: new Date().toISOString(),
+    ranking: [
+      'verified bundle',
+      'adapter conformance',
+      'fullContractPassRate',
+      'decisionAccuracy',
+      'evidenceRecall',
+      'redactionLeaks ascending',
+      'latency.p95Ms ascending',
+      'subject.name',
+    ],
+    rows,
+    failures: rows.flatMap(row => row.verification.failures.map(failure => `${row.subject.name}: ${failure}`)),
+  };
+}
+
+export function writeGuardBenchLeaderboard(options = {}) {
+  const outJson = resolve(options.outJson ?? 'benchmarks/output/leaderboard/guardbench-leaderboard.json');
+  const outMd = resolve(options.outMd ?? 'benchmarks/output/leaderboard/guardbench-leaderboard.md');
+  const schemasDir = resolve(options.schemasDir ?? 'benchmarks/schemas');
+  const leaderboard = buildGuardBenchLeaderboard(options);
+  const schema = readJson(join(schemasDir, 'guardbench-leaderboard.schema.json'));
+  const schemaErrors = validateSchema(leaderboard, schema, 'guardbench-leaderboard');
+  if (schemaErrors.length) {
+    throw new Error(`GuardBench leaderboard schema validation failed: ${schemaErrors.join('; ')}`);
+  }
+  mkdirSync(dirname(outJson), { recursive: true });
+  mkdirSync(dirname(outMd), { recursive: true });
+  writeFileSync(outJson, `${JSON.stringify(leaderboard, null, 2)}\n`, 'utf-8');
+  writeFileSync(outMd, renderMarkdown(leaderboard), 'utf-8');
+  return { leaderboard, outJson, outMd };
+}
+
+export function renderMarkdown(leaderboard) {
+  const lines = [
+    '# GuardBench Leaderboard',
+    '',
+    `Generated: ${leaderboard.generatedAt}`,
+    '',
+    '| Rank | Subject | Verified | Conformant | Full Contract | Decision Accuracy | Evidence Recall | Redaction Leaks | p95 Latency | Bundle |',
+    '|---:|---|---:|---:|---:|---:|---:|---:|---:|---|',
+  ];
+  for (const row of leaderboard.rows) {
+    lines.push([
+      row.rank,
+      row.subject.name,
+      row.verification.ok ? 'yes' : 'no',
+      row.conformance.ok ? 'yes' : 'no',
+      percent(row.score.fullContractPassRate),
+      percent(row.score.decisionAccuracy),
+      percent(row.score.evidenceRecall),
+      number(row.score.redactionLeaks),
+      row.score.latency?.p95Ms == null ? 'n/a' : `${row.score.latency.p95Ms}ms`,
+      row.source.dir,
+    ].join(' | ').replace(/^/, '| ').replace(/$/, ' |'));
+  }
+  if (leaderboard.failures.length) {
+    lines.push('', '## Verification Failures', '');
+    for (const failure of leaderboard.failures) lines.push(`- ${failure}`);
+  }
+  lines.push('');
+  return `${lines.join('\n')}`;
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    bundleDirs: [],
+    outJson: 'benchmarks/output/leaderboard/guardbench-leaderboard.json',
+    outMd: 'benchmarks/output/leaderboard/guardbench-leaderboard.md',
+    json: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if ((token === '--bundle' || token === '--dir') && argv[i + 1]) args.bundleDirs.push(argv[++i]);
+    else if (token === '--out-json' && argv[i + 1]) args.outJson = argv[++i];
+    else if (token === '--out-md' && argv[i + 1]) args.outMd = argv[++i];
+    else if (token === '--schemas-dir' && argv[i + 1]) args.schemasDir = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+  return args;
+}
+
+function usage() {
+  return [
+    'Usage: node benchmarks/build-leaderboard.mjs [--bundle <submission-bundle>] [--json]',
+    '',
+    'Builds ranked JSON and Markdown GuardBench leaderboard artifacts from verified',
+    'submission bundles. Repeat --bundle for multiple systems.',
+  ].join('\n');
+}
+
+async function main() {
+  const args = parseArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+  const result = writeGuardBenchLeaderboard(args);
+  if (args.json) console.log(JSON.stringify(result.leaderboard, null, 2));
+  else {
+    console.log(`GuardBench leaderboard JSON: ${result.outJson}`);
+    console.log(`GuardBench leaderboard Markdown: ${result.outMd}`);
+  }
+  if (result.leaderboard.failures.length) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]).endsWith('build-leaderboard.mjs')) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}