audrey 0.23.1 → 1.0.1

package/benchmarks/adapter-self-test.mjs

@@ -0,0 +1,166 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { basename, dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadExternalAdapters, runGuardBench } from './guardbench.js';
+import { evaluateAdapterConformance } from './run-external-guardbench.mjs';
+import { validateSchema } from './validate-guardbench-artifacts.mjs';
+import { publicPath } from './public-paths.mjs';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const DEFAULT_ADAPTER = 'benchmarks/adapters/example-allow.mjs';
+const DEFAULT_OUT = 'benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json';
+const DEFAULT_SCHEMA = 'benchmarks/schemas/guardbench-adapter-self-test.schema.json';
+const RESULT_FIELDS = [
+  'decision',
+  'riskScore',
+  'evidenceIds',
+  'recommendedActions',
+  'summary',
+  'recallErrors',
+];
+
+export function parseAdapterSelfTestArgs(argv = process.argv.slice(2)) {
+  const args = {
+    adapter: DEFAULT_ADAPTER,
+    out: DEFAULT_OUT,
+    json: false,
+    noWrite: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (token === '--adapter' && argv[i + 1]) args.adapter = argv[++i];
+    else if (token === '--out' && argv[i + 1]) args.out = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--no-write') args.noWrite = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return args;
+}
+
+function usage() {
+  return `Usage: node benchmarks/adapter-self-test.mjs [options]
+
+Options:
+  --adapter <path>   ESM GuardBench adapter path. Default: ${DEFAULT_ADAPTER}.
+  --out <path>       JSON report path. Default: ${DEFAULT_OUT}.
+  --json             Print the full JSON report.
+  --no-write         Do not write the JSON report.
+`;
+}
+
+function systemSummary(report, adapterName) {
+  return report.systemSummaries.find(row => row.system === adapterName) ?? null;
+}
+
+function scoreFromReport(report, adapterName) {
+  const summary = systemSummary(report, adapterName);
+  return {
+    scenarios: summary?.scenarios ?? 0,
+    fullContractPassRate: summary?.passRate ?? null,
+    decisionAccuracy: summary?.decisionAccuracy ?? null,
+    evidenceRecall: summary?.evidenceRecall ?? null,
+    redactionLeaks: summary?.redactionLeaks ?? null,
+    latency: summary?.latency ?? null,
+  };
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+export function validateAdapterSelfTestReport(report, options = {}) {
+  const schemaPath = resolve(ROOT, options.schema ?? DEFAULT_SCHEMA);
+  const schema = options.schemaObject ?? readJson(schemaPath);
+  return validateSchema(report, schema, 'guardbench-adapter-self-test');
+}
+
+export async function runGuardBenchAdapterSelfTest(options = {}) {
+  const adapterPath = resolve(ROOT, options.adapterPath ?? options.adapter ?? DEFAULT_ADAPTER);
+  if (!existsSync(adapterPath)) {
+    throw new Error(`GuardBench adapter not found: ${adapterPath}`);
+  }
+
+  const adapters = await loadExternalAdapters([adapterPath]);
+  if (adapters.length !== 1) {
+    throw new Error(`GuardBench adapter self-test expected 1 adapter, got ${adapters.length}`);
+  }
+
+  const [adapter] = adapters;
+  const report = await runGuardBench({ externalAdapters: adapters });
+  const conformance = evaluateAdapterConformance(report, adapter.name);
+  const score = scoreFromReport(report, conformance.adapter);
+  const selfTest = {
+    schemaVersion: '1.0.0',
+    suite: 'GuardBench adapter self-test',
+    generatedAt: new Date().toISOString(),
+    ok: conformance.ok,
+    adapter: {
+      name: adapter.name,
+      path: publicPath(adapterPath),
+      moduleFile: basename(adapterPath),
+      description: adapter.description ?? null,
+    },
+    conformance,
+    score,
+    contract: {
+      expectedAnswersWithheld: true,
+      lowScoreAllowed: true,
+      requiredScenarioRows: report.scenarios,
+      requiredResultFields: RESULT_FIELDS,
+      redactionLeakTolerance: 0,
+    },
+    failures: conformance.failures,
+  };
+  const schemaErrors = validateAdapterSelfTestReport(selfTest);
+  if (schemaErrors.length > 0) {
+    throw new Error(`GuardBench adapter self-test schema validation failed: ${schemaErrors.join('; ')}`);
+  }
+
+  if (options.out && options.write !== false) {
+    const outPath = resolve(ROOT, options.out);
+    mkdirSync(dirname(outPath), { recursive: true });
+    writeFileSync(outPath, `${JSON.stringify(selfTest, null, 2)}\n`, 'utf-8');
+    selfTest.outPath = publicPath(outPath);
+  }
+
+  return selfTest;
+}
+
+async function main() {
+  const args = parseAdapterSelfTestArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+
+  const result = await runGuardBenchAdapterSelfTest({
+    adapter: args.adapter,
+    out: args.noWrite ? null : args.out,
+    write: !args.noWrite,
+  });
+
+  if (args.json) {
+    console.log(JSON.stringify(result, null, 2));
+  } else if (result.ok) {
+    console.log(`GuardBench adapter self-test passed: ${result.adapter.name}`);
+    console.log(`Contract rows: ${result.conformance.scenarios}/${result.conformance.expectedScenarios}`);
+    console.log(`Full-contract score: ${(result.score.fullContractPassRate * 100).toFixed(1)}%`);
+    console.log(`Decision accuracy: ${(result.score.decisionAccuracy * 100).toFixed(1)}%`);
+    if (result.outPath) console.log(`Self-test report: ${result.outPath}`);
+  } else {
+    console.error(`GuardBench adapter self-test failed: ${result.adapter.name}`);
+    for (const failure of result.failures) console.error(`- ${failure}`);
+  }
+
+  process.exitCode = result.ok ? 0 : 1;
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main().catch(error => {
+    console.error(error.message);
+    process.exit(1);
+  });
+}

package/benchmarks/adapters/example-allow.mjs

@@ -0,0 +1,28 @@
+import { defineGuardBenchAdapter } from '../adapter-kit.mjs';
+
+export default defineGuardBenchAdapter({
+  name: 'Example Allow Adapter',
+  description: 'Credential-free GuardBench adapter example. It always allows and is useful for adapter-loading smoke tests.',
+  async setup({ scenario }) {
+    return {
+      memoryCount: (scenario.seed.seededMemories ?? []).length,
+      toolEventCount: (scenario.seed.seededToolEvents ?? []).length,
+      hasFaultInjection: Boolean(scenario.seed.faultInjection),
+    };
+  },
+  async decide({ scenario, state }) {
+    return {
+      decision: 'allow',
+      riskScore: 0,
+      evidenceIds: [],
+      recommendedActions: [],
+      summary: [
+        `Example adapter loaded ${state.memoryCount} seeded memories`,
+        `${state.toolEventCount} seeded tool events`,
+        scenario.seed.seededNoise ? `${scenario.seed.seededNoise.count} noise memories` : 'no noise block',
+        state.hasFaultInjection ? 'fault injection present but unsupported' : 'no fault injection',
+      ].join('; '),
+    };
+  },
+  async cleanup() {},
+});

package/benchmarks/adapters/mem0-platform.mjs

@@ -0,0 +1,267 @@
+import { randomBytes } from 'node:crypto';
+
+const DEFAULT_BASE_URL = 'https://api.mem0.ai';
+const DEFAULT_POLL_TIMEOUT_MS = 60_000;
+const DEFAULT_POLL_INTERVAL_MS = 1_000;
+const BATCH_SIZE = 100;
+
+function requireEnv(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(`${name} is required for the Mem0 GuardBench adapter.`);
+  }
+  return value;
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function actionText(action) {
+  return [action.action, action.command, action.tool, action.cwd, ...(action.files ?? [])]
+    .filter(Boolean)
+    .join('\n');
+}
+
+function normalize(text) {
+  return String(text || '').toLowerCase();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(token => token.length > 2);
+}
+
+function tokenOverlap(a, b) {
+  const tokens = tokenize(a);
+  if (tokens.length === 0) return 0;
+  const other = new Set(tokenize(b));
+  let matches = 0;
+  for (const token of tokens) {
+    if (other.has(token)) matches++;
+  }
+  return matches / tokens.length;
+}
+
+function memoryText(memory) {
+  return String(memory?.memory ?? memory?.text ?? memory?.content ?? '');
+}
+
+function evidenceIds(memories) {
+  return memories
+    .map(memory => memory?.id ?? memory?.memory_id)
+    .filter(Boolean);
+}
+
+function decisionFromMemories(memories, action, unsupportedFault = null) {
+  if (unsupportedFault) {
+    return {
+      decision: 'warn',
+      riskScore: 0.55,
+      evidenceIds: evidenceIds(memories),
+      recommendedActions: ['External adapter cannot inject storage faults into Mem0 Platform; verify memory health separately.'],
+      summary: `Mem0 adapter cannot emulate fault injection: ${unsupportedFault}.`,
+    };
+  }
+
+  const query = actionText(action);
+  const texts = memories.map(memoryText);
+  const joined = texts.join('\n').toLowerCase();
+  if (/\b(must-follow|never|do not|high-risk|conflicting)\b/i.test(joined)) {
+    const relevant = texts.some(text => tokenOverlap(query, text) >= 0.18);
+    return {
+      decision: relevant ? 'block' : 'warn',
+      riskScore: relevant ? 0.85 : 0.55,
+      evidenceIds: evidenceIds(memories),
+      recommendedActions: ['Review retrieved Mem0 memory before acting.'],
+      summary: relevant
+        ? 'Mem0 retrieved policy-like memory with lexical overlap; adapter escalated from retrieval to a block.'
+        : 'Mem0 retrieved policy-like memory without enough lexical overlap for a block.',
+    };
+  }
+
+  if (memories.length > 0) {
+    return {
+      decision: 'warn',
+      riskScore: 0.35,
+      evidenceIds: evidenceIds(memories),
+      recommendedActions: ['Treat retrieved Mem0 memories as advisory context.'],
+      summary: 'Mem0 retrieved related memory, but no policy-like control evidence was found.',
+    };
+  }
+
+  return {
+    decision: 'allow',
+    riskScore: 0,
+    evidenceIds: [],
+    recommendedActions: [],
+    summary: 'Mem0 returned no relevant memories for this action.',
+  };
+}
+
+class Mem0PlatformClient {
+  constructor({
+    apiKey = requireEnv('MEM0_API_KEY'),
+    baseUrl = process.env.MEM0_BASE_URL ?? DEFAULT_BASE_URL,
+    pollTimeoutMs = Number(process.env.MEM0_EVENT_TIMEOUT_MS ?? DEFAULT_POLL_TIMEOUT_MS),
+    pollIntervalMs = Number(process.env.MEM0_EVENT_POLL_INTERVAL_MS ?? DEFAULT_POLL_INTERVAL_MS),
+    fetchImpl = globalThis.fetch,
+  } = {}) {
+    this.apiKey = apiKey;
+    this.baseUrl = baseUrl.replace(/\/+$/, '');
+    this.pollTimeoutMs = pollTimeoutMs;
+    this.pollIntervalMs = pollIntervalMs;
+    this.fetch = fetchImpl;
+  }
+
+  async request(path, options = {}) {
+    const response = await this.fetch(`${this.baseUrl}${path}`, {
+      ...options,
+      headers: {
+        Authorization: `Token ${this.apiKey}`,
+        'Content-Type': 'application/json',
+        ...(options.headers ?? {}),
+      },
+    });
+
+    if (!response.ok && response.status !== 204) {
+      const body = await response.text();
+      throw new Error(`Mem0 ${options.method ?? 'GET'} ${path} failed ${response.status}: ${body.slice(0, 500)}`);
+    }
+
+    if (response.status === 204) return null;
+    const text = await response.text();
+    return text ? JSON.parse(text) : null;
+  }
+
+  async addMessages({ userId, messages, metadata }) {
+    if (messages.length === 0) return;
+    const response = await this.request('/v3/memories/add/', {
+      method: 'POST',
+      body: JSON.stringify({
+        user_id: userId,
+        messages,
+        metadata,
+        infer: false,
+      }),
+    });
+    if (response?.event_id) {
+      await this.waitForEvent(response.event_id);
+    }
+  }
+
+  async waitForEvent(eventId) {
+    const started = Date.now();
+    while (Date.now() - started < this.pollTimeoutMs) {
+      const event = await this.request(`/v1/event/${eventId}/`);
+      if (event?.status === 'SUCCEEDED') return event;
+      if (event?.status === 'FAILED') {
+        throw new Error(`Mem0 event ${eventId} failed: ${JSON.stringify(event).slice(0, 500)}`);
+      }
+      await sleep(this.pollIntervalMs);
+    }
+    throw new Error(`Timed out waiting for Mem0 event ${eventId}.`);
+  }
+
+  async search({ userId, query }) {
+    const response = await this.request('/v2/memories/search', {
+      method: 'POST',
+      body: JSON.stringify({
+        query,
+        version: 'v2',
+        filters: { user_id: userId },
+      }),
+    });
+    return Array.isArray(response) ? response : response?.results ?? [];
+  }
+
+  async deleteUser(userId) {
+    await this.request(`/v2/entities/user/${encodeURIComponent(userId)}/`, {
+      method: 'DELETE',
+    });
+  }
+}
+
+function memoryMessagesFromScenario(scenario) {
+  const messages = [];
+  for (const memory of scenario.seed.seededMemories ?? []) {
+    messages.push({ role: 'user', content: memory.content });
+  }
+  for (const event of scenario.seed.seededToolEvents ?? []) {
+    const seededSecret = event.errorSummaryPattern && scenario.privateSeed?.seededSecrets?.[0]
+      ? `${'x'.repeat(1990)} ${scenario.privateSeed.seededSecrets[0]}`
+      : '';
+    messages.push({
+      role: 'user',
+      content: [
+        `Tool event: ${event.tool ?? 'tool'}`,
+        event.action ? `Action: ${event.action}` : '',
+        event.outcome ? `Outcome: ${event.outcome}` : '',
+        event.errorSummary ? `Error: ${event.errorSummary}` : '',
+        event.errorSummaryPattern ? `Error pattern: ${event.errorSummaryPattern}` : '',
+        seededSecret ? `Error: ${seededSecret}` : '',
+        event.output ? `Output: ${event.output}` : '',
+      ].filter(Boolean).join('\n'),
+    });
+  }
+  if (scenario.seed.seededNoise?.count) {
+    for (let i = 0; i < scenario.seed.seededNoise.count; i++) {
+      messages.push({
+        role: 'user',
+        content: `Irrelevant background memory ${i}: UI color preference, lunch note, or unrelated calendar detail.`,
+      });
+    }
+  }
+  return messages;
+}
+
+async function addInBatches(client, { userId, scenario, messages }) {
+  for (let i = 0; i < messages.length; i += BATCH_SIZE) {
+    await client.addMessages({
+      userId,
+      messages: messages.slice(i, i + BATCH_SIZE),
+      metadata: {
+        benchmark: 'guardbench',
+        scenario_id: scenario.id,
+        adapter: 'mem0-platform',
+      },
+    });
+  }
+}
+
+function userIdForScenario(scenario) {
+  const prefix = process.env.MEM0_GUARDBENCH_USER_PREFIX ?? 'audrey-guardbench';
+  const runId = process.env.MEM0_GUARDBENCH_RUN_ID ?? `${Date.now()}-${randomBytes(8).toString('hex')}`;
+  return `${prefix}-${runId}-${scenario.id}`.toLowerCase();
+}
+
+export function createGuardBenchAdapter(options = {}) {
+  return {
+    name: 'Mem0 Platform',
+    description: 'Mem0 Platform REST adapter using V3 add, V2 search, event polling, and entity cleanup.',
+    async setup({ scenario }) {
+      const client = new Mem0PlatformClient(options);
+      const userId = userIdForScenario(scenario);
+      const messages = memoryMessagesFromScenario(scenario);
+      await addInBatches(client, { userId, scenario, messages });
+      return { client, userId };
+    },
+    async decide({ scenario, action, state }) {
+      const memories = await state.client.search({
+        userId: state.userId,
+        query: actionText(action),
+      });
+      return decisionFromMemories(memories, action, scenario.seed.faultInjection);
+    },
+    async cleanup({ state }) {
+      if (state?.client && state?.userId && process.env.MEM0_GUARDBENCH_SKIP_CLEANUP !== '1') {
+        await state.client.deleteUser(state.userId);
+      }
+    },
+  };
+}
+
+export default createGuardBenchAdapter();

package/benchmarks/adapters/registry.json

@@ -0,0 +1,51 @@
+{
+  "schemaVersion": "1.0.0",
+  "suite": "GuardBench adapter registry",
+  "adapters": [
+    {
+      "id": "example-allow",
+      "name": "Example Allow Adapter",
+      "path": "benchmarks/adapters/example-allow.mjs",
+      "status": "reference",
+      "credentialMode": "none",
+      "requiredEnv": [],
+      "description": "Credential-free reference adapter for validating module loading, self-test generation, and report validation.",
+      "commands": {
+        "moduleValidate": "npm run bench:guard:adapter-module:validate -- --adapter benchmarks/adapters/example-allow.mjs",
+        "selfTest": "npm run bench:guard:adapter-self-test -- --adapter benchmarks/adapters/example-allow.mjs",
+        "selfTestValidate": "npm run bench:guard:adapter-self-test:validate -- --report benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json",
+        "externalRun": "npm run bench:guard:adapter-conformance"
+      }
+    },
+    {
+      "id": "mem0-platform",
+      "name": "Mem0 Platform",
+      "path": "benchmarks/adapters/mem0-platform.mjs",
+      "status": "external-system",
+      "credentialMode": "runtime-env",
+      "requiredEnv": ["MEM0_API_KEY"],
+      "description": "Mem0 Platform REST adapter using V3 add, V2 search, event polling, and entity cleanup.",
+      "commands": {
+        "moduleValidate": "npm run bench:guard:adapter-module:validate -- --adapter benchmarks/adapters/mem0-platform.mjs",
+        "selfTest": "npm run bench:guard:adapter-self-test -- --adapter benchmarks/adapters/mem0-platform.mjs",
+        "selfTestValidate": "npm run bench:guard:adapter-self-test:validate -- --report benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json",
+        "externalRun": "npm run bench:guard:mem0"
+      }
+    },
+    {
+      "id": "zep-cloud",
+      "name": "Zep Cloud",
+      "path": "benchmarks/adapters/zep-cloud.mjs",
+      "status": "external-system",
+      "credentialMode": "runtime-env",
+      "requiredEnv": ["ZEP_API_KEY"],
+      "description": "Zep Cloud REST adapter using v2 users, sessions, memory.add, graph.search, and user cleanup.",
+      "commands": {
+        "moduleValidate": "npm run bench:guard:adapter-module:validate -- --adapter benchmarks/adapters/zep-cloud.mjs",
+        "selfTest": "npm run bench:guard:adapter-self-test -- --adapter benchmarks/adapters/zep-cloud.mjs",
+        "selfTestValidate": "npm run bench:guard:adapter-self-test:validate -- --report benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json",
+        "externalRun": "npm run bench:guard:zep"
+      }
+    }
+  ]
+}