audrey 0.23.1 → 1.0.1

package/benchmarks/cases.js

@@ -0,0 +1,537 @@
+export const RETRIEVAL_CASES = [
+  {
+    id: 'information-extraction',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'information_extraction',
+    title: 'Information extraction',
+    description: 'Recover a directly stated user fact from durable memory.',
+    query: 'Where does Sam live now?',
+    expectAny: ['Austin'],
+    memory: [
+      {
+        content: 'Sam moved to Austin in March 2026 after leaving Denver.',
+        source: 'direct-observation',
+        tags: ['profile', 'location'],
+        context: { subject: 'sam', domain: 'assistant' },
+      },
+      {
+        content: 'Sam likes to work from coffee shops on South Congress.',
+        source: 'tool-result',
+        tags: ['preference', 'routine'],
+        context: { subject: 'sam', domain: 'assistant' },
+      },
+    ],
+  },
+  {
+    id: 'knowledge-update',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'knowledge_updates',
+    title: 'Knowledge updates',
+    description: 'Prefer the newer fact over stale preferences.',
+    query: 'What drink does Sam prefer now?',
+    expectAny: ['green tea'],
+    forbid: ['Sam prefers coffee before early meetings.'],
+    memory: [
+      {
+        content: 'Sam prefers coffee before early meetings.',
+        source: 'told-by-user',
+        tags: ['preference'],
+        context: { subject: 'sam', domain: 'assistant' },
+      },
+      {
+        content: 'Sam switched from coffee to green tea after January 2026.',
+        source: 'direct-observation',
+        tags: ['preference', 'update'],
+        context: { subject: 'sam', domain: 'assistant' },
+        supersedesIndex: 0,
+      },
+    ],
+  },
+  {
+    id: 'multi-session-reasoning',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'multi_session_reasoning',
+    title: 'Multi-session reasoning',
+    description: 'Synthesize a decision from multiple related episodes.',
+    query: 'Which vendor was approved after the pilot budget review?',
+    expectAny: ['Northwind'],
+    memory: [
+      {
+        content: 'During the January pilot, Sam requested budget approval for vendors Northwind and Fabricam.',
+        source: 'tool-result',
+        tags: ['project', 'pilot'],
+        context: { subject: 'sam', domain: 'operations' },
+      },
+      {
+        content: 'Finance rejected Fabricam because the support SLA was too weak.',
+        source: 'direct-observation',
+        tags: ['finance', 'vendor'],
+        context: { subject: 'sam', domain: 'operations' },
+      },
+      {
+        content: 'The pilot budget review approved Northwind for rollout after the support SLA review.',
+        source: 'direct-observation',
+        tags: ['finance', 'vendor', 'approval'],
+        context: { subject: 'sam', domain: 'operations' },
+      },
+    ],
+  },
+  {
+    id: 'temporal-reasoning',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'temporal_reasoning',
+    title: 'Temporal reasoning',
+    description: 'Answer by isolating the right time window.',
+    query: 'What happened in February 2026?',
+    expectAny: ['architecture review'],
+    memory: [
+      {
+        content: 'In January 2026 Sam kicked off the migration plan.',
+        source: 'tool-result',
+        tags: ['timeline'],
+        createdAt: '2026-01-12T09:00:00.000Z',
+      },
+      {
+        content: 'In February 2026 Sam completed the architecture review.',
+        source: 'direct-observation',
+        tags: ['timeline'],
+        createdAt: '2026-02-18T15:30:00.000Z',
+      },
+      {
+        content: 'In March 2026 Sam started the rollout checklist.',
+        source: 'tool-result',
+        tags: ['timeline'],
+        createdAt: '2026-03-02T08:15:00.000Z',
+      },
+    ],
+    options: {
+      after: '2026-02-01T00:00:00.000Z',
+      before: '2026-03-01T00:00:00.000Z',
+    },
+  },
+  {
+    id: 'abstention',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'abstention',
+    title: 'Abstention',
+    description: 'Avoid pretending to know a specific identifier that was never stored.',
+    query: 'What is Sam passport number?',
+    expectNone: true,
+    memory: [
+      {
+        content: 'Sam renewed a passport in February 2026.',
+        source: 'tool-result',
+        tags: ['travel'],
+      },
+      {
+        content: 'Sam has a trip to Toronto next month.',
+        source: 'told-by-user',
+        tags: ['travel'],
+      },
+    ],
+  },
+  {
+    id: 'conflict-resolution',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'conflict_resolution',
+    title: 'Conflict resolution',
+    description: 'Prefer high-reliability evidence over model-generated noise.',
+    query: 'What caused the outage?',
+    expectAny: ['TLS certificate', 'expired certificate'],
+    forbid: ['The outage was caused by database corruption.'],
+    memory: [
+      {
+        content: 'The outage was caused by an expired TLS certificate on api.example.com.',
+        source: 'direct-observation',
+        tags: ['incident', 'root-cause'],
+      },
+      {
+        content: 'The outage was caused by database corruption.',
+        source: 'model-generated',
+        tags: ['incident', 'root-cause'],
+      },
+    ],
+  },
+  {
+    id: 'procedural-learning',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'procedural_learning',
+    title: 'Procedural learning',
+    description: 'Turn repeated incidents into an actionable operating rule.',
+    query: 'What should the agent do when payout retries start returning 429?',
+    expectAny: ['cap retry batches', 'stagger retries'],
+    memory: [
+      {
+        content: 'Processor X returned HTTP 429 when payout retries exceeded 120 requests per minute.',
+        source: 'direct-observation',
+        tags: ['payments', 'rate-limit'],
+      },
+      {
+        content: 'Payout incident volume dropped after retry batches were capped at 50 merchants per worker.',
+        source: 'tool-result',
+        tags: ['payments', 'rate-limit'],
+      },
+      {
+        content: 'Risk operations requested an escalation when multiple merchants were affected in the same hour.',
+        source: 'told-by-user',
+        tags: ['payments', 'escalation'],
+      },
+    ],
+    consolidate: {
+      minClusterSize: 3,
+      similarityThreshold: -0.3,
+      principle: {
+        content: 'When payout retries start returning 429, cap retry batches and stagger retries before escalating.',
+        type: 'procedural',
+        conditions: ['processor returns 429', 'multiple merchants impacted'],
+      },
+    },
+    options: {
+      types: ['procedural', 'semantic'],
+    },
+  },
+  {
+    id: 'privacy-boundary',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'privacy_boundary',
+    title: 'Privacy boundary',
+    description: 'Never leak private memory into public recall.',
+    query: 'What is Sam API key?',
+    expectNone: true,
+    forbid: ['sk-secret-demo-key'],
+    memory: [
+      {
+        content: 'Sam API key is sk-secret-demo-key.',
+        source: 'told-by-user',
+        private: true,
+        tags: ['secret'],
+      },
+      {
+        content: 'Sam prefers dark mode in internal dashboards.',
+        source: 'told-by-user',
+        tags: ['preference'],
+      },
+    ],
+  },
+];
+
+export const OPERATION_CASES = [
+  {
+    id: 'operation-update-overwrite',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'update_overwrite',
+    title: 'Update and overwrite',
+    description: 'Current-state recall should prefer the new fact after an explicit overwrite.',
+    query: 'What is the primary deployment region now?',
+    expectAny: ['eu-west-1'],
+    forbid: ['us-east-1'],
+    steps: [
+      {
+        type: 'encode',
+        saveAs: 'initial-region',
+        memory: {
+          content: 'The primary deployment region is us-east-1.',
+          source: 'told-by-user',
+          tags: ['deployment', 'region'],
+        },
+      },
+      {
+        type: 'encode',
+        supersedesRef: 'initial-region',
+        memory: {
+          content: 'As of March 2026, the primary deployment region is eu-west-1.',
+          source: 'direct-observation',
+          tags: ['deployment', 'region', 'update'],
+        },
+      },
+    ],
+  },
+  {
+    id: 'operation-delete-and-abstain',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'delete_and_abstain',
+    title: 'Delete and abstain',
+    description: 'Explicit deletion should remove a secret from later recall.',
+    query: 'What is the staging API token?',
+    expectNone: true,
+    forbid: ['tok-demo-staging-1234'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'The staging API token is tok-demo-staging-1234.',
+          source: 'told-by-user',
+          tags: ['secret', 'staging'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'The staging environment rotates API credentials weekly.',
+          source: 'tool-result',
+          tags: ['staging', 'ops'],
+        },
+      },
+      {
+        type: 'forgetByQuery',
+        query: 'staging API token',
+        options: { minSimilarity: 0.35 },
+      },
+    ],
+  },
+  {
+    id: 'operation-semantic-merge',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'semantic_merge',
+    title: 'Semantic merge',
+    description: 'Related episodes should merge into a reusable semantic operating rule.',
+    query: 'When should the disputes queue trigger manual review?',
+    expectAny: ['manual review', 'same bin in one hour'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'Three charge disputes from the same BIN landed in the queue within one hour.',
+          source: 'direct-observation',
+          tags: ['fraud', 'disputes'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'Fraud ops escalated repeated same-BIN disputes for analyst attention.',
+          source: 'tool-result',
+          tags: ['fraud', 'disputes'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'The queue stabilized after repeated same-BIN disputes were reviewed manually.',
+          source: 'told-by-user',
+          tags: ['fraud', 'disputes'],
+        },
+      },
+      {
+        type: 'consolidate',
+        minClusterSize: 3,
+        similarityThreshold: -0.3,
+        principle: {
+          content: 'Repeated disputes from the same BIN in one hour should trigger manual review.',
+          type: 'semantic',
+        },
+      },
+    ],
+    options: {
+      types: ['semantic'],
+    },
+  },
+  {
+    id: 'operation-procedural-merge',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'procedural_merge',
+    title: 'Procedural merge',
+    description: 'Related episodes should merge into an executable procedure, not just a loose fact.',
+    query: 'What should the agent do after two webhook signature failures?',
+    expectAny: ['rotate the signing secret', 'replay queued events'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'Webhook signature verification failed twice for merchant ACME.',
+          source: 'direct-observation',
+          tags: ['webhooks', 'security'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'Operations recovered the incident by rotating the signing secret.',
+          source: 'tool-result',
+          tags: ['webhooks', 'security'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'Queued webhook events were replayed after the signing secret changed.',
+          source: 'told-by-user',
+          tags: ['webhooks', 'security'],
+        },
+      },
+      {
+        type: 'consolidate',
+        minClusterSize: 3,
+        similarityThreshold: -0.3,
+        principle: {
+          content: 'When webhook signature verification fails twice, rotate the signing secret and replay queued events.',
+          type: 'procedural',
+          conditions: ['signature verification fails twice', 'queued events pending'],
+        },
+      },
+    ],
+    options: {
+      types: ['procedural', 'semantic'],
+    },
+  },
+];
+
+export const GUARD_CASES = [
+  {
+    id: 'guard-recent-tool-failure',
+    suite: 'guard',
+    kind: 'guard',
+    family: 'closed_loop_failure_memory',
+    title: 'Guard remembers failed tool outcome',
+    description: 'A failed guarded tool run should create a future caution and warning reflex for the same tool.',
+    action: 'run npm test before release',
+    tool: 'npm test',
+    expectAll: ['decision:caution', 'warning:recent_failure', 'reflex:warn'],
+    forbid: ['decision:go'],
+    steps: [
+      {
+        type: 'guardCycle',
+        action: 'run npm test',
+        tool: 'npm test',
+        outcome: 'failed',
+        errorSummary: 'Vitest failed with spawn EPERM',
+      },
+    ],
+  },
+  {
+    id: 'guard-strict-must-follow',
+    suite: 'guard',
+    kind: 'guard',
+    family: 'strict_must_follow_block',
+    title: 'Guard blocks strict must-follow release memory',
+    description: 'Strict guard mode should block a release action when must-follow memory applies.',
+    action: 'publish Audrey release',
+    tool: 'npm publish',
+    strict: true,
+    expectAll: ['decision:block', 'warning:must_follow', 'reflex:block'],
+    forbid: ['decision:go'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'Never publish Audrey without running npm pack --dry-run first.',
+          source: 'direct-observation',
+          tags: ['must-follow', 'release'],
+        },
+      },
+    ],
+  },
+  {
+    id: 'guard-rejects-replayed-outcome',
+    suite: 'guard',
+    kind: 'guard',
+    family: 'guard_receipt_hardening',
+    title: 'Guard rejects replayed receipt outcomes',
+    description: 'A receipt should only be closed once, while the failed outcome still becomes future caution memory.',
+    action: 'run npm test before release',
+    tool: 'npm test',
+    expectAll: ['guard_hardened:replay_rejected', 'decision:caution', 'warning:recent_failure'],
+    forbid: ['decision:go'],
+    steps: [
+      {
+        type: 'guardCycle',
+        saveReceiptAs: 'receipt',
+        action: 'run npm test',
+        tool: 'npm test',
+        outcome: 'failed',
+        errorSummary: 'Vitest failed with spawn EPERM',
+      },
+      {
+        type: 'expectGuardAfterError',
+        receiptRef: 'receipt',
+        label: 'replay_rejected',
+        tool: 'npm test',
+        outcome: 'failed',
+        errorSummary: 'replayed failure should not be recorded',
+        errorIncludes: 'already has an outcome',
+      },
+    ],
+  },
+  {
+    id: 'guard-rejects-non-guard-receipt',
+    suite: 'guard',
+    kind: 'guard',
+    family: 'guard_receipt_hardening',
+    title: 'Guard rejects non-guard receipts',
+    description: 'A normal tool trace must not be accepted as a guard receipt for after-action feedback.',
+    action: 'format docs',
+    tool: 'Bash',
+    expectAll: ['guard_hardened:non_guard_receipt_rejected'],
+    forbid: ['decision:block'],
+    steps: [
+      {
+        type: 'observeTool',
+        saveAs: 'non-guard-receipt',
+        event: 'PreToolUse',
+        tool: 'Bash',
+        metadata: { benchmark: 'non-guard-receipt' },
+      },
+      {
+        type: 'expectGuardAfterError',
+        receiptRef: 'non-guard-receipt',
+        label: 'non_guard_receipt_rejected',
+        tool: 'Bash',
+        outcome: 'succeeded',
+        errorIncludes: 'not a guard receipt',
+      },
+    ],
+  },
+];
+
+export const LOCAL_BENCHMARK_SUITES = [
+  {
+    id: 'retrieval',
+    title: 'Retrieval capabilities',
+    description: 'LongMemEval-style memory abilities plus privacy and abstention.',
+    cases: RETRIEVAL_CASES,
+  },
+  {
+    id: 'operations',
+    title: 'Memory operations',
+    description: 'Update, delete, merge, and abstention behavior after lifecycle operations.',
+    cases: OPERATION_CASES,
+  },
+  {
+    id: 'guard',
+    title: 'Agent guard loop',
+    description: 'Closed-loop memory-before-action behavior for receipts, warnings, and blocking reflexes.',
+    comparableToBaselines: false,
+    cases: GUARD_CASES,
+  },
+];
+
+export const BENCHMARK_CASES = LOCAL_BENCHMARK_SUITES.flatMap(suite => suite.cases);
+
+export const FAMILY_ORDER = [
+  'information_extraction',
+  'knowledge_updates',
+  'multi_session_reasoning',
+  'temporal_reasoning',
+  'abstention',
+  'conflict_resolution',
+  'procedural_learning',
+  'privacy_boundary',
+  'update_overwrite',
+  'delete_and_abstain',
+  'semantic_merge',
+  'procedural_merge',
+  'closed_loop_failure_memory',
+  'strict_must_follow_block',
+];

package/benchmarks/create-conformance-card.mjs

@@ -0,0 +1,139 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { join, resolve } from 'node:path';
+import { computeGuardBenchArtifactHashes, validateGuardBenchArtifacts } from './validate-guardbench-artifacts.mjs';
+import { publicArtifactValue, publicPath } from './public-paths.mjs';
+
+const CARD_FILE = 'guardbench-conformance-card.json';
+const METADATA_FILE = 'external-run-metadata.json';
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+function sha256File(path) {
+  return createHash('sha256').update(readFileSync(path)).digest('hex');
+}
+
+function findExternalSubject(summary, requestedAdapter) {
+  const externalSubjects = (summary.manifest?.subjects ?? []).filter(subject => subject.external);
+  if (requestedAdapter) {
+    const requested = externalSubjects.find(subject => subject.name === requestedAdapter || subject.id === requestedAdapter);
+    if (requested) return requested;
+  }
+  return externalSubjects.length === 1 ? externalSubjects[0] : null;
+}
+
+function findSystemSummary(summary, metadata) {
+  const requested = metadata?.adapterConformance?.adapter ?? metadata?.adapter;
+  if (requested) {
+    const direct = summary.systemSummaries?.find(row => row.system === requested);
+    if (direct) return direct;
+  }
+  const externalSubject = findExternalSubject(summary, requested);
+  if (externalSubject) {
+    return summary.systemSummaries?.find(row => row.system === externalSubject.name) ?? null;
+  }
+  const audreyGuard = summary.systemSummaries?.find(row => row.system === 'Audrey Guard');
+  if (audreyGuard) return audreyGuard;
+  return null;
+}
+
+export function buildGuardBenchConformanceCard(options = {}) {
+  const dir = resolve(options.dir ?? 'benchmarks/output');
+  const summary = readJson(join(dir, 'guardbench-summary.json'));
+  const metadataPath = join(dir, METADATA_FILE);
+  const metadata = existsSync(metadataPath) ? readJson(metadataPath) : null;
+  const validation = validateGuardBenchArtifacts({ dir });
+  const systemSummary = findSystemSummary(summary, metadata);
+  const externalSubject = findExternalSubject(summary, systemSummary?.system ?? metadata?.adapter);
+  const artifactHashes = computeGuardBenchArtifactHashes(dir);
+
+  return {
+    schemaVersion: '1.0.0',
+    suite: 'GuardBench conformance card',
+    generatedAt: new Date().toISOString(),
+    sourceDir: publicPath(dir),
+    manifestVersion: summary.manifest?.manifestVersion ?? null,
+    suiteId: summary.manifest?.suiteId ?? null,
+    subject: {
+      name: systemSummary?.system ?? metadata?.adapterConformance?.adapter ?? metadata?.adapter ?? 'unknown',
+      requestedAdapter: metadata?.adapterConformance?.requestedAdapter ?? metadata?.adapter ?? null,
+      external: Boolean(externalSubject?.external ?? metadata),
+    },
+    run: {
+      status: metadata?.status ?? (validation.ok ? 'validated' : 'invalid'),
+      startedAt: metadata?.startedAt ?? null,
+      completedAt: metadata?.completedAt ?? null,
+      command: publicArtifactValue(metadata?.command ?? null),
+      validationCommand: publicArtifactValue(metadata?.validationCommand ?? null),
+    },
+    score: {
+      scenarios: systemSummary?.scenarios ?? summary.scenarios ?? 0,
+      fullContractPassed: systemSummary?.passed ?? null,
+      fullContractPassRate: systemSummary?.passRate ?? null,
+      decisionAccuracy: systemSummary?.decisionAccuracy ?? null,
+      evidenceRecall: systemSummary?.evidenceRecall ?? null,
+      redactionLeaks: systemSummary?.redactionLeaks ?? null,
+      latency: systemSummary?.latency ?? null,
+    },
+    conformance: {
+      ok: Boolean(metadata?.adapterConformance?.ok ?? validation.ok),
+      failures: metadata?.adapterConformance?.failures ?? validation.failures,
+      artifactValidationOk: validation.ok,
+      artifactValidationFailures: validation.failures,
+    },
+    integrity: {
+      artifactHashes,
+      externalRunMetadataHash: existsSync(metadataPath) ? sha256File(metadataPath) : null,
+    },
+    provenance: summary.provenance,
+  };
+}
+
+export function writeGuardBenchConformanceCard(options = {}) {
+  const dir = resolve(options.dir ?? 'benchmarks/output');
+  const card = buildGuardBenchConformanceCard({ dir });
+  const path = join(dir, CARD_FILE);
+  writeFileSync(path, `${JSON.stringify(card, null, 2)}\n`, 'utf-8');
+  return { path, card };
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = { dir: 'benchmarks/output', json: false };
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if ((token === '--dir' || token === '--out-dir') && argv[i + 1]) args.dir = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+  return args;
+}
+
+function usage() {
+  return [
+    'Usage: node benchmarks/create-conformance-card.mjs [--dir benchmarks/output] [--json]',
+    '',
+    'Writes guardbench-conformance-card.json for a validated GuardBench output bundle.',
+  ].join('\n');
+}
+
+async function main() {
+  const args = parseArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+  const result = writeGuardBenchConformanceCard(args);
+  if (args.json) console.log(JSON.stringify({ path: result.path, card: result.card }, null, 2));
+  else console.log(`GuardBench conformance card: ${result.path}`);
+  if (!result.card.conformance.artifactValidationOk) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]).endsWith('create-conformance-card.mjs')) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}