audrey 0.16.1 → 0.17.0

package/benchmarks/baselines.js

@@ -0,0 +1,169 @@
+import { createEmbeddingProvider } from '../src/embedding.js';
+import { cosineSimilarity } from '../src/utils.js';
+
+function normalize(text) {
+  return String(text || '').toLowerCase();
+}
+
+function tokenize(text) {
+  return normalize(text)
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+function keywordScore(queryTokens, content) {
+  const contentTokens = new Set(tokenize(content));
+  if (queryTokens.length === 0) return 0;
+  let matches = 0;
+  for (const token of queryTokens) {
+    if (contentTokens.has(token)) matches++;
+  }
+  return matches / queryTokens.length;
+}
+
+function sortByScore(rows) {
+  return rows
+    .filter(row => Number.isFinite(row.score))
+    .sort((a, b) => b.score - a.score || String(b.createdAt || '').localeCompare(String(a.createdAt || '')));
+}
+
+function flattenMemories(benchmarkCase, ids = []) {
+  return benchmarkCase.memory.map((memory, index) => ({
+    id: ids[index] || `memory-${index + 1}`,
+    content: memory.content,
+    source: memory.source,
+    createdAt: memory.createdAt || new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+    private: Boolean(memory.private),
+  }));
+}
+
+function buildSyntheticCase(query, memories, options = {}) {
+  return {
+    query,
+    memory: memories.map(memory => ({
+      content: memory.content,
+      source: memory.source,
+      createdAt: memory.createdAt,
+      private: memory.private,
+    })),
+    options,
+  };
+}
+
+async function runBaselineRetrieval(system, syntheticCase, providerConfig, limit = 5) {
+  switch (system) {
+    case 'Vector Only':
+      return runVectorOnlyBaseline(syntheticCase, providerConfig, limit);
+    case 'Keyword + Recency':
+      return runKeywordRecencyBaseline(syntheticCase, limit);
+    case 'Recent Window':
+      return runRecentWindowBaseline(syntheticCase, limit);
+    default:
+      throw new Error(`Unknown baseline system: ${system}`);
+  }
+}
+
+function createOperationMemory(state, step) {
+  const index = state.counter++;
+  return {
+    id: `memory-${index + 1}`,
+    content: step.memory.content,
+    source: step.memory.source,
+    createdAt: step.memory.createdAt || new Date(Date.UTC(2026, 0, index + 1)).toISOString(),
+    private: Boolean(step.memory.private),
+  };
+}
+
+async function applyBaselineStep(system, state, step, providerConfig) {
+  if (step.type === 'encode') {
+    const memory = createOperationMemory(state, step);
+    state.memories.push(memory);
+    if (step.saveAs) {
+      state.aliases.set(step.saveAs, memory.id);
+    }
+    return;
+  }
+
+  if (step.type === 'forgetByQuery') {
+    const syntheticCase = buildSyntheticCase(step.query, state.memories, step.options);
+    const [match] = await runBaselineRetrieval(system, syntheticCase, providerConfig, 1);
+    if (match && Number.isFinite(match.score) && match.score > 0) {
+      state.memories = state.memories.filter(memory => memory.id !== match.id);
+    }
+    return;
+  }
+
+  if (step.type === 'consolidate') {
+    return;
+  }
+
+  throw new Error(`Unsupported baseline step: ${step.type}`);
+}
+
+export async function runBaselineScenario(system, benchmarkCase, providerConfig, limit = 5) {
+  if (benchmarkCase.kind !== 'operations') {
+    return runBaselineRetrieval(system, benchmarkCase, providerConfig, limit);
+  }
+
+  const state = {
+    counter: 0,
+    memories: [],
+    aliases: new Map(),
+  };
+
+  for (const step of benchmarkCase.steps || []) {
+    await applyBaselineStep(system, state, step, providerConfig);
+  }
+
+  return runBaselineRetrieval(
+    system,
+    buildSyntheticCase(benchmarkCase.query, state.memories, benchmarkCase.options),
+    providerConfig,
+    limit,
+  );
+}
+
+export function runKeywordRecencyBaseline(benchmarkCase, limit = 5) {
+  const queryTokens = tokenize(benchmarkCase.query);
+  return sortByScore(flattenMemories(benchmarkCase).map(memory => ({
+    ...memory,
+    type: 'episodic',
+    score: keywordScore(queryTokens, memory.content),
+  }))).slice(0, limit);
+}
+
+export function runRecentWindowBaseline(benchmarkCase, limit = 3) {
+  return flattenMemories(benchmarkCase)
+    .sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
+    .slice(0, limit)
+    .map((memory, index) => ({
+      ...memory,
+      type: 'episodic',
+      score: 1 - index * 0.1,
+    }));
+}
+
+export async function runVectorOnlyBaseline(benchmarkCase, providerConfig, limit = 5) {
+  const provider = createEmbeddingProvider(providerConfig);
+  if (typeof provider.ready === 'function') {
+    await provider.ready();
+  }
+
+  const queryVector = await provider.embed(benchmarkCase.query);
+  const queryBuffer = provider.vectorToBuffer(queryVector);
+
+  const rows = [];
+  for (const memory of flattenMemories(benchmarkCase)) {
+    const vector = await provider.embed(memory.content);
+    const score = cosineSimilarity(queryBuffer, provider.vectorToBuffer(vector), provider);
+    rows.push({
+      ...memory,
+      type: 'episodic',
+      score,
+    });
+  }
+
+  return sortByScore(rows).slice(0, limit);
+}

package/benchmarks/cases.js

@@ -0,0 +1,421 @@
+export const RETRIEVAL_CASES = [
+  {
+    id: 'information-extraction',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'information_extraction',
+    title: 'Information extraction',
+    description: 'Recover a directly stated user fact from durable memory.',
+    query: 'Where does Sam live now?',
+    expectAny: ['Austin'],
+    memory: [
+      {
+        content: 'Sam moved to Austin in March 2026 after leaving Denver.',
+        source: 'direct-observation',
+        tags: ['profile', 'location'],
+        context: { subject: 'sam', domain: 'assistant' },
+      },
+      {
+        content: 'Sam likes to work from coffee shops on South Congress.',
+        source: 'tool-result',
+        tags: ['preference', 'routine'],
+        context: { subject: 'sam', domain: 'assistant' },
+      },
+    ],
+  },
+  {
+    id: 'knowledge-update',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'knowledge_updates',
+    title: 'Knowledge updates',
+    description: 'Prefer the newer fact over stale preferences.',
+    query: 'What drink does Sam prefer now?',
+    expectAny: ['green tea'],
+    forbid: ['Sam prefers coffee before early meetings.'],
+    memory: [
+      {
+        content: 'Sam prefers coffee before early meetings.',
+        source: 'told-by-user',
+        tags: ['preference'],
+        context: { subject: 'sam', domain: 'assistant' },
+      },
+      {
+        content: 'Sam switched from coffee to green tea after January 2026.',
+        source: 'direct-observation',
+        tags: ['preference', 'update'],
+        context: { subject: 'sam', domain: 'assistant' },
+        supersedesIndex: 0,
+      },
+    ],
+  },
+  {
+    id: 'multi-session-reasoning',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'multi_session_reasoning',
+    title: 'Multi-session reasoning',
+    description: 'Synthesize a decision from multiple related episodes.',
+    query: 'Which vendor was approved after the pilot budget review?',
+    expectAny: ['Northwind'],
+    memory: [
+      {
+        content: 'During the January pilot, Sam requested budget approval for vendors Northwind and Fabricam.',
+        source: 'tool-result',
+        tags: ['project', 'pilot'],
+        context: { subject: 'sam', domain: 'operations' },
+      },
+      {
+        content: 'Finance rejected Fabricam because the support SLA was too weak.',
+        source: 'direct-observation',
+        tags: ['finance', 'vendor'],
+        context: { subject: 'sam', domain: 'operations' },
+      },
+      {
+        content: 'The pilot budget review approved Northwind for rollout after the support SLA review.',
+        source: 'direct-observation',
+        tags: ['finance', 'vendor', 'approval'],
+        context: { subject: 'sam', domain: 'operations' },
+      },
+    ],
+  },
+  {
+    id: 'temporal-reasoning',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'temporal_reasoning',
+    title: 'Temporal reasoning',
+    description: 'Answer by isolating the right time window.',
+    query: 'What happened in February 2026?',
+    expectAny: ['architecture review'],
+    memory: [
+      {
+        content: 'In January 2026 Sam kicked off the migration plan.',
+        source: 'tool-result',
+        tags: ['timeline'],
+        createdAt: '2026-01-12T09:00:00.000Z',
+      },
+      {
+        content: 'In February 2026 Sam completed the architecture review.',
+        source: 'direct-observation',
+        tags: ['timeline'],
+        createdAt: '2026-02-18T15:30:00.000Z',
+      },
+      {
+        content: 'In March 2026 Sam started the rollout checklist.',
+        source: 'tool-result',
+        tags: ['timeline'],
+        createdAt: '2026-03-02T08:15:00.000Z',
+      },
+    ],
+    options: {
+      after: '2026-02-01T00:00:00.000Z',
+      before: '2026-03-01T00:00:00.000Z',
+    },
+  },
+  {
+    id: 'abstention',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'abstention',
+    title: 'Abstention',
+    description: 'Avoid pretending to know a specific identifier that was never stored.',
+    query: 'What is Sam passport number?',
+    expectNone: true,
+    memory: [
+      {
+        content: 'Sam renewed a passport in February 2026.',
+        source: 'tool-result',
+        tags: ['travel'],
+      },
+      {
+        content: 'Sam has a trip to Toronto next month.',
+        source: 'told-by-user',
+        tags: ['travel'],
+      },
+    ],
+  },
+  {
+    id: 'conflict-resolution',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'conflict_resolution',
+    title: 'Conflict resolution',
+    description: 'Prefer high-reliability evidence over model-generated noise.',
+    query: 'What caused the outage?',
+    expectAny: ['TLS certificate', 'expired certificate'],
+    forbid: ['The outage was caused by database corruption.'],
+    memory: [
+      {
+        content: 'The outage was caused by an expired TLS certificate on api.example.com.',
+        source: 'direct-observation',
+        tags: ['incident', 'root-cause'],
+      },
+      {
+        content: 'The outage was caused by database corruption.',
+        source: 'model-generated',
+        tags: ['incident', 'root-cause'],
+      },
+    ],
+  },
+  {
+    id: 'procedural-learning',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'procedural_learning',
+    title: 'Procedural learning',
+    description: 'Turn repeated incidents into an actionable operating rule.',
+    query: 'What should the agent do when payout retries start returning 429?',
+    expectAny: ['cap retry batches', 'stagger retries'],
+    memory: [
+      {
+        content: 'Processor X returned HTTP 429 when payout retries exceeded 120 requests per minute.',
+        source: 'direct-observation',
+        tags: ['payments', 'rate-limit'],
+      },
+      {
+        content: 'Payout incident volume dropped after retry batches were capped at 50 merchants per worker.',
+        source: 'tool-result',
+        tags: ['payments', 'rate-limit'],
+      },
+      {
+        content: 'Risk operations requested an escalation when multiple merchants were affected in the same hour.',
+        source: 'told-by-user',
+        tags: ['payments', 'escalation'],
+      },
+    ],
+    consolidate: {
+      minClusterSize: 3,
+      similarityThreshold: -0.3,
+      principle: {
+        content: 'When payout retries start returning 429, cap retry batches and stagger retries before escalating.',
+        type: 'procedural',
+        conditions: ['processor returns 429', 'multiple merchants impacted'],
+      },
+    },
+    options: {
+      types: ['procedural', 'semantic'],
+    },
+  },
+  {
+    id: 'privacy-boundary',
+    suite: 'retrieval',
+    kind: 'retrieval',
+    family: 'privacy_boundary',
+    title: 'Privacy boundary',
+    description: 'Never leak private memory into public recall.',
+    query: 'What is Sam API key?',
+    expectNone: true,
+    forbid: ['sk-secret-demo-key'],
+    memory: [
+      {
+        content: 'Sam API key is sk-secret-demo-key.',
+        source: 'told-by-user',
+        private: true,
+        tags: ['secret'],
+      },
+      {
+        content: 'Sam prefers dark mode in internal dashboards.',
+        source: 'told-by-user',
+        tags: ['preference'],
+      },
+    ],
+  },
+];
+
+export const OPERATION_CASES = [
+  {
+    id: 'operation-update-overwrite',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'update_overwrite',
+    title: 'Update and overwrite',
+    description: 'Current-state recall should prefer the new fact after an explicit overwrite.',
+    query: 'What is the primary deployment region now?',
+    expectAny: ['eu-west-1'],
+    forbid: ['us-east-1'],
+    steps: [
+      {
+        type: 'encode',
+        saveAs: 'initial-region',
+        memory: {
+          content: 'The primary deployment region is us-east-1.',
+          source: 'told-by-user',
+          tags: ['deployment', 'region'],
+        },
+      },
+      {
+        type: 'encode',
+        supersedesRef: 'initial-region',
+        memory: {
+          content: 'As of March 2026, the primary deployment region is eu-west-1.',
+          source: 'direct-observation',
+          tags: ['deployment', 'region', 'update'],
+        },
+      },
+    ],
+  },
+  {
+    id: 'operation-delete-and-abstain',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'delete_and_abstain',
+    title: 'Delete and abstain',
+    description: 'Explicit deletion should remove a secret from later recall.',
+    query: 'What is the staging API token?',
+    expectNone: true,
+    forbid: ['tok-demo-staging-1234'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'The staging API token is tok-demo-staging-1234.',
+          source: 'told-by-user',
+          tags: ['secret', 'staging'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'The staging environment rotates API credentials weekly.',
+          source: 'tool-result',
+          tags: ['staging', 'ops'],
+        },
+      },
+      {
+        type: 'forgetByQuery',
+        query: 'staging API token',
+        options: { minSimilarity: 0.35 },
+      },
+    ],
+  },
+  {
+    id: 'operation-semantic-merge',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'semantic_merge',
+    title: 'Semantic merge',
+    description: 'Related episodes should merge into a reusable semantic operating rule.',
+    query: 'When should the disputes queue trigger manual review?',
+    expectAny: ['manual review', 'same bin in one hour'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'Three charge disputes from the same BIN landed in the queue within one hour.',
+          source: 'direct-observation',
+          tags: ['fraud', 'disputes'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'Fraud ops escalated repeated same-BIN disputes for analyst attention.',
+          source: 'tool-result',
+          tags: ['fraud', 'disputes'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'The queue stabilized after repeated same-BIN disputes were reviewed manually.',
+          source: 'told-by-user',
+          tags: ['fraud', 'disputes'],
+        },
+      },
+      {
+        type: 'consolidate',
+        minClusterSize: 3,
+        similarityThreshold: -0.3,
+        principle: {
+          content: 'Repeated disputes from the same BIN in one hour should trigger manual review.',
+          type: 'semantic',
+        },
+      },
+    ],
+    options: {
+      types: ['semantic'],
+    },
+  },
+  {
+    id: 'operation-procedural-merge',
+    suite: 'operations',
+    kind: 'operations',
+    family: 'procedural_merge',
+    title: 'Procedural merge',
+    description: 'Related episodes should merge into an executable procedure, not just a loose fact.',
+    query: 'What should the agent do after two webhook signature failures?',
+    expectAny: ['rotate the signing secret', 'replay queued events'],
+    steps: [
+      {
+        type: 'encode',
+        memory: {
+          content: 'Webhook signature verification failed twice for merchant ACME.',
+          source: 'direct-observation',
+          tags: ['webhooks', 'security'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'Operations recovered the incident by rotating the signing secret.',
+          source: 'tool-result',
+          tags: ['webhooks', 'security'],
+        },
+      },
+      {
+        type: 'encode',
+        memory: {
+          content: 'Queued webhook events were replayed after the signing secret changed.',
+          source: 'told-by-user',
+          tags: ['webhooks', 'security'],
+        },
+      },
+      {
+        type: 'consolidate',
+        minClusterSize: 3,
+        similarityThreshold: -0.3,
+        principle: {
+          content: 'When webhook signature verification fails twice, rotate the signing secret and replay queued events.',
+          type: 'procedural',
+          conditions: ['signature verification fails twice', 'queued events pending'],
+        },
+      },
+    ],
+    options: {
+      types: ['procedural', 'semantic'],
+    },
+  },
+];
+
+export const LOCAL_BENCHMARK_SUITES = [
+  {
+    id: 'retrieval',
+    title: 'Retrieval capabilities',
+    description: 'LongMemEval-style memory abilities plus privacy and abstention.',
+    cases: RETRIEVAL_CASES,
+  },
+  {
+    id: 'operations',
+    title: 'Memory operations',
+    description: 'Update, delete, merge, and abstention behavior after lifecycle operations.',
+    cases: OPERATION_CASES,
+  },
+];
+
+export const BENCHMARK_CASES = LOCAL_BENCHMARK_SUITES.flatMap(suite => suite.cases);
+
+export const FAMILY_ORDER = [
+  'information_extraction',
+  'knowledge_updates',
+  'multi_session_reasoning',
+  'temporal_reasoning',
+  'abstention',
+  'conflict_resolution',
+  'procedural_learning',
+  'privacy_boundary',
+  'update_overwrite',
+  'delete_and_abstain',
+  'semantic_merge',
+  'procedural_merge',
+];

package/benchmarks/reference-results.js

@@ -0,0 +1,70 @@
+export const PUBLISHED_LEADERBOARD = [
+  {
+    system: 'MIRIX',
+    benchmark: 'LoCoMo',
+    score: 85.4,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2507.07957',
+    note: 'Published LoCoMo result from the MIRIX paper.',
+  },
+  {
+    system: 'Letta Filesystem',
+    benchmark: 'LoCoMo',
+    score: 74.0,
+    unit: 'accuracy',
+    source: 'https://www.letta.com/blog/benchmarking-ai-agent-memory',
+    note: 'Filesystem-style memory result reported by Letta.',
+  },
+  {
+    system: 'Mem0 Graph Memory',
+    benchmark: 'LoCoMo',
+    score: 68.5,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2504.19413',
+    note: 'Graph memory variant reported in the Mem0 paper.',
+  },
+  {
+    system: 'Mem0',
+    benchmark: 'LoCoMo',
+    score: 66.9,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2504.19413',
+    note: 'Core Mem0 LoCoMo score reported in the Mem0 paper.',
+  },
+  {
+    system: 'OpenAI Memory',
+    benchmark: 'LoCoMo',
+    score: 52.9,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2504.19413',
+    note: 'OpenAI memory baseline as reported by the Mem0 paper.',
+  },
+];
+
+export const MEMORY_TRENDS = [
+  {
+    title: 'Memory is moving from flat retrieval to typed systems',
+    summary: 'Recent work treats episodic, semantic, procedural, and graph memory as separate but cooperating layers.',
+    source: 'https://arxiv.org/abs/2507.03724',
+  },
+  {
+    title: 'Benchmarks now emphasize multi-session realism',
+    summary: 'LongMemEval and LoCoMo push memory systems toward temporal updates, abstraction, and cross-session reasoning instead of single-turn fact recall.',
+    source: 'https://arxiv.org/abs/2410.10813',
+  },
+  {
+    title: 'Context engineering is now competing with retrieval-first designs',
+    summary: 'Letta argues filesystem and memory-block approaches can outperform simpler retrieval-only memory on realistic long-horizon tasks.',
+    source: 'https://www.letta.com/blog/memory-blocks',
+  },
+  {
+    title: 'Production teams care about latency and token footprint, not just recall quality',
+    summary: 'Mem0 frames memory as a cost and latency optimization surface in addition to a personalization surface.',
+    source: 'https://arxiv.org/abs/2504.19413',
+  },
+  {
+    title: 'Temporal and multimodal memory are becoming table stakes',
+    summary: 'MIRIX and Graphiti both model time and state change explicitly instead of assuming memories stay forever true.',
+    source: 'https://arxiv.org/abs/2507.07957',
+  },
+];