audrey 0.23.1 → 1.0.1

package/benchmarks/perf-snapshot.js

@@ -0,0 +1,304 @@
+// Honest performance snapshot for Audrey.
+//
+// What this measures
+// - Encode latency at multiple corpus sizes
+// - Hybrid recall latency at multiple corpus sizes
+// - Post-encode queue processing time (consolidation/interference/validation pipeline)
+//
+// What this does NOT do
+// - Compare against other memory systems. Synthetic head-to-head numbers are
+//   easy to game. If you want a real comparison, run the same workload against
+//   the system you care about and post your own results.
+// - Use cloud embedding providers. Latency to a remote API is dominated by
+//   network round-trip and varies wildly by region and rate-limit state.
+//   We use the in-process mock embedding provider so the numbers reflect
+//   Audrey's own pipeline (SQLite, sqlite-vec, encode/recall logic, hybrid
+//   ranking) without third-party noise. Real-world recall p95 with a local
+//   384-dim provider is typically 5-15x higher; with a hosted provider it is
+//   dominated by the API call.
+//
+// How to read the output
+// - p50 / p95 / p99 are percentile latencies in milliseconds.
+// - The numbers are wall-clock for a single call from a JS caller, including
+//   SQLite work and any post-encode queueing on encode rows.
+// - Run on your own hardware and embedding provider before quoting numbers
+//   anywhere; results scale heavily with CPU, NVMe vs spinning disk, and
+//   embedding dimensionality.
+
+import { performance } from 'node:perf_hooks';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir, cpus, totalmem, arch, platform, release } from 'node:os';
+import { execFileSync } from 'node:child_process';
+import { pathToFileURL } from 'node:url';
+import { Audrey } from '../dist/src/index.js';
+
+const DEFAULT_SIZES = [100, 1000, 5000];
+const DEFAULT_RECALL_RUNS = 50;
+const QUERY_POOL = [
+  'rate limit handling for HTTP 429 retries',
+  'durable agent context across sessions',
+  'safe shell behavior on Windows hosts',
+  'official authority over inferred preferences',
+  'lexical signal for exact identifiers',
+  'deployment region migration steps',
+  'webhook signature recovery procedure',
+  'fraud queue manual review trigger',
+];
+
+const SEED_POOL = [
+  'Stripe API returned HTTP 429 during checkout retry; needs exponential backoff.',
+  'Project memory routing should prefer the local memory layer for durable context.',
+  'Tool trace learning marks repeated spawn EPERM failures as risky on Windows shells.',
+  'Calendar authority should come from the official source before inferred user notes.',
+  'Vector recall is faster but loses BM25 lexical signal on exact identifiers.',
+  'Webhook signature recovery requires rotating the signing secret and replaying queued events.',
+  'Fraud queue stabilizes when repeated same-BIN disputes are escalated for manual review.',
+  'Deployment region migrations should be coordinated against the provider rate-limit window.',
+];
+
+function percentile(values, rank) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.min(sorted.length - 1, Math.ceil((rank / 100) * sorted.length) - 1);
+  return sorted[index];
+}
+
+function round(value) {
+  return Math.round(value * 1000) / 1000;
+}
+
+function summarize(values) {
+  if (values.length === 0) {
+    return { samples: 0, p50: 0, p95: 0, p99: 0, min: 0, max: 0, mean: 0 };
+  }
+  const total = values.reduce((acc, v) => acc + v, 0);
+  return {
+    samples: values.length,
+    p50: round(percentile(values, 50)),
+    p95: round(percentile(values, 95)),
+    p99: round(percentile(values, 99)),
+    min: round(Math.min(...values)),
+    max: round(Math.max(...values)),
+    mean: round(total / values.length),
+  };
+}
+
+function gitSha() {
+  // execFileSync, no shell, fixed argv — provenance only, no user input flows in.
+  try {
+    return execFileSync('git', ['rev-parse', '--short', 'HEAD'], {
+      stdio: ['ignore', 'pipe', 'ignore'],
+    })
+      .toString()
+      .trim();
+  } catch {
+    return null;
+  }
+}
+
+export function resolveAudreyVersion() {
+  if (process.env.npm_package_version) {
+    return process.env.npm_package_version;
+  }
+
+  try {
+    const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
+    return typeof pkg.version === 'string' ? pkg.version : null;
+  } catch {
+    return null;
+  }
+}
+
+function machineProvenance() {
+  const cpuList = cpus();
+  const cpuModel = cpuList[0] ? cpuList[0].model : 'unknown';
+  const totalGb = Math.round((totalmem() / 1024 / 1024 / 1024) * 10) / 10;
+  return {
+    node: process.versions.node,
+    v8: process.versions.v8,
+    platform: platform(),
+    arch: arch(),
+    osRelease: release(),
+    cpuCount: cpuList.length,
+    cpuModel,
+    memoryGb: totalGb,
+  };
+}
+
+function createDataDir() {
+  const parents = [
+    process.env.AUDREY_PERF_PARENT_DIR,
+    tmpdir(),
+    join(process.cwd(), 'benchmarks', '.tmp'),
+  ].filter(Boolean);
+
+  let lastError;
+  for (const parent of parents) {
+    try {
+      mkdirSync(parent, { recursive: true });
+      return mkdtempSync(join(parent, 'audrey-perf-snapshot-'));
+    } catch (err) {
+      lastError = err;
+    }
+  }
+  throw lastError || new Error('Unable to create perf snapshot data directory');
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    sizes: [...DEFAULT_SIZES],
+    recallRuns: DEFAULT_RECALL_RUNS,
+    out: null,
+    json: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (token === '--sizes' && argv[i + 1]) {
+      args.sizes = argv[++i]
+        .split(',')
+        .map((s) => Number.parseInt(s.trim(), 10))
+        .filter((n) => Number.isFinite(n) && n > 0);
+    } else if (token === '--recall-runs' && argv[i + 1]) {
+      args.recallRuns = Number.parseInt(argv[++i], 10);
+    } else if (token === '--out' && argv[i + 1]) {
+      args.out = resolve(argv[++i]);
+    } else if (token === '--json') {
+      args.json = true;
+    }
+  }
+  return args;
+}
+
+async function runOneSize({ size, recallRuns }) {
+  const dataDir = createDataDir();
+  const audrey = new Audrey({
+    dataDir,
+    agent: 'perf-snapshot',
+    embedding: { provider: 'mock', dimensions: 64 },
+    llm: { provider: 'mock' },
+  });
+
+  const queueProcessingTimes = [];
+  audrey.on('post-encode-complete', (event) => {
+    queueProcessingTimes.push(event.processing_ms);
+  });
+
+  try {
+    const encodeTimes = [];
+    for (let i = 0; i < size; i++) {
+      const content = `${SEED_POOL[i % SEED_POOL.length]} (sample ${i})`;
+      const startedAt = performance.now();
+      await audrey.encode({
+        content,
+        source: 'direct-observation',
+        tags: ['perf-snapshot'],
+      });
+      encodeTimes.push(performance.now() - startedAt);
+    }
+
+    const drain = await audrey.drainPostEncodeQueue(60_000);
+    if (!drain.drained) {
+      throw new Error(`post-encode queue did not drain at size=${size}`);
+    }
+
+    const recallTimes = [];
+    for (let i = 0; i < recallRuns; i++) {
+      const query = QUERY_POOL[i % QUERY_POOL.length];
+      const startedAt = performance.now();
+      await audrey.recall(query, { limit: 5, retrieval: 'hybrid' });
+      recallTimes.push(performance.now() - startedAt);
+    }
+
+    return {
+      corpusSize: size,
+      encodeMs: summarize(encodeTimes),
+      hybridRecallMs: summarize(recallTimes),
+      postEncodeQueueMs: summarize(queueProcessingTimes),
+      queueEvents: queueProcessingTimes.length,
+    };
+  } finally {
+    audrey.close();
+    rmSync(dataDir, { recursive: true, force: true });
+  }
+}
+
+export async function runPerfSnapshot({ sizes = DEFAULT_SIZES, recallRuns = DEFAULT_RECALL_RUNS } = {}) {
+  const startedAt = Date.now();
+  const sized = [];
+  for (const size of sizes) {
+    sized.push(await runOneSize({ size, recallRuns }));
+  }
+  return {
+    generatedAt: new Date(startedAt).toISOString(),
+    durationMs: Date.now() - startedAt,
+    audreyVersion: resolveAudreyVersion(),
+    gitSha: gitSha(),
+    methodology: {
+      embedding: 'mock provider, 64 dimensions (in-process, no network)',
+      llm: 'mock provider (in-process)',
+      retrieval: 'hybrid (vector + lexical) with limit=5',
+      sizes,
+      recallRunsPerSize: recallRuns,
+      notes:
+        'Latency is wall-clock for a single call from a JS caller. Cloud and ' +
+        'local 384-dim providers will report higher recall latency dominated by ' +
+        'embedding cost and network. Run on your own hardware before quoting.',
+    },
+    machine: machineProvenance(),
+    sizes: sized,
+  };
+}
+
+function formatMs(value) {
+  if (value === 0) return '0';
+  if (value < 1) return value.toFixed(2);
+  return value.toFixed(1);
+}
+
+export function formatMarkdownTable(snapshot) {
+  const lines = [];
+  lines.push(
+    `Audrey perf snapshot — ${snapshot.audreyVersion || 'dev'} on ${snapshot.machine.platform}/${snapshot.machine.arch}`,
+  );
+  lines.push('');
+  lines.push(
+    `Node ${snapshot.machine.node} · ${snapshot.machine.cpuCount}x ${snapshot.machine.cpuModel} · ${snapshot.machine.memoryGb} GB RAM`,
+  );
+  lines.push(
+    `Generated ${snapshot.generatedAt}${snapshot.gitSha ? ` (${snapshot.gitSha})` : ''}`,
+  );
+  lines.push('');
+  lines.push('| Corpus size | Encode p50 (ms) | Encode p95 (ms) | Recall p50 (ms) | Recall p95 (ms) | Recall p99 (ms) |');
+  lines.push('|---|---|---|---|---|---|');
+  for (const row of snapshot.sizes) {
+    lines.push(
+      `| ${row.corpusSize.toLocaleString()} ` +
+        `| ${formatMs(row.encodeMs.p50)} ` +
+        `| ${formatMs(row.encodeMs.p95)} ` +
+        `| ${formatMs(row.hybridRecallMs.p50)} ` +
+        `| ${formatMs(row.hybridRecallMs.p95)} ` +
+        `| ${formatMs(row.hybridRecallMs.p99)} |`,
+    );
+  }
+  return lines.join('\n');
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const args = parseArgs();
+  runPerfSnapshot({ sizes: args.sizes, recallRuns: args.recallRuns })
+    .then((snapshot) => {
+      if (args.out) {
+        writeFileSync(args.out, JSON.stringify(snapshot, null, 2) + '\n');
+      }
+      if (args.json) {
+        process.stdout.write(JSON.stringify(snapshot, null, 2) + '\n');
+      } else {
+        process.stdout.write(formatMarkdownTable(snapshot) + '\n');
+      }
+    })
+    .catch((err) => {
+      console.error('[audrey] perf snapshot failed:', err);
+      process.exit(1);
+    });
+}

package/benchmarks/perf.bench.js

@@ -0,0 +1,161 @@
+import { performance } from 'node:perf_hooks';
+import { mkdirSync, mkdtempSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pathToFileURL } from 'node:url';
+import { Audrey } from '../dist/src/index.js';
+
+const RUNS = 20;
+
+// Budget source: CHANGELOG.md#0220---2026-04-28, from the Audrey/MemoryGym
+// latency pass. This mock-provider gate catches mechanical regressions in
+// Audrey CI before live GPU benchmarks or MemoryGym release gates find them.
+export const PERF_BUDGETS = Object.freeze({
+  encodeResponseP95Ms: 50,
+  hybridRecallP95Ms: 25,
+  queueProcessingP50Ms: 5,
+});
+
+function roundMs(value) {
+  return Math.round(value * 1000) / 1000;
+}
+
+function percentile(values, percentileRank) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.min(sorted.length - 1, Math.ceil((percentileRank / 100) * sorted.length) - 1);
+  return sorted[index];
+}
+
+function stats(values) {
+  if (values.length === 0) {
+    return { p50: 0, p95: 0, min: 0, max: 0 };
+  }
+  return {
+    p50: roundMs(percentile(values, 50)),
+    p95: roundMs(percentile(values, 95)),
+    min: roundMs(Math.min(...values)),
+    max: roundMs(Math.max(...values)),
+  };
+}
+
+function assertBudget(name, actual, budget) {
+  if (actual >= budget) {
+    throw new Error(`${name} ${actual}ms exceeded budget ${budget}ms`);
+  }
+}
+
+function seedContent(index) {
+  const cases = [
+    'Stripe API returned HTTP 429 during checkout retry and needs exponential backoff.',
+    'Project memory routing should prefer Audrey MCP for durable agent context.',
+    'Tool trace learning marks repeated npm spawn EPERM failures as risky on Windows shells.',
+    'Calendar authority should come from the official source before inferred user notes.',
+    'Vector recall is faster but loses BM25 lexical signal on exact identifiers.',
+  ];
+  return `${cases[index % cases.length]} Perf sample ${index}.`;
+}
+
+function createPerfDataDir() {
+  const parents = [
+    process.env.AUDREY_PERF_PARENT_DIR,
+    tmpdir(),
+    join(process.cwd(), 'benchmarks', '.tmp'),
+  ].filter(Boolean);
+  let lastError;
+
+  for (const parent of parents) {
+    try {
+      mkdirSync(parent, { recursive: true });
+      return mkdtempSync(join(parent, 'audrey-perf-'));
+    } catch (err) {
+      lastError = err;
+    }
+  }
+
+  throw lastError || new Error('Unable to create Audrey perf benchmark data directory');
+}
+
+export async function runPerfBenchmark({
+  runs = RUNS,
+  budgets = PERF_BUDGETS,
+  out = console.log,
+} = {}) {
+  const dataDir = createPerfDataDir();
+  const audrey = new Audrey({
+    dataDir,
+    agent: 'perf-bench',
+    embedding: { provider: 'mock', dimensions: 64 },
+    llm: { provider: 'mock' },
+  });
+
+  const queueProcessingTimes = [];
+  audrey.on('post-encode-complete', event => {
+    queueProcessingTimes.push(event.processing_ms);
+  });
+
+  try {
+    const encodeTimes = [];
+    for (let i = 0; i < runs; i += 1) {
+      const startedAt = performance.now();
+      await audrey.encode({
+        content: seedContent(i),
+        source: 'direct-observation',
+        tags: ['perf-gate'],
+        affect: { valence: i % 2 === 0 ? 0.3 : -0.1, arousal: 0.2 },
+      });
+      encodeTimes.push(performance.now() - startedAt);
+    }
+
+    const drain = await audrey.drainPostEncodeQueue(5000);
+    if (!drain.drained) {
+      throw new Error(`post-encode queue did not drain: ${drain.pendingIds.join(', ')}`);
+    }
+
+    const recallTimes = [];
+    for (let i = 0; i < runs; i += 1) {
+      const startedAt = performance.now();
+      await audrey.recall('Stripe API 429 retry memory routing', {
+        limit: 5,
+        retrieval: 'hybrid',
+      });
+      recallTimes.push(performance.now() - startedAt);
+    }
+
+    const result = {
+      runs,
+      budgets,
+      encode_response_ms: stats(encodeTimes),
+      hybrid_recall_ms: stats(recallTimes),
+      queue_processing_ms: stats(queueProcessingTimes),
+      queue_events: queueProcessingTimes.length,
+      status: {
+        pending_consolidation_count: audrey.memoryStatus().pending_consolidation_count,
+        default_retrieval_mode: audrey.memoryStatus().default_retrieval_mode,
+      },
+    };
+
+    if (queueProcessingTimes.length !== runs) {
+      throw new Error(`expected ${runs} post-encode queue events, got ${queueProcessingTimes.length}`);
+    }
+
+    assertBudget('encode response p95', result.encode_response_ms.p95, budgets.encodeResponseP95Ms);
+    assertBudget('hybrid recall p95', result.hybrid_recall_ms.p95, budgets.hybridRecallP95Ms);
+    assertBudget('queue processing p50', result.queue_processing_ms.p50, budgets.queueProcessingP50Ms);
+
+    out(`Audrey perf gate passed: encode p95=${result.encode_response_ms.p95}ms, `
+      + `hybrid recall p95=${result.hybrid_recall_ms.p95}ms, `
+      + `queue p50=${result.queue_processing_ms.p50}ms`);
+    return result;
+  } finally {
+    audrey.close();
+    rmSync(dataDir, { recursive: true, force: true });
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  runPerfBenchmark().catch(err => {
+    console.error('[audrey] perf gate failed:', err);
+    process.exit(1);
+  });
+}

package/benchmarks/public-paths.mjs

@@ -0,0 +1,78 @@
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const WINDOWS_DRIVE_PATTERN = /(^|[^a-z])[A-Z]:[\\/]/i;
+const EXTENDED_PATH_PATTERN = /\\\\\?\\/;
+const FILE_URL_PATTERN = /file:\/\//i;
+
+function isUrl(value) {
+  return /^[a-z][a-z0-9+.-]*:\/\//i.test(value);
+}
+
+export function publicPath(value) {
+  if (typeof value !== 'string') return value;
+  if (value === process.execPath) return 'node';
+  if (isUrl(value)) return value;
+
+  const resolved = resolve(value);
+  const rel = relative(ROOT, resolved);
+  if (rel && !rel.startsWith('..') && !isAbsolute(rel)) {
+    return rel.replaceAll('\\', '/');
+  }
+  if (value.includes('\\') || value.includes('/') || isAbsolute(value)) {
+    return `[LOCAL-PATH:${basename(value) || 'path'}]`;
+  }
+  return value;
+}
+
+export function publicCommand(command = []) {
+  return command.map(part => publicPath(part));
+}
+
+export function publicArtifactValue(value) {
+  if (Array.isArray(value)) return value.map(item => publicArtifactValue(item));
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, publicArtifactValue(item)]));
+  }
+  return publicPath(value);
+}
+
+export function containsLocalPath(text) {
+  return WINDOWS_DRIVE_PATTERN.test(text)
+    || EXTENDED_PATH_PATTERN.test(text)
+    || FILE_URL_PATTERN.test(text);
+}
+
+export function findLocalPathLeaks(value, path = '$') {
+  if (typeof value === 'string') {
+    return containsLocalPath(value) ? [`${path}: ${value}`] : [];
+  }
+  if (Array.isArray(value)) {
+    return value.flatMap((item, index) => findLocalPathLeaks(item, `${path}[${index}]`));
+  }
+  if (value && typeof value === 'object') {
+    return Object.entries(value).flatMap(([key, item]) => findLocalPathLeaks(item, `${path}.${key}`));
+  }
+  return [];
+}
+
+export function walkFiles(dir, root = dir) {
+  return readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) return walkFiles(path, root);
+    return relative(root, path).replaceAll('\\', '/');
+  });
+}
+
+export function scanFilesForLocalPaths(root, files) {
+  const leaks = [];
+  for (const file of files) {
+    const path = join(root, file);
+    if (!existsSync(path)) continue;
+    const content = readFileSync(path, 'utf-8');
+    if (containsLocalPath(content)) leaks.push(file);
+  }
+  return leaks;
+}

package/benchmarks/reference-results.js

@@ -0,0 +1,70 @@
+export const PUBLISHED_LEADERBOARD = [
+  {
+    system: 'MIRIX',
+    benchmark: 'LoCoMo',
+    score: 85.4,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2507.07957',
+    note: 'Published LoCoMo result from the MIRIX paper.',
+  },
+  {
+    system: 'Letta Filesystem',
+    benchmark: 'LoCoMo',
+    score: 74.0,
+    unit: 'accuracy',
+    source: 'https://www.letta.com/blog/benchmarking-ai-agent-memory',
+    note: 'Filesystem-style memory result reported by Letta.',
+  },
+  {
+    system: 'Mem0 Graph Memory',
+    benchmark: 'LoCoMo',
+    score: 68.5,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2504.19413',
+    note: 'Graph memory variant reported in the Mem0 paper.',
+  },
+  {
+    system: 'Mem0',
+    benchmark: 'LoCoMo',
+    score: 66.9,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2504.19413',
+    note: 'Core Mem0 LoCoMo score reported in the Mem0 paper.',
+  },
+  {
+    system: 'OpenAI Memory',
+    benchmark: 'LoCoMo',
+    score: 52.9,
+    unit: 'accuracy',
+    source: 'https://arxiv.org/abs/2504.19413',
+    note: 'OpenAI memory baseline as reported by the Mem0 paper.',
+  },
+];
+
+export const MEMORY_TRENDS = [
+  {
+    title: 'Memory is moving from flat retrieval to typed systems',
+    summary: 'Recent work treats episodic, semantic, procedural, and graph memory as separate but cooperating layers.',
+    source: 'https://arxiv.org/abs/2507.03724',
+  },
+  {
+    title: 'Benchmarks now emphasize multi-session realism',
+    summary: 'LongMemEval and LoCoMo push memory systems toward temporal updates, abstraction, and cross-session reasoning instead of single-turn fact recall.',
+    source: 'https://arxiv.org/abs/2410.10813',
+  },
+  {
+    title: 'Context engineering is now competing with retrieval-first designs',
+    summary: 'Letta argues filesystem and memory-block approaches can outperform simpler retrieval-only memory on realistic long-horizon tasks.',
+    source: 'https://www.letta.com/blog/memory-blocks',
+  },
+  {
+    title: 'Production teams care about latency and token footprint, not just recall quality',
+    summary: 'Mem0 frames memory as a cost and latency optimization surface in addition to a personalization surface.',
+    source: 'https://arxiv.org/abs/2504.19413',
+  },
+  {
+    title: 'Temporal and multimodal memory are becoming table stakes',
+    summary: 'MIRIX and Graphiti both model time and state change explicitly instead of assuming memories stay forever true.',
+    source: 'https://arxiv.org/abs/2507.07957',
+  },
+];