audrey 1.1.3 → 1.2.1

package/benchmarks/perf-snapshot.js

@@ -1,307 +0,0 @@
-// Honest performance snapshot for Audrey.
-//
-// What this measures
-// - Encode latency at multiple corpus sizes
-// - Hybrid recall latency at multiple corpus sizes
-// - Post-encode queue processing time (consolidation/interference/validation pipeline)
-//
-// What this does NOT do
-// - Compare against other memory systems. Synthetic head-to-head numbers are
-//   easy to game. If you want a real comparison, run the same workload against
-//   the system you care about and post your own results.
-// - Use cloud embedding providers. Latency to a remote API is dominated by
-//   network round-trip and varies wildly by region and rate-limit state.
-//   We use the in-process mock embedding provider so the numbers reflect
-//   Audrey's own pipeline (SQLite, sqlite-vec, encode/recall logic, hybrid
-//   ranking) without third-party noise. Real-world recall p95 with a local
-//   384-dim provider is typically 5-15x higher; with a hosted provider it is
-//   dominated by the API call.
-//
-// How to read the output
-// - p50 / p95 / p99 are percentile latencies in milliseconds.
-// - The numbers are wall-clock for a single call from a JS caller, including
-//   SQLite work and any post-encode queueing on encode rows.
-// - Run on your own hardware and embedding provider before quoting numbers
-//   anywhere; results scale heavily with CPU, NVMe vs spinning disk, and
-//   embedding dimensionality.
-
-import { performance } from 'node:perf_hooks';
-import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
-import { join, resolve } from 'node:path';
-import { tmpdir, cpus, totalmem, arch, platform, release } from 'node:os';
-import { execFileSync } from 'node:child_process';
-import { pathToFileURL } from 'node:url';
-import { Audrey } from '../dist/src/index.js';
-
-const DEFAULT_SIZES = [100, 1000, 5000];
-const DEFAULT_RECALL_RUNS = 50;
-const QUERY_POOL = [
-  'rate limit handling for HTTP 429 retries',
-  'durable agent context across sessions',
-  'safe shell behavior on Windows hosts',
-  'official authority over inferred preferences',
-  'lexical signal for exact identifiers',
-  'deployment region migration steps',
-  'webhook signature recovery procedure',
-  'fraud queue manual review trigger',
-];
-
-const SEED_POOL = [
-  'Stripe API returned HTTP 429 during checkout retry; needs exponential backoff.',
-  'Project memory routing should prefer the local memory layer for durable context.',
-  'Tool trace learning marks repeated spawn EPERM failures as risky on Windows shells.',
-  'Calendar authority should come from the official source before inferred user notes.',
-  'Vector recall is faster but loses BM25 lexical signal on exact identifiers.',
-  'Webhook signature recovery requires rotating the signing secret and replaying queued events.',
-  'Fraud queue stabilizes when repeated same-BIN disputes are escalated for manual review.',
-  'Deployment region migrations should be coordinated against the provider rate-limit window.',
-];
-
-function percentile(values, rank) {
-  if (values.length === 0) return 0;
-  const sorted = [...values].sort((a, b) => a - b);
-  const index = Math.min(sorted.length - 1, Math.ceil((rank / 100) * sorted.length) - 1);
-  return sorted[index];
-}
-
-function round(value) {
-  return Math.round(value * 1000) / 1000;
-}
-
-function summarize(values) {
-  if (values.length === 0) {
-    return { samples: 0, p50: 0, p95: 0, p99: 0, min: 0, max: 0, mean: 0 };
-  }
-  const total = values.reduce((acc, v) => acc + v, 0);
-  return {
-    samples: values.length,
-    p50: round(percentile(values, 50)),
-    p95: round(percentile(values, 95)),
-    p99: round(percentile(values, 99)),
-    min: round(Math.min(...values)),
-    max: round(Math.max(...values)),
-    mean: round(total / values.length),
-  };
-}
-
-function gitSha() {
-  // execFileSync, no shell, fixed argv — provenance only, no user input flows in.
-  try {
-    return execFileSync('git', ['rev-parse', '--short', 'HEAD'], {
-      stdio: ['ignore', 'pipe', 'ignore'],
-    })
-      .toString()
-      .trim();
-  } catch {
-    return null;
-  }
-}
-
-export function resolveAudreyVersion() {
-  if (process.env.npm_package_version) {
-    return process.env.npm_package_version;
-  }
-
-  try {
-    const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
-    return typeof pkg.version === 'string' ? pkg.version : null;
-  } catch {
-    return null;
-  }
-}
-
-function machineProvenance() {
-  const cpuList = cpus();
-  const cpuModel = cpuList[0] ? cpuList[0].model : 'unknown';
-  const totalGb = Math.round((totalmem() / 1024 / 1024 / 1024) * 10) / 10;
-  return {
-    node: process.versions.node,
-    v8: process.versions.v8,
-    platform: platform(),
-    arch: arch(),
-    osRelease: release(),
-    cpuCount: cpuList.length,
-    cpuModel,
-    memoryGb: totalGb,
-  };
-}
-
-function createDataDir() {
-  const parents = [
-    process.env.AUDREY_PERF_PARENT_DIR,
-    tmpdir(),
-    join(process.cwd(), 'benchmarks', '.tmp'),
-  ].filter(Boolean);
-
-  let lastError;
-  for (const parent of parents) {
-    try {
-      mkdirSync(parent, { recursive: true });
-      return mkdtempSync(join(parent, 'audrey-perf-snapshot-'));
-    } catch (err) {
-      lastError = err;
-    }
-  }
-  throw lastError || new Error('Unable to create perf snapshot data directory');
-}
-
-function parseArgs(argv = process.argv.slice(2)) {
-  const args = {
-    sizes: [...DEFAULT_SIZES],
-    recallRuns: DEFAULT_RECALL_RUNS,
-    out: null,
-    json: false,
-  };
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if (token === '--sizes' && argv[i + 1]) {
-      args.sizes = argv[++i]
-        .split(',')
-        .map(s => Number.parseInt(s.trim(), 10))
-        .filter(n => Number.isFinite(n) && n > 0);
-    } else if (token === '--recall-runs' && argv[i + 1]) {
-      args.recallRuns = Number.parseInt(argv[++i], 10);
-    } else if (token === '--out' && argv[i + 1]) {
-      args.out = resolve(argv[++i]);
-    } else if (token === '--json') {
-      args.json = true;
-    }
-  }
-  return args;
-}
-
-async function runOneSize({ size, recallRuns }) {
-  const dataDir = createDataDir();
-  const audrey = new Audrey({
-    dataDir,
-    agent: 'perf-snapshot',
-    embedding: { provider: 'mock', dimensions: 64 },
-    llm: { provider: 'mock' },
-  });
-
-  const queueProcessingTimes = [];
-  audrey.on('post-encode-complete', event => {
-    queueProcessingTimes.push(event.processing_ms);
-  });
-
-  try {
-    const encodeTimes = [];
-    for (let i = 0; i < size; i++) {
-      const content = `${SEED_POOL[i % SEED_POOL.length]} (sample ${i})`;
-      const startedAt = performance.now();
-      await audrey.encode({
-        content,
-        source: 'direct-observation',
-        tags: ['perf-snapshot'],
-      });
-      encodeTimes.push(performance.now() - startedAt);
-    }
-
-    const drain = await audrey.drainPostEncodeQueue(60_000);
-    if (!drain.drained) {
-      throw new Error(`post-encode queue did not drain at size=${size}`);
-    }
-
-    const recallTimes = [];
-    for (let i = 0; i < recallRuns; i++) {
-      const query = QUERY_POOL[i % QUERY_POOL.length];
-      const startedAt = performance.now();
-      await audrey.recall(query, { limit: 5, retrieval: 'hybrid' });
-      recallTimes.push(performance.now() - startedAt);
-    }
-
-    return {
-      corpusSize: size,
-      encodeMs: summarize(encodeTimes),
-      hybridRecallMs: summarize(recallTimes),
-      postEncodeQueueMs: summarize(queueProcessingTimes),
-      queueEvents: queueProcessingTimes.length,
-    };
-  } finally {
-    audrey.close();
-    rmSync(dataDir, { recursive: true, force: true });
-  }
-}
-
-export async function runPerfSnapshot({
-  sizes = DEFAULT_SIZES,
-  recallRuns = DEFAULT_RECALL_RUNS,
-} = {}) {
-  const startedAt = Date.now();
-  const sized = [];
-  for (const size of sizes) {
-    sized.push(await runOneSize({ size, recallRuns }));
-  }
-  return {
-    generatedAt: new Date(startedAt).toISOString(),
-    durationMs: Date.now() - startedAt,
-    audreyVersion: resolveAudreyVersion(),
-    gitSha: gitSha(),
-    methodology: {
-      embedding: 'mock provider, 64 dimensions (in-process, no network)',
-      llm: 'mock provider (in-process)',
-      retrieval: 'hybrid (vector + lexical) with limit=5',
-      sizes,
-      recallRunsPerSize: recallRuns,
-      notes:
-        'Latency is wall-clock for a single call from a JS caller. Cloud and ' +
-        'local 384-dim providers will report higher recall latency dominated by ' +
-        'embedding cost and network. Run on your own hardware before quoting.',
-    },
-    machine: machineProvenance(),
-    sizes: sized,
-  };
-}
-
-function formatMs(value) {
-  if (value === 0) return '0';
-  if (value < 1) return value.toFixed(2);
-  return value.toFixed(1);
-}
-
-export function formatMarkdownTable(snapshot) {
-  const lines = [];
-  lines.push(
-    `Audrey perf snapshot — ${snapshot.audreyVersion || 'dev'} on ${snapshot.machine.platform}/${snapshot.machine.arch}`,
-  );
-  lines.push('');
-  lines.push(
-    `Node ${snapshot.machine.node} · ${snapshot.machine.cpuCount}x ${snapshot.machine.cpuModel} · ${snapshot.machine.memoryGb} GB RAM`,
-  );
-  lines.push(`Generated ${snapshot.generatedAt}${snapshot.gitSha ? ` (${snapshot.gitSha})` : ''}`);
-  lines.push('');
-  lines.push(
-    '| Corpus size | Encode p50 (ms) | Encode p95 (ms) | Recall p50 (ms) | Recall p95 (ms) | Recall p99 (ms) |',
-  );
-  lines.push('|---|---|---|---|---|---|');
-  for (const row of snapshot.sizes) {
-    lines.push(
-      `| ${row.corpusSize.toLocaleString()} ` +
-        `| ${formatMs(row.encodeMs.p50)} ` +
-        `| ${formatMs(row.encodeMs.p95)} ` +
-        `| ${formatMs(row.hybridRecallMs.p50)} ` +
-        `| ${formatMs(row.hybridRecallMs.p95)} ` +
-        `| ${formatMs(row.hybridRecallMs.p99)} |`,
-    );
-  }
-  return lines.join('\n');
-}
-
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
-  const args = parseArgs();
-  runPerfSnapshot({ sizes: args.sizes, recallRuns: args.recallRuns })
-    .then(snapshot => {
-      if (args.out) {
-        writeFileSync(args.out, JSON.stringify(snapshot, null, 2) + '\n');
-      }
-      if (args.json) {
-        process.stdout.write(JSON.stringify(snapshot, null, 2) + '\n');
-      } else {
-        process.stdout.write(formatMarkdownTable(snapshot) + '\n');
-      }
-    })
-    .catch(err => {
-      console.error('[audrey] perf snapshot failed:', err);
-      process.exit(1);
-    });
-}

package/benchmarks/perf.bench.js

@@ -1,169 +0,0 @@
-import { performance } from 'node:perf_hooks';
-import { mkdirSync, mkdtempSync, rmSync } from 'node:fs';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-import { pathToFileURL } from 'node:url';
-import { Audrey } from '../dist/src/index.js';
-
-const RUNS = 20;
-
-// Budget source: CHANGELOG.md#0220---2026-04-28, from the Audrey/MemoryGym
-// latency pass. This mock-provider gate catches mechanical regressions in
-// Audrey CI before live GPU benchmarks or MemoryGym release gates find them.
-export const PERF_BUDGETS = Object.freeze({
-  encodeResponseP95Ms: 50,
-  hybridRecallP95Ms: 25,
-  queueProcessingP50Ms: 5,
-});
-
-function roundMs(value) {
-  return Math.round(value * 1000) / 1000;
-}
-
-function percentile(values, percentileRank) {
-  if (values.length === 0) return 0;
-  const sorted = [...values].sort((a, b) => a - b);
-  const index = Math.min(sorted.length - 1, Math.ceil((percentileRank / 100) * sorted.length) - 1);
-  return sorted[index];
-}
-
-function stats(values) {
-  if (values.length === 0) {
-    return { p50: 0, p95: 0, min: 0, max: 0 };
-  }
-  return {
-    p50: roundMs(percentile(values, 50)),
-    p95: roundMs(percentile(values, 95)),
-    min: roundMs(Math.min(...values)),
-    max: roundMs(Math.max(...values)),
-  };
-}
-
-function assertBudget(name, actual, budget) {
-  if (actual >= budget) {
-    throw new Error(`${name} ${actual}ms exceeded budget ${budget}ms`);
-  }
-}
-
-function seedContent(index) {
-  const cases = [
-    'Stripe API returned HTTP 429 during checkout retry and needs exponential backoff.',
-    'Project memory routing should prefer Audrey MCP for durable agent context.',
-    'Tool trace learning marks repeated npm spawn EPERM failures as risky on Windows shells.',
-    'Calendar authority should come from the official source before inferred user notes.',
-    'Vector recall is faster but loses BM25 lexical signal on exact identifiers.',
-  ];
-  return `${cases[index % cases.length]} Perf sample ${index}.`;
-}
-
-function createPerfDataDir() {
-  const parents = [
-    process.env.AUDREY_PERF_PARENT_DIR,
-    tmpdir(),
-    join(process.cwd(), 'benchmarks', '.tmp'),
-  ].filter(Boolean);
-  let lastError;
-
-  for (const parent of parents) {
-    try {
-      mkdirSync(parent, { recursive: true });
-      return mkdtempSync(join(parent, 'audrey-perf-'));
-    } catch (err) {
-      lastError = err;
-    }
-  }
-
-  throw lastError || new Error('Unable to create Audrey perf benchmark data directory');
-}
-
-export async function runPerfBenchmark({
-  runs = RUNS,
-  budgets = PERF_BUDGETS,
-  out = console.log,
-} = {}) {
-  const dataDir = createPerfDataDir();
-  const audrey = new Audrey({
-    dataDir,
-    agent: 'perf-bench',
-    embedding: { provider: 'mock', dimensions: 64 },
-    llm: { provider: 'mock' },
-  });
-
-  const queueProcessingTimes = [];
-  audrey.on('post-encode-complete', event => {
-    queueProcessingTimes.push(event.processing_ms);
-  });
-
-  try {
-    const encodeTimes = [];
-    for (let i = 0; i < runs; i += 1) {
-      const startedAt = performance.now();
-      await audrey.encode({
-        content: seedContent(i),
-        source: 'direct-observation',
-        tags: ['perf-gate'],
-        affect: { valence: i % 2 === 0 ? 0.3 : -0.1, arousal: 0.2 },
-      });
-      encodeTimes.push(performance.now() - startedAt);
-    }
-
-    const drain = await audrey.drainPostEncodeQueue(5000);
-    if (!drain.drained) {
-      throw new Error(`post-encode queue did not drain: ${drain.pendingIds.join(', ')}`);
-    }
-
-    const recallTimes = [];
-    for (let i = 0; i < runs; i += 1) {
-      const startedAt = performance.now();
-      await audrey.recall('Stripe API 429 retry memory routing', {
-        limit: 5,
-        retrieval: 'hybrid',
-      });
-      recallTimes.push(performance.now() - startedAt);
-    }
-
-    const result = {
-      runs,
-      budgets,
-      encode_response_ms: stats(encodeTimes),
-      hybrid_recall_ms: stats(recallTimes),
-      queue_processing_ms: stats(queueProcessingTimes),
-      queue_events: queueProcessingTimes.length,
-      status: {
-        pending_consolidation_count: audrey.memoryStatus().pending_consolidation_count,
-        default_retrieval_mode: audrey.memoryStatus().default_retrieval_mode,
-      },
-    };
-
-    if (queueProcessingTimes.length !== runs) {
-      throw new Error(
-        `expected ${runs} post-encode queue events, got ${queueProcessingTimes.length}`,
-      );
-    }
-
-    assertBudget('encode response p95', result.encode_response_ms.p95, budgets.encodeResponseP95Ms);
-    assertBudget('hybrid recall p95', result.hybrid_recall_ms.p95, budgets.hybridRecallP95Ms);
-    assertBudget(
-      'queue processing p50',
-      result.queue_processing_ms.p50,
-      budgets.queueProcessingP50Ms,
-    );
-
-    out(
-      `Audrey perf gate passed: encode p95=${result.encode_response_ms.p95}ms, ` +
-        `hybrid recall p95=${result.hybrid_recall_ms.p95}ms, ` +
-        `queue p50=${result.queue_processing_ms.p50}ms`,
-    );
-    return result;
-  } finally {
-    audrey.close();
-    rmSync(dataDir, { recursive: true, force: true });
-  }
-}
-
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
-  runPerfBenchmark().catch(err => {
-    console.error('[audrey] perf gate failed:', err);
-    process.exit(1);
-  });
-}

package/benchmarks/public-paths.mjs

@@ -1,97 +0,0 @@
-import { existsSync, readFileSync, readdirSync } from 'node:fs';
-import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
-const WINDOWS_DRIVE_PATTERN = /(^|[^a-z])[A-Z]:[\\/](?:$|[^\r\n])/i;
-const EXTENDED_PATH_PATTERN = /\\\\\?\\/;
-const FILE_URL_PATTERN = /file:\/\//i;
-function containsBinaryControl(text) {
-  return Array.from(text).some(character => {
-    const code = character.charCodeAt(0);
-    return (
-      (code >= 0 && code <= 8) ||
-      code === 11 ||
-      code === 12 ||
-      (code >= 14 && code <= 31) ||
-      code === 127
-    );
-  });
-}
-
-function isUrl(value) {
-  return /^[a-z][a-z0-9+.-]*:\/\//i.test(value);
-}
-
-export function publicPath(value) {
-  if (typeof value !== 'string') return value;
-  if (value === process.execPath) return 'node';
-  if (isUrl(value)) return value;
-
-  const resolved = resolve(value);
-  const rel = relative(ROOT, resolved);
-  if (rel && !rel.startsWith('..') && !isAbsolute(rel)) {
-    return rel.replaceAll('\\', '/');
-  }
-  if (value.includes('\\') || value.includes('/') || isAbsolute(value)) {
-    return `[LOCAL-PATH:${basename(value) || 'path'}]`;
-  }
-  return value;
-}
-
-export function publicCommand(command = []) {
-  return command.map(part => publicPath(part));
-}
-
-export function publicArtifactValue(value) {
-  if (Array.isArray(value)) return value.map(item => publicArtifactValue(item));
-  if (value && typeof value === 'object') {
-    return Object.fromEntries(
-      Object.entries(value).map(([key, item]) => [key, publicArtifactValue(item)]),
-    );
-  }
-  return publicPath(value);
-}
-
-export function containsLocalPath(text) {
-  if (containsBinaryControl(text)) return false;
-  return (
-    WINDOWS_DRIVE_PATTERN.test(text) ||
-    EXTENDED_PATH_PATTERN.test(text) ||
-    FILE_URL_PATTERN.test(text)
-  );
-}
-
-export function findLocalPathLeaks(value, path = '$') {
-  if (typeof value === 'string') {
-    return containsLocalPath(value) ? [`${path}: ${value}`] : [];
-  }
-  if (Array.isArray(value)) {
-    return value.flatMap((item, index) => findLocalPathLeaks(item, `${path}[${index}]`));
-  }
-  if (value && typeof value === 'object') {
-    return Object.entries(value).flatMap(([key, item]) =>
-      findLocalPathLeaks(item, `${path}.${key}`),
-    );
-  }
-  return [];
-}
-
-export function walkFiles(dir, root = dir) {
-  return readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
-    const path = join(dir, entry.name);
-    if (entry.isDirectory()) return walkFiles(path, root);
-    return relative(root, path).replaceAll('\\', '/');
-  });
-}
-
-export function scanFilesForLocalPaths(root, files) {
-  const leaks = [];
-  for (const file of files) {
-    const path = join(root, file);
-    if (!existsSync(path)) continue;
-    const content = readFileSync(path, 'utf-8');
-    if (containsLocalPath(content)) leaks.push(file);
-  }
-  return leaks;
-}

package/benchmarks/reference-results.js

@@ -1,75 +0,0 @@
-export const PUBLISHED_LEADERBOARD = [
-  {
-    system: 'MIRIX',
-    benchmark: 'LoCoMo',
-    score: 85.4,
-    unit: 'accuracy',
-    source: 'https://arxiv.org/abs/2507.07957',
-    note: 'Published LoCoMo result from the MIRIX paper.',
-  },
-  {
-    system: 'Letta Filesystem',
-    benchmark: 'LoCoMo',
-    score: 74.0,
-    unit: 'accuracy',
-    source: 'https://www.letta.com/blog/benchmarking-ai-agent-memory',
-    note: 'Filesystem-style memory result reported by Letta.',
-  },
-  {
-    system: 'Mem0 Graph Memory',
-    benchmark: 'LoCoMo',
-    score: 68.5,
-    unit: 'accuracy',
-    source: 'https://arxiv.org/abs/2504.19413',
-    note: 'Graph memory variant reported in the Mem0 paper.',
-  },
-  {
-    system: 'Mem0',
-    benchmark: 'LoCoMo',
-    score: 66.9,
-    unit: 'accuracy',
-    source: 'https://arxiv.org/abs/2504.19413',
-    note: 'Core Mem0 LoCoMo score reported in the Mem0 paper.',
-  },
-  {
-    system: 'OpenAI Memory',
-    benchmark: 'LoCoMo',
-    score: 52.9,
-    unit: 'accuracy',
-    source: 'https://arxiv.org/abs/2504.19413',
-    note: 'OpenAI memory baseline as reported by the Mem0 paper.',
-  },
-];
-
-export const MEMORY_TRENDS = [
-  {
-    title: 'Memory is moving from flat retrieval to typed systems',
-    summary:
-      'Recent work treats episodic, semantic, procedural, and graph memory as separate but cooperating layers.',
-    source: 'https://arxiv.org/abs/2507.03724',
-  },
-  {
-    title: 'Benchmarks now emphasize multi-session realism',
-    summary:
-      'LongMemEval and LoCoMo push memory systems toward temporal updates, abstraction, and cross-session reasoning instead of single-turn fact recall.',
-    source: 'https://arxiv.org/abs/2410.10813',
-  },
-  {
-    title: 'Context engineering is now competing with retrieval-first designs',
-    summary:
-      'Letta argues filesystem and memory-block approaches can outperform simpler retrieval-only memory on realistic long-horizon tasks.',
-    source: 'https://www.letta.com/blog/memory-blocks',
-  },
-  {
-    title: 'Production teams care about latency and token footprint, not just recall quality',
-    summary:
-      'Mem0 frames memory as a cost and latency optimization surface in addition to a personalization surface.',
-    source: 'https://arxiv.org/abs/2504.19413',
-  },
-  {
-    title: 'Temporal and multimodal memory are becoming table stakes',
-    summary:
-      'MIRIX and Graphiti both model time and state change explicitly instead of assuming memories stay forever true.',
-    source: 'https://arxiv.org/abs/2507.07957',
-  },
-];