audrey 1.1.3 → 1.2.0

package/benchmarks/snapshots/perf-0.22.2.json

@@ -1,123 +0,0 @@
-{
-  "generatedAt": "2026-05-01T02:15:29.400Z",
-  "durationMs": 4600,
-  "audreyVersion": null,
-  "gitSha": "e2e821b",
-  "methodology": {
-    "embedding": "mock provider, 64 dimensions (in-process, no network)",
-    "llm": "mock provider (in-process)",
-    "retrieval": "hybrid (vector + lexical) with limit=5",
-    "sizes": [
-      100,
-      1000,
-      5000
-    ],
-    "recallRunsPerSize": 50,
-    "notes": "Latency is wall-clock for a single call from a JS caller. Cloud and local 384-dim providers will report higher recall latency dominated by embedding cost and network. Run on your own hardware before quoting."
-  },
-  "machine": {
-    "node": "25.5.0",
-    "v8": "14.1.146.11-node.18",
-    "platform": "win32",
-    "arch": "x64",
-    "osRelease": "10.0.26200",
-    "cpuCount": 24,
-    "cpuModel": "AMD Ryzen 9 7900X3D 12-Core Processor          ",
-    "memoryGb": 62.9
-  },
-  "sizes": [
-    {
-      "corpusSize": 100,
-      "encodeMs": {
-        "samples": 100,
-        "p50": 0.331,
-        "p95": 0.589,
-        "p99": 7.65,
-        "min": 0.214,
-        "max": 10.978,
-        "mean": 0.577
-      },
-      "hybridRecallMs": {
-        "samples": 50,
-        "p50": 0.539,
-        "p95": 1.82,
-        "p99": 2.712,
-        "min": 0.448,
-        "max": 2.712,
-        "mean": 0.659
-      },
-      "postEncodeQueueMs": {
-        "samples": 100,
-        "p50": 0.344,
-        "p95": 0.73,
-        "p99": 9.715,
-        "min": 0.079,
-        "max": 13.218,
-        "mean": 0.533
-      },
-      "queueEvents": 100
-    },
-    {
-      "corpusSize": 1000,
-      "encodeMs": {
-        "samples": 1000,
-        "p50": 0.307,
-        "p95": 2.147,
-        "p99": 9.672,
-        "min": 0.183,
-        "max": 13.514,
-        "mean": 0.639
-      },
-      "hybridRecallMs": {
-        "samples": 50,
-        "p50": 1.566,
-        "p95": 2.364,
-        "p99": 21.177,
-        "min": 0.608,
-        "max": 21.177,
-        "mean": 1.803
-      },
-      "postEncodeQueueMs": {
-        "samples": 1000,
-        "p50": 0.321,
-        "p95": 1.468,
-        "p99": 9.744,
-        "min": 0.07,
-        "max": 15.61,
-        "mean": 0.553
-      },
-      "queueEvents": 1000
-    },
-    {
-      "corpusSize": 5000,
-      "encodeMs": {
-        "samples": 5000,
-        "p50": 0.308,
-        "p95": 1.838,
-        "p99": 10.45,
-        "min": 0.171,
-        "max": 19.247,
-        "mean": 0.653
-      },
-      "hybridRecallMs": {
-        "samples": 50,
-        "p50": 2.091,
-        "p95": 3.417,
-        "p99": 16.58,
-        "min": 2.005,
-        "max": 16.58,
-        "mean": 2.492
-      },
-      "postEncodeQueueMs": {
-        "samples": 5000,
-        "p50": 0.313,
-        "p95": 1.217,
-        "p99": 10.532,
-        "min": 0.074,
-        "max": 19.378,
-        "mean": 0.554
-      },
-      "queueEvents": 5000
-    }
-  ]
-}

package/benchmarks/snapshots/perf-0.23.0.json

@@ -1,123 +0,0 @@
-{
-  "generatedAt": "2026-05-05T17:32:45.578Z",
-  "durationMs": 1042,
-  "audreyVersion": "0.23.0",
-  "gitSha": "20cdde0",
-  "methodology": {
-    "embedding": "mock provider, 64 dimensions (in-process, no network)",
-    "llm": "mock provider (in-process)",
-    "retrieval": "hybrid (vector + lexical) with limit=5",
-    "sizes": [
-      100,
-      1000,
-      5000
-    ],
-    "recallRunsPerSize": 50,
-    "notes": "Latency is wall-clock for a single call from a JS caller. Cloud and local 384-dim providers will report higher recall latency dominated by embedding cost and network. Run on your own hardware before quoting."
-  },
-  "machine": {
-    "node": "25.9.0",
-    "v8": "14.1.146.11-node.25",
-    "platform": "darwin",
-    "arch": "arm64",
-    "osRelease": "25.4.0",
-    "cpuCount": 18,
-    "cpuModel": "Apple M5 Max",
-    "memoryGb": 64
-  },
-  "sizes": [
-    {
-      "corpusSize": 100,
-      "encodeMs": {
-        "samples": 100,
-        "p50": 0.136,
-        "p95": 0.246,
-        "p99": 1.054,
-        "min": 0.084,
-        "max": 2.211,
-        "mean": 0.178
-      },
-      "hybridRecallMs": {
-        "samples": 50,
-        "p50": 0.215,
-        "p95": 0.693,
-        "p99": 1.285,
-        "min": 0.166,
-        "max": 1.285,
-        "mean": 0.261
-      },
-      "postEncodeQueueMs": {
-        "samples": 100,
-        "p50": 0.134,
-        "p95": 0.35,
-        "p99": 1.079,
-        "min": 0.036,
-        "max": 1.244,
-        "mean": 0.156
-      },
-      "queueEvents": 100
-    },
-    {
-      "corpusSize": 1000,
-      "encodeMs": {
-        "samples": 1000,
-        "p50": 0.114,
-        "p95": 0.187,
-        "p99": 0.865,
-        "min": 0.074,
-        "max": 1.378,
-        "mean": 0.139
-      },
-      "hybridRecallMs": {
-        "samples": 50,
-        "p50": 0.272,
-        "p95": 0.48,
-        "p99": 2.081,
-        "min": 0.254,
-        "max": 2.081,
-        "mean": 0.322
-      },
-      "postEncodeQueueMs": {
-        "samples": 1000,
-        "p50": 0.122,
-        "p95": 0.256,
-        "p99": 0.921,
-        "min": 0.035,
-        "max": 1.49,
-        "mean": 0.127
-      },
-      "queueEvents": 1000
-    },
-    {
-      "corpusSize": 5000,
-      "encodeMs": {
-        "samples": 5000,
-        "p50": 0.109,
-        "p95": 0.174,
-        "p99": 0.938,
-        "min": 0.07,
-        "max": 3.618,
-        "mean": 0.136
-      },
-      "hybridRecallMs": {
-        "samples": 50,
-        "p50": 0.735,
-        "p95": 0.867,
-        "p99": 4.228,
-        "min": 0.688,
-        "max": 4.228,
-        "mean": 0.816
-      },
-      "postEncodeQueueMs": {
-        "samples": 5000,
-        "p50": 0.116,
-        "p95": 0.242,
-        "p99": 0.978,
-        "min": 0.034,
-        "max": 6.272,
-        "mean": 0.124
-      },
-      "queueEvents": 5000
-    }
-  ]
-}

package/benchmarks/validate-adapter-module.mjs

@@ -1,107 +0,0 @@
-import { existsSync } from 'node:fs';
-import { basename, resolve } from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'node:url';
-import { validateGuardBenchAdapter } from './guardbench.js';
-import { publicPath } from './public-paths.mjs';
-
-const DEFAULT_ADAPTER = 'benchmarks/adapters/example-allow.mjs';
-
-export function parseAdapterModuleValidatorArgs(argv = process.argv.slice(2)) {
-  const args = {
-    adapter: DEFAULT_ADAPTER,
-    json: false,
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if (token === '--adapter' && argv[i + 1]) args.adapter = argv[++i];
-    else if (token === '--json') args.json = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-
-  return args;
-}
-
-function usage() {
-  return `Usage: node benchmarks/validate-adapter-module.mjs [options]
-
-Options:
-  --adapter <path>   ESM GuardBench adapter module. Default: ${DEFAULT_ADAPTER}.
-  --json             Print the machine-readable validation report.
-`;
-}
-
-export async function validateAdapterModuleFile(options = {}) {
-  const adapterPath = resolve(options.adapter ?? DEFAULT_ADAPTER);
-  const failures = [];
-  let adapter = null;
-
-  if (!existsSync(adapterPath)) {
-    failures.push(`Adapter not found: ${adapterPath}`);
-  } else {
-    try {
-      const mod = await import(pathToFileURL(adapterPath).href);
-      const candidate =
-        typeof mod.createGuardBenchAdapter === 'function'
-          ? await mod.createGuardBenchAdapter()
-          : (mod.default ?? mod.adapter);
-      adapter = validateGuardBenchAdapter(candidate, adapterPath);
-    } catch (error) {
-      failures.push(error.message);
-    }
-  }
-
-  return {
-    ok: failures.length === 0,
-    adapterPath: publicPath(adapterPath),
-    moduleFile: basename(adapterPath),
-    adapter: adapter
-      ? {
-          name: adapter.name,
-          description: adapter.description ?? null,
-          hasSetup: typeof adapter.setup === 'function',
-          hasDecide: typeof adapter.decide === 'function',
-          hasCleanup: typeof adapter.cleanup === 'function',
-        }
-      : null,
-    contract: {
-      moduleFormat: 'ESM',
-      exports: ['default', 'adapter', 'createGuardBenchAdapter'],
-      requiredMethods: ['decide'],
-      optionalMethods: ['setup', 'cleanup'],
-    },
-    failures,
-  };
-}
-
-async function main() {
-  const args = parseAdapterModuleValidatorArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-
-  const validation = await validateAdapterModuleFile(args);
-  if (args.json) {
-    console.log(JSON.stringify(validation, null, 2));
-  } else if (validation.ok) {
-    console.log(`GuardBench adapter module validation passed: ${validation.adapterPath}`);
-    console.log(`Adapter: ${validation.adapter.name}`);
-    console.log(
-      `Methods: setup=${validation.adapter.hasSetup}, decide=${validation.adapter.hasDecide}, cleanup=${validation.adapter.hasCleanup}`,
-    );
-  } else {
-    console.error('GuardBench adapter module validation failed:');
-    for (const failure of validation.failures) console.error(`- ${failure}`);
-  }
-
-  if (!validation.ok) process.exit(1);
-}
-
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
-  main().catch(error => {
-    console.error(error.stack ?? error.message);
-    process.exit(1);
-  });
-}

package/benchmarks/validate-adapter-registry.mjs

@@ -1,145 +0,0 @@
-import { existsSync, readFileSync } from 'node:fs';
-import { resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { validateSchema } from './validate-guardbench-artifacts.mjs';
-import { validateAdapterModuleFile } from './validate-adapter-module.mjs';
-import { publicPath } from './public-paths.mjs';
-
-const DEFAULT_REGISTRY = 'benchmarks/adapters/registry.json';
-const DEFAULT_SCHEMA = 'benchmarks/schemas/guardbench-adapter-registry.schema.json';
-
-export function parseAdapterRegistryValidatorArgs(argv = process.argv.slice(2)) {
-  const args = {
-    registry: DEFAULT_REGISTRY,
-    schema: DEFAULT_SCHEMA,
-    json: false,
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if (token === '--registry' && argv[i + 1]) args.registry = argv[++i];
-    else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
-    else if (token === '--json') args.json = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-
-  return args;
-}
-
-function usage() {
-  return `Usage: node benchmarks/validate-adapter-registry.mjs [options]
-
-Options:
-  --registry <path>   Adapter registry JSON. Default: ${DEFAULT_REGISTRY}.
-  --schema <path>     Adapter registry JSON schema. Default: ${DEFAULT_SCHEMA}.
-  --json              Print the machine-readable validation report.
-`;
-}
-
-function readJson(path) {
-  return JSON.parse(readFileSync(path, 'utf-8'));
-}
-
-export async function validateAdapterRegistry(options = {}) {
-  const registryPath = resolve(options.registry ?? DEFAULT_REGISTRY);
-  const schemaPath = resolve(options.schema ?? DEFAULT_SCHEMA);
-  const failures = [];
-  let registry = null;
-
-  try {
-    registry = readJson(registryPath);
-  } catch (error) {
-    failures.push(error.message);
-  }
-
-  try {
-    const schema = readJson(schemaPath);
-    if (registry) failures.push(...validateSchema(registry, schema, 'guardbench-adapter-registry'));
-  } catch (error) {
-    failures.push(error.message);
-  }
-
-  const ids = new Set();
-  const adapterReports = [];
-  for (const adapter of registry?.adapters ?? []) {
-    if (ids.has(adapter.id)) failures.push(`Duplicate adapter id: ${adapter.id}`);
-    ids.add(adapter.id);
-    if (adapter.credentialMode === 'none' && adapter.requiredEnv.length !== 0) {
-      failures.push(`Adapter ${adapter.id} has credentialMode=none but declares requiredEnv`);
-    }
-    if (adapter.credentialMode === 'runtime-env' && adapter.requiredEnv.length === 0) {
-      failures.push(
-        `Adapter ${adapter.id} has credentialMode=runtime-env but declares no requiredEnv`,
-      );
-    }
-    for (const [commandName, command] of Object.entries(adapter.commands ?? {})) {
-      if (
-        (commandName === 'moduleValidate' || commandName === 'selfTest') &&
-        !command.includes(adapter.path)
-      ) {
-        failures.push(
-          `Adapter ${adapter.id} command ${commandName} does not reference ${adapter.path}`,
-        );
-      }
-    }
-    if (!existsSync(resolve(adapter.path))) {
-      failures.push(`Adapter ${adapter.id} path does not exist: ${adapter.path}`);
-      continue;
-    }
-    const report = await validateAdapterModuleFile({ adapter: adapter.path });
-    adapterReports.push({
-      id: adapter.id,
-      ok: report.ok,
-      adapter: report.adapter,
-      credentialMode: adapter.credentialMode,
-      failures: report.failures,
-    });
-    if (!report.ok) {
-      failures.push(
-        `Adapter ${adapter.id} failed module validation: ${report.failures.join('; ')}`,
-      );
-    }
-    if (report.adapter?.name && report.adapter.name !== adapter.name) {
-      failures.push(
-        `Adapter ${adapter.id} registry name ${adapter.name} does not match module name ${report.adapter.name}`,
-      );
-    }
-  }
-
-  return {
-    ok: failures.length === 0,
-    registry: publicPath(registryPath),
-    schema: publicPath(schemaPath),
-    adapters: adapterReports,
-    failures,
-  };
-}
-
-async function main() {
-  const args = parseAdapterRegistryValidatorArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-
-  const validation = await validateAdapterRegistry(args);
-  if (args.json) {
-    console.log(JSON.stringify(validation, null, 2));
-  } else if (validation.ok) {
-    console.log(`GuardBench adapter registry validation passed: ${validation.registry}`);
-    console.log(`Adapters: ${validation.adapters.length}`);
-  } else {
-    console.error('GuardBench adapter registry validation failed:');
-    for (const failure of validation.failures) console.error(`- ${failure}`);
-  }
-
-  if (!validation.ok) process.exit(1);
-}
-
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
-  main().catch(error => {
-    console.error(error.stack ?? error.message);
-    process.exit(1);
-  });
-}

package/benchmarks/validate-adapter-self-test.mjs

@@ -1,96 +0,0 @@
-import { existsSync, readFileSync } from 'node:fs';
-import { resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { validateAdapterSelfTestReport } from './adapter-self-test.mjs';
-import { publicPath } from './public-paths.mjs';
-
-const DEFAULT_REPORT = 'benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json';
-
-export function parseAdapterSelfTestValidatorArgs(argv = process.argv.slice(2)) {
-  const args = {
-    report: DEFAULT_REPORT,
-    schema: undefined,
-    json: false,
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if ((token === '--report' || token === '--file') && argv[i + 1]) args.report = argv[++i];
-    else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
-    else if (token === '--json') args.json = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-
-  return args;
-}
-
-function usage() {
-  return `Usage: node benchmarks/validate-adapter-self-test.mjs [options]
-
-Options:
-  --report <path>   Adapter self-test JSON report. Default: ${DEFAULT_REPORT}.
-  --schema <path>   Optional alternate schema path.
-  --json            Print the machine-readable validation report.
-`;
-}
-
-function readJson(path) {
-  return JSON.parse(readFileSync(path, 'utf-8'));
-}
-
-export function validateAdapterSelfTestFile(options = {}) {
-  const reportPath = resolve(options.report ?? DEFAULT_REPORT);
-  const failures = [];
-  let report = null;
-
-  if (!existsSync(reportPath)) {
-    failures.push(`Missing adapter self-test report: ${reportPath}`);
-  } else {
-    try {
-      report = readJson(reportPath);
-      failures.push(...validateAdapterSelfTestReport(report, { schema: options.schema }));
-    } catch (error) {
-      failures.push(error.message);
-    }
-  }
-
-  return {
-    ok: failures.length === 0,
-    report: publicPath(reportPath),
-    adapter: report?.adapter?.name ?? null,
-    scenarios: report?.conformance?.scenarios ?? null,
-    expectedScenarios: report?.conformance?.expectedScenarios ?? null,
-    lowScoreAllowed: report?.contract?.lowScoreAllowed ?? null,
-    failures,
-  };
-}
-
-async function main() {
-  const args = parseAdapterSelfTestValidatorArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-
-  const validation = validateAdapterSelfTestFile(args);
-  if (args.json) {
-    console.log(JSON.stringify(validation, null, 2));
-  } else if (validation.ok) {
-    console.log(`GuardBench adapter self-test validation passed: ${validation.report}`);
-    console.log(`Adapter: ${validation.adapter}`);
-    console.log(`Rows: ${validation.scenarios}/${validation.expectedScenarios}`);
-  } else {
-    console.error('GuardBench adapter self-test validation failed:');
-    for (const failure of validation.failures) console.error(`- ${failure}`);
-  }
-
-  if (!validation.ok) process.exit(1);
-}
-
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
-  main().catch(error => {
-    console.error(error.stack ?? error.message);
-    process.exit(1);
-  });
-}