audrey 0.23.1 → 1.0.1

package/benchmarks/snapshots/perf-0.22.2.json

@@ -0,0 +1,123 @@
+{
+  "generatedAt": "2026-05-01T02:15:29.400Z",
+  "durationMs": 4600,
+  "audreyVersion": null,
+  "gitSha": "e2e821b",
+  "methodology": {
+    "embedding": "mock provider, 64 dimensions (in-process, no network)",
+    "llm": "mock provider (in-process)",
+    "retrieval": "hybrid (vector + lexical) with limit=5",
+    "sizes": [
+      100,
+      1000,
+      5000
+    ],
+    "recallRunsPerSize": 50,
+    "notes": "Latency is wall-clock for a single call from a JS caller. Cloud and local 384-dim providers will report higher recall latency dominated by embedding cost and network. Run on your own hardware before quoting."
+  },
+  "machine": {
+    "node": "25.5.0",
+    "v8": "14.1.146.11-node.18",
+    "platform": "win32",
+    "arch": "x64",
+    "osRelease": "10.0.26200",
+    "cpuCount": 24,
+    "cpuModel": "AMD Ryzen 9 7900X3D 12-Core Processor          ",
+    "memoryGb": 62.9
+  },
+  "sizes": [
+    {
+      "corpusSize": 100,
+      "encodeMs": {
+        "samples": 100,
+        "p50": 0.331,
+        "p95": 0.589,
+        "p99": 7.65,
+        "min": 0.214,
+        "max": 10.978,
+        "mean": 0.577
+      },
+      "hybridRecallMs": {
+        "samples": 50,
+        "p50": 0.539,
+        "p95": 1.82,
+        "p99": 2.712,
+        "min": 0.448,
+        "max": 2.712,
+        "mean": 0.659
+      },
+      "postEncodeQueueMs": {
+        "samples": 100,
+        "p50": 0.344,
+        "p95": 0.73,
+        "p99": 9.715,
+        "min": 0.079,
+        "max": 13.218,
+        "mean": 0.533
+      },
+      "queueEvents": 100
+    },
+    {
+      "corpusSize": 1000,
+      "encodeMs": {
+        "samples": 1000,
+        "p50": 0.307,
+        "p95": 2.147,
+        "p99": 9.672,
+        "min": 0.183,
+        "max": 13.514,
+        "mean": 0.639
+      },
+      "hybridRecallMs": {
+        "samples": 50,
+        "p50": 1.566,
+        "p95": 2.364,
+        "p99": 21.177,
+        "min": 0.608,
+        "max": 21.177,
+        "mean": 1.803
+      },
+      "postEncodeQueueMs": {
+        "samples": 1000,
+        "p50": 0.321,
+        "p95": 1.468,
+        "p99": 9.744,
+        "min": 0.07,
+        "max": 15.61,
+        "mean": 0.553
+      },
+      "queueEvents": 1000
+    },
+    {
+      "corpusSize": 5000,
+      "encodeMs": {
+        "samples": 5000,
+        "p50": 0.308,
+        "p95": 1.838,
+        "p99": 10.45,
+        "min": 0.171,
+        "max": 19.247,
+        "mean": 0.653
+      },
+      "hybridRecallMs": {
+        "samples": 50,
+        "p50": 2.091,
+        "p95": 3.417,
+        "p99": 16.58,
+        "min": 2.005,
+        "max": 16.58,
+        "mean": 2.492
+      },
+      "postEncodeQueueMs": {
+        "samples": 5000,
+        "p50": 0.313,
+        "p95": 1.217,
+        "p99": 10.532,
+        "min": 0.074,
+        "max": 19.378,
+        "mean": 0.554
+      },
+      "queueEvents": 5000
+    }
+  ]
+}

package/benchmarks/snapshots/perf-0.23.0.json

@@ -0,0 +1,123 @@
+{
+  "generatedAt": "2026-05-05T17:32:45.578Z",
+  "durationMs": 1042,
+  "audreyVersion": "0.23.0",
+  "gitSha": "20cdde0",
+  "methodology": {
+    "embedding": "mock provider, 64 dimensions (in-process, no network)",
+    "llm": "mock provider (in-process)",
+    "retrieval": "hybrid (vector + lexical) with limit=5",
+    "sizes": [
+      100,
+      1000,
+      5000
+    ],
+    "recallRunsPerSize": 50,
+    "notes": "Latency is wall-clock for a single call from a JS caller. Cloud and local 384-dim providers will report higher recall latency dominated by embedding cost and network. Run on your own hardware before quoting."
+  },
+  "machine": {
+    "node": "25.9.0",
+    "v8": "14.1.146.11-node.25",
+    "platform": "darwin",
+    "arch": "arm64",
+    "osRelease": "25.4.0",
+    "cpuCount": 18,
+    "cpuModel": "Apple M5 Max",
+    "memoryGb": 64
+  },
+  "sizes": [
+    {
+      "corpusSize": 100,
+      "encodeMs": {
+        "samples": 100,
+        "p50": 0.136,
+        "p95": 0.246,
+        "p99": 1.054,
+        "min": 0.084,
+        "max": 2.211,
+        "mean": 0.178
+      },
+      "hybridRecallMs": {
+        "samples": 50,
+        "p50": 0.215,
+        "p95": 0.693,
+        "p99": 1.285,
+        "min": 0.166,
+        "max": 1.285,
+        "mean": 0.261
+      },
+      "postEncodeQueueMs": {
+        "samples": 100,
+        "p50": 0.134,
+        "p95": 0.35,
+        "p99": 1.079,
+        "min": 0.036,
+        "max": 1.244,
+        "mean": 0.156
+      },
+      "queueEvents": 100
+    },
+    {
+      "corpusSize": 1000,
+      "encodeMs": {
+        "samples": 1000,
+        "p50": 0.114,
+        "p95": 0.187,
+        "p99": 0.865,
+        "min": 0.074,
+        "max": 1.378,
+        "mean": 0.139
+      },
+      "hybridRecallMs": {
+        "samples": 50,
+        "p50": 0.272,
+        "p95": 0.48,
+        "p99": 2.081,
+        "min": 0.254,
+        "max": 2.081,
+        "mean": 0.322
+      },
+      "postEncodeQueueMs": {
+        "samples": 1000,
+        "p50": 0.122,
+        "p95": 0.256,
+        "p99": 0.921,
+        "min": 0.035,
+        "max": 1.49,
+        "mean": 0.127
+      },
+      "queueEvents": 1000
+    },
+    {
+      "corpusSize": 5000,
+      "encodeMs": {
+        "samples": 5000,
+        "p50": 0.109,
+        "p95": 0.174,
+        "p99": 0.938,
+        "min": 0.07,
+        "max": 3.618,
+        "mean": 0.136
+      },
+      "hybridRecallMs": {
+        "samples": 50,
+        "p50": 0.735,
+        "p95": 0.867,
+        "p99": 4.228,
+        "min": 0.688,
+        "max": 4.228,
+        "mean": 0.816
+      },
+      "postEncodeQueueMs": {
+        "samples": 5000,
+        "p50": 0.116,
+        "p95": 0.242,
+        "p99": 0.978,
+        "min": 0.034,
+        "max": 6.272,
+        "mean": 0.124
+      },
+      "queueEvents": 5000
+    }
+  ]
+}

package/benchmarks/validate-adapter-module.mjs

@@ -0,0 +1,104 @@
+import { existsSync } from 'node:fs';
+import { basename, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { validateGuardBenchAdapter } from './guardbench.js';
+import { publicPath } from './public-paths.mjs';
+
+const DEFAULT_ADAPTER = 'benchmarks/adapters/example-allow.mjs';
+
+export function parseAdapterModuleValidatorArgs(argv = process.argv.slice(2)) {
+  const args = {
+    adapter: DEFAULT_ADAPTER,
+    json: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (token === '--adapter' && argv[i + 1]) args.adapter = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return args;
+}
+
+function usage() {
+  return `Usage: node benchmarks/validate-adapter-module.mjs [options]
+
+Options:
+  --adapter <path>   ESM GuardBench adapter module. Default: ${DEFAULT_ADAPTER}.
+  --json             Print the machine-readable validation report.
+`;
+}
+
+export async function validateAdapterModuleFile(options = {}) {
+  const adapterPath = resolve(options.adapter ?? DEFAULT_ADAPTER);
+  const failures = [];
+  let adapter = null;
+
+  if (!existsSync(adapterPath)) {
+    failures.push(`Adapter not found: ${adapterPath}`);
+  } else {
+    try {
+      const mod = await import(pathToFileURL(adapterPath).href);
+      const candidate = typeof mod.createGuardBenchAdapter === 'function'
+        ? await mod.createGuardBenchAdapter()
+        : mod.default ?? mod.adapter;
+      adapter = validateGuardBenchAdapter(candidate, adapterPath);
+    } catch (error) {
+      failures.push(error.message);
+    }
+  }
+
+  return {
+    ok: failures.length === 0,
+    adapterPath: publicPath(adapterPath),
+    moduleFile: basename(adapterPath),
+    adapter: adapter
+      ? {
+        name: adapter.name,
+        description: adapter.description ?? null,
+        hasSetup: typeof adapter.setup === 'function',
+        hasDecide: typeof adapter.decide === 'function',
+        hasCleanup: typeof adapter.cleanup === 'function',
+      }
+      : null,
+    contract: {
+      moduleFormat: 'ESM',
+      exports: ['default', 'adapter', 'createGuardBenchAdapter'],
+      requiredMethods: ['decide'],
+      optionalMethods: ['setup', 'cleanup'],
+    },
+    failures,
+  };
+}
+
+async function main() {
+  const args = parseAdapterModuleValidatorArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+
+  const validation = await validateAdapterModuleFile(args);
+  if (args.json) {
+    console.log(JSON.stringify(validation, null, 2));
+  } else if (validation.ok) {
+    console.log(`GuardBench adapter module validation passed: ${validation.adapterPath}`);
+    console.log(`Adapter: ${validation.adapter.name}`);
+    console.log(`Methods: setup=${validation.adapter.hasSetup}, decide=${validation.adapter.hasDecide}, cleanup=${validation.adapter.hasCleanup}`);
+  } else {
+    console.error('GuardBench adapter module validation failed:');
+    for (const failure of validation.failures) console.error(`- ${failure}`);
+  }
+
+  if (!validation.ok) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}

package/benchmarks/validate-adapter-registry.mjs

@@ -0,0 +1,134 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { validateSchema } from './validate-guardbench-artifacts.mjs';
+import { validateAdapterModuleFile } from './validate-adapter-module.mjs';
+import { publicPath } from './public-paths.mjs';
+
+const DEFAULT_REGISTRY = 'benchmarks/adapters/registry.json';
+const DEFAULT_SCHEMA = 'benchmarks/schemas/guardbench-adapter-registry.schema.json';
+
+export function parseAdapterRegistryValidatorArgs(argv = process.argv.slice(2)) {
+  const args = {
+    registry: DEFAULT_REGISTRY,
+    schema: DEFAULT_SCHEMA,
+    json: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (token === '--registry' && argv[i + 1]) args.registry = argv[++i];
+    else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return args;
+}
+
+function usage() {
+  return `Usage: node benchmarks/validate-adapter-registry.mjs [options]
+
+Options:
+  --registry <path>   Adapter registry JSON. Default: ${DEFAULT_REGISTRY}.
+  --schema <path>     Adapter registry JSON schema. Default: ${DEFAULT_SCHEMA}.
+  --json              Print the machine-readable validation report.
+`;
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+export async function validateAdapterRegistry(options = {}) {
+  const registryPath = resolve(options.registry ?? DEFAULT_REGISTRY);
+  const schemaPath = resolve(options.schema ?? DEFAULT_SCHEMA);
+  const failures = [];
+  let registry = null;
+
+  try {
+    registry = readJson(registryPath);
+  } catch (error) {
+    failures.push(error.message);
+  }
+
+  try {
+    const schema = readJson(schemaPath);
+    if (registry) failures.push(...validateSchema(registry, schema, 'guardbench-adapter-registry'));
+  } catch (error) {
+    failures.push(error.message);
+  }
+
+  const ids = new Set();
+  const adapterReports = [];
+  for (const adapter of registry?.adapters ?? []) {
+    if (ids.has(adapter.id)) failures.push(`Duplicate adapter id: ${adapter.id}`);
+    ids.add(adapter.id);
+    if (adapter.credentialMode === 'none' && adapter.requiredEnv.length !== 0) {
+      failures.push(`Adapter ${adapter.id} has credentialMode=none but declares requiredEnv`);
+    }
+    if (adapter.credentialMode === 'runtime-env' && adapter.requiredEnv.length === 0) {
+      failures.push(`Adapter ${adapter.id} has credentialMode=runtime-env but declares no requiredEnv`);
+    }
+    for (const [commandName, command] of Object.entries(adapter.commands ?? {})) {
+      if ((commandName === 'moduleValidate' || commandName === 'selfTest') && !command.includes(adapter.path)) {
+        failures.push(`Adapter ${adapter.id} command ${commandName} does not reference ${adapter.path}`);
+      }
+    }
+    if (!existsSync(resolve(adapter.path))) {
+      failures.push(`Adapter ${adapter.id} path does not exist: ${adapter.path}`);
+      continue;
+    }
+    const report = await validateAdapterModuleFile({ adapter: adapter.path });
+    adapterReports.push({
+      id: adapter.id,
+      ok: report.ok,
+      adapter: report.adapter,
+      credentialMode: adapter.credentialMode,
+      failures: report.failures,
+    });
+    if (!report.ok) {
+      failures.push(`Adapter ${adapter.id} failed module validation: ${report.failures.join('; ')}`);
+    }
+    if (report.adapter?.name && report.adapter.name !== adapter.name) {
+      failures.push(`Adapter ${adapter.id} registry name ${adapter.name} does not match module name ${report.adapter.name}`);
+    }
+  }
+
+  return {
+    ok: failures.length === 0,
+    registry: publicPath(registryPath),
+    schema: publicPath(schemaPath),
+    adapters: adapterReports,
+    failures,
+  };
+}
+
+async function main() {
+  const args = parseAdapterRegistryValidatorArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+
+  const validation = await validateAdapterRegistry(args);
+  if (args.json) {
+    console.log(JSON.stringify(validation, null, 2));
+  } else if (validation.ok) {
+    console.log(`GuardBench adapter registry validation passed: ${validation.registry}`);
+    console.log(`Adapters: ${validation.adapters.length}`);
+  } else {
+    console.error('GuardBench adapter registry validation failed:');
+    for (const failure of validation.failures) console.error(`- ${failure}`);
+  }
+
+  if (!validation.ok) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}

package/benchmarks/validate-adapter-self-test.mjs

@@ -0,0 +1,96 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { validateAdapterSelfTestReport } from './adapter-self-test.mjs';
+import { publicPath } from './public-paths.mjs';
+
+const DEFAULT_REPORT = 'benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json';
+
+export function parseAdapterSelfTestValidatorArgs(argv = process.argv.slice(2)) {
+  const args = {
+    report: DEFAULT_REPORT,
+    schema: undefined,
+    json: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if ((token === '--report' || token === '--file') && argv[i + 1]) args.report = argv[++i];
+    else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return args;
+}
+
+function usage() {
+  return `Usage: node benchmarks/validate-adapter-self-test.mjs [options]
+
+Options:
+  --report <path>   Adapter self-test JSON report. Default: ${DEFAULT_REPORT}.
+  --schema <path>   Optional alternate schema path.
+  --json            Print the machine-readable validation report.
+`;
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+export function validateAdapterSelfTestFile(options = {}) {
+  const reportPath = resolve(options.report ?? DEFAULT_REPORT);
+  const failures = [];
+  let report = null;
+
+  if (!existsSync(reportPath)) {
+    failures.push(`Missing adapter self-test report: ${reportPath}`);
+  } else {
+    try {
+      report = readJson(reportPath);
+      failures.push(...validateAdapterSelfTestReport(report, { schema: options.schema }));
+    } catch (error) {
+      failures.push(error.message);
+    }
+  }
+
+  return {
+    ok: failures.length === 0,
+    report: publicPath(reportPath),
+    adapter: report?.adapter?.name ?? null,
+    scenarios: report?.conformance?.scenarios ?? null,
+    expectedScenarios: report?.conformance?.expectedScenarios ?? null,
+    lowScoreAllowed: report?.contract?.lowScoreAllowed ?? null,
+    failures,
+  };
+}
+
+async function main() {
+  const args = parseAdapterSelfTestValidatorArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+
+  const validation = validateAdapterSelfTestFile(args);
+  if (args.json) {
+    console.log(JSON.stringify(validation, null, 2));
+  } else if (validation.ok) {
+    console.log(`GuardBench adapter self-test validation passed: ${validation.report}`);
+    console.log(`Adapter: ${validation.adapter}`);
+    console.log(`Rows: ${validation.scenarios}/${validation.expectedScenarios}`);
+  } else {
+    console.error('GuardBench adapter self-test validation failed:');
+    for (const failure of validation.failures) console.error(`- ${failure}`);
+  }
+
+  if (!validation.ok) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}