audrey 1.1.2 → 1.2.0

package/benchmarks/create-submission-bundle.mjs

@@ -1,190 +0,0 @@
-import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
-import { createHash } from 'node:crypto';
-import { basename, isAbsolute, join, relative, resolve } from 'node:path';
-import { writeGuardBenchConformanceCard } from './create-conformance-card.mjs';
-import { validateGuardBenchArtifacts } from './validate-guardbench-artifacts.mjs';
-import { publicPath } from './public-paths.mjs';
-
-const REQUIRED_ARTIFACTS = [
-  'guardbench-manifest.json',
-  'guardbench-summary.json',
-  'guardbench-raw.json',
-  'guardbench-conformance-card.json',
-];
-const OPTIONAL_ARTIFACTS = ['external-run-metadata.json'];
-const SCHEMA_FILES = [
-  'guardbench-adapter-registry.schema.json',
-  'guardbench-adapter-self-test.schema.json',
-  'guardbench-external-dry-run.schema.json',
-  'guardbench-external-evidence.schema.json',
-  'guardbench-publication-verification.schema.json',
-  'guardbench-manifest.schema.json',
-  'guardbench-summary.schema.json',
-  'guardbench-raw.schema.json',
-  'guardbench-external-run.schema.json',
-  'guardbench-conformance-card.schema.json',
-  'guardbench-leaderboard.schema.json',
-  'guardbench-submission-manifest.schema.json',
-];
-
-function readJson(path) {
-  return JSON.parse(readFileSync(path, 'utf-8'));
-}
-
-function sha256File(path) {
-  return createHash('sha256').update(readFileSync(path)).digest('hex');
-}
-
-function copyFileInto(sourceDir, outDir, file) {
-  const source = join(sourceDir, file);
-  if (!existsSync(source)) return null;
-  const target = join(outDir, file);
-  cpSync(source, target);
-  return target;
-}
-
-export function bundleRelativeFilePath(path, root) {
-  const relativePath = relative(root, path);
-  if (!relativePath || relativePath.startsWith('..') || isAbsolute(relativePath)) {
-    throw new Error(`Cannot add file outside GuardBench submission bundle: ${path}`);
-  }
-  return relativePath.replaceAll('\\', '/');
-}
-
-function fileRecord(path, root) {
-  return {
-    path: bundleRelativeFilePath(path, root),
-    bytes: readFileSync(path).byteLength,
-    sha256: sha256File(path),
-  };
-}
-
-export function writeGuardBenchSubmissionBundle(options = {}) {
-  const sourceDir = resolve(options.dir ?? 'benchmarks/output');
-  const outDir = resolve(options.outDir ?? join(sourceDir, 'submission-bundle'));
-  const schemasDir = resolve(options.schemasDir ?? 'benchmarks/schemas');
-
-  writeGuardBenchConformanceCard({ dir: sourceDir });
-  const sourceValidation = validateGuardBenchArtifacts({ dir: sourceDir, schemasDir });
-  if (!sourceValidation.ok) {
-    throw new Error(
-      `Cannot create GuardBench submission bundle from invalid artifacts: ${sourceValidation.failures.join('; ')}`,
-    );
-  }
-
-  rmSync(outDir, { recursive: true, force: true });
-  mkdirSync(join(outDir, 'schemas'), { recursive: true });
-
-  const copied = [];
-  for (const file of [...REQUIRED_ARTIFACTS, ...OPTIONAL_ARTIFACTS]) {
-    const target = copyFileInto(sourceDir, outDir, file);
-    if (target) copied.push(target);
-  }
-  for (const file of SCHEMA_FILES) {
-    const target = join(outDir, 'schemas', file);
-    cpSync(join(schemasDir, file), target);
-    copied.push(target);
-  }
-
-  const bundleValidation = validateGuardBenchArtifacts({
-    dir: outDir,
-    schemasDir: join(outDir, 'schemas'),
-  });
-  const validationReportPath = join(outDir, 'validation-report.json');
-  writeFileSync(
-    validationReportPath,
-    `${JSON.stringify(
-      {
-        generatedAt: new Date().toISOString(),
-        sourceValidation,
-        bundleValidation,
-      },
-      null,
-      2,
-    )}\n`,
-    'utf-8',
-  );
-  copied.push(validationReportPath);
-
-  const card = readJson(join(outDir, 'guardbench-conformance-card.json'));
-  const manifestPath = join(outDir, 'submission-manifest.json');
-  const manifest = {
-    schemaVersion: '1.0.0',
-    suite: 'GuardBench submission bundle',
-    generatedAt: new Date().toISOString(),
-    sourceDir: publicPath(sourceDir),
-    subject: card.subject,
-    score: card.score,
-    conformance: card.conformance,
-    validation: bundleValidation,
-    files: copied
-      .map(path => fileRecord(path, outDir))
-      .sort((a, b) => a.path.localeCompare(b.path)),
-  };
-  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
-
-  const finalFiles = copied
-    .map(path => fileRecord(path, outDir))
-    .sort((a, b) => a.path.localeCompare(b.path));
-  manifest.files = finalFiles;
-  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
-
-  return {
-    outDir,
-    manifestPath,
-    validation: bundleValidation,
-    subject: card.subject,
-    files: finalFiles.map(record => record.path),
-  };
-}
-
-function parseArgs(argv = process.argv.slice(2)) {
-  const args = {
-    dir: 'benchmarks/output',
-    outDir: null,
-    schemasDir: 'benchmarks/schemas',
-    json: false,
-  };
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if ((token === '--dir' || token === '--source-dir') && argv[i + 1]) args.dir = argv[++i];
-    else if (token === '--out-dir' && argv[i + 1]) args.outDir = argv[++i];
-    else if (token === '--schemas-dir' && argv[i + 1]) args.schemasDir = argv[++i];
-    else if (token === '--json') args.json = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-  return args;
-}
-
-function usage() {
-  return [
-    'Usage: node benchmarks/create-submission-bundle.mjs [--dir benchmarks/output] [--out-dir <dir>] [--json]',
-    '',
-    'Creates a portable GuardBench submission bundle containing artifacts, schemas,',
-    'a conformance card, validation report, and submission manifest.',
-  ].join('\n');
-}
-
-async function main() {
-  const args = parseArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-  const result = writeGuardBenchSubmissionBundle(args);
-  if (args.json) console.log(JSON.stringify(result, null, 2));
-  else {
-    console.log(`GuardBench submission bundle: ${result.outDir}`);
-    console.log(`Subject: ${result.subject.name}`);
-    console.log(`Validation: ${result.validation.ok ? 'passed' : 'failed'}`);
-  }
-  if (!result.validation.ok) process.exit(1);
-}
-
-if (process.argv[1] && resolve(process.argv[1]).endsWith(basename(import.meta.url))) {
-  main().catch(error => {
-    console.error(error.stack ?? error.message);
-    process.exit(1);
-  });
-}

package/benchmarks/dry-run-external-adapters.mjs

@@ -1,177 +0,0 @@
-import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
-import { dirname, join, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import {
-  buildExternalGuardBenchRun,
-  writeExternalRunMetadata,
-} from './run-external-guardbench.mjs';
-import { validateAdapterRegistry } from './validate-adapter-registry.mjs';
-import { validateSchema } from './validate-guardbench-artifacts.mjs';
-import { publicCommand, publicPath } from './public-paths.mjs';
-
-const DEFAULT_REGISTRY = 'benchmarks/adapters/registry.json';
-const DEFAULT_OUT_ROOT = 'benchmarks/output/external';
-const DEFAULT_OUT = 'benchmarks/output/external/guardbench-external-dry-run.json';
-const DEFAULT_SCHEMA = 'benchmarks/schemas/guardbench-external-dry-run.schema.json';
-
-export function parseExternalDryRunArgs(argv = process.argv.slice(2)) {
-  const args = {
-    registry: DEFAULT_REGISTRY,
-    outRoot: DEFAULT_OUT_ROOT,
-    out: DEFAULT_OUT,
-    includeCredentialFree: false,
-    json: false,
-    noWrite: false,
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if (token === '--registry' && argv[i + 1]) args.registry = argv[++i];
-    else if (token === '--out-root' && argv[i + 1]) args.outRoot = argv[++i];
-    else if (token === '--out' && argv[i + 1]) args.out = argv[++i];
-    else if (token === '--include-credential-free') args.includeCredentialFree = true;
-    else if (token === '--json') args.json = true;
-    else if (token === '--no-write') args.noWrite = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-
-  return args;
-}
-
-function usage() {
-  return `Usage: node benchmarks/dry-run-external-adapters.mjs [options]
-
-Options:
-  --registry <path>              Adapter registry JSON. Default: ${DEFAULT_REGISTRY}.
-  --out-root <path>              Root directory for dry-run metadata. Default: ${DEFAULT_OUT_ROOT}.
-  --out <path>                   Matrix JSON report path. Default: ${DEFAULT_OUT}.
-  --include-credential-free      Include credential-free registry adapters.
-  --json                         Print the machine-readable dry-run matrix.
-  --no-write                     Do not write the matrix JSON report.
-`;
-}
-
-function readJson(path) {
-  return JSON.parse(readFileSync(path, 'utf-8'));
-}
-
-export function validateExternalAdapterDryRunMatrix(matrix, options = {}) {
-  const schema = options.schemaObject ?? readJson(options.schema ?? DEFAULT_SCHEMA);
-  return validateSchema(matrix, schema, 'guardbench-external-dry-run');
-}
-
-export function writeExternalAdapterDryRunMatrix(matrix, out = DEFAULT_OUT) {
-  const outPath = resolve(out);
-  mkdirSync(dirname(outPath), { recursive: true });
-  writeFileSync(outPath, `${JSON.stringify(matrix, null, 2)}\n`, 'utf-8');
-  return outPath;
-}
-
-export async function buildExternalAdapterDryRunMatrix(options = {}) {
-  const registryPath = options.registry ?? DEFAULT_REGISTRY;
-  const outRoot = resolve(options.outRoot ?? DEFAULT_OUT_ROOT);
-  const registryValidation = await validateAdapterRegistry({ registry: registryPath });
-  const registry = readJson(registryPath);
-  const adapters = registry.adapters.filter(
-    adapter =>
-      adapter.status === 'external-system' &&
-      (options.includeCredentialFree || adapter.credentialMode === 'runtime-env'),
-  );
-  const rows = [];
-  const failures = [];
-
-  if (!registryValidation.ok) {
-    failures.push(...registryValidation.failures.map(failure => `registry: ${failure}`));
-  }
-
-  for (const adapter of adapters) {
-    const run = buildExternalGuardBenchRun(
-      {
-        adapter: adapter.id,
-        outDir: join(outRoot, adapter.id),
-        check: true,
-        json: true,
-      },
-      options.env ?? process.env,
-    );
-    const metadata = {
-      suite: 'GuardBench external adapter run',
-      startedAt: new Date().toISOString(),
-      adapter: run.adapter,
-      adapterPath: run.adapterPath,
-      outDir: run.outDir,
-      requiredEnv: run.requiredEnv,
-      missingEnv: run.missingEnv,
-      command: run.command,
-      validationCommand: run.validationCommand,
-      dryRun: true,
-      status: run.missingEnv.length ? 'dry-run-missing-env' : 'dry-run-ready',
-    };
-    const metadataPath = writeExternalRunMetadata(run.outDir, metadata);
-    rows.push({
-      id: adapter.id,
-      name: adapter.name,
-      credentialMode: adapter.credentialMode,
-      requiredEnv: run.requiredEnv,
-      missingEnv: run.missingEnv,
-      status: metadata.status,
-      command: publicCommand(run.command),
-      validationCommand: publicCommand(run.validationCommand),
-      metadataPath: publicPath(metadataPath),
-    });
-  }
-
-  const matrix = {
-    schemaVersion: '1.0.0',
-    suite: 'GuardBench external adapter dry-run matrix',
-    generatedAt: new Date().toISOString(),
-    ok: failures.length === 0,
-    registry: publicPath(resolve(registryPath)),
-    outRoot: publicPath(outRoot),
-    adapters: rows,
-    failures,
-  };
-  const schemaFailures = validateExternalAdapterDryRunMatrix(matrix, options);
-  if (schemaFailures.length > 0) {
-    throw new Error(
-      `GuardBench external adapter dry-run schema validation failed: ${schemaFailures.join('; ')}`,
-    );
-  }
-  return matrix;
-}
-
-async function main() {
-  const args = parseExternalDryRunArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-
-  const matrix = await buildExternalAdapterDryRunMatrix(args);
-  const outPath = args.noWrite ? null : writeExternalAdapterDryRunMatrix(matrix, args.out);
-  if (args.json) {
-    console.log(JSON.stringify(matrix, null, 2));
-  } else if (matrix.ok) {
-    console.log(
-      `GuardBench external adapter dry-run matrix passed: ${matrix.adapters.length} adapter(s)`,
-    );
-    for (const row of matrix.adapters) {
-      const missing = row.missingEnv.length ? `missing ${row.missingEnv.join(', ')}` : 'ready';
-      console.log(`- ${row.id}: ${missing}; metadata ${row.metadataPath}`);
-    }
-    if (outPath) console.log(`Matrix report: ${outPath}`);
-  } else {
-    console.error('GuardBench external adapter dry-run matrix failed:');
-    for (const failure of matrix.failures) console.error(`- ${failure}`);
-  }
-
-  if (!matrix.ok) process.exit(1);
-}
-
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
-  main().catch(error => {
-    console.error(error.stack ?? error.message);
-    process.exit(1);
-  });
-}