audrey 0.23.1 → 1.0.1

package/benchmarks/create-submission-bundle.mjs

@@ -0,0 +1,176 @@
+import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { basename, isAbsolute, join, relative, resolve } from 'node:path';
+import { writeGuardBenchConformanceCard } from './create-conformance-card.mjs';
+import { validateGuardBenchArtifacts } from './validate-guardbench-artifacts.mjs';
+import { publicPath } from './public-paths.mjs';
+
+const REQUIRED_ARTIFACTS = [
+  'guardbench-manifest.json',
+  'guardbench-summary.json',
+  'guardbench-raw.json',
+  'guardbench-conformance-card.json',
+];
+const OPTIONAL_ARTIFACTS = ['external-run-metadata.json'];
+const SCHEMA_FILES = [
+  'guardbench-adapter-registry.schema.json',
+  'guardbench-adapter-self-test.schema.json',
+  'guardbench-external-dry-run.schema.json',
+  'guardbench-external-evidence.schema.json',
+  'guardbench-publication-verification.schema.json',
+  'guardbench-manifest.schema.json',
+  'guardbench-summary.schema.json',
+  'guardbench-raw.schema.json',
+  'guardbench-external-run.schema.json',
+  'guardbench-conformance-card.schema.json',
+  'guardbench-leaderboard.schema.json',
+  'guardbench-submission-manifest.schema.json',
+];
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+function sha256File(path) {
+  return createHash('sha256').update(readFileSync(path)).digest('hex');
+}
+
+function copyFileInto(sourceDir, outDir, file) {
+  const source = join(sourceDir, file);
+  if (!existsSync(source)) return null;
+  const target = join(outDir, file);
+  cpSync(source, target);
+  return target;
+}
+
+export function bundleRelativeFilePath(path, root) {
+  const relativePath = relative(root, path);
+  if (!relativePath || relativePath.startsWith('..') || isAbsolute(relativePath)) {
+    throw new Error(`Cannot add file outside GuardBench submission bundle: ${path}`);
+  }
+  return relativePath.replaceAll('\\', '/');
+}
+
+function fileRecord(path, root) {
+  return {
+    path: bundleRelativeFilePath(path, root),
+    bytes: readFileSync(path).byteLength,
+    sha256: sha256File(path),
+  };
+}
+
+export function writeGuardBenchSubmissionBundle(options = {}) {
+  const sourceDir = resolve(options.dir ?? 'benchmarks/output');
+  const outDir = resolve(options.outDir ?? join(sourceDir, 'submission-bundle'));
+  const schemasDir = resolve(options.schemasDir ?? 'benchmarks/schemas');
+
+  writeGuardBenchConformanceCard({ dir: sourceDir });
+  const sourceValidation = validateGuardBenchArtifacts({ dir: sourceDir, schemasDir });
+  if (!sourceValidation.ok) {
+    throw new Error(`Cannot create GuardBench submission bundle from invalid artifacts: ${sourceValidation.failures.join('; ')}`);
+  }
+
+  rmSync(outDir, { recursive: true, force: true });
+  mkdirSync(join(outDir, 'schemas'), { recursive: true });
+
+  const copied = [];
+  for (const file of [...REQUIRED_ARTIFACTS, ...OPTIONAL_ARTIFACTS]) {
+    const target = copyFileInto(sourceDir, outDir, file);
+    if (target) copied.push(target);
+  }
+  for (const file of SCHEMA_FILES) {
+    const target = join(outDir, 'schemas', file);
+    cpSync(join(schemasDir, file), target);
+    copied.push(target);
+  }
+
+  const bundleValidation = validateGuardBenchArtifacts({
+    dir: outDir,
+    schemasDir: join(outDir, 'schemas'),
+  });
+  const validationReportPath = join(outDir, 'validation-report.json');
+  writeFileSync(validationReportPath, `${JSON.stringify({
+    generatedAt: new Date().toISOString(),
+    sourceValidation,
+    bundleValidation,
+  }, null, 2)}\n`, 'utf-8');
+  copied.push(validationReportPath);
+
+  const card = readJson(join(outDir, 'guardbench-conformance-card.json'));
+  const manifestPath = join(outDir, 'submission-manifest.json');
+  const manifest = {
+    schemaVersion: '1.0.0',
+    suite: 'GuardBench submission bundle',
+    generatedAt: new Date().toISOString(),
+    sourceDir: publicPath(sourceDir),
+    subject: card.subject,
+    score: card.score,
+    conformance: card.conformance,
+    validation: bundleValidation,
+    files: copied.map(path => fileRecord(path, outDir)).sort((a, b) => a.path.localeCompare(b.path)),
+  };
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
+
+  const finalFiles = copied.map(path => fileRecord(path, outDir)).sort((a, b) => a.path.localeCompare(b.path));
+  manifest.files = finalFiles;
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
+
+  return {
+    outDir,
+    manifestPath,
+    validation: bundleValidation,
+    subject: card.subject,
+    files: finalFiles.map(record => record.path),
+  };
+}
+
+function parseArgs(argv = process.argv.slice(2)) {
+  const args = {
+    dir: 'benchmarks/output',
+    outDir: null,
+    schemasDir: 'benchmarks/schemas',
+    json: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if ((token === '--dir' || token === '--source-dir') && argv[i + 1]) args.dir = argv[++i];
+    else if (token === '--out-dir' && argv[i + 1]) args.outDir = argv[++i];
+    else if (token === '--schemas-dir' && argv[i + 1]) args.schemasDir = argv[++i];
+    else if (token === '--json') args.json = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+  return args;
+}
+
+function usage() {
+  return [
+    'Usage: node benchmarks/create-submission-bundle.mjs [--dir benchmarks/output] [--out-dir <dir>] [--json]',
+    '',
+    'Creates a portable GuardBench submission bundle containing artifacts, schemas,',
+    'a conformance card, validation report, and submission manifest.',
+  ].join('\n');
+}
+
+async function main() {
+  const args = parseArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+  const result = writeGuardBenchSubmissionBundle(args);
+  if (args.json) console.log(JSON.stringify(result, null, 2));
+  else {
+    console.log(`GuardBench submission bundle: ${result.outDir}`);
+    console.log(`Subject: ${result.subject.name}`);
+    console.log(`Validation: ${result.validation.ok ? 'passed' : 'failed'}`);
+  }
+  if (!result.validation.ok) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]).endsWith(basename(import.meta.url))) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}

package/benchmarks/dry-run-external-adapters.mjs

@@ -0,0 +1,165 @@
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildExternalGuardBenchRun, writeExternalRunMetadata } from './run-external-guardbench.mjs';
+import { validateAdapterRegistry } from './validate-adapter-registry.mjs';
+import { validateSchema } from './validate-guardbench-artifacts.mjs';
+import { publicCommand, publicPath } from './public-paths.mjs';
+
+const DEFAULT_REGISTRY = 'benchmarks/adapters/registry.json';
+const DEFAULT_OUT_ROOT = 'benchmarks/output/external';
+const DEFAULT_OUT = 'benchmarks/output/external/guardbench-external-dry-run.json';
+const DEFAULT_SCHEMA = 'benchmarks/schemas/guardbench-external-dry-run.schema.json';
+
+export function parseExternalDryRunArgs(argv = process.argv.slice(2)) {
+  const args = {
+    registry: DEFAULT_REGISTRY,
+    outRoot: DEFAULT_OUT_ROOT,
+    out: DEFAULT_OUT,
+    includeCredentialFree: false,
+    json: false,
+    noWrite: false,
+  };
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (token === '--registry' && argv[i + 1]) args.registry = argv[++i];
+    else if (token === '--out-root' && argv[i + 1]) args.outRoot = argv[++i];
+    else if (token === '--out' && argv[i + 1]) args.out = argv[++i];
+    else if (token === '--include-credential-free') args.includeCredentialFree = true;
+    else if (token === '--json') args.json = true;
+    else if (token === '--no-write') args.noWrite = true;
+    else if (token === '--help' || token === '-h') args.help = true;
+    else throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return args;
+}
+
+function usage() {
+  return `Usage: node benchmarks/dry-run-external-adapters.mjs [options]
+
+Options:
+  --registry <path>              Adapter registry JSON. Default: ${DEFAULT_REGISTRY}.
+  --out-root <path>              Root directory for dry-run metadata. Default: ${DEFAULT_OUT_ROOT}.
+  --out <path>                   Matrix JSON report path. Default: ${DEFAULT_OUT}.
+  --include-credential-free      Include credential-free registry adapters.
+  --json                         Print the machine-readable dry-run matrix.
+  --no-write                     Do not write the matrix JSON report.
+`;
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, 'utf-8'));
+}
+
+export function validateExternalAdapterDryRunMatrix(matrix, options = {}) {
+  const schema = options.schemaObject ?? readJson(options.schema ?? DEFAULT_SCHEMA);
+  return validateSchema(matrix, schema, 'guardbench-external-dry-run');
+}
+
+export function writeExternalAdapterDryRunMatrix(matrix, out = DEFAULT_OUT) {
+  const outPath = resolve(out);
+  mkdirSync(dirname(outPath), { recursive: true });
+  writeFileSync(outPath, `${JSON.stringify(matrix, null, 2)}\n`, 'utf-8');
+  return outPath;
+}
+
+export async function buildExternalAdapterDryRunMatrix(options = {}) {
+  const registryPath = options.registry ?? DEFAULT_REGISTRY;
+  const outRoot = resolve(options.outRoot ?? DEFAULT_OUT_ROOT);
+  const registryValidation = await validateAdapterRegistry({ registry: registryPath });
+  const registry = readJson(registryPath);
+  const adapters = registry.adapters.filter(adapter =>
+    adapter.status === 'external-system'
+    && (options.includeCredentialFree || adapter.credentialMode === 'runtime-env'));
+  const rows = [];
+  const failures = [];
+
+  if (!registryValidation.ok) {
+    failures.push(...registryValidation.failures.map(failure => `registry: ${failure}`));
+  }
+
+  for (const adapter of adapters) {
+    const run = buildExternalGuardBenchRun({
+      adapter: adapter.id,
+      outDir: join(outRoot, adapter.id),
+      check: true,
+      json: true,
+    }, options.env ?? process.env);
+    const metadata = {
+      suite: 'GuardBench external adapter run',
+      startedAt: new Date().toISOString(),
+      adapter: run.adapter,
+      adapterPath: run.adapterPath,
+      outDir: run.outDir,
+      requiredEnv: run.requiredEnv,
+      missingEnv: run.missingEnv,
+      command: run.command,
+      validationCommand: run.validationCommand,
+      dryRun: true,
+      status: run.missingEnv.length ? 'dry-run-missing-env' : 'dry-run-ready',
+    };
+    const metadataPath = writeExternalRunMetadata(run.outDir, metadata);
+    rows.push({
+      id: adapter.id,
+      name: adapter.name,
+      credentialMode: adapter.credentialMode,
+      requiredEnv: run.requiredEnv,
+      missingEnv: run.missingEnv,
+      status: metadata.status,
+      command: publicCommand(run.command),
+      validationCommand: publicCommand(run.validationCommand),
+      metadataPath: publicPath(metadataPath),
+    });
+  }
+
+  const matrix = {
+    schemaVersion: '1.0.0',
+    suite: 'GuardBench external adapter dry-run matrix',
+    generatedAt: new Date().toISOString(),
+    ok: failures.length === 0,
+    registry: publicPath(resolve(registryPath)),
+    outRoot: publicPath(outRoot),
+    adapters: rows,
+    failures,
+  };
+  const schemaFailures = validateExternalAdapterDryRunMatrix(matrix, options);
+  if (schemaFailures.length > 0) {
+    throw new Error(`GuardBench external adapter dry-run schema validation failed: ${schemaFailures.join('; ')}`);
+  }
+  return matrix;
+}
+
+async function main() {
+  const args = parseExternalDryRunArgs();
+  if (args.help) {
+    console.log(usage());
+    return;
+  }
+
+  const matrix = await buildExternalAdapterDryRunMatrix(args);
+  const outPath = args.noWrite ? null : writeExternalAdapterDryRunMatrix(matrix, args.out);
+  if (args.json) {
+    console.log(JSON.stringify(matrix, null, 2));
+  } else if (matrix.ok) {
+    console.log(`GuardBench external adapter dry-run matrix passed: ${matrix.adapters.length} adapter(s)`);
+    for (const row of matrix.adapters) {
+      const missing = row.missingEnv.length ? `missing ${row.missingEnv.join(', ')}` : 'ready';
+      console.log(`- ${row.id}: ${missing}; metadata ${row.metadataPath}`);
+    }
+    if (outPath) console.log(`Matrix report: ${outPath}`);
+  } else {
+    console.error('GuardBench external adapter dry-run matrix failed:');
+    for (const failure of matrix.failures) console.error(`- ${failure}`);
+  }
+
+  if (!matrix.ok) process.exit(1);
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main().catch(error => {
+    console.error(error.stack ?? error.message);
+    process.exit(1);
+  });
+}