@pharmatools/opengate 0.1.0

package/src/runner.mjs

@@ -0,0 +1,223 @@
+#!/usr/bin/env node
+// OpenGATE eval runner.
+//
+//   node src/runner.mjs            # offline scorers only
+//   node src/runner.mjs --online   # also run scorers that hit the API
+//   node src/runner.mjs --ci       # exit non-zero on failure or regression
+//   node src/runner.mjs --baseline # save this run as results/baseline.json
+//
+// Discovers gold cases (datasets/cases/*.json) and fixtures (datasets/fixtures/*.json),
+// runs each scorer, prints a summary, writes a timestamped results file, and
+// compares headline metrics against results/baseline.json to catch regressions.
+
+import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join, resolve } from 'node:path';
+import { execSync } from 'node:child_process';
+import { loadAdapter } from './lib/adapter.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EVAL_ROOT = join(__dirname, '..');
+
+const argv = process.argv.slice(2);
+const args = new Set(argv.filter(a => a.startsWith('-')));
+const ONLINE = args.has('--online');
+const CI = args.has('--ci');
+const SAVE_BASELINE = args.has('--baseline');
+
+// Value-taking flag: `--name <value>`, falling back to an env var.
+function argVal(flag, envName) {
+  const i = argv.indexOf(flag);
+  if (i < 0) return process.env[envName] || undefined;
+  const v = argv[i + 1];
+  if (!v || v.startsWith('-')) {
+    console.error(`${flag} requires a value, e.g. ${flag} <path>`);
+    process.exit(2);
+  }
+  return v;
+}
+
+// --adapter <path> overrides the OPENGATE_ADAPTER environment variable.
+const ADAPTER_SPEC = argVal('--adapter', 'OPENGATE_ADAPTER');
+
+// --datasets <dir> points at a directory containing cases/ and fixtures/ —
+// so third-party repos can keep their gold sets in their own tree.
+// --results <dir> relocates snapshots and baseline.json for the same reason.
+const DATASETS_SPEC = argVal('--datasets', 'OPENGATE_DATASETS');
+const RESULTS_SPEC = argVal('--results', 'OPENGATE_RESULTS');
+const DATA_ROOT = DATASETS_SPEC ? resolve(process.cwd(), DATASETS_SPEC) : join(EVAL_ROOT, 'datasets');
+const CASES_DIR = join(DATA_ROOT, 'cases');
+const FIX_DIR = join(DATA_ROOT, 'fixtures');
+// When OpenGATE runs from an installed package (npx / node_modules), snapshots
+// must not be written inside the package — default to ./opengate-results in
+// the caller's working directory instead.
+const RESULTS_DIR = RESULTS_SPEC
+  ? resolve(process.cwd(), RESULTS_SPEC)
+  : (EVAL_ROOT.includes('node_modules')
+      ? join(process.cwd(), 'opengate-results')
+      : join(EVAL_ROOT, 'results'));
+
+if (args.has('--help') || args.has('-h')) {
+  console.log(`OpenGATE — Open-source evaluation for evidence-grounded AI
+
+Usage: opengate [options]        (or: node src/runner.mjs [options])
+
+Options:
+  --online           also run scorers that call the live system
+  --baseline         save this run as results/baseline.json
+  --ci               exit non-zero on any failure or metric regression
+  --adapter <path>   adapter module (overrides OPENGATE_ADAPTER;
+                     default: bundled RefCheckr reference adapter)
+  --datasets <dir>   directory containing cases/ and fixtures/
+                     (overrides OPENGATE_DATASETS; default: bundled datasets/)
+  --results <dir>    where to write snapshots and baseline.json
+                     (overrides OPENGATE_RESULTS; default: bundled results/)
+  -h, --help         show this help
+  -v, --version      show version
+
+Environment:
+  OPENGATE_ADAPTER        adapter module path
+  OPENGATE_DATASETS       datasets directory (cases/ + fixtures/)
+  OPENGATE_RESULTS        results directory
+  OPENGATE_HTTP_CONFIG    config path for the generic HTTP adapter
+  OPENGATE_EVAL_REPEATS   run each verdict pair N times (consistency)
+  OPENGATE_EVAL_MODEL     label the deployment's model in the scorecard
+
+Docs: README.md · ADAPTERS.md`);
+  process.exit(0);
+}
+
+if (args.has('--version') || args.has('-v')) {
+  const pkg = JSON.parse(await readFile(join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json'), 'utf8'));
+  console.log(pkg.version);
+  process.exit(0);
+}
+
+const SCORERS = [
+  './scorers/citation-detection.mjs',
+  './scorers/claim-extraction.mjs',
+  './scorers/verdict-accuracy.mjs',
+];
+
+async function loadJsonDir(dir, { skipPrefix } = {}) {
+  if (!existsSync(dir)) return {};
+  const out = {};
+  for (const f of await readdir(dir)) {
+    if (!f.endsWith('.json')) continue;
+    if (skipPrefix && f.startsWith(skipPrefix)) continue;
+    const key = f.replace(/\.json$/, '');
+    out[key] = JSON.parse(await readFile(join(dir, f), 'utf8'));
+  }
+  return out;
+}
+
+function gitSha() {
+  try { return execSync('git rev-parse --short HEAD', { cwd: EVAL_ROOT }).toString().trim(); }
+  catch { return 'unknown'; }
+}
+
+function fmtPct(x) { return (x * 100).toFixed(1) + '%'; }
+// Percent-format rates/scores; show counts (n_pairs, totalLeaked, repeats, …) raw.
+function isRateKey(k) { return /(rate|accuracy|precision|recall|f1|jaccard|consistency|adjacency|exact|share|score)/i.test(k); }
+
+async function main() {
+  const casesMap = await loadJsonDir(CASES_DIR, { skipPrefix: '_' });
+  const cases = Object.values(casesMap);
+  const fixtures = await loadJsonDir(FIX_DIR);
+
+  if (cases.length === 0) {
+    console.error('No gold cases found in datasets/cases/. Add one (copy _template.json).');
+    process.exit(2);
+  }
+
+  // Load and validate the adapter up front — fail fast on a malformed one,
+  // even for offline runs, so a broken config never produces a partial scorecard.
+  let adapter;
+  try {
+    adapter = await loadAdapter(ADAPTER_SPEC);
+  } catch (err) {
+    console.error(err.message);
+    process.exit(2);
+  }
+
+  console.log(`\nOpenGATE — ${cases.length} case(s), online=${ONLINE}, adapter=${adapter.name}, sha=${gitSha()}\n`);
+
+  const results = [];
+  for (const path of SCORERS) {
+    const mod = await import(path);
+    const isOnline = mod.meta?.mode === 'online';
+    if (isOnline && !ONLINE) {
+      results.push({ id: mod.meta.id, skipped: true, reason: 'online scorer (pass --online)' });
+      continue;
+    }
+    const r = await mod.run({ cases, fixtures, adapter });
+    results.push({ id: mod.meta.id, ...r });
+  }
+
+  // ── Summary table ──
+  for (const r of results) {
+    if (r.skipped) {
+      console.log(`  ⊘ ${r.id.padEnd(20)} SKIPPED — ${r.reason}`);
+      continue;
+    }
+    const status = r.passed ? '✓' : '✗';
+    console.log(`  ${status} ${r.id.padEnd(20)} ${r.passed ? 'PASS' : 'FAIL'}`);
+    for (const [k, v] of Object.entries(r.metrics || {})) {
+      const val = typeof v === 'number' ? (isRateKey(k) ? fmtPct(v) : String(v)) : JSON.stringify(v);
+      console.log(`      ${k.padEnd(26)} ${val}`);
+    }
+    for (const f of r.failures || []) console.log(`      ✗ ${f}`);
+  }
+
+  // ── Persist ──
+  await mkdir(RESULTS_DIR, { recursive: true });
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const snapshot = {
+    timestamp: new Date().toISOString(),
+    sha: gitSha(),
+    online: ONLINE,
+    adapter: adapter.name,
+    results: results.map(({ detail, ...rest }) => rest), // drop bulky detail from snapshot
+  };
+  await writeFile(join(RESULTS_DIR, `${stamp}.json`), JSON.stringify({ ...snapshot, results }, null, 2));
+  if (SAVE_BASELINE) {
+    await writeFile(join(RESULTS_DIR, 'baseline.json'), JSON.stringify(snapshot, null, 2));
+    console.log('\n  saved baseline.json');
+  }
+
+  // ── Regression check vs baseline ──
+  let regressed = false;
+  const baselinePath = join(RESULTS_DIR, 'baseline.json');
+  if (existsSync(baselinePath) && !SAVE_BASELINE) {
+    const base = JSON.parse(await readFile(baselinePath, 'utf8'));
+    const baseById = Object.fromEntries(base.results.map(r => [r.id, r]));
+    console.log('\n  vs baseline:');
+    for (const r of results) {
+      const b = baseById[r.id];
+      if (!b || !r.metrics || !b.metrics) continue;
+      for (const [k, v] of Object.entries(r.metrics)) {
+        if (typeof v !== 'number' || typeof b.metrics[k] !== 'number') continue;
+        const delta = v - b.metrics[k];
+        if (Math.abs(delta) < 1e-9) continue;
+        const arrow = delta > 0 ? '▲' : '▼';
+        // Only rate metrics gate regressions (higher = better); counts shown for info.
+        if (isRateKey(k)) {
+          if (delta < -1e-9) regressed = true;
+          console.log(`      ${r.id}.${k}: ${arrow} ${delta > 0 ? '+' : ''}${(delta * 100).toFixed(1)}pp`);
+        } else {
+          console.log(`      ${r.id}.${k}: ${arrow} ${delta > 0 ? '+' : ''}${delta}`);
+        }
+      }
+    }
+  }
+
+  const anyFail = results.some(r => !r.skipped && r.passed === false);
+  console.log('');
+  if (CI && (anyFail || regressed)) {
+    console.error(`CI gate failed (failures=${anyFail}, regression=${regressed}).`);
+    process.exit(1);
+  }
+}
+
+main().catch(err => { console.error(err); process.exit(2); });

package/src/scorers/citation-detection.mjs

@@ -0,0 +1,69 @@
+// OFFLINE scorer — runs with no API key.
+// Measures the deterministic citation pipeline two ways:
+//   1. Per-claim detection: feed each gold claim's originalText (with markers)
+//      through normalize+parse, compare detected citation set to gold.
+//   2. Style coverage: run the citation-styles fixture, report supported-style
+//      accuracy and surface known-gap styles (e.g. author-year) as a tracked
+//      target rather than a failure.
+
+import { detectCitations } from '../lib/citations.mjs';
+import { exactSetMatch, jaccard, mean } from '../lib/metrics.mjs';
+
+export const meta = { id: 'citation-detection', mode: 'offline' };
+
+export function run({ cases, fixtures }) {
+  // ── 1. Per-claim citation detection across all cases ──
+  const perClaim = [];
+  for (const c of cases) {
+    for (const claim of c.goldClaims || []) {
+      const detected = detectCitations(claim.originalText);
+      perClaim.push({
+        case: c.id,
+        claim: claim.text.slice(0, 60),
+        gold: claim.citations,
+        detected,
+        exact: exactSetMatch(detected, claim.citations),
+        jaccard: jaccard(detected, claim.citations),
+      });
+    }
+  }
+  const exactRate = perClaim.length ? perClaim.filter(p => p.exact).length / perClaim.length : 1;
+  const jaccardMean = mean(perClaim.map(p => p.jaccard));
+
+  // ── 2. Style coverage fixture ──
+  const styles = (fixtures['citation-styles']?.items) || [];
+  const supported = styles.filter(s => !s.knownGap);
+  const knownGaps = styles.filter(s => s.knownGap);
+  const supportedResults = supported.map(s => ({
+    style: s.style,
+    pass: exactSetMatch(detectCitations(s.text), s.expected),
+  }));
+  const supportedAccuracy = supportedResults.length
+    ? supportedResults.filter(r => r.pass).length / supportedResults.length : 1;
+  // For known-gap styles, "still a gap" = we detect nothing where a target exists.
+  const gapResults = knownGaps.map(s => ({
+    style: s.style,
+    stillUnsupported: detectCitations(s.text).length === 0,
+    target: s.targetExpected || [],
+  }));
+
+  const failures = [
+    ...perClaim.filter(p => !p.exact).map(p => `claim "${p.claim}…" expected [${p.gold}] got [${p.detected}]`),
+    ...supportedResults.filter(r => !r.pass).map(r => `style ${r.style} failed`),
+  ];
+
+  return {
+    meta,
+    metrics: {
+      perClaim_exactSetRate: round(exactRate),
+      perClaim_jaccardMean: round(jaccardMean),
+      supportedStyle_accuracy: round(supportedAccuracy),
+      knownGap_styles: knownGaps.map(s => s.style),
+    },
+    detail: { perClaim, supportedResults, gapResults },
+    failures,
+    passed: failures.length === 0,
+  };
+}
+
+function round(x) { return Math.round(x * 1000) / 1000; }

package/src/scorers/claim-extraction.mjs

@@ -0,0 +1,75 @@
+// ONLINE scorer — exercises the adapter's splitClaims().
+// Measures whether the splitter extracts the right verifiable claims:
+//   • recall   — did it find the gold claims?
+//   • precision — did it avoid pulling in non-claims (background/aims)?
+//   • citation agreement on the claims it did extract
+//   • fidelity  — is each extracted claim verbatim-ish from the manuscript?
+
+import { precisionRecallF1, claimMatch, jaccard, normText, mean } from '../lib/metrics.mjs';
+
+export const meta = { id: 'claim-extraction', mode: 'online' };
+
+export async function run({ cases, adapter }) {
+  if (!adapter.onlineAvailable()) {
+    return { meta, skipped: true, reason: adapter.onlineConfigHint() };
+  }
+
+  const perCase = [];
+  const splitErrors = [];
+  for (const c of cases) {
+   try {
+    const resp = await adapter.splitClaims(c.manuscript);
+    const extracted = (resp.claims || []).map(x => (typeof x === 'string' ? { text: x, citations: [] } : x));
+    const goldTexts = (c.goldClaims || []).map(g => g.text);
+
+    // Precision / recall / F1 of extracted claims vs gold claims.
+    const prf = precisionRecallF1(extracted.map(e => e.text), goldTexts, claimMatch);
+
+    // Leakage: any extracted claim that matches a known non-claim is a false positive.
+    const leaked = extracted.filter(e => (c.goldNonClaims || []).some(nc => claimMatch(e.text, nc)));
+
+    // Citation agreement on matched claims.
+    const citJaccards = [];
+    for (const g of c.goldClaims || []) {
+      const m = extracted.find(e => claimMatch(e.text, g.text));
+      if (m) citJaccards.push(jaccard(m.citations || [], g.citations || []));
+    }
+
+    // Fidelity: extracted claim text should appear (normalised) in the manuscript.
+    const src = normText(c.manuscript);
+    const infidelities = extracted.filter(e => {
+      const t = normText(e.text);
+      return t.length >= 12 && !src.includes(t);
+    });
+
+    perCase.push({
+      case: c.id,
+      precision: prf.precision, recall: prf.recall, f1: prf.f1,
+      leakedCount: leaked.length,
+      citationJaccardMean: mean(citJaccards),
+      infidelityCount: infidelities.length,
+      infidelities: infidelities.map(e => e.text.slice(0, 80)),
+    });
+   } catch (err) {
+    splitErrors.push(`case ${c.id}: ${err.message}`);
+   }
+  }
+
+  const agg = {
+    f1: round(mean(perCase.map(p => p.f1))),
+    precision: round(mean(perCase.map(p => p.precision))),
+    recall: round(mean(perCase.map(p => p.recall))),
+    citationJaccardMean: round(mean(perCase.map(p => p.citationJaccardMean))),
+    totalLeaked: perCase.reduce((a, p) => a + p.leakedCount, 0),
+    totalInfidelities: perCase.reduce((a, p) => a + p.infidelityCount, 0),
+  };
+  const failures = [...splitErrors];
+  if (agg.totalInfidelities > 0) failures.push(`${agg.totalInfidelities} extracted claim(s) not verbatim from source`);
+  if (agg.totalLeaked > 0) failures.push(`${agg.totalLeaked} non-claim(s) leaked into extraction`);
+  agg.casesEvaluated = perCase.length;
+  agg.splitErrors = splitErrors.length;
+
+  return { meta, metrics: agg, detail: { perCase }, failures, passed: failures.length === 0 };
+}
+
+function round(x) { return Math.round(x * 1000) / 1000; }

package/src/scorers/verdict-accuracy.mjs

@@ -0,0 +1,163 @@
+// ONLINE scorer — verdict accuracy + passage hallucination + consistency.
+//
+// Verifies ONE claim per call against its single cited reference (sent as a text
+// document) via the adapter's analyzeBatch(). One-claim-per-call deliberately
+// sidesteps multi-claim reassembly bugs in batch endpoints, so this measures the
+// model's verdict QUALITY, not batch-matching behaviour.
+//
+// Metrics:
+//   • exact / adjacency verdict accuracy on the six-point support scale
+//   • confusion matrix (gold vs predicted)
+//   • passage hallucination rate (cited quotes not found verbatim in the source)
+//   • guard_downgrades (verdicts the integrity guard downgraded)
+//   • consistency (when OPENGATE_EVAL_REPEATS > 1): verdict stability across repeats
+
+import { verdictAccuracy, confusionMatrix, normText, mean, percentile } from '../lib/metrics.mjs';
+
+export const meta = { id: 'verdict-accuracy', mode: 'online' };
+
+const REPEATS = Math.max(1, parseInt(process.env.OPENGATE_EVAL_REPEATS || process.env.REFCHECKR_EVAL_REPEATS || '1', 10));
+
+// One claim against its one cited reference.
+function buildPairPayload(caseId, claimText, citation, ref) {
+  return {
+    claims: [claimText],
+    documents: [{ name: ref.name, type: 'text', content: ref.text }],
+    citationMapping: { [citation]: { refId: 'upload:' + ref.name } },
+    claimCitations: [{ citations: [citation] }],
+    document_name: `eval-${caseId}`,
+  };
+}
+
+function readVerdict(resp, refName) {
+  const r = (resp.claims || [])[0];
+  if (!r) return { verdict: undefined, analysis: null };
+  const a = (r.individual_analyses || []).find(x => x.document_name === refName)
+    || (r.individual_analyses || [])[0];
+  return { verdict: a?.verdict, analysis: a || null };
+}
+
+// Quote text from a passage object, tolerant of field naming.
+function passageQuote(p) {
+  return p?.quote ?? p?.text ?? p?.passage ?? p?.snippet ?? '';
+}
+
+// enforceVerdictIntegrity() prepends a fixed reason to the summary when it
+// downgrades a verdict; detecting it tells us the guard fired vs the model judged.
+function isGuardDowngrade(summary) {
+  if (!summary) return false;
+  const s = String(summary).toLowerCase();
+  return s.includes('could not be verified against the document')
+    || s.includes('no explicit textual statement was found')
+    || s.includes('no explicit textual support found');
+}
+
+export async function run({ cases, adapter }) {
+  if (!adapter.onlineAvailable()) {
+    return { meta, skipped: true, reason: adapter.onlineConfigHint() };
+  }
+  const goldCases = cases.filter(c => (c.goldVerdicts || []).length > 0 && c.references);
+  if (goldCases.length === 0) {
+    return { meta, skipped: true, reason: 'No cases include goldVerdicts + references.' };
+  }
+
+  adapter.resetTiming(); // start latency capture for the verify calls below
+  adapter.resetTokens();  // start token-usage capture for real cost/claim
+
+  const pairs = [];
+  const consistencyShares = [];
+  let passageTotal = 0, passageInfidel = 0;
+  const failures = [];
+
+  for (const c of goldCases) {
+    for (const g of c.goldVerdicts) {
+      const ref = c.references[g.citation];
+      if (!ref) { failures.push(`missing reference ${g.citation} in ${c.id}`); continue; }
+      const payload = buildPairPayload(c.id, g.claimText, g.citation, ref);
+
+      const runVerdicts = [];
+      let firstAnalysis = null, firstVerdict;
+      for (let run = 0; run < REPEATS; run++) {
+        let resp;
+        try {
+          resp = await adapter.analyzeBatch(payload);
+        } catch (err) {
+          failures.push(`${c.id} "${g.claimText.slice(0, 30)}…" run ${run + 1}: ${err.message}`);
+          break;
+        }
+        const { verdict, analysis } = readVerdict(resp, ref.name);
+        runVerdicts.push(verdict);
+        if (run === 0) { firstAnalysis = analysis; firstVerdict = verdict; }
+      }
+
+      if (firstVerdict) {
+        pairs.push({
+          case: c.id,
+          claim: g.claimText.slice(0, 45),
+          predicted: firstVerdict,
+          gold: g.verdict,
+          downgraded: isGuardDowngrade(firstAnalysis?.summary),
+          summary: String(firstAnalysis?.summary || '').slice(0, 200),
+          passageCount: (firstAnalysis?.passages || []).length,
+        });
+      } else {
+        failures.push(`no prediction for "${g.claimText.slice(0, 40)}…" in ${c.id}`);
+      }
+
+      // Passage hallucination (first run): cited quote should appear in the source.
+      const refText = normText(ref.text || '');
+      for (const p of (firstAnalysis?.passages || [])) {
+        const q = normText(passageQuote(p));
+        if (q.length < 12) continue;
+        passageTotal++;
+        if (!refText.includes(q.slice(0, Math.min(q.length, 60)))) passageInfidel++;
+      }
+
+      // Consistency across repeats for this pair (modal-verdict share).
+      if (REPEATS > 1 && runVerdicts.length) {
+        const counts = {};
+        for (const v of runVerdicts) counts[v] = (counts[v] || 0) + 1;
+        consistencyShares.push(Math.max(...Object.values(counts)) / runVerdicts.length);
+      }
+    }
+  }
+
+  const acc = verdictAccuracy(pairs);
+  const cm = confusionMatrix(pairs);
+  const hallucinationRate = passageTotal ? passageInfidel / passageTotal : 0;
+  const consistency = consistencyShares.length ? mean(consistencyShares) : null;
+  const downgradedPairs = pairs.filter(p => p.downgraded);
+
+  // Per-verify-call latency (one call verifies one claim against its reference,
+  // so this is p50/p95 latency per claim). Counts as info in the snapshot — not
+  // a pass/fail gate.
+  const verifyMs = adapter.callLatencies();
+  const tok = adapter.tokenTotals();
+  const model = adapter.runModel();
+
+  const metrics = {
+    n_pairs: pairs.length,
+    verdict_exact: round(acc.exact),
+    verdict_adjacency: round(acc.adjacent),
+    hallucination_rate: round(hallucinationRate),
+    guard_downgrades: downgradedPairs.length,
+    ...(consistency != null ? { consistency: round(consistency), repeats: REPEATS } : {}),
+    ...(verifyMs.length ? {
+      latency_p50_ms: Math.round(percentile(verifyMs, 50)),
+      latency_p95_ms: Math.round(percentile(verifyMs, 95)),
+      latency_calls: verifyMs.length,
+    } : {}),
+    ...(tok.calls ? {
+      avg_input_tokens: Math.round(tok.prompt_tokens / tok.calls),
+      avg_output_tokens: Math.round(tok.completion_tokens / tok.calls),
+      avg_reasoning_tokens: Math.round(tok.reasoning_tokens / tok.calls),
+      avg_total_tokens: Math.round(tok.total_tokens / tok.calls),
+      token_calls: tok.calls,
+    } : {}),
+    ...(model ? { run_model: model } : {}),
+  };
+
+  return { meta, metrics, detail: { confusion: cm, pairs }, failures, passed: failures.length === 0 };
+}
+
+function round(x) { return Math.round(x * 1000) / 1000; }