@pharmatools/opengate 0.1.0

package/src/adapters/http.mjs

@@ -0,0 +1,132 @@
+// Generic HTTP adapter — evaluate any REST-backed system without writing code.
+//
+// Point OpenGATE at a JSON config describing your endpoints:
+//
+//   OPENGATE_ADAPTER=./src/adapters/http.mjs npm run eval:online
+//
+// Config is read from OPENGATE_HTTP_CONFIG (default: ./opengate.http.json,
+// resolved from the current working directory):
+//
+//   {
+//     "name": "my-system",
+//     "baseUrl": "${MY_SYSTEM_URL}",
+//     "headers": { "Authorization": "Bearer ${MY_SYSTEM_TOKEN}" },
+//     "endpoints": {
+//       "splitClaims": "/api/claims/split",
+//       "analyzeBatch": "/api/verify/batch"
+//     },
+//     "modelEnv": "MY_SYSTEM_MODEL"
+//   }
+//
+// ${VAR} placeholders are interpolated from the environment at load time.
+// The endpoints must speak the OpenGATE contract shapes (see ADAPTERS.md):
+// if your API uses different shapes, write a thin code adapter instead —
+// this adapter maps transport, not payloads.
+//
+// Latency and token capture are built in: every call's wall-clock time is
+// recorded, and any `usage` block in a response is aggregated for cost/claim.
+
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+// ── Config ──────────────────────────────────────────────────────────────
+const CONFIG_PATH = resolve(process.cwd(), process.env.OPENGATE_HTTP_CONFIG || 'opengate.http.json');
+
+function interpolate(value, missing) {
+  if (typeof value === 'string') {
+    return value.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
+      const v = process.env[name];
+      if (v == null || v === '') { missing.add(name); return ''; }
+      return v;
+    });
+  }
+  if (Array.isArray(value)) return value.map(v => interpolate(v, missing));
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, interpolate(v, missing)]));
+  }
+  return value;
+}
+
+let config = null;
+let configError = null;
+const missingEnv = new Set();
+try {
+  const raw = JSON.parse(readFileSync(CONFIG_PATH, 'utf8'));
+  config = interpolate(raw, missingEnv);
+  if (!config.baseUrl) configError = 'config has no baseUrl';
+  else if (!config.endpoints?.splitClaims || !config.endpoints?.analyzeBatch) {
+    configError = 'config.endpoints must define splitClaims and analyzeBatch paths';
+  }
+} catch (err) {
+  configError = err.code === 'ENOENT'
+    ? `no config found at ${CONFIG_PATH}`
+    : `could not parse ${CONFIG_PATH}: ${err.message}`;
+}
+
+export const meta = { name: config?.name || 'http' };
+
+export function onlineAvailable() {
+  return Boolean(config && !configError && missingEnv.size === 0);
+}
+
+export function onlineConfigHint() {
+  if (missingEnv.size) return `HTTP adapter: unset environment variable(s): ${[...missingEnv].join(', ')}.`;
+  if (configError) return `HTTP adapter: ${configError}. Set OPENGATE_HTTP_CONFIG or create opengate.http.json (see ADAPTERS.md).`;
+  return 'HTTP adapter: configuration incomplete.';
+}
+
+export function runModel() {
+  const envName = config?.modelEnv;
+  return (envName && process.env[envName]) || process.env.OPENGATE_EVAL_MODEL || null;
+}
+
+// ── Timing & token capture (same pattern as the reference adapter) ──────
+const _calls = [];
+export function resetTiming() { _calls.length = 0; }
+export function callLatencies() { return _calls.map(c => c.ms); }
+
+const _tokens = [];
+export function resetTokens() { _tokens.length = 0; }
+export function tokenTotals() {
+  const sum = (k) => _tokens.reduce((a, t) => a + (t[k] || 0), 0);
+  return {
+    calls: _tokens.length,
+    prompt_tokens: sum('prompt_tokens'),
+    completion_tokens: sum('completion_tokens'),
+    reasoning_tokens: sum('reasoning_tokens'),
+    total_tokens: sum('total_tokens'),
+  };
+}
+
+// ── Transport ───────────────────────────────────────────────────────────
+async function post(path, body) {
+  const t0 = performance.now();
+  try {
+    const res = await fetch(`${config.baseUrl}${path}`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', ...(config.headers || {}) },
+      body: JSON.stringify(body),
+    });
+    const data = await res.json().catch(() => ({}));
+    if (!res.ok) throw new Error(data.error || `${path} → HTTP ${res.status}`);
+    if (data && data.usage) {
+      _tokens.push({
+        prompt_tokens: data.usage.prompt_tokens || 0,
+        completion_tokens: data.usage.completion_tokens || 0,
+        reasoning_tokens: data.usage.reasoning_tokens || 0,
+        total_tokens: data.usage.total_tokens || 0,
+      });
+    }
+    return data;
+  } finally {
+    _calls.push({ path, ms: performance.now() - t0 });
+  }
+}
+
+export function splitClaims(text) {
+  return post(config.endpoints.splitClaims, { text });
+}
+
+export function analyzeBatch(payload) {
+  return post(config.endpoints.analyzeBatch, payload);
+}

package/src/adapters/refcheckr.mjs

@@ -0,0 +1,109 @@
+// RefCheckr adapter — the reference implementation of the OpenGATE adapter
+// contract (see ADAPTERS.md). Selected by default; to evaluate a different
+// system, write an adapter with the same exports and select it with
+// OPENGATE_ADAPTER=./path/to/adapter.mjs.
+//
+// OFFLINE scorers don't touch this. ONLINE scorers call it to exercise the real
+// RefCheckr endpoints. Configure via env:
+//   REFCHECKR_BASE_URL   e.g. http://localhost:3848 or https://refcheckr.pharmatools.ai
+//   REFCHECKR_TOKEN      a valid auth token (endpoints are behind checkAuth)
+//
+// Kept deliberately thin: each method maps to one endpoint and returns parsed
+// JSON, so scorers stay focused on measurement, not transport.
+
+export const meta = { name: 'refcheckr' };
+
+const BASE = process.env.REFCHECKR_BASE_URL;
+const TOKEN = process.env.REFCHECKR_TOKEN;
+
+export function onlineAvailable() {
+  return Boolean(BASE && TOKEN);
+}
+
+export function onlineConfigHint() {
+  return 'Set REFCHECKR_BASE_URL and REFCHECKR_TOKEN to run online scorers.';
+}
+
+// Optional label for which model the deployment was running during this eval.
+// The adapter can't infer the server's model, so the operator names it, e.g.
+//   export OPENGATE_EVAL_MODEL=claude-haiku-4-5
+// It's recorded in the scorecard so per-model runs stay comparable.
+// (REFCHECKR_EVAL_MODEL is accepted as a legacy fallback.)
+export function runModel() {
+  return process.env.OPENGATE_EVAL_MODEL || process.env.REFCHECKR_EVAL_MODEL || null;
+}
+
+// ── Timing ──────────────────────────────────────────────────────────────
+// Every request through post() records its wall-clock latency here so scorers
+// can report p50/p95. Call resetTiming() at the start of a scorer, then read
+// callLatencies(pathFilter) after the calls complete.
+const _calls = [];
+export function resetTiming() { _calls.length = 0; }
+export function callLatencies(pathIncludes) {
+  const rows = pathIncludes ? _calls.filter(c => c.path.includes(pathIncludes)) : _calls.slice();
+  return rows.map(c => c.ms);
+}
+
+/** p-th percentile (0–100) of a numeric array; null if empty. */
+export function percentile(values, p) {
+  const xs = values.filter(v => Number.isFinite(v)).sort((a, b) => a - b);
+  if (!xs.length) return null;
+  const idx = Math.min(xs.length - 1, Math.floor((p / 100) * xs.length));
+  return xs[idx];
+}
+
+// ── Token usage ─────────────────────────────────────────────────────────
+// When the server surfaces a `usage` block (model token counts), post() records
+// it here so scorers can compute real cost per claim. Call resetTokens() at the
+// start of a scorer, then tokenTotals(pathFilter) after the calls complete.
+const _tokens = [];
+export function resetTokens() { _tokens.length = 0; }
+export function tokenTotals(pathIncludes) {
+  const rows = pathIncludes ? _tokens.filter(t => t.path.includes(pathIncludes)) : _tokens.slice();
+  const sum = (k) => rows.reduce((a, t) => a + (t[k] || 0), 0);
+  return {
+    calls: rows.length,
+    prompt_tokens: sum('prompt_tokens'),
+    completion_tokens: sum('completion_tokens'),
+    reasoning_tokens: sum('reasoning_tokens'),
+    total_tokens: sum('total_tokens'),
+  };
+}
+
+async function post(path, body) {
+  const t0 = performance.now();
+  try {
+    const res = await fetch(`${BASE}${path}`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${TOKEN}` },
+      body: JSON.stringify(body),
+    });
+    const data = await res.json().catch(() => ({}));
+    if (!res.ok) throw new Error(data.error || `${path} → HTTP ${res.status}`);
+    if (data && data.usage) {
+      _tokens.push({
+        path,
+        prompt_tokens: data.usage.prompt_tokens || 0,
+        completion_tokens: data.usage.completion_tokens || 0,
+        reasoning_tokens: data.usage.reasoning_tokens || 0,
+        total_tokens: data.usage.total_tokens || 0,
+      });
+    }
+    return data;
+  } finally {
+    _calls.push({ path, ms: performance.now() - t0 });
+  }
+}
+
+/** POST /api/claims/split → { claims: [{ text, originalText, citations }] } */
+export function splitClaims(text) {
+  return post('/api/claims/split', { text });
+}
+
+/**
+ * Batch verify — POST /api/verify/batch.
+ * payload: { claims[], documents[{name,type,content}], citationMapping, claimCitations, document_name }
+ */
+export function analyzeBatch(payload) {
+  return post('/api/verify/batch', payload);
+}

package/src/index.mjs

@@ -0,0 +1,29 @@
+// Public programmatic API for @pharmatools/opengate.
+//
+// The CLI (bin: opengate → src/runner.mjs) is the primary interface; these
+// exports let implementations reuse the framework's pure logic directly —
+// e.g. scoring inside an existing test suite, or building custom tooling on
+// the metric primitives.
+
+export {
+  jaccard,
+  exactSetMatch,
+  precisionRecallF1,
+  VERDICT_SCALE,
+  verdictAccuracy,
+  confusionMatrix,
+  mean,
+  stdev,
+  percentile,
+  normText,
+  claimMatch,
+} from './lib/metrics.mjs';
+
+export {
+  normalizeCitations,
+  parseClaimCitations,
+  detectAuthorYear,
+  detectCitations,
+} from './lib/citations.mjs';
+
+export { loadAdapter, validateAdapter } from './lib/adapter.mjs';

package/src/lib/adapter.mjs

@@ -0,0 +1,78 @@
+// Adapter loading + validation.
+//
+// An adapter is the boundary between OpenGATE's scorers and the system under
+// test. The bundled RefCheckr adapter (src/adapters/refcheckr.mjs) is the
+// reference implementation; see ADAPTERS.md for the full contract.
+//
+// Select an adapter with:
+//   OPENGATE_ADAPTER=./path/to/my-adapter.mjs npm run eval:online
+//
+// Relative paths resolve from the current working directory. When the variable
+// is unset, the bundled RefCheckr adapter is used.
+
+import { resolve, basename, dirname, join } from 'node:path';
+import { pathToFileURL, fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DEFAULT_ADAPTER = join(__dirname, '..', 'adapters', 'refcheckr.mjs');
+
+// Required: online scorers cannot run without these.
+const REQUIRED = ['splitClaims', 'analyzeBatch', 'onlineAvailable', 'onlineConfigHint'];
+
+// Optional: validated if present; no-op defaults are supplied if absent, so
+// scorers can call them unconditionally.
+const OPTIONAL = ['runModel', 'resetTiming', 'callLatencies', 'resetTokens', 'tokenTotals'];
+
+const DEFAULTS = {
+  runModel: () => null,
+  resetTiming: () => {},
+  callLatencies: () => [],
+  resetTokens: () => {},
+  tokenTotals: () => ({
+    calls: 0, prompt_tokens: 0, completion_tokens: 0, reasoning_tokens: 0, total_tokens: 0,
+  }),
+};
+
+/** Throws with a readable message listing every problem, or returns silently. */
+export function validateAdapter(mod, source = 'adapter') {
+  const problems = [];
+  for (const fn of REQUIRED) {
+    if (typeof mod[fn] !== 'function') problems.push(`missing required export: ${fn}()`);
+  }
+  for (const fn of OPTIONAL) {
+    if (fn in mod && typeof mod[fn] !== 'function') problems.push(`optional export ${fn} is not a function`);
+  }
+  if (problems.length) {
+    throw new Error(
+      `Invalid adapter (${source}):\n  - ${problems.join('\n  - ')}\n` +
+      'See ADAPTERS.md for the required surface.'
+    );
+  }
+}
+
+/**
+ * Load, validate, and normalise an adapter.
+ * Precedence: explicit argument (--adapter flag) > OPENGATE_ADAPTER env >
+ * bundled RefCheckr reference adapter.
+ */
+export async function loadAdapter(specOverride) {
+  const spec = specOverride || process.env.OPENGATE_ADAPTER;
+  const path = spec ? resolve(process.cwd(), spec) : DEFAULT_ADAPTER;
+  let mod;
+  try {
+    mod = await import(pathToFileURL(path).href);
+  } catch (err) {
+    throw new Error(`Could not load adapter from ${path}: ${err.message}`);
+  }
+  validateAdapter(mod, path);
+  const methods = Object.fromEntries(
+    [...REQUIRED, ...OPTIONAL]
+      .filter(f => typeof mod[f] === 'function')
+      .map(f => [f, mod[f]])
+  );
+  return {
+    name: mod.meta?.name || basename(path, '.mjs'),
+    ...DEFAULTS,
+    ...methods,
+  };
+}

package/src/lib/citations.mjs

@@ -0,0 +1,109 @@
+// Reference implementation of the deterministic citation-detection logic
+// (originally developed for RefCheckr, OpenGATE's first implementation).
+//
+// This MIRRORS RefCheckr production's lib/citations.js, which routes/verify.js
+// imports — the inline duplication in verify.js was promoted to a shared
+// module, so the citation-detection scorer is a true regression test of the
+// production algorithm. If you change behaviour in either copy, port it to the
+// other; the citation-styles fixture catches drift (verified: 104 inputs,
+// byte-identical outputs across both modules).
+
+export function normalizeCitations(text) {
+  // Pass 1: "word.1,2" / "word1,2" → "word.[1,2]" (letter/period/paren/quote + digits)
+  let result = text.replace(/(?<!\d)([a-zA-Z.)"])(\d{1,3}(?:[,\-‐‑‒–]\d{1,3})*)\s/g, (match, prefix, nums) => {
+    if (/\d$/.test(prefix)) return match;
+    const numParts = nums.split(/[,\-‐‑‒–]/);
+    if (numParts.some(p => parseInt(p) > 200)) return match; // guards years / large numbers
+    // Avoid false positives on endpoint/identifier names glued to a single number
+    // (ACR20, PASI90, EASI75, CD4, type2, grade3): only convert when the prefix is
+    // punctuation (e.g. "placebo.1") or it is a list/range (e.g. "outcomes1,2").
+    if (/[a-zA-Z]/.test(prefix) && numParts.length === 1) return match;
+    return `${prefix}[${nums}] `;
+  });
+  // Pass 2: "word,12 " → "word,[12] "
+  result = result.replace(/([a-zA-Z]),(\d{1,3}(?:[\-‐‑‒–]\d{1,3})?)\s/g, (match, prefix, nums) => {
+    if (parseInt(nums) > 200) return match;
+    return `${prefix},[${nums}] `;
+  });
+  // Pass 3: parenthetical "(1,2)" / "(3-9)" → "[1,2]" / "[3-9]"
+  result = result.replace(/\((\d{1,3}(?:[,\-‐‑‒–]\d{1,3})*)\)/g, (match, nums) => {
+    const numParts = nums.split(/[,\-‐‑‒–]/);
+    if (numParts.some(p => parseInt(p) > 200)) return match;
+    return `[${nums}]`;
+  });
+  return result;
+}
+
+/** Parse the set of citation numbers from a (already-normalised) claim string. */
+export function parseClaimCitations(claim) {
+  const citationPattern = /\[(\d[\d,\-‐‑‒–\s]*)\]/g;
+  const citations = new Set();
+  let match;
+  while ((match = citationPattern.exec(claim)) !== null) {
+    match[1].split(',').forEach(part => {
+      part = part.trim();
+      const range = part.match(/^(\d+)\s*[-‐‑‒–]\s*(\d+)$/);
+      if (range) {
+        for (let i = parseInt(range[1]); i <= parseInt(range[2]); i++) citations.add(i);
+      } else if (/^\d+$/.test(part)) {
+        citations.add(parseInt(part));
+      }
+    });
+  }
+  return [...citations].sort((a, b) => a - b);
+}
+
+// ── Author-year detection ───────────────────────────────────────────────
+// Framework capability, additive to the numeric passes above. The numeric
+// functions (normalizeCitations / parseClaimCitations) still mirror RefCheckr
+// production (routes/verify.js) exactly; author-year keys are not yet consumed
+// by RefCheckr's downstream citation mapping, which is numeric-keyed.
+//
+// A detected author-year citation is keyed "Surname YYYY" (first author only),
+// e.g. "(Smith et al., 2020)" → "Smith 2020".
+
+const NAME = String.raw`[A-Z][A-Za-z'’\-]+`;
+const YEAR = String.raw`(?:1[89]|20)\d{2}`;
+const ETAL = String.raw`(?:et al\.?|and colleagues|and coworkers)`;
+
+/** Detect author-year citations in raw text; returns sorted unique keys. */
+export function detectAuthorYear(text) {
+  const found = new Set();
+  const add = (name, year) => found.add(`${name} ${year.slice(0, 4)}`);
+
+  // 1. Narrative with parenthetical year: "Smith et al. (2020)",
+  //    "Smith and Jones (2019)", "Smith (2020)".
+  const narrative = new RegExp(
+    String.raw`\b(${NAME})(?:\s+(?:${ETAL}|(?:and|&)\s+${NAME}))?\s*\((${YEAR}[a-z]?)\)`, 'g');
+  for (const m of text.matchAll(narrative)) add(m[1], m[2]);
+
+  // 2. Parenthetical: "(Smith et al., 2020)", "(Smith & Jones, 2019)",
+  //    "(Kim, 2023)", multiples split on ";": "(Brown 2018; Lee et al., 2022)".
+  const inner = new RegExp(
+    String.raw`^\s*(${NAME})(?:\s+${ETAL}|,?\s+(?:and|&)\s+${NAME})?,?\s+(${YEAR}[a-z]?)\s*$`);
+  for (const m of text.matchAll(/\(([^()]+)\)/g)) {
+    for (const part of m[1].split(';')) {
+      const hit = part.match(inner);
+      if (hit) add(hit[1], hit[2]);
+    }
+  }
+
+  // 3. Narrative prose year: "Jones and colleagues in 2019". Requires an
+  //    et-al-style marker so plain prose years ("began in 2019") never match.
+  const prose = new RegExp(
+    String.raw`\b(${NAME})\s+${ETAL}[^.;()]*?\bin\s+(${YEAR})\b`, 'g');
+  for (const m of text.matchAll(prose)) add(m[1], m[2]);
+
+  return [...found].sort();
+}
+
+/**
+ * Convenience: detect citations directly from raw claim text.
+ * Returns numeric citations (numbers) followed by author-year keys (strings).
+ */
+export function detectCitations(rawClaim) {
+  return [
+    ...parseClaimCitations(normalizeCitations(rawClaim + ' ')),
+    ...detectAuthorYear(rawClaim),
+  ];
+}

package/src/lib/metrics.mjs

@@ -0,0 +1,104 @@
+// Generic scoring primitives used by the eval scorers.
+// Pure functions, no I/O — easy to unit-test and reason about.
+
+/** Set intersection-over-union for two arrays of primitives. */
+export function jaccard(a, b) {
+  const A = new Set(a), B = new Set(b);
+  if (A.size === 0 && B.size === 0) return 1;
+  let inter = 0;
+  for (const x of A) if (B.has(x)) inter++;
+  const union = A.size + B.size - inter;
+  return union === 0 ? 1 : inter / union;
+}
+
+/** True if two arrays contain exactly the same set of values. */
+export function exactSetMatch(a, b) {
+  const A = new Set(a), B = new Set(b);
+  if (A.size !== B.size) return false;
+  for (const x of A) if (!B.has(x)) return false;
+  return true;
+}
+
+/**
+ * Precision / recall / F1 for predicted vs gold items.
+ * `matchFn(pred, gold) => bool` decides whether two items are "the same"
+ * (e.g. fuzzy text match for claims). Greedy one-to-one matching.
+ */
+export function precisionRecallF1(predicted, gold, matchFn = (a, b) => a === b) {
+  const usedGold = new Set();
+  let tp = 0;
+  for (const p of predicted) {
+    const gi = gold.findIndex((g, i) => !usedGold.has(i) && matchFn(p, g));
+    if (gi >= 0) { tp++; usedGold.add(gi); }
+  }
+  const fp = predicted.length - tp;
+  const fn = gold.length - tp;
+  const precision = predicted.length ? tp / predicted.length : (gold.length ? 0 : 1);
+  const recall = gold.length ? tp / gold.length : 1;
+  const f1 = (precision + recall) ? (2 * precision * recall) / (precision + recall) : 0;
+  return { tp, fp, fn, precision, recall, f1 };
+}
+
+// Verdict spectrum, ordered by how well the claim (as written) is borne out by
+// the reference — see datasets/LABELING-GUIDE.md. Used for exact accuracy and
+// "off-by-one" adjacency accuracy.
+export const VERDICT_SCALE = [
+  'strong_support',
+  'partial_support',
+  'implied_by_data',
+  'overclaim',
+  'not_supported',
+  'contradicted',
+];
+
+export function verdictAccuracy(pairs) {
+  // pairs: [{ predicted, gold }]
+  let exact = 0, adjacent = 0;
+  for (const { predicted, gold } of pairs) {
+    if (predicted === gold) { exact++; adjacent++; continue; }
+    const pi = VERDICT_SCALE.indexOf(predicted);
+    const gi = VERDICT_SCALE.indexOf(gold);
+    if (pi >= 0 && gi >= 0 && Math.abs(pi - gi) === 1) adjacent++;
+  }
+  const n = pairs.length || 1;
+  return { n: pairs.length, exact: exact / n, adjacent: adjacent / n };
+}
+
+export function confusionMatrix(pairs, labels = VERDICT_SCALE) {
+  const idx = Object.fromEntries(labels.map((l, i) => [l, i]));
+  const m = labels.map(() => labels.map(() => 0));
+  for (const { predicted, gold } of pairs) {
+    if (idx[gold] != null && idx[predicted] != null) m[idx[gold]][idx[predicted]]++;
+  }
+  return { labels, matrix: m };
+}
+
+export function mean(xs) { return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0; }
+
+/** p-th percentile (0–100) of a numeric array; null if empty. */
+export function percentile(values, p) {
+  const xs = values.filter(v => Number.isFinite(v)).sort((a, b) => a - b);
+  if (!xs.length) return null;
+  const idx = Math.min(xs.length - 1, Math.floor((p / 100) * xs.length));
+  return xs[idx];
+}
+export function stdev(xs) {
+  if (xs.length < 2) return 0;
+  const m = mean(xs);
+  return Math.sqrt(mean(xs.map(x => (x - m) ** 2)));
+}
+
+/** Normalise claim text for fuzzy matching (lowercase, alnum-only). */
+export function normText(s) {
+  return String(s).toLowerCase().replace(/[^a-z0-9]+/g, '');
+}
+
+/** Fuzzy claim match: containment either direction after normalisation. */
+export function claimMatch(a, b) {
+  const x = normText(a), y = normText(b);
+  if (!x || !y) return false;
+  if (x === y) return true;
+  const [short, long] = x.length <= y.length ? [x, y] : [y, x];
+  // Require the shorter to be a substantial substring of the longer.
+  return short.length >= 12 && long.includes(short);
+}