agent-tool-forge 0.3.0

package/lib/api-loader.js

@@ -0,0 +1,260 @@
+/**
+ * API Loader — Fetches endpoints from OpenAPI (URL or file) and manifest.
+ * Merges and dedupes by path+method.
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { resolve } from 'path';
+
+/**
+ * @typedef {Object} ApiEndpoint
+ * @property {string} path - API path (e.g. /api/v1/holdings)
+ * @property {string} method - HTTP method (GET, POST, etc.)
+ * @property {string} [name] - Suggested tool name (snake_case)
+ * @property {string} [description] - Suggested tool description
+ * @property {Record<string,unknown>} [params] - Parameter schema
+ * @property {boolean} [requiresConfirmation] - HITL gate for write ops
+ * @property {string} [source] - 'openapi' | 'manifest'
+ */
+
+/**
+ * Derive tool name from path and method.
+ * @param {string} path
+ * @param {string} method
+ * @returns {string}
+ */
+function deriveName(path, method) {
+  const parts = path
+    .replace(/^\/+/, '')
+    .replace(/\/$/, '')
+    .split('/')
+    .filter(Boolean);
+  const last = parts[parts.length - 1] || 'resource';
+  const base = last.replace(/\{[^}]+\}/g, 'by_id');
+  const verb = method === 'GET' ? 'get' : method === 'POST' ? 'create' : method.toLowerCase();
+  return `${verb}_${base}`.replace(/-/g, '_').replace(/[^a-z0-9_]/gi, '_').toLowerCase();
+}
+
+/**
+ * Parse OpenAPI 3.x paths into ApiEndpoint array.
+ * @param {object} spec - Parsed OpenAPI JSON
+ * @returns {ApiEndpoint[]}
+ */
+function parseOpenApiPaths(spec) {
+  const endpoints = [];
+  const paths = spec.paths || {};
+  const rawBase = spec.servers?.[0]?.url?.replace(/\/$/, '') || '';
+  // Only use relative-path server bases (e.g. /api/v1); ignore full URLs
+  const basePath = rawBase.startsWith('/') ? rawBase : '';
+
+  for (const [path, pathItem] of Object.entries(paths)) {
+    if (typeof pathItem !== 'object' || pathItem === null) continue;
+    const methods = ['get', 'post', 'put', 'patch', 'delete'];
+    for (const method of methods) {
+      const op = pathItem[method];
+      if (!op) continue;
+      const relativePath = path.startsWith('/') ? path : `/${path}`;
+      const fullPath = `${basePath}${relativePath}`;
+      const name = deriveName(relativePath, method.toUpperCase());
+      const params = {};
+      for (const p of op.parameters || []) {
+        if (p?.name) {
+          params[p.name] = {
+            type: p.schema?.type || 'string',
+            description: p.description
+          };
+        }
+      }
+      if (pathItem.parameters) {
+        for (const p of pathItem.parameters) {
+          if (p?.name && !params[p.name]) {
+            params[p.name] = {
+              type: p.schema?.type || 'string',
+              description: p.description
+            };
+          }
+        }
+      }
+      endpoints.push({
+        path: fullPath,
+        method: method.toUpperCase(),
+        name,
+        description: op.summary || op.description || `${method.toUpperCase()} ${fullPath}`,
+        params: Object.keys(params).length ? params : undefined,
+        requiresConfirmation: ['post', 'put', 'patch', 'delete'].includes(method),
+        source: 'openapi'
+      });
+    }
+  }
+  return endpoints;
+}
+
+/**
+ * Load OpenAPI spec from URL.
+ * @param {string} url
+ * @returns {Promise<ApiEndpoint[]>}
+ */
+async function loadFromOpenApiUrl(url, headers = {}) {
+  const res = await fetch(url, {
+    signal: AbortSignal.timeout(10000),
+    headers,
+  });
+  if (!res.ok) throw new Error(`OpenAPI fetch failed: ${res.status} ${url}`);
+  const spec = await res.json();
+  return parseOpenApiPaths(spec);
+}
+
+/**
+ * Load OpenAPI spec from file.
+ * @param {string} filePath
+ * @returns {ApiEndpoint[]}
+ */
+function loadFromOpenApiFile(filePath) {
+  const abs = resolve(process.cwd(), filePath);
+  if (!existsSync(abs)) return [];
+  const raw = readFileSync(abs, 'utf-8');
+  let spec;
+  try { spec = JSON.parse(raw); } catch (err) {
+    throw new Error(`Failed to parse OpenAPI file ${filePath}: ${err.message}`);
+  }
+  return parseOpenApiPaths(spec);
+}
+
+/**
+ * Load endpoints from manifest file.
+ * @param {string} manifestPath
+ * @returns {ApiEndpoint[]}
+ */
+function loadFromManifest(manifestPath) {
+  const abs = resolve(process.cwd(), manifestPath);
+  if (!existsSync(abs)) return [];
+  const raw = readFileSync(abs, 'utf-8');
+  let manifest;
+  try { manifest = JSON.parse(raw); } catch (err) {
+    throw new Error(`Failed to parse manifest ${manifestPath}: ${err.message}`);
+  }
+  const endpoints = manifest.endpoints || [];
+  return endpoints.map((e) => ({
+    path: e.path,
+    method: (e.method || 'GET').toUpperCase(),
+    name: e.name || deriveName(e.path, e.method || 'GET'),
+    description: e.description || `${e.method || 'GET'} ${e.path}`,
+    params: e.params,
+    requiresConfirmation: e.requiresConfirmation ?? false,
+    source: 'manifest'
+  }));
+}
+
+/**
+ * Merge endpoints, dedupe by path+method (manifest overrides openapi).
+ * @param {ApiEndpoint[][]} arrays
+ * @returns {ApiEndpoint[]}
+ */
+function mergeEndpoints(...arrays) {
+  const byKey = new Map();
+  for (const arr of arrays) {
+    for (const e of arr) {
+      const key = `${e.method}:${e.path}`;
+      byKey.set(key, e);
+    }
+  }
+  return Array.from(byKey.values());
+}
+
+/**
+ * Safe JSON.parse — returns null on failure.
+ * @param {string} str
+ * @returns {object|null}
+ */
+function safeParseJson(str) {
+  try { return JSON.parse(str); } catch { return null; }
+}
+
+/**
+ * Normalize a path for comparison: ensure leading slash, trim trailing slash, lowercase.
+ * @param {string} p
+ * @returns {string}
+ */
+function normalizePath(p) {
+  if (!p) return '';
+  const withSlash = p.startsWith('/') ? p : `/${p}`;
+  return withSlash.replace(/\/$/, '').toLowerCase();
+}
+
+/**
+ * Compute API coverage: which spec endpoints have a promoted tool with a matching
+ * mcpRouting.endpoint (path) and mcpRouting.method.
+ *
+ * Matching is path+method, case-insensitive, path-only (no base URL).
+ *
+ * @param {object|null} spec - Parsed OpenAPI spec object
+ * @param {import('better-sqlite3').Database} db
+ * @returns {{ covered: ApiEndpoint[]; uncovered: ApiEndpoint[]; total: number }}
+ */
+export function computeCoverage(spec, db) {
+  if (!spec?.paths) return { covered: [], uncovered: [], total: 0 };
+
+  const endpoints = parseOpenApiPaths(spec);
+
+  const promotedRows = db.prepare(
+    `SELECT spec_json FROM tool_registry WHERE lifecycle_state = 'promoted'`
+  ).all();
+
+  // Build a set of "METHOD:normalizedPath" keys from promoted tools
+  const coveredKeys = new Set();
+  for (const row of promotedRows) {
+    const toolSpec = safeParseJson(row.spec_json);
+    if (!toolSpec?.mcpRouting?.endpoint) continue;
+    const path = normalizePath(toolSpec.mcpRouting.endpoint);
+    const method = (toolSpec.mcpRouting.method || '').toUpperCase();
+    if (!path || !method) continue;
+    coveredKeys.add(`${method}:${path}`);
+  }
+
+  const covered = [];
+  const uncovered = [];
+
+  for (const endpoint of endpoints) {
+    const key = `${endpoint.method.toUpperCase()}:${normalizePath(endpoint.path)}`;
+    if (coveredKeys.has(key)) {
+      covered.push(endpoint);
+    } else {
+      uncovered.push(endpoint);
+    }
+  }
+
+  return { covered, uncovered, total: endpoints.length };
+}
+
+/**
+ * Load all APIs from config.
+ * @param {object} config - forge.config.json api section
+ * @returns {Promise<ApiEndpoint[]>}
+ */
+export async function loadApis(config) {
+  const endpoints = [];
+  const discovery = config?.discovery;
+  const manifestPath = config?.manifestPath;
+
+  if (discovery?.type === 'openapi') {
+    if (discovery.url) {
+      try {
+        const fromUrl = await loadFromOpenApiUrl(discovery.url, discovery.headers);
+        endpoints.push(...fromUrl);
+      } catch (err) {
+        console.error(`OpenAPI URL failed: ${err.message}`);
+      }
+    }
+    if (discovery.file) {
+      const fromFile = loadFromOpenApiFile(discovery.file);
+      endpoints.push(...fromFile);
+    }
+  }
+
+  if (manifestPath) {
+    const fromManifest = loadFromManifest(manifestPath);
+    endpoints.push(...fromManifest);
+  }
+
+  return mergeEndpoints(endpoints);
+}

package/lib/auth.d.ts

@@ -0,0 +1,25 @@
+export interface AuthResult {
+  authenticated: boolean;
+  userId: string | null;
+  claims: Record<string, unknown> | null;
+  error: string | null;
+}
+
+export interface AuthConfig {
+  mode?: 'trust' | 'verify';
+  signingKey?: string;
+  claimsPath?: string;
+}
+
+export interface Authenticator {
+  authenticate(req: object): AuthResult;
+}
+
+export function createAuth(authConfig?: AuthConfig): Authenticator;
+
+export interface AdminAuthResult {
+  authenticated: boolean;
+  error: string | null;
+}
+
+export function authenticateAdmin(req: object, adminKey: string): AdminAuthResult;

package/lib/auth.js

@@ -0,0 +1,158 @@
+/**
+ * Auth module — configurable JWT authentication for the forge sidecar.
+ *
+ * Two modes:
+ *   trust  — decode JWT payload without verifying signature (fast, for local dev / behind reverse proxy)
+ *   verify — verify HMAC-SHA256 (HS256) or RSA-SHA256 (RS256) signature via Node.js built-in crypto
+ *
+ * No external JWT library required.
+ */
+
+import { createHmac, createVerify, timingSafeEqual } from 'crypto';
+
+/**
+ * @typedef {{ authenticated: boolean, userId: string|null, claims: object|null, error: string|null }} AuthResult
+ */
+
+/**
+ * Create an authenticator from config.
+ * @param {{ mode: 'verify'|'trust', signingKey?: string, claimsPath?: string }} authConfig
+ * @returns {{ authenticate(req): AuthResult }}
+ */
+export function createAuth(authConfig = {}) {
+  const mode = authConfig.mode ?? 'trust';
+  const signingKey = authConfig.signingKey ?? null;
+  const claimsPath = authConfig.claimsPath ?? 'sub';
+
+  return {
+    authenticate(req) {
+      let token = null;
+
+      // Priority 1: Authorization header
+      const authHeader = req.headers?.authorization;
+      if (authHeader?.startsWith('Bearer ')) {
+        token = authHeader.slice(7);
+      }
+
+      // Priority 2: ?token= query param (for EventSource which can't set headers)
+      if (!token && req.url) {
+        try {
+          const url = new URL(req.url, 'http://localhost');
+          token = url.searchParams.get('token');
+        } catch { /* malformed URL */ }
+      }
+
+      if (!token) {
+        return { authenticated: false, userId: null, claims: null, error: 'Missing token' };
+      }
+
+      const parts = token.split('.');
+      if (parts.length !== 3) {
+        return { authenticated: false, userId: null, claims: null, error: 'Malformed JWT' };
+      }
+
+      // Verify signature if in verify mode
+      if (mode === 'verify') {
+        if (!signingKey) {
+          return { authenticated: false, userId: null, claims: null, error: 'No signing key configured' };
+        }
+
+        let header;
+        try {
+          header = JSON.parse(base64UrlDecode(parts[0]));
+        } catch {
+          return { authenticated: false, userId: null, claims: null, error: 'Invalid JWT header' };
+        }
+
+        const sigInput = `${parts[0]}.${parts[1]}`;
+        const signature = parts[2];
+        const alg = header.alg ?? 'HS256';
+
+        if (alg === 'HS256') {
+          const expected = base64UrlEncode(
+            createHmac('sha256', signingKey).update(sigInput).digest()
+          );
+          const expectedBuf = Buffer.from(expected);
+          const signatureBuf = Buffer.from(signature);
+          if (expectedBuf.length !== signatureBuf.length || !timingSafeEqual(expectedBuf, signatureBuf)) {
+            return { authenticated: false, userId: null, claims: null, error: 'Invalid signature' };
+          }
+        } else if (alg === 'RS256') {
+          const sigBuf = base64UrlToBuffer(signature);
+          const verifier = createVerify('RSA-SHA256');
+          verifier.update(sigInput);
+          if (!verifier.verify(signingKey, sigBuf)) {
+            return { authenticated: false, userId: null, claims: null, error: 'Invalid signature' };
+          }
+        } else {
+          return { authenticated: false, userId: null, claims: null, error: `Unsupported algorithm: ${alg}` };
+        }
+      }
+
+      // Decode payload
+      let claims;
+      try {
+        claims = JSON.parse(base64UrlDecode(parts[1]));
+      } catch {
+        return { authenticated: false, userId: null, claims: null, error: 'Invalid JWT payload' };
+      }
+
+      const userId = extractClaim(claims, claimsPath);
+      if (!userId) {
+        return { authenticated: false, userId: null, claims, error: `Claim "${claimsPath}" not found in token` };
+      }
+
+      return { authenticated: true, userId: String(userId), claims, error: null };
+    }
+  };
+}
+
+/**
+ * Admin auth — simple Bearer token comparison.
+ * @param {import('http').IncomingMessage} req
+ * @param {string} adminKey
+ * @returns {{ authenticated: boolean, error: string|null }}
+ */
+export function authenticateAdmin(req, adminKey) {
+  if (!adminKey) {
+    return { authenticated: false, error: 'No admin key configured' };
+  }
+  const authHeader = req.headers?.authorization;
+  if (!authHeader || !authHeader.startsWith('Bearer ')) {
+    return { authenticated: false, error: 'Missing or invalid Authorization header' };
+  }
+  const token = authHeader.slice(7);
+  const tokenBuf = Buffer.from(token);
+  const keyBuf = Buffer.from(adminKey);
+  if (tokenBuf.length !== keyBuf.length || !timingSafeEqual(tokenBuf, keyBuf)) {
+    return { authenticated: false, error: 'Invalid admin key' };
+  }
+  return { authenticated: true, error: null };
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function base64UrlDecode(str) {
+  const padded = str.replace(/-/g, '+').replace(/_/g, '/');
+  return Buffer.from(padded, 'base64').toString('utf-8');
+}
+
+function base64UrlEncode(buf) {
+  return Buffer.from(buf).toString('base64')
+    .replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+
+function base64UrlToBuffer(str) {
+  const padded = str.replace(/-/g, '+').replace(/_/g, '/');
+  return Buffer.from(padded, 'base64');
+}
+
+function extractClaim(claims, path) {
+  const parts = path.split('.');
+  let val = claims;
+  for (const p of parts) {
+    if (val == null || typeof val !== 'object') return null;
+    val = val[p];
+  }
+  return val ?? null;
+}

package/lib/checks/check-adapter.js

@@ -0,0 +1,172 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * Map an agent-tool-forge eval case's `expect` object plus run metadata
+ * to a RunChecksInput shape that runChecks() can consume.
+ *
+ * @param {Object} evalCase - the eval case object from the eval JSON file
+ * @param {Object} runMeta - runtime data from executing the case
+ * @param {string[]} runMeta.toolsCalled - actual tools called during execution
+ * @param {string} runMeta.responseText - the model's response text
+ * @param {number} [runMeta.latencyMs] - round-trip latency in ms
+ * @param {number} [runMeta.cost] - actual cost in USD
+ * @returns {import('./types.js').RunChecksInput}
+ */
+export function checkAdapter(evalCase, runMeta) {
+  const expect = evalCase.expect ?? {};
+  const { toolsCalled = [], responseText = '', latencyMs, cost } = runMeta;
+
+  /** @type {import('./types.js').RunChecksInput} */
+  const input = {};
+
+  // Response text is needed for most content checks
+  input.responseText = responseText;
+
+  // Tool selection — exact match (strict mode via expectedTools/actualTools)
+  // toolsAcceptable is handled separately via checkToolsAcceptable() below
+  if (expect.toolsCalled !== undefined) {
+    input.expectedTools = Array.isArray(expect.toolsCalled)
+      ? expect.toolsCalled
+      : [expect.toolsCalled];
+    input.actualTools = toolsCalled;
+  }
+
+  // responseContains → mustContain (array of strings; every item must appear)
+  if (expect.responseContains !== undefined) {
+    input.mustContain = Array.isArray(expect.responseContains)
+      ? expect.responseContains
+      : [expect.responseContains];
+  }
+
+  // responseContainsAny — handled by callers via checkResponseContainsAnyGroups().
+  // Normalization of flat string[] to string[][] is done inside that function.
+  // No field is set on RunChecksInput for this (runChecks has no native anyOf check).
+
+  // responseNotContains → mustNotContain
+  if (expect.responseNotContains !== undefined) {
+    input.mustNotContain = Array.isArray(expect.responseNotContains)
+      ? expect.responseNotContains
+      : [expect.responseNotContains];
+  }
+
+  // responseNonEmpty → nonEmpty check
+  // The eval-runner treats "non-empty" as: text present OR at least one tool called.
+  // RunChecksInput.nonEmpty is a simpler text-only flag; callers should also check
+  // toolsCalled.length when the original semantics matter.
+  if (expect.responseNonEmpty) {
+    input.nonEmpty = true;
+  }
+
+  // requiresPreamble lives on evalCase directly (not inside expect).
+  // If true and the model returned only tool calls (no text), the run should fail.
+  // Map it to nonEmpty so the text-presence check fires.
+  if (evalCase.requiresPreamble === true) {
+    input.nonEmpty = true;
+  }
+
+  // Latency check
+  if (latencyMs !== undefined && expect.maxLatencyMs !== undefined) {
+    input.latencyMs = latencyMs;
+    input.maxLatencyMs = expect.maxLatencyMs;
+  }
+
+  // Cost budget
+  if (cost !== undefined && expect.maxCost !== undefined) {
+    input.actualCost = cost;
+    input.maxCost = expect.maxCost;
+  }
+
+  // Tool call count
+  if (expect.minToolCalls !== undefined || expect.maxToolCalls !== undefined) {
+    input.actualToolCallCount = toolsCalled.length;
+    if (expect.minToolCalls !== undefined) input.minToolCalls = expect.minToolCalls;
+    if (expect.maxToolCalls !== undefined) input.maxToolCalls = expect.maxToolCalls;
+  }
+
+  // jsonValid — check if response is valid JSON
+  if (expect.jsonValid) {
+    input.jsonValid = true;
+  }
+
+  // schemaData — validate response against a schema
+  if (expect.schemaData !== undefined) {
+    input.schemaData = expect.schemaData;
+    input.requiredKeys = expect.requiredKeys ?? [];
+    if (expect.typeChecks) input.typeChecks = expect.typeChecks;
+  }
+
+  // minLength / maxLength — response length bounds
+  if (expect.minLength !== undefined) input.minLength = expect.minLength;
+  if (expect.maxLength !== undefined) input.maxLength = expect.maxLength;
+
+  // regexPattern — response must match pattern
+  if (expect.regexPattern !== undefined) input.regexPattern = expect.regexPattern;
+
+  // copOutPhrases — custom cop-out phrase list for nonEmpty check
+  if (expect.copOutPhrases !== undefined) input.copOutPhrases = expect.copOutPhrases;
+
+  return input;
+}
+
+/**
+ * Handle the responseContainsAny case.
+ *
+ * In eval-runner, responseContainsAny is string[][] — an array of groups where
+ * each group must contribute at least one match.  This mirrors that behaviour.
+ *
+ * @param {string} responseText
+ * @param {string[][]} groups - each inner array is one group; at least one member
+ *   of each group must appear in responseText (case-sensitive, same as eval-runner)
+ * @returns {{ pass: boolean, reason?: string }}
+ */
+export function checkResponseContainsAnyGroups(responseText, groups) {
+  if (!groups?.length) return { pass: true };
+
+  // Normalize: flat string[] → [[...]] (single group). Grouped string[][] passes through.
+  const normalized = Array.isArray(groups[0]) ? groups : [groups];
+
+  const failures = [];
+  for (const group of normalized) {
+    if (!group.some((str) => responseText.includes(str))) {
+      failures.push(`response should contain any of [${group.join(', ')}]`);
+    }
+  }
+
+  if (failures.length === 0) return { pass: true };
+  return { pass: false, reason: failures.join('; ') };
+}
+
+/**
+ * Handle the toolsAcceptable case.
+ *
+ * toolsAcceptable is string[][] — an array of acceptable tool sets.  The run
+ * passes if the actual tools called exactly match ANY of the acceptable sets.
+ * The special token '__none__' inside an acceptable set means no tools called.
+ *
+ * @param {string[]} actualTools
+ * @param {string[][]} acceptable
+ * @returns {{ pass: boolean, reason?: string }}
+ */
+export function checkToolsAcceptable(actualTools, acceptable) {
+  if (!acceptable?.length) return { pass: true };
+
+  function setsEqual(a, b) {
+    const sa = new Set(a);
+    const sb = new Set(b);
+    if (sa.size !== sb.size) return false;
+    for (const v of sa) if (!sb.has(v)) return false;
+    return true;
+  }
+
+  const anyMatch = acceptable.some((set) => {
+    if (set.includes('__none__') && actualTools.length === 0) return true;
+    return setsEqual(set, actualTools);
+  });
+
+  if (anyMatch) return { pass: true };
+  return {
+    pass: false,
+    reason: `tools: [${actualTools.join(', ')}] not in any acceptable set`
+  };
+}

package/lib/checks/compose.js

@@ -0,0 +1,42 @@
+// Adapted from agent-eval-kit by FlanaganSe (https://github.com/FlanaganSe/agent-eval-kit)
+// MIT License — see LICENSE
+
+/**
+ * Compose multiple grader functions — passes only if ALL pass.
+ * @param {Array<(input: unknown) => Promise<{pass: boolean, reason?: string}>>} graders
+ * @returns {(input: unknown) => Promise<{pass: boolean, reason?: string}>}
+ */
+export function all(graders) {
+  return async (input) => {
+    const results = await Promise.all(graders.map(g => g(input)));
+    const failed = results.filter(r => !r.pass);
+    if (failed.length === 0) return { pass: true };
+    return { pass: false, reason: failed.map(r => r.reason).filter(Boolean).join('; ') };
+  };
+}
+
+/**
+ * Compose multiple grader functions — passes if ANY pass.
+ * @param {Array<(input: unknown) => Promise<{pass: boolean, reason?: string}>>} graders
+ * @returns {(input: unknown) => Promise<{pass: boolean, reason?: string}>}
+ */
+export function any(graders) {
+  return async (input) => {
+    const results = await Promise.all(graders.map(g => g(input)));
+    if (results.some(r => r.pass)) return { pass: true };
+    return { pass: false, reason: results.map(r => r.reason).filter(Boolean).join(' | ') };
+  };
+}
+
+/**
+ * Invert a grader function — passes if the original fails.
+ * @param {(input: unknown) => Promise<{pass: boolean, reason?: string}>} grader
+ * @returns {(input: unknown) => Promise<{pass: boolean, reason?: string}>}
+ */
+export function not(grader) {
+  return async (input) => {
+    const result = await grader(input);
+    if (!result.pass) return { pass: true };
+    return { pass: false, reason: `Expected grader to fail but it passed` };
+  };
+}

package/lib/checks/content-match.js

@@ -0,0 +1,14 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * Check that responseText contains all required substrings (case-insensitive).
+ * @param {{responseText: string, mustContain: string[]}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function contentMatch({ responseText, mustContain }) {
+  const lower = responseText.toLowerCase();
+  const missing = mustContain.filter(s => !lower.includes(s.toLowerCase()));
+  if (missing.length === 0) return { pass: true };
+  return { pass: false, reason: `Missing from response: ${missing.join(', ')}` };
+}

package/lib/checks/cost-budget.js

@@ -0,0 +1,11 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{actualCost: number, maxCost: number}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function costBudget({ actualCost, maxCost }) {
+  if (actualCost <= maxCost) return { pass: true };
+  return { pass: false, reason: `Cost $${actualCost.toFixed(6)} exceeded budget $${maxCost.toFixed(6)}` };
+}