agent-tool-forge 0.3.0

package/lib/checks/index.js

@@ -0,0 +1,18 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+export { contentMatch } from './content-match.js';
+export { negativeMatch } from './negative-match.js';
+export { toolSelection } from './tool-selection.js';
+export { latency } from './latency.js';
+export { jsonValid } from './json-valid.js';
+export { schemaMatch } from './schema-match.js';
+export { nonEmpty, DEFAULT_COP_OUT_PHRASES } from './non-empty.js';
+export { lengthBounds } from './length-bounds.js';
+export { regexMatch } from './regex-match.js';
+export { toolCallCount } from './tool-call-count.js';
+export { costBudget } from './cost-budget.js';
+export { runChecks } from './run-checks.js';
+export { noHallucinatedNumbers } from './no-hallucinated-numbers.js';
+export { all, any, not } from './compose.js';
+export { checkAdapter, checkResponseContainsAnyGroups, checkToolsAcceptable } from './check-adapter.js';

package/lib/checks/json-valid.js

@@ -0,0 +1,15 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{responseText: string}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function jsonValid({ responseText }) {
+  try {
+    JSON.parse(responseText);
+    return { pass: true };
+  } catch (e) {
+    return { pass: false, reason: `Invalid JSON: ${e.message}` };
+  }
+}

package/lib/checks/latency.js

@@ -0,0 +1,11 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{latencyMs: number, maxLatencyMs: number}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function latency({ latencyMs, maxLatencyMs }) {
+  if (latencyMs <= maxLatencyMs) return { pass: true };
+  return { pass: false, reason: `Latency ${latencyMs}ms exceeded max ${maxLatencyMs}ms` };
+}

package/lib/checks/length-bounds.js

@@ -0,0 +1,17 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{responseText: string, minLength?: number, maxLength?: number}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function lengthBounds({ responseText, minLength, maxLength }) {
+  const len = responseText.length;
+  if (minLength !== undefined && len < minLength) {
+    return { pass: false, reason: `Response length ${len} is below minimum ${minLength}` };
+  }
+  if (maxLength !== undefined && len > maxLength) {
+    return { pass: false, reason: `Response length ${len} exceeds maximum ${maxLength}` };
+  }
+  return { pass: true };
+}

package/lib/checks/negative-match.js

@@ -0,0 +1,14 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * Check that responseText does NOT contain any of the forbidden substrings (case-insensitive).
+ * @param {{responseText: string, mustNotContain: string[]}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function negativeMatch({ responseText, mustNotContain }) {
+  const lower = responseText.toLowerCase();
+  const found = mustNotContain.filter(s => lower.includes(s.toLowerCase()));
+  if (found.length === 0) return { pass: true };
+  return { pass: false, reason: `Forbidden content found: ${found.join(', ')}` };
+}

package/lib/checks/no-hallucinated-numbers.js

@@ -0,0 +1,63 @@
+// Adapted from agent-eval-kit by FlanaganSe (https://github.com/FlanaganSe/agent-eval-kit)
+// MIT License — see LICENSE
+
+/**
+ * Extract all numbers from a string.
+ * @param {string} text
+ * @returns {number[]}
+ */
+function extractNumbers(text) {
+  const matches = text.match(/-?\d+(?:\.\d+)?(?:e[+-]?\d+)?/gi) ?? [];
+  return matches.map(Number).filter(n => !isNaN(n));
+}
+
+/**
+ * Extract all numbers from a value (recursively for objects/arrays).
+ * @param {unknown} value
+ * @returns {number[]}
+ */
+function extractNumbersDeep(value) {
+  if (typeof value === 'number') return [value];
+  if (typeof value === 'string') return extractNumbers(value);
+  if (Array.isArray(value)) return value.flatMap(extractNumbersDeep);
+  if (value !== null && typeof value === 'object') {
+    return Object.values(value).flatMap(extractNumbersDeep);
+  }
+  return [];
+}
+
+/**
+ * Check that numbers in responseText match numbers from toolResults (within tolerance).
+ * @param {{responseText: string, toolResults: unknown, tolerance?: number}} input
+ * @returns {{pass: boolean, hallucinated: number[], matched: number[], reason?: string}}
+ */
+export function noHallucinatedNumbers({ responseText, toolResults, tolerance = 0.01 }) {
+  const responseNumbers = extractNumbers(responseText);
+  const sourceNumbers = extractNumbersDeep(toolResults);
+
+  const hallucinated = [];
+  const matched = [];
+
+  for (const num of responseNumbers) {
+    // Check if this number is within tolerance of any source number
+    const isMatched = sourceNumbers.some(src => {
+      if (src === 0 && num === 0) return true;
+      if (src === 0) return Math.abs(num) <= tolerance;
+      return Math.abs(num - src) / Math.abs(src) <= tolerance;
+    });
+
+    if (isMatched) {
+      matched.push(num);
+    } else {
+      hallucinated.push(num);
+    }
+  }
+
+  if (hallucinated.length === 0) return { pass: true, hallucinated: [], matched };
+  return {
+    pass: false,
+    hallucinated,
+    matched,
+    reason: `Hallucinated numbers not found in tool results: ${hallucinated.join(', ')}`,
+  };
+}

package/lib/checks/non-empty.js

@@ -0,0 +1,34 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+export const DEFAULT_COP_OUT_PHRASES = [
+  "i'm sorry",
+  "i cannot",
+  "i can't",
+  "i don't know",
+  "i am not able to",
+  "as an ai",
+  "as a language model",
+  "i don't have access",
+  "i don't have information",
+  "i'm not able to",
+  "i am unable to",
+  "unfortunately, i",
+  "i apologize",
+];
+
+/**
+ * @param {{responseText: string, copOutPhrases?: string[]}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function nonEmpty({ responseText, copOutPhrases = DEFAULT_COP_OUT_PHRASES }) {
+  if (!responseText || responseText.trim().length === 0) {
+    return { pass: false, reason: 'Response is empty' };
+  }
+  const lower = responseText.toLowerCase();
+  const found = copOutPhrases.find(phrase => lower.includes(phrase.toLowerCase()));
+  if (found) {
+    return { pass: false, reason: `Response contains cop-out phrase: "${found}"` };
+  }
+  return { pass: true };
+}

package/lib/checks/regex-match.js

@@ -0,0 +1,12 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{responseText: string, pattern: string|RegExp}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function regexMatch({ responseText, pattern }) {
+  const regex = pattern instanceof RegExp ? pattern : new RegExp(pattern);
+  if (regex.test(responseText)) return { pass: true };
+  return { pass: false, reason: `Response did not match pattern: ${pattern}` };
+}

package/lib/checks/run-checks.js

@@ -0,0 +1,84 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+import { contentMatch } from './content-match.js';
+import { negativeMatch } from './negative-match.js';
+import { toolSelection } from './tool-selection.js';
+import { latency } from './latency.js';
+import { jsonValid } from './json-valid.js';
+import { schemaMatch } from './schema-match.js';
+import { nonEmpty } from './non-empty.js';
+import { lengthBounds } from './length-bounds.js';
+import { regexMatch } from './regex-match.js';
+import { toolCallCount } from './tool-call-count.js';
+import { costBudget } from './cost-budget.js';
+import { noHallucinatedNumbers } from './no-hallucinated-numbers.js';
+
+/**
+ * Run all applicable checks based on what inputs are provided.
+ * Only runs a check if the relevant input fields are present.
+ * @param {import('./types.js').RunChecksInput} input
+ * @returns {import('./types.js').CheckSuiteResult}
+ */
+export function runChecks(input) {
+  const checks = {};
+
+  if (input.mustContain?.length && input.responseText !== undefined) {
+    checks.contentMatch = contentMatch({ responseText: input.responseText, mustContain: input.mustContain });
+  }
+
+  if (input.mustNotContain?.length && input.responseText !== undefined) {
+    checks.negativeMatch = negativeMatch({ responseText: input.responseText, mustNotContain: input.mustNotContain });
+  }
+
+  if (input.expectedTools !== undefined && input.actualTools !== undefined) {
+    checks.toolSelection = toolSelection({ expected: input.expectedTools, actual: input.actualTools, mode: input.toolSelectionMode });
+  }
+
+  if (input.latencyMs !== undefined && input.maxLatencyMs !== undefined) {
+    checks.latency = latency({ latencyMs: input.latencyMs, maxLatencyMs: input.maxLatencyMs });
+  }
+
+  if (input.jsonValid && input.responseText !== undefined) {
+    checks.jsonValid = jsonValid({ responseText: input.responseText });
+  }
+
+  if (input.schemaData && (input.requiredKeys !== undefined || input.typeChecks)) {
+    checks.schemaMatch = schemaMatch({ data: input.schemaData, requiredKeys: input.requiredKeys ?? [], typeChecks: input.typeChecks });
+  }
+
+  if (input.nonEmpty && input.responseText !== undefined) {
+    checks.nonEmpty = nonEmpty({ responseText: input.responseText, copOutPhrases: input.copOutPhrases });
+  }
+
+  if ((input.minLength !== undefined || input.maxLength !== undefined) && input.responseText !== undefined) {
+    checks.lengthBounds = lengthBounds({ responseText: input.responseText, minLength: input.minLength, maxLength: input.maxLength });
+  }
+
+  if (input.regexPattern && input.responseText !== undefined) {
+    checks.regexMatch = regexMatch({ responseText: input.responseText, pattern: input.regexPattern });
+  }
+
+  if (input.actualToolCallCount !== undefined && (input.minToolCalls !== undefined || input.maxToolCalls !== undefined)) {
+    checks.toolCallCount = toolCallCount({ actual: input.actualToolCallCount, min: input.minToolCalls, max: input.maxToolCalls });
+  }
+
+  if (input.actualCost !== undefined && input.maxCost !== undefined) {
+    checks.costBudget = costBudget({ actualCost: input.actualCost, maxCost: input.maxCost });
+  }
+
+  if (input.toolResults !== undefined && input.responseText !== undefined) {
+    checks.noHallucinatedNumbers = noHallucinatedNumbers({
+      responseText: input.responseText,
+      toolResults: input.toolResults,
+      tolerance: input.tolerance,
+    });
+  }
+
+  const results = Object.values(checks);
+  const passed = results.filter(r => r.pass).length;
+  const failed = results.filter(r => !r.pass).length;
+  const pass = failed === 0;
+
+  return { pass, checks, total: results.length, passed, failed };
+}

package/lib/checks/schema-match.js

@@ -0,0 +1,26 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{data: Object, requiredKeys: string[], typeChecks?: Object.<string, string>}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function schemaMatch({ data, requiredKeys, typeChecks }) {
+  const missingKeys = requiredKeys.filter(k => !(k in data));
+  if (missingKeys.length > 0) {
+    return { pass: false, reason: `Missing keys: ${missingKeys.join(', ')}` };
+  }
+  if (typeChecks) {
+    const typeErrors = [];
+    for (const [key, expectedType] of Object.entries(typeChecks)) {
+      const actualType = typeof data[key];
+      if (actualType !== expectedType) {
+        typeErrors.push(`${key}: expected ${expectedType}, got ${actualType}`);
+      }
+    }
+    if (typeErrors.length > 0) {
+      return { pass: false, reason: `Type mismatches: ${typeErrors.join('; ')}` };
+    }
+  }
+  return { pass: true };
+}

package/lib/checks/tool-call-count.js

@@ -0,0 +1,16 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{actual: number, min?: number, max?: number}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function toolCallCount({ actual, min, max }) {
+  if (min !== undefined && actual < min) {
+    return { pass: false, reason: `Tool call count ${actual} is below minimum ${min}` };
+  }
+  if (max !== undefined && actual > max) {
+    return { pass: false, reason: `Tool call count ${actual} exceeds maximum ${max}` };
+  }
+  return { pass: true };
+}

package/lib/checks/tool-selection.js

@@ -0,0 +1,34 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @param {{expected: string[], actual: string[], mode?: 'strict'|'subset'|'superset'|'unordered'}} input
+ * @returns {import('./types.js').EvalResult}
+ */
+export function toolSelection({ expected, actual, mode = 'strict' }) {
+  const expectedSet = new Set(expected);
+  const actualSet = new Set(actual);
+
+  if (mode === 'subset') {
+    // expected must be subset of actual
+    const missing = expected.filter(t => !actualSet.has(t));
+    if (missing.length === 0) return { pass: true };
+    return { pass: false, reason: `Expected tools not called: ${missing.join(', ')}` };
+  }
+
+  if (mode === 'superset') {
+    // actual must be subset of expected (expected is superset)
+    const unexpected = actual.filter(t => !expectedSet.has(t));
+    if (unexpected.length === 0) return { pass: true };
+    return { pass: false, reason: `Unexpected tools called: ${unexpected.join(', ')}` };
+  }
+
+  // strict / unordered: exact set equality
+  const missing = expected.filter(t => !actualSet.has(t));
+  const extra = actual.filter(t => !expectedSet.has(t));
+  if (missing.length === 0 && extra.length === 0) return { pass: true };
+  const parts = [];
+  if (missing.length) parts.push(`missing: ${missing.join(', ')}`);
+  if (extra.length) parts.push(`unexpected: ${extra.join(', ')}`);
+  return { pass: false, reason: parts.join('; ') };
+}

package/lib/checks/types.js

@@ -0,0 +1,45 @@
+// Adapted from evalkit by wkhori (https://github.com/wkhori/evalkit)
+// MIT License — see LICENSE
+
+/**
+ * @typedef {Object} EvalResult
+ * @property {boolean} pass
+ * @property {string} [reason]
+ */
+
+/**
+ * @typedef {Object} CheckSuiteResult
+ * @property {boolean} pass - true only if all checks passed
+ * @property {Object.<string, EvalResult>} checks - named check results
+ * @property {number} total
+ * @property {number} passed
+ * @property {number} failed
+ */
+
+/**
+ * @typedef {Object} RunChecksInput
+ * @property {string} [responseText]
+ * @property {string[]} [mustContain]
+ * @property {string[]} [mustNotContain]
+ * @property {string[]} [expectedTools]
+ * @property {string[]} [actualTools]
+ * @property {'strict'|'subset'|'superset'|'unordered'} [toolSelectionMode]
+ * @property {boolean} [nonEmpty]
+ * @property {string[]} [copOutPhrases]
+ * @property {boolean} [jsonValid]
+ * @property {Object} [schemaData]
+ * @property {string[]} [requiredKeys]
+ * @property {Object.<string, string>} [typeChecks]
+ * @property {number} [minLength]
+ * @property {number} [maxLength]
+ * @property {string|RegExp} [regexPattern]
+ * @property {unknown} [toolResults]
+ * @property {number} [tolerance]
+ * @property {number} [latencyMs]
+ * @property {number} [maxLatencyMs]
+ * @property {number} [actualToolCallCount]
+ * @property {number} [minToolCalls]
+ * @property {number} [maxToolCalls]
+ * @property {number} [actualCost]
+ * @property {number} [maxCost]
+ */

package/lib/comparison/compare.js

@@ -0,0 +1,86 @@
+// Adapted from agent-eval-kit by FlanaganSe (https://github.com/FlanaganSe/agent-eval-kit)
+// MIT License — see LICENSE
+
+/**
+ * Compare two eval runs, classifying each case as regression/improvement/unchanged/added/removed.
+ * Uses a configurable absolute significance threshold to classify regressions and improvements.
+ *
+ * @param {import('./types.js').RunSummary} baseRun
+ * @param {import('./types.js').RunSummary} compareRun
+ * @param {import('./types.js').ComparisonOptions} [options]
+ * @returns {import('./types.js').RunComparison}
+ */
+export function compareRuns(baseRun, compareRun, options = {}) {
+  const significanceThreshold = options.significanceThreshold ?? 0.1;
+
+  const baseCaseIds = new Set(Object.keys(baseRun.cases ?? {}));
+  const compareCaseIds = new Set(Object.keys(compareRun.cases ?? {}));
+  const allCaseIds = new Set([...baseCaseIds, ...compareCaseIds]);
+
+  const cases = [];
+  let regressions = 0, improvements = 0, unchanged = 0, added = 0, removed = 0;
+
+  for (const caseId of allCaseIds) {
+    const inBase = baseCaseIds.has(caseId);
+    const inCompare = compareCaseIds.has(caseId);
+
+    if (!inBase) {
+      cases.push({ caseId, status: 'added', comparePassRate: getPassRate(compareRun.cases[caseId]) });
+      added++;
+      continue;
+    }
+    if (!inCompare) {
+      cases.push({ caseId, status: 'removed', basePassRate: getPassRate(baseRun.cases[caseId]) });
+      removed++;
+      continue;
+    }
+
+    const basePassRate = getPassRate(baseRun.cases[caseId]);
+    const comparePassRate = getPassRate(compareRun.cases[caseId]);
+    const diff = comparePassRate - basePassRate;
+
+    let status = 'unchanged';
+    if (Math.abs(diff) >= significanceThreshold) {
+      status = diff < 0 ? 'regression' : 'improvement';
+    }
+
+    if (status === 'regression') regressions++;
+    else if (status === 'improvement') improvements++;
+    else unchanged++;
+
+    const baseMeanLatencyMs = getMeanLatency(baseRun.cases[caseId]);
+    const compareMeanLatencyMs = getMeanLatency(compareRun.cases[caseId]);
+
+    cases.push({ caseId, status, basePassRate, comparePassRate, baseMeanLatencyMs, compareMeanLatencyMs });
+  }
+
+  return {
+    base: baseRun,
+    compare: compareRun,
+    cases,
+    regressions,
+    improvements,
+    unchanged,
+    added,
+    removed,
+  };
+}
+
+/**
+ * @param {{pass: boolean}[]} trials
+ * @returns {number}
+ */
+function getPassRate(trials) {
+  if (!trials?.length) return 0;
+  return trials.filter(t => t.pass).length / trials.length;
+}
+
+/**
+ * @param {{latencyMs?: number}[]} trials
+ * @returns {number}
+ */
+function getMeanLatency(trials) {
+  if (!trials?.length) return 0;
+  const latencies = trials.map(t => t.latencyMs ?? 0);
+  return latencies.reduce((s, l) => s + l, 0) / latencies.length;
+}

package/lib/comparison/format.js

@@ -0,0 +1,104 @@
+// Adapted from agent-eval-kit by FlanaganSe (https://github.com/FlanaganSe/agent-eval-kit)
+// MIT License — see LICENSE
+
+import chalk from 'chalk';
+
+/**
+ * Format a comparison report as a string.
+ * @param {import('./types.js').RunComparison} comparison
+ * @param {{noColor?: boolean, verbose?: boolean}} [options]
+ * @returns {string}
+ */
+export function formatComparisonReport(comparison, options = {}) {
+  const { noColor = false, verbose = false } = options;
+  const c = noColor ? makeNoColor() : chalk;
+
+  const lines = [];
+
+  // Header
+  lines.push(c.bold('\n=== Eval Run Comparison ==='));
+  lines.push(`Base:    ${comparison.base.modelName} (${comparison.base.runId})`);
+  lines.push(`Compare: ${comparison.compare.modelName} (${comparison.compare.runId})`);
+  lines.push('');
+
+  // Summary stats
+  const baseRate = (comparison.base.passRate * 100).toFixed(1);
+  const cmpRate = (comparison.compare.passRate * 100).toFixed(1);
+  const rateDiff = comparison.compare.passRate - comparison.base.passRate;
+  const rateStr = rateDiff >= 0
+    ? c.green(`+${(rateDiff * 100).toFixed(1)}%`)
+    : c.red(`${(rateDiff * 100).toFixed(1)}%`);
+
+  lines.push(`Pass rate: ${baseRate}% → ${cmpRate}% (${rateStr})`);
+  lines.push(`Cases: ${comparison.regressions > 0 ? c.red(`${comparison.regressions} regressions`) : '0 regressions'}, ${comparison.improvements > 0 ? c.green(`${comparison.improvements} improvements`) : '0 improvements'}, ${comparison.unchanged} unchanged`);
+  if (comparison.added > 0) lines.push(`  + ${comparison.added} new cases added`);
+  if (comparison.removed > 0) lines.push(`  - ${comparison.removed} cases removed`);
+  lines.push('');
+
+  // Regressions
+  const regressions = comparison.cases.filter(cas => cas.status === 'regression');
+  if (regressions.length > 0) {
+    lines.push(c.red.bold('Regressions:'));
+    for (const cas of regressions) {
+      const base = (cas.basePassRate * 100).toFixed(0);
+      const cmp = (cas.comparePassRate * 100).toFixed(0);
+      lines.push(`  ${c.red('✗')} ${cas.caseId}: ${base}% → ${cmp}%`);
+    }
+    lines.push('');
+  }
+
+  // Improvements
+  const improvements = comparison.cases.filter(cas => cas.status === 'improvement');
+  if (improvements.length > 0) {
+    lines.push(c.green.bold('Improvements:'));
+    for (const cas of improvements) {
+      const base = (cas.basePassRate * 100).toFixed(0);
+      const cmp = (cas.comparePassRate * 100).toFixed(0);
+      lines.push(`  ${c.green('✓')} ${cas.caseId}: ${base}% → ${cmp}%`);
+    }
+    lines.push('');
+  }
+
+  // Verbose: show all unchanged cases too
+  if (verbose) {
+    const unchanged = comparison.cases.filter(cas => cas.status === 'unchanged');
+    if (unchanged.length > 0) {
+      lines.push(c.gray('Unchanged:'));
+      for (const cas of unchanged) {
+        const rate = (cas.comparePassRate * 100).toFixed(0);
+        lines.push(`  ${c.gray('·')} ${cas.caseId}: ${rate}%`);
+      }
+      lines.push('');
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Create a chalk-compatible no-color proxy.
+ * @returns {typeof chalk}
+ */
+function makeNoColor() {
+  const identity = (...args) => {
+    // Handle tagged template literal calls: args[0] is a TemplateStringsArray
+    if (Array.isArray(args[0]) && args[0].raw) {
+      return String.raw({ raw: args[0].raw }, ...args.slice(1));
+    }
+    return args[0] ?? '';
+  };
+
+  function makeProxy(fn) {
+    return new Proxy(fn, {
+      get(_, prop) {
+        // Return a new proxy for any property access (supports unlimited chaining)
+        return makeProxy(identity);
+      },
+      apply(_, _this, args) {
+        return identity(...args);
+      }
+    });
+  }
+
+  return makeProxy(identity);
+}

package/lib/comparison/index.js

@@ -0,0 +1,6 @@
+// Adapted from agent-eval-kit by FlanaganSe (https://github.com/FlanaganSe/agent-eval-kit)
+// MIT License — see LICENSE
+
+export { wilsonInterval, computeTrialStats, computeAllTrialStats } from './statistics.js';
+export { compareRuns } from './compare.js';
+export { formatComparisonReport } from './format.js';

package/lib/comparison/statistics.js

@@ -0,0 +1,59 @@
+// Adapted from agent-eval-kit by FlanaganSe (https://github.com/FlanaganSe/agent-eval-kit)
+// MIT License — see LICENSE
+
+/**
+ * Compute Wilson score confidence interval for a proportion.
+ * @param {number} passes - number of successes
+ * @param {number} total - total trials
+ * @param {number} [z=1.96] - z-score (1.96 = 95% CI)
+ * @returns {{lower: number, upper: number, center: number}}
+ */
+export function wilsonInterval(passes, total, z = 1.96) {
+  if (total === 0) return { lower: 0, upper: 0, center: 0 };
+  const p = passes / total;
+  const z2 = z * z;
+  const n = total;
+  const center = (p + z2 / (2 * n)) / (1 + z2 / n);
+  const margin = (z / (1 + z2 / n)) * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
+  return {
+    lower: Math.max(0, center - margin),
+    upper: Math.min(1, center + margin),
+    center,
+  };
+}
+
+/**
+ * Compute statistics for a list of trial results.
+ * @param {{pass: boolean, latencyMs?: number}[]} trials
+ * @returns {{passRate: number, lower95: number, upper95: number, meanLatencyMs: number, p95LatencyMs: number}}
+ */
+export function computeTrialStats(trials) {
+  if (trials.length === 0) return { passRate: 0, lower95: 0, upper95: 0, meanLatencyMs: 0, p95LatencyMs: 0 };
+
+  const passes = trials.filter(t => t.pass).length;
+  const { lower, upper } = wilsonInterval(passes, trials.length);
+
+  const latencies = trials.map(t => t.latencyMs ?? 0).sort((a, b) => a - b);
+  const meanLatencyMs = latencies.reduce((s, l) => s + l, 0) / latencies.length;
+  const p95Index = Math.floor((latencies.length - 1) * 0.95);
+  const p95LatencyMs = latencies[Math.min(p95Index, latencies.length - 1)] ?? 0;
+
+  return {
+    passRate: passes / trials.length,
+    lower95: lower,
+    upper95: upper,
+    meanLatencyMs,
+    p95LatencyMs,
+  };
+}
+
+/**
+ * Compute stats for all cases in a run.
+ * @param {Object.<string, {pass: boolean, latencyMs?: number}[]>} allTrials
+ * @returns {Object.<string, ReturnType<typeof computeTrialStats>>}
+ */
+export function computeAllTrialStats(allTrials) {
+  return Object.fromEntries(
+    Object.entries(allTrials).map(([caseId, trials]) => [caseId, computeTrialStats(trials)])
+  );
+}