@plaited/agent-eval-harness 0.7.0 → 0.8.1

package/README.md

@@ -78,6 +78,9 @@ cat prompts.jsonl | \
 
 # Compare runs (built-in strategies: weighted, statistical, custom)
 bunx @plaited/agent-eval-harness compare run1.jsonl run2.jsonl -o comparison.json
+
+# Compare trials for pass@k reliability analysis (auto-detects format)
+bunx @plaited/agent-eval-harness compare trials1.jsonl trials2.jsonl -o comparison.json
 ```
 
 ## Skills for AI Agents

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plaited/agent-eval-harness",
-  "version": "0.7.0",
+  "version": "0.8.1",
   "description": "CLI tool for capturing agent trajectories from headless CLI agents",
   "license": "ISC",
   "engines": {
@@ -42,8 +42,10 @@
     "check:types": "tsc --noEmit",
     "check:write": "biome check --write && format-package --write",
     "prepare": "git rev-parse --git-dir > /dev/null 2>&1 && git config core.hooksPath .hooks || true",
-    "test": "bun test ./**/tests/*.spec.ts",
-    "test:integration": "bun test ./**/integration_tests/*.spec.ts"
+    "test": "bun run test:bin && bun test:src",
+    "test:bin": "bun test bin/tests/*.spec.ts",
+    "test:integration": "bun test ./**/integration_tests/*.spec.ts",
+    "test:src": "bun test src/**/tests/*.spec.ts"
   },
   "lint-staged": {
     "*.{js,cjs,jsx,tsx,ts}": [

package/src/graders/bootstrap.ts

@@ -0,0 +1,135 @@
+/**
+ * Shared bootstrap sampling utilities for confidence interval computation.
+ *
+ * @remarks
+ * Bootstrap resampling provides robust confidence intervals without
+ * assuming a specific distribution. For small samples, it's more
+ * reliable than parametric methods.
+ *
+ * Environment variable configuration:
+ * - `COMPARE_BOOTSTRAP_ITERATIONS` (default: 1000)
+ *
+ * @packageDocumentation
+ */
+
+/** Default number of bootstrap iterations */
+export const DEFAULT_ITERATIONS = 1000
+
+/** Default confidence level (95%) */
+export const DEFAULT_CONFIDENCE_LEVEL = 0.95
+
+/**
+ * Confidence interval as [lower, upper] bounds.
+ */
+export type ConfidenceInterval = [number, number]
+
+/**
+ * Bootstrap confidence interval result.
+ */
+export type BootstrapResult = {
+  /** Median of bootstrap sample means (50th percentile) */
+  median: number
+  /** Confidence interval [lower, upper] */
+  ci: ConfidenceInterval
+}
+
+/**
+ * Configuration for bootstrap sampling.
+ */
+export type BootstrapConfig = {
+  /** Number of bootstrap iterations (default: 1000) */
+  iterations?: number
+  /** Confidence level between 0 and 1 (default: 0.95) */
+  confidenceLevel?: number
+}
+
+/**
+ * Compute bootstrap confidence interval for sample mean.
+ *
+ * @remarks
+ * Bootstrap resampling provides robust confidence intervals without
+ * assuming a specific distribution. For small samples, it's more
+ * reliable than parametric methods.
+ *
+ * @param samples - Array of numeric samples
+ * @param config - Optional bootstrap configuration
+ * @returns Bootstrap median and confidence interval
+ *
+ * @public
+ */
+export const bootstrap = (samples: number[], config?: BootstrapConfig): BootstrapResult => {
+  const iterations = config?.iterations ?? DEFAULT_ITERATIONS
+  const confidenceLevel = config?.confidenceLevel ?? DEFAULT_CONFIDENCE_LEVEL
+
+  if (samples.length === 0) {
+    return { median: 0, ci: [0, 0] }
+  }
+
+  if (samples.length === 1) {
+    const value = samples[0] ?? 0
+    return { median: value, ci: [value, value] }
+  }
+
+  const means: number[] = []
+
+  for (let i = 0; i < iterations; i++) {
+    // Resample with replacement - we know samples.length > 1 at this point
+    const resampled = Array.from(
+      { length: samples.length },
+      () => samples[Math.floor(Math.random() * samples.length)] as number,
+    )
+
+    // Compute mean of resampled data
+    const sum = resampled.reduce((acc, val) => acc + val, 0)
+    means.push(sum / resampled.length)
+  }
+
+  // Sort means for percentile calculation
+  means.sort((a, b) => a - b)
+
+  // Compute percentile indices based on confidence level
+  // For 95% CI: lower = 2.5th percentile, upper = 97.5th percentile
+  const alpha = (1 - confidenceLevel) / 2
+  const lowerIdx = Math.floor(iterations * alpha)
+  const upperIdx = Math.floor(iterations * (1 - alpha))
+
+  return {
+    median: means[Math.floor(iterations / 2)] ?? 0,
+    ci: [means[lowerIdx] ?? 0, means[upperIdx] ?? 0],
+  }
+}
+
+/**
+ * Format confidence interval as string.
+ *
+ * @param ci - Confidence interval [lower, upper]
+ * @param decimals - Number of decimal places (default: 3)
+ * @returns Formatted CI string or empty string if undefined
+ *
+ * @public
+ */
+export const formatCI = (ci: ConfidenceInterval | undefined, decimals: number = 3): string => {
+  if (!ci) return ''
+  return `[${ci[0].toFixed(decimals)}, ${ci[1].toFixed(decimals)}]`
+}
+
+/**
+ * Get bootstrap configuration from environment variables.
+ *
+ * @remarks
+ * Reads configuration from:
+ * - `COMPARE_BOOTSTRAP_ITERATIONS`: Number of iterations (min: 100)
+ *
+ * @returns Bootstrap configuration
+ *
+ * @public
+ */
+export const getBootstrapConfigFromEnv = (): BootstrapConfig => {
+  const envValue = process.env.COMPARE_BOOTSTRAP_ITERATIONS
+  if (!envValue) return { iterations: DEFAULT_ITERATIONS }
+
+  const parsed = Number.parseInt(envValue, 10)
+  const iterations = Number.isNaN(parsed) || parsed < 100 ? DEFAULT_ITERATIONS : parsed
+
+  return { iterations }
+}

package/src/graders/compare-statistical.ts

@@ -12,81 +12,7 @@
  */
 
 import type { ComparisonGrader, ComparisonGraderInput, ComparisonGraderResult } from '../pipeline/pipeline.types.ts'
-
-/** Default number of bootstrap iterations */
-const DEFAULT_ITERATIONS = 1000
-
-/**
- * Bootstrap confidence interval result.
- */
-type BootstrapResult = {
-  /** Estimated mean from bootstrap */
-  mean: number
-  /** 95% confidence interval [lower, upper] */
-  ci95: [number, number]
-}
-
-/**
- * Compute bootstrap confidence interval for sample mean.
- *
- * @remarks
- * Bootstrap resampling provides robust confidence intervals without
- * assuming a specific distribution. For small samples, it's more
- * reliable than parametric methods.
- *
- * @param samples - Array of numeric samples
- * @param iterations - Number of bootstrap iterations
- * @returns Bootstrap mean and 95% confidence interval
- */
-const bootstrap = (samples: number[], iterations: number = DEFAULT_ITERATIONS): BootstrapResult => {
-  if (samples.length === 0) {
-    return { mean: 0, ci95: [0, 0] }
-  }
-
-  if (samples.length === 1) {
-    const value = samples[0] ?? 0
-    return { mean: value, ci95: [value, value] }
-  }
-
-  const means: number[] = []
-
-  for (let i = 0; i < iterations; i++) {
-    // Resample with replacement - we know samples.length > 1 at this point
-    const resampled = Array.from(
-      { length: samples.length },
-      () => samples[Math.floor(Math.random() * samples.length)] as number,
-    )
-
-    // Compute mean of resampled data
-    const sum = resampled.reduce((acc, val) => acc + val, 0)
-    means.push(sum / resampled.length)
-  }
-
-  // Sort means for percentile calculation
-  means.sort((a, b) => a - b)
-
-  // 95% CI: 2.5th and 97.5th percentiles
-  const lowerIdx = Math.floor(iterations * 0.025)
-  const upperIdx = Math.floor(iterations * 0.975)
-
-  return {
-    mean: means[Math.floor(iterations / 2)] ?? 0,
-    ci95: [means[lowerIdx] ?? 0, means[upperIdx] ?? 0],
-  }
-}
-
-/**
- * Get bootstrap iterations from environment variable.
- *
- * @returns Number of bootstrap iterations
- */
-const getIterationsFromEnv = (): number => {
-  const envValue = process.env.COMPARE_BOOTSTRAP_ITERATIONS
-  if (!envValue) return DEFAULT_ITERATIONS
-
-  const parsed = Number.parseInt(envValue, 10)
-  return Number.isNaN(parsed) || parsed < 100 ? DEFAULT_ITERATIONS : parsed
-}
+import { bootstrap, getBootstrapConfigFromEnv } from './bootstrap.ts'
 
 /**
  * Statistical significance comparison grader.
@@ -107,7 +33,7 @@ const getIterationsFromEnv = (): number => {
  * @public
  */
 export const grade: ComparisonGrader = async ({ runs }: ComparisonGraderInput): Promise<ComparisonGraderResult> => {
-  const iterations = getIterationsFromEnv()
+  const config = getBootstrapConfigFromEnv()
 
   // Collect scores for each run
   const runStats = Object.entries(runs).map(([label, run]) => {
@@ -116,13 +42,13 @@ export const grade: ComparisonGrader = async ({ runs }: ComparisonGraderInput):
 
     // For single-prompt comparison, we only have one sample
     // In practice, this grader is most useful when aggregating across prompts
-    const stats = bootstrap([score], iterations)
+    const stats = bootstrap([score], config)
 
     return { label, score, stats }
   })
 
-  // Sort by bootstrap mean descending
-  const sorted = runStats.sort((a, b) => b.stats.mean - a.stats.mean)
+  // Sort by bootstrap median descending
+  const sorted = runStats.sort((a, b) => b.stats.median - a.stats.median)
 
   // Check if winner is statistically significant
   // CIs don't overlap = significant difference (approximately p<0.05)
@@ -131,7 +57,7 @@ export const grade: ComparisonGrader = async ({ runs }: ComparisonGraderInput):
   const second = sorted[1]
   if (first && second) {
     // Non-overlapping: first's lower bound > second's upper bound
-    isSignificant = first.stats.ci95[0] > second.stats.ci95[1]
+    isSignificant = first.stats.ci[0] > second.stats.ci[1]
   }
 
   const reasoning = isSignificant
@@ -142,7 +68,7 @@ export const grade: ComparisonGrader = async ({ runs }: ComparisonGraderInput):
     rankings: sorted.map((s, i) => ({
       run: s.label,
       rank: i + 1,
-      score: s.stats.mean,
+      score: s.stats.median,
     })),
     reasoning,
   }
@@ -156,28 +82,30 @@ export const grade: ComparisonGrader = async ({ runs }: ComparisonGraderInput):
  *
  * @public
  */
-export const createStatisticalGrader = (iterations: number = DEFAULT_ITERATIONS): ComparisonGrader => {
+export const createStatisticalGrader = (iterations?: number): ComparisonGrader => {
+  const config = iterations ? { iterations } : getBootstrapConfigFromEnv()
+
   return async ({ runs }: ComparisonGraderInput): Promise<ComparisonGraderResult> => {
     const runStats = Object.entries(runs).map(([label, run]) => {
       const score = run.score?.score ?? 0
-      const stats = bootstrap([score], iterations)
+      const stats = bootstrap([score], config)
       return { label, score, stats }
     })
 
-    const sorted = runStats.sort((a, b) => b.stats.mean - a.stats.mean)
+    const sorted = runStats.sort((a, b) => b.stats.median - a.stats.median)
 
     let isSignificant = false
     const first = sorted[0]
     const second = sorted[1]
     if (first && second) {
-      isSignificant = first.stats.ci95[0] > second.stats.ci95[1]
+      isSignificant = first.stats.ci[0] > second.stats.ci[1]
     }
 
     return {
       rankings: sorted.map((s, i) => ({
         run: s.label,
         rank: i + 1,
-        score: s.stats.mean,
+        score: s.stats.median,
       })),
       reasoning: isSignificant
         ? `Winner "${first?.label}" is statistically significant (p<0.05)`

package/src/graders/tests/bootstrap.spec.ts

@@ -0,0 +1,169 @@
+/**
+ * Unit tests for bootstrap sampling utilities.
+ */
+
+import { afterEach, describe, expect, test } from 'bun:test'
+import { bootstrap, DEFAULT_CONFIDENCE_LEVEL, DEFAULT_ITERATIONS, getBootstrapConfigFromEnv } from '../bootstrap.ts'
+
+describe('bootstrap', () => {
+  describe('edge cases', () => {
+    test('returns {median: 0, ci: [0, 0]} for empty array', () => {
+      const result = bootstrap([])
+      expect(result.median).toBe(0)
+      expect(result.ci).toEqual([0, 0])
+    })
+
+    test('returns {median: value, ci: [value, value]} for single sample', () => {
+      const result = bootstrap([0.75])
+      expect(result.median).toBe(0.75)
+      expect(result.ci).toEqual([0.75, 0.75])
+    })
+
+    test('handles single sample of 0', () => {
+      const result = bootstrap([0])
+      expect(result.median).toBe(0)
+      expect(result.ci).toEqual([0, 0])
+    })
+
+    test('handles single sample of 1', () => {
+      const result = bootstrap([1])
+      expect(result.median).toBe(1)
+      expect(result.ci).toEqual([1, 1])
+    })
+  })
+
+  describe('confidence interval bounds', () => {
+    test('CI lower bound <= median <= CI upper bound', () => {
+      const samples = [0.5, 0.6, 0.7, 0.8, 0.9]
+      const result = bootstrap(samples, { iterations: 1000 })
+
+      expect(result.ci[0]).toBeLessThanOrEqual(result.median)
+      expect(result.median).toBeLessThanOrEqual(result.ci[1])
+    })
+
+    test('CI contains the true median for uniform samples', () => {
+      // For identical samples, CI should collapse to the value
+      const samples = [0.5, 0.5, 0.5, 0.5, 0.5]
+      const result = bootstrap(samples, { iterations: 1000 })
+
+      expect(result.median).toBeCloseTo(0.5, 2)
+      expect(result.ci[0]).toBeCloseTo(0.5, 2)
+      expect(result.ci[1]).toBeCloseTo(0.5, 2)
+    })
+
+    test('CI widens with more variance in samples', () => {
+      const lowVariance = [0.49, 0.5, 0.51]
+      const highVariance = [0.1, 0.5, 0.9]
+
+      const lowResult = bootstrap(lowVariance, { iterations: 1000 })
+      const highResult = bootstrap(highVariance, { iterations: 1000 })
+
+      const lowWidth = lowResult.ci[1] - lowResult.ci[0]
+      const highWidth = highResult.ci[1] - highResult.ci[0]
+
+      expect(highWidth).toBeGreaterThan(lowWidth)
+    })
+  })
+
+  describe('configuration', () => {
+    test('uses default iterations when not specified', () => {
+      // Just verify it runs without error with defaults
+      const result = bootstrap([0.5, 0.6, 0.7])
+      expect(result.median).toBeGreaterThan(0)
+    })
+
+    test('accepts custom iteration count', () => {
+      const result = bootstrap([0.5, 0.6, 0.7], { iterations: 100 })
+      expect(result.median).toBeGreaterThan(0)
+    })
+
+    test('accepts custom confidence level', () => {
+      const samples = [0.3, 0.4, 0.5, 0.6, 0.7]
+
+      // 90% CI should be narrower than 95% CI
+      const ci90 = bootstrap(samples, { iterations: 1000, confidenceLevel: 0.9 })
+      const ci95 = bootstrap(samples, { iterations: 1000, confidenceLevel: 0.95 })
+
+      const width90 = ci90.ci[1] - ci90.ci[0]
+      const width95 = ci95.ci[1] - ci95.ci[0]
+
+      // 95% CI should generally be wider than 90% CI
+      // Allow some tolerance due to randomness
+      expect(width95).toBeGreaterThanOrEqual(width90 * 0.8)
+    })
+  })
+
+  describe('statistical properties', () => {
+    test('median is close to sample mean', () => {
+      const samples = [0.2, 0.4, 0.6, 0.8, 1.0]
+      const sampleMean = samples.reduce((a, b) => a + b, 0) / samples.length
+
+      const result = bootstrap(samples, { iterations: 10000 })
+
+      // Bootstrap median should be close to sample mean for symmetric distributions
+      expect(result.median).toBeCloseTo(sampleMean, 1)
+    })
+
+    test('is deterministic-ish for large iteration counts', () => {
+      const samples = [0.3, 0.5, 0.7]
+
+      // With many iterations, results should be similar across runs
+      const result1 = bootstrap(samples, { iterations: 10000 })
+      const result2 = bootstrap(samples, { iterations: 10000 })
+
+      expect(result1.median).toBeCloseTo(result2.median, 1)
+    })
+  })
+})
+
+describe('getBootstrapConfigFromEnv', () => {
+  const originalEnv = process.env.COMPARE_BOOTSTRAP_ITERATIONS
+
+  afterEach(() => {
+    if (originalEnv === undefined) {
+      delete process.env.COMPARE_BOOTSTRAP_ITERATIONS
+    } else {
+      process.env.COMPARE_BOOTSTRAP_ITERATIONS = originalEnv
+    }
+  })
+
+  test('returns default iterations when env var not set', () => {
+    delete process.env.COMPARE_BOOTSTRAP_ITERATIONS
+    const config = getBootstrapConfigFromEnv()
+    expect(config.iterations).toBe(DEFAULT_ITERATIONS)
+  })
+
+  test('parses valid iteration count from env', () => {
+    process.env.COMPARE_BOOTSTRAP_ITERATIONS = '5000'
+    const config = getBootstrapConfigFromEnv()
+    expect(config.iterations).toBe(5000)
+  })
+
+  test('returns default for invalid (non-numeric) env value', () => {
+    process.env.COMPARE_BOOTSTRAP_ITERATIONS = 'invalid'
+    const config = getBootstrapConfigFromEnv()
+    expect(config.iterations).toBe(DEFAULT_ITERATIONS)
+  })
+
+  test('returns default for iteration count below minimum (100)', () => {
+    process.env.COMPARE_BOOTSTRAP_ITERATIONS = '50'
+    const config = getBootstrapConfigFromEnv()
+    expect(config.iterations).toBe(DEFAULT_ITERATIONS)
+  })
+
+  test('accepts iteration count at minimum (100)', () => {
+    process.env.COMPARE_BOOTSTRAP_ITERATIONS = '100'
+    const config = getBootstrapConfigFromEnv()
+    expect(config.iterations).toBe(100)
+  })
+})
+
+describe('constants', () => {
+  test('DEFAULT_ITERATIONS is 1000', () => {
+    expect(DEFAULT_ITERATIONS).toBe(1000)
+  })
+
+  test('DEFAULT_CONFIDENCE_LEVEL is 0.95', () => {
+    expect(DEFAULT_CONFIDENCE_LEVEL).toBe(0.95)
+  })
+})