@plaited/agent-eval-harness 0.7.0 → 0.8.0

package/README.md

@@ -78,6 +78,9 @@ cat prompts.jsonl | \
 
 # Compare runs (built-in strategies: weighted, statistical, custom)
 bunx @plaited/agent-eval-harness compare run1.jsonl run2.jsonl -o comparison.json
+
+# Compare trials for pass@k reliability analysis (auto-detects format)
+bunx @plaited/agent-eval-harness compare trials1.jsonl trials2.jsonl -o comparison.json
 ```
 
 ## Skills for AI Agents

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plaited/agent-eval-harness",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "CLI tool for capturing agent trajectories from headless CLI agents",
   "license": "ISC",
   "engines": {

package/src/graders/tests/trials-compare-graders.spec.ts

@@ -0,0 +1,358 @@
+/**
+ * Unit tests for built-in trials comparison graders.
+ *
+ * @remarks
+ * Tests for:
+ * - trials-compare-weighted: Configurable weight grader for trials
+ * - trials-compare-statistical: Bootstrap confidence interval grader for trials
+ *
+ * @packageDocumentation
+ */
+
+import { describe, expect, test } from 'bun:test'
+import type { TrialsComparisonGraderInput, TrialsComparisonRunData } from '../../pipeline/pipeline.types.ts'
+import { createTrialsStatisticalGrader, grade as statisticalGrade } from '../trials-compare-statistical.ts'
+import { createTrialsWeightedGrader, DEFAULT_TRIALS_WEIGHTS, type TrialsWeights } from '../trials-compare-weighted.ts'
+
+// ============================================================================
+// Test Fixtures
+// ============================================================================
+
+const createMockTrialRuns = (
+  overrides: Partial<Record<string, Partial<TrialsComparisonRunData>>> = {},
+): Record<string, TrialsComparisonRunData> => ({
+  baseline: {
+    passRate: 0.67,
+    passAtK: 0.9,
+    passExpK: 0.3,
+    k: 3,
+    trials: [
+      { trialNum: 1, output: 'A', trajectory: [], duration: 100, pass: true, score: 1.0 },
+      { trialNum: 2, output: 'B', trajectory: [], duration: 110, pass: true, score: 0.9 },
+      { trialNum: 3, output: 'C', trajectory: [], duration: 120, pass: false, score: 0.2 },
+    ],
+    ...overrides.baseline,
+  },
+  variant: {
+    passRate: 1.0,
+    passAtK: 1.0,
+    passExpK: 1.0,
+    k: 3,
+    trials: [
+      { trialNum: 1, output: 'X', trajectory: [], duration: 150, pass: true, score: 1.0 },
+      { trialNum: 2, output: 'Y', trajectory: [], duration: 160, pass: true, score: 1.0 },
+      { trialNum: 3, output: 'Z', trajectory: [], duration: 170, pass: true, score: 1.0 },
+    ],
+    ...overrides.variant,
+  },
+})
+
+const createMockTrialInput = (runs: Record<string, TrialsComparisonRunData>): TrialsComparisonGraderInput => ({
+  id: 'test-001',
+  input: 'Test prompt',
+  hint: 'Expected output',
+  runs,
+})
+
+// ============================================================================
+// Weighted Grader Tests
+// ============================================================================
+
+describe('trials-compare-weighted grader', () => {
+  describe('DEFAULT_TRIALS_WEIGHTS', () => {
+    test('has expected default values', () => {
+      expect(DEFAULT_TRIALS_WEIGHTS.capability).toBe(0.4)
+      expect(DEFAULT_TRIALS_WEIGHTS.reliability).toBe(0.4)
+      expect(DEFAULT_TRIALS_WEIGHTS.consistency).toBe(0.2)
+    })
+
+    test('weights sum to 1.0', () => {
+      const sum =
+        DEFAULT_TRIALS_WEIGHTS.capability + DEFAULT_TRIALS_WEIGHTS.reliability + DEFAULT_TRIALS_WEIGHTS.consistency
+      expect(sum).toBe(1.0)
+    })
+  })
+
+  describe('createTrialsWeightedGrader', () => {
+    test('returns higher rank for better passAtK when capability weight is high', async () => {
+      const grader = createTrialsWeightedGrader({ capability: 1.0, reliability: 0.0, consistency: 0.0 })
+      const runs = createMockTrialRuns({
+        baseline: { passAtK: 0.7 },
+        variant: { passAtK: 0.95 },
+      })
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      expect(result.rankings.length).toBe(2)
+      expect(result.rankings[0]?.run).toBe('variant')
+      expect(result.rankings[0]?.rank).toBe(1)
+    })
+
+    test('returns higher rank for better passExpK when reliability weight is high', async () => {
+      const grader = createTrialsWeightedGrader({ capability: 0.0, reliability: 1.0, consistency: 0.0 })
+      const runs = createMockTrialRuns({
+        baseline: { passExpK: 0.9 },
+        variant: { passExpK: 0.3 },
+      })
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      expect(result.rankings[0]?.run).toBe('baseline')
+    })
+
+    test('penalizes flaky runs when consistency weight is high', async () => {
+      const grader = createTrialsWeightedGrader({ capability: 0.0, reliability: 0.0, consistency: 1.0 })
+      const runs = createMockTrialRuns({
+        // baseline: passAtK=0.9, passExpK=0.3, flakiness=0.6
+        baseline: { passAtK: 0.9, passExpK: 0.3 },
+        // variant: passAtK=0.8, passExpK=0.8, flakiness=0.0
+        variant: { passAtK: 0.8, passExpK: 0.8 },
+      })
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      // Variant should win due to lower flakiness (higher consistency)
+      expect(result.rankings[0]?.run).toBe('variant')
+    })
+
+    test('includes weights in reasoning', async () => {
+      const weights: TrialsWeights = { capability: 0.5, reliability: 0.3, consistency: 0.2 }
+      const grader = createTrialsWeightedGrader(weights)
+      const input = createMockTrialInput(createMockTrialRuns())
+
+      const result = await grader(input)
+
+      expect(result.reasoning).toContain('capability=0.5')
+      expect(result.reasoning).toContain('reliability=0.3')
+      expect(result.reasoning).toContain('consistency=0.2')
+    })
+
+    test('handles missing passAtK gracefully (treats as 0)', async () => {
+      const grader = createTrialsWeightedGrader()
+      const runs: Record<string, TrialsComparisonRunData> = {
+        baseline: {
+          k: 3,
+          trials: [],
+        },
+        variant: {
+          passAtK: 0.8,
+          passExpK: 0.5,
+          k: 3,
+          trials: [],
+        },
+      }
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      // Should not throw, variant should rank higher
+      expect(result.rankings.length).toBe(2)
+      expect(result.rankings[0]?.run).toBe('variant')
+    })
+
+    test('handles three or more runs', async () => {
+      const grader = createTrialsWeightedGrader()
+      const runs: Record<string, TrialsComparisonRunData> = {
+        a: { passAtK: 0.9, passExpK: 0.8, k: 3, trials: [] },
+        b: { passAtK: 0.7, passExpK: 0.7, k: 3, trials: [] },
+        c: { passAtK: 0.5, passExpK: 0.2, k: 3, trials: [] },
+      }
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      expect(result.rankings.length).toBe(3)
+      // Ranks should be 1, 2, 3
+      expect(result.rankings.map((r) => r.rank)).toEqual([1, 2, 3])
+    })
+  })
+})
+
+// ============================================================================
+// Statistical Grader Tests
+// ============================================================================
+
+describe('trials-compare-statistical grader', () => {
+  describe('createTrialsStatisticalGrader', () => {
+    test('returns rankings based on bootstrapped passAtK', async () => {
+      const grader = createTrialsStatisticalGrader(100)
+      const runs = createMockTrialRuns({
+        baseline: { passAtK: 0.6 },
+        variant: { passAtK: 0.95 },
+      })
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      expect(result.rankings.length).toBe(2)
+      expect(result.rankings[0]?.run).toBe('variant')
+    })
+
+    test('uses trial outcomes for bootstrap variance estimation', async () => {
+      const grader = createTrialsStatisticalGrader(100)
+      // All trials pass for variant, mixed for baseline
+      const runs: Record<string, TrialsComparisonRunData> = {
+        baseline: {
+          passAtK: 0.9,
+          passExpK: 0.3,
+          k: 5,
+          trials: [
+            { trialNum: 1, output: 'A', trajectory: [], duration: 100, pass: true },
+            { trialNum: 2, output: 'B', trajectory: [], duration: 100, pass: true },
+            { trialNum: 3, output: 'C', trajectory: [], duration: 100, pass: false },
+            { trialNum: 4, output: 'D', trajectory: [], duration: 100, pass: true },
+            { trialNum: 5, output: 'E', trajectory: [], duration: 100, pass: false },
+          ],
+        },
+        variant: {
+          passAtK: 1.0,
+          passExpK: 1.0,
+          k: 5,
+          trials: [
+            { trialNum: 1, output: 'X', trajectory: [], duration: 100, pass: true },
+            { trialNum: 2, output: 'Y', trajectory: [], duration: 100, pass: true },
+            { trialNum: 3, output: 'Z', trajectory: [], duration: 100, pass: true },
+            { trialNum: 4, output: 'W', trajectory: [], duration: 100, pass: true },
+            { trialNum: 5, output: 'V', trajectory: [], duration: 100, pass: true },
+          ],
+        },
+      }
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      // Variant with 100% pass rate should rank higher
+      expect(result.rankings[0]?.run).toBe('variant')
+    })
+
+    test('indicates significance when passAtK differs substantially', async () => {
+      const grader = createTrialsStatisticalGrader(500)
+      // Strong difference: all pass vs all fail
+      const runs: Record<string, TrialsComparisonRunData> = {
+        baseline: {
+          passAtK: 0,
+          k: 5,
+          trials: [
+            { trialNum: 1, output: 'A', trajectory: [], duration: 100, pass: false },
+            { trialNum: 2, output: 'B', trajectory: [], duration: 100, pass: false },
+            { trialNum: 3, output: 'C', trajectory: [], duration: 100, pass: false },
+            { trialNum: 4, output: 'D', trajectory: [], duration: 100, pass: false },
+            { trialNum: 5, output: 'E', trajectory: [], duration: 100, pass: false },
+          ],
+        },
+        variant: {
+          passAtK: 1.0,
+          k: 5,
+          trials: [
+            { trialNum: 1, output: 'X', trajectory: [], duration: 100, pass: true },
+            { trialNum: 2, output: 'Y', trajectory: [], duration: 100, pass: true },
+            { trialNum: 3, output: 'Z', trajectory: [], duration: 100, pass: true },
+            { trialNum: 4, output: 'W', trajectory: [], duration: 100, pass: true },
+            { trialNum: 5, output: 'V', trajectory: [], duration: 100, pass: true },
+          ],
+        },
+      }
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      expect(result.reasoning).toContain('clear separation')
+    })
+
+    test('handles empty trials array', async () => {
+      const grader = createTrialsStatisticalGrader(100)
+      const runs: Record<string, TrialsComparisonRunData> = {
+        baseline: { k: 3, trials: [] },
+        variant: {
+          k: 3,
+          trials: [{ trialNum: 1, output: 'X', trajectory: [], duration: 100, pass: true }],
+        },
+      }
+      const input = createMockTrialInput(runs)
+
+      const result = await grader(input)
+
+      // Should not throw
+      expect(result.rankings.length).toBe(2)
+    })
+  })
+
+  describe('grade function', () => {
+    test('works with default iterations', async () => {
+      const runs = createMockTrialRuns()
+      const input = createMockTrialInput(runs)
+
+      const result = await statisticalGrade(input)
+
+      expect(result.rankings).toBeDefined()
+      expect(result.rankings.length).toBe(2)
+    })
+  })
+})
+
+// ============================================================================
+// Edge Case Tests
+// ============================================================================
+
+describe('trials comparison grader edge cases', () => {
+  test('handles single run gracefully', async () => {
+    const grader = createTrialsWeightedGrader()
+    const runs: Record<string, TrialsComparisonRunData> = {
+      only: { passAtK: 1.0, passExpK: 0.8, k: 3, trials: [] },
+    }
+    const input = createMockTrialInput(runs)
+
+    const result = await grader(input)
+
+    expect(result.rankings.length).toBe(1)
+    expect(result.rankings[0]?.rank).toBe(1)
+  })
+
+  test('handles zero passAtK and passExpK', async () => {
+    const grader = createTrialsWeightedGrader()
+    const runs: Record<string, TrialsComparisonRunData> = {
+      baseline: { passAtK: 0, passExpK: 0, k: 3, trials: [] },
+      variant: { passAtK: 0.5, passExpK: 0.2, k: 3, trials: [] },
+    }
+    const input = createMockTrialInput(runs)
+
+    const result = await grader(input)
+
+    expect(result.rankings[0]?.run).toBe('variant')
+  })
+
+  test('deterministic ordering for equal scores', async () => {
+    const grader = createTrialsWeightedGrader()
+    const runs = createMockTrialRuns({
+      baseline: { passAtK: 0.8, passExpK: 0.6 },
+      variant: { passAtK: 0.8, passExpK: 0.6 },
+    })
+    const input = createMockTrialInput(runs)
+
+    // Run multiple times to check stability
+    const results = await Promise.all([grader(input), grader(input), grader(input)])
+
+    // All should have same ordering
+    const orders = results.map((r) => r.rankings.map((rank) => rank.run).join(','))
+    expect(new Set(orders).size).toBe(1)
+  })
+
+  test('flakiness is clamped to non-negative', async () => {
+    // Edge case: passExpK > passAtK shouldn't happen but handle gracefully
+    const grader = createTrialsWeightedGrader({ capability: 0.0, reliability: 0.0, consistency: 1.0 })
+    const runs: Record<string, TrialsComparisonRunData> = {
+      baseline: { passAtK: 0.5, passExpK: 0.7, k: 3, trials: [] }, // Invalid but should work
+      variant: { passAtK: 0.8, passExpK: 0.8, k: 3, trials: [] },
+    }
+    const input = createMockTrialInput(runs)
+
+    const result = await grader(input)
+
+    // Both should have flakiness 0, so consistency score should be 1.0 for both
+    // Variant has higher capability/reliability so it wins on tiebreaker
+    expect(result.rankings).toBeDefined()
+  })
+})

package/src/graders/trials-compare-statistical.ts

@@ -0,0 +1,188 @@
+/**
+ * Built-in statistical significance comparison grader for trials data.
+ *
+ * @remarks
+ * Uses bootstrap sampling to compute confidence intervals for passAtK and passExpK.
+ * Flags when the winner is statistically significant (p<0.05, non-overlapping CIs).
+ *
+ * Unlike the capture statistical grader which only has one score per prompt,
+ * trials data has multiple trial results per prompt, enabling proper bootstrap
+ * variance estimation.
+ *
+ * Bootstrap iterations can be customized via environment variable:
+ * - `COMPARE_BOOTSTRAP_ITERATIONS` (default: 1000)
+ *
+ * @packageDocumentation
+ */
+
+import type {
+  ComparisonGraderResult,
+  TrialsComparisonGrader,
+  TrialsComparisonGraderInput,
+} from '../pipeline/pipeline.types.ts'
+
+/** Default number of bootstrap iterations */
+const DEFAULT_ITERATIONS = 1000
+
+/**
+ * Bootstrap confidence interval result.
+ */
+type BootstrapResult = {
+  /** Median estimate from bootstrap samples (more robust than mean) */
+  median: number
+  /** 95% confidence interval [lower, upper] */
+  ci95: [number, number]
+}
+
+/**
+ * Compute passAtK estimate from trial pass/fail samples via bootstrap.
+ *
+ * @remarks
+ * passAtK = 1 - (1 - p)^k where p is estimated pass rate.
+ * We bootstrap the pass rate and compute passAtK from each bootstrap sample.
+ *
+ * @param trials - Array of 0/1 values (0=fail, 1=pass)
+ * @param k - Number of trials
+ * @param iterations - Number of bootstrap iterations
+ * @returns Bootstrap estimate and CI for passAtK
+ */
+const bootstrapPassAtK = (trials: number[], k: number, iterations: number): BootstrapResult => {
+  if (trials.length === 0) {
+    return { median: 0, ci95: [0, 0] }
+  }
+
+  const passAtKValues: number[] = []
+
+  for (let i = 0; i < iterations; i++) {
+    // Resample with replacement
+    const resampled = Array.from(
+      { length: trials.length },
+      () => trials[Math.floor(Math.random() * trials.length)] as number,
+    )
+
+    // Compute pass rate from resample
+    const passRate = resampled.reduce((acc, val) => acc + val, 0) / resampled.length
+
+    // Compute passAtK: probability of at least one pass in k samples
+    // passAtK = 1 - (1 - p)^k
+    const passAtK = 1 - (1 - passRate) ** k
+    passAtKValues.push(passAtK)
+  }
+
+  // Sort for percentile calculation
+  passAtKValues.sort((a, b) => a - b)
+
+  const lowerIdx = Math.floor(iterations * 0.025)
+  const upperIdx = Math.floor(iterations * 0.975)
+
+  return {
+    median: passAtKValues[Math.floor(iterations / 2)] ?? 0,
+    ci95: [passAtKValues[lowerIdx] ?? 0, passAtKValues[upperIdx] ?? 0],
+  }
+}
+
+/**
+ * Get bootstrap iterations from environment variable.
+ *
+ * @returns Number of bootstrap iterations
+ */
+const getIterationsFromEnv = (): number => {
+  const envValue = process.env.COMPARE_BOOTSTRAP_ITERATIONS
+  if (!envValue) return DEFAULT_ITERATIONS
+
+  const parsed = Number.parseInt(envValue, 10)
+  return Number.isNaN(parsed) || parsed < 100 ? DEFAULT_ITERATIONS : parsed
+}
+
+/**
+ * Statistical significance trials comparison grader.
+ *
+ * @remarks
+ * Compares runs using bootstrap sampling on trial outcomes to determine
+ * if differences in passAtK are statistically significant.
+ *
+ * Unlike single-sample comparisons, trials data provides multiple samples
+ * per prompt (k trials), enabling meaningful variance estimation.
+ *
+ * @public
+ */
+export const grade: TrialsComparisonGrader = async ({
+  runs,
+}: TrialsComparisonGraderInput): Promise<ComparisonGraderResult> => {
+  const iterations = getIterationsFromEnv()
+
+  // Collect pass/fail outcomes for each run
+  const runStats = Object.entries(runs).map(([label, run]) => {
+    // Convert trials to 0/1 array
+    const trialOutcomes = run.trials.map((t) => (t.pass ? 1 : 0))
+
+    // Bootstrap passAtK estimate
+    const stats = bootstrapPassAtK(trialOutcomes, run.k, iterations)
+
+    return { label, passAtK: run.passAtK ?? 0, stats }
+  })
+
+  // Sort by bootstrap median passAtK descending
+  const sorted = runStats.sort((a, b) => b.stats.median - a.stats.median)
+
+  // Check if winner is statistically significant
+  // CIs don't overlap = significant difference (approximately p<0.05)
+  let isSignificant = false
+  const first = sorted[0]
+  const second = sorted[1]
+  if (first && second) {
+    // Non-overlapping: first's lower bound > second's upper bound
+    isSignificant = first.stats.ci95[0] > second.stats.ci95[1]
+  }
+
+  const reasoning = isSignificant
+    ? `Winner "${first?.label}" shows clear separation (non-overlapping 95% CIs for passAtK)`
+    : 'No clear winner - confidence intervals overlap between top runs'
+
+  return {
+    rankings: sorted.map((s, i) => ({
+      run: s.label,
+      rank: i + 1,
+      score: s.stats.median,
+    })),
+    reasoning,
+  }
+}
+
+/**
+ * Create a statistical grader with custom iteration count.
+ *
+ * @param iterations - Number of bootstrap iterations
+ * @returns Trials comparison grader function
+ *
+ * @public
+ */
+export const createTrialsStatisticalGrader = (iterations: number = DEFAULT_ITERATIONS): TrialsComparisonGrader => {
+  return async ({ runs }: TrialsComparisonGraderInput): Promise<ComparisonGraderResult> => {
+    const runStats = Object.entries(runs).map(([label, run]) => {
+      const trialOutcomes = run.trials.map((t) => (t.pass ? 1 : 0))
+      const stats = bootstrapPassAtK(trialOutcomes, run.k, iterations)
+      return { label, passAtK: run.passAtK ?? 0, stats }
+    })
+
+    const sorted = runStats.sort((a, b) => b.stats.median - a.stats.median)
+
+    let isSignificant = false
+    const first = sorted[0]
+    const second = sorted[1]
+    if (first && second) {
+      isSignificant = first.stats.ci95[0] > second.stats.ci95[1]
+    }
+
+    return {
+      rankings: sorted.map((s, i) => ({
+        run: s.label,
+        rank: i + 1,
+        score: s.stats.median,
+      })),
+      reasoning: isSignificant
+        ? `Winner "${first?.label}" shows clear separation (non-overlapping 95% CIs)`
+        : 'No clear winner - confidence intervals overlap between top runs',
+    }
+  }
+}

package/src/graders/trials-compare-weighted.ts

@@ -0,0 +1,128 @@
+/**
+ * Built-in weighted comparison grader for trials data.
+ *
+ * @remarks
+ * Configurable weights for capability (passAtK), reliability (passExpK),
+ * and consistency (1 - flakiness) dimensions.
+ *
+ * Weights can be customized via environment variables:
+ * - `COMPARE_CAPABILITY` (default: 0.4)
+ * - `COMPARE_RELIABILITY` (default: 0.4)
+ * - `COMPARE_CONSISTENCY` (default: 0.2)
+ *
+ * @packageDocumentation
+ */
+
+import type {
+  ComparisonGraderResult,
+  TrialsComparisonGrader,
+  TrialsComparisonGraderInput,
+} from '../pipeline/pipeline.types.ts'
+
+/**
+ * Weight configuration for trials comparison dimensions.
+ */
+export type TrialsWeights = {
+  /** Weight for capability (passAtK) - can the agent solve this at least once? */
+  capability: number
+  /** Weight for reliability (passExpK) - does the agent solve this consistently? */
+  reliability: number
+  /** Weight for consistency (1 - flakiness) - low gap between capability and reliability */
+  consistency: number
+}
+
+/** Default weights: capability=0.4, reliability=0.4, consistency=0.2 */
+export const DEFAULT_TRIALS_WEIGHTS: TrialsWeights = {
+  capability: 0.4,
+  reliability: 0.4,
+  consistency: 0.2,
+}
+
+/**
+ * Read weights from environment variables with fallback to defaults.
+ *
+ * @remarks
+ * Validates that weights are non-negative. Invalid or negative values
+ * fall back to defaults.
+ *
+ * @returns TrialsWeights configuration
+ *
+ * @public
+ */
+export const getTrialsWeightsFromEnv = (): TrialsWeights => {
+  const parseWeight = (envVar: string | undefined, defaultValue: number): number => {
+    if (!envVar) return defaultValue
+    const parsed = Number.parseFloat(envVar)
+    // Must be a valid non-negative number
+    if (Number.isNaN(parsed) || parsed < 0) return defaultValue
+    return parsed
+  }
+
+  return {
+    capability: parseWeight(process.env.COMPARE_CAPABILITY, DEFAULT_TRIALS_WEIGHTS.capability),
+    reliability: parseWeight(process.env.COMPARE_RELIABILITY, DEFAULT_TRIALS_WEIGHTS.reliability),
+    consistency: parseWeight(process.env.COMPARE_CONSISTENCY, DEFAULT_TRIALS_WEIGHTS.consistency),
+  }
+}
+
+/**
+ * Create a weighted trials comparison grader with custom weights.
+ *
+ * @param weights - Weight configuration for comparison dimensions
+ * @returns Trials comparison grader function
+ *
+ * @public
+ */
+export const createTrialsWeightedGrader = (weights: TrialsWeights = DEFAULT_TRIALS_WEIGHTS): TrialsComparisonGrader => {
+  return async ({ runs }: TrialsComparisonGraderInput): Promise<ComparisonGraderResult> => {
+    const scores = Object.entries(runs).map(([label, run]) => {
+      // Capability score: passAtK (0-1)
+      const capabilityScore = run.passAtK ?? 0
+
+      // Reliability score: passExpK (0-1)
+      const reliabilityScore = run.passExpK ?? 0
+
+      // Consistency score: 1 - flakiness
+      // Flakiness = passAtK - passExpK (how much gap between capability and reliability)
+      const flakiness = Math.max(0, capabilityScore - reliabilityScore)
+      const consistencyScore = 1 - flakiness
+
+      // Weighted combination
+      const weighted =
+        capabilityScore * weights.capability +
+        reliabilityScore * weights.reliability +
+        consistencyScore * weights.consistency
+
+      return { label, weighted, capabilityScore, reliabilityScore, consistencyScore, flakiness }
+    })
+
+    // Sort by weighted score descending (highest = best)
+    const sorted = scores.sort((a, b) => b.weighted - a.weighted)
+
+    return {
+      rankings: sorted.map((s, i) => ({
+        run: s.label,
+        rank: i + 1,
+        score: s.weighted,
+      })),
+      reasoning: `Weighted trials: capability=${weights.capability}, reliability=${weights.reliability}, consistency=${weights.consistency}`,
+    }
+  }
+}
+
+/**
+ * Default weighted trials comparison grader using environment or default weights.
+ *
+ * @remarks
+ * This is the default grader used when `--strategy weighted` is specified
+ * for trials format comparison.
+ *
+ * @public
+ */
+export const grade: TrialsComparisonGrader = async (
+  input: TrialsComparisonGraderInput,
+): Promise<ComparisonGraderResult> => {
+  const weights = getTrialsWeightsFromEnv()
+  const grader = createTrialsWeightedGrader(weights)
+  return grader(input)
+}