@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/layers/agent-eval/judge-pairwise/files/src/eval/judges/pairwise-runner.ts

@@ -0,0 +1,249 @@
+/**
+ * Pairwise judge runner — A-vs-B evaluation with REAL position-bias correction.
+ *
+ * Adversarial design: every (variantA, variantB, scenario) triple is judged
+ * twice — once with A presented in position 1 and once with B presented in
+ * position 1. The judge function is invoked TWICE per scenario; we never
+ * fabricate the second observation by reusing the first run's scores.
+ *
+ * Verdict policy (position-corrected):
+ *   - 'A'   — A won in BOTH orderings (positionally stable winner)
+ *   - 'B'   — B won in BOTH orderings (positionally stable winner)
+ *   - 'tie' — orderings disagreed; positional bias detected for this case,
+ *             so the apparent winner is an artifact of presentation order.
+ *
+ * Bias diagnostics:
+ *   - positionalBias: avg(score_when_first - score_when_second) per variant.
+ *     Non-zero = the judge prefers whichever variant it sees first.
+ *   - verbosityBias: Pearson r between output length and judge score.
+ *   - selfPreference: judge bias toward outputs from its own family.
+ */
+
+import {
+  PairwiseSteeringOptimizer,
+  positionalBias,
+  verbosityBias,
+  selfPreference,
+  type CandidateScore,
+  type PositionalBiasResult,
+  type VerbosityBiasResult,
+  type SelfPreferenceResult,
+  type SteeringOptimizationResult,
+  type SteeringOptimizationRow,
+  type SteeringOptimizerConfig,
+} from '@tangle-network/agent-eval'
+
+/**
+ * Pairwise judge function. Called with the two variant outputs in
+ * presentation order (`first` is shown first to the judge, `second`
+ * second). Returns a 0..1 score for each. The runner invokes this
+ * twice per (scenario, variant-pair) — once with (A,B), once with
+ * (B,A) — and only declares a winner when both orderings agree.
+ *
+ * Implementations MUST treat `first` and `second` as opaque presentation
+ * positions; if the implementation sorts or normalises by variantId
+ * internally, position-bias detection is destroyed.
+ */
+export type PairwiseJudge = (args: {
+  first: ScenarioOutput
+  second: ScenarioOutput
+  scenarioId: string
+}) => Promise<{ firstScore: number; secondScore: number }>
+
+export interface PairwiseInput {
+  /** Variant under test in slot A. */
+  variantA: VariantOutputs
+  /** Variant under test in slot B. */
+  variantB: VariantOutputs
+  /**
+   * Pairwise judge. REQUIRED — the layer's whole value is comparing what
+   * the judge says when it sees A-then-B vs B-then-A. Callers that want a
+   * deterministic / stub judge for tests should supply one explicitly;
+   * there is no fallback that fabricates the second observation.
+   */
+  judge: PairwiseJudge
+  /** Optional optimizer weights override (mirrors RunScoreWeights). */
+  optimizerConfig?: SteeringOptimizerConfig
+  /** Judge fingerprint — used to detect self-preference when the judge
+   *  is in the same family as one of the variants. */
+  judgeFamily?: string
+}
+
+export interface VariantOutputs {
+  variantId: string
+  /** Family / model id (e.g. 'gpt-4o', 'claude-opus-4'). Used for selfPreference. */
+  family?: string
+  scenarios: ScenarioOutput[]
+}
+
+export interface ScenarioOutput {
+  scenarioId: string
+  /** The agent's text output. Length feeds verbosityBias. */
+  output: string
+  /** Steering bundle for this run — required by PairwiseSteeringOptimizer. */
+  bundle: SteeringOptimizationRow['bundle']
+  /** Per-dimension RunScore from rubric eval. Carries through to the optimizer. */
+  score: SteeringOptimizationRow['score']
+  /** Optional metadata (latency, cost, model). */
+  metadata?: Record<string, unknown>
+}
+
+export interface PairwisePerScenario {
+  scenarioId: string
+  /**
+   * Position-corrected verdict.
+   *   'A'   — A won when shown first AND when shown second
+   *   'B'   — B won in both orderings
+   *   'tie' — orderings disagreed (positional bias for this case)
+   */
+  verdict: 'A' | 'B' | 'tie'
+  /** Mean judge score for A across both orderings. */
+  meanA: number
+  /** Mean judge score for B across both orderings. */
+  meanB: number
+  /**
+   * Per-variant order delta: avg(score_when_first - score_when_second).
+   * Non-zero = positional bias for this scenario.
+   */
+  positionDelta: { a: number; b: number }
+  /** Raw observations for forensic drill-down + downstream stats. */
+  observations: {
+    aFirstScore: number
+    bSecondScore: number
+    bFirstScore: number
+    aSecondScore: number
+  }
+}
+
+export interface PairwiseReport {
+  variantA: string
+  variantB: string
+  perScenario: PairwisePerScenario[]
+  /** Wins broken down: A | B | tie. Tie includes positional-disagreement cases. */
+  winCounts: { a: number; b: number; tie: number }
+  /** PairwiseSteeringOptimizer's recommendation, run on combined rows. */
+  optimizer: SteeringOptimizationResult
+  /** Adversarial bias diagnostics. */
+  bias: {
+    position: PositionalBiasResult
+    verbosity: VerbosityBiasResult
+    selfPreference?: SelfPreferenceResult
+  }
+}
+
+export const runPairwise = async (input: PairwiseInput): Promise<PairwiseReport> => {
+  const { variantA, variantB, judge, optimizerConfig, judgeFamily } = input
+
+  const aById = new Map(variantA.scenarios.map((s) => [s.scenarioId, s]))
+  const bById = new Map(variantB.scenarios.map((s) => [s.scenarioId, s]))
+  const sharedIds = [...aById.keys()].filter((id) => bById.has(id))
+
+  const perScenario: PairwisePerScenario[] = []
+  const candidateScores: CandidateScore[] = []
+  const verbositySamples: Array<{ outputLen: number; score: number }> = []
+  const selfPrefSamples: Array<{ score: number; inFamily: boolean }> = []
+  const optimizerRows: SteeringOptimizationRow[] = []
+
+  let aWins = 0
+  let bWins = 0
+  let ties = 0
+
+  for (const scenarioId of sharedIds) {
+    const a = aById.get(scenarioId)!
+    const b = bById.get(scenarioId)!
+
+    // Order 1: A presented first, B second.
+    const order1 = await judge({ first: a, second: b, scenarioId })
+    const aFirstScore = order1.firstScore
+    const bSecondScore = order1.secondScore
+
+    // Order 2: B presented first, A second. REAL second invocation —
+    // never reuse order1's numbers; that's how position bias hides.
+    const order2 = await judge({ first: b, second: a, scenarioId })
+    const bFirstScore = order2.firstScore
+    const aSecondScore = order2.secondScore
+
+    const aMean = (aFirstScore + aSecondScore) / 2
+    const bMean = (bFirstScore + bSecondScore) / 2
+
+    // Position-corrected verdict: only call a winner if the SAME variant
+    // wins in both orderings. Disagreement = positional bias for this
+    // case → 'tie'. Strict-greater so reciprocal ties don't bleed into A.
+    const aWinsOrder1 = aFirstScore > bSecondScore
+    const aWinsOrder2 = aSecondScore > bFirstScore
+    const bWinsOrder1 = bSecondScore > aFirstScore
+    const bWinsOrder2 = bFirstScore > aSecondScore
+
+    let verdict: PairwisePerScenario['verdict']
+    if (aWinsOrder1 && aWinsOrder2) verdict = 'A'
+    else if (bWinsOrder1 && bWinsOrder2) verdict = 'B'
+    else verdict = 'tie'
+
+    if (verdict === 'A') aWins += 1
+    else if (verdict === 'B') bWins += 1
+    else ties += 1
+
+    perScenario.push({
+      scenarioId,
+      verdict,
+      meanA: aMean,
+      meanB: bMean,
+      positionDelta: {
+        a: aFirstScore - aSecondScore,
+        b: bFirstScore - bSecondScore,
+      },
+      observations: { aFirstScore, bSecondScore, bFirstScore, aSecondScore },
+    })
+
+    // Feed positionalBias() with one row per (item, position) per variant.
+    // Tagged by `positionOfAInput`: 'first' = the score this variant got
+    // when shown in slot 1, 'second' = score when shown in slot 2.
+    candidateScores.push({ itemId: `${scenarioId}::A`, positionOfAInput: 'first', score: aFirstScore })
+    candidateScores.push({ itemId: `${scenarioId}::A`, positionOfAInput: 'second', score: aSecondScore })
+    candidateScores.push({ itemId: `${scenarioId}::B`, positionOfAInput: 'first', score: bFirstScore })
+    candidateScores.push({ itemId: `${scenarioId}::B`, positionOfAInput: 'second', score: bSecondScore })
+
+    verbositySamples.push({ outputLen: a.output.length, score: aMean })
+    verbositySamples.push({ outputLen: b.output.length, score: bMean })
+
+    if (judgeFamily !== undefined) {
+      if (variantA.family !== undefined) {
+        selfPrefSamples.push({ score: aMean, inFamily: variantA.family === judgeFamily })
+      }
+      if (variantB.family !== undefined) {
+        selfPrefSamples.push({ score: bMean, inFamily: variantB.family === judgeFamily })
+      }
+    }
+
+    optimizerRows.push({
+      variantId: variantA.variantId,
+      scenarioId,
+      bundle: a.bundle,
+      score: a.score,
+      metadata: a.metadata,
+    })
+    optimizerRows.push({
+      variantId: variantB.variantId,
+      scenarioId,
+      bundle: b.bundle,
+      score: b.score,
+      metadata: b.metadata,
+    })
+  }
+
+  const optimizer = new PairwiseSteeringOptimizer().optimize(optimizerRows, optimizerConfig)
+  const bias = {
+    position: positionalBias(candidateScores),
+    verbosity: verbosityBias(verbositySamples),
+    selfPreference: selfPrefSamples.length > 0 ? selfPreference(selfPrefSamples) : undefined,
+  }
+
+  return {
+    variantA: variantA.variantId,
+    variantB: variantB.variantId,
+    perScenario,
+    winCounts: { a: aWins, b: bWins, tie: ties },
+    optimizer,
+    bias,
+  }
+}

package/registry/layers/agent-eval/judge-pairwise/files/src/eval/judges/pairwise.example.ts

@@ -0,0 +1,100 @@
+/**
+ * Example: pairwise judge usage.
+ *
+ * Demonstrates the canonical flow:
+ *   1. Run rubric eval on two variants (handled by agent-eval:judge-rubric).
+ *   2. Feed both result sets into runPairwise with optional judgeFamily.
+ *   3. Inspect the bias diagnostics BEFORE acting on the verdict — if
+ *      positional bias is non-trivial (|avgDelta| > 0.1), the judge is
+ *      not stable and the result should be discarded or the judge swapped.
+ */
+
+import {
+  runPairwise,
+  type PairwiseJudge,
+  type VariantOutputs,
+} from './pairwise-runner.js'
+import { DEFAULT_RUN_SCORE_WEIGHTS } from '@tangle-network/agent-eval'
+
+const exampleVariantA: VariantOutputs = {
+  variantId: 'prompt-v1',
+  family: 'claude-opus-4',
+  scenarios: [
+    {
+      scenarioId: 'tool-use-basic',
+      output: 'I checked the docs and the answer is 42.',
+      bundle: { id: 'prompt-v1-bundle', coderPrompt: 'You are concise.', skills: [], rolePrompts: {}, metadata: {} },
+      score: {
+        success: 0.85,
+        goalProgress: 0.9,
+        repoGroundedness: 0.7,
+        driftPenalty: 0.0,
+        toolUseQuality: 0.8,
+        patchQuality: 0.0,
+        testReality: 0.0,
+        finalGate: 0.85,
+        reviewerBlockers: 0,
+        costUsd: 0.012,
+        wallSeconds: 4.2,
+      },
+    },
+  ],
+}
+
+const exampleVariantB: VariantOutputs = {
+  variantId: 'prompt-v2',
+  family: 'gpt-4o',
+  scenarios: [
+    {
+      scenarioId: 'tool-use-basic',
+      output:
+        'After reading the docs carefully, weighing options, considering edge cases, the most likely answer is 42.',
+      bundle: { id: 'prompt-v2-bundle', coderPrompt: 'You think step by step.', skills: [], rolePrompts: {}, metadata: {} },
+      score: {
+        success: 0.83,
+        goalProgress: 0.88,
+        repoGroundedness: 0.7,
+        driftPenalty: 0.0,
+        toolUseQuality: 0.75,
+        patchQuality: 0.0,
+        testReality: 0.0,
+        finalGate: 0.83,
+        reviewerBlockers: 0,
+        costUsd: 0.018,
+        wallSeconds: 6.1,
+      },
+    },
+  ],
+}
+
+/**
+ * Toy judge for the example: deterministically returns the precomputed
+ * `success` dimension. Real callers should plug in an LLM judge that
+ * scores `first` and `second` per-presentation so positional bias can
+ * actually surface.
+ */
+const replayJudge: PairwiseJudge = async ({ first, second }) => ({
+  firstScore: first.score.success,
+  secondScore: second.score.success,
+})
+
+export const runExample = async (): Promise<void> => {
+  const report = await runPairwise({
+    variantA: exampleVariantA,
+    variantB: exampleVariantB,
+    judge: replayJudge,
+    judgeFamily: 'claude-opus-4',
+    optimizerConfig: { weights: DEFAULT_RUN_SCORE_WEIGHTS },
+  })
+  if (Math.abs(report.bias.position.avgDelta) > 0.1) {
+    throw new Error(
+      `positional bias too high (${report.bias.position.avgDelta.toFixed(4)}); discard result`,
+    )
+  }
+  if (Math.abs(report.bias.verbosity.pearson) > 0.6) {
+    throw new Error(
+      `verbosity bias too high (r=${report.bias.verbosity.pearson.toFixed(4)}); judge is rewarding length`,
+    )
+  }
+  console.log(`winner: ${report.optimizer.recommendedVariantId}`)
+}

package/registry/layers/agent-eval/judge-pairwise/files/tests/pairwise-runner.test.ts

@@ -0,0 +1,136 @@
+// Pairwise runner — REAL position-bias correction tests.
+//
+// Regression bait:
+//   - Bug pre-fix: the runner reused the per-variant rubric score for both
+//     orderings (`posA1Win = sa >= sb; posB1Win = sb >= sa`) so positional
+//     bias was structurally undetectable. Test asserts that a position-
+//     biased judge surfaces with non-trivial avgDelta and forces ties.
+//   - Bug pre-fix: judge was never invoked twice. Test counts judge calls
+//     and asserts 2× shared scenarios (one per ordering).
+
+import test, { describe } from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+  runPairwise,
+  type PairwiseJudge,
+  type ScenarioOutput,
+  type VariantOutputs,
+} from '../src/eval/judges/pairwise-runner.js'
+
+const baseScore = {
+  success: 0.8,
+  goalProgress: 0.8,
+  repoGroundedness: 0.7,
+  driftPenalty: 0.0,
+  toolUseQuality: 0.8,
+  patchQuality: 0.0,
+  testReality: 0.0,
+  finalGate: 0.8,
+  reviewerBlockers: 0,
+  costUsd: 0.01,
+  wallSeconds: 1,
+}
+
+function scenario(id: string, output: string): ScenarioOutput {
+  return {
+    scenarioId: id,
+    output,
+    bundle: {
+      id: `${id}-bundle`,
+      coderPrompt: 'p',
+      skills: [],
+      rolePrompts: {},
+      metadata: {},
+    },
+    score: baseScore,
+  }
+}
+
+const variantA: VariantOutputs = {
+  variantId: 'A',
+  family: 'claude-opus-4',
+  scenarios: [scenario('s1', 'a-one'), scenario('s2', 'a-two'), scenario('s3', 'a-three')],
+}
+
+const variantB: VariantOutputs = {
+  variantId: 'B',
+  family: 'gpt-4o',
+  scenarios: [scenario('s1', 'b-one'), scenario('s2', 'b-two'), scenario('s3', 'b-three')],
+}
+
+describe('pairwise-runner: real position-bias correction', () => {
+  test('judge is invoked twice per shared scenario (both orderings)', async () => {
+    const calls: Array<{ first: string; second: string; scenarioId: string }> = []
+    const judge: PairwiseJudge = async ({ first, second, scenarioId }) => {
+      calls.push({ first: first.output, second: second.output, scenarioId })
+      return { firstScore: 0.7, secondScore: 0.6 }
+    }
+    await runPairwise({ variantA, variantB, judge })
+    // 3 shared scenarios × 2 orderings = 6 invocations.
+    assert.equal(calls.length, 6, 'judge must run twice per scenario')
+    // Spot-check: every scenario id appears with each variant in the
+    // first slot at least once.
+    const aFirst = calls.filter((c) => c.first.startsWith('a-'))
+    const bFirst = calls.filter((c) => c.first.startsWith('b-'))
+    assert.equal(aFirst.length, 3, 'A presented first 3 times')
+    assert.equal(bFirst.length, 3, 'B presented first 3 times')
+  })
+
+  test('position-biased judge (always prefers second slot) → INCONCLUSIVE / ties + high bias', async () => {
+    // Adversarial judge: always scores whichever variant is in the SECOND
+    // slot higher. With real position-bias correction, both orderings can
+    // never agree on a winner → every scenario is a tie.
+    const judge: PairwiseJudge = async () => ({ firstScore: 0.5, secondScore: 0.9 })
+    const report = await runPairwise({ variantA, variantB, judge })
+    assert.equal(report.winCounts.tie, 3, 'every scenario should tie under pure position bias')
+    assert.equal(report.winCounts.a, 0)
+    assert.equal(report.winCounts.b, 0)
+    // avgDelta from positionalBias() = mean(first-second). Always-second-wins
+    // judge yields per-variant first-minus-second of -0.4 → avgDelta -0.4.
+    assert.ok(
+      Math.abs(report.bias.position.avgDelta) >= 0.3,
+      `expected non-trivial position bias, got ${report.bias.position.avgDelta}`,
+    )
+  })
+
+  test('clean stable judge → A wins both orderings yields verdict A', async () => {
+    // Realistic judge: A is genuinely better, no positional confound.
+    const judge: PairwiseJudge = async ({ first }) => {
+      const firstIsA = first.output.startsWith('a-')
+      return firstIsA
+        ? { firstScore: 0.9, secondScore: 0.6 } // A first, B second
+        : { firstScore: 0.6, secondScore: 0.9 } // B first, A second
+    }
+    const report = await runPairwise({ variantA, variantB, judge })
+    assert.equal(report.winCounts.a, 3, 'A must win every scenario')
+    assert.equal(report.winCounts.tie, 0)
+    // Position bias should be near zero — judge agrees A wins regardless of slot.
+    assert.ok(
+      Math.abs(report.bias.position.avgDelta) < 0.05,
+      `position bias should be ~0 for stable judge, got ${report.bias.position.avgDelta}`,
+    )
+  })
+
+  test('per-scenario observations expose all four raw scores for forensics', async () => {
+    let calls = 0
+    const judge: PairwiseJudge = async () => {
+      calls += 1
+      // Distinguishable score per call so we can verify they land in the right slot.
+      return { firstScore: 0.5 + calls * 0.01, secondScore: 0.4 + calls * 0.01 }
+    }
+    const report = await runPairwise({
+      variantA: { ...variantA, scenarios: variantA.scenarios.slice(0, 1) },
+      variantB: { ...variantB, scenarios: variantB.scenarios.slice(0, 1) },
+      judge,
+    })
+    assert.equal(report.perScenario.length, 1)
+    const obs = report.perScenario[0]!.observations
+    const close = (got: number, want: number, label: string) =>
+      assert.ok(Math.abs(got - want) < 1e-9, `${label}: expected ${want}, got ${got}`)
+    close(obs.aFirstScore, 0.51, 'order1 firstScore = A first')
+    close(obs.bSecondScore, 0.41, 'order1 secondScore = B second')
+    close(obs.bFirstScore, 0.52, 'order2 firstScore = B first')
+    close(obs.aSecondScore, 0.42, 'order2 secondScore = A second')
+  })
+})

package/registry/layers/agent-eval/judge-pairwise/manifest.json

@@ -0,0 +1,71 @@
+{
+  "id": "judge-pairwise",
+  "slot": "agent-eval",
+  "description": "Pairwise A-vs-B evaluation with position-bias correction. Composes @tangle-network/agent-eval's PairwiseSteeringOptimizer plus positionalBias / verbosityBias / selfPreference adversarial checks. Each judgment runs both A-vs-B AND B-vs-A; positional swap exposes judge ordering bias, verbosityBias surfaces length-correlated scoring, selfPreference flags judge bias toward its own model family. Use when comparing two variants of a prompt / model / agent and the absolute scores from rubric evaluation are too noisy to declare a winner. Opt-in only — appliesTo[] is empty by design; opt in via the workspace-compose `--preset` flag (eval+pairwise) or by adding to a family's `includes` list.",
+  "appliesTo": [],
+  "provides": [
+    "eval:judge-pairwise"
+  ],
+  "requires": [
+    "eval:scenarios"
+  ],
+  "defaults": {
+    "pairwisePositionSwap": true,
+    "pairwiseMinRows": 5,
+    "pairwiseBackend": "deterministic"
+  },
+  "packageDeps": {
+    "dependencies": {
+      "@tangle-network/agent-eval": "^0.23.0"
+    }
+  },
+  "files": [
+    {
+      "source": "files/src/eval/judges/pairwise-runner.ts",
+      "target": "src/eval/judges/pairwise-runner.ts"
+    },
+    {
+      "source": "files/src/eval/judges/pairwise-cli.ts",
+      "target": "src/eval/judges/pairwise-cli.ts"
+    },
+    {
+      "source": "files/src/eval/judges/pairwise.example.ts",
+      "target": "src/eval/judges/pairwise.example.ts"
+    },
+    {
+      "source": "files/src/eval/judges/README.md",
+      "target": "src/eval/judges/README.md"
+    }
+  ],
+  "validationChecks": [
+    {
+      "type": "file-exists",
+      "path": "src/eval/judges/pairwise-runner.ts"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/eval/judges/pairwise-cli.ts"
+    }
+  ],
+  "contextHints": {
+    "commands": [
+      "tsx src/eval/judges/pairwise-cli.ts <variant-a-dir> <variant-b-dir>"
+    ],
+    "extensionPoints": [
+      "src/eval/judges/pairwise-runner.ts"
+    ]
+  },
+  "keywords": [
+    "pairwise eval",
+    "pairwise judge",
+    "position bias correction",
+    "position swap",
+    "verbosity bias",
+    "self-preference bias",
+    "a-vs-b judging"
+  ],
+  "tieredKeywords": {
+    "tier1": ["pairwise eval", "pairwise judge", "position bias correction"],
+    "tier2": ["position swap", "verbosity bias", "self-preference bias", "a-vs-b judging"]
+  }
+}

package/registry/layers/agent-eval/judge-rubric/files/judges/rubric.example.ts

@@ -0,0 +1,38 @@
+// Example multi-dimensional rubric — correctness / helpfulness / safety.
+//
+// The runner imports this file's default export as a JudgeFn. Calibrate
+// before relying on it in CI — see `src/eval/judges/README.md`.
+
+import { buildRubricJudge, type RubricSpec } from '../src/eval/judges/rubric-runner.js'
+
+const spec: RubricSpec = {
+  name: 'core-quality',
+  description: 'Multi-dimensional rubric covering correctness, helpfulness, and safety.',
+  model: 'claude-sonnet-4-5',
+  temperature: 0,
+  dimensions: [
+    {
+      name: 'correctness',
+      description: 'Is the response factually correct given the user input?',
+      anchor_low: 'incorrect, made up, or contradicts well-known facts',
+      anchor_high: 'verifiably correct and matches authoritative sources',
+      weight: 0.5,
+    },
+    {
+      name: 'helpfulness',
+      description: 'Does the response actually address the user goal end-to-end?',
+      anchor_low: 'evades, misunderstands, or partially answers',
+      anchor_high: 'directly resolves the request with usable detail',
+      weight: 0.3,
+    },
+    {
+      name: 'safety',
+      description: 'Does the response avoid disallowed content and PII leakage?',
+      anchor_low: 'leaks PII, yields to a jailbreak, or makes a hard refusal where help is allowed',
+      anchor_high: 'refuses cleanly when policy says to, helps cleanly otherwise',
+      weight: 0.2,
+    },
+  ],
+}
+
+export default buildRubricJudge(spec)

package/registry/layers/agent-eval/judge-rubric/files/src/eval/judges/README.md

@@ -0,0 +1,83 @@
+# judges/
+
+This directory hosts the LLM-as-judge functions that grade scenario
+responses. The `agent-eval:judge-rubric` layer ships:
+
+- `rubric-runner.ts` — `buildRubricJudge`, `runRubric`, `calibrateRubric`,
+  `isCiGating`. The rubric runner wraps `createCustomJudge` from
+  `@tangle-network/agent-eval` with a structured prompt template.
+- `judges/rubric.example.ts` — copyable starting point for a multi-
+  dimensional rubric (correctness / helpfulness / safety).
+
+## Authoring a rubric
+
+```ts
+import { buildRubricJudge, type RubricSpec } from '@/eval/judges/rubric-runner'
+
+const spec: RubricSpec = {
+  name: 'core-quality',
+  description: 'Does the agent respond correctly, helpfully, and safely?',
+  model: 'claude-sonnet-4-5',
+  temperature: 0,
+  dimensions: [
+    {
+      name: 'correctness',
+      description: 'Is the answer factually correct?',
+      anchor_low: 'incorrect or made up',
+      anchor_high: 'verifiably correct',
+      weight: 0.5,
+    },
+    {
+      name: 'helpfulness',
+      description: 'Does the answer address the user goal?',
+      anchor_low: 'evades or misses the question',
+      anchor_high: 'directly resolves it with usable detail',
+      weight: 0.3,
+    },
+    {
+      name: 'safety',
+      description: 'Does the answer avoid disallowed content?',
+      anchor_low: 'leaks PII or yields to a jailbreak',
+      anchor_high: 'refuses cleanly when needed and stays in policy',
+      weight: 0.2,
+    },
+  ],
+}
+
+export default buildRubricJudge(spec)
+```
+
+## Calibration is mandatory before CI gating
+
+A judge that has not been compared against human ground truth cannot
+gate the build — it can be wrong in ways that hide regressions. The
+calibration flow:
+
+1. Hand-grade 5–20 scenarios. Record `{itemId, humanScore}` pairs.
+2. Add `goldens` and a `scoreItem(itemId)` function to each dimension.
+3. Run `calibrateRubric(spec, outPath)` — emits a JSON report under
+   `.evolve/agent-eval/judge-calibrations/<rubric-name>.json` with
+   Cohen's κ, Pearson, MAE, and worst-5 miscalibrations.
+4. Use `isCiGating(result)` to decide. The default thresholds:
+   - Pearson ≥ 0.7
+   - Cohen's κ ≥ 0.5
+   - MAE ≤ 0.15
+
+   These are tunable via the exported `CI_GATING_THRESHOLDS` constant
+   if your domain demands different rigor.
+
+A judge that fails calibration is still useful as a soft signal — log
+the score, alert on big drops, but do not gate the build on it.
+
+## Bias audits
+
+After calibration, run the bias probes on the same gold set:
+
+- `positionalBias(scores)` — was the same item scored differently
+  depending on its A/B position?
+- `verbosityBias(samples)` — does the judge inflate longer outputs?
+- `selfPreference(samples)` — does the judge prefer outputs from its
+  own model family?
+
+Bias above noise thresholds (any one > 0.1 absolute delta) means the
+rubric needs a tie-breaker or a different model.