@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/layers/agent-eval/auto-research/files/src/eval/auto-research/README.md

@@ -0,0 +1,95 @@
+# auto-research
+
+Composable optimization layer over `@tangle-network/agent-eval@^0.77.0`.
+
+Wraps the upstream primitives (`runOptimization`, `runImprovementLoop`,
+`PairwiseSteeringOptimizer`, `runProposeReview`, `paretoFrontier`,
+`paretoFrontierWithCrowding`) plus the run-analysis bridge (`analyzeRuns`)
+into a small surface a research-harness family can compose without
+re-deriving the agent-eval API.
+
+## Capabilities
+
+- `provides`: `eval:auto-research`
+- `requires`: `eval:scenarios`, `eval:judge-rubric`, `eval:regression`
+
+The three required capabilities supply the measurement substrate
+(scenarios to run, judges to score, gates to promote). This layer
+supplies the optimizer that drives variants against that substrate.
+
+## How a family composes this
+
+```ts
+import {
+  runSteeringOptimization,
+  runMultiShotTrajectoryOptimization,
+  runEvolution,
+  proposeReview,
+  frontier,
+  DEFAULT_OBJECTIVES,
+} from './eval/auto-research/index.js'
+
+// 1. Steering bundles → winner (FDR-corrected pairwise).
+const result = await runSteeringOptimization({
+  variants: [bundleA, bundleB, bundleC],
+  examples: scenarios.map((s) => ({ scenarioId: s.id, ... })),
+  evaluate: async ({ variant, scenarioId }) => harness.run(variant, scenarioId),
+  trialsPerScenario: 3,
+})
+
+// 2. Variable-length agent trajectory optimization.
+const optimized = await runMultiShotTrajectoryOptimization({
+  runId: `research-${Date.now()}`,
+  baselineSurface: baselinePrompt,
+  scenarios,
+  judges,
+  dispatchWithSurface,
+  driver,
+  reps: 2,
+  maxGenerations: 3,
+  populationSize: 4,
+})
+deploy(optimized.winnerSurface)
+
+// 3. Reflective hypothesis → final state via propose/verify/review.
+const report = await proposeReview({
+  goal: 'reduce judge-rubric failures on category=tool-use',
+  initialState: { promptVersion: 'v1' },
+  propose: ...,
+  verify: ...,
+  review: ...,
+})
+
+// 4. Pareto-non-dominated points across (quality, cost, latency).
+const f = frontier([
+  { variantId: 'A', quality: 0.81, costUsd: 0.12, wallSeconds: 4.1 },
+  { variantId: 'B', quality: 0.83, costUsd: 0.40, wallSeconds: 6.0 },
+])
+
+// 5. Convert captured RunRecords into a launch decision report. Call AFTER
+//     the optimizer emits real records; do NOT invent records from aggregates.
+import { analyzeOptimization } from './eval/auto-research/index.js'
+
+const rl = await analyzeOptimization({
+  runs,
+  baselineCandidateId: 'baseline',
+  candidateCandidateId: optimized.winnerSurfaceHash,
+  split: 'holdout',
+})
+// rl.recommendations → ranked launch / hold / investigate guidance
+```
+
+The run-analysis bridge closes the auto-research loop into an evidence-backed
+decision packet. It preserves the rule that analysis starts from real
+`RunRecord[]`, not synthesized aggregate scores.
+
+## What this layer does NOT do
+
+- Run the actual eval (that's `eval:scenarios` + `eval:judge-rubric`)
+- Persist results (that's the family's runner — see
+  `agent-research-harness-ts`)
+- Decide promotions (that's `eval:regression`)
+- Generate hypotheses (that's the family's proposer)
+
+This layer is the OPTIMIZER. Measurement and bookkeeping are separate
+concerns and live in separate layers / families on purpose.

package/registry/layers/agent-eval/auto-research/files/src/eval/auto-research/index.ts

@@ -0,0 +1,49 @@
+/**
+ * auto-research barrel — re-exports the primitives a research-harness
+ * family composes:
+ *
+ *   - `runSteeringOptimization`, `runMultiShotTrajectoryOptimization`,
+ *     `runEvolution`, `runPromotedImprovementLoop` (loop.ts)
+ *   - `analyzeOptimization` — captured runs → decision report with lift,
+ *     cost/quality frontier, and ranked recommendations (loop.ts)
+ *   - `proposeReview` (propose-review.ts)
+ *   - `frontier`, `diverseFrontier`, `DEFAULT_OBJECTIVES` (pareto.ts)
+ */
+
+export {
+  runSteeringOptimization,
+  runMultiShotTrajectoryOptimization,
+  runEvolution,
+  runPromotedImprovementLoop,
+  analyzeOptimization,
+  type SteeringOptimizationInput,
+  type SteeringOptimizationOutput,
+  type MultiShotOptimizationConfig,
+  type MultiShotOptimizationResult,
+  type PromptEvolutionConfig,
+  type PromptEvolutionResult,
+  type PromotedImprovementConfig,
+  type PromotedImprovementResult,
+  type SteeringOptimizationRow,
+  type SteeringOptimizationResult,
+  type SteeringOptimizerConfig,
+  type AnalyzeOptimizationResultOptions,
+  type AnalyzeOptimizationResultReport,
+} from './loop.js'
+
+export {
+  proposeReview,
+  type ProposeReviewInput,
+  type ProposeReviewOutput,
+  type ProposeReviewConfig,
+  type ProposeReviewReport,
+} from './propose-review.js'
+
+export {
+  frontier,
+  diverseFrontier,
+  DEFAULT_OBJECTIVES,
+  type VariantPoint,
+  type Objective,
+  type ParetoResult,
+} from './pareto.js'

package/registry/layers/agent-eval/auto-research/files/src/eval/auto-research/loop.ts

@@ -0,0 +1,151 @@
+/**
+ * auto-research:loop — composable optimization loop.
+ *
+ * Wraps the optimization primitives from `@tangle-network/agent-eval@^0.77.0`.
+ * The eval-harness layers (scenarios, judge-rubric, regression) supply the
+ * measurement substrate; this module supplies the optimizer that drives
+ * variants against it.
+ *
+ * Three optimization entrypoints:
+ *
+ *   - `runSteeringOptimization` — N steering bundles × M scenarios → winner
+ *     by observed aggregate score. Use when variants are already enumerated
+ *     and the question is "which one wins?".
+ *
+ *   - `runMultiShotTrajectoryOptimization` — campaign optimization for a
+ *     variable-length agent task. Use this for chat agents, browser/coding
+ *     agents, and autoresearch loops where one trial is a whole trajectory.
+ *
+ *   - `runEvolution` — population-based reflective mutation. Use when the
+ *     consumer wants the loop to GENERATE variants for a narrow prompt-only
+ *     surface. Prefer `runMultiShotTrajectoryOptimization` for product loops.
+ *
+ * Plus the current run-analysis bridge:
+ *
+ *   - `analyzeOptimization` — converts captured `RunRecord[]` into the
+ *     canonical insight report: score distributions, cost/quality frontier,
+ *     lift analysis when baseline/candidate ids are present, and ranked
+ *     recommendations. Idempotent and read-only with respect to the runs.
+ *
+ * Both optimization entrypoints pass through to agent-eval primitives
+ * unchanged so callers can read the upstream docs and trust their
+ * semantics. The RL bridge is wired here so families inherit the
+ * launch-decision-grade artifact set by default — see
+ * `agent-builder/src/lib/.server/eval/auto-research-runner.ts` for the
+ * canonical reference wiring.
+ */
+
+import {
+  PairwiseSteeringOptimizer,
+  type SteeringOptimizationRow,
+  type SteeringOptimizationResult,
+  type SteeringOptimizerConfig,
+} from '@tangle-network/agent-eval'
+import {
+  runImprovementLoop,
+  runOptimization,
+  type RunImprovementLoopOptions,
+  type RunImprovementLoopResult,
+  type RunOptimizationOptions,
+  type RunOptimizationResult,
+  type Scenario,
+} from '@tangle-network/agent-eval/campaign'
+import {
+  analyzeRuns,
+  type AnalyzeRunsOptions,
+  type InsightReport,
+} from '@tangle-network/agent-eval/contract'
+
+export interface SteeringOptimizationInput {
+  rows: SteeringOptimizationRow[]
+  config?: SteeringOptimizerConfig
+}
+export interface SteeringOptimizationOutput extends SteeringOptimizationResult {}
+
+/**
+ * Run a single-shot steering-bundle optimization. The provided `evaluate`
+ * function is the bridge to the eval-harness — it should call into the
+ * scenarios + judge-rubric layers and return a `RunScore`.
+ */
+export async function runSteeringOptimization(
+  input: SteeringOptimizationInput,
+): Promise<SteeringOptimizationOutput> {
+  return new PairwiseSteeringOptimizer().optimize(input.rows, input.config)
+}
+
+/**
+ * Run GEPA-style optimization over full agent trajectories.
+ *
+ * This is the default for real agents. A single trial may contain one turn or
+ * many turns; the runner owns execution, the scorer emits actionable side
+ * information, and agent-eval owns paired seeds, Pareto selection, and optional
+ * holdout promotion.
+ */
+export async function runMultiShotTrajectoryOptimization<P>(
+  config: MultiShotOptimizationConfig<P>,
+): Promise<MultiShotOptimizationResult<P>> {
+  return runOptimization(config)
+}
+
+/**
+ * Run a population-based prompt evolution. Consumer supplies a `scoreAdapter`
+ * (run a (variant, scenario, rep) trial) and a `mutateAdapter` (produce
+ * children given trace evidence). Pareto-selected, crowding-distance
+ * tie-broken, generation-by-generation.
+ *
+ * NOTE: use this for narrow prompt/signature surfaces. Product agents should
+ * usually use `runMultiShotTrajectoryOptimization` so the optimizer sees the
+ * whole task trajectory and ASI, not a surrogate scalar.
+ */
+export async function runEvolution<P>(
+  config: PromptEvolutionConfig<P>,
+): Promise<PromptEvolutionResult<P>> {
+  return runOptimization(config)
+}
+
+/**
+ * Run the full promotion shell: optimize on train, re-score baseline vs winner
+ * on holdout, and return the gate decision plus promoted diff. Use this when
+ * the caller has the held-out split required for a launch decision.
+ */
+export async function runPromotedImprovementLoop<P>(
+  config: PromotedImprovementConfig<P>,
+): Promise<PromotedImprovementResult<P>> {
+  return runImprovementLoop(config)
+}
+
+/**
+ * Analyze captured run records into the canonical decision packet —
+ * score distributions, lift when paired baseline/candidate ids exist,
+ * cost/quality frontier, failure clusters when an analyst is configured,
+ * and ranked recommendations.
+ *
+ * Idempotent and read-only with respect to the captured runs. Call after the
+ * optimization or promotion loop has emitted `RunRecord[]`; do NOT invent
+ * records from aggregate scores.
+ *
+ * @example
+ *   const rl = await analyzeOptimization({
+ *     runs,
+ *     baselineCandidateId: 'baseline',
+ *     candidateCandidateId: optimized.winnerSurfaceHash,
+ *     split: 'holdout',
+ *   })
+ *   // rl.recommendations → ranked launch / hold / investigate guidance
+ */
+export async function analyzeOptimization(
+  options: AnalyzeOptimizationResultOptions,
+): Promise<AnalyzeOptimizationResultReport> {
+  return analyzeRuns(options)
+}
+
+export type MultiShotOptimizationConfig<P = unknown> = RunOptimizationOptions<Scenario, P>
+export type MultiShotOptimizationResult<P = unknown> = RunOptimizationResult<P, Scenario>
+export type PromptEvolutionConfig<P = unknown> = RunOptimizationOptions<Scenario, P>
+export type PromptEvolutionResult<P = unknown> = RunOptimizationResult<P, Scenario>
+export type PromotedImprovementConfig<P = unknown> = RunImprovementLoopOptions<Scenario, P>
+export type PromotedImprovementResult<P = unknown> = RunImprovementLoopResult<P, Scenario>
+export type AnalyzeOptimizationResultOptions = AnalyzeRunsOptions
+export type AnalyzeOptimizationResultReport = InsightReport
+
+export type { SteeringOptimizationRow, SteeringOptimizationResult, SteeringOptimizerConfig }

package/registry/layers/agent-eval/auto-research/files/src/eval/auto-research/pareto.ts

@@ -0,0 +1,60 @@
+/**
+ * auto-research:pareto — multi-objective frontier over (quality, cost, latency).
+ *
+ * A research-harness rarely has a single scalar goal. A variant that
+ * raises pass-rate by 2pp but doubles cost is not strictly an
+ * improvement; the validator should surface BOTH points on the frontier
+ * and let the operator decide.
+ *
+ * This module composes `paretoFrontier` and `paretoFrontierWithCrowding`
+ * from agent-eval. The default objectives are the three axes every
+ * research-harness should be tracking; consumers can override.
+ */
+
+import {
+  paretoFrontier,
+  paretoFrontierWithCrowding,
+  type Objective,
+  type ParetoResult,
+} from '@tangle-network/agent-eval'
+
+export interface VariantPoint {
+  variantId: string
+  /** 0..1 quality (pass-rate, judge mean, whatever the harness reports). */
+  quality: number
+  /** USD spent across the run. Lower is better. */
+  costUsd: number
+  /** Wall-clock seconds. Lower is better. */
+  wallSeconds: number
+}
+
+export const DEFAULT_OBJECTIVES: Objective<VariantPoint>[] = [
+  { name: 'quality', direction: 'maximize', value: (p) => p.quality },
+  { name: 'cost', direction: 'minimize', value: (p) => p.costUsd },
+  { name: 'latency', direction: 'minimize', value: (p) => p.wallSeconds },
+]
+
+/**
+ * Filter a candidate list to the Pareto-non-dominated set across
+ * (quality, cost, latency). Pass-through to agent-eval's `paretoFrontier`.
+ */
+export function frontier(
+  candidates: VariantPoint[],
+  objectives: Objective<VariantPoint>[] = DEFAULT_OBJECTIVES,
+): ParetoResult<VariantPoint> {
+  return paretoFrontier(candidates, objectives)
+}
+
+/**
+ * NSGA-II frontier with crowding-distance tie-break. Returns the frontier
+ * sorted by descending crowding distance — `.slice(0, k)` to pick K
+ * diverse winners when a single scalar isn't desired.
+ */
+export function diverseFrontier(
+  candidates: VariantPoint[],
+  objectives: Objective<VariantPoint>[] = DEFAULT_OBJECTIVES,
+): Array<{ candidate: VariantPoint; distance: number }> {
+  return paretoFrontierWithCrowding(candidates, objectives)
+}
+
+export type { Objective, ParetoResult } from '@tangle-network/agent-eval'

package/registry/layers/agent-eval/auto-research/files/src/eval/auto-research/propose-review.ts

@@ -0,0 +1,36 @@
+/**
+ * auto-research:propose-review — typed wrapper around `runProposeReview`.
+ *
+ * The propose/verify/review primitive is the inner loop of any
+ * hypothesis-driven research run: given a goal + a verifier, iterate up to
+ * N shots, each refining the proposal using the prior verification output.
+ *
+ * This wrapper enforces the shape every research-harness consumer cares
+ * about (goal description in, typed final state + score out) so families
+ * compose this layer without re-deriving the agent-eval surface.
+ */
+
+import {
+  runProposeReview,
+  type ProposeReviewConfig,
+  type ProposeReviewReport,
+} from '@tangle-network/agent-eval'
+
+export interface ProposeReviewInput<State, Summary = unknown>
+  extends ProposeReviewConfig<State, Summary> {}
+
+export interface ProposeReviewOutput<State, Summary = unknown>
+  extends ProposeReviewReport<State, Summary> {}
+
+/**
+ * Drive a propose / verify / review loop until the verifier passes, the
+ * shot cap is hit, or the reviewer signals stop. Returns the final state +
+ * the per-shot trace for downstream reporting.
+ */
+export async function proposeReview<State, Summary = unknown>(
+  config: ProposeReviewInput<State, Summary>,
+): Promise<ProposeReviewOutput<State, Summary>> {
+  return runProposeReview<State, Summary>(config)
+}
+
+export type { ProposeReviewConfig, ProposeReviewReport } from '@tangle-network/agent-eval'

package/registry/layers/agent-eval/auto-research/manifest.json

@@ -0,0 +1,86 @@
+{
+  "id": "auto-research",
+  "group": "agent-eval",
+  "description": "Composable auto-research layer — wires the eval-harness's scenarios + judges + regression gates into agent-eval's current campaign optimization primitives (runOptimization, runImprovementLoop, runProposeReview, PairwiseSteeringOptimizer, paretoFrontier). Lets a family run hypothesis-driven prompt + steering evolution against the same harness used for measurement. Required by agent-research-harness-ts.",
+  "appliesTo": [
+    "agent-research-harness-ts"
+  ],
+  "provides": [
+    "eval:auto-research"
+  ],
+  "requires": [
+    "eval:scenarios",
+    "eval:judge-rubric",
+    "eval:regression"
+  ],
+  "packageDeps": {
+    "dependencies": {
+      "@tangle-network/agent-eval": "^0.23.0"
+    }
+  },
+  "files": [
+    {
+      "source": "files/src/eval/auto-research/loop.ts",
+      "target": "src/eval/auto-research/loop.ts"
+    },
+    {
+      "source": "files/src/eval/auto-research/propose-review.ts",
+      "target": "src/eval/auto-research/propose-review.ts"
+    },
+    {
+      "source": "files/src/eval/auto-research/pareto.ts",
+      "target": "src/eval/auto-research/pareto.ts"
+    },
+    {
+      "source": "files/src/eval/auto-research/index.ts",
+      "target": "src/eval/auto-research/index.ts"
+    },
+    {
+      "source": "files/src/eval/auto-research/README.md",
+      "target": "src/eval/auto-research/README.md"
+    }
+  ],
+  "validationChecks": [
+    {
+      "type": "file-exists",
+      "path": "src/eval/auto-research/loop.ts"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/eval/auto-research/propose-review.ts"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/eval/auto-research/pareto.ts"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/eval/auto-research/index.ts"
+    }
+  ],
+  "contextHints": {
+    "extensionPoints": [
+      "src/eval/auto-research/loop.ts",
+      "src/eval/auto-research/pareto.ts"
+    ]
+  },
+  "keywords": [
+    "auto-research",
+    "prompt-evolution",
+    "optimization-loop",
+    "pareto-frontier",
+    "propose-review"
+  ],
+  "tieredKeywords": {
+    "tier1": [
+      "auto-research-loop",
+      "prompt-evolution-loop",
+      "optimization-loop-eval"
+    ],
+    "tier2": [
+      "pareto-frontier",
+      "propose-review",
+      "ax-gepa-steering"
+    ]
+  }
+}

package/registry/layers/agent-eval/judge-pairwise/files/src/eval/judges/README.md

@@ -0,0 +1,86 @@
+# Pairwise judge
+
+Position-bias-aware A-vs-B evaluation built on `@tangle-network/agent-eval`'s
+`PairwiseSteeringOptimizer` plus the `positionalBias` / `verbosityBias` /
+`selfPreference` adversarial-bias suite.
+
+## Why pairwise
+
+Absolute rubric scores are noisy when variants are close (delta < 0.05). A
+ranked decision ("v2 is better than v1 on this scenario") is more reliable
+than absolute calibration, especially across heterogeneous prompts where
+the rubric scale isn't preserved.
+
+## Why position-corrected
+
+LLM judges score the candidate they see first higher than the same
+candidate shown second. Without position correction, every comparison is
+biased toward whichever variant the harness happens to put in slot A.
+
+The runner invokes the supplied `judge` function **twice per scenario** —
+once with `(first: A, second: B)` and once with `(first: B, second: A)` —
+and only declares a winner when the SAME variant wins in both orderings.
+Disagreement collapses to `'tie'` and surfaces as a positional bias
+signal in the report. Score reuse across orderings is forbidden by
+construction; the API requires a real `judge` callable.
+
+## Judge contract
+
+```ts
+const judge: PairwiseJudge = async ({ first, second, scenarioId }) => {
+  // Run your LLM/human/learned judge here. Return both scores in 0..1.
+  return { firstScore: 0.82, secondScore: 0.79 }
+}
+```
+
+Implementations MUST treat `first` and `second` as opaque presentation
+slots; sorting/normalising by `variantId` inside the judge erases the
+position-bias signal that this layer exists to surface.
+
+## When to use
+
+- After rubric eval (`agent-eval:judge-rubric`) shortlists 2-4 candidates.
+- When absolute scores cluster within 0.05 (signal lost in rubric noise).
+- For prompt iteration where you have many sibling variants to triage.
+
+## When NOT to use
+
+- For absolute thresholds. Pairwise tells you "A > B", not "A meets the
+  bar". Use rubric eval for thresholds.
+- When variants disagree on scenario coverage. Pairwise only scores
+  scenarios both variants ran.
+
+## Running
+
+```bash
+tsx src/eval/judges/pairwise-cli.ts ./runs/v1 ./runs/v2 --judge-family claude-opus-4
+```
+
+Both directories must contain `outputs.json` with `{ variantId, family,
+scenarios: [...] }`. Set `PAIRWISE_JSON_OUT=path/to/report.json` to also
+emit a structured sidecar for CI ingestion.
+
+## Bias gates
+
+The example in `pairwise.example.ts` shows the canonical guard:
+
+```ts
+if (Math.abs(report.bias.position.avgDelta) > 0.1) throw new Error(...)
+if (Math.abs(report.bias.verbosity.pearson) > 0.6) throw new Error(...)
+```
+
+`selfPreference` is computed only when `judgeFamily` is provided AND the
+variants declare `family`. Non-trivial `deltaMean` (>0.05) means the
+judge prefers outputs from its own family — swap to a heterogeneous
+judge fleet (`runJudgeFleet` from agent-eval) before promoting.
+
+## What composes here
+
+| Capability         | Source                                                |
+| ------------------ | ----------------------------------------------------- |
+| `eval:scenarios`   | `agent-eval:scenarios` (Worker 1) — corpus this reads |
+| `eval:judge-rubric`| `agent-eval:judge-rubric` (Worker 1) — produces `RunScore` per variant |
+| `eval:judge-pairwise` | this layer — composes both for ranked decision    |
+
+This layer does NOT re-score outputs — it consumes `RunScore` from rubric
+eval. If rubric eval hasn't run, run it first.

package/registry/layers/agent-eval/judge-pairwise/files/src/eval/judges/pairwise-cli.ts

@@ -0,0 +1,118 @@
+#!/usr/bin/env tsx
+/**
+ * Pairwise eval CLI.
+ *
+ *   tsx pairwise-cli.ts <variant-a-dir> <variant-b-dir> [--judge-family <id>]
+ *
+ * Each variant dir is expected to contain `outputs.json`:
+ *   {
+ *     "variantId": "v1",
+ *     "family": "claude-opus-4",
+ *     "scenarios": [{ "scenarioId": "s1", "output": "...", "bundle": {...},
+ *                     "score": {...RunScore}, "metadata": {...} }]
+ *   }
+ *
+ * Emits a markdown report to stdout + a structured JSON sidecar for CI.
+ */
+
+import { readFileSync, writeFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import process from 'node:process'
+
+import {
+  runPairwise,
+  type PairwiseJudge,
+  type PairwiseReport,
+  type VariantOutputs,
+} from './pairwise-runner.js'
+
+const parseArgs = (argv: string[]): { aDir: string; bDir: string; judgeFamily?: string } => {
+  const args = argv.slice(2)
+  const positional: string[] = []
+  let judgeFamily: string | undefined
+  for (let i = 0; i < args.length; i += 1) {
+    const a = args[i]
+    if (a === '--judge-family') {
+      judgeFamily = args[++i]
+    } else if (a === '--help' || a === '-h') {
+      console.log('Usage: pairwise-cli.ts <variant-a-dir> <variant-b-dir> [--judge-family <id>]')
+      process.exit(0)
+    } else {
+      positional.push(a)
+    }
+  }
+  if (positional.length !== 2) {
+    console.error('error: expected exactly two positional args (variant-a-dir, variant-b-dir)')
+    process.exit(2)
+  }
+  return { aDir: positional[0], bDir: positional[1], judgeFamily }
+}
+
+const loadVariant = (dir: string): VariantOutputs => {
+  const path = resolve(dir, 'outputs.json')
+  const raw = readFileSync(path, 'utf8')
+  const parsed = JSON.parse(raw) as VariantOutputs
+  if (!parsed.variantId || !Array.isArray(parsed.scenarios)) {
+    throw new Error(`invalid variant outputs at ${path}: missing variantId or scenarios[]`)
+  }
+  return parsed
+}
+
+const renderMarkdown = (r: PairwiseReport): string => {
+  const lines: string[] = []
+  lines.push(`# Pairwise eval — ${r.variantA} vs ${r.variantB}`)
+  lines.push('')
+  lines.push(`## Wins  A: ${r.winCounts.a}  B: ${r.winCounts.b}  ties: ${r.winCounts.tie}`)
+  lines.push('')
+  lines.push(`Optimizer recommends: **${r.optimizer.recommendedVariantId}**`)
+  lines.push(`> ${r.optimizer.rationale}`)
+  lines.push('')
+  lines.push('## Bias diagnostics')
+  lines.push(`- **positional**: avgDelta=${r.bias.position.avgDelta.toFixed(4)} (n=${r.bias.position.n})`)
+  lines.push(`- **verbosity**: pearson=${r.bias.verbosity.pearson.toFixed(4)} (n=${r.bias.verbosity.n})`)
+  if (r.bias.selfPreference) {
+    const sp = r.bias.selfPreference
+    lines.push(`- **self-preference**: deltaMean=${sp.deltaMean.toFixed(4)} (in=${sp.inFamilyMean.toFixed(3)}, out=${sp.outOfFamilyMean.toFixed(3)}, n=${sp.n})`)
+  }
+  lines.push('')
+  lines.push('## Per-scenario verdicts')
+  for (const s of r.perScenario) {
+    lines.push(`- ${s.scenarioId}: **${s.verdict}** (A=${s.meanA.toFixed(3)}, B=${s.meanB.toFixed(3)}, Δ=${s.positionDelta.toFixed(3)})`)
+  }
+  return lines.join('\n')
+}
+
+/**
+ * CLI judge: replays the pre-computed `score.success` from each variant's
+ * outputs.json. Because the score is precomputed (rubric eval already ran
+ * per-variant), there is no presentation-order signal to recover at this
+ * stage — both orderings will return the same numbers, position-bias will
+ * be reported as 0, and the verdict reflects only the rubric-score gap.
+ *
+ * To exercise REAL position-bias correction at the CLI layer, callers
+ * must run a true pairwise judge (e.g. invokeMetaJudge with both outputs
+ * in a single prompt) and write a CLI variant that wraps it. The library
+ * `runPairwise` is the integration point.
+ */
+const replayJudge: PairwiseJudge = async ({ first, second }) => ({
+  firstScore: first.score.success,
+  secondScore: second.score.success,
+})
+
+const main = async (): Promise<void> => {
+  const { aDir, bDir, judgeFamily } = parseArgs(process.argv)
+  const variantA = loadVariant(aDir)
+  const variantB = loadVariant(bDir)
+  const report = await runPairwise({ variantA, variantB, judge: replayJudge, judgeFamily })
+  process.stdout.write(renderMarkdown(report) + '\n')
+  // Write structured sidecar for CI ingestion.
+  const jsonOut = process.env.PAIRWISE_JSON_OUT
+  if (jsonOut) {
+    writeFileSync(jsonOut, JSON.stringify(report, null, 2))
+  }
+}
+
+main().catch((err) => {
+  console.error('[pairwise-cli] fatal:', err)
+  process.exit(2)
+})