dsh-llm-verifier 0.3.0 → 0.4.0

package/src/decisions.ts

@@ -0,0 +1,244 @@
+import { randomUUID } from 'node:crypto'
+import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { sanitizeVerifierText } from './session.ts'
+
+/**
+ * One model call captured inside an invocation.
+ *
+ * Everything the plugin decides is decided inside a prompt: "why did the progress
+ * judge answer K again" is a question about the exact text that was sent and the raw
+ * answer that came back. The statistics store answers "how much did it cost"; this one
+ * answers "why did it say that".
+ */
+export interface DecisionCall {
+  /** Which part of the invocation made the call (criterion, repeat, phase). */
+  label: string
+  /** Scoring channel that answered (top-logprobs | explicit-tag). */
+  channel: string
+  prompt: string
+  output: string
+  /** The score this call produced, when it produced one. */
+  score?: number
+}
+
+/** Sink the engine reports every REAL model call to (cache hits make no call). */
+export type DecisionTrace = (call: DecisionCall) => void
+
+export interface DecisionRecord {
+  id: string
+  toolName: string
+  phase: string
+  startedAt: number
+  provider: string
+  model: string
+  calls: DecisionCall[]
+}
+
+interface DecisionsDocument {
+  version: 1
+  records: DecisionRecord[]
+}
+
+/**
+ * Upper bound on captured calls per invocation.
+ *
+ * A session acceptance makes six; an explicit best-of-N makes about 27 (N drafts, the
+ * tournament, and the winner-vs-baseline comparison). Keeping the old 12 would have dropped
+ * the drafts from the snapshot entirely — judge labels sort before `draft N` — so the one
+ * record that explains "which draft won and why" could not show the drafts at all. The
+ * per-record character budget is unchanged and shared equally, so the extra calls shrink each
+ * window instead of growing the file. 32 is the ceiling the equal-share floor allows:
+ * 32 x (512 prompt + 256 output) still fits in {@link MAX_RECORD_CHARS}.
+ */
+const MAX_CALLS = 32
+const MAX_PROMPT_CHARS = 8000
+const MAX_OUTPUT_CHARS = 4000
+/** Upper bound on one record's captured text, so a wide select cannot fill the file. */
+const MAX_RECORD_CHARS = 30000
+/** Smallest window worth storing; at MAX_CALLS the equal share stays above this floor. */
+const MIN_PROMPT_CHARS = 512
+const MIN_OUTPUT_CHARS = 256
+
+export function resolveDecisionsFile(cacheFile: string): string {
+  return join(dirname(cacheFile), 'decisions-v1.json')
+}
+
+/**
+ * Redact and bound captured calls.
+ *
+ * Runs the same sanitizer the prompts do, so a snapshot can never persist a secret the
+ * judge itself never saw, and drops calls past the per-record budget instead of writing
+ * an unbounded file.
+ * @param calls - captured calls, oldest first.
+ * @returns The bounded calls; empty when nothing was captured.
+ */
+export function boundCaptureText(text: string, maxChars: number): string {
+  if (!Number.isSafeInteger(maxChars) || maxChars < 1) throw new Error('llm-verifier: decision capture cap must be a positive integer')
+  const clean = sanitizeVerifierText(text, Math.max(1, text.length))
+  if (clean.length <= maxChars || maxChars < 256) return sanitizeVerifierText(text, maxChars)
+  const head = Math.floor(maxChars / 2)
+  const tail = maxChars - head
+  const omitted = Math.max(0, clean.length - head - tail)
+  const marker = '\n[… ' + omitted + ' characters omitted …]\n'
+  // The marker is charged to the tail's share, so the sum can never exceed the cap.
+  const keepTail = Math.max(0, tail - marker.length)
+  return clean.slice(0, head) + marker + (keepTail === 0 ? '' : clean.slice(-keepTail))
+}
+
+/**
+ * Redact and bound captured calls.
+ *
+ * Runs the same sanitizer the prompts do, so a snapshot can never persist a secret the
+ * judge itself never saw. The record is bounded twice: at {@link MAX_CALLS} calls, and at
+ * {@link MAX_RECORD_CHARS} characters shared EQUALLY between the calls it keeps — a six-call
+ * session acceptance must not shrink to its first three calls, and which three survived must not
+ * depend on completion order. Text uses {@link boundCaptureText}, which keeps both ends: a
+ * session-acceptance prompt runs past 100k characters, and head-only truncation kept the
+ * instructions while dropping the trajectory tail the judge actually graded.
+ * @param calls - captured calls; callers sort them by label so the bounded set is deterministic.
+ * @returns The bounded calls; empty when nothing was captured.
+ */
+export function boundDecisionCalls(calls: readonly DecisionCall[]): DecisionCall[] {
+  if (calls.length === 0) return []
+  // Over the count cap, keep an EVEN SPREAD (first and last included) rather than a prefix.
+  // A prefix is the wrong sample whenever the labels cluster: the engine labels its own calls
+  // in sorted order, so an oversized best-of-N (n=4 makes ~46 calls: the baseline, then the
+  // tournament, then `draft N` last) would lose every draft and keep only judge calls.
+  const selected = calls.length <= MAX_CALLS
+    ? [...calls]
+    : Array.from({ length: MAX_CALLS }, (_, index) => calls[Math.round(index * (calls.length - 1) / (MAX_CALLS - 1))]!)
+  // Equal share per call, not first-come-first-served. A session acceptance makes six calls on
+  // ~8k-char prompts; a fixed 8k + 4k per call let only the FIRST THREE through, and which three
+  // survived depended on completion order, i.e. on the network — so the most expensive path in
+  // the plugin stored a different (and partial) set of criteria every time. Every call now keeps
+  // a smaller window: boundCaptureText keeps both ends, so the instruction head and the graded
+  // trajectory tail still survive.
+  const overhead = selected.reduce((sum, call) => sum + Math.min(call.label.length, 200) + Math.min(call.channel.length, 40), 0)
+  const perCall = Math.max(MIN_PROMPT_CHARS + MIN_OUTPUT_CHARS, Math.floor((MAX_RECORD_CHARS - overhead) / selected.length))
+  const bounded: DecisionCall[] = []
+  let used = 0
+  for (const call of selected) {
+    // The two windows must SUM to at most perCall, which the previous 80/20 split did not
+    // guarantee: once perCall fell below ~1000 (a wide invocation), the output floor pushed the
+    // pair over its share and the safety net below silently dropped the tail — the drafts.
+    const promptChars = Math.min(MAX_PROMPT_CHARS, Math.max(MIN_PROMPT_CHARS, Math.min(Math.floor(perCall * 0.8), perCall - MIN_OUTPUT_CHARS)))
+    const outputChars = Math.min(MAX_OUTPUT_CHARS, Math.max(MIN_OUTPUT_CHARS, perCall - promptChars))
+    const value: DecisionCall = {
+      label: call.label.slice(0, 200),
+      channel: call.channel.slice(0, 40),
+      prompt: boundCaptureText(call.prompt, promptChars),
+      output: boundCaptureText(call.output, outputChars),
+      ...(typeof call.score === 'number' && Number.isFinite(call.score) ? { score: call.score } : {}),
+    }
+    const cost = value.label.length + value.channel.length + value.prompt.length + value.output.length
+    // Safety net only: the equal share already fits at MAX_CALLS, unless perCall is below the
+    // two floors (unreachable for realistic label lengths).
+    if (bounded.length > 0 && used + cost > MAX_RECORD_CHARS) break
+    bounded.push(value)
+    used += cost
+  }
+  return bounded
+}
+
+export interface DecisionInput {
+  toolName: string
+  phase: string
+  startedAt: number
+  provider: string
+  model: string
+  calls: readonly DecisionCall[]
+}
+
+/**
+ * Per-topic ring buffer of decision snapshots.
+ *
+ * Mirrors {@link import('./statistics.ts').StatisticsStore}: one JSON file beside the
+ * statistics of the same topic, atomic replace on write, a fixed number of records, so
+ * the observable cost of "keep the evidence that explains a verdict" is bounded.
+ */
+export class DecisionStore {
+  private loaded = false
+  private hydrating: Promise<void> | undefined
+  private records: DecisionRecord[] = []
+  private writing: Promise<void> = Promise.resolve()
+
+  constructor(private readonly file: string, private readonly maxEntries = 40) {
+    if (!Number.isSafeInteger(maxEntries) || maxEntries <= 0) throw new Error('llm-verifier: decision maxEntries must be a positive integer')
+  }
+
+  async record(input: DecisionInput): Promise<DecisionRecord | undefined> {
+    const calls = boundDecisionCalls(input.calls)
+    if (calls.length === 0) return undefined
+    const record: DecisionRecord = {
+      id: randomUUID(),
+      toolName: input.toolName,
+      phase: input.phase,
+      startedAt: input.startedAt,
+      provider: input.provider,
+      model: input.model,
+      calls,
+    }
+    const operation = async () => {
+      await this.load()
+      this.records.push(record)
+      if (this.records.length > this.maxEntries) this.records.splice(0, this.records.length - this.maxEntries)
+      await this.persist()
+    }
+    this.writing = this.writing.then(operation, operation)
+    await this.writing
+    return record
+  }
+
+  /** One snapshot by invocation id, or undefined when it was pruned or never captured. */
+  async find(id: string): Promise<DecisionRecord | undefined> {
+    await this.writing.catch(() => {})
+    await this.load()
+    return this.records.find(record => record.id === id)
+  }
+
+  async load(): Promise<void> {
+    if (this.loaded) return
+    this.hydrating ??= (async () => {
+      try {
+        const document = JSON.parse(await readFile(this.file, 'utf8')) as Partial<DecisionsDocument>
+        if (document.version === 1 && Array.isArray(document.records)) {
+          this.records = document.records.filter(isDecisionRecord).slice(-this.maxEntries)
+        }
+        this.loaded = true
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+          this.loaded = true
+          return
+        }
+        throw error
+      } finally {
+        this.hydrating = undefined
+      }
+    })()
+    await this.hydrating
+  }
+
+  private async persist(): Promise<void> {
+    const snapshot: DecisionsDocument = { version: 1, records: this.records }
+    await mkdir(dirname(this.file), { recursive: true })
+    const temporary = this.file + '.tmp-' + process.pid + '-' + randomUUID()
+    await writeFile(temporary, JSON.stringify(snapshot), 'utf8')
+    try { await rename(temporary, this.file) } catch (error) { await unlink(temporary).catch(() => {}); throw error }
+  }
+}
+
+/**
+ * Loose validation, so a record written by a newer/older plugin still loads.
+ * @param value - parsed JSON entry.
+ * @returns True when the entry carries the fields the dashboard needs.
+ */
+function isDecisionRecord(value: unknown): value is DecisionRecord {
+  if (typeof value !== 'object' || value === null) return false
+  const row = value as Partial<DecisionRecord>
+  return typeof row.id === 'string'
+    && typeof row.toolName === 'string'
+    && typeof row.startedAt === 'number'
+    && Array.isArray(row.calls)
+    && row.calls.every(call => typeof call === 'object' && call !== null && typeof (call as DecisionCall).prompt === 'string' && typeof (call as DecisionCall).output === 'string')
+}

package/src/engine.test.ts

@@ -166,6 +166,18 @@ describe('VerifierEngine cache identity', () => {
     expect(result.index).toBe(0)
     expect(result.pivots).toEqual([])
   })
+  it('prefixes snapshot labels so two comparisons on the same criteria stay distinguishable', async () => {
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream([]) } as any }), 4)
+    const labels: string[] = []
+    const trace = (call: { label: string }) => { labels.push(call.label) }
+    await engine.compare({ problem: 'task', candidateA: 'STRONG', candidateB: 'WEAK', repeats: 1, trace })
+    // Regression: without the prefix, a best-of-N baseline comparison was indistinguishable from
+    // the tournament it followed, and the snapshot read as if the judge had contradicted itself.
+    await engine.compare({ problem: 'task', candidateA: 'STRONG', candidateB: 'WEAK', repeats: 1, trace, traceLabelPrefix: 'baseline: ' })
+    expect(labels.filter(label => label.startsWith('baseline: '))).toHaveLength(3)
+    expect(labels.filter(label => !label.startsWith('baseline: '))).toHaveLength(3)
+  })
+
   it('uses version 6 in scoreOne cache identity with temperature (regression FIX 1)', async () => {
     const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-v6-'))
     try {
@@ -599,3 +611,122 @@ describe('VerifierEngine N-judge ensemble', () => {
     await expect(engine.track('Test problem', ['step 1'], [1], 1)).rejects.toThrow('track failed completely 1')
   })
 })
+
+/** Records start/end of every model call, tagged with the A/B slot orientation of its prompt. */
+function recordingStream(events: string[]): (options: any) => AsyncIterable<any[]> {
+  return function (options: any) {
+    const message = options.messages[0]
+    const prompt = typeof message.content === 'string' ? message.content : message.content.filter((block: any) => block.type === 'text').map((block: any) => block.text).join('')
+    const orientation = prompt.indexOf('ALPHA') < prompt.indexOf('BRAVO') ? 'AB' : 'BA'
+    return (async function* () {
+      events.push('start:' + orientation)
+      await new Promise(resolve => setTimeout(resolve, 1))
+      events.push('end:' + orientation)
+      yield* streamOf(chunks('<score_A> A </score_A>\n<score_B> T </score_B>'))
+    })()
+  }
+}
+
+/** Drops any call that is not already covered by the score cache. */
+function failIfCalled(): (options: any) => AsyncIterable<any[]> {
+  return function () {
+    throw new Error('unexpected model call')
+  }
+}
+
+describe('prefix warm-up', () => {
+  it('warms one call per distinct A/B slot orientation before fanning out, without extra calls', async () => {
+    const events: string[] = []
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: recordingStream(events) } as any }), 8)
+    const result = await engine.compare({ problem: 'task', candidateA: 'ALPHA', candidateB: 'BRAVO', repeats: 2 })
+
+    // Three default criteria x two repeats = six calls. The warm-up only reorders them, and
+    // odd repeats swap the slots, so each of the two prompt prefixes needs its own warm call.
+    expect(result.stats.calls).toBe(6)
+    // Indices, not values: the two orientations repeat, so looking a start up by its text would
+    // find the earlier warm call instead of the later fan-out one.
+    const startIndices = events.map((entry, index) => (entry.startsWith('start:') ? index : -1)).filter(index => index >= 0)
+    expect(startIndices).toHaveLength(6)
+    expect(new Set(startIndices.slice(0, 2).map(index => events[index])).size).toBe(2)
+    // Nothing else starts until BOTH warm calls (one per orientation) have finished: with a
+    // single warm job the whole swapped-slot half of the fan-out was a cold cache miss.
+    expect(events.slice(0, startIndices[2]!).filter(entry => entry.startsWith('end:'))).toHaveLength(2)
+  })
+
+  it('warms the identical track prompt once before the repeated calls', async () => {
+    const events: string[] = []
+    const stream = () => (async function* () {
+      events.push('start')
+      await new Promise(resolve => setTimeout(resolve, 1))
+      events.push('end')
+      yield* streamOf(chunks('<c1> T </c1>'))
+    })()
+    const engine = new VerifierEngine(clientConfig({ llm: { stream } as any }), 8)
+    const result = await engine.track('problem', ['step 1'], [1], 3)
+
+    // Every repeat sends the same prompt, so repeats 2 and 3 must not start before repeat 1
+    // has returned and populated the provider prefix cache.
+    expect(result.stats.calls).toBe(3)
+    const starts = events.map((entry, index) => (entry === 'start' ? index : -1)).filter(index => index >= 0)
+    expect(starts).toHaveLength(3)
+    expect(events.slice(0, starts[1]!).filter(entry => entry === 'end')).toHaveLength(1)
+  })
+})
+
+describe('byte-identical candidates', () => {
+  it('compares identical sides as an uninformative tie without calling the model', async () => {
+    const judged: string[] = []
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged) } as any }), 4)
+    const result = await engine.compare({ problem: 'task', candidateA: 'SAME', candidateB: 'SAME', repeats: 2 })
+
+    expect(judged).toHaveLength(0)
+    expect(result.calls).toBe(0)
+    expect(result.identical).toBe(true)
+    expect(result.winner).toBe('tie')
+    // 0.5 on both sides, not a confident 1.0: the acceptance gate must stay closed.
+    expect(result.scoreA).toBe(0.5)
+    expect(result.scoreB).toBe(0.5)
+    expect(result.criteria.map(row => row.scoreA)).toEqual([0.5, 0.5, 0.5])
+    expect(result.judges[0]).toMatchObject({ ok: true, calls: 0, winner: 'tie' })
+  })
+
+  it('ranks all-identical candidates without a single model call', async () => {
+    const judged: string[] = []
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged) } as any }), 4)
+    const result = await engine.select({ problem: 'task', candidates: ['SAME', 'SAME', 'SAME'], repeats: 1 })
+
+    expect(judged).toHaveLength(0)
+    expect(result.calls).toBe(0)
+    expect(result.comparisons).toBe(0)
+    expect(result.identical).toBe(true)
+    expect(result.best).toBe('SAME')
+    expect(result.scores).toEqual([0.5, 0.5, 0.5])
+    expect(result.ranking).toEqual([0, 1, 2])
+    expect(result.judges[0]).toMatchObject({ ok: true, calls: 0, scores: [0.5, 0.5, 0.5] })
+  })
+
+  it('judges duplicated candidates once and maps the verdict back onto every index', async () => {
+    const judged: string[] = []
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged) } as any }), 4)
+    const result = await engine.select({ problem: 'task', candidates: ['STRONG-0', 'WEAK-1', 'STRONG-0'], repeats: 1 })
+
+    // One compared pair (two distinct candidates) x three default criteria = three calls,
+    // where the duplicate would otherwise have doubled the tournament.
+    expect(result.comparisons).toBe(1)
+    expect(result.stats.calls).toBe(3)
+    expect(judged).toHaveLength(3)
+    expect(result.best).toBe('STRONG-0')
+    expect(result.index).toBe(0)
+    expect(result.ranking[0]).toBe(0)
+    expect(result.scores).toHaveLength(3)
+    // The duplicate inherits its representative's score, so no index is left undefined.
+    expect(result.scores[2]).toBe(result.scores[0])
+    expect(result.scores[0]).toBeGreaterThan(result.scores[1]!)
+  })
+
+  it('rejects a blank candidate instead of spending calls on it', async () => {
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: failIfCalled() } as any }), 4)
+    await expect(engine.select({ problem: 'task', candidates: ['real work', '   '] })).rejects.toThrow(/blank/u)
+    await expect(engine.select({ problem: 'task', candidates: [] })).rejects.toThrow(/must not be empty/u)
+  })
+})

package/src/engine.ts

@@ -1,11 +1,13 @@
 import { addUsage, callVerifier, emptyUsage, predictScoringChannel, type ScoringMode, type UsageStats, type VerifierClientConfig, type VerifierImage } from './caller.ts'
 import { ScoreCache, SingleFlight, stableHash, type CachedPairScore } from './cache.ts'
+import type { DecisionTrace } from './decisions.ts'
 import {
   DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, accumulatePairs, buildPairwisePrompt, buildProgressPrompt,
-  extractProgressScore, extractScore, pivotRoundPairs, rankScores, ringCycle, topPivots, type Criterion,
+  dedupeCriterionId, extractProgressScore, extractScore, pivotRoundPairs, rankScores, ringCycle, slugCriterionId,
+  topPivots, type Criterion,
 } from './core.ts'
 
-export interface CompareOptions { problem: string; candidateA: string; candidateB: string; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; images?: readonly VerifierImage[] }
+export interface CompareOptions { problem: string; candidateA: string; candidateB: string; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; images?: readonly VerifierImage[]; trace?: DecisionTrace; /** Prefix for this comparison's decision-snapshot labels; one invocation that judges twice on the same criteria needs them distinguishable. */ traceLabelPrefix?: string }
 export interface CriterionResult { id: string; name: string; scoreA: number; scoreB: number }
 export interface RunStats extends UsageStats { cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
 
@@ -32,9 +34,11 @@ export interface CompareResult {
   stats: RunStats
   judges: JudgeScore[]
   agreement: number
+  /** Set when both sides were byte-identical: no model call was made and both sides score 0.5. */
+  identical?: true
 }
 
-export interface SelectOptions { problem: string; candidates: readonly string[]; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; pivots?: number; seed?: number; images?: readonly VerifierImage[] }
+export interface SelectOptions { problem: string; candidates: readonly string[]; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; pivots?: number; seed?: number; images?: readonly VerifierImage[]; trace?: DecisionTrace }
 
 export interface SelectResult {
   index: number
@@ -46,6 +50,8 @@ export interface SelectResult {
   calls: number
   stats: RunStats
   judges: JudgeScore[]
+  /** Set when every candidate was byte-identical: no pair was judged and every score is 0.5. */
+  identical?: true
 }
 
 function average(values: readonly number[]): number { return values.reduce((sum, value) => sum + value, 0) / (values.length || 1) }
@@ -58,6 +64,8 @@ function median(values: readonly number[]): number {
 }
 
 function blankStats(): RunStats { return { ...emptyUsage(), cacheHits: 0, cacheMisses: 0, estimatedCostUsd: 0, topLogprobScores: 0, explicitTagScores: 0 } }
+/** Candidate indices best-first by score, ties broken by index. */
+function rankByScore(scores: readonly number[]): number[] { return Array.from({ length: scores.length }, (_, index) => index).sort((a, b) => (scores[b] ?? 0) - (scores[a] ?? 0) || a - b) }
 function unorderedPair(a: number, b: number): string { return a < b ? a + ',' + b : b + ',' + a }
 
 /**
@@ -160,7 +168,12 @@ export class VerifierEngine {
     const keyForMode = (scoringMode: ScoringMode) => stableHash({ ...identity, scoringMode })
     const create = async () => {
       const completion = await callVerifier(client, prompt, signal, options.images)
-      return { scoreA: extractScore(completion, '<score_A>'), scoreB: extractScore(completion, '<score_B>'), usage: completion.usage, scoringMode: completion.scoringMode, createdAt: Date.now() }
+      const scoreA = extractScore(completion, '<score_A>')
+      const scoreB = extractScore(completion, '<score_B>')
+      // Traced here, not in scoreOne(): a cache hit or a merged in-flight call makes no
+      // model call, and a snapshot that showed one anyway would be a fabrication.
+      options.trace?.({ label: (options.traceLabelPrefix ?? '') + criterion.name + ' repeat ' + (repeat + 1), channel: completion.scoringMode, prompt, output: completion.text, score: scoreA })
+      return { scoreA, scoreB, usage: completion.usage, scoringMode: completion.scoringMode, createdAt: Date.now() }
     }
     const cache = this.cache
     if (cache === undefined) { const value = await create(); return { scores: [value.scoreA, value.scoreB], usage: value.usage, scoringMode: value.scoringMode, hit: false } }
@@ -181,12 +194,48 @@ export class VerifierEngine {
     await Promise.all(runners); return results
   }
 
+  /**
+   * Verdict for a comparison whose two sides are byte-identical.
+   *
+   * Deliberately uninformative: no model call is made, both sides score 0.5 and the winner is
+   * a tie. Judging identical text would ask the model to break a tie it cannot break, and any
+   * confident score would let the acceptance gate pass a session indistinguishable from the
+   * empty-work baseline. Callers that need "these are the same" read {@link CompareResult.identical}.
+   * @param criteria - the criteria the comparison would have scored.
+   * @returns A tie carrying 0.5 per criterion, zero calls and one agreeing judge per client.
+   */
+  private informationalTie(criteria: readonly Criterion[]): CompareResult {
+    const judges: JudgeScore[] = this.clients.map(client => ({ provider: client.provider, model: client.model, label: judgeLabel(client), ok: true, calls: 0, scoreA: 0.5, scoreB: 0.5, winner: 'tie' as const }))
+    return {
+      scoreA: 0.5,
+      scoreB: 0.5,
+      winner: 'tie',
+      criteria: criteria.map(criterion => ({ id: criterion.id, name: criterion.name, scoreA: 0.5, scoreB: 0.5 })),
+      calls: 0,
+      stats: this.finishStats(blankStats()),
+      judges,
+      agreement: 1,
+      identical: true,
+    }
+  }
+
   async compare(options: CompareOptions, signal?: AbortSignal): Promise<CompareResult> {
     const criteria = options.criteria?.length ? options.criteria : DEFAULT_CRITERIA
+    if (options.candidateA === options.candidateB) return this.informationalTie(criteria)
     const repeats = options.repeats ?? 2
     const jobs = criteria.flatMap(criterion => Array.from({ length: repeats }, (_, repeat) => ({ criterion, repeat })))
-    // Prefix warm-up: one criterion/repeat runs first, then the shared-prefix fan-out.
-    const warm = jobs.slice(0, 1); const rest = jobs.slice(1)
+    // Prefix warm-up: one job per DISTINCT prompt prefix runs first, then the rest fan out.
+    // Only the A/B slot order changes the prompt prefix (the criterion sits at the tail, see
+    // core.buildPairwisePrompt) and odd repeats swap the slots, so each orientation needs its own
+    // warm call: warming a single job left every swapped-slot call cold, which is where half the
+    // input tokens of a session acceptance went. The warm jobs are calls that had to happen anyway.
+    const warm: typeof jobs = []; const rest: typeof jobs = []
+    const warmedOrientations = new Set<number>()
+    for (const job of jobs) {
+      const orientation = job.repeat % 2
+      if (warmedOrientations.has(orientation)) rest.push(job)
+      else { warmedOrientations.add(orientation); warm.push(job) }
+    }
     const run = async (batch: typeof jobs) => this.mapLimited(batch, async ({ criterion, repeat }) => {
       const swapped = repeat % 2 === 1
       const candA = swapped ? options.candidateB : options.candidateA
@@ -338,6 +387,7 @@ export class VerifierEngine {
         groundTruthNote: options.groundTruthNote,
         repeats: options.repeats,
         images: options.images,
+        trace: options.trace,
       }, signal),
     }))
     const rewards = new Map<string, readonly [number, number]>()
@@ -372,7 +422,7 @@ export class VerifierEngine {
     return { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats: this.finishStats(stats) }
   }
 
-  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats; judges: JudgeScore[] }> {
+  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[], trace?: DecisionTrace): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats; judges: JudgeScore[] }> {
     if (!steps.length || !checkpoints.length) throw new Error('llm-verifier: steps and checkpoints must not be empty')
     for (const checkpoint of checkpoints) if (!Number.isSafeInteger(checkpoint) || checkpoint < 1 || checkpoint > steps.length) throw new Error('llm-verifier: each checkpoint must be an integer between 1 and steps.length')
     const prompt = buildProgressPrompt(problem, steps, checkpoints)
@@ -383,12 +433,16 @@ export class VerifierEngine {
     const judgePerRepeatScores: Array<number[][]> = Array.from({ length: this.clients.length }, () => [])
 
     const repeatIndices = Array.from({ length: repeats }, (_, index) => index)
-    const runs = await this.mapLimited(repeatIndices, async () => {
+    const runRepeat = async (repeatIndex: number) => {
       const judgeResults = await Promise.all(
         this.clients.map(async (client, k) => {
           try {
             const completion = await callVerifier(client, prompt, signal, images)
             const scores = checkpoints.map((_, index) => extractProgressScore(completion, '<c' + (index + 1) + '>'))
+            // One snapshot per repeat for the primary judge only: extra judges would
+            // multiply the record past its budget, and the primary is what the verdict
+            // reports.
+            if (k === 0) trace?.({ label: 'progress repeat ' + (repeatIndex + 1) + '/' + repeats + (this.clients.length > 1 ? ' judge 1/' + this.clients.length : ''), channel: completion.scoringMode, prompt, output: completion.text, score: scores[scores.length - 1] })
             return {
               k,
               ok: true as const,
@@ -427,7 +481,13 @@ export class VerifierEngine {
       }
 
       return checkpoints.map((_, cIndex) => median(successful.map(r => r.scores[cIndex]!)))
-    })
+    }
+    // Every repeat sends the SAME prompt, so the first repeat warms the provider prefix cache for
+    // the others; fanning all repeats out at once left every call cold. The repeat count is
+    // unchanged — only the order, and therefore the cache hits, change.
+    const runs: number[][] = []
+    if (repeatIndices.length > 0) runs.push(...await this.mapLimited(repeatIndices.slice(0, 1), runRepeat))
+    runs.push(...await this.mapLimited(repeatIndices.slice(1), runRepeat))
 
     const scores = checkpoints.map((_, index) => average(runs.map(run => run[index]!)))
     const judges: JudgeScore[] = this.clients.map((client, k) => {
@@ -457,8 +517,79 @@ export class VerifierEngine {
     return { scores, perRepeat: runs, calls: stats.calls, stats: this.finishStats(stats), judges }
   }
 
+  /**
+   * Verdict for a candidate list whose entries are all byte-identical.
+   *
+   * Ranking identical text is a coin flip, so the result is deliberately uninformative
+   * (0.5 everywhere) rather than a confident 1.0. No model call is made. This is the
+   * cost-saving half of upstream's majority-vote shortcut without its semantics: we never
+   * declare an unjudged candidate the winner, we only decline to spend calls on a tie.
+   * @param candidates - the identical candidates (length >= 2).
+   * @returns A ranking with every score at 0.5 and zero calls.
+   */
+  private identicalCandidates(candidates: readonly string[]): SelectResult {
+    const scores = candidates.map(() => 0.5)
+    const ranking = candidates.map((_, index) => index)
+    return {
+      index: 0,
+      best: candidates[0]!,
+      scores,
+      ranking,
+      pivots: [],
+      comparisons: 0,
+      calls: 0,
+      stats: blankStats(),
+      judges: this.clients.map(client => ({ provider: client.provider, model: client.model, label: judgeLabel(client), ok: true, calls: 0, scores: [...scores], ranking: [...ranking] })),
+      identical: true,
+    }
+  }
+
+  /**
+   * Judge only the DISTINCT candidates, then expand the verdict back onto the caller's list.
+   *
+   * Duplicated candidates are the common case when an agent pastes several drafts of the same
+   * artifact: every pair that touches a duplicate is a comparison that cannot change the
+   * ranking but still costs model calls. The tournament runs on the distinct list and every
+   * duplicate inherits its representative's score, so no index, score or ranking entry shifts.
+   * @param options - the original select options, duplicates included.
+   * @param signal - caller's abort signal.
+   * @returns The tournament verdict mapped back onto the original candidate list.
+   */
+  private async selectUnique(options: SelectOptions, signal?: AbortSignal): Promise<SelectResult> {
+    const unique: string[] = []
+    const representative: number[] = []
+    const seen = new Map<string, number>()
+    for (const candidate of options.candidates) {
+      let index = seen.get(candidate)
+      if (index === undefined) { index = unique.length; unique.push(candidate); seen.set(candidate, index) }
+      representative.push(index)
+    }
+    const result = await this.select({ ...options, candidates: unique }, signal)
+    const expand = (values: readonly number[]): number[] => representative.map(index => values[index] ?? 0)
+    const scores = expand(result.scores)
+    const judges = result.judges.map(judge => {
+      if (!judge.ok || judge.scores === undefined) return judge
+      const judgeScores = expand(judge.scores)
+      return { ...judge, scores: judgeScores, ...(judge.ranking === undefined ? {} : { ranking: rankByScore(judgeScores) }) }
+    })
+    return {
+      index: options.candidates.indexOf(result.best),
+      best: result.best,
+      scores,
+      ranking: rankByScore(scores),
+      pivots: result.pivots,
+      comparisons: result.comparisons,
+      calls: result.calls,
+      stats: result.stats,
+      judges,
+    }
+  }
+
   async select(options: SelectOptions, signal?: AbortSignal): Promise<SelectResult> {
     if (!options.candidates.length) throw new Error('llm-verifier: candidates must not be empty')
+    if (options.candidates.some(candidate => candidate.trim() === '')) throw new Error('llm-verifier: candidates must not contain blank entries')
+    if (options.candidates.length > 1 && options.candidates.every(candidate => candidate === options.candidates[0])) return this.identicalCandidates(options.candidates)
+    if (options.candidates.length > 1 && new Set(options.candidates).size !== options.candidates.length) return this.selectUnique(options, signal)
     if (options.candidates.length === 1) {
       const judges: JudgeScore[] = this.clients.map(client => ({
         provider: client.provider,
@@ -569,9 +700,35 @@ export class VerifierEngine {
   }
 }
 
+/**
+ * Normalize a caller-supplied `criteria` argument into the engine's canonical shape.
+ *
+ * Mirrors upstream's `normalize_criteria` so a caller does not have to fill in every field: a
+ * plain string is both the name and the instruction, and a missing `id` is slugged from the
+ * name (the score cache keys on the rendered prompt, so a slug is cosmetic). Ids are
+ * de-duplicated because `compare` groups per-criterion results by id — a collision would
+ * silently merge two criteria into one line of the verdict.
+ * @param input - undefined (the default rubric), or a non-empty array of strings/objects.
+ * @returns The criteria the engine will score with.
+ */
 export function normalizeCriteria(input: unknown): Criterion[] {
   if (input === undefined) return DEFAULT_CRITERIA
   if (!Array.isArray(input) || !input.length) throw new Error('llm-verifier: criteria must be a non-empty array')
-  return input.map((value, index) => { if (typeof value !== 'object' || value === null) throw new Error('llm-verifier: criteria[' + index + '] must be an object'); const row = value as Record<string, unknown>; for (const key of ['id', 'name', 'description']) if (typeof row[key] !== 'string' || row[key].trim().length === 0) throw new Error('llm-verifier: criteria[' + index + '].' + key + ' must be non-empty'); return { id: String(row.id), name: String(row.name), description: String(row.description) } })
+  const seen = new Set<string>()
+  return input.map((value, index) => {
+    if (typeof value === 'string') {
+      const text = value.trim()
+      if (!text) throw new Error('llm-verifier: criteria[' + index + '] must not be blank')
+      return { id: dedupeCriterionId(slugCriterionId(text), seen), name: text, description: text }
+    }
+    if (typeof value !== 'object' || value === null) throw new Error('llm-verifier: criteria[' + index + '] must be a string or an object')
+    const row = value as Record<string, unknown>
+    const field = (key: string) => (typeof row[key] === 'string' ? (row[key] as string).trim() : '')
+    const description = field('description')
+    if (!description) throw new Error('llm-verifier: criteria[' + index + '].description must be a non-empty string')
+    const name = field('name') || field('id') || slugCriterionId(description)
+    const id = field('id') || slugCriterionId(name)
+    return { id: dedupeCriterionId(id, seen), name, description }
+  })
 }
 export { DEFAULT_GROUND_TRUTH_NOTE }