dsh-llm-verifier 0.4.0 → 0.6.0

package/src/engine.ts

@@ -1,15 +1,77 @@
-import { addUsage, callVerifier, emptyUsage, predictScoringChannel, type ScoringMode, type UsageStats, type VerifierClientConfig, type VerifierImage } from './caller.ts'
+import { addUsage, attachUsage, callVerifier, emptyUsage, partialUsage, predictScoringChannel, requestAttempts, type ScoringMode, type UsageStats, type VerifierClientConfig, type VerifierImage } from './caller.ts'
 import { ScoreCache, SingleFlight, stableHash, type CachedPairScore } from './cache.ts'
 import type { DecisionTrace } from './decisions.ts'
 import {
-  DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, accumulatePairs, buildPairwisePrompt, buildProgressPrompt,
-  dedupeCriterionId, extractProgressScore, extractScore, pivotRoundPairs, rankScores, ringCycle, slugCriterionId,
-  topPivots, type Criterion,
+  DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, PROPOSAL_CRITERIA, accumulatePairs, buildPairwisePrompt, buildProgressPrompt,
+  dedupeCriterionId, diagnosticKey, extractProgressScore, extractScore, parseDiagnostics, pivotRoundPairs, rankScores,
+  ringCycle, slugCriterionId, swapDiagnosticEvidence, topPivots, type Criterion, type Diagnostic, type ReviewStage,
 } from './core.ts'
 
-export interface CompareOptions { problem: string; candidateA: string; candidateB: string; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; images?: readonly VerifierImage[]; trace?: DecisionTrace; /** Prefix for this comparison's decision-snapshot labels; one invocation that judges twice on the same criteria needs them distinguishable. */ traceLabelPrefix?: string }
+/** Evidence tokens one pairwise prompt shows the judge, exactly as the prompt lists them. */
+const PAIRWISE_EVIDENCE = ['TASK', 'A', 'B'] as const
+
+/**
+ * Cap on the findings ONE invocation reports, after de-duplication across criteria, repeats and judges.
+ *
+ * The per-call cap is {@link MAX_DIAGNOSTICS}; the default final acceptance is 3 criteria x 2 repeats,
+ * so the aggregate is deliberately a small multiple: the feedback budget (4000 characters) is the real
+ * limit, and a list that covers every criterion is not a location, it is a second rubric.
+ */
+const MAX_AGGREGATED_DIAGNOSTICS = 6
+
+/** Append findings in order, de-duplicated, never exceeding the aggregate cap. */
+function mergeDiagnostics(target: Diagnostic[], seen: Set<string>, incoming: readonly Diagnostic[] | undefined): void {
+  for (const diagnostic of incoming ?? []) {
+    if (target.length >= MAX_AGGREGATED_DIAGNOSTICS) return
+    const key = diagnosticKey(diagnostic)
+    if (seen.has(key)) continue
+    seen.add(key)
+    target.push(diagnostic)
+  }
+}
+
+export interface CompareOptions {
+  problem: string
+  candidateA: string
+  candidateB: string
+  criteria?: readonly Criterion[]
+  groundTruthNote?: string
+  repeats?: number
+  images?: readonly VerifierImage[]
+  trace?: DecisionTrace
+  /** Prefix for this comparison's decision-snapshot labels; one invocation that judges twice on the same criteria needs them distinguishable. */
+  traceLabelPrefix?: string
+  /**
+   * Which review stage both sides belong to.
+   *
+   * `proposal` switches the default rubric to {@link PROPOSAL_CRITERIA} and the prompt to the
+   * unexecuted-draft framing; omitted (or `artifact`) keeps the historical semantics, so every
+   * existing caller, cache key and verdict is unchanged.
+   */
+  reviewStage?: ReviewStage
+  /** Task domain for the prompt's role sentence; normally the criteria preset id. */
+  domain?: string
+  /**
+   * Optional reference context rendered as a separate data-only block between the task and the
+   * two candidates (P06's bounded process evidence pack). Omitted renders nothing.
+   */
+  context?: string
+}
 export interface CriterionResult { id: string; name: string; scoreA: number; scoreB: number }
-export interface RunStats extends UsageStats { cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
+export interface RunStats extends UsageStats {
+  cacheHits: number
+  cacheMisses: number
+  estimatedCostUsd: number
+  topLogprobScores: number
+  explicitTagScores: number
+  /**
+   * At least one request failed before returning usage. The tokens it may have spent are
+   * UNKNOWN, not zero: a failed call must not make the invocation look free.
+   */
+  usageIncomplete?: boolean
+  /** Calls that were attempted on the direct logprob transport and downgraded to explicit tags. */
+  channelFallbacks?: number
+}
 
 export interface JudgeScore {
   provider: string
@@ -36,9 +98,33 @@ export interface CompareResult {
   agreement: number
   /** Set when both sides were byte-identical: no model call was made and both sides score 0.5. */
   identical?: true
+  /**
+   * Verified findings the judge located, de-duplicated and capped.
+   *
+   * Empty is the normal case and means "nothing locatable was reported" — never "nothing is wrong".
+   * A score served from the cache carries none: replaying a finding from an older run would present a
+   * past observation as current evidence.
+   */
+  diagnostics: Diagnostic[]
 }
 
-export interface SelectOptions { problem: string; candidates: readonly string[]; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; pivots?: number; seed?: number; images?: readonly VerifierImage[]; trace?: DecisionTrace }
+export interface SelectOptions {
+  problem: string
+  candidates: readonly string[]
+  criteria?: readonly Criterion[]
+  groundTruthNote?: string
+  repeats?: number
+  pivots?: number
+  seed?: number
+  images?: readonly VerifierImage[]
+  trace?: DecisionTrace
+  /** Review stage applied to every pair of the tournament; see {@link CompareOptions.reviewStage}. */
+  reviewStage?: ReviewStage
+  /** Task domain for the prompt's role sentence; normally the criteria preset id. */
+  domain?: string
+  /** Reference context shown to every pair; see {@link CompareOptions.context}. */
+  context?: string
+}
 
 export interface SelectResult {
   index: number
@@ -52,6 +138,8 @@ export interface SelectResult {
   judges: JudgeScore[]
   /** Set when every candidate was byte-identical: no pair was judged and every score is 0.5. */
   identical?: true
+  /** Findings the judges located, aggregated across every judged pair (see {@link CompareResult.diagnostics}). */
+  diagnostics: Diagnostic[]
 }
 
 function average(values: readonly number[]): number { return values.reduce((sum, value) => sum + value, 0) / (values.length || 1) }
@@ -64,8 +152,105 @@ function median(values: readonly number[]): number {
 }
 
 function blankStats(): RunStats { return { ...emptyUsage(), cacheHits: 0, cacheMisses: 0, estimatedCostUsd: 0, topLogprobScores: 0, explicitTagScores: 0 } }
+
+/**
+ * Where an invocation records the usage it accumulated before it finally failed.
+ *
+ * The engine aggregates per-judge usage while jobs are still running, so a later job that
+ * throws used to discard everything the earlier jobs had already spent: a run with two
+ * successful calls then one failure was persisted as one attempt and zero tokens. Attaching
+ * the live stats object to the error keeps every known request and token.
+ */
+/**
+ * Usage accumulated before an invocation failed; undefined when the error carries none.
+ *
+ * The carrier may be a bare UsageStats (a caller-level response) or a full RunStats (an engine
+ * accumulator). It is normalized here so no caller can merge a partial shape and turn the
+ * RunStats-only counters into NaN (which the host serializes as null and rejects).
+ */
+export function partialStats(error: unknown): RunStats | undefined {
+  const value = partialUsage<Partial<RunStats> & UsageStats>(error)
+  if (value === undefined) return undefined
+  return {
+    ...blankStats(),
+    ...value,
+    cacheHits: value.cacheHits ?? 0,
+    cacheMisses: value.cacheMisses ?? 0,
+    estimatedCostUsd: value.estimatedCostUsd ?? 0,
+    topLogprobScores: value.topLogprobScores ?? 0,
+    explicitTagScores: value.explicitTagScores ?? 0,
+  }
+}
+
+/** A failed judge result knows its usage through one of two carriers; fold whichever it has. */
+function accountFailure(stats: RunStats, error: unknown): void {
+  const partial = partialStats(error)
+  if (partial !== undefined) {
+    mergeRunStats(stats, partial)
+    return
+  }
+  // The request never returned usage. The attempt count is the one known fact.
+  const attempts = requestAttempts(error)
+  stats.attempts += attempts
+  stats.retries += Math.max(0, attempts - 1)
+}
+
+/**
+ * Fold one nested run's counters into an accumulator (usage, cache, channel, incompleteness).
+ *
+ * Shared with index.ts so a multi-phase tool (best-of-N) accumulates generation, tournament and
+ * baseline usage the same way. Never folds a value into itself, and zero-fills the RunStats-only
+ * counters so a partial source cannot poison the totals with NaN.
+ */
+export function mergeRunStats(target: RunStats, source: (UsageStats & Partial<RunStats>) | undefined): void {
+  if (source === undefined || source === target) return
+  addUsage(target, source)
+  target.cacheHits += source.cacheHits ?? 0
+  target.cacheMisses += source.cacheMisses ?? 0
+  target.topLogprobScores += source.topLogprobScores ?? 0
+  target.explicitTagScores += source.explicitTagScores ?? 0
+  if (source.usageIncomplete) target.usageIncomplete = true
+  if (source.channelFallbacks) target.channelFallbacks = (target.channelFallbacks ?? 0) + source.channelFallbacks
+}
+
+/** Keep a billed-but-unusable response's usage on its error. */
+function attachBilled(error: unknown, usage: UsageStats): void {
+  const billed = blankStats()
+  addUsage(billed, usage)
+  if (usage.usageIncomplete) billed.usageIncomplete = true
+  attachUsage(error, billed)
+}
 /** Candidate indices best-first by score, ties broken by index. */
 function rankByScore(scores: readonly number[]): number[] { return Array.from({ length: scores.length }, (_, index) => index).sort((a, b) => (scores[b] ?? 0) - (scores[a] ?? 0) || a - b) }
+
+/**
+ * Rewrite one pair's `A`/`B` finding into the tournament's candidate identity.
+ *
+ * The label is 1-based on purpose: the automatic selection feedback locates candidates as `[N]`,
+ * and the tool result is read by the same agent that reads that feedback.
+ */
+function locatePairDiagnostic(diagnostic: Diagnostic, a: number, b: number): Diagnostic {
+  if (diagnostic.evidence === 'A') return { ...diagnostic, evidence: 'candidate ' + (a + 1) }
+  if (diagnostic.evidence === 'B') return { ...diagnostic, evidence: 'candidate ' + (b + 1) }
+  return diagnostic
+}
+
+/**
+ * Renumber one deduplicated selection's finding onto the caller's original candidate list.
+ *
+ * {@link locatePairDiagnostic} numbers the DISTINCT candidates the tournament judged; the caller
+ * passed a list that may repeat entries, so `candidate 2` of the compressed list can be
+ * `candidate 3` of the caller's. Anything that is not a candidate reference is left alone.
+ * @param diagnostic - one aggregated finding.
+ * @param originals - distinct index -> the caller's first index of that candidate.
+ * @returns The finding, renumbered when it names a candidate.
+ */
+function remapCandidateDiagnostic(diagnostic: Diagnostic, originals: readonly number[]): Diagnostic {
+  const match = /^candidate (\d+)$/u.exec(diagnostic.evidence)
+  if (match === null) return diagnostic
+  const original = originals[Number(match[1]) - 1]
+  return original === undefined ? diagnostic : { ...diagnostic, evidence: 'candidate ' + (original + 1) }
+}
 function unorderedPair(a: number, b: number): string { return a < b ? a + ',' + b : b + ',' + a }
 
 /**
@@ -154,9 +339,26 @@ export class VerifierEngine {
     return stats
   }
 
-  private async scoreOne(client: VerifierClientConfig, options: CompareOptions, candidateA: string, candidateB: string, criterion: Criterion, repeat: number, signal?: AbortSignal): Promise<{ scores: readonly [number, number]; usage: UsageStats; scoringMode: 'top-logprobs' | 'explicit-tag'; hit: boolean }> {
+  /**
+   * Prompt framing for one comparison.
+   *
+   * Rebuilt from the stage/domain the caller declared. Both fields are part of the RENDERED
+   * prompt, so the score cache keys on them automatically; the cache's `version` does not need
+   * to move when only this framing changes.
+   * @param options - the comparison's options.
+   * @returns Stage and domain to forward to {@link buildPairwisePrompt}.
+   */
+  private framing(options: CompareOptions): { stage?: ReviewStage; domain?: string; context?: string } {
+    return {
+      ...(options.reviewStage === undefined ? {} : { stage: options.reviewStage }),
+      ...(options.domain === undefined ? {} : { domain: options.domain }),
+      ...(options.context === undefined ? {} : { context: options.context }),
+    }
+  }
+
+  private async scoreOne(client: VerifierClientConfig, options: CompareOptions, candidateA: string, candidateB: string, criterion: Criterion, repeat: number, signal?: AbortSignal): Promise<{ scores: readonly [number, number]; usage: UsageStats; scoringMode: 'top-logprobs' | 'explicit-tag'; hit: boolean; channelFallback: boolean; diagnostics: Diagnostic[] }> {
     const ground = options.groundTruthNote ?? DEFAULT_GROUND_TRUTH_NOTE
-    const prompt = buildPairwisePrompt(options.problem, candidateA, candidateB, criterion, ground)
+    const prompt = buildPairwisePrompt(options.problem, candidateA, candidateB, criterion, ground, this.framing(options))
     const imageKey = options.images?.map(image => stableHash([image.mediaType, Buffer.from(image.data).toString('base64')]))
     // Cache identity pins the scoring channel, but the channel is only predictable
     // before the call when the capability cache already knows the answer. Lookups use
@@ -166,17 +368,30 @@ export class VerifierEngine {
     // expectation. The channel stays OUT of the in-flight dedup key below.
     const identity = { version: 6, provider: client.provider, model: client.model, effort: client.reasoningEffort, maxTokens: client.maxTokens, temperature: client.temperature, repeat, promptHash: stableHash(prompt), imageKey }
     const keyForMode = (scoringMode: ScoringMode) => stableHash({ ...identity, scoringMode })
+    // Captured out of band: a fallback describes the REQUEST, not the score, so it must not
+    // enter the cached value (a cache hit later must not claim a downgrade that never happened).
+    let fellBack = false
     const create = async () => {
       const completion = await callVerifier(client, prompt, signal, options.images)
-      const scoreA = extractScore(completion, '<score_A>')
-      const scoreB = extractScore(completion, '<score_B>')
-      // Traced here, not in scoreOne(): a cache hit or a merged in-flight call makes no
-      // model call, and a snapshot that showed one anyway would be a fabrication.
-      options.trace?.({ label: (options.traceLabelPrefix ?? '') + criterion.name + ' repeat ' + (repeat + 1), channel: completion.scoringMode, prompt, output: completion.text, score: scoreA })
-      return { scoreA, scoreB, usage: completion.usage, scoringMode: completion.scoringMode, createdAt: Date.now() }
+      try {
+        const scoreA = extractScore(completion, '<score_A>')
+        const scoreB = extractScore(completion, '<score_B>')
+        fellBack = completion.channelFallback === true
+        // Traced here, not in scoreOne(): a cache hit or a merged in-flight call makes no
+        // model call, and a snapshot that showed one anyway would be a fabrication.
+        options.trace?.({ label: (options.traceLabelPrefix ?? '') + criterion.name + ' repeat ' + (repeat + 1), channel: completion.scoringMode, prompt, output: completion.text, score: scoreA })
+        // Parsed against exactly what this prompt offered; anything unverifiable is dropped.
+        const diagnostics = parseDiagnostics(completion.text, { criteria: [criterion.name, criterion.id], evidence: PAIRWISE_EVIDENCE })
+        return { scoreA, scoreB, usage: completion.usage, scoringMode: completion.scoringMode, diagnostics, createdAt: Date.now() }
+      } catch (error) {
+        // The response came back and was billed, but carried no usable score. Keep its usage
+        // on the error so the failed row reports known requests and tokens, not zero.
+        attachBilled(error, completion.usage)
+        throw error
+      }
     }
     const cache = this.cache
-    if (cache === undefined) { const value = await create(); return { scores: [value.scoreA, value.scoreB], usage: value.usage, scoringMode: value.scoringMode, hit: false } }
+    if (cache === undefined) { const value = await create(); return { scores: [value.scoreA, value.scoreB], usage: value.usage, scoringMode: value.scoringMode, hit: false, channelFallback: fellBack, diagnostics: value.diagnostics } }
     // Registration happens in the synchronous segment before any await, so a request
     // that arrives while the first one is still resolving its channel prediction
     // still merges instead of duplicating the model call. The flight table is shared
@@ -185,13 +400,37 @@ export class VerifierEngine {
     const outcome = await this.flights.run(dedupeKey, async () => cache.getOrCreate(keyForMode(await predictScoringChannel(client)), create, value => keyForMode(value.scoringMode)))
     const landed = outcome.value
     const reused = outcome.joined || landed.hit
-    return { scores: [landed.value.scoreA, landed.value.scoreB], usage: reused ? emptyUsage() : landed.value.usage, scoringMode: landed.value.scoringMode, hit: reused }
+    // Findings ride along with the cache entry: the key is the rendered prompt, so they describe the
+    // very evidence that produced this score. An entry written before P04 simply has none.
+    return { scores: [landed.value.scoreA, landed.value.scoreB], usage: reused ? emptyUsage() : landed.value.usage, scoringMode: landed.value.scoringMode, hit: reused, channelFallback: reused ? false : fellBack, diagnostics: landed.value.diagnostics ?? [] }
   }
 
   private async mapLimited<T, R>(items: readonly T[], worker: (item: T) => Promise<R>): Promise<R[]> {
     const results = new Array<R>(items.length); let cursor = 0
-    const runners = Array.from({ length: Math.min(this.maxConcurrency, items.length) }, async () => { while (cursor < items.length) { const index = cursor++; results[index] = await worker(items[index]!) } })
-    await Promise.all(runners); return results
+    let failed = false
+    let failure: unknown
+    const runners = Array.from({ length: Math.min(this.maxConcurrency, items.length) }, async () => {
+      while (!failed && cursor < items.length) {
+        const index = cursor++
+        try { results[index] = await worker(items[index]!) }
+        catch (error) {
+          // Stop buying NEW work, but let every in-flight call settle first: it was already
+          // paid for, and its usage is merged into the shared accumulator the error carries.
+          // Rejecting immediately (the old Promise.all) discarded those calls' usage.
+          if (!failed) { failed = true; failure = error }
+        }
+      }
+    })
+    await Promise.all(runners)
+    if (failed) {
+      const partial = partialStats(failure)
+      // All workers have settled, so the accumulator is final. partialStats returns a normalized
+      // COPY, so pricing it in place would be discarded: price it and write it back onto the
+      // error the failure row actually reads.
+      if (partial !== undefined) attachUsage(failure, this.finishStats(partial))
+      throw failure
+    }
+    return results
   }
 
   /**
@@ -216,11 +455,16 @@ export class VerifierEngine {
       judges,
       agreement: 1,
       identical: true,
+      // No judge ran, so nothing was located: an informational tie has nothing to report.
+      diagnostics: [],
     }
   }
 
   async compare(options: CompareOptions, signal?: AbortSignal): Promise<CompareResult> {
-    const criteria = options.criteria?.length ? options.criteria : DEFAULT_CRITERIA
+    // An unexecuted proposal has no observed output to match, so scoring it with the artifact
+    // rubric ("Output Match") fails it by construction. The stage therefore selects the default
+    // rubric; an explicit `criteria` always wins and is used verbatim.
+    const criteria = options.criteria?.length ? options.criteria : options.reviewStage === 'proposal' ? PROPOSAL_CRITERIA : DEFAULT_CRITERIA
     if (options.candidateA === options.candidateB) return this.informationalTie(criteria)
     const repeats = options.repeats ?? 2
     const jobs = criteria.flatMap(criterion => Array.from({ length: repeats }, (_, repeat) => ({ criterion, repeat })))
@@ -236,6 +480,18 @@ export class VerifierEngine {
       if (warmedOrientations.has(orientation)) rest.push(job)
       else { warmedOrientations.add(orientation); warm.push(job) }
     }
+    // Shared with the workers so a job that fails still reports what the earlier jobs spent:
+    // aggregating only after every job resolved discarded all of it when one threw. A run with
+    // two successful judge calls and one failure used to persist one attempt and zero tokens.
+    const stats = blankStats()
+    const judgeCalls = new Array<number>(this.clients.length).fill(0)
+    const judgeOk = new Array<boolean>(this.clients.length).fill(true)
+    const judgeErrors = new Array<string | undefined>(this.clients.length).fill(undefined)
+    const judgeJobScores: Array<Array<{ criterionId: string; scoreA: number; scoreB: number }>> = Array.from(
+      { length: this.clients.length },
+      () => []
+    )
+
     const run = async (batch: typeof jobs) => this.mapLimited(batch, async ({ criterion, repeat }) => {
       const swapped = repeat % 2 === 1
       const candA = swapped ? options.candidateB : options.candidateA
@@ -252,6 +508,10 @@ export class VerifierEngine {
               usage: res.usage,
               scoringMode: res.scoringMode,
               hit: res.hit,
+              channelFallback: res.channelFallback,
+              // This round rendered the swapped slots; the scores are mapped back above, so the
+              // findings must be too, or a defect in the caller's A is reported against B.
+              diagnostics: swapped ? res.diagnostics.map(swapDiagnosticEvidence) : res.diagnostics,
             }
           } catch (error) {
             return {
@@ -262,44 +522,47 @@ export class VerifierEngine {
           }
         })
       )
-      const successful = judgeResults.filter((r): r is Extract<typeof r, { ok: true }> => r.ok)
-      if (successful.length === 0) {
-        const firstFail = judgeResults.find(r => !r.ok)! as Extract<typeof judgeResults[number], { ok: false }>
-        throw firstFail.error
-      }
-      const leafScoreA = median(successful.map(r => r.scoreA))
-      const leafScoreB = median(successful.map(r => r.scoreB))
-      return { criterion, repeat, leafScoreA, leafScoreB, judgeResults }
-    })
-
-    const jobOutcomes = [...await run(warm), ...await run(rest)]
-    const stats = blankStats()
-    const judgeCalls = new Array<number>(this.clients.length).fill(0)
-    const judgeOk = new Array<boolean>(this.clients.length).fill(true)
-    const judgeErrors = new Array<string | undefined>(this.clients.length).fill(undefined)
-    const judgeJobScores: Array<Array<{ criterionId: string; scoreA: number; scoreB: number }>> = Array.from(
-      { length: this.clients.length },
-      () => []
-    )
-
-    for (const outcome of jobOutcomes) {
-      for (const r of outcome.judgeResults) {
+      for (const r of judgeResults) {
         if (r.ok) {
           judgeCalls[r.k] += r.usage.calls
-          judgeJobScores[r.k]!.push({ criterionId: outcome.criterion.id, scoreA: r.scoreA, scoreB: r.scoreB })
+          judgeJobScores[r.k]!.push({ criterionId: criterion.id, scoreA: r.scoreA, scoreB: r.scoreB })
           addUsage(stats, r.usage)
+          if (r.usage.usageIncomplete) stats.usageIncomplete = true
           if (r.hit) stats.cacheHits++
           else stats.cacheMisses++
           if (r.scoringMode === 'top-logprobs') stats.topLogprobScores++
           else stats.explicitTagScores++
+          if (r.channelFallback) stats.channelFallbacks = (stats.channelFallbacks ?? 0) + 1
         } else {
           judgeOk[r.k] = false
+          // The failed request really happened; fold in whatever accounting it carries (billed
+          // but unusable usage, or just its attempt count) and mark the row incomplete rather
+          // than reporting a confident zero-cost call.
+          accountFailure(stats, r.error)
+          stats.usageIncomplete = true
           if (judgeErrors[r.k] === undefined) {
             judgeErrors[r.k] = r.error instanceof Error ? r.error.message : String(r.error)
           }
         }
       }
-    }
+      const successful = judgeResults.filter((r): r is Extract<typeof r, { ok: true }> => r.ok)
+      if (successful.length === 0) {
+        const firstFail = judgeResults.find(r => !r.ok)! as Extract<typeof judgeResults[number], { ok: false }>
+        // Carry the usage the earlier jobs already spent on this error.
+        attachUsage(firstFail.error, stats)
+        throw firstFail.error
+      }
+      const leafScoreA = median(successful.map(r => r.scoreA))
+      const leafScoreB = median(successful.map(r => r.scoreB))
+      return { criterion, repeat, leafScoreA, leafScoreB, judgeResults, diagnostics: successful.flatMap(r => r.diagnostics) }
+    })
+
+    const jobOutcomes = [...await run(warm), ...await run(rest)]
+    // Findings are located information, not a score: they are aggregated across criteria, repeats and
+    // judges but de-duplicated and capped, and they never enter the score arithmetic.
+    const diagnostics: Diagnostic[] = []
+    const diagnosticSeen = new Set<string>()
+    for (const row of jobOutcomes) mergeDiagnostics(diagnostics, diagnosticSeen, row.diagnostics)
 
     const byCriterion = criteria.map(criterion => {
       const rows = jobOutcomes.filter(row => row.criterion.id === criterion.id)
@@ -364,6 +627,7 @@ export class VerifierEngine {
       stats: this.finishStats(stats),
       judges,
       agreement,
+      diagnostics,
     }
   }
 
@@ -374,40 +638,29 @@ export class VerifierEngine {
     judgeErrors: Array<string | undefined>
     judgeCalls: number[]
     stats: RunStats
+    diagnostics: Diagnostic[]
   }> {
     const unique = [...new Map(pairs.map(pair => [pair[0] + ',' + pair[1], pair])).values()]
-    const values = await this.mapLimited(unique, async ([a, b]) => ({
-      a,
-      b,
-      result: await this.compare({
-        problem: options.problem,
-        candidateA: options.candidates[a]!,
-        candidateB: options.candidates[b]!,
-        criteria: options.criteria,
-        groundTruthNote: options.groundTruthNote,
-        repeats: options.repeats,
-        images: options.images,
-        trace: options.trace,
-      }, signal),
-    }))
+    // Aggregated inside each worker so a pair that fails still reports what the other pairs
+    // already spent; the outer loop used to run only after every pair had resolved.
     const rewards = new Map<string, readonly [number, number]>()
     const judgeRewards: Array<Map<string, readonly [number, number]>> = Array.from({ length: this.clients.length }, () => new Map())
     const judgeOk = new Array<boolean>(this.clients.length).fill(true)
     const judgeErrors = new Array<string | undefined>(this.clients.length).fill(undefined)
     const judgeCalls = new Array<number>(this.clients.length).fill(0)
     const stats = blankStats()
-
-    for (const value of values) {
-      const pairKey = value.a + ',' + value.b
-      rewards.set(pairKey, [value.result.scoreA, value.result.scoreB])
-      addUsage(stats, value.result.stats)
-      stats.cacheHits += value.result.stats.cacheHits
-      stats.cacheMisses += value.result.stats.cacheMisses
-      stats.topLogprobScores += value.result.stats.topLogprobScores
-      stats.explicitTagScores += value.result.stats.explicitTagScores
-
+    const diagnostics: Diagnostic[] = []
+    const diagnosticSeen = new Set<string>()
+    const recordPair = (a: number, b: number, result: CompareResult): void => {
+      const pairKey = a + ',' + b
+      rewards.set(pairKey, [result.scoreA, result.scoreB])
+      mergeRunStats(stats, result.stats)
+      // A pair's findings reference ITS two slots. A tournament ranks N candidates, so the slots
+      // are rewritten into the original candidate identity before they are aggregated: reporting
+      // "evidence B" for a pair the caller never saw as A/B points the agent at the wrong object.
+      mergeDiagnostics(diagnostics, diagnosticSeen, result.diagnostics.map(diagnostic => locatePairDiagnostic(diagnostic, a, b)))
       for (let k = 0; k < this.clients.length; k++) {
-        const js = value.result.judges[k]!
+        const js = result.judges[k]!
         judgeCalls[k] += js.calls
         if (js.ok && js.scoreA !== undefined && js.scoreB !== undefined) {
           judgeRewards[k]!.set(pairKey, [js.scoreA, js.scoreB])
@@ -419,10 +672,34 @@ export class VerifierEngine {
         }
       }
     }
-    return { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats: this.finishStats(stats) }
+    await this.mapLimited(unique, async ([a, b]) => {
+      try {
+        const result = await this.compare({
+          problem: options.problem,
+          candidateA: options.candidates[a]!,
+          candidateB: options.candidates[b]!,
+          criteria: options.criteria,
+          groundTruthNote: options.groundTruthNote,
+          repeats: options.repeats,
+          images: options.images,
+          trace: options.trace,
+          ...(options.reviewStage === undefined ? {} : { reviewStage: options.reviewStage }),
+          ...(options.domain === undefined ? {} : { domain: options.domain }),
+          ...(options.context === undefined ? {} : { context: options.context }),
+        }, signal)
+        recordPair(a, b, result)
+      } catch (error) {
+        // Merge this pair's own partial usage into the shared accumulator, then hand the union
+        // to the caller's failure row.
+        mergeRunStats(stats, partialStats(error))
+        attachUsage(error, stats)
+        throw error
+      }
+    })
+    return { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats: this.finishStats(stats), diagnostics }
   }
 
-  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[], trace?: DecisionTrace): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats; judges: JudgeScore[] }> {
+  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[], trace?: DecisionTrace): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats; judges: JudgeScore[]; diagnostics: Diagnostic[] }> {
     if (!steps.length || !checkpoints.length) throw new Error('llm-verifier: steps and checkpoints must not be empty')
     for (const checkpoint of checkpoints) if (!Number.isSafeInteger(checkpoint) || checkpoint < 1 || checkpoint > steps.length) throw new Error('llm-verifier: each checkpoint must be an integer between 1 and steps.length')
     const prompt = buildProgressPrompt(problem, steps, checkpoints)
@@ -438,16 +715,27 @@ export class VerifierEngine {
         this.clients.map(async (client, k) => {
           try {
             const completion = await callVerifier(client, prompt, signal, images)
-            const scores = checkpoints.map((_, index) => extractProgressScore(completion, '<c' + (index + 1) + '>'))
-            // One snapshot per repeat for the primary judge only: extra judges would
-            // multiply the record past its budget, and the primary is what the verdict
-            // reports.
-            if (k === 0) trace?.({ label: 'progress repeat ' + (repeatIndex + 1) + '/' + repeats + (this.clients.length > 1 ? ' judge 1/' + this.clients.length : ''), channel: completion.scoringMode, prompt, output: completion.text, score: scores[scores.length - 1] })
-            return {
-              k,
-              ok: true as const,
-              completion,
-              scores,
+            try {
+              const scores = checkpoints.map((_, index) => extractProgressScore(completion, '<c' + (index + 1) + '>'))
+              // One snapshot per repeat for the primary judge only: extra judges would
+              // multiply the record past its budget, and the primary is what the verdict
+              // reports.
+              if (k === 0) trace?.({ label: 'progress repeat ' + (repeatIndex + 1) + '/' + repeats + (this.clients.length > 1 ? ' judge 1/' + this.clients.length : ''), channel: completion.scoringMode, prompt, output: completion.text, score: scores[scores.length - 1] })
+              // The prompt offered the checkpoint labels c1..cN and the task block as citable
+              // evidence; anything else the judge cites is dropped by the parser.
+              const labels = checkpoints.map((_, index) => 'c' + (index + 1))
+              const diagnostics = parseDiagnostics(completion.text, { checkpoints: labels, evidence: ['TASK', ...labels] })
+              return {
+                k,
+                ok: true as const,
+                completion,
+                scores,
+                diagnostics,
+              }
+            } catch (error) {
+              // The response came back and was billed; a failed progress parse must keep its usage.
+              attachBilled(error, completion.usage)
+              throw error
             }
           } catch (error) {
             return {
@@ -459,35 +747,53 @@ export class VerifierEngine {
         })
       )
 
-      const successful = judgeResults.filter((r): r is Extract<typeof r, { ok: true }> => r.ok)
-      if (successful.length === 0) {
-        const firstFail = judgeResults.find(r => !r.ok)! as Extract<typeof judgeResults[number], { ok: false }>
-        throw firstFail.error
-      }
-
+      // Aggregate BEFORE the all-failed check so an earlier repeat's usage survives a later
+      // repeat throwing; the error then carries it to the failure row.
       for (const r of judgeResults) {
         if (r.ok) {
           judgeCalls[r.k] += r.completion.usage.calls
           judgePerRepeatScores[r.k]!.push(r.scores)
           addUsage(stats, r.completion.usage)
+          if (r.completion.usage.usageIncomplete) stats.usageIncomplete = true
           if (r.completion.scoringMode === 'top-logprobs') stats.topLogprobScores++
           else stats.explicitTagScores++
+          if (r.completion.channelFallback) stats.channelFallbacks = (stats.channelFallbacks ?? 0) + 1
         } else {
           judgeOk[r.k] = false
+          accountFailure(stats, r.error)
+          stats.usageIncomplete = true
           if (judgeErrors[r.k] === undefined) {
             judgeErrors[r.k] = r.error instanceof Error ? r.error.message : String(r.error)
           }
         }
       }
 
-      return checkpoints.map((_, cIndex) => median(successful.map(r => r.scores[cIndex]!)))
+      const successful = judgeResults.filter((r): r is Extract<typeof r, { ok: true }> => r.ok)
+      if (successful.length === 0) {
+        const firstFail = judgeResults.find(r => !r.ok)! as Extract<typeof judgeResults[number], { ok: false }>
+        attachUsage(firstFail.error, stats)
+        throw firstFail.error
+      }
+
+      const diagnostics: Diagnostic[] = []
+      const diagnosticSeen = new Set<string>()
+      for (const row of successful) mergeDiagnostics(diagnostics, diagnosticSeen, row.diagnostics)
+      return { scores: checkpoints.map((_, cIndex) => median(successful.map(r => r.scores[cIndex]!))), diagnostics }
     }
     // Every repeat sends the SAME prompt, so the first repeat warms the provider prefix cache for
     // the others; fanning all repeats out at once left every call cold. The repeat count is
     // unchanged — only the order, and therefore the cache hits, change.
     const runs: number[][] = []
-    if (repeatIndices.length > 0) runs.push(...await this.mapLimited(repeatIndices.slice(0, 1), runRepeat))
-    runs.push(...await this.mapLimited(repeatIndices.slice(1), runRepeat))
+    const diagnostics: Diagnostic[] = []
+    const diagnosticSeen = new Set<string>()
+    const collect = (outcomes: Array<{ scores: number[]; diagnostics: Diagnostic[] }>): void => {
+      for (const outcome of outcomes) {
+        runs.push(outcome.scores)
+        mergeDiagnostics(diagnostics, diagnosticSeen, outcome.diagnostics)
+      }
+    }
+    if (repeatIndices.length > 0) collect(await this.mapLimited(repeatIndices.slice(0, 1), runRepeat))
+    collect(await this.mapLimited(repeatIndices.slice(1), runRepeat))
 
     const scores = checkpoints.map((_, index) => average(runs.map(run => run[index]!)))
     const judges: JudgeScore[] = this.clients.map((client, k) => {
@@ -514,7 +820,7 @@ export class VerifierEngine {
       }
     })
 
-    return { scores, perRepeat: runs, calls: stats.calls, stats: this.finishStats(stats), judges }
+    return { scores, perRepeat: runs, calls: stats.calls, stats: this.finishStats(stats), judges, diagnostics }
   }
 
   /**
@@ -541,6 +847,7 @@ export class VerifierEngine {
       stats: blankStats(),
       judges: this.clients.map(client => ({ provider: client.provider, model: client.model, label: judgeLabel(client), ok: true, calls: 0, scores: [...scores], ranking: [...ranking] })),
       identical: true,
+      diagnostics: [],
     }
   }
 
@@ -557,11 +864,13 @@ export class VerifierEngine {
    */
   private async selectUnique(options: SelectOptions, signal?: AbortSignal): Promise<SelectResult> {
     const unique: string[] = []
+    /** unique index -> the caller's FIRST index of that candidate (the inverse of `representative`). */
+    const originals: number[] = []
     const representative: number[] = []
     const seen = new Map<string, number>()
     for (const candidate of options.candidates) {
       let index = seen.get(candidate)
-      if (index === undefined) { index = unique.length; unique.push(candidate); seen.set(candidate, index) }
+      if (index === undefined) { index = unique.length; unique.push(candidate); seen.set(candidate, index); originals.push(representative.length) }
       representative.push(index)
     }
     const result = await this.select({ ...options, candidates: unique }, signal)
@@ -572,6 +881,12 @@ export class VerifierEngine {
       const judgeScores = expand(judge.scores)
       return { ...judge, scores: judgeScores, ...(judge.ranking === undefined ? {} : { ranking: rankByScore(judgeScores) }) }
     })
+    // The tournament numbered the DISTINCT candidates. The caller never saw that list, so a finding
+    // has to be renumbered onto the original indices exactly like the scores and rankings above —
+    // otherwise "candidate 2" of a deduplicated list points at a different entry of the caller's.
+    const diagnostics: Diagnostic[] = []
+    const diagnosticSeen = new Set<string>()
+    for (const diagnostic of result.diagnostics) mergeDiagnostics(diagnostics, diagnosticSeen, [remapCandidateDiagnostic(diagnostic, originals)])
     return {
       index: options.candidates.indexOf(result.best),
       best: result.best,
@@ -582,6 +897,7 @@ export class VerifierEngine {
       calls: result.calls,
       stats: result.stats,
       judges,
+      diagnostics,
     }
   }
 
@@ -600,13 +916,13 @@ export class VerifierEngine {
         scores: [1],
         ranking: [0],
       }))
-      return { index: 0, best: options.candidates[0]!, scores: [1], ranking: [0], pivots: [0], comparisons: 0, calls: 0, stats: blankStats(), judges }
+      return { index: 0, best: options.candidates[0]!, scores: [1], ranking: [0], pivots: [0], comparisons: 0, calls: 0, stats: blankStats(), judges, diagnostics: [] }
     }
     if (options.candidates.length === 2) {
       // ringCycle(2) would judge the single unordered pair in both directions; play it
       // once, in the orientation orientPair() picks so the slot is not always candidate 0.
       const single = orientPair(0, 1)
-      const { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats } = await this.scorePairs(options, [single], signal)
+      const { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats, diagnostics } = await this.scorePairs(options, [single], signal)
       const wins = [0, 0]; const counts = [0, 0]
       accumulatePairs([single], rewards, wins, counts)
       const ranked = rankScores(wins, counts); const index = ranked[0]!.index
@@ -635,7 +951,7 @@ export class VerifierEngine {
           ranking: jRanked.map(v => v.index),
         }
       })
-      return { index, best: options.candidates[index]!, scores: wins.map((value, candidate) => value / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots: [], comparisons: 1, calls: stats.calls, stats, judges }
+      return { index, best: options.candidates[index]!, scores: wins.map((value, candidate) => value / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots: [], comparisons: 1, calls: stats.calls, stats, judges, diagnostics }
     }
     const ring = ringCycle(options.candidates.length, options.seed ?? 0)
     const ringScores = await this.scorePairs(options, ring, signal)
@@ -648,7 +964,16 @@ export class VerifierEngine {
     // single match so wins/counts are not double-weighted and no pair is judged twice.
     const ringPairs = new Set(ring.map(pair => unorderedPair(pair[0], pair[1])))
     const rounds = orientRoundPairs(pivotRoundPairs(options.candidates.length, pivots).filter(pair => !ringPairs.has(unorderedPair(pair[0], pair[1]))))
-    const roundScores = await this.scorePairs(options, rounds, signal)
+    // The ring phase is already paid for. Accumulate it BEFORE the pivot phase so a pivot
+    // failure keeps the successful pairs' usage on the thrown error instead of losing it.
+    const stats = blankStats()
+    mergeRunStats(stats, ringScores.stats)
+    const roundScores = await this.scorePairs(options, rounds, signal).catch((error: unknown) => {
+      mergeRunStats(stats, partialStats(error))
+      attachUsage(error, this.finishStats(stats))
+      throw error
+    })
+    mergeRunStats(stats, roundScores.stats)
     const allRewards = new Map([...ringScores.rewards, ...roundScores.rewards])
     const wins = new Array<number>(options.candidates.length).fill(0)
     const counts = new Array<number>(options.candidates.length).fill(0)
@@ -656,14 +981,6 @@ export class VerifierEngine {
     accumulatePairs(rounds, allRewards, wins, counts)
     const ranked = rankScores(wins, counts)
     const index = ranked[0]!.index
-    const stats = blankStats()
-    for (const source of [ringScores.stats, roundScores.stats]) {
-      addUsage(stats, source)
-      stats.cacheHits += source.cacheHits
-      stats.cacheMisses += source.cacheMisses
-      stats.topLogprobScores += source.topLogprobScores
-      stats.explicitTagScores += source.explicitTagScores
-    }
 
     const judges: JudgeScore[] = this.clients.map((client, k) => {
       const isOk = ringScores.judgeOk[k]! && roundScores.judgeOk[k]!
@@ -696,7 +1013,11 @@ export class VerifierEngine {
       }
     })
 
-    return { index, best: options.candidates[index]!, scores: Array.from({ length: options.candidates.length }, (_, candidate) => wins[candidate]! / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots, comparisons: ring.length + rounds.length, calls: stats.calls, stats: this.finishStats(stats), judges }
+    const diagnostics: Diagnostic[] = []
+    const diagnosticSeen = new Set<string>()
+    mergeDiagnostics(diagnostics, diagnosticSeen, ringScores.diagnostics)
+    mergeDiagnostics(diagnostics, diagnosticSeen, roundScores.diagnostics)
+    return { index, best: options.candidates[index]!, scores: Array.from({ length: options.candidates.length }, (_, candidate) => wins[candidate]! / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots, comparisons: ring.length + rounds.length, calls: stats.calls, stats: this.finishStats(stats), judges, diagnostics }
   }
 }