dsh-llm-verifier 0.1.13 → 0.1.16

package/src/engine.test.ts

@@ -1,136 +1,136 @@
-import { mkdtempSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, describe, expect, it, vi } from 'vitest'
-import type { VerifierClientConfig } from './caller.ts'
-import { ScoreCache, SingleFlight, type CachedPairScore } from './cache.ts'
-import { VerifierEngine } from './engine.ts'
-import { TopLogprobCapabilityCache } from './top-logprobs.ts'
-
-function chunks(text: string) { return [{ type: 'block-start', index: 0, blockType: 'text' }, { type: 'text-delta', index: 0, text }, { type: 'block-end', index: 0, block: { type: 'text', text } }, { type: 'usage', usage: { inputTokens: 7, cacheReadTokens: 3, outputTokens: 4, reasoningTokens: 2 } }, { type: 'finish', reason: { kind: 'stop' } }] as any[] }
-async function* streamOf(items: any[]) { for (const item of items) yield item }
-
-function clientConfig(overrides: Partial<VerifierClientConfig> = {}): VerifierClientConfig {
-  return { ctx: { get: () => undefined } as any, llm: { stream: async function* () { throw new Error('unexpected model call') } } as any, attachments: { saveImage: async () => ({}) } as any, topLogprobCapabilities: new TopLogprobCapabilityCache(), provider: 'openai', model: 'gpt-5', reasoningEffort: 'high', maxTokens: 100, timeoutMs: 1000, maxRetries: 0, retryBaseDelayMs: 1, ...overrides }
-}
-
-/** Streams an explicit-tag verdict derived from which named candidate sits in trajectory A/B. */
-function scriptedStream(judged: Array<string>, gate?: Array<() => void>): (options: any) => AsyncIterable<any[]> {
-  return function (options: any) {
-    const message = options.messages[0]
-    const prompt = typeof message.content === 'string' ? message.content : message.content.filter((block: any) => block.type === 'text').map((block: any) => block.text).join('')
-    const section = (marker: string) => { const at = prompt.indexOf(marker); if (at < 0) return ''; const end = prompt.indexOf('\n\n**', at); return prompt.slice(at + marker.length, end < 0 ? undefined : end) }
-    const traceA = section('**Trajectory A:**\n')
-    const traceB = section('**Trajectory B:**\n')
-    const letter = (trace: string) => trace.includes('STRONG') ? 'A' : 'T'
-    const criterion = /\*\*Evaluation Guideline — (.+?):\*\*/.exec(prompt)?.[1] ?? '?'
-    judged.push(criterion + '|' + [traceA.trim(), traceB.trim()].sort().join('|'))
-    const text = '<score_A> ' + letter(traceA) + ' </score_A>\n<score_B> ' + letter(traceB) + ' </score_B>'
-    if (gate === undefined) return streamOf(chunks(text))
-    return (async function* () { await new Promise<void>(resolve => { gate.push(resolve) }); yield* streamOf(chunks(text)) })()
-  }
-}
-
-afterEach(() => vi.unstubAllGlobals())
-
-describe('VerifierEngine tournament', () => {
-  it('judges each unordered candidate pair at most once and ranks the strong candidate first', async () => {
-    const judged: Array<string> = []
-    const llm = { stream: scriptedStream(judged) } as any
-    const engine = new VerifierEngine(clientConfig({ llm }), 4)
-    const candidates = ['STRONG-0', 'WEAK-1', 'WEAK-2', 'WEAK-3']
-    const result = await engine.select({ problem: 'Pick the better implementation.', candidates, repeats: 1 })
-    // Ring edges incident to a pivot used to be re-judged (reversed) in the pivot round.
-    expect(new Set(judged).size).toBe(judged.length)
-    expect(result.comparisons).toBe(new Set(judged.map(entry => entry.split('|')[1] + '|' + entry.split('|')[2])).size)
-    expect(result.stats.calls).toBe(judged.length)
-    expect(result.ranking[0]).toBe(0)
-    expect(result.best).toBe('STRONG-0')
-  })
-})
-
-describe('VerifierEngine cache identity', () => {
-  it('does not reuse an explicit-tag cache entry for top-logprobs scoring', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))
-    try {
-      const file = join(dir, 'scores.json')
-      const options = { problem: 'task', candidateA: 'AAA', candidateB: 'BBB', repeats: 1 }
-      const fallbackCaps = new TopLogprobCapabilityCache()
-      fallbackCaps.markUnsupported('openai', 'gpt-5')
-      const fallback = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream([]) } as any, topLogprobCapabilities: fallbackCaps }), 4, new ScoreCache(file, 100))
-      const first = await fallback.compare(options)
-      expect(first.stats.explicitTagScores).toBe(3)
-      expect(first.stats.cacheMisses).toBe(3)
-
-      const alternative = (token: string, probability: number) => ({ token, logprob: Math.log(probability) })
-      const body = { choices: [{ message: { content: '<score_A> A </score_A> <score_B> T </score_B>' }, logprobs: { content: [{ token: '<score_A>', logprob: 0, top_logprobs: [] }, { token: 'A', logprob: -0.1, top_logprobs: [alternative('A', 0.7), alternative('T', 0.3)] }, { token: '<score_B>', logprob: 0, top_logprobs: [] }, { token: 'T', logprob: -0.1, top_logprobs: [alternative('T', 0.8), alternative('A', 0.2)] }] } }], usage: { prompt_tokens: 10, completion_tokens: 4 } }
-      vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(body), { status: 200 })))
-      const context = { get(name: string) { if (name === 'settings') return { get: () => ({ providers: { openai: { api: 'openai-completions', baseURL: 'https://example.test/v1', apiKeyEnv: 'OPENAI_API_KEY' } } }) }; if (name === 'credentials') return { resolve: async () => ({ value: 'secret' }) }; return undefined } }
-      const live = new VerifierEngine(clientConfig({ ctx: context as any }), 4, new ScoreCache(file, 100))
-      const second = await live.compare(options)
-      expect(second.stats.topLogprobScores).toBe(3)
-      expect(second.stats.cacheHits).toBe(0)
-      expect(second.stats.cacheMisses).toBe(3)
-    } finally {
-      rmSync(dir, { recursive: true, force: true })
-    }
-  })
-  it('files a first-call explicit-tag downgrade under explicit-tag cache keys', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))
-    try {
-      const file = join(dir, 'scores.json')
-      const options = { problem: 'task', candidateA: 'AAA', candidateB: 'BBB', repeats: 1 }
-      // No settings in ctx, so the first call predicts top-logprobs but discovers no
-      // route at runtime and downgrades; nothing was pre-marked unsupported.
-      const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream([]) } as any }), 4, new ScoreCache(file, 100))
-      const first = await engine.compare(options)
-      expect(first.stats.explicitTagScores).toBe(3)
-      expect(first.stats.cacheMisses).toBe(3)
-      // The downgraded entries must be found by the now-explicit-tag prediction.
-      const second = await engine.compare(options)
-      expect(second.stats.cacheHits).toBe(3)
-      expect(second.stats.calls).toBe(0)
-      expect(second.stats.explicitTagScores).toBe(3)
-    } finally {
-      rmSync(dir, { recursive: true, force: true })
-    }
-  })
-  it('plays a single match when select is called with two candidates', async () => {
-    const judged: Array<string> = []
-    const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged) } as any }), 4)
-    const result = await engine.select({ problem: 'Pick the better implementation.', candidates: ['STRONG-0', 'WEAK-1'], repeats: 1 })
-    expect(new Set(judged).size).toBe(judged.length)
-    expect(result.comparisons).toBe(1)
-    expect(result.index).toBe(0)
-    expect(result.pivots).toEqual([])
-  })
-  it('merges concurrent identical requests across engine instances inside the first-call downgrade window', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))
-    try {
-      const judged: Array<string> = []
-      const gate: Array<() => void> = []
-      const caps = new TopLogprobCapabilityCache()
-      // Separate engines, shared topic flight table + cache — the index.ts assembly shape.
-      const flights = new SingleFlight<{ value: CachedPairScore; hit: boolean }>()
-      const cache = new ScoreCache(join(dir, 'scores.json'), 100)
-      const makeEngine = () => new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged, gate) } as any, topLogprobCapabilities: caps }), 4, cache, { input: 0, output: 0 }, flights)
-      const options = { problem: 'task', candidateA: 'AAA', candidateB: 'BBB', repeats: 1, criteria: [{ id: 'one', name: 'One', description: 'single criterion' }] }
-      // No settings in ctx, so t1 predicts top-logprobs but downgrades at runtime.
-      const first = makeEngine().compare(options)
-      caps.markUnsupported('openai', 'gpt-5')
-      const second = makeEngine().compare(options)
-      // Wait until the first engine's stream is actually parked on the gate (cache
-      // load does real fs I/O, so a single macrotask tick is not enough), then release.
-      for (let index = 0; index < 200 && gate.length === 0; index += 1) await new Promise<void>(resolve => setTimeout(resolve, 5))
-      while (gate.length > 0) gate.shift()!()
-      const [r1, r2] = await Promise.all([first, second])
-      // One criterion, one merged model call — without the shared flight the second engine would re-run it.
-      expect(judged).toHaveLength(1)
-      expect(r1.stats.cacheMisses).toBe(1)
-      expect(r2.stats.cacheHits).toBe(1)
-      expect(r2.stats.calls).toBe(0)
-      expect(r2.scoreA).toBe(r1.scoreA)
-      expect(r2.scoreB).toBe(r1.scoreB)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-})
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type { VerifierClientConfig } from './caller.ts'
+import { ScoreCache, SingleFlight, type CachedPairScore } from './cache.ts'
+import { VerifierEngine } from './engine.ts'
+import { TopLogprobCapabilityCache } from './top-logprobs.ts'
+
+function chunks(text: string) { return [{ type: 'block-start', index: 0, blockType: 'text' }, { type: 'text-delta', index: 0, text }, { type: 'block-end', index: 0, block: { type: 'text', text } }, { type: 'usage', usage: { inputTokens: 7, cacheReadTokens: 3, outputTokens: 4, reasoningTokens: 2 } }, { type: 'finish', reason: { kind: 'stop' } }] as any[] }
+async function* streamOf(items: any[]) { for (const item of items) yield item }
+
+function clientConfig(overrides: Partial<VerifierClientConfig> = {}): VerifierClientConfig {
+  return { ctx: { get: () => undefined } as any, llm: { stream: async function* () { throw new Error('unexpected model call') } } as any, attachments: { saveImage: async () => ({}) } as any, topLogprobCapabilities: new TopLogprobCapabilityCache(), provider: 'openai', model: 'gpt-5', reasoningEffort: 'high', maxTokens: 100, timeoutMs: 1000, maxRetries: 0, retryBaseDelayMs: 1, ...overrides }
+}
+
+/** Streams an explicit-tag verdict derived from which named candidate sits in trajectory A/B. */
+function scriptedStream(judged: Array<string>, gate?: Array<() => void>): (options: any) => AsyncIterable<any[]> {
+  return function (options: any) {
+    const message = options.messages[0]
+    const prompt = typeof message.content === 'string' ? message.content : message.content.filter((block: any) => block.type === 'text').map((block: any) => block.text).join('')
+    const section = (marker: string) => { const at = prompt.indexOf(marker); if (at < 0) return ''; const end = prompt.indexOf('\n\n**', at); return prompt.slice(at + marker.length, end < 0 ? undefined : end) }
+    const traceA = section('**Trajectory A:**\n')
+    const traceB = section('**Trajectory B:**\n')
+    const letter = (trace: string) => trace.includes('STRONG') ? 'A' : 'T'
+    const criterion = /\*\*Evaluation Guideline — (.+?):\*\*/.exec(prompt)?.[1] ?? '?'
+    judged.push(criterion + '|' + [traceA.trim(), traceB.trim()].sort().join('|'))
+    const text = '<score_A> ' + letter(traceA) + ' </score_A>\n<score_B> ' + letter(traceB) + ' </score_B>'
+    if (gate === undefined) return streamOf(chunks(text))
+    return (async function* () { await new Promise<void>(resolve => { gate.push(resolve) }); yield* streamOf(chunks(text)) })()
+  }
+}
+
+afterEach(() => vi.unstubAllGlobals())
+
+describe('VerifierEngine tournament', () => {
+  it('judges each unordered candidate pair at most once and ranks the strong candidate first', async () => {
+    const judged: Array<string> = []
+    const llm = { stream: scriptedStream(judged) } as any
+    const engine = new VerifierEngine(clientConfig({ llm }), 4)
+    const candidates = ['STRONG-0', 'WEAK-1', 'WEAK-2', 'WEAK-3']
+    const result = await engine.select({ problem: 'Pick the better implementation.', candidates, repeats: 1 })
+    // Ring edges incident to a pivot used to be re-judged (reversed) in the pivot round.
+    expect(new Set(judged).size).toBe(judged.length)
+    expect(result.comparisons).toBe(new Set(judged.map(entry => entry.split('|')[1] + '|' + entry.split('|')[2])).size)
+    expect(result.stats.calls).toBe(judged.length)
+    expect(result.ranking[0]).toBe(0)
+    expect(result.best).toBe('STRONG-0')
+  })
+})
+
+describe('VerifierEngine cache identity', () => {
+  it('does not reuse an explicit-tag cache entry for top-logprobs scoring', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))
+    try {
+      const file = join(dir, 'scores.json')
+      const options = { problem: 'task', candidateA: 'AAA', candidateB: 'BBB', repeats: 1 }
+      const fallbackCaps = new TopLogprobCapabilityCache()
+      fallbackCaps.markUnsupported('openai', 'gpt-5')
+      const fallback = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream([]) } as any, topLogprobCapabilities: fallbackCaps }), 4, new ScoreCache(file, 100))
+      const first = await fallback.compare(options)
+      expect(first.stats.explicitTagScores).toBe(3)
+      expect(first.stats.cacheMisses).toBe(3)
+
+      const alternative = (token: string, probability: number) => ({ token, logprob: Math.log(probability) })
+      const body = { choices: [{ message: { content: '<score_A> A </score_A> <score_B> T </score_B>' }, logprobs: { content: [{ token: '<score_A>', logprob: 0, top_logprobs: [] }, { token: 'A', logprob: -0.1, top_logprobs: [alternative('A', 0.7), alternative('T', 0.3)] }, { token: '<score_B>', logprob: 0, top_logprobs: [] }, { token: 'T', logprob: -0.1, top_logprobs: [alternative('T', 0.8), alternative('A', 0.2)] }] } }], usage: { prompt_tokens: 10, completion_tokens: 4 } }
+      vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(body), { status: 200 })))
+      const context = { get(name: string) { if (name === 'settings') return { get: () => ({ providers: { openai: { api: 'openai-completions', baseURL: 'https://example.test/v1', apiKeyEnv: 'OPENAI_API_KEY' } } }) }; if (name === 'credentials') return { resolve: async () => ({ value: 'secret' }) }; return undefined } }
+      const live = new VerifierEngine(clientConfig({ ctx: context as any }), 4, new ScoreCache(file, 100))
+      const second = await live.compare(options)
+      expect(second.stats.topLogprobScores).toBe(3)
+      expect(second.stats.cacheHits).toBe(0)
+      expect(second.stats.cacheMisses).toBe(3)
+    } finally {
+      rmSync(dir, { recursive: true, force: true })
+    }
+  })
+  it('files a first-call explicit-tag downgrade under explicit-tag cache keys', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))
+    try {
+      const file = join(dir, 'scores.json')
+      const options = { problem: 'task', candidateA: 'AAA', candidateB: 'BBB', repeats: 1 }
+      // No settings in ctx, so the first call predicts top-logprobs but discovers no
+      // route at runtime and downgrades; nothing was pre-marked unsupported.
+      const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream([]) } as any }), 4, new ScoreCache(file, 100))
+      const first = await engine.compare(options)
+      expect(first.stats.explicitTagScores).toBe(3)
+      expect(first.stats.cacheMisses).toBe(3)
+      // The downgraded entries must be found by the now-explicit-tag prediction.
+      const second = await engine.compare(options)
+      expect(second.stats.cacheHits).toBe(3)
+      expect(second.stats.calls).toBe(0)
+      expect(second.stats.explicitTagScores).toBe(3)
+    } finally {
+      rmSync(dir, { recursive: true, force: true })
+    }
+  })
+  it('plays a single match when select is called with two candidates', async () => {
+    const judged: Array<string> = []
+    const engine = new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged) } as any }), 4)
+    const result = await engine.select({ problem: 'Pick the better implementation.', candidates: ['STRONG-0', 'WEAK-1'], repeats: 1 })
+    expect(new Set(judged).size).toBe(judged.length)
+    expect(result.comparisons).toBe(1)
+    expect(result.index).toBe(0)
+    expect(result.pivots).toEqual([])
+  })
+  it('merges concurrent identical requests across engine instances inside the first-call downgrade window', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))
+    try {
+      const judged: Array<string> = []
+      const gate: Array<() => void> = []
+      const caps = new TopLogprobCapabilityCache()
+      // Separate engines, shared topic flight table + cache — the index.ts assembly shape.
+      const flights = new SingleFlight<{ value: CachedPairScore; hit: boolean }>()
+      const cache = new ScoreCache(join(dir, 'scores.json'), 100)
+      const makeEngine = () => new VerifierEngine(clientConfig({ llm: { stream: scriptedStream(judged, gate) } as any, topLogprobCapabilities: caps }), 4, cache, { input: 0, output: 0 }, flights)
+      const options = { problem: 'task', candidateA: 'AAA', candidateB: 'BBB', repeats: 1, criteria: [{ id: 'one', name: 'One', description: 'single criterion' }] }
+      // No settings in ctx, so t1 predicts top-logprobs but downgrades at runtime.
+      const first = makeEngine().compare(options)
+      caps.markUnsupported('openai', 'gpt-5')
+      const second = makeEngine().compare(options)
+      // Wait until the first engine's stream is actually parked on the gate (cache
+      // load does real fs I/O, so a single macrotask tick is not enough), then release.
+      for (let index = 0; index < 200 && gate.length === 0; index += 1) await new Promise<void>(resolve => setTimeout(resolve, 5))
+      while (gate.length > 0) gate.shift()!()
+      const [r1, r2] = await Promise.all([first, second])
+      // One criterion, one merged model call — without the shared flight the second engine would re-run it.
+      expect(judged).toHaveLength(1)
+      expect(r1.stats.cacheMisses).toBe(1)
+      expect(r2.stats.cacheHits).toBe(1)
+      expect(r2.stats.calls).toBe(0)
+      expect(r2.scoreA).toBe(r1.scoreA)
+      expect(r2.scoreB).toBe(r1.scoreB)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+})

package/src/engine.ts

@@ -1,140 +1,140 @@
-import { addUsage, callVerifier, emptyUsage, predictScoringChannel, type ScoringMode, type UsageStats, type VerifierClientConfig, type VerifierImage } from './caller.ts'
-import { ScoreCache, SingleFlight, stableHash, type CachedPairScore } from './cache.ts'
-import {
-  DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, accumulatePairs, buildPairwisePrompt, buildProgressPrompt,
-  extractProgressScore, extractScore, pivotRoundPairs, rankScores, ringCycle, topPivots, type Criterion,
-} from './core.ts'
-
-export interface CompareOptions { problem: string; candidateA: string; candidateB: string; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; images?: readonly VerifierImage[] }
-export interface CriterionResult { id: string; name: string; scoreA: number; scoreB: number }
-export interface RunStats extends UsageStats { cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
-export interface CompareResult { scoreA: number; scoreB: number; winner: 'A' | 'B' | 'tie'; criteria: CriterionResult[]; calls: number; stats: RunStats }
-export interface SelectOptions { problem: string; candidates: readonly string[]; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; pivots?: number; seed?: number; images?: readonly VerifierImage[] }
-export interface SelectResult { index: number; best: string; scores: number[]; ranking: number[]; pivots: number[]; comparisons: number; calls: number; stats: RunStats }
-
-function average(values: readonly number[]): number { return values.reduce((sum, value) => sum + value, 0) / (values.length || 1) }
-function blankStats(): RunStats { return { ...emptyUsage(), cacheHits: 0, cacheMisses: 0, estimatedCostUsd: 0, topLogprobScores: 0, explicitTagScores: 0 } }
-function unorderedPair(a: number, b: number): string { return a < b ? a + ',' + b : b + ',' + a }
-
-export class VerifierEngine {
-  readonly client: VerifierClientConfig
-  readonly maxConcurrency: number
-  readonly cache: ScoreCache | undefined
-  readonly inputPrice: number
-  readonly outputPrice: number
-  private readonly flights: SingleFlight<{ value: CachedPairScore; hit: boolean }>
-
-  constructor(client: VerifierClientConfig, maxConcurrency = 8, cache?: ScoreCache, prices: { input: number; output: number } = { input: 0, output: 0 }, flights: SingleFlight<{ value: CachedPairScore; hit: boolean }> = new SingleFlight()) {
-    this.client = client; this.maxConcurrency = maxConcurrency; this.cache = cache; this.inputPrice = prices.input; this.outputPrice = prices.output; this.flights = flights
-  }
-
-  private finishStats(stats: RunStats): RunStats {
-    stats.estimatedCostUsd = ((stats.inputTokens + stats.cachedInputTokens) * this.inputPrice + stats.outputTokens * this.outputPrice) / 1_000_000
-    return stats
-  }
-
-  private async scoreOne(options: CompareOptions, candidateA: string, candidateB: string, criterion: Criterion, repeat: number, signal?: AbortSignal): Promise<{ scores: readonly [number, number]; usage: UsageStats; scoringMode: 'top-logprobs' | 'explicit-tag'; hit: boolean }> {
-    const ground = options.groundTruthNote ?? DEFAULT_GROUND_TRUTH_NOTE
-    const prompt = buildPairwisePrompt(options.problem, candidateA, candidateB, criterion, ground)
-    const imageKey = options.images?.map(image => stableHash([image.mediaType, Buffer.from(image.data).toString('base64')]))
-    // Cache identity pins the scoring channel, but the channel is only predictable
-    // before the call when the capability cache already knows the answer. Lookups use
-    // the predicted channel while entries are always stored under the ACTUAL
-    // completion mode, so a first-call downgrade lands under explicit-tag keys and is
-    // found by later explicit-tag calls instead of being misread as a top-logprobs
-    // expectation. The channel stays OUT of the in-flight dedup key below.
-    const identity = { version: 4, provider: this.client.provider, model: this.client.model, effort: this.client.reasoningEffort, maxTokens: this.client.maxTokens, repeat, promptHash: stableHash(prompt), imageKey }
-    const keyForMode = (scoringMode: ScoringMode) => stableHash({ ...identity, scoringMode })
-    const create = async () => {
-      const completion = await callVerifier(this.client, prompt, signal, options.images)
-      return { scoreA: extractScore(completion, '<score_A>'), scoreB: extractScore(completion, '<score_B>'), usage: completion.usage, scoringMode: completion.scoringMode, createdAt: Date.now() }
-    }
-    const cache = this.cache
-    if (cache === undefined) { const value = await create(); return { scores: [value.scoreA, value.scoreB], usage: value.usage, scoringMode: value.scoringMode, hit: false } }
-    // Registration happens in the synchronous segment before any await, so a request
-    // that arrives while the first one is still resolving its channel prediction
-    // still merges instead of duplicating the model call. The flight table is shared
-    // per topic, so concurrent tool calls through separate engine instances merge too.
-    const dedupeKey = stableHash(identity)
-    const outcome = await this.flights.run(dedupeKey, async () => cache.getOrCreate(keyForMode(await predictScoringChannel(this.client)), create, value => keyForMode(value.scoringMode)))
-    const landed = outcome.value
-    const reused = outcome.joined || landed.hit
-    return { scores: [landed.value.scoreA, landed.value.scoreB], usage: reused ? emptyUsage() : landed.value.usage, scoringMode: landed.value.scoringMode, hit: reused }
-  }
-
-  private async mapLimited<T, R>(items: readonly T[], worker: (item: T) => Promise<R>): Promise<R[]> {
-    const results = new Array<R>(items.length); let cursor = 0
-    const runners = Array.from({ length: Math.min(this.maxConcurrency, items.length) }, async () => { while (cursor < items.length) { const index = cursor++; results[index] = await worker(items[index]!) } })
-    await Promise.all(runners); return results
-  }
-
-  async compare(options: CompareOptions, signal?: AbortSignal): Promise<CompareResult> {
-    const criteria = options.criteria?.length ? options.criteria : DEFAULT_CRITERIA
-    const repeats = options.repeats ?? 2
-    const jobs = criteria.flatMap(criterion => Array.from({ length: repeats }, (_, repeat) => ({ criterion, repeat })))
-    // Prefix warm-up: one criterion/repeat runs first, then the shared-prefix fan-out.
-    const warm = jobs.slice(0, 1); const rest = jobs.slice(1)
-    const run = async (batch: typeof jobs) => this.mapLimited(batch, async ({ criterion, repeat }) => {
-      const swapped = repeat % 2 === 1
-      const result = await this.scoreOne(options, swapped ? options.candidateB : options.candidateA, swapped ? options.candidateA : options.candidateB, criterion, repeat, signal)
-      return { criterion, scoreA: swapped ? result.scores[1] : result.scores[0], scoreB: swapped ? result.scores[0] : result.scores[1], usage: result.usage, scoringMode: result.scoringMode, hit: result.hit }
-    })
-    const values = [...await run(warm), ...await run(rest)]
-    const stats = blankStats()
-    for (const value of values) { addUsage(stats, value.usage); value.hit ? stats.cacheHits++ : stats.cacheMisses++; value.scoringMode === 'top-logprobs' ? stats.topLogprobScores++ : stats.explicitTagScores++ }
-    const byCriterion = criteria.map(criterion => { const rows = values.filter(value => value.criterion.id === criterion.id); return { id: criterion.id, name: criterion.name, scoreA: average(rows.map(row => row.scoreA)), scoreB: average(rows.map(row => row.scoreB)) } })
-    const scoreA = average(byCriterion.map(value => value.scoreA)); const scoreB = average(byCriterion.map(value => value.scoreB))
-    return { scoreA, scoreB, winner: Math.abs(scoreA - scoreB) < 1e-12 ? 'tie' : scoreA > scoreB ? 'A' : 'B', criteria: byCriterion, calls: stats.calls, stats: this.finishStats(stats) }
-  }
-
-  private async scorePairs(options: SelectOptions, pairs: readonly [number, number][], signal?: AbortSignal): Promise<{ rewards: Map<string, readonly [number, number]>; stats: RunStats }> {
-    const unique = [...new Map(pairs.map(pair => [pair[0] + ',' + pair[1], pair])).values()]
-    const values = await this.mapLimited(unique, async ([a, b]) => ({ a, b, result: await this.compare({ problem: options.problem, candidateA: options.candidates[a]!, candidateB: options.candidates[b]!, criteria: options.criteria, groundTruthNote: options.groundTruthNote, repeats: options.repeats, images: options.images }, signal) }))
-    const rewards = new Map<string, readonly [number, number]>(); const stats = blankStats()
-    for (const value of values) { rewards.set(value.a + ',' + value.b, [value.result.scoreA, value.result.scoreB]); addUsage(stats, value.result.stats); stats.cacheHits += value.result.stats.cacheHits; stats.cacheMisses += value.result.stats.cacheMisses; stats.topLogprobScores += value.result.stats.topLogprobScores; stats.explicitTagScores += value.result.stats.explicitTagScores }
-    return { rewards, stats: this.finishStats(stats) }
-  }
-
-  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats }> {
-    if (!steps.length || !checkpoints.length) throw new Error('llm-verifier: steps and checkpoints must not be empty')
-    for (const checkpoint of checkpoints) if (!Number.isSafeInteger(checkpoint) || checkpoint < 1 || checkpoint > steps.length) throw new Error('llm-verifier: each checkpoint must be an integer between 1 and steps.length')
-    const prompt = buildProgressPrompt(problem, steps, checkpoints)
-    const completions = await this.mapLimited(Array.from({ length: repeats }, (_, index) => index), async () => callVerifier(this.client, prompt, signal, images))
-    const stats = blankStats(); for (const completion of completions) { addUsage(stats, completion.usage); completion.scoringMode === 'top-logprobs' ? stats.topLogprobScores++ : stats.explicitTagScores++ }
-    const runs = completions.map(completion => checkpoints.map((_, index) => extractProgressScore(completion, '<c' + (index + 1) + '>')))
-    return { scores: checkpoints.map((_, index) => average(runs.map(run => run[index]!))), perRepeat: runs, calls: stats.calls, stats: this.finishStats(stats) }
-  }
-
-  async select(options: SelectOptions, signal?: AbortSignal): Promise<SelectResult> {
-    if (!options.candidates.length) throw new Error('llm-verifier: candidates must not be empty')
-    if (options.candidates.length === 1) return { index: 0, best: options.candidates[0]!, scores: [1], ranking: [0], pivots: [0], comparisons: 0, calls: 0, stats: blankStats() }
-    if (options.candidates.length === 2) {
-      // ringCycle(2) would judge the single unordered pair in both directions; play it once.
-      const { rewards, stats } = await this.scorePairs(options, [[0, 1]], signal)
-      const wins = [0, 0]; const counts = [0, 0]
-      accumulatePairs([[0, 1]], rewards, wins, counts)
-      const ranked = rankScores(wins, counts); const index = ranked[0]!.index
-      return { index, best: options.candidates[index]!, scores: wins.map((value, candidate) => value / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots: [], comparisons: 1, calls: stats.calls, stats }
-    }
-    const ring = ringCycle(options.candidates.length, options.seed ?? 0); const ringScores = await this.scorePairs(options, ring, signal)
-    const firstWins = new Array<number>(options.candidates.length).fill(0); const firstCounts = new Array<number>(options.candidates.length).fill(0); accumulatePairs(ring, ringScores.rewards, firstWins, firstCounts)
-    const pivots = topPivots(firstWins, firstCounts, options.pivots ?? 2)
-    // pivotRoundPairs regenerates every ring edge that touches a pivot (reversed for
-    // pivot→neighbour edges). Dropping those duplicates keeps each unordered pair to a
-    // single match so wins/counts are not double-weighted and no pair is judged twice.
-    const ringPairs = new Set(ring.map(pair => unorderedPair(pair[0], pair[1])))
-    const rounds = pivotRoundPairs(options.candidates.length, pivots).filter(pair => !ringPairs.has(unorderedPair(pair[0], pair[1])))
-    const roundScores = await this.scorePairs(options, rounds, signal)
-    const allRewards = new Map([...ringScores.rewards, ...roundScores.rewards]); const wins = new Array<number>(options.candidates.length).fill(0); const counts = new Array<number>(options.candidates.length).fill(0)
-    accumulatePairs(ring, allRewards, wins, counts); accumulatePairs(rounds, allRewards, wins, counts); const ranked = rankScores(wins, counts); const index = ranked[0]!.index
-    const stats = blankStats(); for (const source of [ringScores.stats, roundScores.stats]) { addUsage(stats, source); stats.cacheHits += source.cacheHits; stats.cacheMisses += source.cacheMisses; stats.topLogprobScores += source.topLogprobScores; stats.explicitTagScores += source.explicitTagScores }
-    return { index, best: options.candidates[index]!, scores: Array.from({ length: options.candidates.length }, (_, candidate) => wins[candidate]! / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots, comparisons: ring.length + rounds.length, calls: stats.calls, stats: this.finishStats(stats) }
-  }
-}
-
-export function normalizeCriteria(input: unknown): Criterion[] {
-  if (input === undefined) return DEFAULT_CRITERIA
-  if (!Array.isArray(input) || !input.length) throw new Error('llm-verifier: criteria must be a non-empty array')
-  return input.map((value, index) => { if (typeof value !== 'object' || value === null) throw new Error('llm-verifier: criteria[' + index + '] must be an object'); const row = value as Record<string, unknown>; for (const key of ['id', 'name', 'description']) if (typeof row[key] !== 'string' || row[key].trim().length === 0) throw new Error('llm-verifier: criteria[' + index + '].' + key + ' must be non-empty'); return { id: String(row.id), name: String(row.name), description: String(row.description) } })
-}
-export { DEFAULT_GROUND_TRUTH_NOTE }
+import { addUsage, callVerifier, emptyUsage, predictScoringChannel, type ScoringMode, type UsageStats, type VerifierClientConfig, type VerifierImage } from './caller.ts'
+import { ScoreCache, SingleFlight, stableHash, type CachedPairScore } from './cache.ts'
+import {
+  DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, accumulatePairs, buildPairwisePrompt, buildProgressPrompt,
+  extractProgressScore, extractScore, pivotRoundPairs, rankScores, ringCycle, topPivots, type Criterion,
+} from './core.ts'
+
+export interface CompareOptions { problem: string; candidateA: string; candidateB: string; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; images?: readonly VerifierImage[] }
+export interface CriterionResult { id: string; name: string; scoreA: number; scoreB: number }
+export interface RunStats extends UsageStats { cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
+export interface CompareResult { scoreA: number; scoreB: number; winner: 'A' | 'B' | 'tie'; criteria: CriterionResult[]; calls: number; stats: RunStats }
+export interface SelectOptions { problem: string; candidates: readonly string[]; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; pivots?: number; seed?: number; images?: readonly VerifierImage[] }
+export interface SelectResult { index: number; best: string; scores: number[]; ranking: number[]; pivots: number[]; comparisons: number; calls: number; stats: RunStats }
+
+function average(values: readonly number[]): number { return values.reduce((sum, value) => sum + value, 0) / (values.length || 1) }
+function blankStats(): RunStats { return { ...emptyUsage(), cacheHits: 0, cacheMisses: 0, estimatedCostUsd: 0, topLogprobScores: 0, explicitTagScores: 0 } }
+function unorderedPair(a: number, b: number): string { return a < b ? a + ',' + b : b + ',' + a }
+
+export class VerifierEngine {
+  readonly client: VerifierClientConfig
+  readonly maxConcurrency: number
+  readonly cache: ScoreCache | undefined
+  readonly inputPrice: number
+  readonly outputPrice: number
+  private readonly flights: SingleFlight<{ value: CachedPairScore; hit: boolean }>
+
+  constructor(client: VerifierClientConfig, maxConcurrency = 8, cache?: ScoreCache, prices: { input: number; output: number } = { input: 0, output: 0 }, flights: SingleFlight<{ value: CachedPairScore; hit: boolean }> = new SingleFlight()) {
+    this.client = client; this.maxConcurrency = maxConcurrency; this.cache = cache; this.inputPrice = prices.input; this.outputPrice = prices.output; this.flights = flights
+  }
+
+  private finishStats(stats: RunStats): RunStats {
+    stats.estimatedCostUsd = ((stats.inputTokens + stats.cachedInputTokens) * this.inputPrice + stats.outputTokens * this.outputPrice) / 1_000_000
+    return stats
+  }
+
+  private async scoreOne(options: CompareOptions, candidateA: string, candidateB: string, criterion: Criterion, repeat: number, signal?: AbortSignal): Promise<{ scores: readonly [number, number]; usage: UsageStats; scoringMode: 'top-logprobs' | 'explicit-tag'; hit: boolean }> {
+    const ground = options.groundTruthNote ?? DEFAULT_GROUND_TRUTH_NOTE
+    const prompt = buildPairwisePrompt(options.problem, candidateA, candidateB, criterion, ground)
+    const imageKey = options.images?.map(image => stableHash([image.mediaType, Buffer.from(image.data).toString('base64')]))
+    // Cache identity pins the scoring channel, but the channel is only predictable
+    // before the call when the capability cache already knows the answer. Lookups use
+    // the predicted channel while entries are always stored under the ACTUAL
+    // completion mode, so a first-call downgrade lands under explicit-tag keys and is
+    // found by later explicit-tag calls instead of being misread as a top-logprobs
+    // expectation. The channel stays OUT of the in-flight dedup key below.
+    const identity = { version: 4, provider: this.client.provider, model: this.client.model, effort: this.client.reasoningEffort, maxTokens: this.client.maxTokens, repeat, promptHash: stableHash(prompt), imageKey }
+    const keyForMode = (scoringMode: ScoringMode) => stableHash({ ...identity, scoringMode })
+    const create = async () => {
+      const completion = await callVerifier(this.client, prompt, signal, options.images)
+      return { scoreA: extractScore(completion, '<score_A>'), scoreB: extractScore(completion, '<score_B>'), usage: completion.usage, scoringMode: completion.scoringMode, createdAt: Date.now() }
+    }
+    const cache = this.cache
+    if (cache === undefined) { const value = await create(); return { scores: [value.scoreA, value.scoreB], usage: value.usage, scoringMode: value.scoringMode, hit: false } }
+    // Registration happens in the synchronous segment before any await, so a request
+    // that arrives while the first one is still resolving its channel prediction
+    // still merges instead of duplicating the model call. The flight table is shared
+    // per topic, so concurrent tool calls through separate engine instances merge too.
+    const dedupeKey = stableHash(identity)
+    const outcome = await this.flights.run(dedupeKey, async () => cache.getOrCreate(keyForMode(await predictScoringChannel(this.client)), create, value => keyForMode(value.scoringMode)))
+    const landed = outcome.value
+    const reused = outcome.joined || landed.hit
+    return { scores: [landed.value.scoreA, landed.value.scoreB], usage: reused ? emptyUsage() : landed.value.usage, scoringMode: landed.value.scoringMode, hit: reused }
+  }
+
+  private async mapLimited<T, R>(items: readonly T[], worker: (item: T) => Promise<R>): Promise<R[]> {
+    const results = new Array<R>(items.length); let cursor = 0
+    const runners = Array.from({ length: Math.min(this.maxConcurrency, items.length) }, async () => { while (cursor < items.length) { const index = cursor++; results[index] = await worker(items[index]!) } })
+    await Promise.all(runners); return results
+  }
+
+  async compare(options: CompareOptions, signal?: AbortSignal): Promise<CompareResult> {
+    const criteria = options.criteria?.length ? options.criteria : DEFAULT_CRITERIA
+    const repeats = options.repeats ?? 2
+    const jobs = criteria.flatMap(criterion => Array.from({ length: repeats }, (_, repeat) => ({ criterion, repeat })))
+    // Prefix warm-up: one criterion/repeat runs first, then the shared-prefix fan-out.
+    const warm = jobs.slice(0, 1); const rest = jobs.slice(1)
+    const run = async (batch: typeof jobs) => this.mapLimited(batch, async ({ criterion, repeat }) => {
+      const swapped = repeat % 2 === 1
+      const result = await this.scoreOne(options, swapped ? options.candidateB : options.candidateA, swapped ? options.candidateA : options.candidateB, criterion, repeat, signal)
+      return { criterion, scoreA: swapped ? result.scores[1] : result.scores[0], scoreB: swapped ? result.scores[0] : result.scores[1], usage: result.usage, scoringMode: result.scoringMode, hit: result.hit }
+    })
+    const values = [...await run(warm), ...await run(rest)]
+    const stats = blankStats()
+    for (const value of values) { addUsage(stats, value.usage); value.hit ? stats.cacheHits++ : stats.cacheMisses++; value.scoringMode === 'top-logprobs' ? stats.topLogprobScores++ : stats.explicitTagScores++ }
+    const byCriterion = criteria.map(criterion => { const rows = values.filter(value => value.criterion.id === criterion.id); return { id: criterion.id, name: criterion.name, scoreA: average(rows.map(row => row.scoreA)), scoreB: average(rows.map(row => row.scoreB)) } })
+    const scoreA = average(byCriterion.map(value => value.scoreA)); const scoreB = average(byCriterion.map(value => value.scoreB))
+    return { scoreA, scoreB, winner: Math.abs(scoreA - scoreB) < 1e-12 ? 'tie' : scoreA > scoreB ? 'A' : 'B', criteria: byCriterion, calls: stats.calls, stats: this.finishStats(stats) }
+  }
+
+  private async scorePairs(options: SelectOptions, pairs: readonly [number, number][], signal?: AbortSignal): Promise<{ rewards: Map<string, readonly [number, number]>; stats: RunStats }> {
+    const unique = [...new Map(pairs.map(pair => [pair[0] + ',' + pair[1], pair])).values()]
+    const values = await this.mapLimited(unique, async ([a, b]) => ({ a, b, result: await this.compare({ problem: options.problem, candidateA: options.candidates[a]!, candidateB: options.candidates[b]!, criteria: options.criteria, groundTruthNote: options.groundTruthNote, repeats: options.repeats, images: options.images }, signal) }))
+    const rewards = new Map<string, readonly [number, number]>(); const stats = blankStats()
+    for (const value of values) { rewards.set(value.a + ',' + value.b, [value.result.scoreA, value.result.scoreB]); addUsage(stats, value.result.stats); stats.cacheHits += value.result.stats.cacheHits; stats.cacheMisses += value.result.stats.cacheMisses; stats.topLogprobScores += value.result.stats.topLogprobScores; stats.explicitTagScores += value.result.stats.explicitTagScores }
+    return { rewards, stats: this.finishStats(stats) }
+  }
+
+  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats }> {
+    if (!steps.length || !checkpoints.length) throw new Error('llm-verifier: steps and checkpoints must not be empty')
+    for (const checkpoint of checkpoints) if (!Number.isSafeInteger(checkpoint) || checkpoint < 1 || checkpoint > steps.length) throw new Error('llm-verifier: each checkpoint must be an integer between 1 and steps.length')
+    const prompt = buildProgressPrompt(problem, steps, checkpoints)
+    const completions = await this.mapLimited(Array.from({ length: repeats }, (_, index) => index), async () => callVerifier(this.client, prompt, signal, images))
+    const stats = blankStats(); for (const completion of completions) { addUsage(stats, completion.usage); completion.scoringMode === 'top-logprobs' ? stats.topLogprobScores++ : stats.explicitTagScores++ }
+    const runs = completions.map(completion => checkpoints.map((_, index) => extractProgressScore(completion, '<c' + (index + 1) + '>')))
+    return { scores: checkpoints.map((_, index) => average(runs.map(run => run[index]!))), perRepeat: runs, calls: stats.calls, stats: this.finishStats(stats) }
+  }
+
+  async select(options: SelectOptions, signal?: AbortSignal): Promise<SelectResult> {
+    if (!options.candidates.length) throw new Error('llm-verifier: candidates must not be empty')
+    if (options.candidates.length === 1) return { index: 0, best: options.candidates[0]!, scores: [1], ranking: [0], pivots: [0], comparisons: 0, calls: 0, stats: blankStats() }
+    if (options.candidates.length === 2) {
+      // ringCycle(2) would judge the single unordered pair in both directions; play it once.
+      const { rewards, stats } = await this.scorePairs(options, [[0, 1]], signal)
+      const wins = [0, 0]; const counts = [0, 0]
+      accumulatePairs([[0, 1]], rewards, wins, counts)
+      const ranked = rankScores(wins, counts); const index = ranked[0]!.index
+      return { index, best: options.candidates[index]!, scores: wins.map((value, candidate) => value / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots: [], comparisons: 1, calls: stats.calls, stats }
+    }
+    const ring = ringCycle(options.candidates.length, options.seed ?? 0); const ringScores = await this.scorePairs(options, ring, signal)
+    const firstWins = new Array<number>(options.candidates.length).fill(0); const firstCounts = new Array<number>(options.candidates.length).fill(0); accumulatePairs(ring, ringScores.rewards, firstWins, firstCounts)
+    const pivots = topPivots(firstWins, firstCounts, options.pivots ?? 2)
+    // pivotRoundPairs regenerates every ring edge that touches a pivot (reversed for
+    // pivot→neighbour edges). Dropping those duplicates keeps each unordered pair to a
+    // single match so wins/counts are not double-weighted and no pair is judged twice.
+    const ringPairs = new Set(ring.map(pair => unorderedPair(pair[0], pair[1])))
+    const rounds = pivotRoundPairs(options.candidates.length, pivots).filter(pair => !ringPairs.has(unorderedPair(pair[0], pair[1])))
+    const roundScores = await this.scorePairs(options, rounds, signal)
+    const allRewards = new Map([...ringScores.rewards, ...roundScores.rewards]); const wins = new Array<number>(options.candidates.length).fill(0); const counts = new Array<number>(options.candidates.length).fill(0)
+    accumulatePairs(ring, allRewards, wins, counts); accumulatePairs(rounds, allRewards, wins, counts); const ranked = rankScores(wins, counts); const index = ranked[0]!.index
+    const stats = blankStats(); for (const source of [ringScores.stats, roundScores.stats]) { addUsage(stats, source); stats.cacheHits += source.cacheHits; stats.cacheMisses += source.cacheMisses; stats.topLogprobScores += source.topLogprobScores; stats.explicitTagScores += source.explicitTagScores }
+    return { index, best: options.candidates[index]!, scores: Array.from({ length: options.candidates.length }, (_, candidate) => wins[candidate]! / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots, comparisons: ring.length + rounds.length, calls: stats.calls, stats: this.finishStats(stats) }
+  }
+}
+
+export function normalizeCriteria(input: unknown): Criterion[] {
+  if (input === undefined) return DEFAULT_CRITERIA
+  if (!Array.isArray(input) || !input.length) throw new Error('llm-verifier: criteria must be a non-empty array')
+  return input.map((value, index) => { if (typeof value !== 'object' || value === null) throw new Error('llm-verifier: criteria[' + index + '] must be an object'); const row = value as Record<string, unknown>; for (const key of ['id', 'name', 'description']) if (typeof row[key] !== 'string' || row[key].trim().length === 0) throw new Error('llm-verifier: criteria[' + index + '].' + key + ' must be non-empty'); return { id: String(row.id), name: String(row.name), description: String(row.description) } })
+}
+export { DEFAULT_GROUND_TRUTH_NOTE }