dsh-llm-verifier 0.1.0

package/src/core.test.ts

@@ -0,0 +1,67 @@
+import { describe, expect, it } from 'vitest'
+import {
+  accumulatePairs,
+  bradleyTerry,
+  buildPairwisePrompt,
+  buildProgressPrompt,
+  extractProgressScore,
+  extractScore,
+  pivotRoundPairs,
+  rankScores,
+  ringCycle,
+  topPivots,
+} from './core.ts'
+
+function completion(text: string, tokens: string[] = [], positions: Array<Array<{ token: string; logprob: number }>> = []) {
+  return { text, tokens, positions }
+}
+
+describe('score extraction', () => {
+  it('parses literal final tags', () => {
+    expect(extractScore(completion('<score_A> A </score_A>'), '<score_A>')).toBe(1)
+    expect(extractScore(completion('<score_A> T </score_A>'), '<score_A>')).toBe(0)
+  })
+  it('uses the final literal tag', () => {
+    expect(extractScore(completion('<score_A> A </score_A> blah <score_A> T </score_A>'), '<score_A>')).toBe(0)
+  })
+  it('computes normalized top-logprob expectation', () => {
+    const value = extractScore(completion('', ['<score_A>'], [[], [{ token: 'A', logprob: Math.log(0.75) }, { token: 'T', logprob: Math.log(0.25) }]]), '<score_A>')
+    expect(value).toBeCloseTo(0.75)
+  })
+  it('reverses the progress scale', () => {
+    expect(extractProgressScore(completion('<c1> T </c1>'), '<c1>')).toBe(1)
+    expect(extractProgressScore(completion('<c1> A </c1>'), '<c1>')).toBe(0)
+  })
+})
+
+describe('prompts', () => {
+  it('puts criterion at the tail', () => {
+    const prompt = buildPairwisePrompt('task', 'a', 'b', { id: 'x', name: 'Criterion X', description: 'tail marker' })
+    expect(prompt.indexOf('**Trajectory B:**')).toBeLessThan(prompt.indexOf('tail marker'))
+    expect(prompt).toContain('<score_A> LETTER_A_TO_T </score_A>')
+  })
+  it('emits exact progress tags', () => {
+    expect(buildProgressPrompt('task', ['one', 'two'], [1, 2])).toContain('<c1>LETTER</c1>\n<c2>LETTER</c2>')
+  })
+})
+
+describe('pivot tournament', () => {
+  it('makes a Hamiltonian directed ring', () => {
+    const pairs = ringCycle(5, 7)
+    expect(pairs).toHaveLength(5)
+    expect(new Set(pairs.map(pair => pair[0])).size).toBe(5)
+    expect(new Set(pairs.map(pair => pair[1])).size).toBe(5)
+  })
+  it('generates linear pivot rounds', () => {
+    expect(pivotRoundPairs(5, [1, 3])).toEqual([[0, 1], [0, 3], [2, 1], [2, 3], [4, 1], [4, 3], [1, 3]])
+  })
+  it('ranks soft wins', () => {
+    expect(bradleyTerry(0.9, 0.1)).toBeGreaterThan(0.5)
+    const rewards = new Map<string, readonly [number, number]>([['0,1', [1, 0]]])
+    const wins = [0, 0]
+    const counts = [0, 0]
+    accumulatePairs([[0, 1]], rewards, wins, counts)
+    expect(topPivots(wins, counts, 1)).toEqual([0])
+    expect(rankScores(wins, counts)[0]?.index).toBe(0)
+  })
+})

package/src/core.ts

@@ -0,0 +1,204 @@
+/** Pure scoring and Probabilistic Pivot Tournament primitives. */
+
+export interface Criterion {
+  id: string
+  name: string
+  description: string
+}
+
+export interface TokenAlternative {
+  token: string
+  logprob: number
+}
+
+export interface CompletionLogprobs {
+  text: string
+  tokens: string[]
+  positions: TokenAlternative[][]
+}
+
+export interface CandidateScore {
+  index: number
+  score: number
+}
+
+export const GRANULARITY = 20
+export const LETTERS = Array.from({ length: GRANULARITY }, (_, index) => String.fromCharCode(65 + index))
+export const SCALE_DESCRIPTION = [
+  'Rate how likely the agent correctly solved the task on a 20-point scale using letters A through T:',
+  '  A = clearly and completely succeeded with verified output (best)',
+  '  B-D = succeeded with only minor issues',
+  '  E-G = above average, mostly correct with some issues',
+  '  H-J = uncertain, leans toward success',
+  '  K-M = uncertain, leans toward failure',
+  '  N-P = below average, significant issues remain',
+  '  Q-S = failed with some partial progress',
+  '  T = clearly and completely failed (worst)',
+].join('\n')
+
+export const DEFAULT_CRITERIA: Criterion[] = [
+  {
+    id: 'specification',
+    name: 'Specification Adherence',
+    description: 'Re-read the task description and check exact requirements: file paths, output formats, naming, and explicit constraints. Penalize a solution that solves a similar but different problem.',
+  },
+  {
+    id: 'output_match',
+    name: 'Output Match',
+    description: 'Find the final verification command and compare its actual stdout/stderr to the required output. Reward only evidence literally visible in observed output; do not trust narration.',
+  },
+  {
+    id: 'error_signals',
+    name: 'Error Signal Detection',
+    description: 'Scan especially later steps for unresolved errors, tracebacks, non-zero exits, command-not-found, missing files, compilation failures, and test failures. Score only unresolved error evidence.',
+  },
+]
+
+export const DEFAULT_GROUND_TRUTH_NOTE = "**IMPORTANT:** Focus on observed tool and terminal output as ground truth. Do NOT trust the agent's self-assessment or claims of success."
+
+export function normalizeScoreLetter(token: string): string | undefined {
+  let value = token.trim()
+  if (value.startsWith('>')) value = value.slice(1).trim()
+  const match = /^([A-T])$/i.exec(value)
+  return match?.[1]?.toUpperCase()
+}
+
+function letterValue(letter: string): number {
+  return GRANULARITY - (letter.charCodeAt(0) - 65)
+}
+
+function findTagLogprobs(tokens: readonly string[], positions: readonly TokenAlternative[][], tag: string): TokenAlternative[] | undefined {
+  if (tokens.length === 0 || positions.length === 0) return undefined
+  for (const suffix of [tag, tag.slice(0, -1)]) {
+    let found: TokenAlternative[] | undefined
+    let text = ''
+    for (let index = 0; index < tokens.length; index += 1) {
+      text += tokens[index]
+      if (text.trimEnd().endsWith(suffix) && index + 1 < positions.length) found = positions[index + 1]
+    }
+    if (found !== undefined) return found
+  }
+  return undefined
+}
+
+export function extractScore(completion: CompletionLogprobs, tag: string): number {
+  const alternatives = findTagLogprobs(completion.tokens, completion.positions, tag)
+  const probabilities = new Map<number, number>()
+  for (const alternative of alternatives ?? []) {
+    const letter = normalizeScoreLetter(alternative.token)
+    if (letter === undefined || !Number.isFinite(alternative.logprob)) continue
+    const value = letterValue(letter)
+    probabilities.set(value, Math.max(probabilities.get(value) ?? 0, Math.exp(alternative.logprob)))
+  }
+  if (probabilities.size > 0) {
+    let probability = 0
+    let expectation = 0
+    for (const [value, p] of probabilities) {
+      probability += p
+      expectation += value * p
+    }
+    if (probability > 0) return (expectation / probability - 1) / (GRANULARITY - 1)
+  }
+  const name = tag.slice(1, -1)
+  const regex = new RegExp('<' + name + '>\\s*(.+?)\\s*</' + name + '>', 'gi')
+  let last: RegExpExecArray | null = null
+  for (let match = regex.exec(completion.text); match !== null; match = regex.exec(completion.text)) last = match
+  const letter = normalizeScoreLetter(last?.[1] ?? '')
+  return letter === undefined ? 0.5 : (letterValue(letter) - 1) / (GRANULARITY - 1)
+}
+
+export function buildPairwisePrompt(problem: string, traceA: string, traceB: string, criterion: Criterion, groundTruthNote = DEFAULT_GROUND_TRUTH_NOTE): string {
+  return [
+    'You are an expert evaluator of AI coding agents. You will see a task description and two agent trajectories, then evaluate them on ONE specific criterion, stated at the end.',
+    groundTruthNote,
+    '**Task:**\n' + problem,
+    '**Trajectory A:**\n' + traceA,
+    '**Trajectory B:**\n' + traceB,
+    '**Rating Scale:**\n' + SCALE_DESCRIPTION,
+    '**Evaluation Guideline — ' + criterion.name + ':**\n' + criterion.description,
+    'Score each trajectory ONLY on this specific criterion ("' + criterion.name + '"). Ignore other aspects that are not relevant to it.',
+    'Reason it through first, then END your reply with exactly these two lines and nothing after them. Replace each placeholder with a single letter A-T, keeping the spaces around the letter exactly as shown:\n<score_A> LETTER_A_TO_T </score_A>\n<score_B> LETTER_A_TO_T </score_B>',
+    'Begin your analysis now.',
+  ].join('\n\n')
+}
+
+export function buildProgressPrompt(problem: string, steps: readonly string[], checkpoints: readonly number[]): string {
+  const trajectory = steps.map((step, index) => '=== Agent Step ' + (index + 1) + ' ===\n' + step.trim()).join('\n\n')
+  const tags = checkpoints.map((_, index) => '<c' + (index + 1) + '>LETTER</c' + (index + 1) + '>').join('\n')
+  return [
+    "You are a strict, skeptical evaluator of agent task attempts. Trust observed output — NOT the agent's narration.",
+    '**Task instruction:**\n' + problem.trim(),
+    '**Agent trajectory (' + steps.length + ' agent steps):**\n' + trajectory,
+    "Score whether the agent's CURRENT state at each checkpoint would actually satisfy the task's hidden grader.",
+    'Use A through T where A = certainly NO, H-M = uncertain, N-S = leans YES, and T = essentially certain YES with matching observed verification.',
+    'Effort and confident narration are not evidence. A state without real verification should not exceed K. Scores may decrease after regressions.',
+    'The checkpoints are:\n' + checkpoints.map((step, index) => '  Checkpoint ' + (index + 1) + ' = state right after Agent Step ' + step).join('\n'),
+    'Output EXACTLY these lines and nothing else:\n' + tags,
+  ].join('\n\n')
+}
+
+/** Progress uses A=NO..T=YES, the reverse of pairwise success scoring. */
+export function extractProgressScore(completion: CompletionLogprobs, tag: string): number {
+  return 1 - extractScore(completion, tag)
+}
+
+export function bradleyTerry(rewardA: number, rewardB: number): number {
+  return 1 / (1 + Math.exp(-(rewardA - rewardB)))
+}
+
+export function seededRandom(seed: number): () => number {
+  let state = seed >>> 0
+  return () => {
+    state = (state + 0x6D2B79F5) >>> 0
+    let value = state
+    value = Math.imul(value ^ value >>> 15, value | 1)
+    value ^= value + Math.imul(value ^ value >>> 7, value | 61)
+    return ((value ^ value >>> 14) >>> 0) / 4294967296
+  }
+}
+
+export function ringCycle(count: number, seed = 0): Array<[number, number]> {
+  if (count <= 1) return []
+  const permutation = Array.from({ length: count }, (_, index) => index)
+  const random = seededRandom(seed)
+  for (let index = count - 1; index > 0; index -= 1) {
+    const other = Math.floor(random() * (index + 1))
+    ;[permutation[index], permutation[other]] = [permutation[other]!, permutation[index]!]
+  }
+  return permutation.map((candidate, index) => [candidate, permutation[(index + 1) % count]!] as [number, number])
+}
+
+export function pivotRoundPairs(count: number, pivots: readonly number[]): Array<[number, number]> {
+  const pivotSet = new Set(pivots)
+  const pairs: Array<[number, number]> = []
+  for (let candidate = 0; candidate < count; candidate += 1) {
+    if (!pivotSet.has(candidate)) for (const pivot of pivots) pairs.push([candidate, pivot])
+  }
+  const sorted = [...pivots].sort((a, b) => a - b)
+  for (let left = 0; left < sorted.length; left += 1) {
+    for (let right = left + 1; right < sorted.length; right += 1) pairs.push([sorted[left]!, sorted[right]!])
+  }
+  return pairs
+}
+
+export function accumulatePairs(pairs: readonly [number, number][], rewards: ReadonlyMap<string, readonly [number, number]>, wins: number[], counts: number[]): void {
+  for (const [a, b] of pairs) {
+    const reward = rewards.get(a + ',' + b) ?? [0.5, 0.5]
+    const probability = bradleyTerry(reward[0], reward[1])
+    wins[a] = (wins[a] ?? 0) + probability
+    counts[a] = (counts[a] ?? 0) + 1
+    wins[b] = (wins[b] ?? 0) + 1 - probability
+    counts[b] = (counts[b] ?? 0) + 1
+  }
+}
+
+export function topPivots(wins: readonly number[], counts: readonly number[], requested: number): number[] {
+  return Array.from({ length: wins.length }, (_, index) => index)
+    .sort((a, b) => ((wins[b] ?? 0) / (counts[b] || 1)) - ((wins[a] ?? 0) / (counts[a] || 1)) || a - b)
+    .slice(0, Math.min(requested, wins.length))
+}
+
+export function rankScores(wins: readonly number[], counts: readonly number[]): CandidateScore[] {
+  return Array.from({ length: wins.length }, (_, index) => ({ index, score: (wins[index] ?? 0) / (counts[index] || 1) }))
+    .sort((a, b) => b.score - a.score || a.index - b.index)
+}

package/src/engine.ts

@@ -0,0 +1,109 @@
+import { addUsage, callVerifier, emptyUsage, type UsageStats, type VerifierClientConfig, type VerifierImage } from './caller.ts'
+import { ScoreCache, stableHash } from './cache.ts'
+import {
+  DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, accumulatePairs, buildPairwisePrompt, buildProgressPrompt,
+  extractProgressScore, extractScore, pivotRoundPairs, rankScores, ringCycle, topPivots, type Criterion,
+} from './core.ts'
+
+export interface CompareOptions { problem: string; candidateA: string; candidateB: string; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; images?: readonly VerifierImage[] }
+export interface CriterionResult { id: string; name: string; scoreA: number; scoreB: number }
+export interface RunStats extends UsageStats { cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
+export interface CompareResult { scoreA: number; scoreB: number; winner: 'A' | 'B' | 'tie'; criteria: CriterionResult[]; calls: number; stats: RunStats }
+export interface SelectOptions { problem: string; candidates: readonly string[]; criteria?: readonly Criterion[]; groundTruthNote?: string; repeats?: number; pivots?: number; seed?: number; images?: readonly VerifierImage[] }
+export interface SelectResult { index: number; best: string; scores: number[]; ranking: number[]; pivots: number[]; comparisons: number; calls: number; stats: RunStats }
+
+function average(values: readonly number[]): number { return values.reduce((sum, value) => sum + value, 0) / (values.length || 1) }
+function blankStats(): RunStats { return { ...emptyUsage(), cacheHits: 0, cacheMisses: 0, estimatedCostUsd: 0, topLogprobScores: 0, explicitTagScores: 0 } }
+
+export class VerifierEngine {
+  readonly client: VerifierClientConfig
+  readonly maxConcurrency: number
+  readonly cache: ScoreCache | undefined
+  readonly inputPrice: number
+  readonly outputPrice: number
+
+  constructor(client: VerifierClientConfig, maxConcurrency = 8, cache?: ScoreCache, prices: { input: number; output: number } = { input: 0, output: 0 }) {
+    this.client = client; this.maxConcurrency = maxConcurrency; this.cache = cache; this.inputPrice = prices.input; this.outputPrice = prices.output
+  }
+
+  private finishStats(stats: RunStats): RunStats {
+    stats.estimatedCostUsd = ((stats.inputTokens + stats.cachedInputTokens) * this.inputPrice + stats.outputTokens * this.outputPrice) / 1_000_000
+    return stats
+  }
+
+  private async scoreOne(options: CompareOptions, candidateA: string, candidateB: string, criterion: Criterion, repeat: number, signal?: AbortSignal): Promise<{ scores: readonly [number, number]; usage: UsageStats; scoringMode: 'top-logprobs' | 'explicit-tag'; hit: boolean }> {
+    const ground = options.groundTruthNote ?? DEFAULT_GROUND_TRUTH_NOTE
+    const prompt = buildPairwisePrompt(options.problem, candidateA, candidateB, criterion, ground)
+    const imageKey = options.images?.map(image => stableHash([image.mediaType, Buffer.from(image.data).toString('base64')]))
+    const key = stableHash({ version: 2, scoringPolicy: 'auto-top-logprobs', provider: this.client.provider, model: this.client.model, effort: this.client.reasoningEffort, maxTokens: this.client.maxTokens, problem: options.problem, candidateA, candidateB, criterion, ground, repeat, imageKey })
+    const create = async () => {
+      const completion = await callVerifier(this.client, prompt, signal, options.images)
+      return { scoreA: extractScore(completion, '<score_A>'), scoreB: extractScore(completion, '<score_B>'), usage: completion.usage, scoringMode: completion.scoringMode, createdAt: Date.now() }
+    }
+    if (this.cache === undefined) { const value = await create(); return { scores: [value.scoreA, value.scoreB], usage: value.usage, scoringMode: value.scoringMode, hit: false } }
+    const cached = await this.cache.getOrCreate(key, create)
+    return { scores: [cached.value.scoreA, cached.value.scoreB], usage: cached.hit ? emptyUsage() : cached.value.usage, scoringMode: cached.value.scoringMode, hit: cached.hit }
+  }
+
+  private async mapLimited<T, R>(items: readonly T[], worker: (item: T) => Promise<R>): Promise<R[]> {
+    const results = new Array<R>(items.length); let cursor = 0
+    const runners = Array.from({ length: Math.min(this.maxConcurrency, items.length) }, async () => { while (cursor < items.length) { const index = cursor++; results[index] = await worker(items[index]!) } })
+    await Promise.all(runners); return results
+  }
+
+  async compare(options: CompareOptions, signal?: AbortSignal): Promise<CompareResult> {
+    const criteria = options.criteria?.length ? options.criteria : DEFAULT_CRITERIA
+    const repeats = options.repeats ?? 2
+    const jobs = criteria.flatMap(criterion => Array.from({ length: repeats }, (_, repeat) => ({ criterion, repeat })))
+    // Prefix warm-up: one criterion/repeat runs first, then the shared-prefix fan-out.
+    const warm = jobs.slice(0, 1); const rest = jobs.slice(1)
+    const run = async (batch: typeof jobs) => this.mapLimited(batch, async ({ criterion, repeat }) => {
+      const swapped = repeat % 2 === 1
+      const result = await this.scoreOne(options, swapped ? options.candidateB : options.candidateA, swapped ? options.candidateA : options.candidateB, criterion, repeat, signal)
+      return { criterion, scoreA: swapped ? result.scores[1] : result.scores[0], scoreB: swapped ? result.scores[0] : result.scores[1], usage: result.usage, scoringMode: result.scoringMode, hit: result.hit }
+    })
+    const values = [...await run(warm), ...await run(rest)]
+    const stats = blankStats()
+    for (const value of values) { addUsage(stats, value.usage); value.hit ? stats.cacheHits++ : stats.cacheMisses++; value.scoringMode === 'top-logprobs' ? stats.topLogprobScores++ : stats.explicitTagScores++ }
+    const byCriterion = criteria.map(criterion => { const rows = values.filter(value => value.criterion.id === criterion.id); return { id: criterion.id, name: criterion.name, scoreA: average(rows.map(row => row.scoreA)), scoreB: average(rows.map(row => row.scoreB)) } })
+    const scoreA = average(byCriterion.map(value => value.scoreA)); const scoreB = average(byCriterion.map(value => value.scoreB))
+    return { scoreA, scoreB, winner: Math.abs(scoreA - scoreB) < 1e-12 ? 'tie' : scoreA > scoreB ? 'A' : 'B', criteria: byCriterion, calls: stats.calls, stats: this.finishStats(stats) }
+  }
+
+  private async scorePairs(options: SelectOptions, pairs: readonly [number, number][], signal?: AbortSignal): Promise<{ rewards: Map<string, readonly [number, number]>; stats: RunStats }> {
+    const unique = [...new Map(pairs.map(pair => [pair[0] + ',' + pair[1], pair])).values()]
+    const values = await this.mapLimited(unique, async ([a, b]) => ({ a, b, result: await this.compare({ problem: options.problem, candidateA: options.candidates[a]!, candidateB: options.candidates[b]!, criteria: options.criteria, groundTruthNote: options.groundTruthNote, repeats: options.repeats, images: options.images }, signal) }))
+    const rewards = new Map<string, readonly [number, number]>(); const stats = blankStats()
+    for (const value of values) { rewards.set(value.a + ',' + value.b, [value.result.scoreA, value.result.scoreB]); addUsage(stats, value.result.stats); stats.cacheHits += value.result.stats.cacheHits; stats.cacheMisses += value.result.stats.cacheMisses; stats.topLogprobScores += value.result.stats.topLogprobScores; stats.explicitTagScores += value.result.stats.explicitTagScores }
+    return { rewards, stats: this.finishStats(stats) }
+  }
+
+  async track(problem: string, steps: readonly string[], checkpoints: readonly number[], repeats = 2, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<{ scores: number[]; perRepeat: number[][]; calls: number; stats: RunStats }> {
+    if (!steps.length || !checkpoints.length) throw new Error('llm-verifier: steps and checkpoints must not be empty')
+    for (const checkpoint of checkpoints) if (!Number.isSafeInteger(checkpoint) || checkpoint < 1 || checkpoint > steps.length) throw new Error('llm-verifier: each checkpoint must be an integer between 1 and steps.length')
+    const prompt = buildProgressPrompt(problem, steps, checkpoints)
+    const completions = await this.mapLimited(Array.from({ length: repeats }, (_, index) => index), async () => callVerifier(this.client, prompt, signal, images))
+    const stats = blankStats(); for (const completion of completions) { addUsage(stats, completion.usage); completion.scoringMode === 'top-logprobs' ? stats.topLogprobScores++ : stats.explicitTagScores++ }
+    const runs = completions.map(completion => checkpoints.map((_, index) => extractProgressScore(completion, '<c' + (index + 1) + '>')))
+    return { scores: checkpoints.map((_, index) => average(runs.map(run => run[index]!))), perRepeat: runs, calls: stats.calls, stats: this.finishStats(stats) }
+  }
+
+  async select(options: SelectOptions, signal?: AbortSignal): Promise<SelectResult> {
+    if (!options.candidates.length) throw new Error('llm-verifier: candidates must not be empty')
+    if (options.candidates.length === 1) return { index: 0, best: options.candidates[0]!, scores: [1], ranking: [0], pivots: [0], comparisons: 0, calls: 0, stats: blankStats() }
+    const ring = ringCycle(options.candidates.length, options.seed ?? 0); const ringScores = await this.scorePairs(options, ring, signal)
+    const firstWins = new Array<number>(options.candidates.length).fill(0); const firstCounts = new Array<number>(options.candidates.length).fill(0); accumulatePairs(ring, ringScores.rewards, firstWins, firstCounts)
+    const pivots = topPivots(firstWins, firstCounts, options.pivots ?? 2); const rounds = pivotRoundPairs(options.candidates.length, pivots); const roundScores = await this.scorePairs(options, rounds, signal)
+    const allRewards = new Map([...ringScores.rewards, ...roundScores.rewards]); const wins = new Array<number>(options.candidates.length).fill(0); const counts = new Array<number>(options.candidates.length).fill(0)
+    accumulatePairs(ring, allRewards, wins, counts); accumulatePairs(rounds, allRewards, wins, counts); const ranked = rankScores(wins, counts); const index = ranked[0]!.index
+    const stats = blankStats(); for (const source of [ringScores.stats, roundScores.stats]) { addUsage(stats, source); stats.cacheHits += source.cacheHits; stats.cacheMisses += source.cacheMisses; stats.topLogprobScores += source.topLogprobScores; stats.explicitTagScores += source.explicitTagScores }
+    return { index, best: options.candidates[index]!, scores: Array.from({ length: options.candidates.length }, (_, candidate) => wins[candidate]! / (counts[candidate] || 1)), ranking: ranked.map(value => value.index), pivots, comparisons: ring.length + rounds.length, calls: stats.calls, stats: this.finishStats(stats) }
+  }
+}
+
+export function normalizeCriteria(input: unknown): Criterion[] {
+  if (input === undefined) return DEFAULT_CRITERIA
+  if (!Array.isArray(input) || !input.length) throw new Error('llm-verifier: criteria must be a non-empty array')
+  return input.map((value, index) => { if (typeof value !== 'object' || value === null) throw new Error('llm-verifier: criteria[' + index + '] must be an object'); const row = value as Record<string, unknown>; for (const key of ['id', 'name', 'description']) if (typeof row[key] !== 'string' || row[key].trim().length === 0) throw new Error('llm-verifier: criteria[' + index + '].' + key + ' must be non-empty'); return { id: String(row.id), name: String(row.name), description: String(row.description) } })
+}
+export { DEFAULT_GROUND_TRUTH_NOTE }

package/src/images.ts

@@ -0,0 +1,33 @@
+import type { VerifierImage } from './caller.ts'
+
+const TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif'])
+const MAX_IMAGE_BYTES = 20 * 1024 * 1024
+
+function parseDataUrl(value: string): VerifierImage | undefined {
+  const match = /^data:(image\/(?:png|jpeg|webp|gif));base64,([A-Za-z0-9+/=\s]+)$/i.exec(value)
+  if (!match) return undefined
+  const data = Buffer.from(match[2]!.replace(/\s/g, ''), 'base64')
+  if (data.byteLength > MAX_IMAGE_BYTES) throw new Error('llm-verifier: image exceeds 20 MiB')
+  return { mediaType: match[1]!.toLowerCase() as VerifierImage['mediaType'], data }
+}
+
+export async function loadVerifierImages(inputs: readonly string[] | undefined, signal?: AbortSignal): Promise<VerifierImage[]> {
+  const images: VerifierImage[] = []
+  for (const input of inputs ?? []) {
+    const data = parseDataUrl(input)
+    if (data !== undefined) { images.push(data); continue }
+    let url: URL
+    try { url = new URL(input) } catch { throw new Error('llm-verifier: images accept only HTTPS URLs or data:image/...;base64 URLs') }
+    if (url.protocol !== 'https:') throw new Error('llm-verifier: remote images must use HTTPS')
+    const response = await fetch(url, { redirect: 'error', signal })
+    if (!response.ok) throw new Error('llm-verifier: image fetch returned HTTP ' + response.status)
+    const type = (response.headers.get('content-type') ?? '').split(';')[0]!.toLowerCase()
+    if (!TYPES.has(type)) throw new Error('llm-verifier: unsupported image media type ' + type)
+    const declared = Number(response.headers.get('content-length') ?? 0)
+    if (declared > MAX_IMAGE_BYTES) throw new Error('llm-verifier: image exceeds 20 MiB')
+    const bytes = new Uint8Array(await response.arrayBuffer())
+    if (bytes.byteLength > MAX_IMAGE_BYTES) throw new Error('llm-verifier: image exceeds 20 MiB')
+    images.push({ mediaType: type as VerifierImage['mediaType'], data: bytes })
+  }
+  return images
+}

package/src/index.ts

@@ -0,0 +1,48 @@
+import type { Context } from '@deepseek-ai/cordis'
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
+import { Config, installVerifierSettings, resolveConfig } from './config.ts'
+import { RequestLimiter } from './caller.ts'
+import { TopLogprobCapabilityCache } from './top-logprobs.ts'
+import { ScoreCache, resolveCacheFile } from './cache.ts'
+import { VerifierEngine, normalizeCriteria } from './engine.ts'
+import { loadVerifierImages } from './images.ts'
+import { extractSession } from './session.ts'
+
+export const name = 'llm-verifier'
+export const inject = ['tools', 'agents', 'attachments', 'llm']
+export { Config }
+export * from './core.ts'
+export * from './engine.ts'
+export * from './cache.ts'
+export { callVerifier, RequestLimiter, type VerifierClientConfig, type VerifierImage, type UsageStats, type VerifierCompletion } from './caller.ts'
+
+const criterionSchema = { type: 'object' as const, additionalProperties: false, properties: { id: { type: 'string' as const, required: true as const }, name: { type: 'string' as const, required: true as const }, description: { type: 'string' as const, required: true as const } } }
+const statsSchema = { type: 'object' as const, additionalProperties: false, properties: { calls: { type: 'integer' as const, required: true as const }, attempts: { type: 'integer' as const, required: true as const }, retries: { type: 'integer' as const, required: true as const }, inputTokens: { type: 'integer' as const, required: true as const }, cachedInputTokens: { type: 'integer' as const, required: true as const }, outputTokens: { type: 'integer' as const, required: true as const }, reasoningTokens: { type: 'integer' as const, required: true as const }, cacheHits: { type: 'integer' as const, required: true as const }, cacheMisses: { type: 'integer' as const, required: true as const }, estimatedCostUsd: { type: 'number' as const, required: true as const }, topLogprobScores: { type: 'integer' as const, required: true as const }, explicitTagScores: { type: 'integer' as const, required: true as const } } }
+const criterionResultSchema = { type: 'object' as const, additionalProperties: false, properties: { id: { type: 'string' as const, required: true as const }, name: { type: 'string' as const, required: true as const }, scoreA: { type: 'number' as const, required: true as const }, scoreB: { type: 'number' as const, required: true as const } } }
+const commonParams = { criteria: { type: 'array' as const, items: criterionSchema }, repeats: { type: 'integer' as const }, images: { type: 'array' as const, items: { type: 'string' as const }, description: 'Optional HTTPS or data:image/...;base64 images. The selected DSH model must accept image input.' } }
+function renderJson(value: unknown) { return [{ type: 'text' as const, text: JSON.stringify(value, null, 2) }] }
+function positive(value: number | undefined, fallback: number, field: string): number { const result = value ?? fallback; if (!Number.isSafeInteger(result) || result <= 0) throw new Error('llm-verifier: ' + field + ' must be a positive integer'); return result }
+
+export function apply(ctx: Context, config: Config = {}): void {
+  const entry = resolveConfig(config)
+  let limiter = new RequestLimiter(entry.maxConcurrency)
+  const current = installVerifierSettings(ctx, entry, () => { limiter = new RequestLimiter(current().maxConcurrency) })
+  const cache = new ScoreCache(resolveCacheFile(entry.cacheDir), entry.cacheMaxEntries)
+  const topLogprobCapabilities = new TopLogprobCapabilityCache()
+  const engine = async () => {
+    const selected = current()
+    await ctx.llm.resolveCallConfig({ provider: selected.provider, model: selected.model, ...(selected.reasoningEffort ? { reasoningEffort: selected.reasoningEffort as never } : {}), maxTokens: selected.maxTokens })
+    return { verifier: new VerifierEngine({ ...selected, ctx, llm: ctx.llm, attachments: ctx.attachments, topLogprobCapabilities, limiter }, selected.maxConcurrency, cache, { input: selected.estimatedInputUsdPerMillion, output: selected.estimatedOutputUsdPerMillion }), selected }
+  }
+  const images = (values: readonly string[] | undefined, signal: AbortSignal) => loadVerifierImages(values, signal)
+  const route = (selected: { provider: string; model: string }) => ({ provider: selected.provider, model: selected.model })
+
+  ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: args.problem, candidateA: args.candidate_a, candidateB: args.candidate_b, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) } } }))
+
+  ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Uses the configured DSH verifier model and the O(Nk) Probabilistic Pivot Tournament.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { const { verifier, selected } = await engine(); const result = await verifier.select({ problem: args.problem, candidates: args.candidates, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), pivots: positive(args.pivots, 2, 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) } } }))
+
+  ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Do not use for a single completed answer or invent checkpoints that were not supplied by the task history. Trusts observed output rather than narration.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { const { verifier, selected } = await engine(); const result = await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { ...result, ...route(selected) } } }))
+
+  ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Use autonomously near the end of a non-trivial coding, debugging, migration, deployment, or operations task when independent completion verification would materially reduce risk and the session contains real tool evidence. Do not use for routine conversation, simple factual answers, or every turn. Extracts the current DSH session, applies secret redaction, bounds and truncation, then sends the evidence to the verifier model selected in Settings.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { const agent = exec.agent ?? ctx.agents.currentInitiator(); if (agent === undefined) throw new Error('llm-verifier: verifier_current_session requires an agent-owned tool call'); const extracted = await extractSession(agent, async (ref: ImageAttachmentRef) => { const stored = await ctx.attachments.readImage(ref, exec.signal); return { data: stored.data, mediaType: stored.ref.mediaType } }, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars }); const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(args.repeats, 2, 'repeats'), images: extracted.images }, exec.signal); return { sessionId: extracted.sessionId, problem: extracted.problem, score: result.scoreA, baselineScore: result.scoreB, winner: result.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: result.calls, stats: result.stats, ...route(selected) } } }))
+}

package/src/parity.test.ts

@@ -0,0 +1,71 @@
+import { execFileSync } from 'node:child_process'
+import { existsSync, mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { ScoreCache } from './cache.ts'
+import { extractProgressScore, extractScore, pivotRoundPairs } from './core.ts'
+import { resolveConfig } from './config.ts'
+
+const PYTHON_ROOT = join(import.meta.dirname, '..', '..', 'llm-as-a-verifier')
+const hasPythonUpstream = existsSync(join(PYTHON_ROOT, 'llm_verifier'))
+
+function python(payload: unknown): unknown {
+  const script = [
+    'import json,sys',
+    'sys.path.insert(0, sys.argv[1])',
+    'from llm_verifier.fine_grained_reward import extract_score',
+    'from llm_verifier.pivot_tournament import pivot_round_pairs',
+    'd=json.loads(sys.stdin.read())',
+    "result = extract_score(d['text'], d['tokens'], d['positions'], d['tag']) if d['kind']=='score' else pivot_round_pairs(d['n'], d['pivots'])",
+    'print(json.dumps(result))',
+  ].join(';')
+  const source = payload as Record<string, unknown>
+  const compatible = source.kind === 'score' ? { ...source, positions: (source.positions as Array<Array<{ token: string; logprob: number }>>).map(position => position.map(item => [item.token, item.logprob])) } : source
+  const result = execFileSync('py', ['-3', '-c', script, PYTHON_ROOT], { input: JSON.stringify(compatible), encoding: 'utf8' })
+  return JSON.parse(result)
+}
+
+describe('Python parity fixtures', () => {
+  it.skipIf(!hasPythonUpstream)('matches literal and distribution score extraction', () => {
+    const fixtures = [
+      { text: '<score_A> A </score_A>', tokens: [], positions: [], tag: '<score_A>' },
+      { text: '<score_A> A </score_A> then <score_A> T </score_A>', tokens: [], positions: [], tag: '<score_A>' },
+      { text: '', tokens: ['<score_A>'], positions: [[], [{ token: 'A', logprob: Math.log(0.7) }, { token: 'T', logprob: Math.log(0.3) }]], tag: '<score_A>' },
+      { text: '', tokens: ['<score_A'], positions: [[], [{ token: '>B', logprob: Math.log(0.8) }, { token: '>S', logprob: Math.log(0.2) }]], tag: '<score_A>' },
+    ]
+    for (const fixture of fixtures) {
+      const ts = extractScore(fixture, fixture.tag)
+      const py = python({ kind: 'score', ...fixture }) as number
+      expect(ts).toBeCloseTo(py, 12)
+    }
+  })
+  it.skipIf(!hasPythonUpstream)('matches pivot pair generation', () => {
+    expect(pivotRoundPairs(7, [1, 4, 5])).toEqual(python({ kind: 'pivot', n: 7, pivots: [1, 4, 5] }))
+  })
+  it('matches reversed progress convention', () => {
+    expect(extractProgressScore({ text: '<c1> T </c1>', tokens: [], positions: [] }, '<c1>')).toBe(1)
+  })
+})
+
+describe('DSH model routing and cache', () => {
+  it('accepts any non-empty DSH provider and model route', () => {
+    expect(resolveConfig({ provider: 'openai', model: 'gpt-5' })).toMatchObject({ provider: 'openai', model: 'gpt-5' })
+    expect(resolveConfig({ provider: 'anthropic', model: 'claude-sonnet' })).toMatchObject({ provider: 'anthropic', model: 'claude-sonnet' })
+    expect(() => resolveConfig({ provider: '', model: 'gpt-5' })).toThrow(/provider must be non-empty/)
+  })
+  it('persists successful results and avoids duplicate creation', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-'))
+    try {
+      const file = join(dir, 'scores.json')
+      const first = new ScoreCache(file, 100)
+      let creates = 0
+      const create = async () => { creates++; return { scoreA: 1, scoreB: 0, usage: { calls: 1, attempts: 1, retries: 0, inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, reasoningTokens: 0 }, scoringMode: 'top-logprobs' as const, createdAt: Date.now() } }
+      expect((await first.getOrCreate('key', create)).hit).toBe(false)
+      expect((await first.getOrCreate('key', create)).hit).toBe(true)
+      const second = new ScoreCache(file, 100)
+      expect((await second.getOrCreate('key', create)).hit).toBe(true)
+      expect(creates).toBe(1)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+})

package/src/session.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'vitest'
+import { Session } from '@deepseek-ai/dsh-session'
+import { createUserMessage, createAssistantMessage, createToolResultMessage } from '@deepseek-ai/dsh-llm'
+import { extractSession } from './session.ts'
+
+describe('current session extraction', () => {
+  it('keeps direct evidence, skips plugin instructions, and redacts secrets', async () => {
+    const session = Session.create('session-00000000-0000-4000-8000-000000000001' as never)
+    session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Fix task token=abc123' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
+    session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'hidden plugin instruction' }], source: { kind: 'plugin', plugin: 'test' } }), { surfaceOp: 'append' })
+    session.append('assistant/message', { turn: 1, step: 1, message: createAssistantMessage({ content: [{ type: 'text', text: 'running checks' }], source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' } }) }, { surfaceOp: 'append' })
+    session.append('tool/call', { turn: 1, step: 1, callId: 'call-1' as never, name: 'pwsh', arguments: '{"command":"test"}' })
+    session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'call-1' as never, content: [{ type: 'text', text: 'exit 0 password=hunter2' }], isError: false }) }, { surfaceOp: 'append' })
+    const agent = { id: session.id, session } as never
+    const result = await extractSession(agent, async () => { throw new Error('no image expected') })
+    expect(result.problem).toContain('Fix task')
+    expect(result.problem).toContain('[REDACTED]')
+    expect(result.trace).not.toContain('hidden plugin instruction')
+    expect(result.trace).toContain('exit 0')
+    expect(result.trace).not.toContain('hunter2')
+  })
+})