dsh-llm-verifier 0.3.0 → 0.4.0

package/src/core.ts

@@ -54,8 +54,119 @@ export const DEFAULT_CRITERIA: Criterion[] = [
   },
 ]
 
+/** Task classes a rubric can be chosen for; `custom` lives at the config layer, not here. */
+export const CRITERIA_PRESET_IDS = ['coding', 'debug', 'research', 'ops', 'writing'] as const
+export type CriteriaPresetId = typeof CRITERIA_PRESET_IDS[number]
+
+/**
+ * Rubric per task class.
+ *
+ * Upstream ships one criteria file per benchmark (`criteria/swe_bench.md`, `terminal_bench.md`,
+ * `medagentbench.md`) and its TEMPLATE states the rule this table follows: 2-4 narrow criteria
+ * beat one broad one. Judging a research answer with "Output Match"/"Error Signal Detection"
+ * measures the wrong thing, and the automatic gate has no per-task override.
+ *
+ * `coding` is byte-identical to {@link DEFAULT_CRITERIA}: the default preset must not change
+ * any existing verdict, prompt or cache key.
+ */
+export const CRITERIA_PRESETS: Record<CriteriaPresetId, Criterion[]> = {
+  coding: DEFAULT_CRITERIA,
+  debug: [
+    {
+      id: 'reproduction',
+      name: 'Failure Reproduction',
+      description: 'Did the agent reproduce the reported failure BEFORE changing code? Look for a command or test whose observed output shows the failure happening. Penalize edits made without any reproduction, and treat "the user said it is broken" as no evidence.',
+    },
+    {
+      id: 'root_cause',
+      name: 'Root Cause',
+      description: 'Compare the stated cause with the evidence: does the diagnosis point at code that the observed output actually implicates, rather than at the last error message? Penalize symptom patches and guesses presented as findings.',
+    },
+    {
+      id: 'fix_verification',
+      name: 'Fix Verification',
+      description: 'Find the command that exercised the fix AFTER the last code change. Reward only observed output showing the previously failing case now passing with no new failures. Penalize "should be fixed" assertions and fixes verified before the final edit.',
+    },
+  ],
+  research: [
+    {
+      id: 'question_addressed',
+      name: 'Question Addressed',
+      description: 'Does the answer address exactly what was asked, including every part of a multi-part question? Penalize thorough answers to a nearby but different question, and unanswered sub-questions.',
+    },
+    {
+      id: 'source_grounding',
+      name: 'Source Grounding',
+      description: 'Is every material claim traceable to evidence the answer names (file, URL, command output)? Reward claims tied to a specific source; penalize confident claims with no traceable basis and citations that do not actually support the claim.',
+    },
+    {
+      id: 'limits_stated',
+      name: 'Limits Stated',
+      description: 'Does the answer separate what was verified from what is inferred, and state assumptions, uncertainty and missing data? Penalize unqualified certainty that goes beyond the observed evidence.',
+    },
+  ],
+  ops: [
+    {
+      id: 'change_specification',
+      name: 'Change Specification',
+      description: 'Compare the executed commands with the requested operation: target, environment, arguments and scope. Penalize actions against the wrong target, and side effects beyond the requested scope.',
+    },
+    {
+      id: 'observed_result',
+      name: 'Observed Result',
+      description: 'Reward commands whose observed output shows the intended state (service up, file present, config applied). Penalize inferring success from an exit code without inspecting the resulting state.',
+    },
+    {
+      id: 'reversibility',
+      name: 'Reversibility',
+      description: 'Does the work leave a way back: a backup, a recorded previous value, a dry run first, or a stated rollback path? Penalize irreversible changes made without one.',
+    },
+  ],
+  writing: [
+    {
+      id: 'brief_adherence',
+      name: 'Brief Adherence',
+      description: 'Check the requested deliverable: format, length, audience and every explicit constraint. Penalize a well-written piece that answers a different brief.',
+    },
+    {
+      id: 'structure_clarity',
+      name: 'Structure And Clarity',
+      description: 'Judge whether the structure carries the argument: ordering, sections, and a point the reader can follow. Penalize padding, repetition and unsupported assertions used as filler.',
+    },
+    {
+      id: 'factual_grounding',
+      name: 'Factual Grounding',
+      description: 'Are factual statements supported by the material the task supplied, or invented? Penalize fabricated specifics such as names, numbers, dates and quotes that do not appear in the evidence.',
+    },
+  ],
+}
+
+/** Derive a criterion id from free text: lowercase, alphanumerics and underscores, max 40 chars. */
+export function slugCriterionId(text: string): string {
+  const slug = text.toLowerCase().replace(/[^a-z0-9]+/gu, '_').replace(/^_+|_+$/gu, '')
+  return slug.slice(0, 40).replace(/_+$/gu, '') || 'criterion'
+}
+
+/** Make an id unique against the ids already used, by appending _2, _3, ... */
+export function dedupeCriterionId(id: string, seen: Set<string>): string {
+  let candidate = id
+  let suffix = 1
+  while (seen.has(candidate)) { suffix += 1; candidate = id + '_' + suffix }
+  seen.add(candidate)
+  return candidate
+}
+
 export const DEFAULT_GROUND_TRUTH_NOTE = "**IMPORTANT:** Focus on observed tool and terminal output as ground truth. Do NOT trust the agent's self-assessment or claims of success."
 
+/**
+ * The fixed baseline a session acceptance measures itself against.
+ *
+ * One definition, read by the automatic final gate and by `verifier_best_of_n`. The tool
+ * advertises an "absolute" score with the gate's own threshold, so if these two ever drifted
+ * apart the tool would be measuring against a different reference than the gate it predicts.
+ */
+export const EMPTY_WORK_BASELINE = '(No useful work or verification was performed.)'
+
 /**
  * Deterministic per-prompt delimiter token.
  *
@@ -166,6 +277,23 @@ export function extractScore(completion: CompletionLogprobs, tag: string): numbe
   return (letterValue(letter) - 1) / (GRANULARITY - 1)
 }
 
+/**
+ * One pairwise prompt focused on a single criterion.
+ *
+ * Everything that does not depend on the criterion (task, both trajectories, the rating
+ * scale) comes first and ONLY the criterion varies at the tail. That is not cosmetic:
+ * it maximizes the shared prompt prefix across the criteria of one comparison, so a
+ * prefix-caching backend serves the trace-heavy body from cache. Upstream documents the
+ * same constraint on its `build_prompt` ("Keep criterion-specific text strictly at the
+ * end when editing"), and `VerifierEngine.compare` warms the prefix with one job before
+ * fanning out the rest. Keep it that way.
+ * @param problem - task statement shown to the judge.
+ * @param traceA - candidate A's trajectory.
+ * @param traceB - candidate B's trajectory.
+ * @param criterion - the single criterion this call scores.
+ * @param groundTruthNote - note prepended to every judge prompt.
+ * @returns The rendered prompt.
+ */
 export function buildPairwisePrompt(problem: string, traceA: string, traceB: string, criterion: Criterion, groundTruthNote = DEFAULT_GROUND_TRUTH_NOTE): string {
   const token = evidenceNonce(problem, traceA, traceB)
   return [
@@ -193,13 +321,39 @@ export function buildProgressPrompt(problem: string, steps: readonly string[], c
     '**Task instruction:**\n' + renderDelimitedBlock('TASK', token, problem.trim()),
     '**Agent trajectory (' + steps.length + ' agent steps):**\n' + renderDelimitedBlock('AGENT_TRACE', token, trajectory),
     "Score whether the agent's CURRENT state at each checkpoint would actually satisfy the task's hidden grader.",
-    'Use A through T where A = certainly NO, H-M = uncertain, N-S = leans YES, and T = essentially certain YES with matching observed verification.',
-    'Effort and confident narration are not evidence. A state without real verification should not exceed K. Scores may decrease after regressions.',
+    'Use the 20-letter A-T scale: A = certainly NO (nothing useful done yet); B-G = leans NO (partial work, key pieces missing or broken); H-M = uncertain (a plausible solution is taking shape, but no convincing verification yet); N-S = leans YES (the right artifacts appear to be in place and partial verification has worked, with minor concerns); T = essentially certain YES (the relevant verification ran and its observed output literally matches what the task calls for, with no outstanding errors).',
+    "Calibration: effort, exploration, step count and confident narration are NOT progress — an agent that ran twenty commands without producing the right output deserves a score near A. Default to skepticism: a result with no real verification step should not exceed ~K, and even a verified-looking one should rarely exceed ~R unless the verification clearly matches the task's stated success criterion. Treat the agent's prose declarations (\"done\", \"all tests pass\") as ZERO evidence, and ground the score in the actions and the observed output you can see.",
+    'Successive checkpoints do not have to rise: a trajectory committed to a wrong approach should plateau, and a regression should score lower than the state it broke.',
     'The checkpoints are:\n' + checkpoints.map((step, index) => '  Checkpoint ' + (index + 1) + ' = state right after Agent Step ' + step).join('\n'),
     'Output EXACTLY these lines and nothing else:\n' + tags,
   ].join('\n\n')
 }
 
+/**
+ * One best-of-N drafting prompt.
+ *
+ * Deliberately NOT the judge contract: here the request IS the instruction to follow and the
+ * answer is free-form work product, so there is no data-only delimiter and no A–T verdict tag.
+ * The untrusted-content boundary is re-established downstream instead — every draft is embedded
+ * as delimited, nonce-terminated evidence when the judges score it.
+ *
+ * Only the final line varies between the N drafts, so the shared prompt prefix (which carries
+ * the whole request) is maximal. Same tail-only convention as `buildPairwisePrompt`'s
+ * criterion, and for the same prefix-caching reason.
+ * @param task - the request the drafts must answer.
+ * @param index - 0-based draft index.
+ * @param total - how many drafts are being generated.
+ * @returns The rendered prompt.
+ */
+export function buildGenerationPrompt(task: string, index: number, total: number): string {
+  return [
+    'You are one of ' + total + ' independent experts asked to answer the same request. Answer it yourself, completely and concretely, in the form the request calls for — code, patch, plan, or prose.',
+    'Do not ask clarifying questions, do not defer the work to a later step, and do not refer to the other attempts. Where the answer depends on a command, a test, or other evidence, state exactly what it is and what output would prove it; never claim a verification you cannot describe.',
+    '**Request:**\n' + task.trim(),
+    'Draft ' + (index + 1) + ' of ' + total + '.',
+  ].join('\n\n')
+}
+
 /** Progress uses A=NO..T=YES, the reverse of pairwise success scoring. */
 export function extractProgressScore(completion: CompletionLogprobs, tag: string): number {
   return 1 - extractScore(completion, tag)
@@ -265,3 +419,64 @@ export function rankScores(wins: readonly number[], counts: readonly number[]):
   return Array.from({ length: wins.length }, (_, index) => ({ index, score: (wins[index] ?? 0) / (counts[index] || 1) }))
     .sort((a, b) => b.score - a.score || a.index - b.index)
 }
+
+/** Optional trailing `{#id}` anchor on a criterion heading, e.g. `### Root Cause {#cause}`. */
+const CRITERION_ID_ANCHOR = /^(.*?)\s*\{#([A-Za-z0-9_-]+)\}\s*$/
+const HTML_COMMENT = /<!--[\s\S]*?-->/gu
+
+/**
+ * Parse a criteria file (or an inline markdown string) into a ground-truth note and criteria.
+ *
+ * Upstream's criteria files are the reason this exists: a rubric you can edit, review and
+ * version beats a hard-coded list, and its TEMPLATE pins the layout. Format:
+ *
+ *     # <title>                        (ignored)
+ *     ## Ground Truth Note             (optional)
+ *     <one paragraph the verifier always sees>
+ *     ## Criteria
+ *     ### <Criterion Name> {#id}       (the anchor is optional; the id is slugged from the name)
+ *     <instruction for this criterion>
+ *
+ * HTML comments are stripped, so a file can carry author notes the judge never sees. The
+ * parser fails closed: no criteria, a heading without a body, or a blank instruction throws
+ * rather than silently scoring with fewer criteria than authored.
+ * @param text - criteria markdown.
+ * @returns The optional ground-truth note and the parsed criteria, in file order.
+ */
+export function parseCriteriaMarkdown(text: string): { groundTruthNote: string; criteria: Criterion[] } {
+  const lines = text.replace(HTML_COMMENT, '').split(/\r?\n/u)
+  const criteria: Criterion[] = []
+  const seen = new Set<string>()
+  let groundTruthNote = ''
+  let section: 'ground_truth' | 'criteria' | undefined
+  let current: { name: string; id: string } | undefined
+  let buffer: string[] = []
+  const flush = () => {
+    const body = buffer.join('\n').trim()
+    if (section === 'ground_truth') { if (!groundTruthNote) groundTruthNote = body }
+    else if (current !== undefined) { criteria.push({ id: current.id, name: current.name, description: body }); current = undefined }
+    buffer = []
+  }
+  for (const line of lines) {
+    if (line.startsWith('## ') && !line.startsWith('### ')) {
+      flush()
+      const heading = line.slice(3).trim().toLowerCase()
+      section = heading.includes('ground truth') ? 'ground_truth' : heading.includes('criteri') ? 'criteria' : undefined
+    } else if (line.startsWith('### ') && section === 'criteria') {
+      flush()
+      const heading = line.slice(4).trim()
+      const anchored = CRITERION_ID_ANCHOR.exec(heading)
+      const name = (anchored?.[1] ?? heading).trim()
+      current = { name, id: dedupeCriterionId(slugCriterionId(anchored?.[2] ?? name), seen) }
+    } else if (line.startsWith('# ')) {
+      continue
+    } else {
+      buffer.push(line)
+    }
+  }
+  flush()
+  if (criteria.length === 0) throw new Error('llm-verifier: criteria file has no criteria — add a "## Criteria" section with one "### Criterion Name" heading per criterion')
+  const empty = criteria.filter(criterion => criterion.description.length === 0).map(criterion => criterion.id)
+  if (empty.length > 0) throw new Error('llm-verifier: criteria file has criteria with no instruction: ' + empty.join(', '))
+  return { groundTruthNote, criteria }
+}

package/src/criteria.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from 'vitest'
+import { CRITERIA_PRESETS, DEFAULT_CRITERIA } from './core.ts'
+import { CriteriaResolver } from './criteria.ts'
+
+const FILE = ['# Rubric', '## Ground Truth Note', '', 'Trust raw output only.', '', '## Criteria', '', '### Alpha', '', 'Judge alpha carefully.', '', '### Beta {#b2}', '', 'Judge beta carefully.', ''].join('\n')
+
+describe('CriteriaResolver', () => {
+  it('returns the bundled preset by reference, so switching presets changes the rubric', async () => {
+    const resolver = new CriteriaResolver()
+    const coding = await resolver.resolve('coding', undefined)
+    expect(coding.criteria).toBe(DEFAULT_CRITERIA)
+    expect(coding.source).toBe('coding')
+    const research = await resolver.resolve('research', undefined)
+    expect(research.criteria).toBe(CRITERIA_PRESETS.research)
+    expect(research.error).toBeUndefined()
+  })
+
+  it('reads a custom markdown file and carries its ground-truth note', async () => {
+    const resolver = new CriteriaResolver(async () => FILE)
+    const resolved = await resolver.resolve('custom', 'rubric.md')
+    expect(resolved.source).toBe('custom')
+    expect(resolved.groundTruthNote).toBe('Trust raw output only.')
+    expect(resolved.criteria.map(criterion => criterion.id)).toEqual(['alpha', 'b2'])
+  })
+
+  it('falls back to the coding rubric instead of failing the gate when the file is unusable', async () => {
+    const missing = new CriteriaResolver(async () => { throw new Error('ENOENT: no such file') })
+    const resolved = await missing.resolve('custom', 'nope.md')
+    expect(resolved.source).toBe('fallback')
+    expect(resolved.criteria).toBe(DEFAULT_CRITERIA)
+    expect(resolved.error).toMatch(/ENOENT/u)
+    expect(resolved.file).toBe('nope.md')
+
+    const unparsable = new CriteriaResolver(async () => '# no criteria in here')
+    const broken = await unparsable.resolve('custom', 'broken.md')
+    expect(broken.source).toBe('fallback')
+    expect(broken.error).toMatch(/no criteria/u)
+  })
+
+  it('reports a missing file path rather than reading an empty one', async () => {
+    const resolver = new CriteriaResolver(async () => FILE)
+    const resolved = await resolver.resolve('custom', '   ')
+    expect(resolved.source).toBe('fallback')
+    expect(resolved.error).toMatch(/no criteria file configured/u)
+  })
+
+  it('re-reads an edited file, and caches an unchanged one', async () => {
+    let text = FILE
+    let reads = 0
+    const resolver = new CriteriaResolver(async () => { reads += 1; return text })
+    const first = await resolver.resolve('custom', 'rubric.md')
+    expect(first.criteria.map(criterion => criterion.id)).toEqual(['alpha', 'b2'])
+    // Same content: the parse is reused, but the file is re-read so an edit is never missed.
+    await resolver.resolve('custom', 'rubric.md')
+    expect(reads).toBe(2)
+    text = FILE.replace('### Alpha', '### Gamma')
+    const third = await resolver.resolve('custom', 'rubric.md')
+    expect(third.criteria.map(criterion => criterion.id)).toEqual(['gamma', 'b2'])
+  })
+})

package/src/criteria.ts

@@ -0,0 +1,76 @@
+import { readFile } from 'node:fs/promises'
+import { CRITERIA_PRESETS, DEFAULT_CRITERIA, parseCriteriaMarkdown, type Criterion, type CriteriaPresetId } from './core.ts'
+
+/** Everything the settings page can select: a bundled task-class preset, or a markdown file. */
+export type CriteriaSelection = CriteriaPresetId | 'custom'
+
+export interface ResolvedCriteria {
+  criteria: Criterion[]
+  /** Note prepended to every judge prompt, when the rubric carries one. */
+  groundTruthNote?: string
+  /** The preset id, `custom`, or `fallback` when a custom file could not be used. */
+  source: string
+  file?: string
+  /** Why the custom file was rejected; the `coding` rubric is used instead of failing the gate. */
+  error?: string
+}
+
+/**
+ * Resolve the configured rubric to the criteria the engine scores with.
+ *
+ * A custom markdown file is read on every resolve (a rubric file is a few kilobytes and it is
+ * resolved once per turn boundary), so editing it takes effect immediately. The parse is cached
+ * against the file text, so repeated resolves of an unchanged file cost one read.
+ *
+ * A broken custom file does NOT fail verification: a rubric typo must not disable the acceptance
+ * gate, so the coding preset is used and the reason travels in {@link ResolvedCriteria.error}
+ * for the settings page and the log. This mirrors the plugin's rule that scoring-channel
+ * capability problems degrade instead of aborting a verification.
+ */
+export class CriteriaResolver {
+  private cache: { key: string; value: ResolvedCriteria } | undefined
+  constructor(private readonly load: (path: string) => Promise<string> = path => readFile(path, 'utf8')) {}
+
+  async resolve(selection: CriteriaSelection, file: string | undefined): Promise<ResolvedCriteria> {
+    if (selection !== 'custom') {
+      const preset = CRITERIA_PRESETS[selection]
+      return { criteria: preset ?? DEFAULT_CRITERIA, source: preset ? selection : 'fallback', ...(preset ? {} : { error: 'unknown criteria preset: ' + String(selection) }) }
+    }
+    const path = (file ?? '').trim()
+    let text: string
+    try {
+      if (!path) throw new Error('no criteria file configured (set criteriaFile, or switch criteriaPreset away from custom)')
+      text = await this.load(path)
+    } catch (error) {
+      return this.remember('custom\u0000' + path + '\u0000!' + String(error), {
+        criteria: DEFAULT_CRITERIA,
+        source: 'fallback',
+        ...(path ? { file: path } : {}),
+        error: error instanceof Error ? error.message : String(error),
+      })
+    }
+    const key = 'custom\u0000' + path + '\u0000' + text
+    if (this.cache?.key === key) return this.cache.value
+    try {
+      const parsed = parseCriteriaMarkdown(text)
+      return this.remember(key, {
+        criteria: parsed.criteria,
+        ...(parsed.groundTruthNote ? { groundTruthNote: parsed.groundTruthNote } : {}),
+        source: 'custom',
+        file: path,
+      })
+    } catch (error) {
+      return this.remember(key + '\u0000!' + String(error), {
+        criteria: DEFAULT_CRITERIA,
+        source: 'fallback',
+        file: path,
+        error: error instanceof Error ? error.message : String(error),
+      })
+    }
+  }
+
+  private remember(key: string, value: ResolvedCriteria): ResolvedCriteria {
+    this.cache = { key, value }
+    return value
+  }
+}

package/src/decisions.test.ts

@@ -0,0 +1,150 @@
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { describe, expect, it } from 'vitest'
+import { DecisionStore, boundDecisionCalls, resolveDecisionsFile } from './decisions.ts'
+
+const call = (overrides: Partial<{ label: string; channel: string; prompt: string; output: string; score?: number }> = {}) => ({
+  label: 'Specification Adherence repeat 1',
+  channel: 'explicit-tag',
+  prompt: 'score this',
+  output: '<score_A> K </score_A>',
+  ...overrides,
+})
+
+describe('boundDecisionCalls', () => {
+  it('keeps a small capture intact', () => {
+    const bounded = boundDecisionCalls([call({ score: 0.5263 })])
+    expect(bounded).toHaveLength(1)
+    expect(bounded[0]!.score).toBe(0.5263)
+    expect(bounded[0]!.channel).toBe('explicit-tag')
+  })
+
+  it('redacts and truncates every captured field', () => {
+    const bounded = boundDecisionCalls([call({ prompt: 'Authorization: Bearer abcdefghijklmnop ' + 'p'.repeat(20000), output: 'o'.repeat(9000) })])
+    expect(bounded[0]!.prompt).not.toContain('abcdefghijklmnop')
+    expect(bounded[0]!.prompt.length).toBeLessThanOrEqual(8000)
+    expect(bounded[0]!.output.length).toBeLessThanOrEqual(4000)
+  })
+
+  it('bounds a record by call count and by the per-record character budget', () => {
+    // 40 calls are capped at MAX_CALLS = 32, and the equal per-call share keeps every one of
+    // them inside the character budget rather than dropping the tail.
+    const many = Array.from({ length: 40 }, (_, index) => call({ label: 'c' + index, prompt: 'p'.repeat(6000), output: 'o'.repeat(3000) }))
+    const bounded = boundDecisionCalls(many)
+    expect(bounded).toHaveLength(32)
+    const total = bounded.reduce((sum, value) => sum + value.prompt.length + value.output.length + value.label.length + value.channel.length, 0)
+    expect(total).toBeLessThanOrEqual(30000)
+    // The first call is always kept, so a snapshot never comes back empty.
+    expect(bounded[0]!.label).toBe('c0')
+  })
+
+  it('keeps the drafts and the tournament of a 27-call best-of-N snapshot', () => {
+    // Regression: with MAX_CALLS = 12 and judge labels sorting BEFORE the draft labels, every
+    // draft was cut and the snapshot could not show what the judges had chosen between.
+    const calls = [
+      ...Array.from({ length: 3 }, (_, index) => call({ label: 'draft ' + (index + 1), prompt: 'draft prompt ' + index, output: 'DRAFT-' + index + ' ' + 'd'.repeat(3000) })),
+      ...Array.from({ length: 18 }, (_, index) => call({ label: 'criterion ' + (index % 3) + ' repeat ' + (index % 2 + 1) + ' #' + index, prompt: 'p'.repeat(6000), output: 'o'.repeat(3000) })),
+      ...Array.from({ length: 6 }, (_, index) => call({ label: 'baseline ' + index, prompt: 'b'.repeat(6000), output: 'o'.repeat(3000) })),
+    ]
+    const bounded = boundDecisionCalls(calls)
+    expect(bounded).toHaveLength(27)
+    for (const index of [0, 1, 2]) expect(bounded.some(value => value.label === 'draft ' + (index + 1))).toBe(true)
+    const total = bounded.reduce((sum, value) => sum + value.prompt.length + value.output.length + value.label.length + value.channel.length, 0)
+    expect(total).toBeLessThanOrEqual(30000)
+  })
+
+  it('keeps every call of a six-call session acceptance instead of the first three', () => {
+    // Regression: a fixed 8000-char prompt window meant 30000 / 8000 drove the loop to break
+    // after three calls, and only the calls that finished first were stored — the acceptance
+    // path, the plugin's most expensive decision, lost half its criteria.
+    const acceptance = Array.from({ length: 6 }, (_, index) => call({
+      label: 'criterion ' + Math.floor(index / 2) + ' repeat ' + (index % 2 + 1),
+      prompt: 'HEAD\n' + 'p'.repeat(120000) + '\nTAIL',
+      output: '<score_A> K </score_A>\n' + 'o'.repeat(900),
+    }))
+    const bounded = boundDecisionCalls(acceptance)
+    expect(bounded).toHaveLength(6)
+    expect(new Set(bounded.map(value => value.label)).size).toBe(6)
+    // Both ends of the (much shorter) window still survive.
+    expect(bounded[0]!.prompt.startsWith('HEAD')).toBe(true)
+    expect(bounded[0]!.prompt.endsWith('TAIL')).toBe(true)
+    const total = bounded.reduce((sum, value) => sum + value.prompt.length + value.output.length + value.label.length + value.channel.length, 0)
+    expect(total).toBeLessThanOrEqual(30000)
+  })
+
+  it('ignores a non-finite score instead of persisting NaN', () => {
+    const bounded = boundDecisionCalls([call({ score: Number.NaN })])
+    expect(bounded[0]!.score).toBeUndefined()
+  })
+})
+
+describe('boundCaptureText', () => {
+  it('keeps both ends of an over-long prompt instead of only the head', () => {
+    // A session-acceptance prompt runs past 100k characters: head-only truncation saved
+    // the instructions and dropped the trajectory tail, i.e. the part the judge graded.
+    const prompt = 'HEAD-MARKER\n' + 'x'.repeat(30000) + '\nTAIL-MARKER'
+    const kept = boundDecisionCalls([call({ prompt })])[0]!.prompt
+    expect(kept.length).toBeLessThanOrEqual(8000)
+    expect(kept.startsWith('HEAD-MARKER')).toBe(true)
+    expect(kept.endsWith('TAIL-MARKER')).toBe(true)
+    expect(kept).toContain('characters omitted')
+  })
+
+  it('leaves a prompt that already fits untouched', () => {
+    const kept = boundDecisionCalls([call({ prompt: 'short prompt' })])[0]!.prompt
+    expect(kept).toBe('short prompt')
+    expect(kept).not.toContain('omitted')
+  })
+})
+
+describe('DecisionStore', () => {
+  it('stores a snapshot and finds it again from a fresh instance', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-decisions-'))
+    const file = join(root, 'decisions.json')
+    const store = new DecisionStore(file)
+    const record = await store.record({ toolName: 'verifier_track', phase: 'track', startedAt: 1, provider: 'p', model: 'm', calls: [call()] })
+    expect(record?.calls).toHaveLength(1)
+    // Survives a restart: the second store loads the same file.
+    const reopened = new DecisionStore(file)
+    expect(await reopened.find(record!.id)).toMatchObject({ toolName: 'verifier_track', phase: 'track' })
+    const document = JSON.parse(await readFile(file, 'utf8'))
+    expect(document.version).toBe(1)
+    expect(document.records).toHaveLength(1)
+  })
+
+  it('writes nothing when no call was captured', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-decisions-'))
+    const file = join(root, 'decisions.json')
+    const store = new DecisionStore(file)
+    expect(await store.record({ toolName: 'verifier_track', phase: 'track', startedAt: 1, provider: 'p', model: 'm', calls: [] })).toBeUndefined()
+    await expect(readFile(file, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' })
+  })
+
+  it('keeps only the newest records and returns undefined for a pruned id', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-decisions-'))
+    const file = join(root, 'decisions.json')
+    const store = new DecisionStore(file, 2)
+    const first = await store.record({ toolName: 'a', phase: 'explicit', startedAt: 1, provider: 'p', model: 'm', calls: [call()] })
+    const second = await store.record({ toolName: 'b', phase: 'explicit', startedAt: 2, provider: 'p', model: 'm', calls: [call()] })
+    const third = await store.record({ toolName: 'c', phase: 'explicit', startedAt: 3, provider: 'p', model: 'm', calls: [call()] })
+    expect(await store.find(first!.id)).toBeUndefined()
+    expect(await store.find(second!.id)).toMatchObject({ toolName: 'b' })
+    expect(await store.find(third!.id)).toMatchObject({ toolName: 'c' })
+    const document = JSON.parse(await readFile(file, 'utf8'))
+    expect(document.records.map((row: { toolName: string }) => row.toolName)).toEqual(['b', 'c'])
+  })
+
+  it('skips a corrupt document instead of failing the invocation', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-decisions-'))
+    const file = join(root, 'decisions.json')
+    await writeFile(file, JSON.stringify({ version: 1, records: [{ id: 5 }, { id: 'ok', toolName: 'a', startedAt: 1, calls: [] }] }), 'utf8')
+    const store = new DecisionStore(file)
+    await store.load()
+    expect(await store.find('ok')).toMatchObject({ toolName: 'a' })
+  })
+
+  it('resolves the snapshot file beside the statistics of the same topic', () => {
+    expect(resolveDecisionsFile(join('topic', 'verifier', 'scores-v1.json'))).toBe(join('topic', 'verifier', 'decisions-v1.json'))
+  })
+})