dsh-llm-verifier 0.2.1 → 0.3.1

package/src/router.ts

@@ -98,6 +98,23 @@ function blockText(blocks: readonly ContentBlock[]): string {
   return parts.join('\n').trim()
 }
 
+/**
+ * Text of one assistant turn, excluding reasoning blocks.
+ *
+ * Used only as the newest checkpoint's narration. Prose is not evidence, but for a
+ * deliverable that lives in prose (a review, an analysis) it is the only thing that
+ * describes the current state at all, so the judge receives it explicitly labelled
+ * as a claim instead of being shown nothing about the deliverable.
+ * @param blocks - content blocks of an `assistant/message` event.
+ * @returns The joined text, trimmed.
+ */
+function narrativeText(blocks: readonly ContentBlock[]): string {
+  const parts: string[] = []
+  const visit = (items: readonly ContentBlock[]) => { for (const block of items) { if (block.type === 'text') parts.push(block.text); else if (block.type === 'tool-result') visit(block.content) } }
+  visit(blocks)
+  return parts.join('\n').trim()
+}
+
 function successful(event: SessionEvent<'tool/result'>): boolean {
   return event.data.error === undefined && event.data.message.content.every(block => block.isError !== true)
 }
@@ -129,6 +146,8 @@ interface EvidenceIndex {
   calls: Map<string, EvidenceCall>
   todos: Map<number, TodoItem[]>
   teamTasks: Map<number, TeamTaskItem[]>
+  /** Newest assistant prose in the task; attached to the current checkpoint as a claim, never as evidence. */
+  narration?: { seq: number; text: string }
 }
 
 export function buildEvidenceIndex(events: readonly SessionEvent[]): EvidenceIndex | undefined {
@@ -143,6 +162,12 @@ export function buildEvidenceIndex(events: readonly SessionEvent[]): EvidenceInd
 
   // Track the rolling view of team tasks across the session
   const currentTeamTasks = new Map<string, TeamTaskItem>()
+  let narration: { seq: number; text: string } | undefined
+  // Tool names each wrapper call (a `run_code` program) dispatched, keyed by the
+  // wrapper's callId. Session V3 carries rootCallId on the dispatch event, which is the
+  // only reliable parent link: the wrapper's own result text is a concatenation of its
+  // sub-results, so the tool name of the wrapper says nothing about what it did.
+  const dispatchedTools = new Map<string, Set<string>>()
 
   for (const rawEvent of relevant) {
     const event = rawEvent as unknown as { type: string; seq: number; data: any }
@@ -156,8 +181,21 @@ export function buildEvidenceIndex(events: readonly SessionEvent[]): EvidenceInd
         teamTasks.set(event.seq, [...currentTeamTasks.values()])
       }
     }
+    else if (event.type === 'assistant/message') {
+      // Session V3 carries the blocks under message.content (same shape as tool/result);
+      // the bare content fallback keeps older/mock event streams working.
+      const blocks = event.data?.message?.content ?? event.data?.content
+      const text = narrativeText(Array.isArray(blocks) ? blocks : [])
+      if (text) narration = { seq: event.seq, text }
+    }
     else if (event.type === 'tool/ptc-dispatch' || event.type === 'tool/code-dispatch') {
-      const data = event.data as { subCallId?: string; name: string; isError?: boolean; content?: readonly ContentBlock[] }
+      const data = event.data as { rootCallId?: string; subCallId?: string; name: string; isError?: boolean; content?: readonly ContentBlock[] }
+      const rootCallId = typeof data.rootCallId === 'string' && data.rootCallId ? data.rootCallId : undefined
+      if (rootCallId) {
+        const names = dispatchedTools.get(rootCallId) ?? new Set<string>()
+        names.add(data.name)
+        dispatchedTools.set(rootCallId, names)
+      }
       const isOk = data.isError !== true && (!Array.isArray(data.content) || data.content.every(b => (b as { isError?: boolean }).isError !== true))
       if (isOk) {
         const subCallId = String(data.subCallId ?? ('code:' + event.seq))
@@ -169,9 +207,15 @@ export function buildEvidenceIndex(events: readonly SessionEvent[]): EvidenceInd
 
   for (const [callId, call] of calls) {
     const result = results.get(callId)
-    if (result) paired.set(callId, { name: call.data.name, callSeq: call.seq, resultSeq: result.seq, text: blockText(result.data.message.content) })
+    if (!result) continue
+    // A wrapper that dispatched nothing but bookkeeping carries only their payloads
+    // (typically the todo list echoed straight back), so it is bookkeeping too. Its
+    // nested dispatches stay in the index under their own names, so real work done by
+    // the same program is still available as evidence.
+    if (onlyBookkeepingDispatches(dispatchedTools.get(callId))) continue
+    paired.set(callId, { name: call.data.name, callSeq: call.seq, resultSeq: result.seq, text: blockText(result.data.message.content) })
   }
-  return { problemSeq: taskStartSeq, calls: paired, todos, teamTasks }
+  return { problemSeq: taskStartSeq, calls: paired, todos, teamTasks, narration }
 }
 
 function parseTrustedWorkflow(value: unknown, callId: string, callSeq: number, resultSeq: number, maxCandidates: number, maxItemChars: number, maxInputChars: number): CandidateArtifact[] {
@@ -218,6 +262,35 @@ function canonicalTodoSnapshots(index: EvidenceIndex): Array<{ seq: number; todo
   return values
 }
 
+/**
+ * Tools whose successful output only maintains the agent's own bookkeeping.
+ *
+ * A checkpoint already renders the todo/team snapshot these tools wrote, so their
+ * own result repeats it while displacing the real work output that came just before
+ * them.
+ *
+ * `present` belongs here for the same reason: declaring deliverables produces no
+ * independent output, and it is always the LAST call of a turn. A live session ran
+ * typecheck + the full suite, committed, then presented — and every stop boundary
+ * showed the judge "Latest observed tool output at routing time (run_code):
+ * presented: 8" instead of the verification run one call earlier. Following its own
+ * rule ("a state without real verification should not exceed K"), the judge capped
+ * the newest checkpoint at exactly K = 52.6% against a 0.8 threshold on four
+ * consecutive routes, so each one steered "continue the unfinished work" for work
+ * that was finished and verified. The `deliverables/presented` event still records
+ * what was presented; the tool result adds nothing the judge can grade.
+ */
+const BOOKKEEPING_TOOLS = new Set([
+  'todo_write', 'create_goal', 'get_goal', 'update_goal', 'interrupt_agent', 'list_agents', 'exit_plan_mode', 'skill', 'present',
+])
+
+/** Whether a wrapper's dispatches were all bookkeeping; undefined/empty means it dispatched nothing. */
+function onlyBookkeepingDispatches(names: ReadonlySet<string> | undefined): boolean {
+  if (names === undefined || names.size === 0) return false
+  for (const name of names) if (!BOOKKEEPING_TOOLS.has(name)) return false
+  return true
+}
+
 /**
  * Observed tool evidence available at one checkpoint.
  *
@@ -225,22 +298,32 @@ function canonicalTodoSnapshots(index: EvidenceIndex): Array<{ seq: number; todo
  * threshold: the judge prompt explicitly refuses to credit a state that carries
  * no observed output. The most recent successful tool result at or before the
  * checkpoint is therefore attached as evidence.
+ *
+ * Bookkeeping tools are skipped: the checkpoint already renders the todo/team
+ * snapshot they wrote, so their own result repeats it while displacing the real
+ * output produced just before them (a live session attached `create_goal`,
+ * `update_goal` and `interrupt_agent` output to checkpoints that had already run
+ * the task's test suite).
  * @param index - evidence index of the current task.
- * @param seq - checkpoint sequence number.
+ * @param seq - checkpoint sequence number, or `Infinity` for the current state.
  * @param budget - maximum characters the evidence may occupy.
+ * @param current - render the newest output in the task instead of the newest one before `seq`.
  * @returns Evidence block, or '' when the task produced none yet.
  */
-function checkpointEvidence(index: EvidenceIndex, seq: number, budget: number): string {
+function checkpointEvidence(index: EvidenceIndex, seq: number, budget: number, current = false): string {
   if (budget < 64) return ''
   let latest: EvidenceCall | undefined
   for (const pair of index.calls.values()) {
+    if (BOOKKEEPING_TOOLS.has(pair.name) || !pair.text.trim()) continue
     if (pair.resultSeq <= seq && (latest === undefined || pair.resultSeq > latest.resultSeq)) latest = pair
   }
   if (latest === undefined) return ''
   // The prefix length depends on the tool name, so measure it instead of assuming a
   // fixed overhead: with a long tool name the old "- 60" let the rendered step exceed
   // maxItemChars, and boundDecision() then dropped the whole track decision silently.
-  const prefix = '\n\nLatest observed tool output before this checkpoint (' + latest.name + '):\n'
+  const prefix = current
+    ? '\n\nLatest observed tool output at routing time (' + latest.name + '):\n'
+    : '\n\nLatest observed tool output before this checkpoint (' + latest.name + '):\n'
   if (prefix.length >= budget) return ''
   return prefix + sanitizeVerifierText(latest.text, budget - prefix.length)
 }
@@ -280,6 +363,12 @@ interface CheckpointSource { seq: number; label: string; body: string }
  * The result fits both caps by construction: each step is at most min(maxItemChars,
  * maxInputChars / kept.length) characters, so the combined length can never exceed
  * maxInputChars and boundDecision() no longer drops the whole route.
+ *
+ * The newest checkpoint is also the state the route is judging, so it is rendered as
+ * the CURRENT state: its evidence is the newest observed output in the task rather
+ * than the newest one before the last todo snapshot (which is often several tool calls
+ * stale), and it carries the agent's latest prose as an explicitly labelled claim (see
+ * {@link currentNarration}), because prose deliverables never reach a tool.
  * @param index - evidence index of the current task.
  * @param sources - checkpoints in chronological order.
  * @param maxItemChars - hard per-item cap enforced by boundDecision().
@@ -294,12 +383,40 @@ function renderCheckpointSteps(index: EvidenceIndex, sources: readonly Checkpoin
   const evidenceBudget = Math.max(0, Math.min(Math.floor(maxInputChars / 2), Math.floor(maxItemChars / 2), Math.floor(stepCap / 2)))
   const steps = kept.map((source, position) => {
     const note = position === 0 && omitted > 0 ? 'Earlier ' + omitted + ' checkpoint(s) omitted; showing the ' + kept.length + ' most recent.\n' : ''
-    const evidence = checkpointEvidence(index, source.seq, evidenceBudget)
-    return sanitizeVerifierText(note + source.label + source.body, Math.max(1, stepCap - evidence.length)) + evidence
+    // Only the newest checkpoint describes the state the route is judging, so only it
+    // carries the agent's own latest narration and the newest observed output. Half of
+    // the evidence budget is held back for the narration: it is much longer than a tool
+    // result and used to be the deliverable for tasks (reviews, analyses) whose output
+    // never reaches a tool.
+    const isCurrent = position === kept.length - 1
+    const observed = checkpointEvidence(index, isCurrent ? Number.POSITIVE_INFINITY : source.seq, isCurrent ? Math.floor(evidenceBudget / 2) : evidenceBudget, isCurrent)
+    const narration = isCurrent ? currentNarration(index, evidenceBudget - observed.length) : ''
+    return sanitizeVerifierText(note + source.label + source.body, Math.max(1, stepCap - observed.length - narration.length)) + observed + narration
   })
   return { steps, evidenceSeqs: kept.map(source => source.seq), omitted }
 }
 
+/**
+ * The agent's newest unverified prose, attached to the current checkpoint.
+ *
+ * Progress checkpoints are graded against "would this state satisfy the task", and
+ * for work whose deliverable is prose the todo snapshot plus one tool result say
+ * nothing about it: every checkpoint then scores "certainly NO" even though the
+ * deliverable exists (a live review task scored 0% on all four checkpoints while the
+ * final session acceptance of the same work passed). The block is labelled as a claim
+ * so the judge can weigh it without treating it as observed output.
+ * @param index - evidence index of the current task.
+ * @param budget - maximum characters the narration may occupy.
+ * @returns Narration block, or '' when the task produced no prose or has no budget.
+ */
+function currentNarration(index: EvidenceIndex, budget: number): string {
+  const narration = index.narration
+  if (narration === undefined || budget < 64) return ''
+  const prefix = "\n\nNewest agent narration in this task (the agent's own claim — NOT observed evidence):\n"
+  if (prefix.length >= budget) return ''
+  return prefix + sanitizeVerifierText(narration.text, budget - prefix.length)
+}
+
 function canonicalTeamTaskSnapshots(index: EvidenceIndex): Array<{ seq: number; tasks: TeamTaskItem[] }> {
   const values: Array<{ seq: number; tasks: TeamTaskItem[] }> = []
   let previous = ''
@@ -329,23 +446,46 @@ export function analyzeStructuredRoute(events: readonly SessionEvent[], maxCandi
     const snapshots = canonicalTodoSnapshots(index)
     if (snapshots.length >= 2 && snapshots.some(snapshot => snapshot.todos.length >= 2)) {
       const rendered = renderCheckpointSteps(index, snapshots.map(snapshot => ({ seq: snapshot.seq, label: 'Todo checkpoint seq ' + snapshot.seq + ':\n', body: snapshot.todos.map(todo => '- [' + todo.status + '] ' + todo.content).join('\n') })), maxItemChars, maxInputChars)
-      return { kind: 'track', source: 'structured', confidence: 1, reason: 'changed durable todo snapshots', fingerprint: stableHash({ kind: 'track', snapshots }), steps: rendered.steps, checkpoints: rendered.steps.map((_, i) => i + 1), evidenceSeqs: rendered.evidenceSeqs }
+      // The fingerprint covers the RENDERED steps, not just the snapshots: the evidence
+      // attached to a checkpoint (and the newest narration) changes with the work that
+      // followed it, so a route whose prompt would differ must not be refused as
+      // "already routed". The semantic track route hashes its steps for the same reason.
+      return { kind: 'track', source: 'structured', confidence: 1, reason: 'changed durable todo snapshots', fingerprint: stableHash({ kind: 'track', snapshots, steps: rendered.steps }), steps: rendered.steps, checkpoints: rendered.steps.map((_, i) => i + 1), evidenceSeqs: rendered.evidenceSeqs }
     }
     const teamSnapshots = canonicalTeamTaskSnapshots(index)
     if (teamSnapshots.length >= 2) {
       const rendered = renderCheckpointSteps(index, teamSnapshots.map(snapshot => ({ seq: snapshot.seq, label: 'Team task checkpoint seq ' + snapshot.seq + ':\n', body: snapshot.tasks.map(task => '- [' + task.status + '] ' + task.subject + (task.description ? ' (' + task.description + ')' : '')).join('\n') })), maxItemChars, maxInputChars)
-      return { kind: 'track', source: 'structured', confidence: 1, reason: 'changed durable team tasks', fingerprint: stableHash({ kind: 'track', teamSnapshots }), steps: rendered.steps, checkpoints: rendered.steps.map((_, i) => i + 1), evidenceSeqs: rendered.evidenceSeqs }
+      return { kind: 'track', source: 'structured', confidence: 1, reason: 'changed durable team tasks', fingerprint: stableHash({ kind: 'track', teamSnapshots, steps: rendered.steps }), steps: rendered.steps, checkpoints: rendered.steps.map((_, i) => i + 1), evidenceSeqs: rendered.evidenceSeqs }
     }
   }
   return undefined
 }
 
+/**
+ * Artifacts that only the classifier can turn into candidates.
+ *
+ * Todo and team snapshots are deliberately absent. They are the structured track
+ * route's own input, and that route runs first — so listing them here could only make
+ * the hint true in shapes the structured pass already claimed (or in a task that
+ * already ran an explicit `verifier_track`, where re-classifying the same snapshots is
+ * not wanted). The one shape left out is a snapshot series whose lists are all shorter
+ * than two items, which is not worth a classification call.
+ */
+const HINT_ARTIFACTS = new Set(['subagent', 'subagent_fork', 'workflow', 'exit_plan_mode'])
+
+/**
+ * Whether a smart-mode stop boundary is worth a semantic classification call.
+ *
+ * The semantic phase only runs when the structured pass produced nothing, so this
+ * answers "is there material the structured pass never consumes?" — never "are there
+ * todo/team snapshots?", which the structured pass would have used already.
+ * @param events - Session event log.
+ * @returns True when a subagent/workflow/plan artifact exists.
+ */
 export function semanticRouteHint(events: readonly SessionEvent[]): boolean {
   const index = buildEvidenceIndex(events)
   if (!index) return false
-  if ([...index.calls.values()].some(pair => pair.name === 'subagent' || pair.name === 'subagent_fork' || pair.name === 'workflow' || pair.name === 'exit_plan_mode')) return true
-  if (canonicalTodoSnapshots(index).length >= 2) return true
-  if (canonicalTeamTaskSnapshots(index).length >= 2) return true
+  for (const pair of index.calls.values()) if (HINT_ARTIFACTS.has(pair.name)) return true
   return false
 }
 
@@ -359,6 +499,9 @@ export function buildSemanticRoutePrompt(problem: string, events: readonly Sessi
   let used = 0
   let omitted = 0
   for (const [callId, pair] of [...index.calls.entries()].reverse()) {
+    // Bookkeeping results are not alternatives to anything; offering them let the
+    // classifier cite e.g. two goal/agent-control calls as competing candidates.
+    if (BOOKKEEPING_TOOLS.has(pair.name)) continue
     const text = sanitizeVerifierText(pair.text, maxItemChars)
     if (artifacts.length > 0 && used + text.length > maxInputChars) { omitted += 1; continue }
     used += text.length
@@ -416,7 +559,11 @@ export function semanticDecision(output: SemanticRouteOutput, events: readonly S
   const perItem = itemBudget(output.candidateCallIds.length, maxItemChars, maxInputChars)
   const candidates = output.candidateCallIds.map((callId, i) => {
     const pair = index.calls.get(callId)
-    return pair ? { id: callId, groupId: 'semantic', label: pair.name + ' ' + (i + 1), content: sanitizeVerifierText(pair.text, perItem), callId, fromSeq: pair.callSeq, toSeq: pair.resultSeq } : undefined
+    // Bookkeeping calls are not candidates (they were never offered to the classifier);
+    // a citation of one is an invalid reference, so the whole decision is rejected
+    // instead of comparing metadata as if it were alternative work.
+    if (!pair || BOOKKEEPING_TOOLS.has(pair.name)) return undefined
+    return { id: callId, groupId: 'semantic', label: pair.name + ' ' + (i + 1), content: sanitizeVerifierText(pair.text, perItem), callId, fromSeq: pair.callSeq, toSeq: pair.resultSeq }
   }).filter((candidate): candidate is CandidateArtifact => candidate !== undefined)
   if (candidates.length !== output.candidateCallIds.length) return undefined
   const fingerprint = stableHash({ kind: output.kind, candidates })
@@ -435,6 +582,27 @@ export function semanticDecision(output: SemanticRouteOutput, events: readonly S
  * @param criteriaCount - number of criteria evaluated per comparison.
  * @returns The planned model-call count, never below 1.
  */
+/**
+ * Scoring repeats a routed decision actually runs.
+ *
+ * `compare` judges ONE unordered pair, and `VerifierEngine.compare` only swaps the
+ * candidates on odd repeat indices — with the shipped default of a single round the
+ * first candidate therefore always sat in slot A, so the winner was partly decided by
+ * listing order. Rounding its count up to an even number averages a swapped round and
+ * cancels that.
+ *
+ * `select` does not need it: its ring is symmetric by construction and the pivot round
+ * is oriented per pair by the engine, so one round is already unbiased. `track` scores
+ * a single checkpoint list and has no slots at all.
+ * @param decision - the routed decision about to run.
+ * @param configured - the configured auto-route repeat count.
+ * @returns The repeat count to pass to the engine.
+ */
+export function routedRepeats(decision: RouteDecision, configured: number): number {
+  if (decision.kind !== 'compare') return configured
+  return configured % 2 === 0 ? configured : configured + 1
+}
+
 export function estimateRoutedCalls(decision: RouteDecision, repeats: number, criteriaCount: number): number {
   if (decision.kind === 'compare') return Math.max(1, criteriaCount * repeats)
   if (decision.kind === 'track') return Math.max(1, repeats)

package/src/statistics.test.ts

@@ -7,6 +7,8 @@ import {
   emptyRunStats,
   mergeStatisticsOverviews,
   parseStatisticsQuery,
+  summarizeVerdict,
+  type VerdictThresholds,
 } from './statistics.ts'
 
 const readTracker = {
@@ -188,6 +190,120 @@ describe('StatisticsStore', () => {
     })
   })
 
+  it('summarizes every tool verdict from its own result shape', () => {
+    const thresholds: VerdictThresholds = { autoVerifyThreshold: 0.65, autoTrackCompletionThreshold: 0.8 }
+    // Route classification reports no score at all.
+    expect(summarizeVerdict('verifier_route_classify', { kind: 'none' }, 'semantic', thresholds)).toEqual({ phase: 'semantic', outcome: 'classified' })
+    // Select reports the ranked winner's score.
+    expect(summarizeVerdict('verifier_select', { index: 1, scores: [0.2, 0.9], ranking: [1, 0] }, 'select', thresholds)).toEqual({ phase: 'select', outcome: 'ranked', score: 0.9 })
+    // Track reports the newest checkpoint (not the historical minimum) plus the curve.
+    expect(summarizeVerdict('verifier_track', { scores: [0, 0.10526315789473684, 0.7894736842105262] }, 'track', thresholds)).toEqual({
+      phase: 'track',
+      outcome: 'below-threshold',
+      score: 0.7894736842105262,
+      scores: [0, 0.10526315789473684, 0.7894736842105262],
+      threshold: 0.8,
+    })
+    expect(summarizeVerdict('verifier_track', { scores: [0.9] }, 'track', thresholds)).toEqual({ phase: 'track', outcome: 'passed', score: 0.9, threshold: 0.8 })
+  })
+
+  it('reports the winning side of a comparison instead of the loser under a threshold', () => {
+    const thresholds: VerdictThresholds = { autoVerifyThreshold: 0.65, autoTrackCompletionThreshold: 0.8 }
+    // Regression: a comparison has no threshold, so "below-threshold" used to describe
+    // candidate A's (the loser's) score as a failure of the comparison itself.
+    expect(summarizeVerdict('verifier_compare', { scoreA: 0.2, scoreB: 0.9, winner: 'B' }, 'compare', thresholds)).toEqual({ phase: 'compare', outcome: 'compared', score: 0.9, scoreB: 0.9, winner: 'B' })
+    expect(summarizeVerdict('verifier_compare', { scoreA: 0.9, scoreB: 0.2, winner: 'A' }, 'compare', thresholds)).toEqual({ phase: 'compare', outcome: 'compared', score: 0.9, scoreB: 0.2, winner: 'A' })
+    expect(summarizeVerdict('verifier_compare', { scoreA: 0.5, scoreB: 0.5, winner: 'tie' }, 'compare', thresholds)).toEqual({ phase: 'compare', outcome: 'tie', score: 0.5, scoreB: 0.5, winner: 'tie' })
+    // Session acceptance keeps A = the session and B = the empty-work baseline.
+    expect(summarizeVerdict('verifier_current_session', { score: 0.4, baselineScore: 0, winner: 'A' }, 'final', thresholds)).toEqual({
+      phase: 'final', outcome: 'below-threshold', score: 0.4, baselineScore: 0, winner: 'A', threshold: 0.65,
+    })
+    expect(summarizeVerdict('verifier_current_session', { score: 0.7, baselineScore: 0, winner: 'A' }, 'final', thresholds)).toEqual({
+      phase: 'final', outcome: 'passed', score: 0.7, baselineScore: 0, winner: 'A', threshold: 0.65,
+    })
+    // Losing to the baseline can never pass, however high the raw score is.
+    expect(summarizeVerdict('verifier_current_session', { score: 0.9, baselineScore: 0.95, winner: 'B' }, 'final', thresholds)).toMatchObject({ outcome: 'below-threshold' })
+    // One criterion below the threshold fails the record even though the mean passes.
+    expect(summarizeVerdict('verifier_current_session', { score: 0.6667, baselineScore: 0, winner: 'A', criteria: [{ id: 'specification', scoreA: 1 }, { id: 'error_signals', scoreA: 0 }] }, 'final', thresholds)).toEqual({
+      phase: 'final',
+      outcome: 'below-threshold',
+      score: 0.6667,
+      baselineScore: 0,
+      criteria: [{ id: 'specification', score: 1 }, { id: 'error_signals', score: 0 }],
+      winner: 'A',
+      threshold: 0.65,
+    })
+  })
+
+  it('keeps the track checkpoint progression and drops invalid entries', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
+    const file = join(root, 'statistics.json')
+    const store = new StatisticsStore(file)
+    const record = await store.record({
+      toolName: 'verifier_track',
+      startedAt: 100,
+      finishedAt: 200,
+      success: true,
+      provider: 'p',
+      model: 'm',
+      stats: stats(),
+      verdict: {
+        phase: 'track',
+        outcome: 'below-threshold',
+        // The newest checkpoint drives the outcome; the whole curve explains it.
+        score: 0.7894736842105262,
+        scores: [0, Number.NaN, Number.POSITIVE_INFINITY, 0.7894736842105262],
+        threshold: 0.8,
+      },
+    })
+    expect(record.verdict?.score).toBe(0.7894736842105262)
+    expect(record.verdict?.scores).toEqual([0, 0.7894736842105262])
+
+    const fresh = new StatisticsStore(file)
+    const result = await fresh.overview({ fromMs: 0, toMs: 1_000 })
+    expect(result.recent[0]?.verdict?.scores).toEqual([0, 0.7894736842105262])
+  })
+
+  it('caps a stored checkpoint progression at the boundary', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
+    const store = new StatisticsStore(join(root, 'statistics.json'))
+    const scores = Array.from({ length: 70 }, (_, index) => index / 100)
+    const record = await store.record({
+      toolName: 'verifier_track', startedAt: 100, finishedAt: 200, success: true, provider: 'p', model: 'm', stats: stats(),
+      verdict: { phase: 'track', outcome: 'below-threshold', score: 0.69, scores, threshold: 0.8 },
+    })
+    expect(record.verdict?.scores).toHaveLength(64)
+    expect(record.verdict?.scores?.at(-1)).toBe(0.63)
+  })
+
+  it('stores the per-criterion breakdown and drops malformed entries', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
+    const store = new StatisticsStore(join(root, 'statistics.json'))
+    const record = await store.record({
+      toolName: 'verifier_current_session', startedAt: 1, finishedAt: 2, success: true, provider: 'p', model: 'm', stats: stats(),
+      verdict: {
+        phase: 'final', outcome: 'below-threshold', score: 0.67, baselineScore: 0, winner: 'A', threshold: 0.65,
+        criteria: [{ id: 'specification', score: 1 }, { id: 42, score: 1 } as never, { id: 'error_signals', score: Number.NaN }, { id: 'output_match', score: 0 }],
+      },
+    })
+    expect(record.verdict?.criteria).toEqual([{ id: 'specification', score: 1 }, { id: 'output_match', score: 0 }])
+  })
+
+  it('stores scoreB for comparisons and drops a non-finite one', async () => {
+    const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
+    const store = new StatisticsStore(join(root, 'statistics.json'))
+    const dropped = await store.record({
+      toolName: 'verifier_compare', startedAt: 1, finishedAt: 2, success: true, provider: 'p', model: 'm', stats: stats(),
+      verdict: { phase: 'compare', outcome: 'compared', score: 0.9, scoreB: Number.NaN, winner: 'A' },
+    })
+    expect(dropped.verdict?.scoreB).toBeUndefined()
+    const kept = await store.record({
+      toolName: 'verifier_compare', startedAt: 3, finishedAt: 4, success: true, provider: 'p', model: 'm', stats: stats(),
+      verdict: { phase: 'compare', outcome: 'compared', score: 0.9, scoreB: 0.15789473684210525, winner: 'A' },
+    })
+    expect(kept.verdict?.scoreB).toBe(0.15789473684210525)
+  })
+
   it('loads old records that have no verdict', async () => {
     const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
     const file = join(root, 'statistics.json')

package/src/statistics.ts

@@ -10,11 +10,111 @@ export interface VerdictSummary {
   phase?: string
   outcome?: string
   score?: number
+  /** Per-checkpoint progression of a `verifier_track` verdict, oldest first. Optional: old records do not carry it. */
+  scores?: number[]
+  /** Candidate B's score of a two-way comparison; `score` is the winning side. */
+  scoreB?: number
+  /** Per-criterion A-side scores of a session acceptance; the mean alone can hide a failed requirement. */
+  criteria?: Array<{ id: string; score: number }>
   baselineScore?: number
   winner?: 'A' | 'B' | 'tie'
   threshold?: number
 }
 
+/** Upper bound on the stored checkpoint progression; one explicit call can legitimately carry 32 steps. */
+const MAX_VERDICT_SCORES = 64
+
+/** Upper bound on the stored per-criterion breakdown; the default rubric has three criteria. */
+const MAX_VERDICT_CRITERIA = 16
+
+/** Thresholds the verdict summary needs; plain values keep the mapping a pure function. */
+export interface VerdictThresholds {
+  autoVerifyThreshold: number
+  autoTrackCompletionThreshold: number
+}
+
+/**
+ * Compact summary of what the judges decided, stored beside the call counters so
+ * the dashboard can answer "why did this fail?" instead of only "how much did it cost".
+ *
+ * Pure on purpose: this mapping used to live inside the plugin closure, where the two
+ * bugs it carried (a track verdict reported as the historical minimum, a comparison
+ * reported as the loser's score under a threshold it never used) could not be tested.
+ * @param toolName - the verifier tool that produced the value.
+ * @param value - its rendered result.
+ * @param phase - which stage produced it (explicit | compare | select | track | final | ...).
+ * @param thresholds - resolved acceptance thresholds.
+ * @returns A verdict summary; score fields are omitted when the tool has none.
+ */
+export function summarizeVerdict(toolName: VerifierToolName, value: unknown, phase: string, thresholds: VerdictThresholds): VerdictSummary {
+  const row = typeof value === 'object' && value !== null ? value as Record<string, unknown> : {}
+  const numberAt = (key: string): number | undefined => (typeof row[key] === 'number' && Number.isFinite(row[key]) ? row[key] as number : undefined)
+  const scores = Array.isArray(row.scores) ? row.scores.filter((entry): entry is number => typeof entry === 'number' && Number.isFinite(entry)) : []
+  const winner = row.winner === 'A' || row.winner === 'B' || row.winner === 'tie' ? row.winner : undefined
+  if (toolName === 'verifier_route_classify') return { phase, outcome: 'classified' }
+  if (toolName === 'verifier_select') {
+    const index = numberAt('index')
+    const best = index === undefined ? undefined : scores[index]
+    return { phase, outcome: 'ranked', ...(best !== undefined ? { score: best } : {}) }
+  }
+  if (toolName === 'verifier_track') {
+    // The verdict reports the newest checkpoint (the one the continuation decision
+    // uses) plus the whole progression. Reporting only Math.min() made every routed
+    // track look like a 0% failure on the dashboard, because the first checkpoint of
+    // a task is always the untouched plan.
+    const latest = scores.length > 0 ? scores[scores.length - 1] : undefined
+    const threshold = thresholds.autoTrackCompletionThreshold
+    return {
+      phase,
+      outcome: latest !== undefined && latest >= threshold ? 'passed' : 'below-threshold',
+      ...(latest !== undefined ? { score: latest } : {}),
+      ...(scores.length > 1 ? { scores: [...scores] } : {}),
+      threshold,
+    }
+  }
+  if (toolName === 'verifier_compare') {
+    // A comparison has no threshold, so "below-threshold" used to describe the loser's
+    // score as a failure of the call itself. Report the winning side and keep both.
+    const scoreA = numberAt('score') ?? numberAt('scoreA')
+    const scoreB = numberAt('scoreB')
+    const score = winner === 'B' ? scoreB : scoreA
+    return {
+      phase,
+      outcome: winner === 'tie' ? 'tie' : 'compared',
+      ...(score !== undefined ? { score } : {}),
+      ...(scoreB !== undefined ? { scoreB } : {}),
+      ...(winner !== undefined ? { winner } : {}),
+    }
+  }
+  // Session acceptance: `score` is the session's own score and `baselineScore` the
+  // empty-work baseline it has to beat.
+  const score = numberAt('score') ?? numberAt('scoreA')
+  const baselineScore = numberAt('baselineScore')
+  const threshold = thresholds.autoVerifyThreshold
+  const criteria = Array.isArray(row.criteria)
+    ? (row.criteria as unknown[])
+        .map(entry => (typeof entry === 'object' && entry !== null ? entry as Record<string, unknown> : {}))
+        .filter(entry => typeof entry.id === 'string' && typeof entry.scoreA === 'number' && Number.isFinite(entry.scoreA))
+        .slice(0, MAX_VERDICT_CRITERIA)
+        .map(entry => ({ id: entry.id as string, score: entry.scoreA as number }))
+    : []
+  // A session does not pass on the mean alone: one criterion below the threshold fails
+  // the record too, which is exactly what the live gate now enforces.
+  const belowThreshold = criteria.filter(entry => !(entry.score >= threshold))
+  const outcome = winner === 'tie'
+    ? 'tie'
+    : winner === 'A' && score !== undefined && score >= threshold && belowThreshold.length === 0 ? 'passed' : 'below-threshold'
+  return {
+    phase,
+    outcome,
+    ...(score !== undefined ? { score } : {}),
+    ...(baselineScore !== undefined ? { baselineScore } : {}),
+    ...(criteria.length > 0 ? { criteria } : {}),
+    ...(winner !== undefined ? { winner } : {}),
+    threshold,
+  }
+}
+
 export interface InvocationRecord {
   id: string
   toolName: VerifierToolName
@@ -132,6 +232,18 @@ function cleanVerdict(input: VerdictSummary | undefined): VerdictSummary | undef
   if (typeof input.phase === 'string') verdict.phase = input.phase
   if (typeof input.outcome === 'string') verdict.outcome = input.outcome
   if (typeof input.score === 'number' && Number.isFinite(input.score)) verdict.score = input.score
+  if (Array.isArray(input.scores)) {
+    const scores = input.scores.filter((entry): entry is number => typeof entry === 'number' && Number.isFinite(entry)).slice(0, MAX_VERDICT_SCORES)
+    if (scores.length > 0) verdict.scores = scores
+  }
+  if (typeof input.scoreB === 'number' && Number.isFinite(input.scoreB)) verdict.scoreB = input.scoreB
+  if (Array.isArray(input.criteria)) {
+    const criteria = input.criteria
+      .filter((entry): entry is { id: string; score: number } => typeof entry === 'object' && entry !== null && typeof (entry as { id?: unknown }).id === 'string' && typeof (entry as { score?: unknown }).score === 'number' && Number.isFinite((entry as { score: number }).score))
+      .slice(0, MAX_VERDICT_CRITERIA)
+      .map(entry => ({ id: entry.id, score: entry.score }))
+    if (criteria.length > 0) verdict.criteria = criteria
+  }
   if (typeof input.baselineScore === 'number' && Number.isFinite(input.baselineScore)) verdict.baselineScore = input.baselineScore
   if (input.winner === 'A' || input.winner === 'B' || input.winner === 'tie') verdict.winner = input.winner
   if (typeof input.threshold === 'number' && Number.isFinite(input.threshold)) verdict.threshold = input.threshold