dsh-llm-verifier 0.2.1 → 0.3.1

package/src/engine.test.ts

@@ -4,7 +4,8 @@ import { join } from 'node:path'
 import { afterEach, describe, expect, it, vi } from 'vitest'
 import type { VerifierClientConfig } from './caller.ts'
 import { ScoreCache, SingleFlight, type CachedPairScore } from './cache.ts'
-import { VerifierEngine } from './engine.ts'
+import { VerifierEngine, orientRoundPairs } from './engine.ts'
+import { pivotRoundPairs } from './core.ts'
 import { TopLogprobCapabilityCache } from './top-logprobs.ts'
 
 function chunks(text: string) { return [{ type: 'block-start', index: 0, blockType: 'text' }, { type: 'text-delta', index: 0, text }, { type: 'block-end', index: 0, block: { type: 'text', text } }, { type: 'usage', usage: { inputTokens: 7, cacheReadTokens: 3, outputTokens: 4, reasoningTokens: 2 } }, { type: 'finish', reason: { kind: 'stop' } }] as any[] }
@@ -72,6 +73,44 @@ describe('VerifierEngine tournament', () => {
   })
 })
 
+describe('pivot round orientation', () => {
+  it('gives every ring leader both slots instead of always trajectory B', () => {
+    // Regression: the pivot round emitted [candidate, pivot] for every edge, so the ring
+    // leaders sat in trajectory B in ALL their extra matches — a judge that merely
+    // prefers slot A then averaged 0.36 for the pivots against 0.60 for everyone else
+    // and pushed the leaders to the bottom of the final ranking.
+    const key = ([a, b]: readonly [number, number]) => (a < b ? a + ',' + b : b + ',' + a)
+    for (const count of [4, 5, 6, 8]) {
+      const pivots = [0, 1]
+      const raw = pivotRoundPairs(count, pivots)
+      const oriented = orientRoundPairs(raw)
+      // Same unordered pairs — only the presentation order may change.
+      expect(oriented).toHaveLength(raw.length)
+      expect(oriented.map(key).sort()).toEqual(raw.map(key).sort())
+      expect(new Set(oriented.map(key)).size).toBe(oriented.length)
+      const slotA = new Map<number, number>()
+      const slotB = new Map<number, number>()
+      for (const [a, b] of oriented) {
+        slotA.set(a, (slotA.get(a) ?? 0) + 1)
+        slotB.set(b, (slotB.get(b) ?? 0) + 1)
+      }
+      for (const pivot of pivots) {
+        const a = slotA.get(pivot) ?? 0
+        const b = slotB.get(pivot) ?? 0
+        expect(a, 'count=' + count + ' pivot=' + pivot + ' must not always sit in slot B').toBeGreaterThan(0)
+        expect(Math.abs(a - b), 'count=' + count + ' pivot=' + pivot + ' slot balance').toBeLessThanOrEqual(1)
+      }
+    }
+  })
+
+  it('leaves a two-candidate select with exactly one match', () => {
+    // The orientation must not invent or duplicate matches.
+    const oriented = orientRoundPairs([[0, 1]])
+    expect(oriented).toHaveLength(1)
+    expect([oriented[0]![0], oriented[0]![1]].sort()).toEqual([0, 1])
+  })
+})
+
 describe('VerifierEngine cache identity', () => {
   it('does not reuse an explicit-tag cache entry for top-logprobs scoring', async () => {
     const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-engine-'))

package/src/engine.ts

@@ -60,6 +60,56 @@ function median(values: readonly number[]): number {
 function blankStats(): RunStats { return { ...emptyUsage(), cacheHits: 0, cacheMisses: 0, estimatedCostUsd: 0, topLogprobScores: 0, explicitTagScores: 0 } }
 function unorderedPair(a: number, b: number): string { return a < b ? a + ',' + b : b + ',' + a }
 
+/**
+ * Deterministic A/B slot for one pivot-round pair, balanced by construction.
+ *
+ * The round used to emit `[candidate, pivot]` for every edge, so the ring leaders sat
+ * in trajectory B in all of their extra matches: with a judge that merely prefers slot A
+ * the pivots averaged 0.36 against 0.60 for everyone else and sank in the final ranking.
+ * Alternating on the sum of the two RANKS — not the raw indices: the pivot set is
+ * selected by score, so its indices can share a parity and a parity rule would then
+ * handicap half the field — gives every pivot and every candidate both slots within one
+ * edge of each other, without a second model call.
+ *
+ * Deliberately NOT in core.ts: `pivotRoundPairs` is compared against the upstream Python
+ * reference in `parity.test.ts`, so the orientation stays an orchestration-side decision.
+ * @param pair - one unordered pair from the pivot round.
+ * @param pivotRanks - pivot index → its rank among the pivots.
+ * @param nonPivotRanks - candidate index → its rank among the non-pivots.
+ * @returns The pair in the order it should be presented.
+ */
+export function orientRoundPairs(pairs: readonly (readonly [number, number])[]): Array<[number, number]> {
+  const balance = new Map<number, number>()
+  return pairs.map(([a, b]) => {
+    const first = balance.get(a) ?? 0
+    const second = balance.get(b) ?? 0
+    // Slot A goes to whichever endpoint has held it less often; an equal count keeps the
+    // incoming order, so the result is deterministic for a given pair sequence.
+    const [x, y] = second < first ? [b, a] : [a, b]
+    balance.set(x, (balance.get(x) ?? 0) + 1)
+    balance.set(y, (balance.get(y) ?? 0) - 1)
+    return [x, y] as [number, number]
+  })
+}
+
+/**
+ * Arbitrary but fixed A/B slot for the single pair of a two-candidate select.
+ *
+ * There is only one match, so nothing can cancel a preference; the mix at least stops
+ * candidate 0 from always sitting in slot A. Real comparisons go through `compare`,
+ * where `routedRepeats` runs an even number of rounds instead.
+ * @param a - first candidate index of the pair.
+ * @param b - second candidate index of the pair.
+ * @returns The pair in the order it should be presented.
+ */
+function orientPair(a: number, b: number): [number, number] {
+  const lo = Math.min(a, b)
+  const hi = Math.max(a, b)
+  let mixed = Math.imul(lo + 1, 0x9e3779b1) ^ Math.imul(hi + 1, 0x85ebca77)
+  mixed = Math.imul(mixed ^ (mixed >>> 15), 0x2545f491)
+  return ((mixed ^ (mixed >>> 13)) & 1) === 0 ? [a, b] : [b, a]
+}
+
 function judgeLabel(client: VerifierClientConfig): string {
   return client.label?.trim() || client.provider + '/' + client.model
 }
@@ -422,10 +472,12 @@ export class VerifierEngine {
       return { index: 0, best: options.candidates[0]!, scores: [1], ranking: [0], pivots: [0], comparisons: 0, calls: 0, stats: blankStats(), judges }
     }
     if (options.candidates.length === 2) {
-      // ringCycle(2) would judge the single unordered pair in both directions; play it once.
-      const { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats } = await this.scorePairs(options, [[0, 1]], signal)
+      // ringCycle(2) would judge the single unordered pair in both directions; play it
+      // once, in the orientation orientPair() picks so the slot is not always candidate 0.
+      const single = orientPair(0, 1)
+      const { rewards, judgeRewards, judgeOk, judgeErrors, judgeCalls, stats } = await this.scorePairs(options, [single], signal)
       const wins = [0, 0]; const counts = [0, 0]
-      accumulatePairs([[0, 1]], rewards, wins, counts)
+      accumulatePairs([single], rewards, wins, counts)
       const ranked = rankScores(wins, counts); const index = ranked[0]!.index
       const judges: JudgeScore[] = this.clients.map((client, k) => {
         const isOk = judgeOk[k]!
@@ -464,7 +516,7 @@ export class VerifierEngine {
     // pivot→neighbour edges). Dropping those duplicates keeps each unordered pair to a
     // single match so wins/counts are not double-weighted and no pair is judged twice.
     const ringPairs = new Set(ring.map(pair => unorderedPair(pair[0], pair[1])))
-    const rounds = pivotRoundPairs(options.candidates.length, pivots).filter(pair => !ringPairs.has(unorderedPair(pair[0], pair[1])))
+    const rounds = orientRoundPairs(pivotRoundPairs(options.candidates.length, pivots).filter(pair => !ringPairs.has(unorderedPair(pair[0], pair[1]))))
     const roundScores = await this.scorePairs(options, rounds, signal)
     const allRewards = new Map([...ringScores.rewards, ...roundScores.rewards])
     const wins = new Array<number>(options.candidates.length).fill(0)

package/src/index.ts

@@ -12,12 +12,12 @@ import { ScoreCache, SingleFlight, resolveCacheFile, stableHash, type CachedPair
 import { VerifierEngine, normalizeCriteria, type JudgeScore, type RunStats } from './engine.ts'
 import { loadVerifierImages } from './images.ts'
 import { extractSession, sanitizeVerifierText, sessionEvents } from './session.ts'
-import { analyzeAutoTask, automaticFeedback, isSubagentSession } from './auto.ts'
-import { AutoVerifierRouter, analyzeStructuredRoute, boundDecision, buildSemanticRoutePrompt, estimateRoutedCalls, parseSemanticRoute, semanticDecision, semanticRouteHint, type RouteDecision, type RoutedVerifierKind } from './router.ts'
+import { analyzeAutoTask, automaticFeedback, failedAcceptanceCriteria, isSubagentSession, sessionAccepted, type AcceptanceCriterion } from './auto.ts'
+import { AutoVerifierRouter, analyzeStructuredRoute, boundDecision, buildSemanticRoutePrompt, estimateRoutedCalls, parseSemanticRoute, routedRepeats, semanticDecision, semanticRouteHint, type RouteDecision, type RoutedVerifierKind } from './router.ts'
 import { DEFAULT_CRITERIA } from './core.ts'
 import { buildPlanPreReviewPrompt, parseVerdictLetter, planFromArguments } from './plan-gate.ts'
 import { inspectTeamTasks, buildTeamTaskVerificationPrompt } from './team-gate.ts'
-import { StatisticsStore, emptyRunStats, errorDetails, mergeStatisticsOverviews, parseStatisticsQuery, resolveStatisticsFile, type StatisticsOverview, type VerifierToolName, type VerdictSummary } from './statistics.ts'
+import { StatisticsStore, emptyRunStats, errorDetails, mergeStatisticsOverviews, parseStatisticsQuery, resolveStatisticsFile, summarizeVerdict, type StatisticsOverview, type VerifierToolName } from './statistics.ts'
 import { resolveTopicDataDir, type SessionArtifactLocator } from './topic-storage.ts'
 
 export const name = 'llm-verifier'
@@ -98,7 +98,7 @@ function rpcSuccess<T>(value: T) { return { ok: true as const, value } }
 function rpcFailure(message: string) { return { ok: false as const, error: { code: 'bad-request' as const, message, details: { issues: [] } } } }
 
 interface SessionVerificationOptions { fromSeq?: number; toSeq?: number; includeAssistantText?: boolean; redactPatterns?: readonly string[]; maxChars?: number; repeats?: number }
-interface SessionVerificationResult { sessionId: string; problem: string; score: number; baselineScore: number; winner: 'A' | 'B' | 'tie'; fromSeq: number; toSeq: number; omittedCharacters: number; calls: number; stats: RunStats; judges: JudgeScore[]; agreement: number }
+interface SessionVerificationResult { sessionId: string; problem: string; score: number; baselineScore: number; winner: 'A' | 'B' | 'tie'; criteria: AcceptanceCriterion[]; fromSeq: number; toSeq: number; omittedCharacters: number; calls: number; stats: RunStats; judges: JudgeScore[]; agreement: number }
 
 export function apply(ctx: Context, config: Config = {}): void {
   const services = ctx as Context & { attachments: AttachmentStore; connection: HostConnectionHandle; sessionPersistence: SessionArtifactLocator & { list(signal?: AbortSignal): Promise<readonly unknown[]> } }
@@ -139,36 +139,12 @@ export function apply(ctx: Context, config: Config = {}): void {
   const images = (values: readonly string[] | undefined, signal: AbortSignal) => loadVerifierImages(values, signal)
   const route = (selected: { provider: string; model: string }) => ({ provider: selected.provider, model: selected.model })
   const requireEnabled = () => { if (!current().enabled) throw new Error('llm-verifier: verifier tools are disabled — enable them in Settings → LLM Verifier') }
-  /**
-   * Compact summary of what the judges decided, stored beside the call counters so
-   * the dashboard can answer "why did this fail?" instead of only "how much did it cost".
-   * @param toolName - the verifier tool that produced the value.
-   * @param value - its rendered result.
-   * @param phase - which stage produced it (explicit | compare | select | track | final | ...).
-   * @returns A verdict summary; score fields are omitted when the tool has none.
-   */
-  const verdictFrom = (toolName: VerifierToolName, value: unknown, phase: string): VerdictSummary => {
+  const verdictFrom = (toolName: VerifierToolName, value: unknown, phase: string) => {
     const selected = current()
-    const row = typeof value === 'object' && value !== null ? value as Record<string, unknown> : {}
-    const numberAt = (key: string): number | undefined => (typeof row[key] === 'number' && Number.isFinite(row[key]) ? row[key] as number : undefined)
-    const scores = Array.isArray(row.scores) ? row.scores.filter((entry): entry is number => typeof entry === 'number' && Number.isFinite(entry)) : []
-    const winner = row.winner === 'A' || row.winner === 'B' || row.winner === 'tie' ? row.winner : undefined
-    if (toolName === 'verifier_route_classify') return { phase, outcome: 'classified' }
-    if (toolName === 'verifier_select') {
-      const index = numberAt('index')
-      const best = index === undefined ? undefined : scores[index]
-      return { phase, outcome: 'ranked', ...(best !== undefined ? { score: best } : {}) }
-    }
-    if (toolName === 'verifier_track') {
-      const worst = scores.length > 0 ? Math.min(...scores) : undefined
-      const threshold = selected.autoTrackCompletionThreshold
-      return { phase, outcome: worst !== undefined && worst >= threshold ? 'passed' : 'below-threshold', ...(worst !== undefined ? { score: worst } : {}), threshold }
-    }
-    const score = numberAt('score') ?? numberAt('scoreA')
-    const baselineScore = numberAt('baselineScore')
-    const threshold = selected.autoVerifyThreshold
-    const outcome = winner === 'tie' ? 'tie' : winner === 'A' && score !== undefined && score >= threshold ? 'passed' : 'below-threshold'
-    return { phase, outcome, ...(score !== undefined ? { score } : {}), ...(baselineScore !== undefined ? { baselineScore } : {}), ...(winner !== undefined ? { winner } : {}), threshold }
+    return summarizeVerdict(toolName, value, phase, {
+      autoVerifyThreshold: selected.autoVerifyThreshold,
+      autoTrackCompletionThreshold: selected.autoTrackCompletionThreshold,
+    })
   }
   const record = async <T>(toolName: VerifierToolName, agent: Agent, operation: () => Promise<{ result: T; selected: { provider: string; model: string } }>, phase = 'explicit'): Promise<T & { provider: string; model: string }> => {
     const startedAt = Date.now()
@@ -218,7 +194,7 @@ export function apply(ctx: Context, config: Config = {}): void {
     if (!extracted.problem.trim()) throw new Error('llm-verifier: no direct user task found in the selected session range — widen from_seq so the task statement is included')
     const { verifier, selected } = await engine(agent)
     const compared = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(options.repeats, 2, 'repeats'), images: extracted.images }, signal)
-    const result: SessionVerificationResult = { sessionId: extracted.sessionId, problem: extracted.problem, score: compared.scoreA, baselineScore: compared.scoreB, winner: compared.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: compared.calls, stats: compared.stats, judges: compared.judges, agreement: compared.agreement }
+    const result: SessionVerificationResult = { sessionId: extracted.sessionId, problem: extracted.problem, score: compared.scoreA, baselineScore: compared.scoreB, winner: compared.winner, criteria: compared.criteria.map(row => ({ id: row.id, name: row.name, score: row.scoreA })), fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: compared.calls, stats: compared.stats, judges: compared.judges, agreement: compared.agreement }
     return { result, selected }
   }, phase)
   const compareCandidates = async (agent: Agent, problem: string, candidateA: string, candidateB: string, repeats: number, signal: AbortSignal, routedImages: readonly import('./caller.ts').VerifierImage[] = [], phase = 'explicit') => record('verifier_compare', agent, async () => {
@@ -393,8 +369,11 @@ export function apply(ctx: Context, config: Config = {}): void {
           if (!stillCurrent()) { autoRouter.fail(agent, taskReservation, false); return }
           const verdict = parseVerdictLetter(classified.text)
           if (verdict === undefined) {
-            autoRouter.fail(agent, taskReservation, false)
-            ctx.logger.warn('llm-verifier team task verification produced no verdict line; task ' + task.id + ' was not gated')
+            // Fail closed in strict mode like every other routed phase: an unparseable
+            // verdict must not silently leave the task ungated.
+            const strict = selected.autoVerifyMode === 'strict'
+            autoRouter.fail(agent, taskReservation, strict)
+            ctx.logger.warn('llm-verifier team task verification produced no verdict line; task ' + task.id + (strict ? ' stays blocked' : ' was not gated'))
             continue
           }
           if (verdict.score >= selected.autoVerifyThreshold) {
@@ -462,8 +441,10 @@ export function apply(ctx: Context, config: Config = {}): void {
 
     if (!stillCurrent()) return
     if (decision) {
-      // Every judge scores every match, so the reservation has to cover the ensemble.
-      const expectedCalls = estimateRoutedCalls(decision, selected.autoVerifyRepeats, DEFAULT_CRITERIA.length) * selected.judges.length
+      // Every judge scores every match, and compare/select round the repeat count up to an
+      // even number so the A/B slots get swapped, so the reservation has to cover both.
+      const repeats = routedRepeats(decision, selected.autoVerifyRepeats)
+      const expectedCalls = estimateRoutedCalls(decision, repeats, DEFAULT_CRITERIA.length) * selected.judges.length
       const reservation = autoRouter.reserve(agent, decision.kind, decision.fingerprint, expectedCalls, policy)
       if (reservation === undefined) {
         // A refused reservation used to drop the whole routed decision silently.
@@ -483,7 +464,7 @@ export function apply(ctx: Context, config: Config = {}): void {
         try {
           const extracted = await extractTask(agent, evidence.taskStartSeq, admittedLastSeq, selected.autoVerifyMaxChars, signal)
           if (decision.kind === 'compare') {
-            const result = await compareCandidates(agent, extracted.problem, decision.candidates[0].content, decision.candidates[1].content, selected.autoVerifyRepeats, signal, extracted.images, 'compare')
+            const result = await compareCandidates(agent, extracted.problem, decision.candidates[0].content, decision.candidates[1].content, repeats, signal, extracted.images, 'compare')
             if (!stillCurrent()) { autoRouter.fail(agent, reservation, false); return }
             if (!autoRouter.commit(agent, reservation, admittedLastSeq)) return
             const winner = result.winner === 'A' ? decision.candidates[0].label : result.winner === 'B' ? decision.candidates[1].label : 'tie'
@@ -491,18 +472,23 @@ export function apply(ctx: Context, config: Config = {}): void {
             return
           }
           if (decision.kind === 'select') {
-            const result = await selectCandidates(agent, extracted.problem, decision.candidates.map(candidate => candidate.content), selected.autoVerifyRepeats, signal, extracted.images, 'select')
+            const result = await selectCandidates(agent, extracted.problem, decision.candidates.map(candidate => candidate.content), repeats, signal, extracted.images, 'select')
             if (!stillCurrent()) { autoRouter.fail(agent, reservation, false); return }
             if (!autoRouter.commit(agent, reservation, admittedLastSeq)) return
             const ranking = result.ranking.map((index, rank) => (rank + 1) + '. ' + decision.candidates[index]!.label).join('\n')
             agent.steer(routeFeedback(decision, 'Ranking:\n' + ranking + '\nProceed with ' + decision.candidates[result.index]!.label + '.'))
             return
           }
-          const result = await trackProgress(agent, extracted.problem, decision.steps, decision.checkpoints, selected.autoVerifyRepeats, signal, extracted.images, 'track')
+          const result = await trackProgress(agent, extracted.problem, decision.steps, decision.checkpoints, repeats, signal, extracted.images, 'track')
           if (!stillCurrent()) { autoRouter.fail(agent, reservation, false); return }
           if (!autoRouter.commit(agent, reservation, admittedLastSeq)) return
           const detail = result.scores.map((score, index) => 'Checkpoint step ' + decision.checkpoints[index] + ': ' + (score * 100).toFixed(1) + '%').join('\n')
-          const continuation = result.scores.some(score => score < selected.autoTrackCompletionThreshold) ? '\nContinue the unfinished work.' : '\nPrepare final delivery evidence; final session verification is mandatory.'
+          // Judge the CURRENT state, not the whole history: the first checkpoint is the
+          // state right after the first todo snapshot, which is always "nothing done
+          // yet", so "any checkpoint below threshold" made the continue branch
+          // unconditional (5/5 live routes) and the threshold meaningless.
+          const latest = result.scores.length > 0 ? result.scores[result.scores.length - 1]! : 0
+          const continuation = latest < selected.autoTrackCompletionThreshold ? '\nContinue the unfinished work.' : '\nPrepare final delivery evidence; final session verification is mandatory.'
           agent.steer(routeFeedback(decision, detail + continuation))
           return
         } catch (error) {
@@ -542,11 +528,15 @@ export function apply(ctx: Context, config: Config = {}): void {
     try {
       const result = await verifySession(agent, { fromSeq: finalFromSeq, toSeq: admittedLastSeq, includeAssistantText: true, maxChars: selected.autoVerifyMaxChars, repeats: selected.autoVerifyFinalRepeats }, signal, 'final')
       if (!stillCurrent()) { autoRouter.fail(agent, finalReservation, false); return }
-      const passed = result.winner === 'A' && result.score >= selected.autoVerifyThreshold
+      // The mean over criteria used to hide a single failed requirement, and the fixed
+      // empty-work baseline can never win on its own, so the gate is (a) the winner,
+      // (b) the mean score and (c) every criterion on its own.
+      const failed = failedAcceptanceCriteria(result.criteria, selected.autoVerifyThreshold)
+      const passed = sessionAccepted({ score: result.score, winner: result.winner, criteria: result.criteria }, selected.autoVerifyThreshold)
       if (passed) autoRouter.commit(agent, finalReservation)
       else {
         autoRouter.fail(agent, finalReservation, selected.autoVerifyMode === 'strict')
-        agent.steer(createUserMessage({ content: [{ type: 'text', text: automaticFeedback(result.score, result.baselineScore, result.winner, selected.autoVerifyThreshold) }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }))
+        agent.steer(createUserMessage({ content: [{ type: 'text', text: automaticFeedback(result.score, result.baselineScore, result.winner, selected.autoVerifyThreshold, failed) }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }))
       }
     } catch (error) {
       autoRouter.fail(agent, finalReservation, selected.autoVerifyMode === 'strict')

package/src/router.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from 'vitest'
 import { Session } from '@deepseek-ai/dsh-session'
-import { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
-import { analyzeStructuredRoute, AutoVerifierRouter, boundDecision, buildSemanticRoutePrompt, estimateRoutedCalls, latestDirectUserSeq, MAX_ROUTED_CHECKPOINTS, parseSemanticRoute, semanticDecision, semanticRouteHint, type RouterPolicy } from './router.ts'
+import { createAssistantMessage, createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
+import { analyzeStructuredRoute, AutoVerifierRouter, boundDecision, buildSemanticRoutePrompt, estimateRoutedCalls, latestDirectUserSeq, MAX_ROUTED_CHECKPOINTS, parseSemanticRoute, routedRepeats, semanticDecision, semanticRouteHint, type RouterPolicy } from './router.ts'
 import { sanitizeVerifierText } from './session.ts'
 
 function session() {
@@ -13,6 +13,9 @@ function tool(value: ReturnType<typeof session>, name: string, id: string, text:
   value.append('tool/call', { turn, step, callId: id as never, name, arguments: '{}' })
   value.append('tool/result', { turn, step, message: createToolResultMessage({ callId: id as never, content: [{ type: 'text', text }], isError: false }) }, { surfaceOp: 'append' })
 }
+function assistant(value: ReturnType<typeof session>, text: string, turn = 1, step = 1) {
+  value.append('assistant/message', { turn, step, message: createAssistantMessage({ content: [{ type: 'text', text }], source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' } }) }, { surfaceOp: 'append' })
+}
 const policy: RouterPolicy = { mode: 'smart', minConfidence: .9, maxCandidates: 8, maxPerTask: 5, maxPerSession: 20, maxModelCallsPerTask: 48, maxModelCallsPerSession: 160, maxInputChars: 60000, maxItemChars: 20000 }
 const envelope = (count: number) => JSON.stringify({ protocol: 'dsh-verifier-candidates', version: 1, groupId: 'auth', candidates: Array.from({ length: count }, (_, i) => ({ id: String(i + 1), label: 'C' + (i + 1), status: 'completed', content: 'candidate ' + (i + 1) })) })
 
@@ -45,6 +48,158 @@ describe('production structured routing', () => {
       expect(decision.steps.every(step => step.length <= 400)).toBe(true)
     }
   })
+  it('never attaches bookkeeping output as checkpoint evidence', () => {
+    // Regression: the newest successful result before a todo snapshot is usually the
+    // bookkeeping call that wrote it (create_goal/update_goal/todo_write/interrupt_agent),
+    // so every checkpoint showed the router its own metadata instead of the work that
+    // had just run the task's tests.
+    const value = session()
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    tool(value, 'pwsh', 'run', 'all tests passed: 91 passed')
+    tool(value, 'create_goal', 'goal', '{"goal":{"id":"g1","phase":"active"}}')
+    tool(value, 'interrupt_agent', 'stop', 'interrupt requested for agent abc')
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    const decision = analyzeStructuredRoute(value.events, 8, 400, 800)
+    expect(decision?.kind).toBe('track')
+    if (decision?.kind !== 'track') return
+    // The real run is older than both bookkeeping results, and must still win.
+    expect(decision.steps[1]).toContain('all tests passed')
+    expect(decision.steps[1]).not.toContain('"phase":"active"')
+    expect(decision.steps[1]).not.toContain('interrupt requested')
+  })
+
+  it('attaches the newest narration to the current checkpoint only', () => {
+    // A prose deliverable (review, analysis) never reaches a tool result, so every
+    // checkpoint scored "certainly NO" while the same session's final acceptance
+    // passed. The narration is labelled as a claim, and only the checkpoint the route
+    // actually judges carries it.
+    const value = session()
+    assistant(value, 'Early plan: read the parser first.')
+    value.append('todo/write', { todos: [{ content: 'Review', status: 'in_progress' }, { content: 'Report', status: 'pending' }] })
+    assistant(value, 'Deliverable: the review found two blocking issues.', 1, 2)
+    value.append('todo/write', { todos: [{ content: 'Review', status: 'completed' }, { content: 'Report', status: 'completed' }] })
+    const decision = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(decision?.kind).toBe('track')
+    if (decision?.kind !== 'track') return
+    expect(decision.steps[0]).not.toContain('Newest agent narration')
+    expect(decision.steps[1]).toContain('Newest agent narration')
+    expect(decision.steps[1]).toContain('two blocking issues')
+    expect(decision.steps[1]).not.toContain('Early plan')
+  })
+
+  it('keeps narration and evidence inside the checkpoint caps', () => {
+    const value = session()
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    tool(value, 'pwsh', 'run', 'Z'.repeat(4000))
+    assistant(value, 'Q'.repeat(4000))
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    for (const maxItemChars of [128, 200, 2000, 20000]) {
+      const maxInputChars = Math.max(60000, maxItemChars * 2)
+      const decision = analyzeStructuredRoute(value.events, 8, maxItemChars, maxInputChars)
+      expect(decision, 'maxItemChars=' + maxItemChars).toMatchObject({ kind: 'track' })
+      if (decision?.kind !== 'track') continue
+      for (const step of decision.steps) expect(step.length, 'maxItemChars=' + maxItemChars).toBeLessThanOrEqual(maxItemChars)
+      expect(decision.steps.reduce((sum, step) => sum + step.length, 0)).toBeLessThanOrEqual(maxInputChars)
+      // Regression guard for the caps that boundDecision() enforces on the whole route.
+      expect(boundDecision(decision, { ...policy, maxItemChars, maxInputChars }), 'maxItemChars=' + maxItemChars).toBeDefined()
+    }
+  })
+
+  it('re-routes the same todo snapshots once the rendered evidence changed', () => {
+    // Regression: the fingerprint covered only the todo snapshots, so a route whose
+    // prompt had changed (new evidence, new narration) was refused as "already routed".
+    const value = session()
+    tool(value, 'pwsh', 'run-a', 'first output')
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    tool(value, 'pwsh', 'run-b', 'second output')
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    const first = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    const again = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(first?.kind).toBe('track')
+    // An unchanged log keeps its identity, so the budget guard still works.
+    expect(again?.fingerprint).toBe(first?.fingerprint)
+    assistant(value, 'Report: the deliverable is ready.', 2, 1)
+    const afterNarration = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(afterNarration?.fingerprint).not.toBe(first?.fingerprint)
+    if (first?.kind !== 'track' || afterNarration?.kind !== 'track') return
+    expect(afterNarration.steps.at(-1)).toContain('Report: the deliverable is ready.')
+  })
+
+  it('renders the newest checkpoint as the state at routing time', () => {
+    // Regression: evidence was frozen at the last todo snapshot's seq, so work done
+    // after it — often the actual verification run — never reached the judge.
+    const value = session()
+    tool(value, 'pwsh', 'first', 'first run output')
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    tool(value, 'pwsh', 'second', 'verification run output')
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    tool(value, 'pwsh', 'third', 'output after the last snapshot')
+    const decision = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(decision?.kind).toBe('track')
+    if (decision?.kind !== 'track') return
+    // Historical checkpoints keep the output that was current for them...
+    expect(decision.steps[0]).toContain('first run output')
+    expect(decision.steps[0]).not.toContain('verification run output')
+    // ...while the newest one carries the newest output and says so.
+    expect(decision.steps[1]).toContain('at routing time')
+    expect(decision.steps[1]).toContain('output after the last snapshot')
+  })
+
+  it('treats a wrapper that only dispatched bookkeeping as bookkeeping', () => {
+    // Regression: a PTC program that only called todo_write produced a run_code result
+    // whose text was the todo list echoed back, and the wrapper's name hid what it did.
+    const value = session()
+    tool(value, 'pwsh', 'run', 'all tests passed: 91 passed')
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    value.append('tool/call', { turn: 1, step: 1, callId: 'wrap' as never, name: 'run_code', arguments: '{}' })
+    value.append('tool/ptc-dispatch' as never, { rootCallId: 'wrap', subCallId: 'wrap:ptc:1', name: 'todo_write', arguments: '{}', isError: false, content: [{ type: 'text', text: '{todos:[{content:"Implement",status:"completed"}]}' }] } as never)
+    value.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'wrap' as never, content: [{ type: 'text', text: '{ todos: [ { content: "Implement", status: "completed" } ], counts: {} }' }], isError: false }) }, { surfaceOp: 'append' })
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    const decision = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(decision?.kind).toBe('track')
+    if (decision?.kind !== 'track') return
+    expect(decision.steps[1]).toContain('all tests passed')
+    expect(decision.steps[1]).not.toContain('counts:')
+  })
+
+  it('never attaches a deliverable presentation as checkpoint evidence', () => {
+    // Regression (live session, four routes in a row): `present` is the last call of a
+    // turn, so the newest "observed tool output" reaching the judge was the wrapper's
+    // `presented: 1` — the paths the agent had just declared — while the typecheck and
+    // test run sat one call earlier. The judge's own rule ("a state without real
+    // verification should not exceed K") then capped the newest checkpoint at exactly
+    // K = 52.6% against a 0.8 threshold, so every route steered "continue the
+    // unfinished work" for work that was finished and verified, until the per-task
+    // route budget ran out.
+    const value = session()
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    tool(value, 'pwsh', 'verify', 'TYPECHECK_EXIT=0\nTests 309 passed')
+    value.append('tool/call', { turn: 1, step: 1, callId: 'wrap' as never, name: 'run_code', arguments: '{}' })
+    value.append('tool/ptc-dispatch' as never, { rootCallId: 'wrap', subCallId: 'wrap:ptc:1', name: 'present', arguments: '{}', isError: false, content: [{ type: 'text', text: 'Presented C:\\repo\\src\\mapper.ts' }] } as never)
+    value.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'wrap' as never, content: [{ type: 'text', text: 'presented: 1' }], isError: false }) }, { surfaceOp: 'append' })
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    const decision = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(decision?.kind).toBe('track')
+    if (decision?.kind !== 'track') return
+    // The verification run is older than the presentation, and must still win.
+    expect(decision.steps[1]).toContain('309 passed')
+    expect(decision.steps[1]).not.toContain('Presented C:')
+    expect(decision.steps[1]).not.toContain('presented: 1')
+  })
+
+  it('keeps a wrapper that dispatched real work', () => {
+    const value = session()
+    value.append('tool/call', { turn: 1, step: 1, callId: 'wrap' as never, name: 'run_code', arguments: '{}' })
+    value.append('tool/ptc-dispatch' as never, { rootCallId: 'wrap', subCallId: 'wrap:ptc:1', name: 'pwsh', arguments: '{}', isError: false, content: [{ type: 'text', text: 'tests passed' }] } as never)
+    value.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'wrap' as never, content: [{ type: 'text', text: 'tests passed in the wrapper output' }], isError: false }) }, { surfaceOp: 'append' })
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'in_progress' }, { content: 'Test', status: 'pending' }] })
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'completed' }] })
+    const decision = analyzeStructuredRoute(value.events, 8, 20000, 60000)
+    expect(decision?.kind).toBe('track')
+    if (decision?.kind !== 'track') return
+    expect(decision.steps[1]).toContain('tests passed in the wrapper output')
+  })
+
   it('deduplicates identical todo snapshots', () => {
     const value = session()
     const todos = [{ content: 'Implement', status: 'in_progress' as const }, { content: 'Test', status: 'pending' as const }]
@@ -99,6 +254,32 @@ describe('semantic evidence references', () => {
     expect(semanticDecision(parsed, value.events)).toMatchObject({ kind: 'compare', source: 'semantic' })
   })
 
+  it('never offers bookkeeping calls as semantic candidates', () => {
+    // Regression: the artifact list carried every paired tool result, so the classifier
+    // could cite two goal/agent-control calls as "competing alternatives".
+    const value = session()
+    tool(value, 'subagent', 'a', 'candidate A from a real subagent')
+    tool(value, 'create_goal', 'goal', '{"goal":{"id":"g1","phase":"active"}}')
+    tool(value, 'skill', 'skill', 'loaded review skill body')
+    tool(value, 'present', 'present', 'Presented C:\\repo\\src\\mapper.ts')
+    const prompt = buildSemanticRoutePrompt('pick the better one', value.events, 8, 20000, 60000)
+    expect(prompt).toContain('candidate A from a real subagent')
+    expect(prompt).not.toContain('"phase":"active"')
+    expect(prompt).not.toContain('loaded review skill body')
+    expect(prompt).not.toContain('Presented C:')
+  })
+
+  it('rejects a semantic decision that cites bookkeeping evidence', () => {
+    const value = session()
+    tool(value, 'create_goal', 'goal-1', '{"goal":{"id":"g1"}}')
+    tool(value, 'create_goal', 'goal-2', '{"goal":{"id":"g2"}}')
+    const parsed = parseSemanticRoute(JSON.stringify({ kind: 'compare', confidence: .95, reason: 'alternatives', candidateCallIds: ['goal-1', 'goal-2'], checkpointSeqs: [] }))
+    expect(parsed?.kind).toBe('compare')
+    // Fail closed: the whole decision is dropped, and the caller records the reference
+    // as invalid instead of comparing metadata.
+    expect(semanticDecision(parsed!, value.events)).toBeUndefined()
+  })
+
   it('resolves candidates emitted via Session V3 tool/ptc-dispatch', () => {
     const value = session()
     value.append('tool/ptc-dispatch' as never, { subCallId: 'ptc-1', name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate A from ptc' }] } as never)
@@ -210,10 +391,28 @@ describe('semantic evidence references', () => {
     const value = session()
     value.append('team/task' as never, { task: { id: 'task-1', revision: 1, subject: 'Backend API', status: 'in_progress' } } as never)
     value.append('team/task' as never, { task: { id: 'task-1', revision: 2, subject: 'Backend API', status: 'completed' } } as never)
-    expect(semanticRouteHint(value.events)).toBe(true)
+    // Snapshots are structured material, not a semantic hint: the structured pass claims
+    // them first, so asking the classifier about them would only ever pay for a route
+    // that could not have been produced anyway.
+    expect(semanticRouteHint(value.events)).toBe(false)
     const decision = analyzeStructuredRoute(value.events)
     expect(decision).toMatchObject({ kind: 'track', source: 'structured', reason: 'changed durable team tasks' })
   })
+
+  it('hints a classification only for material the structured pass never consumes', () => {
+    const subagent = session(); tool(subagent, 'subagent', 'a', 'candidate A')
+    expect(semanticRouteHint(subagent.events)).toBe(true)
+    const plan = session()
+    plan.append('tool/call', { turn: 1, step: 1, callId: 'plan' as never, name: 'exit_plan_mode', arguments: '{}' })
+    plan.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'plan' as never, content: [{ type: 'text', text: 'approved' }], isError: false }) }, { surfaceOp: 'append' })
+    expect(semanticRouteHint(plan.events)).toBe(true)
+    // Todo snapshots alone (all lists shorter than two items) are still structured-only.
+    const todos = session()
+    todos.append('todo/write', { todos: [{ content: 'Only step', status: 'in_progress' }] })
+    todos.append('todo/write', { todos: [{ content: 'Only step', status: 'completed' }] })
+    expect(analyzeStructuredRoute(todos.events)).toBeUndefined()
+    expect(semanticRouteHint(todos.events)).toBe(false)
+  })
 })
 
 describe('transactional router state', () => {
@@ -256,6 +455,22 @@ describe('transactional router state', () => {
     value.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'next task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
     expect(router.claimExhaustedNotice(agent)).toBe(true)
   })
+  it('rounds compare repeats up to an even count and leaves the other routes alone', () => {
+    // compare judges ONE pair, so an odd count leaves its A/B preference uncorrected
+    // (the engine only swaps positions on odd repeat indices). select is already
+    // symmetric per pair and track has no slots, so both keep the configured count.
+    const compare = { kind: 'compare', candidates: [{}, {}] } as never
+    const select = { kind: 'select', candidates: [{}, {}, {}] } as never
+    const track = { kind: 'track', steps: [] } as never
+    expect(routedRepeats(compare, 1)).toBe(2)
+    expect(routedRepeats(compare, 2)).toBe(2)
+    expect(routedRepeats(compare, 3)).toBe(4)
+    expect(routedRepeats(select, 1)).toBe(1)
+    expect(routedRepeats(select, 3)).toBe(3)
+    expect(routedRepeats(track, 1)).toBe(1)
+    expect(routedRepeats(track, 3)).toBe(3)
+  })
+
   it('estimates routed calls from the real tournament shape', () => {
     const candidate = { id: 'c', groupId: 'g', label: 'c', content: 'x', callId: 'a', fromSeq: 1, toSeq: 2 }
     const base = { source: 'structured' as const, confidence: 1, reason: 'r', fingerprint: 'f' }