dsh-llm-verifier 0.1.13 → 0.1.16

package/src/parity.test.ts

@@ -1,85 +1,85 @@
-import { execFileSync } from 'node:child_process'
-import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { describe, expect, it } from 'vitest'
-import { ScoreCache } from './cache.ts'
-import { extractProgressScore, extractScore, pivotRoundPairs } from './core.ts'
-import { resolveConfig } from './config.ts'
-
-const PYTHON_ROOT = join(import.meta.dirname, '..', '..', 'llm-as-a-verifier')
-const hasPythonUpstream = existsSync(join(PYTHON_ROOT, 'llm_verifier'))
-
-function python(payload: unknown): unknown {
-  const script = [
-    'import json,sys',
-    'sys.path.insert(0, sys.argv[1])',
-    'from llm_verifier.fine_grained_reward import extract_score',
-    'from llm_verifier.pivot_tournament import pivot_round_pairs',
-    'd=json.loads(sys.stdin.read())',
-    "result = extract_score(d['text'], d['tokens'], d['positions'], d['tag']) if d['kind']=='score' else pivot_round_pairs(d['n'], d['pivots'])",
-    'print(json.dumps(result))',
-  ].join(';')
-  const source = payload as Record<string, unknown>
-  const compatible = source.kind === 'score' ? { ...source, positions: (source.positions as Array<Array<{ token: string; logprob: number }>>).map(position => position.map(item => [item.token, item.logprob])) } : source
-  const result = execFileSync('py', ['-3', '-c', script, PYTHON_ROOT], { input: JSON.stringify(compatible), encoding: 'utf8' })
-  return JSON.parse(result)
-}
-
-describe('Python parity fixtures', () => {
-  it.skipIf(!hasPythonUpstream)('matches literal and distribution score extraction', () => {
-    const fixtures = [
-      { text: '<score_A> A </score_A>', tokens: [], positions: [], tag: '<score_A>' },
-      { text: '<score_A> A </score_A> then <score_A> T </score_A>', tokens: [], positions: [], tag: '<score_A>' },
-      { text: '', tokens: ['<score_A>'], positions: [[], [{ token: 'A', logprob: Math.log(0.7) }, { token: 'T', logprob: Math.log(0.3) }]], tag: '<score_A>' },
-      { text: '', tokens: ['<score_A'], positions: [[], [{ token: '>B', logprob: Math.log(0.8) }, { token: '>S', logprob: Math.log(0.2) }]], tag: '<score_A>' },
-    ]
-    for (const fixture of fixtures) {
-      const ts = extractScore(fixture, fixture.tag)
-      const py = python({ kind: 'score', ...fixture }) as number
-      expect(ts).toBeCloseTo(py, 12)
-    }
-  })
-  it.skipIf(!hasPythonUpstream)('matches pivot pair generation', () => {
-    expect(pivotRoundPairs(7, [1, 4, 5])).toEqual(python({ kind: 'pivot', n: 7, pivots: [1, 4, 5] }))
-  })
-  it('matches reversed progress convention', () => {
-    expect(extractProgressScore({ text: '<c1> T </c1>', tokens: [], positions: [] }, '<c1>')).toBe(1)
-  })
-})
-
-describe('DSH model routing and cache', () => {
-  it('accepts any non-empty DSH provider and model route', () => {
-    expect(resolveConfig({ provider: 'openai', model: 'gpt-5' })).toMatchObject({ provider: 'openai', model: 'gpt-5' })
-    expect(resolveConfig({ provider: 'anthropic', model: 'claude-sonnet' })).toMatchObject({ provider: 'anthropic', model: 'claude-sonnet' })
-    expect(() => resolveConfig({ provider: '', model: 'gpt-5' })).toThrow(/provider must be non-empty/)
-    expect(resolveConfig({ autoRouteMaxItemChars: 1000, autoRouteMaxInputChars: 2000, autoMaxModelCallsPerTask: 4, autoMaxModelCallsPerSession: 8 })).toMatchObject({ autoRouteMaxItemChars: 1000, autoRouteMaxInputChars: 2000, autoMaxModelCallsPerTask: 4, autoMaxModelCallsPerSession: 8 })
-    expect(() => resolveConfig({ autoRouteMaxCandidates: 17 })).toThrow(/between 3 and 16/)
-    expect(() => resolveConfig({ autoRouteMaxItemChars: 2000, autoRouteMaxInputChars: 3000 })).toThrow(/fit at least two/)
-  })
-  it('rejects malformed persisted entries and recreates them', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-invalid-'))
-    try {
-      const file = join(dir, 'scores.json')
-      writeFileSync(file, JSON.stringify({ version: 1, entries: { bad: { scoreA: 'NaN', scoreB: 0, usage: {}, createdAt: -1 } } }))
-      const cache = new ScoreCache(file, 100)
-      let creates = 0
-      const result = await cache.getOrCreate('bad', async () => { creates++; return { scoreA: .8, scoreB: .2, usage: { calls: 1, attempts: 1, retries: 0, inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, reasoningTokens: 0 }, scoringMode: 'explicit-tag' as const, createdAt: Date.now() } })
-      expect(result.hit).toBe(false); expect(creates).toBe(1)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-  it('persists successful results and avoids duplicate creation', async () => {
-    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-'))
-    try {
-      const file = join(dir, 'scores.json')
-      const first = new ScoreCache(file, 100)
-      let creates = 0
-      const create = async () => { creates++; return { scoreA: 1, scoreB: 0, usage: { calls: 1, attempts: 1, retries: 0, inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, reasoningTokens: 0 }, scoringMode: 'top-logprobs' as const, createdAt: Date.now() } }
-      expect((await first.getOrCreate('key', create)).hit).toBe(false)
-      expect((await first.getOrCreate('key', create)).hit).toBe(true)
-      const second = new ScoreCache(file, 100)
-      expect((await second.getOrCreate('key', create)).hit).toBe(true)
-      expect(creates).toBe(1)
-    } finally { rmSync(dir, { recursive: true, force: true }) }
-  })
-})
+import { execFileSync } from 'node:child_process'
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { ScoreCache } from './cache.ts'
+import { extractProgressScore, extractScore, pivotRoundPairs } from './core.ts'
+import { resolveConfig } from './config.ts'
+
+const PYTHON_ROOT = join(import.meta.dirname, '..', '..', 'llm-as-a-verifier')
+const hasPythonUpstream = existsSync(join(PYTHON_ROOT, 'llm_verifier'))
+
+function python(payload: unknown): unknown {
+  const script = [
+    'import json,sys',
+    'sys.path.insert(0, sys.argv[1])',
+    'from llm_verifier.fine_grained_reward import extract_score',
+    'from llm_verifier.pivot_tournament import pivot_round_pairs',
+    'd=json.loads(sys.stdin.read())',
+    "result = extract_score(d['text'], d['tokens'], d['positions'], d['tag']) if d['kind']=='score' else pivot_round_pairs(d['n'], d['pivots'])",
+    'print(json.dumps(result))',
+  ].join(';')
+  const source = payload as Record<string, unknown>
+  const compatible = source.kind === 'score' ? { ...source, positions: (source.positions as Array<Array<{ token: string; logprob: number }>>).map(position => position.map(item => [item.token, item.logprob])) } : source
+  const result = execFileSync('py', ['-3', '-c', script, PYTHON_ROOT], { input: JSON.stringify(compatible), encoding: 'utf8' })
+  return JSON.parse(result)
+}
+
+describe('Python parity fixtures', () => {
+  it.skipIf(!hasPythonUpstream)('matches literal and distribution score extraction', () => {
+    const fixtures = [
+      { text: '<score_A> A </score_A>', tokens: [], positions: [], tag: '<score_A>' },
+      { text: '<score_A> A </score_A> then <score_A> T </score_A>', tokens: [], positions: [], tag: '<score_A>' },
+      { text: '', tokens: ['<score_A>'], positions: [[], [{ token: 'A', logprob: Math.log(0.7) }, { token: 'T', logprob: Math.log(0.3) }]], tag: '<score_A>' },
+      { text: '', tokens: ['<score_A'], positions: [[], [{ token: '>B', logprob: Math.log(0.8) }, { token: '>S', logprob: Math.log(0.2) }]], tag: '<score_A>' },
+    ]
+    for (const fixture of fixtures) {
+      const ts = extractScore(fixture, fixture.tag)
+      const py = python({ kind: 'score', ...fixture }) as number
+      expect(ts).toBeCloseTo(py, 12)
+    }
+  })
+  it.skipIf(!hasPythonUpstream)('matches pivot pair generation', () => {
+    expect(pivotRoundPairs(7, [1, 4, 5])).toEqual(python({ kind: 'pivot', n: 7, pivots: [1, 4, 5] }))
+  })
+  it('matches reversed progress convention', () => {
+    expect(extractProgressScore({ text: '<c1> T </c1>', tokens: [], positions: [] }, '<c1>')).toBe(1)
+  })
+})
+
+describe('DSH model routing and cache', () => {
+  it('accepts any non-empty DSH provider and model route', () => {
+    expect(resolveConfig({ provider: 'openai', model: 'gpt-5' })).toMatchObject({ provider: 'openai', model: 'gpt-5' })
+    expect(resolveConfig({ provider: 'anthropic', model: 'claude-sonnet' })).toMatchObject({ provider: 'anthropic', model: 'claude-sonnet' })
+    expect(() => resolveConfig({ provider: '', model: 'gpt-5' })).toThrow(/provider must be non-empty/)
+    expect(resolveConfig({ autoRouteMaxItemChars: 1000, autoRouteMaxInputChars: 2000, autoMaxModelCallsPerTask: 4, autoMaxModelCallsPerSession: 8 })).toMatchObject({ autoRouteMaxItemChars: 1000, autoRouteMaxInputChars: 2000, autoMaxModelCallsPerTask: 4, autoMaxModelCallsPerSession: 8 })
+    expect(() => resolveConfig({ autoRouteMaxCandidates: 17 })).toThrow(/between 3 and 16/)
+    expect(() => resolveConfig({ autoRouteMaxItemChars: 2000, autoRouteMaxInputChars: 3000 })).toThrow(/fit at least two/)
+  })
+  it('rejects malformed persisted entries and recreates them', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-invalid-'))
+    try {
+      const file = join(dir, 'scores.json')
+      writeFileSync(file, JSON.stringify({ version: 1, entries: { bad: { scoreA: 'NaN', scoreB: 0, usage: {}, createdAt: -1 } } }))
+      const cache = new ScoreCache(file, 100)
+      let creates = 0
+      const result = await cache.getOrCreate('bad', async () => { creates++; return { scoreA: .8, scoreB: .2, usage: { calls: 1, attempts: 1, retries: 0, inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, reasoningTokens: 0 }, scoringMode: 'explicit-tag' as const, createdAt: Date.now() } })
+      expect(result.hit).toBe(false); expect(creates).toBe(1)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+  it('persists successful results and avoids duplicate creation', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'dsh-verifier-'))
+    try {
+      const file = join(dir, 'scores.json')
+      const first = new ScoreCache(file, 100)
+      let creates = 0
+      const create = async () => { creates++; return { scoreA: 1, scoreB: 0, usage: { calls: 1, attempts: 1, retries: 0, inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, reasoningTokens: 0 }, scoringMode: 'top-logprobs' as const, createdAt: Date.now() } }
+      expect((await first.getOrCreate('key', create)).hit).toBe(false)
+      expect((await first.getOrCreate('key', create)).hit).toBe(true)
+      const second = new ScoreCache(file, 100)
+      expect((await second.getOrCreate('key', create)).hit).toBe(true)
+      expect(creates).toBe(1)
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+})

package/src/plan-gate.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest'
+import { buildPlanPreReviewPrompt, parseVerdictLetter, planFromArguments } from './plan-gate.ts'
+
+describe('plan-gate', () => {
+  it('reads the plan markdown out of exit_plan_mode arguments', () => {
+    expect(planFromArguments({ plan: '  # Plan\nStep 1  ' })).toBe('# Plan\nStep 1')
+    expect(planFromArguments({ plan: '   ' })).toBe('')
+    expect(planFromArguments({})).toBe('')
+    expect(planFromArguments({ plan: 42 })).toBe('')
+    expect(planFromArguments('plan')).toBe('')
+    expect(planFromArguments(null)).toBe('')
+    expect(planFromArguments(undefined)).toBe('')
+  })
+
+  it('builds plan review prompt with requirements and an exact verdict line', () => {
+    const prompt = buildPlanPreReviewPrompt('Implement Redis cache', '1. Connect redis 2. Cache queries', 5000)
+    expect(prompt).toContain('You are an expert independent technical plan verifier')
+    expect(prompt).toContain('Implement Redis cache')
+    expect(prompt).toContain('1. Connect redis 2. Cache queries')
+    expect(prompt).toContain('Verdict: <single uppercase letter A-T>')
+  })
+
+  it('parses a whole verdict line into the A-T score scale', () => {
+    const perfect = parseVerdictLetter('Verdict: A\nSummary: Outstanding plan.\nVerification is complete.')
+    expect(perfect?.verdict).toBe('A')
+    expect(perfect?.score).toBe(1.0)
+    expect(perfect?.feedback).toContain('Outstanding plan')
+
+    const failed = parseVerdictLetter('Verdict: T\nSummary: Completely unfeasible.')
+    expect(failed?.verdict).toBe('T')
+    expect(failed?.score).toBe(0.0)
+
+    const middle = parseVerdictLetter('Verdict: C\nSummary: Mostly good.')
+    expect(middle?.verdict).toBe('C')
+    expect(middle?.score).toBeCloseTo(1 - 2 / 19, 3)
+
+    expect(parseVerdictLetter('Summary: fine\n  verdict:  d  \nNotes: ok')?.score).toBeCloseTo(1 - 3 / 19, 3)
+  })
+
+  it('reads a verdict line wrapped in markdown decoration', () => {
+    expect(parseVerdictLetter('**Verdict: A**\nSummary: ok')?.verdict).toBe('A')
+    expect(parseVerdictLetter('**Verdict**: B')?.score).toBeCloseTo(1 - 1 / 19, 3)
+    expect(parseVerdictLetter('- Verdict: C.')?.verdict).toBe('C')
+    expect(parseVerdictLetter('1. Verdict: D)')?.verdict).toBe('D')
+    expect(parseVerdictLetter('`Verdict: E`')?.verdict).toBe('E')
+    expect(parseVerdictLetter('### Verdict: T')?.score).toBe(0)
+  })
+
+  it('never scores prose, a missing verdict, or a trailing explanation', () => {
+    expect(parseVerdictLetter('Verdict: Failed\nSummary: the plan cannot work')).toBeUndefined()
+    expect(parseVerdictLetter('Verdict: Approved')).toBeUndefined()
+    expect(parseVerdictLetter('Verdict: A because the plan is great')).toBeUndefined()
+    expect(parseVerdictLetter('Summary: no verdict line at all')).toBeUndefined()
+    expect(parseVerdictLetter('')).toBeUndefined()
+  })
+})

package/src/plan-gate.ts

@@ -0,0 +1,66 @@
+import { sanitizeVerifierText } from './session.ts'
+
+export interface PlanVerdict {
+  verdict: string
+  score: number
+  feedback: string
+}
+
+/**
+ * Read the plan markdown out of one `exit_plan_mode` argument object.
+ * @param args - Parsed tool arguments; anything but `{ plan: string }` yields ''.
+ * @returns Trimmed plan markdown, or '' when the call carries no plan.
+ */
+export function planFromArguments(args: unknown): string {
+  if (typeof args !== 'object' || args === null) return ''
+  const plan = (args as { plan?: unknown }).plan
+  return typeof plan === 'string' ? plan.trim() : ''
+}
+
+export function buildPlanPreReviewPrompt(problem: string, planText: string, maxChars = 20000): string {
+  return [
+    'You are an expert independent technical plan verifier. A candidate implementation plan is about to be submitted to the human for approval.',
+    'Evaluate whether the plan is sound, executable, and comprehensive.',
+    'Scoring criteria:',
+    'A: Flawless, thorough, with clear steps, rigorous verification, edge cases handled.',
+    'B-J: Mostly sound, minor gaps or nuances missing.',
+    'K-S: Noticeable architectural gaps, missing regression checks, or ambiguities.',
+    'T: Fundamentally flawed, dangerous, or missing core requirements.',
+    '',
+    'Task requirement:',
+    sanitizeVerifierText(problem, 4000),
+    '',
+    'Proposed Plan:',
+    sanitizeVerifierText(planText, maxChars),
+    '',
+    'Output format:',
+    'The first line must be exactly "Verdict: <single uppercase letter A-T>" — one letter, nothing else on the line.',
+    'Line 2: Summary: <One sentence assessment>',
+    'Line 3+: Key strengths, blind spots, and verification guidance.',
+  ].join('\n\n')
+}
+
+/**
+ * Parse the `Verdict: <A-T>` line the judge prompts require.
+ *
+ * Markdown decoration around the required line is stripped first, so "**Verdict:
+ * A**", "- Verdict: B." or a numbered list marker still produce a grade — a
+ * formatting habit must never silently switch the gate off. Prose that is not a
+ * grade is still rejected: "Verdict: Failed" and "Verdict: Approved" carry no
+ * verdict letter, and an answer without a verdict line has no score at all —
+ * callers must skip such a review, never treat it as a pass. A is 1.0 and T is
+ * 0.0, matching the top-logprob A-T scale used by the judge prompts.
+ * @param text - Raw judge model output.
+ * @returns The parsed verdict, or undefined when no verdict line is present.
+ */
+export function parseVerdictLetter(text: string): PlanVerdict | undefined {
+  for (const rawLine of text.split(/\r?\n/)) {
+    const line = rawLine.replace(/[*_`~#>]/g, ' ').replace(/^\s*(?:[-+]|\d+[.)])\s+/, ' ').trim()
+    const match = /^verdict\s*:\s*([A-Ta-t])\s*[.)]?$/i.exec(line)
+    if (match === null) continue
+    const verdict = (match[1] as string).toUpperCase()
+    const score = Math.max(0, Math.min(1, 1 - (verdict.charCodeAt(0) - 65) / 19))
+    return { verdict, score, feedback: text.trim() }
+  }
+  return undefined
+}

package/src/router.test.ts

@@ -1,89 +1,107 @@
-import { describe, expect, it } from 'vitest'
-import { Session } from '@deepseek-ai/dsh-session'
-import { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
-import { analyzeStructuredRoute, AutoVerifierRouter, buildSemanticRoutePrompt, parseSemanticRoute, semanticDecision, semanticRouteHint, type RouterPolicy } from './router.ts'
-
-function session() {
-  const value = Session.create('session-00000000-0000-4000-8000-000000000077' as never)
-  value.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Choose and implement the best solution' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
-  return value
-}
-function tool(value: ReturnType<typeof session>, name: string, id: string, text: string, turn = 1, step = 1) {
-  value.append('tool/call', { turn, step, callId: id as never, name, arguments: '{}' })
-  value.append('tool/result', { turn, step, message: createToolResultMessage({ callId: id as never, content: [{ type: 'text', text }], isError: false }) }, { surfaceOp: 'append' })
-}
-const policy: RouterPolicy = { mode: 'smart', minConfidence: .9, maxCandidates: 8, maxPerTask: 5, maxPerSession: 20, maxModelCallsPerTask: 48, maxModelCallsPerSession: 160, maxInputChars: 60000, maxItemChars: 20000 }
-const envelope = (count: number) => JSON.stringify({ protocol: 'dsh-verifier-candidates', version: 1, groupId: 'auth', candidates: Array.from({ length: count }, (_, i) => ({ id: String(i + 1), label: 'C' + (i + 1), status: 'completed', content: 'candidate ' + (i + 1) })) })
-
-describe('production structured routing', () => {
-  it('requires a trusted versioned workflow envelope', () => {
-    const value = session(); tool(value, 'workflow', 'w', envelope(3))
-    expect(analyzeStructuredRoute(value.events)).toMatchObject({ kind: 'select', source: 'structured' })
-    const untrusted = session(); tool(untrusted, 'workflow', 'w', JSON.stringify({ verifier_candidates: ['a', 'b'] }))
-    expect(analyzeStructuredRoute(untrusted.events)).toBeUndefined()
-  })
-  it('does not guess that unrelated synchronous subagents are alternatives', () => {
-    const value = session(); tool(value, 'subagent', 'a', 'frontend analysis', 1, 2); tool(value, 'subagent', 'b', 'backend analysis', 1, 2)
-    expect(analyzeStructuredRoute(value.events)).toBeUndefined()
-    expect(semanticRouteHint(value.events)).toBe(true)
-  })
-  it('deduplicates identical todo snapshots', () => {
-    const value = session()
-    const todos = [{ content: 'Implement', status: 'in_progress' as const }, { content: 'Test', status: 'pending' as const }]
-    value.append('todo/write', { todos }); value.append('todo/write', { todos })
-    expect(analyzeStructuredRoute(value.events)).toBeUndefined()
-    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'in_progress' }] })
-    expect(analyzeStructuredRoute(value.events)).toMatchObject({ kind: 'track', evidenceSeqs: expect.any(Array) })
-  })
-  it('redacts and bounds trusted candidate content', () => {
-    const value = session(); tool(value, 'workflow', 'w', JSON.stringify({ protocol: 'dsh-verifier-candidates', version: 1, groupId: 'g', candidates: [{ id: 'a', status: 'completed', content: 'token = abc ' + 'x'.repeat(100) }, { id: 'b', status: 'completed', content: 'password: secret ' + 'y'.repeat(100) }] }))
-    const decision = analyzeStructuredRoute(value.events, 8, 40)
-    expect(decision?.kind).toBe('compare')
-    if (decision?.kind === 'compare') { expect(decision.candidates[0].content).not.toContain('abc'); expect(decision.candidates[0].content.length).toBeLessThan(80) }
-  })
-})
-
-describe('semantic evidence references', () => {
-  it('accepts exact strict JSON references and rejects prose/unknown keys', () => {
-    const valid = JSON.stringify({ kind: 'compare', confidence: .95, reason: 'alternatives', candidateCallIds: ['a', 'b'], checkpointSeqs: [] })
-    expect(parseSemanticRoute(valid)).toMatchObject({ kind: 'compare', candidateCallIds: ['a', 'b'] })
-    expect(parseSemanticRoute('```json\n' + valid + '\n```')).toBeUndefined()
-    expect(parseSemanticRoute(JSON.stringify({ ...JSON.parse(valid), extra: true }))).toBeUndefined()
-  })
-  it('resolves only real paired call ids', () => {
-    const value = session(); tool(value, 'subagent', 'a', 'candidate A'); tool(value, 'subagent', 'b', 'candidate B')
-    const parsed = parseSemanticRoute(JSON.stringify({ kind: 'compare', confidence: .95, reason: 'same task alternatives', candidateCallIds: ['a', 'b'], checkpointSeqs: [] }))!
-    expect(semanticDecision(parsed, value.events)).toMatchObject({ kind: 'compare', source: 'semantic' })
-    expect(semanticDecision({ ...parsed, candidateCallIds: ['a', 'missing'] }, value.events)).toBeUndefined()
-    expect(buildSemanticRoutePrompt('problem', value.events, 5)).toContain('candidateCallIds')
-  })
-  it('resolves candidates emitted via PTC mode tool/code-dispatch', () => {
-    const value = session()
-    value.append('tool/code-dispatch', { subCallId: 'c-1' as never, name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate 1 content' }] })
-    value.append('tool/code-dispatch', { subCallId: 'c-2' as never, name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate 2 content' }] })
-    expect(semanticRouteHint(value.events)).toBe(true)
-    const parsed = parseSemanticRoute(JSON.stringify({ kind: 'compare', confidence: 0.92, reason: 'PTC alternatives', candidateCallIds: ['c-1', 'c-2'], checkpointSeqs: [] }))!
-    expect(semanticDecision(parsed, value.events)).toMatchObject({ kind: 'compare', source: 'semantic' })
-  })
-})
-
-describe('transactional router state', () => {
-  it('commits, requires final verification, and clears it only after final commit', () => {
-    const value = session(); const agent = { id: value.id, session: value }; const router = new AutoVerifierRouter()
-    const route = router.reserve(agent, 'compare', 'route', 4, policy)!
-    expect(router.commit(agent, route, 9)).toBe(true); expect(router.finalRequired(agent)).toBe(9)
-    const final = router.reserve(agent, 'final', 'final', 4, policy)!
-    expect(router.commit(agent, final)).toBe(true); expect(router.finalRequired(agent)).toBeUndefined()
-  })
-  it('releases in-flight state after failure and strict remains blocked', () => {
-    const value = session(); const agent = { id: value.id, session: value }; const router = new AutoVerifierRouter()
-    const first = router.reserve(agent, 'compare', 'route-a', 4, { ...policy, mode: 'strict' })!
-    router.fail(agent, first, true); expect(router.strictBlocked(agent)).toBe(true)
-    expect(router.reserve(agent, 'compare', 'route-b', 4, { ...policy, mode: 'strict' })).toBeDefined()
-  })
-  it('enforces manual mode and unified model-call budgets', () => {
-    const value = session(); const agent = { id: value.id, session: value }; const router = new AutoVerifierRouter()
-    expect(router.reserve(agent, 'compare', 'manual', 1, { ...policy, mode: 'manual' })).toBeUndefined()
-    expect(router.reserve(agent, 'select', 'expensive', 49, policy)).toBeUndefined()
-  })
-})
+import { describe, expect, it } from 'vitest'
+import { Session } from '@deepseek-ai/dsh-session'
+import { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
+import { analyzeStructuredRoute, AutoVerifierRouter, buildSemanticRoutePrompt, parseSemanticRoute, semanticDecision, semanticRouteHint, type RouterPolicy } from './router.ts'
+
+function session() {
+  const value = Session.create('session-00000000-0000-4000-8000-000000000077' as never)
+  value.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Choose and implement the best solution' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
+  return value
+}
+function tool(value: ReturnType<typeof session>, name: string, id: string, text: string, turn = 1, step = 1) {
+  value.append('tool/call', { turn, step, callId: id as never, name, arguments: '{}' })
+  value.append('tool/result', { turn, step, message: createToolResultMessage({ callId: id as never, content: [{ type: 'text', text }], isError: false }) }, { surfaceOp: 'append' })
+}
+const policy: RouterPolicy = { mode: 'smart', minConfidence: .9, maxCandidates: 8, maxPerTask: 5, maxPerSession: 20, maxModelCallsPerTask: 48, maxModelCallsPerSession: 160, maxInputChars: 60000, maxItemChars: 20000 }
+const envelope = (count: number) => JSON.stringify({ protocol: 'dsh-verifier-candidates', version: 1, groupId: 'auth', candidates: Array.from({ length: count }, (_, i) => ({ id: String(i + 1), label: 'C' + (i + 1), status: 'completed', content: 'candidate ' + (i + 1) })) })
+
+describe('production structured routing', () => {
+  it('requires a trusted versioned workflow envelope', () => {
+    const value = session(); tool(value, 'workflow', 'w', envelope(3))
+    expect(analyzeStructuredRoute(value.events)).toMatchObject({ kind: 'select', source: 'structured' })
+    const untrusted = session(); tool(untrusted, 'workflow', 'w', JSON.stringify({ verifier_candidates: ['a', 'b'] }))
+    expect(analyzeStructuredRoute(untrusted.events)).toBeUndefined()
+  })
+  it('does not guess that unrelated synchronous subagents are alternatives', () => {
+    const value = session(); tool(value, 'subagent', 'a', 'frontend analysis', 1, 2); tool(value, 'subagent', 'b', 'backend analysis', 1, 2)
+    expect(analyzeStructuredRoute(value.events)).toBeUndefined()
+    expect(semanticRouteHint(value.events)).toBe(true)
+  })
+  it('deduplicates identical todo snapshots', () => {
+    const value = session()
+    const todos = [{ content: 'Implement', status: 'in_progress' as const }, { content: 'Test', status: 'pending' as const }]
+    value.append('todo/write', { todos }); value.append('todo/write', { todos })
+    expect(analyzeStructuredRoute(value.events)).toBeUndefined()
+    value.append('todo/write', { todos: [{ content: 'Implement', status: 'completed' }, { content: 'Test', status: 'in_progress' }] })
+    expect(analyzeStructuredRoute(value.events)).toMatchObject({ kind: 'track', evidenceSeqs: expect.any(Array) })
+  })
+  it('redacts and bounds trusted candidate content', () => {
+    const value = session(); tool(value, 'workflow', 'w', JSON.stringify({ protocol: 'dsh-verifier-candidates', version: 1, groupId: 'g', candidates: [{ id: 'a', status: 'completed', content: 'token = abc ' + 'x'.repeat(100) }, { id: 'b', status: 'completed', content: 'password: secret ' + 'y'.repeat(100) }] }))
+    const decision = analyzeStructuredRoute(value.events, 8, 40)
+    expect(decision?.kind).toBe('compare')
+    if (decision?.kind === 'compare') { expect(decision.candidates[0].content).not.toContain('abc'); expect(decision.candidates[0].content.length).toBeLessThan(80) }
+  })
+})
+
+describe('semantic evidence references', () => {
+  it('accepts exact strict JSON references and rejects prose/unknown keys', () => {
+    const valid = JSON.stringify({ kind: 'compare', confidence: .95, reason: 'alternatives', candidateCallIds: ['a', 'b'], checkpointSeqs: [] })
+    expect(parseSemanticRoute(valid)).toMatchObject({ kind: 'compare', candidateCallIds: ['a', 'b'] })
+    expect(parseSemanticRoute('```json\n' + valid + '\n```')).toBeUndefined()
+    expect(parseSemanticRoute(JSON.stringify({ ...JSON.parse(valid), extra: true }))).toBeUndefined()
+  })
+  it('resolves only real paired call ids', () => {
+    const value = session(); tool(value, 'subagent', 'a', 'candidate A'); tool(value, 'subagent', 'b', 'candidate B')
+    const parsed = parseSemanticRoute(JSON.stringify({ kind: 'compare', confidence: .95, reason: 'same task alternatives', candidateCallIds: ['a', 'b'], checkpointSeqs: [] }))!
+    expect(semanticDecision(parsed, value.events)).toMatchObject({ kind: 'compare', source: 'semantic' })
+    expect(semanticDecision({ ...parsed, candidateCallIds: ['a', 'missing'] }, value.events)).toBeUndefined()
+    expect(buildSemanticRoutePrompt('problem', value.events, 5)).toContain('candidateCallIds')
+  })
+  it('resolves candidates emitted via PTC mode tool/code-dispatch', () => {
+    const value = session()
+    value.append('tool/code-dispatch', { subCallId: 'c-1' as never, name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate 1 content' }] })
+    value.append('tool/code-dispatch', { subCallId: 'c-2' as never, name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate 2 content' }] })
+    expect(semanticRouteHint(value.events)).toBe(true)
+    const parsed = parseSemanticRoute(JSON.stringify({ kind: 'compare', confidence: 0.92, reason: 'PTC alternatives', candidateCallIds: ['c-1', 'c-2'], checkpointSeqs: [] }))!
+    expect(semanticDecision(parsed, value.events)).toMatchObject({ kind: 'compare', source: 'semantic' })
+  })
+
+  it('resolves candidates emitted via Session V3 tool/ptc-dispatch', () => {
+    const value = session()
+    value.append('tool/ptc-dispatch' as never, { subCallId: 'ptc-1', name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate A from ptc' }] } as never)
+    value.append('tool/ptc-dispatch' as never, { subCallId: 'ptc-2', name: 'subagent', arguments: '{}', isError: false, content: [{ type: 'text', text: 'candidate B from ptc' }] } as never)
+    expect(semanticRouteHint(value.events)).toBe(true)
+    const parsed = parseSemanticRoute(JSON.stringify({ kind: 'compare', confidence: 0.95, reason: 'V3 PTC alternatives', candidateCallIds: ['ptc-1', 'ptc-2'], checkpointSeqs: [] }))!
+    expect(semanticDecision(parsed, value.events)).toMatchObject({ kind: 'compare', source: 'semantic' })
+  })
+
+  it('routes progress tracking based on changed team/task snapshots', () => {
+    const value = session()
+    value.append('team/task' as never, { task: { id: 'task-1', revision: 1, subject: 'Backend API', status: 'in_progress' } } as never)
+    value.append('team/task' as never, { task: { id: 'task-1', revision: 2, subject: 'Backend API', status: 'completed' } } as never)
+    expect(semanticRouteHint(value.events)).toBe(true)
+    const decision = analyzeStructuredRoute(value.events)
+    expect(decision).toMatchObject({ kind: 'track', source: 'structured', reason: 'changed durable team tasks' })
+  })
+})
+
+describe('transactional router state', () => {
+  it('commits, requires final verification, and clears it only after final commit', () => {
+    const value = session(); const agent = { id: value.id, session: value }; const router = new AutoVerifierRouter()
+    const route = router.reserve(agent, 'compare', 'route', 4, policy)!
+    expect(router.commit(agent, route, 9)).toBe(true); expect(router.finalRequired(agent)).toBe(9)
+    const final = router.reserve(agent, 'final', 'final', 4, policy)!
+    expect(router.commit(agent, final)).toBe(true); expect(router.finalRequired(agent)).toBeUndefined()
+  })
+  it('releases in-flight state after failure and strict remains blocked', () => {
+    const value = session(); const agent = { id: value.id, session: value }; const router = new AutoVerifierRouter()
+    const first = router.reserve(agent, 'compare', 'route-a', 4, { ...policy, mode: 'strict' })!
+    router.fail(agent, first, true); expect(router.strictBlocked(agent)).toBe(true)
+    expect(router.reserve(agent, 'compare', 'route-b', 4, { ...policy, mode: 'strict' })).toBeDefined()
+  })
+  it('enforces manual mode and unified model-call budgets', () => {
+    const value = session(); const agent = { id: value.id, session: value }; const router = new AutoVerifierRouter()
+    expect(router.reserve(agent, 'compare', 'manual', 1, { ...policy, mode: 'manual' })).toBeUndefined()
+    expect(router.reserve(agent, 'select', 'expensive', 49, policy)).toBeUndefined()
+  })
+})