dsh-llm-verifier 0.3.0 → 0.4.0

package/src/client.tsx

@@ -4,15 +4,16 @@ import type {} from '@deepseek-ai/dsh-client-ui-settings/client'
 import type {} from '@deepseek-ai/dsh-client-ui-conversation/client'
 import type {} from '@deepseek-ai/dsh-client-ui-slots'
 import { Button, IconDataOutline16, IconRefreshOutline16, Input } from '@deepseek-ai/dsh-client-ui-primitives'
-import { useEffect, useMemo, useRef, useState } from 'react'
+import { Fragment, useEffect, useMemo, useRef, useState } from 'react'
 import {
   zh, en, dictionaries, toolLabels, tFormat, useLanguage, detectLanguage,
   compact, money, duration, dateTime, type I18nDict,
   type VerdictSummary, resolveCacheDirOnSave, sameSettingValue, sectionForSave,
-  WORST_CASE_ROUTE_CALLS_PER_JUDGE, WORST_CASE_FINAL_CALLS_PER_JUDGE, WORST_CASE_TASK_PER_JUDGE, WORST_CASE_SESSION_PER_JUDGE,
+  WORST_CASE_ROUTE_CALLS_PER_JUDGE, WORST_CASE_FINAL_CALLS_PER_JUDGE, WORST_CASE_TASK_PER_JUDGE, WORST_CASE_SESSION_PER_JUDGE, WORST_CASE_CRITERIA_PER_COMPARISON,
   computeJudgeCount, computeWorstCaseBudget, type BudgetWarningState,
   evaluateBudgetWarning, isVerdictFailed, formatPercentage, formatVerdictDetails,
 } from './client-i18n.ts'
+import { CRITERIA_PRESETS, DEFAULT_GROUND_TRUTH_NOTE, buildPairwisePrompt } from './core.ts'
 import {
   type ExtraJudgeDraft,
   MAX_EXTRA_JUDGES,
@@ -32,14 +33,17 @@ export {
 }
 
 const NS = 'llm-verifier'
-export interface Values { enabled: boolean; autoVerifyMode: 'manual'|'smart'|'strict'; autoVerifyThreshold: number; autoVerifyRepeats: number; autoVerifyFinalRepeats: number; autoVerifyMinToolCalls: number; autoVerifyMaxChars: number; autoVerifyMaxPerTask: number; autoVerifyMaxPerSession: number; autoRouteSemantic: boolean; autoRouteMinConfidence: number; autoRouteMaxCandidates: number; autoRouteMaxPerTask: number; autoRouteMaxPerSession: number; autoTrackCompletionThreshold: number; autoRouteMaxItemChars: number; autoRouteMaxInputChars: number; autoMaxModelCallsPerTask: number; autoMaxModelCallsPerSession: number; autoVerifyTeamTasks: boolean; autoVerifyPlanMode: boolean; provider: string; model: string; reasoningEffort?: string; maxTokens: number; temperature: number; label?: string; maxConcurrency: number; maxRetries: number; retryBaseDelayMs: number; timeoutMs: number; cacheDir: string; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number; autoVerifySubagents: boolean; extraJudges: ExtraJudgeDraft[] }
+interface DecisionCallView { label: string; channel: string; prompt: string; output: string; score?: number }
+interface DecisionRecordView { id: string; toolName: string; phase: string; provider: string; model: string; startedAt: number; calls: DecisionCallView[] }
+
+export interface Values { enabled: boolean; captureDecisions: boolean; autoVerifyMode: 'manual'|'smart'|'strict'; autoVerifyThreshold: number; autoVerifyRepeats: number; autoTrackRepeats: number; autoVerifyFinalRepeats: number; autoVerifyMinToolCalls: number; autoVerifyMaxChars: number; autoVerifyMaxPerTask: number; autoVerifyMaxPerSession: number; autoRouteSemantic: boolean; autoRouteMinConfidence: number; autoRouteMaxCandidates: number; autoRouteMaxPerTask: number; autoRouteMaxPerSession: number; autoTrackCompletionThreshold: number; autoRouteMaxItemChars: number; autoRouteMaxInputChars: number; autoMaxModelCallsPerTask: number; autoMaxModelCallsPerSession: number; autoVerifyTeamTasks: boolean; autoVerifyPlanMode: boolean; criteriaPreset: 'coding'|'debug'|'research'|'ops'|'writing'|'custom'; criteriaFile: string; provider: string; model: string; reasoningEffort?: string; maxTokens: number; temperature: number; label?: string; maxConcurrency: number; maxRetries: number; retryBaseDelayMs: number; timeoutMs: number; cacheDir: string; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number; autoVerifySubagents: boolean; extraJudges: ExtraJudgeDraft[] }
 export interface Loaded { groups: readonly ModelProviderGroup[]; settings: SettingsNamespaceView; writable: boolean; failures: string[] }
 export interface RunStats { calls: number; attempts: number; retries: number; inputTokens: number; cachedInputTokens: number; outputTokens: number; reasoningTokens: number; cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
 export interface InvocationRecord { id: string; toolName: string; sessionId?: string; startedAt: number; finishedAt: number; durationMs: number; success: boolean; errorName?: string; errorMessage?: string; provider: string; model: string; stats: RunStats; verdict?: VerdictSummary }
 interface DailyStatistics { date: string; invocations: number; successes: number; failures: number; calls: number; tokens: number; estimatedCostUsd: number; byTool: Record<string, number> }
 interface ToolStatistics { toolName: string; invocations: number; successes: number; failures: number; successRate: number; averageDurationMs: number; calls: number; tokens: number; cacheHits: number; cacheMisses: number; estimatedCostUsd: number }
 interface ModelStatistics { provider: string; model: string; invocations: number; calls: number; tokens: number; estimatedCostUsd: number }
-interface Totals extends RunStats { invocations: number; successes: number; failures: number; successRate: number; averageDurationMs: number; tokens: number; cacheHitRate: number }
+interface Totals extends RunStats { invocations: number; successes: number; failures: number; successRate: number; averageDurationMs: number; tokens: number; cacheHitRate: number; prefixCacheHitRate: number }
 interface StatisticsOverview { generatedAt: number; fromMs: number; toMs: number; sessionId?: string; totals: Totals; daily: DailyStatistics[]; tools: ToolStatistics[]; models: ModelStatistics[]; recent: InvocationRecord[] }
 interface VerifierRemote {
   session: {
@@ -67,6 +71,9 @@ interface VerifierRemote {
   }
 }
 
+export interface JudgeProbeView { label: string; provider: string; model: string; ok: boolean; channel?: string; channelProbed?: boolean; scoreA?: number; scoreB?: number; latencyMs: number; calls?: number; inputTokens?: number; cachedInputTokens?: number; outputTokens?: number; error?: string }
+export interface ProbeResultView { judges: JudgeProbeView[]; channelProbed?: boolean; rubric: { source: string; count: number; file?: string; error?: string } }
+
 interface VerifierSettingsProps { remote: VerifierRemote }
 interface StatisticsPageProps {
   sessionId?: string
@@ -87,10 +94,10 @@ const toggleStyle = (enabled: boolean): React.CSSProperties => ({ position: 'rel
 const toggleThumbStyle = (enabled: boolean): React.CSSProperties => ({ position: 'absolute', top: 3, left: enabled ? 21 : 3, width: 16, height: 16, borderRadius: '50%', background: 'var(--dsw-static-neutral-00, #fff)', boxShadow: '0 1px 3px rgba(0,0,0,.28)', transition: 'left .15s ease' })
 const dashboardCard: React.CSSProperties = { border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.13))', background: 'color-mix(in srgb, var(--dsw-alias-bg-module, #171925) 88%, transparent)', borderRadius: 16, boxShadow: '0 12px 36px rgba(0,0,0,.12)' }
 const muted: React.CSSProperties = { color: 'var(--dsw-text-secondary)', fontSize: 12 }
-const toolColors: Record<string, string> = { verifier_route_classify: '#d97706', verifier_compare: '#4f8cff', verifier_select: '#8b6df6', verifier_track: '#2fc5c9', verifier_current_session: '#f5a524' }
+const toolColors: Record<string, string> = { verifier_route_classify: '#d97706', verifier_compare: '#4f8cff', verifier_select: '#8b6df6', verifier_track: '#2fc5c9', verifier_best_of_n: '#e2569b', verifier_current_session: '#f5a524' }
 
 function record(value: unknown): Record<string, unknown> { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {} }
-function values(view: SettingsNamespaceView): Values { const v=record(view.value); const mode=v.autoVerifyMode==='manual'||v.autoVerifyMode==='strict'?v.autoVerifyMode:'smart'; return { enabled:v.enabled!==false,autoVerifyMode:mode,autoVerifyThreshold:Number(v.autoVerifyThreshold??.65),autoVerifyRepeats:Number(v.autoVerifyRepeats??1),autoVerifyFinalRepeats:Number(v.autoVerifyFinalRepeats??2),autoVerifyMinToolCalls:Number(v.autoVerifyMinToolCalls??3),autoVerifyMaxChars:Number(v.autoVerifyMaxChars??80000),autoVerifyMaxPerTask:Number(v.autoVerifyMaxPerTask??2),autoVerifyMaxPerSession:Number(v.autoVerifyMaxPerSession??8),autoRouteSemantic:v.autoRouteSemantic!==false,autoRouteMinConfidence:Number(v.autoRouteMinConfidence??.9),autoRouteMaxCandidates:Number(v.autoRouteMaxCandidates??8),autoRouteMaxPerTask:Number(v.autoRouteMaxPerTask??2),autoRouteMaxPerSession:Number(v.autoRouteMaxPerSession??8),autoTrackCompletionThreshold:Number(v.autoTrackCompletionThreshold??.8),autoRouteMaxItemChars:Number(v.autoRouteMaxItemChars??20000),autoRouteMaxInputChars:Number(v.autoRouteMaxInputChars??60000),autoMaxModelCallsPerTask:Number(v.autoMaxModelCallsPerTask??96),autoMaxModelCallsPerSession:Number(v.autoMaxModelCallsPerSession??240),autoVerifyTeamTasks:v.autoVerifyTeamTasks!==false,autoVerifyPlanMode:v.autoVerifyPlanMode!==false,provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),temperature:Number(v.temperature??0.2),...(typeof v.label==='string'&&v.label.trim()?{label:v.label.trim()}:{}),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),retryBaseDelayMs:Number(v.retryBaseDelayMs??500),timeoutMs:Number(v.timeoutMs??300000),cacheDir:typeof v.cacheDir==='string'&&v.cacheDir.trim()?v.cacheDir.trim():'verifier',cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0),autoVerifySubagents:v.autoVerifySubagents===true,extraJudges:normalizeExtraJudges(v.extraJudges) } }
+function values(view: SettingsNamespaceView): Values { const v=record(view.value); const mode=v.autoVerifyMode==='manual'||v.autoVerifyMode==='strict'?v.autoVerifyMode:'smart'; return { enabled:v.enabled!==false,captureDecisions:v.captureDecisions!==false,autoVerifyMode:mode,autoVerifyThreshold:Number(v.autoVerifyThreshold??.65),autoVerifyRepeats:Number(v.autoVerifyRepeats??1),autoTrackRepeats:Number(v.autoTrackRepeats??3),autoVerifyFinalRepeats:Number(v.autoVerifyFinalRepeats??2),autoVerifyMinToolCalls:Number(v.autoVerifyMinToolCalls??3),autoVerifyMaxChars:Number(v.autoVerifyMaxChars??80000),autoVerifyMaxPerTask:Number(v.autoVerifyMaxPerTask??2),autoVerifyMaxPerSession:Number(v.autoVerifyMaxPerSession??8),autoRouteSemantic:v.autoRouteSemantic!==false,autoRouteMinConfidence:Number(v.autoRouteMinConfidence??.9),autoRouteMaxCandidates:Number(v.autoRouteMaxCandidates??8),autoRouteMaxPerTask:Number(v.autoRouteMaxPerTask??2),autoRouteMaxPerSession:Number(v.autoRouteMaxPerSession??8),autoTrackCompletionThreshold:Number(v.autoTrackCompletionThreshold??.684),autoRouteMaxItemChars:Number(v.autoRouteMaxItemChars??20000),autoRouteMaxInputChars:Number(v.autoRouteMaxInputChars??60000),autoMaxModelCallsPerTask:Number(v.autoMaxModelCallsPerTask??96),autoMaxModelCallsPerSession:Number(v.autoMaxModelCallsPerSession??240),autoVerifyTeamTasks:v.autoVerifyTeamTasks!==false,autoVerifyPlanMode:v.autoVerifyPlanMode!==false,criteriaPreset:(v.criteriaPreset==='debug'||v.criteriaPreset==='research'||v.criteriaPreset==='ops'||v.criteriaPreset==='writing'||v.criteriaPreset==='custom')?v.criteriaPreset:'coding',criteriaFile:typeof v.criteriaFile==='string'?v.criteriaFile.trim():'',provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),temperature:Number(v.temperature??0.2),...(typeof v.label==='string'&&v.label.trim()?{label:v.label.trim()}:{}),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),retryBaseDelayMs:Number(v.retryBaseDelayMs??500),timeoutMs:Number(v.timeoutMs??300000),cacheDir:typeof v.cacheDir==='string'&&v.cacheDir.trim()?v.cacheDir.trim():'verifier',cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0),autoVerifySubagents:v.autoVerifySubagents===true,extraJudges:normalizeExtraJudges(v.extraJudges) } }
 function message(error: unknown): string { return error instanceof Error ? error.message : String(error) }
 /** The endpoint answered but rejected the request: a transport fallback would only repeat it. */
 class EndpointError extends Error {}
@@ -165,13 +172,18 @@ export function VerifierSettings({ remote }: VerifierSettingsProps) {
   }
   const budgetWarning = useMemo(() => {
     if (!draft) return null
+    // The engine reserves budget from the REAL criteria count. A built-in preset's count is known
+    // here; a custom file's is not (it lives on the server), so the estimate falls back to the
+    // three-criteria default and the file field's help says as much.
+    const criteria = draft.criteriaPreset === 'custom' ? WORST_CASE_CRITERIA_PER_COMPARISON : CRITERIA_PRESETS[draft.criteriaPreset].length
     return evaluateBudgetWarning(
       draft.autoVerifyMode,
       draft.extraJudges.length,
       draft.autoMaxModelCallsPerTask,
       draft.autoMaxModelCallsPerSession,
+      criteria,
     )
-  }, [draft?.autoVerifyMode, draft?.extraJudges.length, draft?.autoMaxModelCallsPerTask, draft?.autoMaxModelCallsPerSession])
+  }, [draft?.autoVerifyMode, draft?.criteriaPreset, draft?.extraJudges.length, draft?.autoMaxModelCallsPerTask, draft?.autoMaxModelCallsPerSession])
   // The saved section holds only real overrides: a draft field equal to the
   // composition base is dropped so a later plugin default still reaches this
   // install (see sectionForSave).
@@ -231,8 +243,12 @@ export function VerifierSettings({ remote }: VerifierSettingsProps) {
 
     <section style={group}><GroupTitle>{t['section.autoVerify']}</GroupTitle>
       <div style={row}><Label title={t['field.autoVerifyMode.title']} help={t['field.autoVerifyMode.help']}/><select style={selectStyle} disabled={busy} aria-label={t['field.autoVerifyMode.title']} value={draft.autoVerifyMode} onChange={e=>patch('autoVerifyMode',e.target.value as Values['autoVerifyMode'])}><option value="manual">{t['field.autoVerifyMode.manual']}</option><option value="smart">{t['field.autoVerifyMode.smart']}</option><option value="strict">{t['field.autoVerifyMode.strict']}</option></select></div>
+      <div style={row}><Label title={t['field.criteriaPreset.title']} help={t['field.criteriaPreset.help']}/><select style={selectStyle} disabled={busy} aria-label={t['field.criteriaPreset.title']} value={draft.criteriaPreset} onChange={e=>patch('criteriaPreset',e.target.value as Values['criteriaPreset'])}>{(['coding','debug','research','ops','writing','custom'] as const).map(id=><option key={id} value={id}>{t[('field.criteriaPreset.'+id) as keyof I18nDict] as string}</option>)}</select></div>
+      {draft.criteriaPreset==='custom'&&<div style={row}><Label title={t['field.criteriaFile.title']} help={t['field.criteriaFile.help']}/>{textField('criteriaFile','criteria/my-task.md')}</div>}
+      {draft.criteriaPreset!=='custom'&&(()=>{const preset=CRITERIA_PRESETS[draft.criteriaPreset];const first=preset[0];if(!first)return null;return <details style={{margin:'2px 0 0 4px',fontSize:12}}><summary style={{cursor:'pointer',color:'var(--dsw-text-secondary)'}}>{tFormat(t['field.criteriaPreset.previewSummary'],{count:preset.length})}</summary><pre style={{whiteSpace:'pre-wrap',wordBreak:'break-word',maxHeight:240,overflow:'auto',background:'var(--dsw-surface-sunken)',border:'1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.12))',borderRadius:8,padding:10,marginTop:8}}>{buildPairwisePrompt(t['field.criteriaPreset.sampleTask'],t['field.criteriaPreset.sampleA'],t['field.criteriaPreset.sampleB'],first,DEFAULT_GROUND_TRUTH_NOTE)}</pre></details>})()}
       <div style={row}><Label title={t['field.autoRouteSemantic.title']} help={t['field.autoRouteSemantic.help']}/><button type="button" role="switch" aria-checked={draft.autoRouteSemantic} aria-label={t['field.autoRouteSemantic.title']} onClick={()=>patch('autoRouteSemantic',!draft.autoRouteSemantic)} style={toggleStyle(draft.autoRouteSemantic)}><span style={toggleThumbStyle(draft.autoRouteSemantic)}/></button></div>
       <div style={row}><Label title={t['field.autoVerifyTeamTasks.title']} help={t['field.autoVerifyTeamTasks.help']}/><button type="button" role="switch" aria-checked={draft.autoVerifyTeamTasks} aria-label={t['field.autoVerifyTeamTasks.title']} onClick={()=>patch('autoVerifyTeamTasks',!(draft.autoVerifyTeamTasks))} style={toggleStyle(draft.autoVerifyTeamTasks)}><span style={toggleThumbStyle(draft.autoVerifyTeamTasks)}/></button></div>
+      <div style={row}><Label title={t['field.captureDecisions.title']} help={t['field.captureDecisions.help']}/><button type="button" role="switch" aria-checked={draft.captureDecisions} aria-label={t['field.captureDecisions.title']} onClick={()=>patch('captureDecisions',!draft.captureDecisions)} style={toggleStyle(draft.captureDecisions)}><span style={toggleThumbStyle(draft.captureDecisions)}/></button></div>
       <div style={row}><Label title={t['field.autoVerifySubagents.title']} help={t['field.autoVerifySubagents.help']}/><button type="button" role="switch" aria-checked={draft.autoVerifySubagents} aria-label={t['field.autoVerifySubagents.title']} onClick={()=>patch('autoVerifySubagents',!draft.autoVerifySubagents)} style={toggleStyle(draft.autoVerifySubagents)}><span style={toggleThumbStyle(draft.autoVerifySubagents)}/></button></div>
       <div style={row}><Label title={t['field.autoVerifyPlanMode.title']} help={t['field.autoVerifyPlanMode.help']}/><button type="button" role="switch" aria-checked={draft.autoVerifyPlanMode} aria-label={t['field.autoVerifyPlanMode.title']} onClick={()=>patch('autoVerifyPlanMode',!(draft.autoVerifyPlanMode))} style={toggleStyle(draft.autoVerifyPlanMode)}><span style={toggleThumbStyle(draft.autoVerifyPlanMode)}/></button></div>
       <div style={row}><Label title={t['field.autoRouteMinConfidence.title']} help={t['field.autoRouteMinConfidence.help']}/>{numeric('autoRouteMinConfidence',0)}</div>
@@ -248,6 +264,7 @@ export function VerifierSettings({ remote }: VerifierSettingsProps) {
       {budgetWarning?.warnSession&&<p style={{margin:'-4px 0 10px',fontSize:12,lineHeight:'18px',color:'var(--dsw-alias-state-warn-label)'}}>{tFormat(t['field.autoMaxModelCallsPerSession.warnBudget'],{current:draft.autoMaxModelCallsPerSession,required:budgetWarning.worstCaseSession,judges:budgetWarning.judgeCount})}</p>}
       <div style={row}><Label title={t['field.autoVerifyThreshold.title']} help={t['field.autoVerifyThreshold.help']}/>{numeric('autoVerifyThreshold',0)}</div>
       <div style={row}><Label title={t['field.autoVerifyRepeats.title']} help={t['field.autoVerifyRepeats.help']}/>{numeric('autoVerifyRepeats',1)}</div>
+      <div style={row}><Label title={t['field.autoTrackRepeats.title']} help={t['field.autoTrackRepeats.help']}/>{numeric('autoTrackRepeats',1)}</div>
       <div style={row}><Label title={t['field.autoVerifyFinalRepeats.title']} help={t['field.autoVerifyFinalRepeats.help']}/>{numeric('autoVerifyFinalRepeats',1)}</div>
       <div style={row}><Label title={t['field.autoVerifyMinToolCalls.title']} help={t['field.autoVerifyMinToolCalls.help']}/>{numeric('autoVerifyMinToolCalls',1)}</div>
       <div style={row}><Label title={t['field.autoVerifyMaxChars.title']} help={t['field.autoVerifyMaxChars.help']}/>{numeric('autoVerifyMaxChars',1000)}</div>
@@ -363,6 +380,63 @@ export function StatisticsPage({ sessionId, rpc, isGlobal }: StatisticsPageProps
   const [loading, setLoading] = useState(true)
   const [error, setError] = useState<string | null>(null)
   const [refresh, setRefresh] = useState(0)
+  // Decision snapshots are fetched on demand: they carry whole prompts, and a dashboard
+  // that loaded all of them would ship megabytes of text for rows nobody opened.
+  const [snapshot, setSnapshot] = useState<{ id: string; record?: DecisionRecordView; error?: string } | null>(null)
+  const snapshotRequest = useRef(0)
+  const toggleSnapshot = async (id: string) => {
+    if (snapshot?.id === id) { setSnapshot(null); return }
+    const request = ++snapshotRequest.current
+    setSnapshot({ id })
+    const payload = { kind: 'decision', id }
+    try {
+      let record: DecisionRecordView | undefined
+      if (rpc && typeof rpc.call === 'function') {
+        try {
+          const result = await rpc.call('/api', 'llm-verifier/statistics', payload)
+          if (result && result.ok === true) record = (result.value as { decision?: DecisionRecordView }).decision
+        } catch (rpcError) { console.warn('[llm-verifier] decision rpc.call failed, trying fetch fallback:', rpcError) }
+      }
+      if (record === undefined) {
+        const response = await fetch('/api/llm-verifier/statistics', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(payload) })
+        const body = await response.json().catch(() => undefined)
+        const value = body?.ok === true ? body.value : body?.result?.ok === true ? body.result.value : undefined
+        if (value === undefined || value === null) throw new EndpointError(body?.error?.message ?? body?.result?.error?.message ?? t['stats.requestFailed'])
+        record = (value as { decision?: DecisionRecordView }).decision
+      }
+      if (snapshotRequest.current === request) setSnapshot({ id, ...(record === undefined ? { error: t['recent.decisionMissing'] } : { record }) })
+    } catch (cause) {
+      if (snapshotRequest.current === request) setSnapshot({ id, error: message(cause) })
+    }
+  }
+  // Judge probe: one real, bounded call per configured judge. Not recorded in the statistics —
+  // it answers "which channel is this judge on, and is my rubric the one in effect", which the
+  // aggregate counters cannot.
+  const [probe, setProbe] = useState<{ busy: boolean; value?: ProbeResultView; error?: string } | null>(null)
+  const runProbe = async () => {
+    if (probe?.busy) return
+    setProbe({ busy: true })
+    const payload = { kind: 'probe' }
+    try {
+      let value: ProbeResultView | undefined
+      if (rpc && typeof rpc.call === 'function') {
+        try {
+          const result = await rpc.call('/api', 'llm-verifier/statistics', payload)
+          if (result && result.ok === true) value = result.value as ProbeResultView
+        } catch (rpcError) { console.warn('[llm-verifier] probe rpc.call failed, trying fetch fallback:', rpcError) }
+      }
+      if (value === undefined) {
+        const response = await fetch('/api/llm-verifier/statistics', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(payload) })
+        const body = await response.json().catch(() => undefined)
+        const fallback = body?.ok === true ? body.value : body?.result?.ok === true ? body.result.value : undefined
+        if (fallback === undefined || fallback === null) throw new EndpointError(body?.error?.message ?? body?.result?.error?.message ?? t['stats.requestFailed'])
+        value = fallback as ProbeResultView
+      }
+      setProbe({ busy: false, value })
+    } catch (cause) {
+      setProbe({ busy: false, error: message(cause) })
+    }
+  }
   // Data belongs to the range/session that produced it: keeping the previous
   // range's numbers under an error banner reads as if they were current.
   const queryKey = days + '|' + sessionOnly + '|' + String(sessionId ?? '')
@@ -462,10 +536,32 @@ export function StatisticsPage({ sessionId, rpc, isGlobal }: StatisticsPageProps
             {[7, 30, 90].map(value => <button key={value} aria-pressed={days === value} onClick={() => setDays(value)} style={{ border: 0, borderRadius: 7, padding: '6px 10px', cursor: 'pointer', color: days === value ? '#fff' : 'var(--dsw-text-secondary)', background: days === value ? '#3f68d8' : 'transparent' }}>{tFormat(t['stats.daysUnit'], { days: value })}</button>)}
           </div>
           {Boolean(sessionId) && <button aria-pressed={sessionOnly} onClick={() => setSessionOnly(value => !value)} style={{ border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.15))', borderRadius: 9, padding: '7px 11px', cursor: 'pointer', color: 'var(--dsw-text-primary)', background: sessionOnly ? 'rgba(79,140,255,.18)' : 'var(--dsw-surface-sunken)' }}>{sessionOnly ? t['stats.currentSession'] : t['stats.allSessions']}</button>}
+          <button type="button" disabled={probe?.busy === true} onClick={() => void runProbe()} style={{ border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.15))', borderRadius: 9, padding: '7px 11px', cursor: 'pointer', color: 'var(--dsw-text-primary)', background: 'var(--dsw-surface-sunken)' }}>{probe?.busy === true ? t['probe.running'] : t['probe.button']}</button>
           <button title={t['stats.refresh']} onClick={() => setRefresh(value => value + 1)} style={{ display: 'grid', placeItems: 'center', width: 34, height: 34, borderRadius: 9, border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.15))', color: 'var(--dsw-text-primary)', background: 'var(--dsw-surface-sunken)', cursor: 'pointer' }}><IconRefreshOutline16 size={16} /></button>
         </div>
       </header>
       {error && <div style={{ ...dashboardCard, padding: 18, borderColor: 'var(--dsw-danger, #e85858)', color: 'var(--dsw-danger, #e85858)' }}>{error}<div style={{ ...muted, marginTop: 6 }}>{t['stats.hostRestartHint']}</div></div>}
+      {probe !== null && <section style={{ ...dashboardCard, padding: '16px 18px' }}>
+        <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}><strong>{t['probe.title']}</strong><span style={muted}>{t['probe.note']}</span></div>
+        {probe.error !== undefined && <div style={{ marginTop: 10, fontSize: 12, color: '#e76565' }}>{probe.error}</div>}
+        {probe.value !== undefined && <>
+          <div style={{ ...muted, marginTop: 8 }}>{tFormat(t['probe.rubric'], { source: probe.value.rubric.source, count: probe.value.rubric.count, file: probe.value.rubric.file ?? '' })}</div>
+          {probe.value.rubric.error !== undefined && <div style={{ marginTop: 6, fontSize: 12, color: '#e3bd63' }}>{tFormat(t['probe.rubricFallback'], { error: probe.value.rubric.error })}</div>}
+          <div style={{ marginTop: 10, display: 'flex', flexDirection: 'column', gap: 8 }}>
+            {probe.value.judges.map(judge => <div key={judge.label + judge.model} style={{ display: 'flex', flexWrap: 'wrap', gap: 10, alignItems: 'center', fontSize: 12, padding: '9px 11px', borderRadius: 9, background: 'var(--dsw-surface-sunken)' }}>
+              <span style={{ color: judge.ok ? '#77d49b' : '#e76565', fontWeight: 600 }}>{judge.ok ? 'OK' : t['probe.failed']}</span>
+              <strong>{judge.label}</strong>
+              <span style={muted}>{judge.provider}/{judge.model}</span>
+              {judge.ok ? <>
+                {judge.channel !== undefined && <span style={muted}>{tFormat(t['probe.channel'], { channel: judge.channel })}{judge.channelProbed === true ? ' · ' + t['probe.reprobed'] : ''}</span>}
+                {judge.scoreA !== undefined && judge.scoreB !== undefined && <span style={muted}>{tFormat(t['probe.scores'], { a: formatPercentage(judge.scoreA), b: formatPercentage(judge.scoreB) })}</span>}
+                <span style={muted}>{tFormat(t['probe.latency'], { ms: String(judge.latencyMs) })}</span>
+                {judge.calls !== undefined && <span style={muted}>{tFormat(t['probe.calls'], { calls: String(judge.calls) })}</span>}
+              </> : <span style={{ color: '#e76565' }}>{judge.error}</span>}
+            </div>)}
+          </div>
+        </>}
+      </section>}
       {loading && !data ? <div style={{ ...dashboardCard, padding: 32, textAlign: 'center', ...muted }}>{t['stats.loading']}</div> : <>
         <section style={{ ...dashboardCard, padding: '22px 24px', display: 'grid', gridTemplateColumns: 'minmax(220px,1.4fr) minmax(240px,1fr)', gap: 24, alignItems: 'center' }}>
           <div>
@@ -481,6 +577,7 @@ export function StatisticsPage({ sessionId, rpc, isGlobal }: StatisticsPageProps
         </section>
         <section style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit,minmax(180px,1fr))', gap: 12 }}>
           <Metric label={t['metric.cacheHitRate']} value={((totals?.cacheHitRate ?? 0) * 100).toFixed(1) + '%'} note={tFormat(t['metric.cacheHitNote'], { hits: compact(totals?.cacheHits ?? 0, lang), total: compact((totals?.cacheHits ?? 0) + (totals?.cacheMisses ?? 0), lang) })} accent="#b7dd64" />
+          <Metric label={t['metric.prefixCacheHitRate']} value={((totals?.prefixCacheHitRate ?? 0) * 100).toFixed(1) + '%'} note={tFormat(t['metric.prefixCacheHitNote'], { cached: compact(totals?.cachedInputTokens ?? 0, lang), input: compact((totals?.inputTokens ?? 0) + (totals?.cachedInputTokens ?? 0), lang) })} />
           <Metric label={t['metric.tokens']} value={compact(totals?.tokens ?? 0, lang)} note={tFormat(t['metric.tokensNote'], { input: compact((totals?.inputTokens ?? 0) + (totals?.cachedInputTokens ?? 0), lang), output: compact(totals?.outputTokens ?? 0, lang) })} />
           <Metric label={t['metric.avgModelCalls']} value={(totals?.invocations ?? 0) > 0 ? ((totals?.calls ?? 0) / (totals?.invocations ?? 1)).toFixed(1) : '0'} note={tFormat(t['metric.avgModelCallsNote'], { attempts: compact(totals?.attempts ?? 0, lang), retries: compact(totals?.retries ?? 0, lang) })} />
           <Metric label={t['metric.scoringMode']} value={compact(totals?.topLogprobScores ?? 0, lang)} note={tFormat(t['metric.scoringModeNote'], { explicit: compact(totals?.explicitTagScores ?? 0, lang) })} />
@@ -506,6 +603,47 @@ export function StatisticsPage({ sessionId, rpc, isGlobal }: StatisticsPageProps
                   {verdictInfo.criteriaText && <span style={{ color: 'var(--dsw-text-secondary)' }}>{verdictInfo.criteriaText}</span>}
                   {verdictInfo.winnerText && <span style={{ color: 'var(--dsw-text-secondary)' }}>{verdictInfo.winnerText}</span>}
                 </div>}
+                <div style={{ margin: '6px 0 0 15px' }}>
+                  <button type="button" onClick={() => void toggleSnapshot(item.id)} style={{ fontSize: 11, padding: '2px 7px', borderRadius: 5, cursor: 'pointer', color: 'var(--dsw-text-secondary)', background: 'var(--dsw-surface-sunken)', border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.14))' }}>{snapshot?.id === item.id ? t['recent.decisionHide'] : t['recent.decision']}</button>
+                </div>
+                <details style={{ margin: '6px 0 0 15px' }}>
+                  <summary style={{ cursor: 'pointer', fontSize: 11, color: 'var(--dsw-text-secondary)' }}>{t['recent.details']}</summary>
+                  <div style={{ marginTop: 8, border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.14))', borderRadius: 8, padding: '8px 10px', background: 'var(--dsw-surface-sunken)', fontSize: 11 }}>
+                    {(item.verdict?.criteria?.length ?? 0) > 0 && <div style={{ display: 'grid', gridTemplateColumns: 'minmax(120px,1fr) auto auto', gap: '4px 14px', marginBottom: 8 }}>
+                      <strong style={{ color: 'var(--dsw-text-secondary)' }}>{t['recent.detail.criterion']}</strong>
+                      <strong style={{ color: 'var(--dsw-text-secondary)' }}>{t['recent.detail.score']}</strong>
+                      <strong style={{ color: 'var(--dsw-text-secondary)' }}>{t['recent.detail.threshold']}</strong>
+                      {item.verdict!.criteria!.map(criterion => {
+                        const threshold = typeof item.verdict!.threshold === 'number' ? item.verdict!.threshold : undefined
+                        const missed = threshold !== undefined && criterion.score < threshold
+                        return <Fragment key={criterion.id}>
+                          <span>{criterion.id}</span>
+                          <span style={{ color: missed ? '#e76565' : '#77d49b' }}>{formatPercentage(criterion.score)}</span>
+                          <span style={{ color: 'var(--dsw-text-secondary)' }}>{threshold === undefined ? '—' : formatPercentage(threshold)}</span>
+                        </Fragment>
+                      })}
+                    </div>}
+                    <div style={{ display: 'flex', flexWrap: 'wrap', gap: 10, color: 'var(--dsw-text-secondary)' }}>
+                      <span>{tFormat(t['recent.detail.calls'], { calls: compact(item.stats.calls, lang) })}</span>
+                      <span>{tFormat(t['recent.detail.tokens'], { input: compact(item.stats.inputTokens, lang), cached: compact(item.stats.cachedInputTokens, lang), output: compact(item.stats.outputTokens, lang) })}</span>
+                      <span>{tFormat(t['recent.detail.scoreCache'], { hits: compact(item.stats.cacheHits, lang), misses: compact(item.stats.cacheMisses, lang) })}</span>
+                      <span>{tFormat(t['recent.detail.prefixCache'], { rate: ((item.stats.inputTokens + item.stats.cachedInputTokens) > 0 ? (100 * item.stats.cachedInputTokens / (item.stats.inputTokens + item.stats.cachedInputTokens)).toFixed(0) : '0') + '%' })}</span>
+                      <span>{tFormat(t['recent.detail.cost'], { cost: money(item.stats.estimatedCostUsd) })}</span>
+                    </div>
+                  </div>
+                </details>
+                {snapshot?.id === item.id && <div style={{ margin: '8px 0 2px 15px', border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.14))', borderRadius: 8, padding: '8px 10px', background: 'var(--dsw-surface-sunken)' }}>
+                  {snapshot.error !== undefined && <div style={{ fontSize: 11, color: '#e76565' }}>{snapshot.error}</div>}
+                  {snapshot.record === undefined && snapshot.error === undefined && <div style={{ ...muted, fontSize: 11 }}>{t['recent.decisionLoading']}</div>}
+                  {snapshot.record !== undefined && snapshot.record.calls.length === 0 && <div style={{ ...muted, fontSize: 11 }}>{t['recent.decisionEmpty']}</div>}
+                  {(snapshot.record?.calls ?? []).map((call, index) => <div key={index} style={{ marginBottom: index === snapshot.record!.calls.length - 1 ? 0 : 10 }}>
+                    <div style={{ fontSize: 11, color: 'var(--dsw-text-secondary)' }}>{call.label} · {call.channel}{call.score === undefined ? '' : ' · ' + formatPercentage(call.score)}</div>
+                    <div style={{ fontSize: 11, marginTop: 4 }}>{t['recent.decisionPrompt']}</div>
+                    <pre style={{ margin: 0, maxHeight: 180, overflow: 'auto', fontSize: 11, whiteSpace: 'pre-wrap', wordBreak: 'break-word', background: 'rgba(0,0,0,.2)', padding: '6px 8px', borderRadius: 6 }}>{call.prompt}</pre>
+                    <div style={{ fontSize: 11, marginTop: 4 }}>{t['recent.decisionOutput']}</div>
+                    <pre style={{ margin: 0, maxHeight: 140, overflow: 'auto', fontSize: 11, whiteSpace: 'pre-wrap', wordBreak: 'break-word', background: 'rgba(0,0,0,.2)', padding: '6px 8px', borderRadius: 6 }}>{call.output}</pre>
+                  </div>)}
+                </div>}
               </div>
               <div style={{ textAlign: 'right' }}><div style={{ fontSize: 12 }}>{duration(item.durationMs)}</div><div style={{ ...muted, marginTop: 3 }}>{dateTime(item.startedAt, lang)}</div></div>
             </div>

package/src/config.test.ts

@@ -311,6 +311,20 @@ describe('config - judges ensemble resolution', () => {
     ).toThrow('llm-verifier: extraJudges[0] must be an object')
   })
 
+  it('resolves the criteria preset and defaults to the historical coding rubric', () => {
+    const resolved = resolveConfig({})
+    // Default MUST stay 'coding': it is the only preset byte-identical to DEFAULT_CRITERIA, so an
+    // existing installation's verdicts and cache keys do not move.
+    expect(resolved.criteriaPreset).toBe('coding')
+    expect(resolved.criteriaFile).toBe('')
+    expect(resolveConfig({ criteriaPreset: 'research' }).criteriaPreset).toBe('research')
+    expect(resolveConfig({ criteriaPreset: 'custom', criteriaFile: '  rubric.md ' }).criteriaFile).toBe('rubric.md')
+    // A custom preset with no file is NOT a config error: the resolver degrades to coding and
+    // reports why, so a typo in a rubric path cannot disable the gate.
+    expect(resolveConfig({ criteriaPreset: 'custom' }).criteriaFile).toBe('')
+    expect(() => resolveConfig({ criteriaPreset: 'nonsense' as any })).toThrow('llm-verifier: criteriaPreset must be one of coding, debug, research, ops, writing, custom')
+  })
+
   it('regression: primary configuration still validates and behaves as before', () => {
     expect(() => resolveConfig({ provider: '' })).toThrow('llm-verifier: provider must be non-empty')
     expect(() => resolveConfig({ provider: '   ' })).toThrow('llm-verifier: provider must be non-empty')
@@ -377,6 +391,24 @@ describe('config - settings payload seam', () => {
     expect(resolved.judges.map(judge => judge.maxTokens)).toEqual([32768, 32768, 32768, 32768])
   })
 
+  it('carries the criteria preset across the settings schema, JSON boundary and resolveConfig', () => {
+    // The host validates the stored section with the schema and the RPC carries plain JSON, so the
+    // new keys must survive Config -> JSON -> Config -> resolveConfig, not just resolveConfig.
+    const payload = JSON.parse(JSON.stringify(Config({ provider: 'deepseek-official', model: 'deepseek-flash', criteriaPreset: 'ops', criteriaFile: ' criteria/ops.md ' })))
+    expect(payload.criteriaPreset).toBe('ops')
+    expect(payload.criteriaFile).toBe(' criteria/ops.md ')
+    const resolved = resolveConfig(payload)
+    expect(resolved.criteriaPreset).toBe('ops')
+    // The schema preserves what the user typed; resolveConfig is what trims.
+    expect(resolved.criteriaFile).toBe('criteria/ops.md')
+    // Every selectable value round-trips, and unset stays on the historical default.
+    for (const preset of ['coding', 'debug', 'research', 'ops', 'writing', 'custom'] as const) {
+      expect(resolveConfig(Config({ criteriaPreset: preset })).criteriaPreset).toBe(preset)
+    }
+    expect(Config({}).criteriaPreset).toBe('coding')
+    expect(resolveConfig({}).criteriaFile).toBe('')
+  })
+
   it('rejects the section the settings page blocks: a duplicate of the primary judge', () => {
     const drafts: ExtraJudgeDraft[] = [{ provider: 'deepseek-official', model: 'deepseek-flash' }]
     const payload = JSON.parse(JSON.stringify(serializeExtraJudges(drafts))) as NonNullable<Config['extraJudges']>

package/src/config.ts

@@ -1,5 +1,9 @@
 import type { Context } from '@deepseek-ai/cordis'
 import z from 'schemastery'
+import { CRITERIA_PRESET_IDS, type CriteriaPresetId } from './core.ts'
+
+/** A configured rubric: a bundled task-class preset, or a markdown file. */
+export type CriteriaPresetSelection = CriteriaPresetId | 'custom'
 
 export const VERIFIER_SETTINGS_NAMESPACE = 'llm-verifier' as never
 
@@ -28,6 +32,16 @@ export interface Config {
   autoVerifyMode?: AutoVerifyMode
   autoVerifyThreshold?: number
   autoVerifyRepeats?: number
+  /**
+   * Scoring repeats for an automatic `track` route only.
+   *
+   * Progress scores on a model without token logprobs come from the explicit-tag
+   * channel, i.e. ONE sampled letter per call, and one letter is worth 5.3% of the
+   * A–T scale — adjacent bands are a couple of samples apart. Averaging repeats is
+   * what the upstream `n_evaluations` does for exactly this reason, and a track call
+   * is the cheapest kind (one prompt, no tournament), so it defaults to 3.
+   */
+  autoTrackRepeats?: number
   /**
    * Scoring repeats for the FINAL session acceptance only.
    *
@@ -50,6 +64,22 @@ export interface Config {
   autoRouteMaxInputChars?: number
   autoMaxModelCallsPerTask?: number
   autoMaxModelCallsPerSession?: number
+  /**
+   * Persist a bounded snapshot (prompt + raw answer) of every verifier model call.
+   *
+   * Answers "why did the judge say that?" without replaying a session from its event
+   * log; capped per call, per record and per invocation by `decisions.ts`.
+   */
+  captureDecisions?: boolean
+  /**
+   * Rubric the automatic gate (final acceptance, routed compare/select/track) scores with.
+   *
+   * Defaults to `coding` = the historical DEFAULT_CRITERIA, so an existing installation is
+   * unaffected. Judging a research or ops task with the coding rubric measures the wrong thing.
+   */
+  criteriaPreset?: CriteriaPresetSelection
+  /** Markdown rubric file, read when `criteriaPreset` is `custom`. See README for the format. */
+  criteriaFile?: string
   autoVerifyTeamTasks?: boolean
   autoVerifyPlanMode?: boolean
   autoVerifySubagents?: boolean
@@ -75,6 +105,7 @@ export interface ResolvedConfig {
   autoVerifyMode: AutoVerifyMode
   autoVerifyThreshold: number
   autoVerifyRepeats: number
+  autoTrackRepeats: number
   autoVerifyFinalRepeats: number
   autoVerifyMinToolCalls: number
   autoVerifyMaxChars: number
@@ -90,6 +121,9 @@ export interface ResolvedConfig {
   autoRouteMaxInputChars: number
   autoMaxModelCallsPerTask: number
   autoMaxModelCallsPerSession: number
+  captureDecisions: boolean
+  criteriaPreset: CriteriaPresetSelection
+  criteriaFile: string
   autoVerifyTeamTasks: boolean
   autoVerifyPlanMode: boolean
   autoVerifySubagents: boolean
@@ -122,6 +156,7 @@ export const Config: z<Config> = z.object({
   autoVerifyMode: z.union(['manual', 'smart', 'strict']).default('smart'),
   autoVerifyThreshold: z.number().min(0).max(1).default(0.65),
   autoVerifyRepeats: z.number().step(1).min(1).default(1),
+  autoTrackRepeats: z.number().step(1).min(1).default(3),
   autoVerifyFinalRepeats: z.number().step(1).min(1).default(2),
   autoVerifyMinToolCalls: z.number().step(1).min(1).default(3),
   autoVerifyMaxChars: z.number().step(1).min(1000).default(80000),
@@ -132,11 +167,14 @@ export const Config: z<Config> = z.object({
   autoRouteMaxCandidates: z.number().step(1).min(3).default(8),
   autoRouteMaxPerTask: z.number().step(1).min(1).default(2),
   autoRouteMaxPerSession: z.number().step(1).min(1).default(8),
-  autoTrackCompletionThreshold: z.number().min(0).max(1).default(0.8),
+  autoTrackCompletionThreshold: z.number().min(0).max(1).default(0.684),
   autoRouteMaxItemChars: z.number().step(1).min(100).default(20000),
   autoRouteMaxInputChars: z.number().step(1).min(1000).default(60000),
   autoMaxModelCallsPerTask: z.number().step(1).min(1).default(96),
   autoMaxModelCallsPerSession: z.number().step(1).min(1).default(240),
+  captureDecisions: z.boolean().default(true),
+  criteriaPreset: z.union([...CRITERIA_PRESET_IDS, 'custom'] as const).default('coding'),
+  criteriaFile: z.string().default(''),
   autoVerifyTeamTasks: z.boolean().default(true),
   autoVerifyPlanMode: z.boolean().default(true),
   autoVerifySubagents: z.boolean().default(false),
@@ -201,6 +239,7 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
   if (!Number.isFinite(temperature) || temperature < 0 || temperature > 2) throw new Error('llm-verifier: temperature must be between 0 and 2')
   const values = {
     autoVerifyRepeats: config.autoVerifyRepeats ?? 1,
+    autoTrackRepeats: config.autoTrackRepeats ?? 3,
     autoVerifyFinalRepeats: config.autoVerifyFinalRepeats ?? 2,
     autoVerifyMinToolCalls: config.autoVerifyMinToolCalls ?? 3,
     autoVerifyMaxChars: config.autoVerifyMaxChars ?? 80000,
@@ -224,7 +263,7 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
   if (values.autoRouteMaxCandidates < 3 || values.autoRouteMaxCandidates > 16) throw new Error('llm-verifier: autoRouteMaxCandidates must be between 3 and 16')
   if (values.autoRouteMaxInputChars < values.autoRouteMaxItemChars * 2) throw new Error('llm-verifier: autoRouteMaxInputChars must fit at least two route items')
   const autoRouteMinConfidence = config.autoRouteMinConfidence ?? 0.9
-  const autoTrackCompletionThreshold = config.autoTrackCompletionThreshold ?? 0.8
+  const autoTrackCompletionThreshold = config.autoTrackCompletionThreshold ?? 0.684
   if (![autoRouteMinConfidence, autoTrackCompletionThreshold].every(value => Number.isFinite(value) && value >= 0 && value <= 1)) throw new Error('llm-verifier: auto route thresholds must be between 0 and 1')
   const maxRetries = config.maxRetries ?? 3
   if (!Number.isSafeInteger(maxRetries) || maxRetries < 0) throw new Error('llm-verifier: maxRetries must be a non-negative safe integer')
@@ -235,6 +274,11 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
   const estimatedInputUsdPerMillion = config.estimatedInputUsdPerMillion ?? 0
   const estimatedOutputUsdPerMillion = config.estimatedOutputUsdPerMillion ?? 0
   if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every(value => Number.isFinite(value) && value >= 0)) throw new Error('llm-verifier: estimated token prices must be finite non-negative numbers')
+  const criteriaPreset = config.criteriaPreset ?? 'coding'
+  if (criteriaPreset !== 'custom' && !CRITERIA_PRESET_IDS.includes(criteriaPreset as CriteriaPresetId)) throw new Error('llm-verifier: criteriaPreset must be one of ' + [...CRITERIA_PRESET_IDS, 'custom'].join(', '))
+  // A half-configured custom rubric is NOT a config error: CriteriaResolver falls back to the
+  // coding preset and reports why, so a typo in a rubric path cannot disable the gate.
+  const criteriaFile = (config.criteriaFile ?? '').trim()
   const reasoningEffort = config.reasoningEffort?.trim()
 
   const extraJudges = config.extraJudges ?? []
@@ -306,6 +350,9 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
     autoRouteSemantic: config.autoRouteSemantic ?? true,
     autoRouteMinConfidence,
     autoTrackCompletionThreshold,
+    captureDecisions: config.captureDecisions ?? true,
+    criteriaPreset,
+    criteriaFile,
     autoVerifyTeamTasks: config.autoVerifyTeamTasks ?? true,
     autoVerifyPlanMode: config.autoVerifyPlanMode ?? true,
     autoVerifySubagents: config.autoVerifySubagents ?? false,

package/src/core.test.ts

@@ -1,17 +1,22 @@
 import { describe, expect, it } from 'vitest'
 import {
+  CRITERIA_PRESETS,
   DEFAULT_CRITERIA,
+  EMPTY_WORK_BASELINE,
   accumulatePairs,
   bradleyTerry,
+  buildGenerationPrompt,
   buildPairwisePrompt,
   buildProgressPrompt,
   evidenceNonce,
   extractProgressScore,
   extractScore,
+  parseCriteriaMarkdown,
   pivotRoundPairs,
   rankScores,
   renderDelimitedBlock,
   ringCycle,
+  slugCriterionId,
   topPivots,
 } from './core.ts'
 
@@ -145,3 +150,93 @@ describe('pivot tournament', () => {
     expect(rankScores(wins, counts)[0]?.index).toBe(0)
   })
 })
+
+describe('criteria presets', () => {
+  it('keeps the default preset byte-identical to the historical default rubric', () => {
+    // The default preset must not change a single verdict, prompt or cache key.
+    expect(CRITERIA_PRESETS.coding).toEqual(DEFAULT_CRITERIA)
+    expect(CRITERIA_PRESETS.coding).toBe(DEFAULT_CRITERIA)
+  })
+
+  it('gives every preset 2-4 narrow, uniquely identified criteria', () => {
+    for (const [name, criteria] of Object.entries(CRITERIA_PRESETS)) {
+      expect(criteria.length, name).toBeGreaterThanOrEqual(2)
+      expect(criteria.length, name).toBeLessThanOrEqual(4)
+      expect(new Set(criteria.map(criterion => criterion.id)).size, name).toBe(criteria.length)
+      for (const criterion of criteria) {
+        expect(criterion.id, name).toMatch(/^[a-z0-9_]+$/)
+        expect(criterion.name.trim().length, name).toBeGreaterThan(0)
+        expect(criterion.description.trim().length, name).toBeGreaterThan(40)
+      }
+    }
+  })
+})
+
+describe('criteria markdown', () => {
+  const file = [
+    '# Task — Verifier Criteria',
+    '<!-- an author note the judge never sees -->',
+    '',
+    '## Ground Truth Note',
+    '',
+    'Do NOT trust the agent self-assessment.',
+    '',
+    '## Criteria',
+    '',
+    '### Final Answer Correctness',
+    '',
+    'Check the answer against what the task asked for.',
+    '',
+    '### Empirical Verification {#verification}',
+    '',
+    'Look at the commands the agent actually ran.',
+    '',
+  ].join('\n')
+
+  it('parses the note, the slugged ids and the pinned anchor', () => {
+    const parsed = parseCriteriaMarkdown(file)
+    expect(parsed.groundTruthNote).toBe('Do NOT trust the agent self-assessment.')
+    expect(parsed.criteria).toEqual([
+      { id: 'final_answer_correctness', name: 'Final Answer Correctness', description: 'Check the answer against what the task asked for.' },
+      { id: 'verification', name: 'Empirical Verification', description: 'Look at the commands the agent actually ran.' },
+    ])
+  })
+
+  it('slugs and de-duplicates colliding ids instead of merging two criteria', () => {
+    const parsed = parseCriteriaMarkdown('## Criteria\n\n### Same Name\n\nfirst body\n\n### Same Name\n\nsecond body\n')
+    expect(parsed.criteria.map(criterion => criterion.id)).toEqual(['same_name', 'same_name_2'])
+    expect(parsed.criteria).toHaveLength(2)
+  })
+
+  it('fails closed on a file with no criteria or with an empty instruction', () => {
+    expect(() => parseCriteriaMarkdown('# nothing here\n')).toThrow(/no criteria/u)
+    expect(() => parseCriteriaMarkdown('## Criteria\n\n### Empty\n\n### Other\n\nbody\n')).toThrow(/no instruction: empty/u)
+  })
+
+  it('derives ids with the same slug rule the parser uses', () => {
+    expect(slugCriterionId('Final Answer Correctness')).toBe('final_answer_correctness')
+    expect(slugCriterionId('***')).toBe('criterion')
+    expect(slugCriterionId('x'.repeat(80)).length).toBe(40)
+  })
+
+  it('renders a drafting prompt that cannot be mistaken for the judge contract', () => {
+    const prompt = buildGenerationPrompt('Fix the parser\nexactly.', 1, 3)
+    expect(prompt).toContain('Draft 2 of 3.')
+    expect(prompt).toContain('Fix the parser\nexactly.')
+    // Here the request IS the instruction to follow and the answer is free-form work product,
+    // so it is deliberately NOT wrapped in a data-only block nor given an A-T verdict contract.
+    expect(prompt).not.toContain('<<<TASK:')
+    expect(prompt).not.toContain('SECURITY')
+    expect(prompt).not.toContain('<score_A>')
+    // The only text that varies between the N drafts is the final line, so all of them share a
+    // maximal prompt prefix (the request itself) and a prefix cache can serve it.
+    const other = buildGenerationPrompt('Fix the parser\nexactly.', 2, 3)
+    let shared = 0
+    while (shared < prompt.length && prompt[shared] === other[shared]) shared += 1
+    expect(shared).toBeGreaterThan(prompt.length - 20)
+  })
+
+  it('keeps exactly one definition of the gate baseline', () => {
+    expect(EMPTY_WORK_BASELINE).toBe('(No useful work or verification was performed.)')
+  })
+})