dsh-llm-verifier 0.3.0 → 0.4.0

package/src/caller.test.ts

@@ -1,8 +1,10 @@
 import { afterEach, describe, expect, it, vi } from 'vitest'
-import { RequestLimiter, callVerifier } from './caller.ts'
+import { RequestLimiter, callVerifier, generateCandidate, generationClient } from './caller.ts'
 import { TopLogprobCapabilityCache } from './top-logprobs.ts'
 
 function chunks(text = '<score_A> A </score_A>') { return [{ type: 'block-start', index: 0, blockType: 'text' }, { type: 'text-delta', index: 0, text }, { type: 'block-end', index: 0, block: { type: 'text', text } }, { type: 'usage', usage: { inputTokens: 7, cacheReadTokens: 3, outputTokens: 4, reasoningTokens: 2 } }, { type: 'finish', reason: { kind: 'stop' } }] as any[] }
+/** The same stream, but the model stopped because it ran into the output ceiling. */
+function truncatedChunks(text = 'a draft that ran out of room') { const value = chunks(text); value[value.length - 1] = { type: 'finish', reason: { kind: 'max-tokens' } }; return value }
 function ctx(settingsValue?: unknown) { return { get(name: string) { if (name === 'settings' && settingsValue !== undefined) return { get: () => settingsValue }; if (name === 'credentials') return { resolve: async () => ({ value: 'secret' }) }; return undefined } } as any }
 function config(stream: (options: any) => AsyncIterable<any>, saveImage = vi.fn(), context = ctx(), overrides: Record<string, any> = {}) { return { ctx: context, llm: { stream } as any, attachments: { saveImage } as any, topLogprobCapabilities: new TopLogprobCapabilityCache(), provider: 'openai', model: 'gpt-5', reasoningEffort: 'high', maxTokens: 100, temperature: 0.2, timeoutMs: 1000, maxRetries: 2, retryBaseDelayMs: 1, ...overrides } }
 async function* streamOf(items: any[]) { for (const item of items) yield item }
@@ -243,3 +245,62 @@ describe('imageRefs eviction on rejection (FIX 3)', () => {
     expect(saveImage).toHaveBeenCalledTimes(2)
   })
 })
+
+describe('best-of-N generation seam', () => {
+  it('re-points a judge client at the session model and forces diversity sampling', () => {
+    const base = config(async function* () { yield* streamOf(chunks()) }, vi.fn(), ctx(), { reasoningEffort: 'high', temperature: 0.2, maxTokens: 100 })
+    const target = generationClient(base, { provider: 'session-p', model: 'session-m' })
+    expect(target.provider).toBe('session-p')
+    expect(target.model).toBe('session-m')
+    expect(target.temperature).toBe(1)
+    expect(target.maxTokens).toBe(16384)
+    // The base is a judge: its reasoning effort must NOT leak into the drafts, because the
+    // session model's own effort is the one the drafts have to be produced with.
+    expect(target.reasoningEffort).toBeUndefined()
+    expect(target.limiter).toBeUndefined()
+    // Transport identity is inherited, so generation uses the same timeout and retry budget.
+    expect(target.timeoutMs).toBe(base.timeoutMs)
+    expect(target.maxRetries).toBe(base.maxRetries)
+    expect(target.retryBaseDelayMs).toBe(base.retryBaseDelayMs)
+    expect(generationClient(base, { provider: 'p', model: 'm', reasoningEffort: 'low' }).reasoningEffort).toBe('low')
+    // A caller can override the ceiling for one attempt, which is what makes it testable.
+    expect(generationClient(base, { provider: 'p', model: 'm' }, 4096).maxTokens).toBe(4096)
+  })
+
+  it('returns plain text, complete, without asking for or parsing an A-T verdict', async () => {
+    let seen: any
+    const base = config(async function* (options: any) { seen = options; yield* streamOf(chunks('a complete draft, with no score tags at all')) })
+    const result = await generateCandidate(base, { provider: 'p', model: 'm' }, 'draft this')
+    expect(result.text).toBe('a complete draft, with no score tags at all')
+    expect(result.truncated).toBe(false)
+    expect(result.usage.inputTokens).toBe(7)
+    // No probability channel is involved: generation is not a scored call.
+    expect(result.scoringMode).toBe('explicit-tag')
+    expect(result.tokens).toEqual([])
+    expect(seen.temperature).toBe(1)
+    expect(seen.maxTokens).toBe(16384)
+  })
+
+  it('keeps a truncated draft instead of discarding a paid generation', async () => {
+    // Regression from the first real end-to-end acceptance: all three drafts of a
+    // "function + 12 test cases + a note" task hit the ceiling, callExplicitTag's max-tokens error
+    // (correct for a judge) propagated through the generator, and the tool returned zero candidates.
+    const ceilings: Array<number | undefined> = []
+    const base = config(async function* (options: any) { ceilings.push(options.maxTokens); yield* streamOf(truncatedChunks('a draft that ran out of room')) })
+    const result = await generateCandidate(base, { provider: 'p', model: 'm' }, 'draft this')
+    expect(result.truncated).toBe(true)
+    expect(result.text).toBe('a draft that ran out of room')
+    expect(ceilings).toEqual([16384])
+  })
+
+  it('keeps the judge path fail-closed on a truncated answer', async () => {
+    // The tolerance above must not leak into scoring: a truncated verdict has no usable tags.
+    const judge = config(async function* () { yield* streamOf(truncatedChunks('<score_A> A </score_A>')) }, vi.fn(), ctx(), { maxRetries: 0 })
+    await expect(callVerifier(judge, 'prompt')).rejects.toThrow(/max tokens/u)
+  })
+
+  it('fails closed on an empty draft instead of returning one', async () => {
+    const base = config(async function* () { yield* streamOf(chunks('   ')) }, vi.fn(), ctx(), { maxRetries: 0 })
+    await expect(generateCandidate(base, { provider: 'p', model: 'm' }, 'draft this')).rejects.toThrow(/produced no text/u)
+  })
+})

package/src/caller.ts

@@ -117,7 +117,23 @@ async function retrying<T>(config: VerifierClientConfig, signal: AbortSignal | u
 /** One attachment per image object per process: retries must not duplicate attachments. */
 const imageRefs = new WeakMap<VerifierImage, Promise<unknown>>()
 
-async function callExplicitTag(config: VerifierClientConfig, prompt: string, signal: AbortSignal | undefined, images: readonly VerifierImage[] | undefined, attempt: number): Promise<VerifierCompletion> {
+/**
+ * One plain-text completion through the DSH stream.
+ *
+ * Shared by the explicit-tag judge path and by best-of-N generation, which differ in exactly one
+ * place: a `max-tokens` finish. A truncated JUDGE answer is unusable — its verdict tags may never
+ * have been emitted — so it stays a hard error. A truncated DRAFT is still a candidate the operator
+ * already paid for, and the judge can see that the text stops mid-sentence and score it accordingly,
+ * so generation receives the text plus a `truncated` flag.
+ * @param config - resolved verifier client configuration.
+ * @param prompt - the rendered prompt.
+ * @param signal - the attempt's deadline signal.
+ * @param images - attachments to send, if any.
+ * @param attempt - 1-based attempt number for usage accounting.
+ * @param tolerateTruncation - return the text of a max-tokens finish instead of throwing.
+ * @returns The completion (with `truncated`) when it produced text.
+ */
+async function callTextCompletion(config: VerifierClientConfig, prompt: string, signal: AbortSignal | undefined, images: readonly VerifierImage[] | undefined, attempt: number, tolerateTruncation: boolean): Promise<VerifierCompletion & { truncated: boolean }> {
   const content: ContentBlock[] = [{ type: 'text', text: prompt }]
   for (const image of images ?? []) {
     let pending = imageRefs.get(image)
@@ -147,13 +163,24 @@ async function callExplicitTag(config: VerifierClientConfig, prompt: string, sig
     signal,
   })
   for await (const chunk of config.llm.stream(options)) assembler.push(chunk)
-  const failed = failureMessage(assembler.finish)
-  if (failed !== undefined) throw new Error('llm-verifier: model call failed: ' + failed)
+  let truncated = false
+  if (assembler.finish.kind === 'max-tokens') {
+    if (!tolerateTruncation) throw new Error('llm-verifier: model call failed: ' + failureMessage(assembler.finish))
+    truncated = true
+  } else {
+    const failed = failureMessage(assembler.finish)
+    if (failed !== undefined) throw new Error('llm-verifier: model call failed: ' + failed)
+  }
   const text = assembler.blocks().filter((block): block is Extract<ContentBlock, { type: 'text' }> => block.type === 'text').map(block => block.text).join('')
   if (!text.trim()) throw new Error('llm-verifier: selected DSH model produced no text')
   // DSH adapters expose provider-neutral text/usage but not top-logprob candidates.
   // extractScore() therefore uses the model's explicit final A–T tags.
-  return { text, tokens: [], positions: [], scoringMode: 'explicit-tag', usage: usage(attempt, assembler.usage) }
+  return { text, tokens: [], positions: [], scoringMode: 'explicit-tag', usage: usage(attempt, assembler.usage), truncated }
+}
+
+/** Explicit-tag judge completion: fail-closed, so a truncated answer is an error and never a score. */
+async function callExplicitTag(config: VerifierClientConfig, prompt: string, signal: AbortSignal | undefined, images: readonly VerifierImage[] | undefined, attempt: number): Promise<VerifierCompletion> {
+  return callTextCompletion(config, prompt, signal, images, attempt, false)
 }
 
 export class RequestLimiter {
@@ -205,6 +232,99 @@ export async function callVerifier(config: VerifierClientConfig, prompt: string,
   return config.limiter === undefined ? invoke() : config.limiter.run(invoke, signal)
 }
 
+/**
+ * Best-of-N generation sampling constants.
+ *
+ * Deliberately NOT the judge's temperature: best-of-N only pays off when the drafts actually
+ * differ, and the judge's low default (0.2) would produce N near-copies — the tool would then
+ * spend N times the money choosing between the same answer.
+ */
+export const GENERATION_TEMPERATURE = 1
+/**
+ * Output ceiling for one draft.
+ *
+ * 4096 was a guess in the plan, and the first real end-to-end acceptance disproved it: a
+ * "function + 12 test cases + a note" task truncated ALL THREE drafts, so the tool returned
+ * nothing at all. The measured cause is reasoning tokens: on a trivial 2-draft run the session
+ * model (deepseek-official/deepseek-flash) spent 16363 reasoning tokens out of 17254 output
+ * tokens — roughly 8k of reasoning PER DRAFT, which eats any 4096 budget before the answer starts.
+ *
+ * 16384 is about 2x that measured per-draft spend, and it is also the largest ceiling that keeps
+ * a pairwise judge prompt inside the plugin's own evidence limit: two drafts at 16384 tokens are
+ * roughly 130k characters against a 240k explicit-evidence ceiling (EXPLICIT_MAX_TOTAL_CHARS).
+ * `maxTokens` is a ceiling rather than a reservation, so a draft that needs less room costs
+ * exactly what it did before. A task whose answer genuinely needs more is too large for this tool,
+ * and its truncation is REPORTED instead of being silently returned.
+ */
+export const GENERATION_MAX_TOKENS = 16384
+
+export interface GenerationTarget {
+  provider: string
+  model: string
+  reasoningEffort?: string
+}
+
+/** One draft: plain text, never scored, plus whether it ran into the output ceiling. */
+export interface GeneratedCandidate extends VerifierCompletion { truncated: boolean }
+
+/**
+ * Re-point an existing verifier client at the model that should draft the candidates.
+ *
+ * Everything transport-shaped (context, llm runtime, attachments, capability memory, limiter,
+ * timeout, retry budget) is inherited; only the route and the two generation-specific scalars
+ * are replaced. `reasoningEffort` comes from the target and is never inherited from the base,
+ * because the base is a judge: a leftover judge effort would silently draft with the wrong model
+ * settings.
+ * @param base - any configured verifier client (the primary judge is the cheapest source).
+ * @param target - the model that writes the drafts.
+ * @param maxTokens - output ceiling for this attempt.
+ * @returns A client config safe to pass to {@link callGeneratedText}.
+ */
+export function generationClient(base: VerifierClientConfig, target: GenerationTarget, maxTokens = GENERATION_MAX_TOKENS): VerifierClientConfig {
+  return {
+    ctx: base.ctx,
+    llm: base.llm,
+    attachments: base.attachments,
+    topLogprobCapabilities: base.topLogprobCapabilities,
+    provider: target.provider,
+    model: target.model,
+    label: target.provider + '/' + target.model,
+    ...(target.reasoningEffort === undefined ? {} : { reasoningEffort: target.reasoningEffort }),
+    maxTokens,
+    temperature: GENERATION_TEMPERATURE,
+    timeoutMs: base.timeoutMs,
+    maxRetries: base.maxRetries,
+    retryBaseDelayMs: base.retryBaseDelayMs,
+    ...(base.limiter === undefined ? {} : { limiter: base.limiter }),
+  }
+}
+
+/**
+ * One best-of-N draft.
+ *
+ * A named seam over the plain-text path: generation has no A–T contract and no scoring channel, and
+ * the name keeps it distinct from the judge calls at every call site and in the decision snapshot.
+ *
+ * Hitting the output ceiling is deliberately NOT an error here. The operator already paid for those
+ * tokens, and the judge can see for itself that the text stops mid-sentence. That is exactly why the
+ * judge's fail-closed max-tokens handling was split out of this path: before the split, one long
+ * task made the whole tool return zero candidates.
+ * @param base - any configured verifier client (the primary judge is the cheapest source).
+ * @param target - the model that writes the draft (normally the session model).
+ * @param prompt - the rendered drafting prompt.
+ * @param signal - caller's abort signal.
+ * @returns The draft text, token usage, and whether it ran into the ceiling.
+ */
+export async function generateCandidate(base: VerifierClientConfig, target: GenerationTarget, prompt: string, signal?: AbortSignal): Promise<GeneratedCandidate> {
+  return callGeneratedText(generationClient(base, target), prompt, signal)
+}
+
+/** Generation transport: the judge's timeout/retry/limiter wrappers, with truncation as a result. */
+async function callGeneratedText(config: VerifierClientConfig, prompt: string, signal: AbortSignal | undefined): Promise<GeneratedCandidate> {
+  const invoke = () => retrying(config, signal, (attemptSignal, attempt) => callTextCompletion(config, prompt, attemptSignal, undefined, attempt, true))
+  return config.limiter === undefined ? invoke() : config.limiter.run(invoke, signal)
+}
+
 /** Plain-text verifier call for conservative JSON routing; probability labels are intentionally bypassed. */
 export async function callVerifierText(config: VerifierClientConfig, prompt: string, signal?: AbortSignal): Promise<VerifierCompletion> {
   const invoke = () => retrying(config, signal, (attemptSignal, attempt) => callExplicitTag(config, prompt, attemptSignal, undefined, attempt))

package/src/client-i18n.ts

@@ -13,7 +13,7 @@ export const zh = {
   'field.enabled.title': '启用 Verifier 工具',
   'field.enabled.helpOn': '允许 Agent 调用四个 verifier 工具并向裁判模型发起请求。',
   'field.enabled.helpOff': '停用后，所有 verifier 工具都会立即返回错误。',
-  'field.enabled.warnDisabled': 'verifier_compare、verifier_select、verifier_track 和 verifier_current_session 当前不可用。',
+  'field.enabled.warnDisabled': 'verifier_compare、verifier_select、verifier_track、verifier_best_of_n 和 verifier_current_session 当前不可用。',
 
   // Section: Automatic Verification
   'section.autoVerify': '自动验收',
@@ -22,6 +22,20 @@ export const zh = {
   'field.autoVerifyMode.manual': '手动',
   'field.autoVerifyMode.smart': '智能（推荐）',
   'field.autoVerifyMode.strict': '严格',
+  'field.criteriaPreset.title': '验收判据',
+  'field.criteriaPreset.help': '自动门控（最终验收、自动 compare/select/track）与未显式传 criteria 的工具所采用的评分标准。默认「编码」与历史内置判据逐字相同，切换会改变验收松紧与缓存键。',
+  'field.criteriaPreset.coding': '编码（默认）',
+  'field.criteriaPreset.debug': '调试排错',
+  'field.criteriaPreset.research': '研究问答',
+  'field.criteriaPreset.ops': '运维操作',
+  'field.criteriaPreset.writing': '文档写作',
+  'field.criteriaPreset.custom': '自定义文件',
+  'field.criteriaPreset.previewSummary': '预览判官提示词（{count} 条判据中的第 1 条）',
+  'field.criteriaPreset.sampleTask': '（示例任务）修复登录接口返回 500 的问题。',
+  'field.criteriaPreset.sampleA': '（示例轨迹 A）运行 pytest -k login，输出 1 failed … 修改 auth.py … 再次运行，输出 1 passed。',
+  'field.criteriaPreset.sampleB': '（示例轨迹 B）已经改好了，应该没问题。',
+  'field.criteriaFile.title': '判据文件',
+  'field.criteriaFile.help': 'Markdown 判据文件路径：## Criteria 下每个 ### 判据名 一条判据，可选 {#id} 锚定 id，可选 ## Ground Truth Note。文件缺失或解析失败时退回「编码」判据并在日志中告警，不会让验收失效。注意：预算告警按 3 条判据估算，自定义文件条数更多时请按比例调高「任务/会话模型调用预算」，否则可能在验收中途耗尽。',
   'field.autoRouteSemantic.title': '混合语义路由',
   'field.autoRouteSemantic.help': '结构化候选或检查点不足时，由裁判模型保守识别真实的 compare/select/track 对象。智能模式仅在有候选线索时运行；严格模式每次结束边界都会检查。',
   'field.autoRouteMinConfidence.title': '语义路由置信度',
@@ -29,11 +43,11 @@ export const zh = {
   'field.autoRouteMaxCandidates.title': '最多候选数',
   'field.autoRouteMaxCandidates.help': '一次自动 select 最多纳入的真实候选数量，至少 3。',
   'field.autoRouteMaxPerTask.title': '每任务最多路由',
-  'field.autoRouteMaxPerTask.help': '自动路由（compare/select/track 与语义分类）的任务尝试上限；与「每任务最多验收」共享同一个计数器，实际可用次数是两者之和。',
+  'field.autoRouteMaxPerTask.help': '自动路由（compare/select/track、语义分类、团队任务闸与计划预审）的任务尝试上限；与「每任务最多验收」各自独立计数，路由无论跑多少次都不会占用最终验收的额度。',
   'field.autoRouteMaxPerSession.title': '每会话最多路由',
-  'field.autoRouteMaxPerSession.help': '同一会话中自动路由的尝试上限；与「每会话最多验收」共享计数器，实际可用次数同样是两者之和。',
+  'field.autoRouteMaxPerSession.help': '同一会话中自动路由的尝试上限；与「每会话最多验收」各自独立计数。',
   'field.autoTrackCompletionThreshold.title': '进度完成阈值',
-  'field.autoTrackCompletionThreshold.help': 'track 最新检查点低于该值时 steering 要求 Agent 继续执行；更早的检查点只作展示，范围 0–1。',
+  'field.autoTrackCompletionThreshold.help': 'track 最新检查点低于该值时 steering 要求 Agent 继续执行；更早的检查点只作展示，范围 0–1。默认 0.684 = 标尺上 N（“偏成功”）的起点，与判官提示词的档位描述自洽。',
   'field.autoRouteMaxItemChars.title': '单项证据字符上限',
   'field.autoRouteMaxItemChars.help': '每个候选或检查点在脱敏后允许发送给裁判的最大字符数。',
   'field.autoRouteMaxInputChars.title': '路由证据总字符上限',
@@ -45,7 +59,9 @@ export const zh = {
   'field.autoVerifyThreshold.title': '通过阈值',
   'field.autoVerifyThreshold.help': '会话证据分数达到该值且胜过空工作基线才允许结束；范围 0–1。',
   'field.autoVerifyRepeats.title': '路由评估轮次',
-  'field.autoVerifyRepeats.help': '自动路由（compare / select / track）每项标准的评分重复次数；1 为低成本初筛。',
+  'field.autoVerifyRepeats.help': '自动路由（compare / select）每项标准的评分重复次数；1 为低成本初筛。',
+  'field.autoTrackRepeats.title': '进度跟踪轮次',
+  'field.autoTrackRepeats.help': 'track 路由的评分重复次数并取平均。判官模型没有 logprobs 时每次只采到一个字母（A–T 每档 5.3%），取平均可压住采样噪声；默认 3。',
   'field.autoVerifyFinalRepeats.title': '最终验收轮次',
   'field.autoVerifyFinalRepeats.help': '最终会话验收每项标准的评分重复次数；偶数轮会交换 A/B 位置以抵消位置偏好，默认 2。',
   'field.autoVerifyMinToolCalls.title': '智能模式最少工具调用',
@@ -53,9 +69,9 @@ export const zh = {
   'field.autoVerifyMaxChars.title': '最大证据字符',
   'field.autoVerifyMaxChars.help': '发送给裁判前保留的最近会话证据字符数。',
   'field.autoVerifyMaxPerTask.title': '每任务最多验收',
-  'field.autoVerifyMaxPerTask.help': '最终验收的任务尝试上限；与「每任务最多路由」共享同一个计数器，实际可用次数是两者之和，用于防止低分反馈形成无限修复循环。',
+  'field.autoVerifyMaxPerTask.help': '最终验收的任务尝试上限；与「每任务最多路由」各自独立计数，保证被路由要求过的最终验收一定有额度可用，同时防止低分反馈形成无限修复循环。',
   'field.autoVerifyMaxPerSession.title': '每会话最多验收',
-  'field.autoVerifyMaxPerSession.help': '同一会话中最终验收的尝试上限；与「每会话最多路由」共享计数器，实际可用次数同样是两者之和。',
+  'field.autoVerifyMaxPerSession.help': '同一会话中最终验收的尝试上限；与「每会话最多路由」各自独立计数。',
   'field.autoVerify.warnNotice': '自动路由按 select → compare → track → current_session 的阶段顺序处理；脱敏后的任务、Assistant 轨迹与真实工具输出会发送给所选裁判模型。候选选择或进度不足会 steering 继续工作，最终验收未通过会要求修复并重新验证。',
   'field.autoMaxModelCallsPerTask.warnBudget': '当前单任务预算为 {current} 次模型调用，低于 {judges} 位裁判完整锦标赛所需的最差情况（{required} 次）。建议调大该预算以避免自动路由过早耗尽。',
   'field.autoMaxModelCallsPerSession.warnBudget': '当前会话预算为 {current} 次模型调用，低于 {judges} 位裁判推荐的最差情况（{required} 次）。建议调大该预算以避免会话验证过早耗尽。',
@@ -127,6 +143,18 @@ export const zh = {
   'stats.currentSession': '当前会话',
   'stats.allSessions': '全部会话',
   'stats.refresh': '刷新',
+  'probe.button': '判官自检',
+  'probe.running': '检测中…',
+  'probe.title': '判官自检',
+  'probe.note': '每个判官发一次真实调用，并强制重新探测评分通道；结果不计入统计',
+  'probe.reprobed': '已重新探测',
+  'probe.rubric': '生效判据：{source}（{count} 条）{file}',
+  'probe.rubricFallback': '自定义判据文件不可用：{error}（本次已退回编码判据）',
+  'probe.channel': '通道 {channel}',
+  'probe.scores': 'A {a} · B {b}',
+  'probe.latency': '{ms} ms',
+  'probe.calls': '{calls} 次调用',
+  'probe.failed': '失败',
   'stats.requestFailed': '统计接口请求失败',
   'stats.hostRestartHint': '请确认宿主已重启并加载最新版本的 dsh-llm-verifier。',
   'stats.loading': '正在读取工具运行统计…',
@@ -158,8 +186,10 @@ export const zh = {
   'stats.verdict.criteria': '标准 {criteria}',
 
   // Metrics
-  'metric.cacheHitRate': '缓存命中率',
-  'metric.cacheHitNote': '{hits} 命中 / {total} 评分',
+  'metric.cacheHitRate': '评分缓存命中率',
+  'metric.cacheHitNote': '{hits} 命中 / {total} 次评分查询（本地复用，未调用模型）',
+  'metric.prefixCacheHitRate': '前缀缓存命中率',
+  'metric.prefixCacheHitNote': '输入 {cached} / {input} 命中（由模型服务计费口径统计）',
   'metric.tokens': 'Token',
   'metric.tokensNote': '输入 {input} · 输出 {output}',
   'metric.avgModelCalls': '平均模型请求',
@@ -190,6 +220,22 @@ export const zh = {
   'recent.title': '最近调用',
   'recent.maxCount': '最多 50 条',
   'recent.empty': '暂无记录',
+  'recent.decision': '决策快照',
+  'recent.decisionHide': '收起快照',
+  'recent.decisionLoading': '正在读取…',
+  'recent.decisionMissing': '找不到这次调用的快照（可能已被淘汰或从未捕获）',
+  'recent.decisionEmpty': '这次调用没有捕获到模型调用（命中缓存或已关闭捕获）',
+  'recent.decisionPrompt': '提示词',
+  'recent.decisionOutput': '原始回答',
+  'recent.details': '查看详情',
+  'recent.detail.criterion': '判据',
+  'recent.detail.score': '分数',
+  'recent.detail.threshold': '阈值',
+  'recent.detail.calls': '模型调用 {calls}',
+  'recent.detail.tokens': '输入 {input} · 缓存 {cached} · 输出 {output}',
+  'recent.detail.scoreCache': '评分缓存 {hits} 命中 / {misses} 未命中',
+  'recent.detail.prefixCache': '前缀缓存命中 {rate}',
+  'recent.detail.cost': '预估成本 {cost}',
 
   // Model summary
   'models.title': '模型汇总',
@@ -208,6 +254,8 @@ export const zh = {
   'field.autoVerifyTeamTasks.help': '当 Agent Teams 中的任务状态变更或进入完成状态时，自动路由进度追踪与任务验收。',
   'field.autoVerifyPlanMode.title': '计划模式自动预审',
   'field.autoVerifyPlanMode.help': '在 Agent 调用 exit_plan_mode 提交计划交付人类评审前，自动对计划完整度与风险进行独立预审。',
+  'field.captureDecisions.title': '保存决策快照',
+  'field.captureDecisions.help': '把每次裁判调用的提示词与原始回答脱敏后限量存进本话题的 verifier/decisions-v1.json（每次调用 ≤ 12 次模型调用、单条记录 ≤ 3 万字符、按话题保留最近 40 条）。用于在统计看板里回答"裁判为什么这么判"，不参与任何判定。',
   'field.autoVerifySubagents.title': '同时验收子 Agent',
   'field.autoVerifySubagents.help': '子 Agent（subagent / fork）的会话也执行自动路由与最终验收。默认关闭：子会话同样以真实用户消息播种，被门控会额外消耗裁判预算并可能反复 steering 子 Agent。',
 }
@@ -227,7 +275,7 @@ export const en: I18nDict = {
   'field.enabled.title': 'Enable Verifier Tools',
   'field.enabled.helpOn': 'Allows the Agent to call verifier tools and make requests to the judge model.',
   'field.enabled.helpOff': 'When disabled, all verifier tools will return an error immediately.',
-  'field.enabled.warnDisabled': 'verifier_compare, verifier_select, verifier_track, and verifier_current_session are currently unavailable.',
+  'field.enabled.warnDisabled': 'verifier_compare, verifier_select, verifier_track, verifier_best_of_n, and verifier_current_session are currently unavailable.',
 
   // Section: Automatic Verification
   'section.autoVerify': 'Automatic Verification',
@@ -236,6 +284,20 @@ export const en: I18nDict = {
   'field.autoVerifyMode.manual': 'Manual',
   'field.autoVerifyMode.smart': 'Smart (Recommended)',
   'field.autoVerifyMode.strict': 'Strict',
+  'field.criteriaPreset.title': 'Acceptance Rubric',
+  'field.criteriaPreset.help': 'The rubric the automatic gate (final acceptance, routed compare/select/track) and any tool call without an explicit criteria argument scores with. The default "Coding" preset is the historical rubric, word for word; switching it changes acceptance strictness and the cache keys.',
+  'field.criteriaPreset.coding': 'Coding (default)',
+  'field.criteriaPreset.debug': 'Debugging',
+  'field.criteriaPreset.research': 'Research Q&A',
+  'field.criteriaPreset.ops': 'Operations',
+  'field.criteriaPreset.writing': 'Writing',
+  'field.criteriaPreset.custom': 'Custom file',
+  'field.criteriaPreset.previewSummary': 'Preview the judge prompt (criterion 1 of {count})',
+  'field.criteriaPreset.sampleTask': '(sample task) Fix the login endpoint returning 500.',
+  'field.criteriaPreset.sampleA': '(sample trajectory A) Ran pytest -k login: 1 failed … changed auth.py … ran it again: 1 passed.',
+  'field.criteriaPreset.sampleB': '(sample trajectory B) Already fixed it, should be fine.',
+  'field.criteriaFile.title': 'Criteria File',
+  'field.criteriaFile.help': 'Path to a markdown rubric: one ### Criterion Name per criterion under ## Criteria, an optional {#id} anchor, an optional ## Ground Truth Note. A missing or unparsable file falls back to the coding rubric with a logged warning instead of disabling the gate. Note: the budget warning assumes three criteria, so raise the task/session call budget proportionally when your file has more.',
   'field.autoRouteSemantic.title': 'Hybrid Semantic Routing',
   'field.autoRouteSemantic.help': 'When structured candidates or checkpoints are lacking, the judge model conservatively identifies real compare/select/track targets. Smart mode only runs when candidate clues exist; Strict mode checks at every task boundary.',
   'field.autoRouteMinConfidence.title': 'Semantic Route Confidence',
@@ -243,11 +305,11 @@ export const en: I18nDict = {
   'field.autoRouteMaxCandidates.title': 'Max Candidates',
   'field.autoRouteMaxCandidates.help': 'Maximum number of real candidates included in an automatic select, at least 3.',
   'field.autoRouteMaxPerTask.title': 'Max Routes per Task',
-  'field.autoRouteMaxPerTask.help': 'Task attempt cap for automatic routing (compare/select/track and semantic classification); shares one counter with "Max Verifications per Task", so the effective limit is the sum of both.',
+  'field.autoRouteMaxPerTask.help': 'Task attempt cap for automatic routing (compare/select/track, semantic classification, team-task gates and plan pre-review); counted separately from "Max Verifications per Task", so routing can never spend the final acceptance budget.',
   'field.autoRouteMaxPerSession.title': 'Max Routes per Session',
-  'field.autoRouteMaxPerSession.help': 'Session attempt cap for automatic routing; shares its counter with "Max Verifications per Session" (effective limit is the sum of both).',
+  'field.autoRouteMaxPerSession.help': 'Session attempt cap for automatic routing; counted separately from "Max Verifications per Session".',
   'field.autoTrackCompletionThreshold.title': 'Progress Completion Threshold',
-  'field.autoTrackCompletionThreshold.help': 'When the newest track checkpoint falls below this score, steering instructs the Agent to continue; earlier checkpoints are shown only; range 0–1.',
+  'field.autoTrackCompletionThreshold.help': 'When the newest track checkpoint falls below this score, steering instructs the Agent to continue; earlier checkpoints are shown only; range 0–1. The 0.684 default is the start of band N ("leans YES") on the rating scale, so it matches what the judge prompt tells the judge.',
   'field.autoRouteMaxItemChars.title': 'Max Chars per Route Item',
   'field.autoRouteMaxItemChars.help': 'Maximum sanitized character count allowed per candidate or checkpoint when sent to the judge.',
   'field.autoRouteMaxInputChars.title': 'Total Route Input Chars Cap',
@@ -257,7 +319,9 @@ export const en: I18nDict = {
   'field.autoMaxModelCallsPerSession.title': 'Model Call Budget per Session',
   'field.autoMaxModelCallsPerSession.help': 'Estimated model call cap shared across all automatic verification phases throughout the entire session.',
   'field.autoVerifyRepeats.title': 'Routing Repeats',
-  'field.autoVerifyRepeats.help': 'Scoring repeats per criterion for automatic routing (compare / select / track); 1 is low-cost preliminary screening.',
+  'field.autoVerifyRepeats.help': 'Scoring repeats per criterion for automatic routing (compare / select); 1 is low-cost preliminary screening.',
+  'field.autoTrackRepeats.title': 'Progress Repeats',
+  'field.autoTrackRepeats.help': 'Scoring repeats for an automatic track route, averaged. Without token logprobs each call samples ONE letter (5.3% of the A-T scale per letter), so repeats keep the progress curve from flipping bands on noise; default 3.',
   'field.autoVerifyFinalRepeats.title': 'Final Acceptance Repeats',
   'field.autoVerifyFinalRepeats.help': 'Scoring repeats per criterion for the final session acceptance; even rounds swap A/B positions to cancel position bias. Defaults to 2.',
   'field.autoVerifyThreshold.title': 'Pass Threshold',
@@ -267,9 +331,9 @@ export const en: I18nDict = {
   'field.autoVerifyMaxChars.title': 'Max Evidence Characters',
   'field.autoVerifyMaxChars.help': 'Maximum recent session evidence characters retained before sending to the judge.',
   'field.autoVerifyMaxPerTask.title': 'Max Verifications per Task',
-  'field.autoVerifyMaxPerTask.help': 'Final-acceptance attempt cap per task; shares one counter with "Max Routes per Task", so the effective limit is the sum of both. Prevents endless low-score repair loops.',
+  'field.autoVerifyMaxPerTask.help': 'Final-acceptance attempt cap per task; counted separately from "Max Routes per Task", so an armed final gate always has an attempt reserved. Prevents endless low-score repair loops.',
   'field.autoVerifyMaxPerSession.title': 'Max Verifications per Session',
-  'field.autoVerifyMaxPerSession.help': 'Final-acceptance attempt cap within one session; shares its counter with "Max Routes per Session" (effective limit is the sum of both).',
+  'field.autoVerifyMaxPerSession.help': 'Final-acceptance attempt cap within one session; counted separately from "Max Routes per Session".',
   'field.autoVerify.warnNotice': 'Automatic routing follows the phase order: select → compare → track → current_session. Sanitized tasks, Assistant trajectories, and real tool outputs are sent to the judge model. Candidate selection or insufficient progress steers the Agent to keep working; failed final acceptance requires remediation and re-verification.',
   'field.autoMaxModelCallsPerTask.warnBudget': 'Current task budget ({current} calls) is below the worst-case requirement ({required} calls) for {judges} judge(s) in a full tournament. Consider increasing it to avoid premature budget exhaustion.',
   'field.autoMaxModelCallsPerSession.warnBudget': 'Current session budget ({current} calls) is below the worst-case requirement ({required} calls) for {judges} judge(s). Consider increasing it to avoid premature budget exhaustion.',
@@ -341,6 +405,18 @@ export const en: I18nDict = {
   'stats.currentSession': 'Current Session',
   'stats.allSessions': 'All Sessions',
   'stats.refresh': 'Refresh',
+  'probe.button': 'Probe judges',
+  'probe.running': 'Probing…',
+  'probe.title': 'Judge probe',
+  'probe.note': 'One real call per judge, forcing a fresh scoring-channel probe; not counted in the statistics',
+  'probe.reprobed': 're-probed',
+  'probe.rubric': 'Rubric in effect: {source} ({count} criteria) {file}',
+  'probe.rubricFallback': 'Custom criteria file unusable: {error} (fell back to coding for this run)',
+  'probe.channel': 'channel {channel}',
+  'probe.scores': 'A {a} · B {b}',
+  'probe.latency': '{ms} ms',
+  'probe.calls': '{calls} calls',
+  'probe.failed': 'Failed',
   'stats.requestFailed': 'Failed to query statistics API',
   'stats.hostRestartHint': 'Please make sure the host has restarted and loaded the latest version of dsh-llm-verifier.',
   'stats.loading': 'Loading tool execution statistics…',
@@ -372,8 +448,10 @@ export const en: I18nDict = {
   'stats.verdict.criteria': 'Criteria {criteria}',
 
   // Metrics
-  'metric.cacheHitRate': 'Cache Hit Rate',
-  'metric.cacheHitNote': '{hits} hits / {total} scored',
+  'metric.cacheHitRate': 'Score Cache Hit Rate',
+  'metric.cacheHitNote': '{hits} hits / {total} lookups (reused locally, no model call)',
+  'metric.prefixCacheHitRate': 'Prefix Cache Hit Rate',
+  'metric.prefixCacheHitNote': '{cached} of {input} input tokens served from cache',
   'metric.tokens': 'Tokens',
   'metric.tokensNote': 'Input {input} · Output {output}',
   'metric.avgModelCalls': 'Avg Model Calls',
@@ -404,6 +482,22 @@ export const en: I18nDict = {
   'recent.title': 'Recent Invocations',
   'recent.maxCount': 'Max 50',
   'recent.empty': 'No records',
+  'recent.decision': 'Decision snapshot',
+  'recent.decisionHide': 'Hide snapshot',
+  'recent.decisionLoading': 'Loading…',
+  'recent.decisionMissing': 'No snapshot for this invocation (pruned, or never captured)',
+  'recent.decisionEmpty': 'No model calls were captured for this invocation (cache hit, or capture disabled)',
+  'recent.decisionPrompt': 'Prompt',
+  'recent.decisionOutput': 'Raw answer',
+  'recent.details': 'Details',
+  'recent.detail.criterion': 'Criterion',
+  'recent.detail.score': 'Score',
+  'recent.detail.threshold': 'Threshold',
+  'recent.detail.calls': '{calls} model calls',
+  'recent.detail.tokens': 'input {input} · cached {cached} · output {output}',
+  'recent.detail.scoreCache': 'score cache {hits} hits / {misses} misses',
+  'recent.detail.prefixCache': 'prefix cache {rate}',
+  'recent.detail.cost': 'est. {cost}',
 
   // Model summary
   'models.title': 'Model Summary',
@@ -422,6 +516,8 @@ export const en: I18nDict = {
   'field.autoVerifyTeamTasks.help': 'Automatically route progress and verify tasks when Agent Teams task status changes or completes.',
   'field.autoVerifyPlanMode.title': 'Plan Mode Pre-verification',
   'field.autoVerifyPlanMode.help': 'Automatically pre-verify plan feasibility and risks before exit_plan_mode presents it for user review.',
+  'field.captureDecisions.title': 'Keep decision snapshots',
+  'field.captureDecisions.help': 'Store the redacted prompt and raw answer of every judge call in this topic (verifier/decisions-v1.json): at most 12 model calls per invocation, 30k characters per record, the newest 40 records per topic. It answers "why did the judge decide that" in the dashboard and never affects a verdict.',
   'field.autoVerifySubagents.title': 'Verify Subagent Sessions',
   'field.autoVerifySubagents.help': 'Also run automatic routing and final acceptance on delegated child sessions. Off by default: child sessions are seeded with a real user message, so gating them spends extra judge budget and can repeatedly steer the subagent.',
 }
@@ -434,6 +530,7 @@ export const toolLabels: Record<'zh' | 'en', Record<string, string>> = {
     verifier_compare: '两项对比',
     verifier_select: '多项优选',
     verifier_track: '进度跟踪',
+    verifier_best_of_n: '生成选优',
     verifier_current_session: '会话验收',
   },
   en: {
@@ -441,6 +538,7 @@ export const toolLabels: Record<'zh' | 'en', Record<string, string>> = {
     verifier_compare: 'Pairwise Comparison',
     verifier_select: 'Candidate Selection',
     verifier_track: 'Progress Tracking',
+    verifier_best_of_n: 'Best-of-N Drafting',
     verifier_current_session: 'Session Acceptance',
   },
 }
@@ -578,10 +676,23 @@ export function sectionForSave(
   return section
 }
 
-/** An eight-candidate select: ring + pivot rounds (18 pairs) x three criteria, one round (the per-pair orientation removes the slot bias). */
-export const WORST_CASE_ROUTE_CALLS_PER_JUDGE = 54
-/** Final acceptance: three criteria x the default two repeats (one per A/B position). */
-export const WORST_CASE_FINAL_CALLS_PER_JUDGE = 6
+/**
+ * Criteria per comparison the worst-case estimate assumes.
+ *
+ * All five built-in presets have three, which is what keeps the historical constants below
+ * unchanged. A CUSTOM rubric file can have any number, and the engine reserves budget from the
+ * real count (\`rubric.criteria.length\`), so a custom file with more criteria needs a larger
+ * budget than a warning built on three would suggest — pass the count in when it is known.
+ */
+export const WORST_CASE_CRITERIA_PER_COMPARISON = 3
+/** An eight-candidate select: 18 pairs (ring + pivot rounds) x criteria, one round (the per-pair orientation removes the slot bias). */
+export function worstCaseRouteCallsPerJudge(criteria = WORST_CASE_CRITERIA_PER_COMPARISON): number { return 18 * Math.max(1, criteria) }
+/** Final acceptance: criteria x the default two repeats (one per A/B position). */
+export function worstCaseFinalCallsPerJudge(criteria = WORST_CASE_CRITERIA_PER_COMPARISON, repeats = 2): number { return Math.max(1, criteria) * Math.max(1, repeats) }
+/** @deprecated Kept at the three-criteria value; prefer {@link worstCaseRouteCallsPerJudge}. */
+export const WORST_CASE_ROUTE_CALLS_PER_JUDGE = worstCaseRouteCallsPerJudge()
+/** @deprecated Kept at the three-criteria value; prefer {@link worstCaseFinalCallsPerJudge}. */
+export const WORST_CASE_FINAL_CALLS_PER_JUDGE = worstCaseFinalCallsPerJudge()
 export const WORST_CASE_TASK_PER_JUDGE = WORST_CASE_ROUTE_CALLS_PER_JUDGE + WORST_CASE_FINAL_CALLS_PER_JUDGE
 export const WORST_CASE_SESSION_PER_JUDGE = 160
 
@@ -589,13 +700,13 @@ export function computeJudgeCount(extraJudgesCount: number): number {
   return 1 + Math.max(0, extraJudgesCount)
 }
 
-export function computeWorstCaseBudget(judgeCount: number): {
+export function computeWorstCaseBudget(judgeCount: number, criteria = WORST_CASE_CRITERIA_PER_COMPARISON): {
   worstCaseTask: number
   worstCaseSession: number
 } {
   const count = Math.max(1, judgeCount)
   return {
-    worstCaseTask: count * WORST_CASE_TASK_PER_JUDGE,
+    worstCaseTask: count * (worstCaseRouteCallsPerJudge(criteria) + worstCaseFinalCallsPerJudge(criteria)),
     worstCaseSession: count * WORST_CASE_SESSION_PER_JUDGE,
   }
 }
@@ -613,10 +724,11 @@ export function evaluateBudgetWarning(
   extraJudgesCount: number,
   autoMaxModelCallsPerTask: number,
   autoMaxModelCallsPerSession: number,
+  criteria = WORST_CASE_CRITERIA_PER_COMPARISON,
 ): BudgetWarningState | null {
   if (autoVerifyMode === 'manual') return null
   const judgeCount = computeJudgeCount(extraJudgesCount)
-  const { worstCaseTask, worstCaseSession } = computeWorstCaseBudget(judgeCount)
+  const { worstCaseTask, worstCaseSession } = computeWorstCaseBudget(judgeCount, criteria)
   const warnTask = autoMaxModelCallsPerTask < worstCaseTask
   const warnSession = autoMaxModelCallsPerSession < worstCaseSession
   if (!warnTask && !warnSession) return null

package/src/client.test.ts

@@ -8,6 +8,9 @@ import {
   WORST_CASE_FINAL_CALLS_PER_JUDGE,
   WORST_CASE_TASK_PER_JUDGE,
   WORST_CASE_SESSION_PER_JUDGE,
+  WORST_CASE_CRITERIA_PER_COMPARISON,
+  worstCaseRouteCallsPerJudge,
+  worstCaseFinalCallsPerJudge,
   computeJudgeCount,
   computeWorstCaseBudget,
   evaluateBudgetWarning,
@@ -52,6 +55,7 @@ describe('client i18n dictionaries', () => {
       'verifier_compare',
       'verifier_select',
       'verifier_track',
+      'verifier_best_of_n',
       'verifier_current_session',
     ]
 
@@ -475,6 +479,23 @@ describe('budget worst case and sanity warnings', () => {
     expect(computeWorstCaseBudget(3)).toEqual({ worstCaseTask: 180, worstCaseSession: 480 })
   })
 
+  it('scales the route estimate with the criteria count the engine actually reserves for', () => {
+    // All built-in presets have three criteria, which is why the historical constants still hold.
+    expect(WORST_CASE_CRITERIA_PER_COMPARISON).toBe(3)
+    expect(worstCaseRouteCallsPerJudge()).toBe(WORST_CASE_ROUTE_CALLS_PER_JUDGE)
+    expect(worstCaseFinalCallsPerJudge()).toBe(WORST_CASE_FINAL_CALLS_PER_JUDGE)
+    // A custom rubric file with more criteria costs proportionally more.
+    expect(worstCaseRouteCallsPerJudge(6)).toBe(108)
+    expect(worstCaseFinalCallsPerJudge(6)).toBe(12)
+    expect(computeWorstCaseBudget(1, 6)).toEqual({ worstCaseTask: 120, worstCaseSession: 160 })
+    // Degenerate counts are clamped, never zero or negative.
+    expect(worstCaseRouteCallsPerJudge(0)).toBe(18)
+    expect(worstCaseFinalCallsPerJudge(0)).toBe(2)
+    // The warning follows: the shipped 96/240 clears three criteria but not six.
+    expect(evaluateBudgetWarning('smart', 0, 96, 240, 3)).toBeNull()
+    expect(evaluateBudgetWarning('smart', 0, 96, 240, 6)).toMatchObject({ warnTask: true, warnSession: false, worstCaseTask: 120 })
+  })
+
   it('suppresses budget warnings in manual mode', () => {
     const warning = evaluateBudgetWarning('manual', 2, 10, 10)
     expect(warning).toBeNull()