dsh-llm-verifier 0.3.0 → 0.4.0

package/lib/types/index.js

@@ -1,19 +1,21 @@
 import { defineTool } from '@deepseek-ai/dsh-tools';
 import { createUserMessage } from '@deepseek-ai/dsh-llm';
 import { Config, installVerifierSettings, resolveConfig } from "./config.js";
-import { RequestLimiter, callVerifierText } from "./caller.js";
+import { RequestLimiter, addUsage, callVerifier, callVerifierText, generateCandidate } from "./caller.js";
 import { TopLogprobCapabilityCache, resolveCapabilityFile } from "./top-logprobs.js";
 import { ScoreCache, SingleFlight, resolveCacheFile, stableHash } from "./cache.js";
 import { VerifierEngine, normalizeCriteria } from "./engine.js";
 import { loadVerifierImages } from "./images.js";
 import { extractSession, sanitizeVerifierText, sessionEvents } from "./session.js";
+import { CriteriaResolver } from "./criteria.js";
 import { analyzeAutoTask, automaticFeedback, failedAcceptanceCriteria, isSubagentSession, sessionAccepted } from "./auto.js";
 import { AutoVerifierRouter, analyzeStructuredRoute, boundDecision, buildSemanticRoutePrompt, estimateRoutedCalls, parseSemanticRoute, routedRepeats, semanticDecision, semanticRouteHint } from "./router.js";
-import { DEFAULT_CRITERIA } from "./core.js";
+import { DEFAULT_GROUND_TRUTH_NOTE, EMPTY_WORK_BASELINE, buildGenerationPrompt, buildPairwisePrompt, extractScore } from "./core.js";
 import { buildPlanPreReviewPrompt, parseVerdictLetter, planFromArguments } from "./plan-gate.js";
 import { inspectTeamTasks, buildTeamTaskVerificationPrompt } from "./team-gate.js";
 import { StatisticsStore, emptyRunStats, errorDetails, mergeStatisticsOverviews, parseStatisticsQuery, resolveStatisticsFile, summarizeVerdict } from "./statistics.js";
 import { resolveTopicDataDir } from "./topic-storage.js";
+import { DecisionStore, boundDecisionCalls, resolveDecisionsFile } from "./decisions.js";
 export const name = 'llm-verifier';
 export const inject = ['tools', 'agents', 'attachments', 'llm', 'connection', 'sessionPersistence'];
 export { Config };
@@ -22,6 +24,7 @@ export * from "./engine.js";
 export * from "./cache.js";
 export * from "./statistics.js";
 export * from "./topic-storage.js";
+export * from "./decisions.js";
 export * from "./auto.js";
 export * from "./router.js";
 export * from "./plan-gate.js";
@@ -74,11 +77,30 @@ const EXPLICIT_MAX_TOTAL_CHARS = 240_000;
 const EXPLICIT_MIN_TOTAL_CHARS = 120_000;
 const MAX_EXPLICIT_CANDIDATES = 16;
 const MAX_EXPLICIT_PLANNED_CALLS = 500;
+/**
+ * Best-of-N bounds. The ceiling is 4 because cost grows ~linearly in N while the marginal
+ * value of the 5th draft does not, and the tool is meant for a final deliverable rather than
+ * routine work. The floor is 2 because "choose the best of one" is not a choice.
+ */
+const MIN_BEST_OF_N = 2;
+const MAX_BEST_OF_N = 4;
+const DEFAULT_BEST_OF_N = 3;
 function explicitItemChars(selected) { return Math.max(selected.autoRouteMaxItemChars, EXPLICIT_MIN_ITEM_CHARS); }
 function explicitBudget(selected) { return Math.min(Math.max(selected.autoRouteMaxInputChars * 2, EXPLICIT_MIN_TOTAL_CHARS), EXPLICIT_MAX_TOTAL_CHARS); }
 function explicitCandidateLimit(selected) { return Math.max(selected.autoRouteMaxCandidates, MAX_EXPLICIT_CANDIDATES); }
 /** Ring + pivot-round comparisons for an explicitly requested selection. */
 function plannedComparisons(count) { return count <= 2 ? 1 : count + Math.max(0, (count - 2) * 2 + 1 - 3); }
+/**
+ * Worst-case tournament pairs for a selection of `count` candidates.
+ *
+ * `plannedComparisons` is an ESTIMATE (it assumes the ring already covers three of the pivot-round
+ * pairs), and best-of-N spends N generations first, so its guard has to fail closed on the true
+ * bound instead: ring (N) plus every pivot pair (non-pivots x pivots, plus pivot-vs-pivot), since
+ * de-duplicating against the ring can only ever remove pairs.
+ * @param count - candidate count.
+ * @returns The largest number of pairs the tournament can judge.
+ */
+function selectComparisonsUpperBound(count) { return count <= 2 ? 1 : count + (count - 2) * 2 + 1; }
 function numberField(value, fallback) { return typeof value === 'number' && Number.isFinite(value) ? value : fallback; }
 function statsFrom(value) { if (typeof value !== 'object' || value === null || !('stats' in value))
     return emptyRunStats(); const source = value.stats; if (typeof source !== 'object' || source === null)
@@ -94,6 +116,18 @@ const judgeScoreSchema = { type: 'object', additionalProperties: false, properti
 const judgesSchema = { type: 'array', items: judgeScoreSchema, required: true };
 function rpcSuccess(value) { return { ok: true, value }; }
 function rpcFailure(message) { return { ok: false, error: { code: 'bad-request', message, details: { issues: [] } } }; }
+/**
+ * Small but real pair for the settings-page probe.
+ *
+ * The probe asks each judge for an actual A–T verdict instead of a "ping": that is what proves
+ * the response PARSES, which is the failure a reachability check misses. A pair where one side is
+ * obviously better keeps the exercise honest without making the expected score a secret.
+ */
+const PROBE_TASK = 'Fix the failing parser test and prove it passes.';
+const PROBE_A = 'Ran the test: 1 failed. Patched the separator handling. Ran it again: 1 passed, no other failures.';
+const PROBE_B = 'The test should pass now.';
+/** Bound the probe: a diagnostics button must not sit on the configured 5-minute timeout plus retries. */
+const PROBE_TIMEOUT_MS = 30000;
 export function apply(ctx, config = {}) {
     const services = ctx;
     const entry = resolveConfig(config);
@@ -109,7 +143,7 @@ export function apply(ctx, config = {}) {
         if (existing?.dataDir === dataDir)
             return existing;
         const cacheFile = resolveCacheFile(dataDir);
-        const created = { dataDir, cache: new ScoreCache(cacheFile, selected.cacheMaxEntries), capabilities: new TopLogprobCapabilityCache(resolveCapabilityFile(dataDir)), flights: new SingleFlight(), statistics: new StatisticsStore(resolveStatisticsFile(cacheFile)) };
+        const created = { dataDir, cache: new ScoreCache(cacheFile, selected.cacheMaxEntries), capabilities: new TopLogprobCapabilityCache(resolveCapabilityFile(dataDir)), flights: new SingleFlight(), statistics: new StatisticsStore(resolveStatisticsFile(cacheFile)), decisions: new DecisionStore(resolveDecisionsFile(cacheFile)) };
         topics.set(id, created);
         return created;
     };
@@ -119,9 +153,40 @@ export function apply(ctx, config = {}) {
             throw new Error('llm-verifier: verifier tools require an agent-owned topic so their data can follow topic deletion');
         return selected;
     };
-    const engine = async (agent) => {
+    // The configured rubric (preset, or a markdown file) is the one every automatic decision and
+    // every explicit tool without its own `criteria` argument scores with. Resolved through a
+    // cache so an unchanged file costs one read per boundary, and a broken file degrades to the
+    // coding preset instead of disabling the gate.
+    const criteriaResolver = new CriteriaResolver();
+    const configuredCriteria = () => criteriaResolver.resolve(current().criteriaPreset, current().criteriaFile);
+    /**
+     * Every known topic header, newest first.
+     *
+     * The statistics and decision dashboards merge all topics; anything that needs exactly ONE
+     * topic (the judge probe's per-topic capability memory) takes the newest, because that is where
+     * the operator's current work lives. Legacy backends handed back the header directly instead of
+     * wrapping it in \`{ header }`, so both shapes are accepted.
+     * @returns Session headers, newest first; entries without an id are dropped.
+     */
+    const sessionHeaders = async () => {
+        const items = await services.sessionPersistence.list();
+        return items
+            .map(item => item && typeof item === 'object' && 'header' in item ? item.header : item)
+            .filter(header => header !== undefined && header.id !== undefined)
+            .sort((a, b) => Number(b.createdAt ?? 0) - Number(a.createdAt ?? 0));
+    };
+    /**
+     * Build the judge ensemble for one topic.
+     *
+     * Takes a session header rather than an Agent because the judge probe is started from the global
+     * statistics dashboard, where there is no current initiator. The diagnostic still needs a topic
+     * for the per-topic capability memory, so it attaches to the most recent one (see handleProbe).
+     * @param header - session header owning the topic whose sidecars the judges use.
+     * @returns The engine plus the resolved configuration it was built from.
+     */
+    const engineForHeader = async (header) => {
         const selected = current();
-        const topicEntry = topic(agent.session.header);
+        const topicEntry = topic(header);
         // One client per configured judge: the judge identity is part of the scoring
         // cache key, so each judge caches and de-duplicates independently. `judges[0]`
         // is the primary, which keeps a single-judge configuration on exactly the old path.
@@ -132,6 +197,7 @@ export function apply(ctx, config = {}) {
         }
         return { verifier: new VerifierEngine(clients, selected.maxConcurrency, topicEntry.cache, { input: selected.estimatedInputUsdPerMillion, output: selected.estimatedOutputUsdPerMillion }, topicEntry.flights), selected };
     };
+    const engine = async (agent) => engineForHeader(agent.session.header);
     const images = (values, signal) => loadVerifierImages(values, signal);
     const route = (selected) => ({ provider: selected.provider, model: selected.model });
     const requireEnabled = () => { if (!current().enabled)
@@ -146,12 +212,24 @@ export function apply(ctx, config = {}) {
     const record = async (toolName, agent, operation, phase = 'explicit') => {
         const startedAt = Date.now();
         let selected = current();
-        const statistics = topic(agent.session.header).statistics;
+        const topicEntry = topic(agent.session.header);
+        const statistics = topicEntry.statistics;
+        // Every real model call is captured here, so the dashboard can answer "why did the
+        // judge say that" from the exact prompt and raw answer instead of a replay. Calls
+        // are bounded twice (per call, per record) before anything reaches the disk.
+        const capture = current().captureDecisions;
+        const calls = [];
+        const trace = capture ? call => { calls.push(call); } : undefined;
         try {
-            const completed = await operation();
+            const completed = await operation(trace);
             selected = completed.selected;
             const value = { ...completed.result, ...route(selected) };
             await statistics.record({ toolName, sessionId: String(agent.id), startedAt, success: true, provider: selected.provider, model: selected.model, stats: statsFrom(value), verdict: verdictFrom(toolName, value, phase) }).catch(() => { });
+            // Order by label before storing: the engine reports calls as they complete, so a concurrent
+            // fan-out reports them in network order. Sorting makes "which calls are in the snapshot"
+            // reproducible even when the record has to bound its text.
+            if (calls.length > 0)
+                await topicEntry.decisions.record({ toolName, phase, startedAt, provider: selected.provider, model: selected.model, calls: boundDecisionCalls([...calls].sort((a, b) => (a.label < b.label ? -1 : a.label > b.label ? 1 : 0))) }).catch(() => { });
             return value;
         }
         catch (error) {
@@ -187,35 +265,160 @@ export function apply(ctx, config = {}) {
             verdict: { phase, outcome },
         }).catch(() => { });
     };
-    const verifySession = async (agent, options, signal, phase = 'explicit') => record('verifier_current_session', agent, async () => {
+    const verifySession = async (agent, options, signal, phase = 'explicit', rubricOverride) => record('verifier_current_session', agent, async (trace) => {
         const extracted = await extractSession(agent, async (ref) => { const stored = await services.attachments.readImage(ref, signal); return { data: stored.data, mediaType: stored.ref.mediaType }; }, { fromSeq: options.fromSeq, toSeq: options.toSeq, includeAssistantText: options.includeAssistantText, redactPatterns: options.redactPatterns, maxChars: options.maxChars });
         if (!extracted.problem.trim())
             throw new Error('llm-verifier: no direct user task found in the selected session range — widen from_seq so the task statement is included');
         const { verifier, selected } = await engine(agent);
-        const compared = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(options.repeats, 2, 'repeats'), images: extracted.images }, signal);
+        const rubric = rubricOverride ?? await configuredCriteria();
+        const compared = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: EMPTY_WORK_BASELINE, criteria: rubric.criteria, ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats: positive(options.repeats, 2, 'repeats'), images: extracted.images, ...(trace ? { trace } : {}) }, signal);
         const result = { sessionId: extracted.sessionId, problem: extracted.problem, score: compared.scoreA, baselineScore: compared.scoreB, winner: compared.winner, criteria: compared.criteria.map(row => ({ id: row.id, name: row.name, score: row.scoreA })), fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: compared.calls, stats: compared.stats, judges: compared.judges, agreement: compared.agreement };
         return { result, selected };
     }, phase);
-    const compareCandidates = async (agent, problem, candidateA, candidateB, repeats, signal, routedImages = [], phase = 'explicit') => record('verifier_compare', agent, async () => {
+    const compareCandidates = async (agent, problem, candidateA, candidateB, repeats, signal, rubric, routedImages = [], phase = 'explicit') => record('verifier_compare', agent, async (trace) => {
         const { verifier, selected } = await engine(agent);
-        return { result: await verifier.compare({ problem, candidateA, candidateB, repeats, images: routedImages }, signal), selected };
+        return { result: await verifier.compare({ problem, candidateA, candidateB, criteria: rubric.criteria, ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats, images: routedImages, ...(trace ? { trace } : {}) }, signal), selected };
     }, phase);
-    const selectCandidates = async (agent, problem, candidates, repeats, signal, routedImages = [], phase = 'explicit') => record('verifier_select', agent, async () => {
+    const selectCandidates = async (agent, problem, candidates, repeats, signal, rubric, routedImages = [], phase = 'explicit') => record('verifier_select', agent, async (trace) => {
         const { verifier, selected } = await engine(agent);
-        return { result: await verifier.select({ problem, candidates, repeats, pivots: Math.min(2, candidates.length), seed: 0, images: routedImages }, signal), selected };
+        return { result: await verifier.select({ problem, candidates, criteria: rubric.criteria, ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats, pivots: Math.min(2, candidates.length), seed: 0, images: routedImages, ...(trace ? { trace } : {}) }, signal), selected };
     }, phase);
-    const trackProgress = async (agent, problem, steps, checkpoints, repeats, signal, routedImages = [], phase = 'explicit') => record('verifier_track', agent, async () => {
+    const trackProgress = async (agent, problem, steps, checkpoints, repeats, signal, routedImages = [], phase = 'explicit') => record('verifier_track', agent, async (trace) => {
         const { verifier, selected } = await engine(agent);
-        return { result: await verifier.track(problem, steps, checkpoints, repeats, signal, routedImages), selected };
+        return { result: await verifier.track(problem, steps, checkpoints, repeats, signal, routedImages, trace), selected };
     }, phase);
-    const classifyRoute = async (agent, prompt, signal, phase) => record('verifier_route_classify', agent, async () => {
+    /**
+     * Explicit best-of-N: draft N candidates with the session model, rank them with the
+     * configured judges, then re-measure the winner against the gate's own baseline.
+     *
+     * The tournament's `scores` are relative preference shares (wins/counts) and cannot be
+     * compared with `autoVerifyThreshold`; only the extra winner-vs-baseline comparison produces
+     * an absolute score in the gate's units. That comparison is why this tool exists instead of
+     * "generate candidates yourself and call verifier_select".
+     *
+     * Fail-closed by construction: fewer than {@link MIN_BEST_OF_N} surviving drafts is an error
+     * that names every failure, never a silent "best" picked out of a single survivor.
+     * @param agent - Agent owning the topic, and the session whose model writes the drafts.
+     * @param task - the request the drafts answer.
+     * @param count - how many drafts to request ({@link MIN_BEST_OF_N}..{@link MAX_BEST_OF_N}).
+     * @param repeats - caller's repeat count, or undefined for the default; used by both the
+     *   tournament and the baseline comparison.
+     * @param signal - tool-call abort signal.
+     * @param criteriaInput - caller-supplied criteria, or undefined for the configured rubric.
+     */
+    const bestOfN = async (agent, task, count, repeats, signal, criteriaInput) => record('verifier_best_of_n', agent, async (trace) => {
+        const { verifier, selected } = await engine(agent);
+        if (!Number.isSafeInteger(count) || count < MIN_BEST_OF_N || count > MAX_BEST_OF_N)
+            throw new Error('llm-verifier: n must be an integer between ' + MIN_BEST_OF_N + ' and ' + MAX_BEST_OF_N);
+        const rounds = capped(repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats');
+        const rubric = criteriaInput === undefined ? await configuredCriteria() : { criteria: normalizeCriteria(criteriaInput), source: 'explicit' };
+        // Bound the plan before spending the generation: with repeats and an ensemble this tool can
+        // reach the shared explicit ceiling, and an unbounded custom rubric would blow past it.
+        const planned = (selectComparisonsUpperBound(count) + 1) * rubric.criteria.length * rounds * selected.judges.length;
+        if (planned > MAX_EXPLICIT_PLANNED_CALLS)
+            throw new Error('llm-verifier: best-of-n with n=' + count + ' would issue about ' + planned + ' judge calls; reduce n, repeats, criteria or judges');
+        // The drafting model is the session's OWN model, read from the logged request header: no new
+        // configuration, and the drafts come from exactly the model the task is being done with.
+        const call = agent.session.requestHeader()?.config;
+        if (call === undefined)
+            throw new Error('llm-verifier: this session has no logged request header yet, so best-of-n cannot tell which model should write the drafts — generate candidates with parallel subagents and rank them with verifier_select instead');
+        const target = { provider: call.provider, model: call.model, ...(call.reasoningEffort === undefined ? {} : { reasoningEffort: String(call.reasoningEffort) }) };
+        const problem = explicitEvidence([task], explicitItemChars(selected), explicitBudget(selected), 'task')[0];
+        // N independent drafts, in parallel. A failure is captured per draft so the error below can
+        // name it; a draft that fails is never substituted with another one.
+        const attempts = await Promise.all(Array.from({ length: count }, async (_unused, index) => {
+            const prompt = buildGenerationPrompt(problem, index, count);
+            try {
+                const completion = await generateCandidate(verifier.client, target, prompt, signal);
+                trace?.({ label: 'draft ' + (index + 1), channel: completion.scoringMode, prompt, output: completion.text });
+                return { ok: true, text: completion.text, usage: completion.usage, truncated: completion.truncated };
+            }
+            catch (error) {
+                return { ok: false, error: error instanceof Error ? error.message : String(error) };
+            }
+        }));
+        const survivors = [];
+        const failures = [];
+        attempts.forEach((attempt, index) => {
+            if (attempt.ok)
+                survivors.push({ attempt: index + 1, text: attempt.text, usage: attempt.usage, truncated: attempt.truncated });
+            else
+                failures.push('draft ' + (index + 1) + ': ' + attempt.error);
+        });
+        if (survivors.length < MIN_BEST_OF_N)
+            throw new Error('llm-verifier: best-of-n produced ' + survivors.length + ' usable draft(s) out of ' + count + '; at least ' + MIN_BEST_OF_N + ' are required to choose between them' + (failures.length === 0 ? '' : ' — ' + failures.join('; ')));
+        const ranked = await verifier.select({ problem, candidates: survivors.map(survivor => survivor.text), criteria: rubric.criteria, ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats: rounds, pivots: Math.min(2, survivors.length), seed: 0, ...(trace ? { trace } : {}) }, signal);
+        // The absolute, gate-comparable measurement. Same fixed baseline, same threshold rule as the
+        // automatic final acceptance, so "passes" here means the same thing it means at the gate.
+        // The label prefix keeps the snapshot readable: this compare reuses the same criteria as the
+        // tournament above, so without it two different comparisons share one set of labels.
+        const compared = await verifier.compare({ problem, candidateA: ranked.best, candidateB: EMPTY_WORK_BASELINE, criteria: rubric.criteria, traceLabelPrefix: 'baseline: ', ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats: rounds, ...(trace ? { trace } : {}) }, signal);
+        const criteria = compared.criteria.map(row => ({ id: row.id, name: row.name, score: row.scoreA }));
+        const threshold = selected.autoVerifyThreshold;
+        const passesThreshold = sessionAccepted({ score: compared.scoreA, winner: compared.winner, criteria }, threshold);
+        // One cost line for the whole invocation, generation included: the drafts are real tokens the
+        // operator paid for even though no judge scored them. Their price is estimated with the
+        // configured verifier table (the only one the plugin has), which the README states.
+        const stats = { ...ranked.stats };
+        addUsage(stats, compared.stats);
+        stats.cacheHits += compared.stats.cacheHits;
+        stats.cacheMisses += compared.stats.cacheMisses;
+        stats.topLogprobScores += compared.stats.topLogprobScores;
+        stats.explicitTagScores += compared.stats.explicitTagScores;
+        for (const survivor of survivors)
+            addUsage(stats, survivor.usage);
+        stats.estimatedCostUsd = ((stats.inputTokens + stats.cachedInputTokens) * selected.estimatedInputUsdPerMillion + stats.outputTokens * selected.estimatedOutputUsdPerMillion) / 1_000_000;
+        const result = {
+            best: ranked.best,
+            index: ranked.index,
+            /** 1-based draft numbers of the survivors, in `scores`/`ranking` order. */
+            sources: survivors.map(survivor => survivor.attempt),
+            scores: ranked.scores,
+            ranking: ranked.ranking,
+            comparisons: ranked.comparisons,
+            pivots: ranked.pivots,
+            generated: survivors.length,
+            failed: failures.length,
+            failures,
+            /** 1-based draft numbers still truncated after their escalated retry; empty is the normal case. */
+            truncated: survivors.filter(survivor => survivor.truncated).map(survivor => survivor.attempt),
+            score: compared.scoreA,
+            baselineScore: compared.scoreB,
+            winner: compared.winner,
+            criteria: compared.criteria,
+            threshold,
+            passesThreshold,
+            failedCriteria: failedAcceptanceCriteria(criteria, threshold).map(criterion => criterion.id),
+            calls: stats.calls,
+            stats,
+            generatorProvider: target.provider,
+            generatorModel: target.model,
+            // Every judge call of this invocation, not just the tournament's: the winner-vs-baseline
+            // comparison runs on the same clients, and the top-level call count already includes it.
+            // `ok` therefore covers both comparisons while `scores`/`ranking` stay tournament-shaped.
+            judges: ranked.judges.map((judge, index) => {
+                const baseline = compared.judges[index];
+                if (baseline === undefined)
+                    return judge;
+                return {
+                    ...judge,
+                    ok: judge.ok && baseline.ok,
+                    calls: judge.calls + baseline.calls,
+                    ...(judge.error === undefined && baseline.error !== undefined ? { error: baseline.error } : {}),
+                };
+            }),
+        };
+        return { result, selected };
+    });
+    const classifyRoute = async (agent, prompt, signal, phase) => record('verifier_route_classify', agent, async (trace) => {
         const { verifier, selected } = await engine(agent);
         const completion = await callVerifierText(verifier.client, prompt, signal);
+        trace?.({ label: 'route classify', channel: completion.scoringMode, prompt, output: completion.text });
         const stats = { ...completion.usage, cacheHits: 0, cacheMisses: 0, estimatedCostUsd: ((completion.usage.inputTokens + completion.usage.cachedInputTokens) * selected.estimatedInputUsdPerMillion + completion.usage.outputTokens * selected.estimatedOutputUsdPerMillion) / 1_000_000, topLogprobScores: 0, explicitTagScores: 1 };
         return { result: { text: completion.text, stats }, selected };
     }, phase);
     const extractTask = async (agent, fromSeq, toSeq, maxChars, signal) => extractSession(agent, async (ref) => { const stored = await services.attachments.readImage(ref, signal); return { data: stored.data, mediaType: stored.ref.mediaType }; }, { fromSeq, toSeq, includeAssistantText: true, maxChars });
-    const routePolicy = (selected) => ({ mode: selected.autoVerifyMode, minConfidence: selected.autoRouteMinConfidence, maxCandidates: selected.autoRouteMaxCandidates, maxPerTask: selected.autoRouteMaxPerTask + selected.autoVerifyMaxPerTask, maxPerSession: selected.autoRouteMaxPerSession + selected.autoVerifyMaxPerSession, maxModelCallsPerTask: selected.autoMaxModelCallsPerTask, maxModelCallsPerSession: selected.autoMaxModelCallsPerSession, maxInputChars: selected.autoRouteMaxInputChars, maxItemChars: selected.autoRouteMaxItemChars });
+    const routePolicy = (selected) => ({ mode: selected.autoVerifyMode, minConfidence: selected.autoRouteMinConfidence, maxCandidates: selected.autoRouteMaxCandidates, maxRoutePerTask: selected.autoRouteMaxPerTask, maxRoutePerSession: selected.autoRouteMaxPerSession, maxFinalPerTask: selected.autoVerifyMaxPerTask, maxFinalPerSession: selected.autoVerifyMaxPerSession, maxModelCallsPerTask: selected.autoMaxModelCallsPerTask, maxModelCallsPerSession: selected.autoMaxModelCallsPerSession, maxInputChars: selected.autoRouteMaxInputChars, maxItemChars: selected.autoRouteMaxItemChars });
     const routeFeedback = (decision, detail) => createUserMessage({ content: [{ type: 'text', text: '[Automatic verifier routing: ' + decision.kind + ']\n' + detail + '\nUse this independent result to continue the actual task. Do not merely restate the ranking or progress score; implement, correct, and verify the required work.' }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier', form: 'notice', summary: 'Automatic verifier routed ' + decision.kind } });
     const handleStatisticsQuery = async (payload) => {
         // A malformed range used to be swallowed by the Promise.allSettled fan-out below and
@@ -225,10 +428,7 @@ export function apply(ctx, config = {}) {
             return rpcFailure(parsed.message);
         const query = parsed.query;
         try {
-            const items = await services.sessionPersistence.list();
-            const headers = items
-                .map(item => item && typeof item === 'object' && 'header' in item ? item.header : item)
-                .filter(header => header !== undefined && (query.sessionId === undefined || String(header.id) === query.sessionId));
+            const headers = (await sessionHeaders()).filter(header => query.sessionId === undefined || String(header.id) === query.sessionId);
             const settled = await Promise.allSettled(headers.map(async (header) => topic(header).statistics.overview(query)));
             const overviews = [];
             for (const result of settled) {
@@ -244,6 +444,93 @@ export function apply(ctx, config = {}) {
             return rpcFailure(error instanceof Error ? error.message : String(error));
         }
     };
+    /**
+     * One decision snapshot by invocation id, across every topic.
+     *
+     * The dashboard shows merged topics, so the lookup fans out the same way the
+     * statistics query does instead of guessing which topic owns the record.
+     * @param id - invocation id the snapshot was stored under.
+     */
+    const handleDecisionQuery = async (id) => {
+        if (typeof id !== 'string' || id.length === 0)
+            return rpcFailure('decision id must be a non-empty string');
+        try {
+            for (const header of await sessionHeaders()) {
+                const found = await topic(header).decisions.find(id).catch(() => undefined);
+                if (found)
+                    return rpcSuccess({ decision: found });
+            }
+            return rpcFailure('decision snapshot not found: it was pruned, never captured, or belongs to a deleted topic');
+        }
+        catch (error) {
+            return rpcFailure(error instanceof Error ? error.message : String(error));
+        }
+    };
+    /** Whether a payload asks for a decision snapshot instead of a statistics overview. */
+    const isDecisionQuery = (payload) => typeof payload === 'object' && payload !== null && payload.kind === 'decision';
+    /**
+     * Judge diagnostics: one real, bounded judge call per configured judge plus the resolved rubric.
+     *
+     * Answers the two questions a user actually has after configuring the plugin: "which scoring
+     * channel is this judge using" (a silent explicit-tag downgrade is invisible otherwise) and
+     * "is the rubric I configured the one in effect" (a broken custom file degrades quietly).
+     * Deliberately NOT recorded in the statistics: it is a diagnostic, not a verification.
+     * @returns Per-judge probe results and the rubric in effect.
+     */
+    const handleProbe = async () => {
+        try {
+            // The dashboard is a GLOBAL page: there is usually no current initiator, so requiring one
+            // made the probe unusable exactly where its button lives. Fall back to the newest topic,
+            // which is also where the re-probed channel verdict should be remembered.
+            const initiator = ctx.agents.currentInitiator();
+            const header = initiator?.session.header ?? (await sessionHeaders())[0];
+            if (header === undefined)
+                return rpcFailure('llm-verifier: the judge probe needs one session to attach its capability memory to, and no session exists yet — start a session, then probe again');
+            const { verifier } = await engineForHeader(header);
+            const rubric = await configuredCriteria();
+            const criterion = rubric.criteria[0];
+            if (criterion === undefined)
+                return rpcFailure('llm-verifier: the configured rubric has no criteria');
+            const prompt = buildPairwisePrompt(PROBE_TASK, PROBE_A, PROBE_B, criterion, rubric.groundTruthNote ?? DEFAULT_GROUND_TRUTH_NOTE);
+            const judges = [];
+            for (const client of verifier.clients) {
+                const label = client.label ?? client.provider + '/' + client.model;
+                const startedAt = Date.now();
+                // Forget the cached channel verdict for this judge before probing. A mark written up to 24h
+                // ago — possibly before the provider was (re)configured, and certainly before a host
+                // restart — would make the probe replay a stale answer instead of checking, which is the one
+                // thing a diagnostic must not do. It also means a provider that CAN return logprobs starts
+                // using that better channel for subsequent real verifications.
+                client.topLogprobCapabilities.forget(client.provider, client.model);
+                try {
+                    const completion = await callVerifier({ ...client, timeoutMs: Math.min(client.timeoutMs, PROBE_TIMEOUT_MS), maxRetries: 0 }, prompt);
+                    judges.push({
+                        label,
+                        provider: client.provider,
+                        model: client.model,
+                        ok: true,
+                        channelProbed: true,
+                        channel: completion.scoringMode,
+                        scoreA: extractScore(completion, '<score_A>'),
+                        scoreB: extractScore(completion, '<score_B>'),
+                        latencyMs: Date.now() - startedAt,
+                        ...completion.usage,
+                    });
+                }
+                catch (error) {
+                    judges.push({ label, provider: client.provider, model: client.model, ok: false, latencyMs: Date.now() - startedAt, error: error instanceof Error ? error.message : String(error) });
+                }
+            }
+            return rpcSuccess({
+                judges,
+                channelProbed: true,
+                rubric: { source: rubric.source, count: rubric.criteria.length, ...(rubric.file ? { file: rubric.file } : {}), ...(rubric.error ? { error: rubric.error } : {}) },
+            });
+        }
+        catch (error) {
+            return rpcFailure(error instanceof Error ? error.message : String(error));
+        }
+    };
     // 1. 优先注册到官方 /api 共享通道 (connection.fetch.register)
     const anyConn = services.connection;
     if (anyConn && anyConn.fetch && typeof anyConn.fetch.register === 'function') {
@@ -263,7 +550,8 @@ export function apply(ctx, config = {}) {
                     const isRpcEnvelope = typeof body === 'object' && body !== null && body.type === 'client-request' && typeof body.rpcId === 'string';
                     const rpcId = isRpcEnvelope ? body.rpcId : 'direct';
                     const payload = isRpcEnvelope ? body.payload : body;
-                    const outcome = await handleStatisticsQuery(payload);
+                    const isProbe = typeof payload === 'object' && payload !== null && payload.kind === 'probe';
+                    const outcome = isDecisionQuery(payload) ? await handleDecisionQuery(payload.id) : isProbe ? await handleProbe() : await handleStatisticsQuery(payload);
                     if (isRpcEnvelope) {
                         return Response.json({
                             type: 'server-response',
@@ -286,6 +574,10 @@ export function apply(ctx, config = {}) {
     if (typeof legacyRpc.rpc?.handle === 'function') {
         try {
             legacyRpc.rpc.handle('/llm-verifier', async (endpoint, payload) => {
+                if (endpoint === 'decision')
+                    return handleDecisionQuery(payload?.id);
+                if (endpoint === 'probe')
+                    return handleProbe();
                 if (endpoint !== 'statistics')
                     return rpcFailure('unknown llm-verifier endpoint');
                 return handleStatisticsQuery(payload);
@@ -414,13 +706,18 @@ export function apply(ctx, config = {}) {
                 }
             }
         }
-        const structured = analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars, selected.autoRouteMaxInputChars);
-        let decision = boundDecision(structured, policy);
+        // A previous track route already cleared the completion threshold (preferFinal): the
+        // final gate is mandatory and already armed, so buying another route first would only
+        // delay it with evidence whose steered text would be identical. The preference is
+        // consumed by the final reservation, so a failed acceptance releases routing again.
+        const finalPreferred = autoRouter.finalPreferred(agent);
+        const structured = finalPreferred ? undefined : analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars, selected.autoRouteMaxInputChars);
+        let decision = finalPreferred ? undefined : boundDecision(structured, policy);
         if (structured !== undefined && decision === undefined) {
             ctx.logger.warn('llm-verifier automatic ' + structured.kind + ' route dropped: its evidence exceeds the per-item/total routing caps (' + selected.autoRouteMaxItemChars + '/' + selected.autoRouteMaxInputChars + ' characters)');
             await recordSkippedRoute(agent, structured.kind, 'structured', 'dropped-over-budget');
         }
-        if (decision === undefined && selected.autoRouteSemantic && (selected.autoVerifyMode === 'strict' || semanticRouteHint(snapshot))) {
+        if (!finalPreferred && decision === undefined && selected.autoRouteSemantic && (selected.autoVerifyMode === 'strict' || semanticRouteHint(snapshot))) {
             const fingerprint = stableHash({ phase: 'semantic', from: evidence.taskStartSeq, to: admittedLastSeq, model: selected.provider + '/' + selected.model });
             const reservation = autoRouter.reserve(agent, 'semantic', fingerprint, 1, policy);
             if (reservation) {
@@ -462,8 +759,9 @@ export function apply(ctx, config = {}) {
         if (decision) {
             // Every judge scores every match, and compare/select round the repeat count up to an
             // even number so the A/B slots get swapped, so the reservation has to cover both.
-            const repeats = routedRepeats(decision, selected.autoVerifyRepeats);
-            const expectedCalls = estimateRoutedCalls(decision, repeats, DEFAULT_CRITERIA.length) * selected.judges.length;
+            const repeats = routedRepeats(decision, selected.autoVerifyRepeats, selected.autoTrackRepeats);
+            const rubric = await configuredCriteria();
+            const expectedCalls = estimateRoutedCalls(decision, repeats, rubric.criteria.length) * selected.judges.length;
             const reservation = autoRouter.reserve(agent, decision.kind, decision.fingerprint, expectedCalls, policy);
             if (reservation === undefined) {
                 // A refused reservation used to drop the whole routed decision silently.
@@ -483,7 +781,7 @@ export function apply(ctx, config = {}) {
                 try {
                     const extracted = await extractTask(agent, evidence.taskStartSeq, admittedLastSeq, selected.autoVerifyMaxChars, signal);
                     if (decision.kind === 'compare') {
-                        const result = await compareCandidates(agent, extracted.problem, decision.candidates[0].content, decision.candidates[1].content, repeats, signal, extracted.images, 'compare');
+                        const result = await compareCandidates(agent, extracted.problem, decision.candidates[0].content, decision.candidates[1].content, repeats, signal, rubric, extracted.images, 'compare');
                         if (!stillCurrent()) {
                             autoRouter.fail(agent, reservation, false);
                             return;
@@ -495,7 +793,7 @@ export function apply(ctx, config = {}) {
                         return;
                     }
                     if (decision.kind === 'select') {
-                        const result = await selectCandidates(agent, extracted.problem, decision.candidates.map(candidate => candidate.content), repeats, signal, extracted.images, 'select');
+                        const result = await selectCandidates(agent, extracted.problem, decision.candidates.map(candidate => candidate.content), repeats, signal, rubric, extracted.images, 'select');
                         if (!stillCurrent()) {
                             autoRouter.fail(agent, reservation, false);
                             return;
@@ -519,7 +817,12 @@ export function apply(ctx, config = {}) {
                     // yet", so "any checkpoint below threshold" made the continue branch
                     // unconditional (5/5 live routes) and the threshold meaningless.
                     const latest = result.scores.length > 0 ? result.scores[result.scores.length - 1] : 0;
-                    const continuation = latest < selected.autoTrackCompletionThreshold ? '\nContinue the unfinished work.' : '\nPrepare final delivery evidence; final session verification is mandatory.';
+                    const completed = latest >= selected.autoTrackCompletionThreshold;
+                    // The work already reads as done: arm the next boundary to run the mandatory final
+                    // gate instead of paying for another progress route that would say the same thing.
+                    if (completed)
+                        autoRouter.preferFinal(agent);
+                    const continuation = completed ? '\nPrepare final delivery evidence; final session verification is mandatory.' : '\nContinue the unfinished work.';
                     agent.steer(routeFeedback(decision, detail + continuation));
                     return;
                 }
@@ -549,7 +852,8 @@ export function apply(ctx, config = {}) {
         const finalFingerprint = stableHash({ phase: 'final', from: finalFromSeq, to: admittedLastSeq });
         // Exact planning instead of a flat 4-per-repeat guess: one comparison per
         // criterion per repeat, repeated for every judge in the ensemble.
-        const finalReservation = autoRouter.reserve(agent, 'final', finalFingerprint, Math.max(1, DEFAULT_CRITERIA.length * selected.autoVerifyFinalRepeats) * selected.judges.length, policy);
+        const finalRubric = await configuredCriteria();
+        const finalReservation = autoRouter.reserve(agent, 'final', finalFingerprint, Math.max(1, finalRubric.criteria.length * selected.autoVerifyFinalRepeats) * selected.judges.length, policy);
         if (!finalReservation) {
             // Same one-shot rule: a spent budget never grants another reservation, so
             // an unconditional steer here would livelock the turn.
@@ -562,7 +866,7 @@ export function apply(ctx, config = {}) {
             return;
         }
         try {
-            const result = await verifySession(agent, { fromSeq: finalFromSeq, toSeq: admittedLastSeq, includeAssistantText: true, maxChars: selected.autoVerifyMaxChars, repeats: selected.autoVerifyFinalRepeats }, signal, 'final');
+            const result = await verifySession(agent, { fromSeq: finalFromSeq, toSeq: admittedLastSeq, includeAssistantText: true, maxChars: selected.autoVerifyMaxChars, repeats: selected.autoVerifyFinalRepeats }, signal, 'final', finalRubric);
             if (!stillCurrent()) {
                 autoRouter.fail(agent, finalReservation, false);
                 return;
@@ -586,12 +890,62 @@ export function apply(ctx, config = {}) {
                 agent.steer(createUserMessage({ content: [{ type: 'text', text: '[Automatic verifier gate]\nStrict final verification failed: ' + (error instanceof Error ? error.message : String(error)) + '\nDo not conclude until verification succeeds.' }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }));
         }
     });
-    ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings (or the configured judge ensemble) with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, agreement: { type: 'number', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return record('verifier_compare', agent, async () => { const { verifier, selected } = await engine(agent); const [problem, candidateA, candidateB] = explicitEvidence([args.problem, args.candidate_a, args.candidate_b], explicitItemChars(selected), explicitBudget(selected), 'compare input'); const result = await verifier.compare({ problem, candidateA, candidateB, criteria: normalizeCriteria(args.criteria), repeats: capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { result, selected }; }); } }));
-    ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Deterministic orchestrators should call this directly once they have three or more real candidates.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return record('verifier_select', agent, async () => { const { verifier, selected } = await engine(agent); const limit = explicitCandidateLimit(selected); if (args.candidates.length > limit)
-            throw new Error('llm-verifier: candidates must contain at most ' + limit + ' entries'); const candidates = explicitEvidence(args.candidates, explicitItemChars(selected), explicitBudget(selected), 'candidates'); const criteria = normalizeCriteria(args.criteria); const repeats = capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats'); const planned = plannedComparisons(candidates.length) * (criteria?.length || 3) * repeats; if (planned > MAX_EXPLICIT_PLANNED_CALLS)
-            throw new Error('llm-verifier: this selection would issue about ' + planned + ' judge calls; reduce candidates or repeats'); const result = await verifier.select({ problem: sanitizeVerifierText(args.problem, explicitItemChars(selected)), candidates, criteria, repeats, pivots: capped(args.pivots, 2, Math.max(1, candidates.length), 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { result, selected }; }); } }));
-    ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Deterministic goal/workflow orchestrators should call this directly when real checkpoints already exist. Do not use for a single completed answer or invent checkpoints.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return record('verifier_track', agent, async () => { const { verifier, selected } = await engine(agent); if (args.steps.length > MAX_TRACK_STEPS)
-            throw new Error('llm-verifier: steps must contain at most ' + MAX_TRACK_STEPS + ' entries'); const steps = explicitEvidence(args.steps, explicitItemChars(selected), explicitBudget(selected), 'steps'); const result = await verifier.track(sanitizeVerifierText(args.problem, explicitItemChars(selected)), steps, args.checkpoints, capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { result, selected }; }); } }));
+    ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings (or the configured judge ensemble) with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, agreement: { type: 'number', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return record('verifier_compare', agent, async (trace) => { const { verifier, selected } = await engine(agent); const [problem, candidateA, candidateB] = explicitEvidence([args.problem, args.candidate_a, args.candidate_b], explicitItemChars(selected), explicitBudget(selected), 'compare input'); const rubric = args.criteria === undefined ? await configuredCriteria() : { criteria: normalizeCriteria(args.criteria), source: 'explicit' }; const result = await verifier.compare({ problem, candidateA, candidateB, criteria: rubric.criteria, ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats: capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats'), images: await images(args.images, exec.signal), ...(trace ? { trace } : {}) }, exec.signal); return { result, selected }; }); } }));
+    ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates. Generating extra candidates pays off only when the artifact is a final deliverable and choosing wrong is expensive: produce them (for example with parallel subagents), then rank the real ones here. Do not pad the list with near-duplicates. Deterministic orchestrators should call this directly once they have three or more real candidates.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return record('verifier_select', agent, async (trace) => { const { verifier, selected } = await engine(agent); const limit = explicitCandidateLimit(selected); if (args.candidates.length > limit)
+            throw new Error('llm-verifier: candidates must contain at most ' + limit + ' entries'); const candidates = explicitEvidence(args.candidates, explicitItemChars(selected), explicitBudget(selected), 'candidates'); const rubric = args.criteria === undefined ? await configuredCriteria() : { criteria: normalizeCriteria(args.criteria), source: 'explicit' }; const criteria = rubric.criteria; const repeats = capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats'); const planned = plannedComparisons(candidates.length) * (criteria?.length || 3) * repeats; if (planned > MAX_EXPLICIT_PLANNED_CALLS)
+            throw new Error('llm-verifier: this selection would issue about ' + planned + ' judge calls; reduce candidates or repeats'); const result = await verifier.select({ problem: sanitizeVerifierText(args.problem, explicitItemChars(selected)), candidates, criteria, ...(rubric.groundTruthNote ? { groundTruthNote: rubric.groundTruthNote } : {}), repeats, pivots: capped(args.pivots, 2, Math.max(1, candidates.length), 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal), ...(trace ? { trace } : {}) }, exec.signal); return { result, selected }; }); } }));
+    ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Deterministic goal/workflow orchestrators should call this directly when real checkpoints already exist. Do not use for a single completed answer or invent checkpoints.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return record('verifier_track', agent, async (trace) => { const { verifier, selected } = await engine(agent); if (args.steps.length > MAX_TRACK_STEPS)
+            throw new Error('llm-verifier: steps must contain at most ' + MAX_TRACK_STEPS + ' entries'); const steps = explicitEvidence(args.steps, explicitItemChars(selected), explicitBudget(selected), 'steps'); const result = await verifier.track(sanitizeVerifierText(args.problem, explicitItemChars(selected)), steps, args.checkpoints, capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats'), exec.signal, await images(args.images, exec.signal), trace); return { result, selected }; }); } }));
+    ctx.tools.register(defineTool({
+        name: 'verifier_best_of_n',
+        description: 'Use ONLY when a final deliverable is expensive to get wrong and no candidate exists yet: this drafts n independent candidates with the current session model, has the independent verifier rank them, and then re-scores the winner against the same fixed empty-work baseline the automatic acceptance gate uses. A draft that hits its output ceiling is kept and listed in truncated — the judge sees the incomplete text and scores it as such, and discarding it would waste a generation already paid for. It is by far the most expensive verifier tool: on the default 3-criterion rubric it costs 2 generations + 12 judge calls at n=2, 3 + 24 at n=3, and 4 + 36..60 at n=4 — never call it per turn, for trivial questions, or to compare candidates you already have (rank those with verifier_select or verifier_compare). scores are RELATIVE tournament shares; only score, criteria, threshold and passesThreshold are absolute and comparable with the acceptance gate. Requires a session whose model is already known; if the session has no logged request header, write the candidates with parallel subagents and call verifier_select instead.',
+        parameters: {
+            task: { type: 'string', required: true, description: 'The request every draft must answer. Bound to the explicit per-item evidence cap after redaction.' },
+            n: { type: 'integer', description: 'How many independent drafts to generate; between 2 and 4, default 3. Cost grows with n.' },
+            criteria: commonParams.criteria,
+            repeats: commonParams.repeats,
+        },
+        output: {
+            schema: {
+                type: 'object',
+                additionalProperties: false,
+                properties: {
+                    best: { type: 'string', required: true },
+                    index: { type: 'integer', required: true },
+                    sources: { type: 'array', items: { type: 'integer' }, required: true },
+                    scores: { type: 'array', items: { type: 'number' }, required: true },
+                    ranking: { type: 'array', items: { type: 'integer' }, required: true },
+                    comparisons: { type: 'integer', required: true },
+                    pivots: { type: 'array', items: { type: 'integer' }, required: true },
+                    generated: { type: 'integer', required: true },
+                    failed: { type: 'integer', required: true },
+                    failures: { type: 'array', items: { type: 'string' }, required: true },
+                    truncated: { type: 'array', items: { type: 'integer' }, required: true },
+                    score: { type: 'number', required: true },
+                    baselineScore: { type: 'number', required: true },
+                    winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true },
+                    criteria: { type: 'array', items: criterionResultSchema, required: true },
+                    threshold: { type: 'number', required: true },
+                    passesThreshold: { type: 'boolean', required: true },
+                    failedCriteria: { type: 'array', items: { type: 'string' }, required: true },
+                    calls: { type: 'integer', required: true },
+                    stats: { ...statsSchema, required: true },
+                    generatorProvider: { type: 'string', required: true },
+                    generatorModel: { type: 'string', required: true },
+                    provider: { type: 'string', required: true },
+                    model: { type: 'string', required: true },
+                    judges: judgesSchema,
+                },
+            },
+            render: (_args, value) => renderJson(value),
+        },
+        timeoutMs: entry.timeoutMs * 100,
+        async execute(args, exec) {
+            requireEnabled();
+            const agent = requireAgent(exec.agent);
+            return bestOfN(agent, args.task, args.n ?? DEFAULT_BEST_OF_N, args.repeats, exec.signal, args.criteria);
+        },
+    }));
     ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Explicitly verify the current DSH session. Smart/strict policy can also invoke this gate automatically at the turn-stopping lifecycle boundary after consequential work with real tool evidence. Extracts the session, applies redaction and bounds, then sends the evidence to the configured verifier model.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, agreement: { type: 'number', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true }, judges: judgesSchema } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = requireAgent(exec.agent); return verifySession(agent, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars === undefined ? undefined : capped(args.max_chars, 200000, MAX_SESSION_CHARS, 'max_chars'), repeats: capped(args.repeats, 2, MAX_EXPLICIT_REPEATS, 'repeats') }, exec.signal); } }));
 }
 //# sourceMappingURL=index.js.map