dsh-llm-verifier 0.1.20 → 0.2.0

package/lib/index.js

@@ -24,6 +24,7 @@ const Config = z.object({
 	]).default("smart"),
 	autoVerifyThreshold: z.number().min(0).max(1).default(.65),
 	autoVerifyRepeats: z.number().step(1).min(1).default(1),
+	autoVerifyFinalRepeats: z.number().step(1).min(1).default(2),
 	autoVerifyMinToolCalls: z.number().step(1).min(1).default(3),
 	autoVerifyMaxChars: z.number().step(1).min(1e3).default(8e4),
 	autoVerifyMaxPerTask: z.number().step(1).min(1).default(2),
@@ -36,7 +37,7 @@ const Config = z.object({
 	autoTrackCompletionThreshold: z.number().min(0).max(1).default(.8),
 	autoRouteMaxItemChars: z.number().step(1).min(100).default(2e4),
 	autoRouteMaxInputChars: z.number().step(1).min(1e3).default(6e4),
-	autoMaxModelCallsPerTask: z.number().step(1).min(1).default(64),
+	autoMaxModelCallsPerTask: z.number().step(1).min(1).default(96),
 	autoMaxModelCallsPerSession: z.number().step(1).min(1).default(240),
 	autoVerifyTeamTasks: z.boolean().default(true),
 	autoVerifyPlanMode: z.boolean().default(true),
@@ -97,6 +98,7 @@ function resolveConfig(config = {}) {
 	if (!Number.isFinite(temperature) || temperature < 0 || temperature > 2) throw new Error("llm-verifier: temperature must be between 0 and 2");
 	const values = {
 		autoVerifyRepeats: config.autoVerifyRepeats ?? 1,
+		autoVerifyFinalRepeats: config.autoVerifyFinalRepeats ?? 2,
 		autoVerifyMinToolCalls: config.autoVerifyMinToolCalls ?? 3,
 		autoVerifyMaxChars: config.autoVerifyMaxChars ?? 8e4,
 		autoVerifyMaxPerTask: config.autoVerifyMaxPerTask ?? 2,
@@ -106,7 +108,7 @@ function resolveConfig(config = {}) {
 		autoRouteMaxPerSession: config.autoRouteMaxPerSession ?? 8,
 		autoRouteMaxItemChars: config.autoRouteMaxItemChars ?? 2e4,
 		autoRouteMaxInputChars: config.autoRouteMaxInputChars ?? 6e4,
-		autoMaxModelCallsPerTask: config.autoMaxModelCallsPerTask ?? 64,
+		autoMaxModelCallsPerTask: config.autoMaxModelCallsPerTask ?? 96,
 		autoMaxModelCallsPerSession: config.autoMaxModelCallsPerSession ?? 240,
 		maxTokens: config.maxTokens ?? 32768,
 		timeoutMs: config.timeoutMs ?? 3e5,
@@ -1169,14 +1171,16 @@ function buildEvidenceIndex(events) {
 		teamTasks
 	};
 }
-function parseTrustedWorkflow(value, callId, callSeq, resultSeq, maxCandidates, maxItemChars) {
+function parseTrustedWorkflow(value, callId, callSeq, resultSeq, maxCandidates, maxItemChars, maxInputChars) {
 	if (typeof value !== "object" || value === null || Array.isArray(value)) return [];
 	const envelope = value;
 	if (envelope.protocol !== "dsh-verifier-candidates" || envelope.version !== TRUSTED_WORKFLOW_VERSION || typeof envelope.groupId !== "string" || !envelope.groupId.trim() || !Array.isArray(envelope.candidates)) return [];
 	const groupId = envelope.groupId.trim();
 	const seen = /* @__PURE__ */ new Set();
 	const candidates = [];
-	for (const item of envelope.candidates.slice(0, maxCandidates)) {
+	const considered = envelope.candidates.slice(0, maxCandidates);
+	const perItem = itemBudget(considered.length, maxItemChars, maxInputChars);
+	for (const item of considered) {
 		if (typeof item !== "object" || item === null || Array.isArray(item)) return [];
 		const row = item;
 		if (row.status !== "completed" || typeof row.id !== "string" || !row.id.trim() || seen.has(row.id.trim()) || typeof row.content !== "string" || !row.content.trim()) return [];
@@ -1186,8 +1190,8 @@ function parseTrustedWorkflow(value, callId, callSeq, resultSeq, maxCandidates,
 		candidates.push({
 			id,
 			groupId,
-			label: sanitizeVerifierText(label, 120),
-			content: sanitizeVerifierText(row.content, maxItemChars),
+			label: sanitizeVerifierText(label, Math.min(120, perItem)),
+			content: sanitizeVerifierText(row.content, perItem),
 			callId,
 			fromSeq: callSeq,
 			toSeq: resultSeq
@@ -1242,17 +1246,61 @@ function checkpointEvidence(index, seq, budget) {
 	return prefix + sanitizeVerifierText(latest.text, budget - prefix.length);
 }
 /**
-* Render one checkpoint step as "state + the output that proves it".
+* Per-item character budget for a decision with a known item count.
+*
+* boundDecision() enforces the COMBINED cap, so spending maxItemChars per item
+* drops the whole decision as soon as the items are numerous or large. Splitting
+* the combined budget keeps both caps satisfied by construction; a single item
+* still gets the full maxItemChars.
+* @param count - number of items that will be rendered.
+* @param maxItemChars - hard per-item cap enforced by boundDecision().
+* @param maxInputChars - hard combined cap enforced by boundDecision().
+* @returns The per-item character budget, never below 1.
+*/
+function itemBudget(count, maxItemChars, maxInputChars) {
+	return Math.max(1, Math.min(maxItemChars, Math.floor(maxInputChars / Math.max(1, count))));
+}
+/**
+* Upper bound on the checkpoints rendered into one routed track decision.
+*
+* boundDecision() rejects the WHOLE decision once the rendered steps exceed
+* maxInputChars, while the number of durable snapshots is unbounded (every changed
+* todo/team snapshot becomes a checkpoint). A long task therefore used to lose
+* progress routing exactly when it needed it, so only the most recent checkpoints
+* are kept and the combined input budget is split across them.
+*/
+const MAX_ROUTED_CHECKPOINTS = 6;
+/**
+* Render progress checkpoints as "state + the output that proves it".
+*
+* The result fits both caps by construction: each step is at most min(maxItemChars,
+* maxInputChars / kept.length) characters, so the combined length can never exceed
+* maxInputChars and boundDecision() no longer drops the whole route.
 * @param index - evidence index of the current task.
-* @param label - checkpoint heading.
-* @param body - todo/team task rendering.
-* @param seq - checkpoint sequence number.
+* @param sources - checkpoints in chronological order.
 * @param maxItemChars - hard per-item cap enforced by boundDecision().
-* @returns A step that always fits the per-item cap.
+* @param maxInputChars - hard combined cap enforced by boundDecision().
+* @returns The rendered steps plus the sequence numbers they were built from.
 */
-function checkpointStep(index, label, body, seq, maxItemChars, maxInputChars) {
-	const evidence = checkpointEvidence(index, seq, Math.max(0, Math.min(Math.floor(maxInputChars / 2), Math.floor(maxItemChars / 2))));
-	return sanitizeVerifierText(label + body, Math.max(1, maxItemChars - evidence.length)) + evidence;
+function renderCheckpointSteps(index, sources, maxItemChars, maxInputChars) {
+	const kept = sources.slice(-6);
+	const omitted = sources.length - kept.length;
+	if (kept.length === 0) return {
+		steps: [],
+		evidenceSeqs: [],
+		omitted: 0
+	};
+	const stepCap = itemBudget(kept.length, maxItemChars, maxInputChars);
+	const evidenceBudget = Math.max(0, Math.min(Math.floor(maxInputChars / 2), Math.floor(maxItemChars / 2), Math.floor(stepCap / 2)));
+	return {
+		steps: kept.map((source, position) => {
+			const note = position === 0 && omitted > 0 ? "Earlier " + omitted + " checkpoint(s) omitted; showing the " + kept.length + " most recent.\n" : "";
+			const evidence = checkpointEvidence(index, source.seq, evidenceBudget);
+			return sanitizeVerifierText(note + source.label + source.body, Math.max(1, stepCap - evidence.length)) + evidence;
+		}),
+		evidenceSeqs: kept.map((source) => source.seq),
+		omitted
+	};
 }
 function canonicalTeamTaskSnapshots(index) {
 	const values = [];
@@ -1278,7 +1326,7 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4, m
 	const groups = [];
 	for (const [callId, pair] of index.calls) {
 		if (pair.name !== "workflow") continue;
-		const candidates = parseTrustedWorkflow(strictJson(pair.text), callId, pair.callSeq, pair.resultSeq, maxCandidates, maxItemChars);
+		const candidates = parseTrustedWorkflow(strictJson(pair.text), callId, pair.callSeq, pair.resultSeq, maxCandidates, maxItemChars, maxInputChars);
 		if (candidates.length >= 2) groups.push(candidates);
 	}
 	groups.sort((a, b) => b.length - a.length || b[0].toSeq - a[0].toSeq);
@@ -1308,8 +1356,11 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4, m
 	if (!explicit.has("track")) {
 		const snapshots = canonicalTodoSnapshots(index);
 		if (snapshots.length >= 2 && snapshots.some((snapshot) => snapshot.todos.length >= 2)) {
-			const steps = snapshots.map((snapshot) => checkpointStep(index, "Todo checkpoint seq " + snapshot.seq + ":\n", snapshot.todos.map((todo) => "- [" + todo.status + "] " + todo.content).join("\n"), snapshot.seq, maxItemChars, maxInputChars));
-			const checkpoints = steps.map((_, i) => i + 1);
+			const rendered = renderCheckpointSteps(index, snapshots.map((snapshot) => ({
+				seq: snapshot.seq,
+				label: "Todo checkpoint seq " + snapshot.seq + ":\n",
+				body: snapshot.todos.map((todo) => "- [" + todo.status + "] " + todo.content).join("\n")
+			})), maxItemChars, maxInputChars);
 			return {
 				kind: "track",
 				source: "structured",
@@ -1319,15 +1370,18 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4, m
 					kind: "track",
 					snapshots
 				}),
-				steps,
-				checkpoints,
-				evidenceSeqs: snapshots.map((snapshot) => snapshot.seq)
+				steps: rendered.steps,
+				checkpoints: rendered.steps.map((_, i) => i + 1),
+				evidenceSeqs: rendered.evidenceSeqs
 			};
 		}
 		const teamSnapshots = canonicalTeamTaskSnapshots(index);
 		if (teamSnapshots.length >= 2) {
-			const steps = teamSnapshots.map((snapshot) => checkpointStep(index, "Team task checkpoint seq " + snapshot.seq + ":\n", snapshot.tasks.map((task) => "- [" + task.status + "] " + task.subject + (task.description ? " (" + task.description + ")" : "")).join("\n"), snapshot.seq, maxItemChars, maxInputChars));
-			const checkpoints = steps.map((_, i) => i + 1);
+			const rendered = renderCheckpointSteps(index, teamSnapshots.map((snapshot) => ({
+				seq: snapshot.seq,
+				label: "Team task checkpoint seq " + snapshot.seq + ":\n",
+				body: snapshot.tasks.map((task) => "- [" + task.status + "] " + task.subject + (task.description ? " (" + task.description + ")" : "")).join("\n")
+			})), maxItemChars, maxInputChars);
 			return {
 				kind: "track",
 				source: "structured",
@@ -1337,9 +1391,9 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4, m
 					kind: "track",
 					teamSnapshots
 				}),
-				steps,
-				checkpoints,
-				evidenceSeqs: teamSnapshots.map((snapshot) => snapshot.seq)
+				steps: rendered.steps,
+				checkpoints: rendered.steps.map((_, i) => i + 1),
+				evidenceSeqs: rendered.evidenceSeqs
 			};
 		}
 	}
@@ -1436,7 +1490,11 @@ function semanticDecision(output, events, maxItemChars = 2e4, maxInputChars = 6e
 			todos: index.todos.get(seq)
 		})).filter((item) => item.todos !== void 0);
 		if (snapshots.length !== output.checkpointSeqs.length) return void 0;
-		const steps = snapshots.map((snapshot) => checkpointStep(index, "Todo checkpoint seq " + snapshot.seq + ":\n", snapshot.todos.map((todo) => "- [" + todo.status + "] " + todo.content).join("\n"), snapshot.seq, maxItemChars, maxInputChars));
+		const rendered = renderCheckpointSteps(index, snapshots.map((snapshot) => ({
+			seq: snapshot.seq,
+			label: "Todo checkpoint seq " + snapshot.seq + ":\n",
+			body: snapshot.todos.map((todo) => "- [" + todo.status + "] " + todo.content).join("\n")
+		})), maxItemChars, maxInputChars);
 		return {
 			kind: "track",
 			source: "semantic",
@@ -1445,20 +1503,21 @@ function semanticDecision(output, events, maxItemChars = 2e4, maxInputChars = 6e
 			fingerprint: stableHash({
 				kind: "track",
 				seqs: output.checkpointSeqs,
-				steps
+				steps: rendered.steps
 			}),
-			steps,
-			checkpoints: steps.map((_, i) => i + 1),
-			evidenceSeqs: output.checkpointSeqs
+			steps: rendered.steps,
+			checkpoints: rendered.steps.map((_, i) => i + 1),
+			evidenceSeqs: rendered.evidenceSeqs
 		};
 	}
+	const perItem = itemBudget(output.candidateCallIds.length, maxItemChars, maxInputChars);
 	const candidates = output.candidateCallIds.map((callId, i) => {
 		const pair = index.calls.get(callId);
 		return pair ? {
 			id: callId,
 			groupId: "semantic",
 			label: pair.name + " " + (i + 1),
-			content: sanitizeVerifierText(pair.text, maxItemChars),
+			content: sanitizeVerifierText(pair.text, perItem),
 			callId,
 			fromSeq: pair.callSeq,
 			toSeq: pair.resultSeq
@@ -2873,6 +2932,40 @@ function apply(ctx, config = {}) {
 			throw error;
 		}
 	};
+	/** The registered tool each routable decision kind maps to. */
+	const ROUTED_TOOL_BY_KIND = {
+		compare: "verifier_compare",
+		select: "verifier_select",
+		track: "verifier_track"
+	};
+	/**
+	* Record a routed decision the plugin built but could not execute.
+	*
+	* A decision rejected by the evidence caps (or one whose semantic references
+	* disappeared) used to vanish without a trace: the dashboard showed a task that
+	* was never verified and gave no reason why. Nothing failed here — no model call
+	* was made — so the row is a successful invocation carrying an explanatory verdict.
+	* @param agent - Agent whose task produced the decision.
+	* @param kind - The routed decision kind that was dropped.
+	* @param phase - Routing phase that produced it (structured | semantic).
+	* @param outcome - Why it was dropped.
+	*/
+	const recordSkippedRoute = async (agent, kind, phase, outcome) => {
+		const selected = current();
+		await topic(agent.session.header).statistics.record({
+			toolName: ROUTED_TOOL_BY_KIND[kind],
+			sessionId: String(agent.id),
+			startedAt: Date.now(),
+			success: true,
+			provider: selected.provider,
+			model: selected.model,
+			stats: emptyRunStats(),
+			verdict: {
+				phase,
+				outcome
+			}
+		}).catch(() => {});
+	};
 	const verifySession = async (agent, options, signal, phase = "explicit") => record("verifier_current_session", agent, async () => {
 		const extracted = await extractSession(agent, async (ref) => {
 			const stored = await services.attachments.readImage(ref, signal);
@@ -3184,7 +3277,12 @@ function apply(ctx, config = {}) {
 				}
 			}
 		}
-		let decision = boundDecision(analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars, selected.autoRouteMaxInputChars), policy);
+		const structured = analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars, selected.autoRouteMaxInputChars);
+		let decision = boundDecision(structured, policy);
+		if (structured !== void 0 && decision === void 0) {
+			ctx.logger.warn("llm-verifier automatic " + structured.kind + " route dropped: its evidence exceeds the per-item/total routing caps (" + selected.autoRouteMaxItemChars + "/" + selected.autoRouteMaxInputChars + " characters)");
+			await recordSkippedRoute(agent, structured.kind, "structured", "dropped-over-budget");
+		}
 		if (decision === void 0 && selected.autoRouteSemantic && (selected.autoVerifyMode === "strict" || semanticRouteHint(snapshot))) {
 			const fingerprint = stableHash({
 				phase: "semantic",
@@ -3202,7 +3300,17 @@ function apply(ctx, config = {}) {
 				}
 				const parsed = parseSemanticRoute(classified.text, selected.autoRouteMaxCandidates);
 				if (!parsed) throw new Error("semantic router returned invalid strict JSON");
-				decision = parsed.confidence >= selected.autoRouteMinConfidence ? boundDecision(semanticDecision(parsed, snapshot, selected.autoRouteMaxItemChars, selected.autoRouteMaxInputChars), policy) : void 0;
+				if (parsed.confidence >= selected.autoRouteMinConfidence) {
+					const resolved = semanticDecision(parsed, snapshot, selected.autoRouteMaxItemChars, selected.autoRouteMaxInputChars);
+					decision = boundDecision(resolved, policy);
+					if (resolved !== void 0 && decision === void 0) {
+						ctx.logger.warn("llm-verifier semantic " + resolved.kind + " route dropped: its evidence exceeds the per-item/total routing caps (" + selected.autoRouteMaxItemChars + "/" + selected.autoRouteMaxInputChars + " characters)");
+						await recordSkippedRoute(agent, resolved.kind, "semantic", "dropped-over-budget");
+					} else if (resolved === void 0 && parsed.kind !== "none") {
+						ctx.logger.warn("llm-verifier semantic router returned " + parsed.kind + " but referenced evidence missing from the session snapshot");
+						await recordSkippedRoute(agent, parsed.kind, "semantic", "invalid-references");
+					}
+				}
 				autoRouter.commit(agent, reservation);
 			} catch (error) {
 				autoRouter.fail(agent, reservation, selected.autoVerifyMode === "strict");
@@ -3226,7 +3334,9 @@ function apply(ctx, config = {}) {
 			const reservation = autoRouter.reserve(agent, decision.kind, decision.fingerprint, expectedCalls, policy);
 			if (reservation === void 0) {
 				const exhausted = autoRouter.budgetExhausted(agent, expectedCalls, policy);
-				ctx.logger.warn("llm-verifier automatic " + decision.kind + " route skipped: " + (exhausted ? "the task/session budget cannot cover " + expectedCalls + " model calls" : "another verifier is active or this decision already ran"));
+				const alreadyRan = autoRouter.completedFingerprint(agent, decision.fingerprint);
+				const refusal = exhausted ? "the task/session budget cannot cover " + expectedCalls + " model calls" : alreadyRan ? "this exact evidence was already routed" : "another verifier is active";
+				ctx.logger.warn("llm-verifier automatic " + decision.kind + " route skipped: " + refusal);
 				if (exhausted && selected.autoVerifyMode === "strict" && autoRouter.claimExhaustedNotice(agent)) {
 					agent.steer(createUserMessage({
 						content: [{
@@ -3312,7 +3422,7 @@ function apply(ctx, config = {}) {
 			from: finalFromSeq,
 			to: admittedLastSeq
 		});
-		const finalReservation = autoRouter.reserve(agent, "final", finalFingerprint, Math.max(1, 4 * selected.autoVerifyRepeats) * selected.judges.length, policy);
+		const finalReservation = autoRouter.reserve(agent, "final", finalFingerprint, Math.max(1, DEFAULT_CRITERIA.length * selected.autoVerifyFinalRepeats) * selected.judges.length, policy);
 		if (!finalReservation) {
 			if (selected.autoVerifyMode === "strict" && (forcedFromSeq !== void 0 || autoRouter.strictBlocked(agent))) if (autoRouter.claimExhaustedNotice(agent)) agent.steer(createUserMessage({
 				content: [{
@@ -3333,7 +3443,7 @@ function apply(ctx, config = {}) {
 				toSeq: admittedLastSeq,
 				includeAssistantText: true,
 				maxChars: selected.autoVerifyMaxChars,
-				repeats: selected.autoVerifyRepeats
+				repeats: selected.autoVerifyFinalRepeats
 			}, signal, "final");
 			if (!stillCurrent()) {
 				autoRouter.fail(agent, finalReservation, false);
@@ -3733,6 +3843,6 @@ function apply(ctx, config = {}) {
 	}));
 }
 //#endregion
-export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, MAX_TEAM_TASK_METADATA_CHARS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, UNTRUSTED_EVIDENCE_NOTE, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildPlanPreReviewPrompt, buildProgressPrompt, buildSemanticRoutePrompt, buildTeamTaskVerificationPrompt, callVerifier, emptyRunStats, errorDetails, estimateRoutedCalls, evidenceNonce, extractProgressScore, extractScore, inject, inspectTeamTasks, isSubagentSession, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, parseStatisticsQuery, parseVerdictLetter, pivotRoundPairs, planFromArguments, rankScores, renderDelimitedBlock, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
+export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, MAX_ROUTED_CHECKPOINTS, MAX_TEAM_TASK_METADATA_CHARS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, UNTRUSTED_EVIDENCE_NOTE, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildPlanPreReviewPrompt, buildProgressPrompt, buildSemanticRoutePrompt, buildTeamTaskVerificationPrompt, callVerifier, emptyRunStats, errorDetails, estimateRoutedCalls, evidenceNonce, extractProgressScore, extractScore, inject, inspectTeamTasks, isSubagentSession, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, parseStatisticsQuery, parseVerdictLetter, pivotRoundPairs, planFromArguments, rankScores, renderDelimitedBlock, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
 
 //# sourceMappingURL=index.js.map