dsh-llm-verifier 0.1.2 → 0.1.4

package/README.md

@@ -8,12 +8,13 @@
 
 `dsh-llm-verifier` 为 DeepSeek Harness（DSH）引入了一套**独立的裁判复核机制**。在主 Agent 负责生成代码、执行命令与工具交互的同时，Verifier 会收集当前任务目标、各候选方案过程以及真实的终端执行结果，交由你在设置中指定的独立 DSH 模型进行仲裁：评估哪个方案更可靠、当前任务的实际完成进度、以及是否存在未发现的潜在错误。
 
-只有在主动调用以下工具时，插件才会向裁判模型发起请求：
+插件提供四个显式工具，并支持可配置的宿主级自动会话验收：
 
 - `verifier_compare`：对两个候选执行过程进行成对比较（Pairwise Comparison）；
-- `verifier_select`：在多个候选方案中通过锦标赛机制选出最优解；
-- `verifier_track`：评估任务在不同检查点（Checkpoint）的完成度与进展；
-- `verifier_current_session`：显式提取当前 DSH 会话记录，进行脱敏并执行复核。
+- `verifier_select`：在多个候选方案中通过锦标赛机制选出最优解，供 Best-of-N / 多候选编排器直接调用；
+- `verifier_track`：评估任务在已有检查点（Checkpoint）的完成度与进展，供 Goal / Workflow 等长任务编排器直接调用；
+- `verifier_current_session`：显式提取当前 DSH 会话记录，进行脱敏并执行复核；
+- **自动验收门控**：智能或严格策略在 `agent/turn-stopping` 生命周期边界检测真实工具证据，宿主直接运行同一会话验收逻辑；未通过时以插件 steering 反馈要求 Agent 修复并重新验证，而不是依赖模型是否主动想起工具。
 
 ## 安装与启用 (Installation & Usage)
 
@@ -164,7 +165,13 @@ flowchart LR
 
 | 配置项 | 说明 |
 |---|---|
-| **启用工具 (Enabled)** | 是否允许 Agent 调用 verifier 工具；关闭后调用会立即返回错误，不产生任何模型请求（设置页「工具开关」按钮可切换） |
+| **启用工具 (Enabled)** | 是否允许显式工具与自动验收向裁判模型发起请求；关闭后显式调用立即报错且自动门控不运行 |
+| **调用策略** | `manual` 仅显式调用；`smart` 仅在达到工具证据数量且含写入/执行类操作时自动验收；`strict` 对任一已完成的关键操作自动验收 |
+| **通过阈值** | 自动会话验收要求证据分数达到阈值且胜过“未执行有效工作”基线 |
+| **自动评估轮次** | 自动验收每个标准的重复轮次；默认 1，作为低成本初筛 |
+| **智能模式最少工具调用** | `smart` 策略需要的最少非 Verifier 工具调用数 |
+| **最大证据字符** | 自动发送给裁判的最近会话轨迹字符上限 |
+| **每任务/每会话最多验收** | 防止低分反馈形成无限修复循环并限制成本 |
 | **供应商 (Provider)** | 从 DSH 当前已配置且可路由的 Provider 列表中选择 |
 | **模型 (Model)** | 从所选 Provider 的模型目录中指定具体裁判模型 |
 | **推理强度 (Reasoning Effort)** | 使用 Adapter 为该模型声明的思考强度，或保留模型默认值 |
@@ -189,7 +196,7 @@ flowchart LR
 
 ## 隐私与数据安全边界
 
-`verifier_current_session` 工具**仅在被显式调用时**才会读取当前 DSH 会话内容：
+`verifier_current_session` 会在显式调用时读取当前 DSH 会话；启用 `smart` / `strict` 策略后，宿主也会在满足证据门槛的任务结束边界自动调用同一提取与复核路径：
 
 - **提取范围**：仅提取直接的用户消息、Assistant 回复、工具调用（Tool Call）及实际工具执行结果（Tool Result）；自动排除插件和系统内置指令（Plugin/System Instructions）；
 - **敏感信息脱敏**：默认对常见 Bearer Token、API Key、通用 Token、Password 及 Secret 进行脱敏替换；

package/lib/caller-BXrUTAoL.js

@@ -0,0 +1,324 @@
+import { BlockAssembler, ReasoningEffortId, createUserMessage, deepFreeze } from "@deepseek-ai/dsh-llm";
+import { settingsNamespace } from "@deepseek-ai/dsh-settings";
+import { credentialRef } from "@deepseek-ai/dsh-credentials";
+//#region src/top-logprobs.ts
+var TopLogprobsUnsupportedError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "TopLogprobsUnsupportedError";
+	}
+};
+function object(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+function text(value) {
+	return typeof value === "string" && value.trim() ? value.trim() : void 0;
+}
+function endpoint(baseURL) {
+	return baseURL.replace(/\/+$/, "") + "/chat/completions";
+}
+function dataUrl(image) {
+	return "data:" + image.mediaType + ";base64," + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).toString("base64");
+}
+async function credential(ctx, name) {
+	if (!name) return void 0;
+	return (await ctx.get("credentials")?.resolve(credentialRef(name)))?.value;
+}
+async function resolveTopLogprobRoute(ctx, provider) {
+	const settings = ctx.get("settings");
+	if (provider === "deepseek-official") {
+		const value = settings ? object(settings.get(settingsNamespace("llm-deepseek"))) ?? {} : {};
+		const apiKey = await credential(ctx, text(value.apiKeyEnv) ?? "DEEPSEEK_API_KEY");
+		if (!apiKey) return void 0;
+		return {
+			baseURL: text(value.baseURL) ?? "https://api.deepseek.com",
+			apiKey,
+			deepSeekThinking: true
+		};
+	}
+	if (!settings) return void 0;
+	const profile = object(object(object(settings.get(settingsNamespace("llm-pi-ai")))?.providers)?.[provider]);
+	if (!profile || profile.api !== "openai-completions") return void 0;
+	const baseURL = text(profile.baseURL);
+	if (!baseURL || !/^https:\/\//i.test(baseURL)) return void 0;
+	const apiKey = await credential(ctx, text(profile.apiKeyEnv));
+	const rawHeaders = object(profile.headers);
+	const headers = rawHeaders === void 0 ? void 0 : Object.fromEntries(Object.entries(rawHeaders).filter((entry) => typeof entry[1] === "string"));
+	return {
+		baseURL,
+		...apiKey ? { apiKey } : {},
+		...headers ? { headers } : {},
+		deepSeekThinking: false
+	};
+}
+async function callTopLogprobs(route, model, prompt, maxTokens, reasoningEffort, signal, images) {
+	const content = images?.length ? [{
+		type: "text",
+		text: prompt
+	}, ...images.map((image) => ({
+		type: "image_url",
+		image_url: { url: dataUrl(image) }
+	}))] : prompt;
+	const thinking = route.deepSeekThinking && reasoningEffort ? reasoningEffort === "off" ? { thinking: { type: "disabled" } } : {
+		thinking: { type: "enabled" },
+		reasoning_effort: reasoningEffort
+	} : {};
+	const response = await fetch(endpoint(route.baseURL), {
+		method: "POST",
+		redirect: "error",
+		signal,
+		headers: {
+			"content-type": "application/json",
+			...route.apiKey ? { authorization: "Bearer " + route.apiKey } : {},
+			...route.headers
+		},
+		body: JSON.stringify({
+			model,
+			messages: [{
+				role: "user",
+				content
+			}],
+			max_tokens: maxTokens,
+			temperature: 1,
+			logprobs: true,
+			top_logprobs: 20,
+			...thinking
+		})
+	});
+	const raw = await response.text();
+	if (!response.ok) {
+		const excerpt = raw.slice(0, 1e3);
+		if ([
+			400,
+			404,
+			405,
+			415,
+			422
+		].includes(response.status) && /logprob|top_logprobs|unsupported|unknown (?:field|parameter)|unrecognized (?:field|parameter)|not support/i.test(excerpt)) throw new TopLogprobsUnsupportedError("provider rejected top_logprobs: HTTP " + response.status + " " + excerpt);
+		throw new Error("llm-verifier: top_logprobs request failed with HTTP " + response.status + ": " + excerpt);
+	}
+	let body;
+	try {
+		body = object(JSON.parse(raw)) ?? {};
+	} catch {
+		throw new Error("llm-verifier: top_logprobs endpoint returned invalid JSON");
+	}
+	const choice = object((Array.isArray(body.choices) ? body.choices : [])[0]);
+	const message = object(choice?.message);
+	const answer = typeof message?.content === "string" ? message.content : "";
+	const logprobs = object(choice?.logprobs);
+	const rows = Array.isArray(logprobs?.content) ? logprobs.content : [];
+	if (!rows.length) throw new TopLogprobsUnsupportedError("provider returned no token logprobs");
+	const tokens = [];
+	const positions = [];
+	for (const rawRow of rows) {
+		const row = object(rawRow) ?? {};
+		const token = typeof row.token === "string" ? row.token : "";
+		tokens.push(token);
+		const alternatives = (Array.isArray(row.top_logprobs) ? row.top_logprobs : []).flatMap((value) => {
+			const item = object(value);
+			return item && typeof item.token === "string" && typeof item.logprob === "number" ? [{
+				token: item.token,
+				logprob: item.logprob
+			}] : [];
+		});
+		if (!alternatives.length && typeof row.logprob === "number") alternatives.push({
+			token,
+			logprob: row.logprob
+		});
+		positions.push(alternatives);
+	}
+	const rawUsage = object(body.usage) ?? {};
+	const promptDetails = object(rawUsage.prompt_tokens_details) ?? {};
+	const completionDetails = object(rawUsage.completion_tokens_details) ?? {};
+	const cached = Number(rawUsage.prompt_cache_hit_tokens ?? promptDetails.cached_tokens ?? 0) || 0;
+	const input = Number(rawUsage.prompt_tokens ?? 0) || 0;
+	return {
+		text: answer,
+		tokens,
+		positions,
+		scoringMode: "top-logprobs",
+		usage: {
+			calls: 1,
+			attempts: 1,
+			retries: 0,
+			inputTokens: Math.max(0, input - cached),
+			cachedInputTokens: cached,
+			outputTokens: Number(rawUsage.completion_tokens ?? 0) || 0,
+			reasoningTokens: Number(completionDetails.reasoning_tokens ?? 0) || 0
+		}
+	};
+}
+var TopLogprobCapabilityCache = class {
+	unsupported = /* @__PURE__ */ new Set();
+	isUnsupported(provider, model) {
+		return this.unsupported.has(provider + "\0" + model);
+	}
+	markUnsupported(provider, model) {
+		this.unsupported.add(provider + "\0" + model);
+	}
+};
+//#endregion
+//#region src/caller.ts
+function failureMessage(finish) {
+	if (finish.kind === "error" || finish.kind === "aborted") return finish.failure.message;
+	if (finish.kind === "max-tokens") return "verifier response reached max tokens before completing its answer";
+}
+async function delay(ms, signal) {
+	if (signal?.aborted) throw signal.reason;
+	await new Promise((resolve, reject) => {
+		const timer = setTimeout(resolve, ms);
+		const abort = () => {
+			clearTimeout(timer);
+			reject(signal?.reason);
+		};
+		signal?.addEventListener("abort", abort, { once: true });
+	});
+}
+function usage(attempts, value = {}) {
+	return {
+		calls: 1,
+		attempts,
+		retries: attempts - 1,
+		inputTokens: value.inputTokens ?? 0,
+		cachedInputTokens: (value.cacheReadTokens ?? 0) + (value.cacheWriteTokens ?? 0),
+		outputTokens: value.outputTokens ?? 0,
+		reasoningTokens: value.reasoningTokens ?? 0
+	};
+}
+async function callExplicitTag(config, prompt, signal, images) {
+	let attempt = 0;
+	while (true) {
+		attempt += 1;
+		const controller = new AbortController();
+		const timeout = setTimeout(() => controller.abort(/* @__PURE__ */ new Error("llm-verifier: request timed out")), config.timeoutMs);
+		const abort = () => controller.abort(signal?.reason);
+		signal?.addEventListener("abort", abort, { once: true });
+		try {
+			const content = [{
+				type: "text",
+				text: prompt
+			}];
+			for (const image of images ?? []) {
+				const ref = await config.attachments.saveImage({
+					data: image.data,
+					mediaType: image.mediaType
+				});
+				content.push({
+					type: "image",
+					attachment: ref
+				});
+			}
+			const messages = [createUserMessage({
+				content,
+				source: {
+					kind: "plugin",
+					plugin: "dsh-llm-verifier"
+				}
+			})];
+			const assembler = new BlockAssembler();
+			const options = deepFreeze({
+				provider: config.provider,
+				model: config.model,
+				...config.reasoningEffort ? { reasoningEffort: ReasoningEffortId(config.reasoningEffort) } : {},
+				messages,
+				maxTokens: config.maxTokens,
+				temperature: 1,
+				signal: controller.signal
+			});
+			for await (const chunk of config.llm.stream(options)) assembler.push(chunk);
+			const failed = failureMessage(assembler.finish);
+			if (failed !== void 0) throw new Error("llm-verifier: model call failed: " + failed);
+			const text = assembler.blocks().filter((block) => block.type === "text").map((block) => block.text).join("");
+			if (!text.trim()) throw new Error("llm-verifier: selected DSH model produced no text");
+			return {
+				text,
+				tokens: [],
+				positions: [],
+				scoringMode: "explicit-tag",
+				usage: usage(attempt, assembler.usage)
+			};
+		} catch (error) {
+			if (signal?.aborted) throw signal.reason;
+			if (attempt > config.maxRetries || !(error instanceof Error) || !/rate|quota|timeout|timed out|temporar|network|fetch|socket|5dd/i.test(error.message)) throw error;
+			await delay(Math.min(3e4, config.retryBaseDelayMs * 2 ** (attempt - 1) * (.8 + Math.random() * .4)), signal);
+		} finally {
+			clearTimeout(timeout);
+			signal?.removeEventListener("abort", abort);
+		}
+	}
+}
+var RequestLimiter = class {
+	limit;
+	active = 0;
+	queue = [];
+	constructor(limit) {
+		this.limit = limit;
+	}
+	async run(operation, signal) {
+		if (this.active >= this.limit) await new Promise((resolve, reject) => {
+			const enter = () => {
+				signal?.removeEventListener("abort", abort);
+				resolve();
+			};
+			const abort = () => {
+				const index = this.queue.indexOf(enter);
+				if (index >= 0) this.queue.splice(index, 1);
+				reject(signal?.reason);
+			};
+			this.queue.push(enter);
+			signal?.addEventListener("abort", abort, { once: true });
+		});
+		if (signal?.aborted) throw signal.reason;
+		this.active += 1;
+		try {
+			return await operation();
+		} finally {
+			this.active -= 1;
+			this.queue.shift()?.();
+		}
+	}
+};
+async function callAutomatic(config, prompt, signal, images) {
+	if (!config.topLogprobCapabilities.isUnsupported(config.provider, config.model)) {
+		const route = await resolveTopLogprobRoute(config.ctx, config.provider);
+		if (route !== void 0) try {
+			return await callTopLogprobs(route, config.model, prompt, config.maxTokens, config.reasoningEffort, signal, images);
+		} catch (error) {
+			if (!(error instanceof TopLogprobsUnsupportedError)) throw error;
+			config.topLogprobCapabilities.markUnsupported(config.provider, config.model);
+		}
+		else config.topLogprobCapabilities.markUnsupported(config.provider, config.model);
+	}
+	return callExplicitTag(config, prompt, signal, images);
+}
+async function callVerifier(config, prompt, signal, images) {
+	const invoke = () => callAutomatic(config, prompt, signal, images);
+	return config.limiter === void 0 ? invoke() : config.limiter.run(invoke, signal);
+}
+function addUsage(target, source) {
+	for (const key of [
+		"calls",
+		"attempts",
+		"retries",
+		"inputTokens",
+		"cachedInputTokens",
+		"outputTokens",
+		"reasoningTokens"
+	]) target[key] += source[key];
+}
+function emptyUsage() {
+	return {
+		calls: 0,
+		attempts: 0,
+		retries: 0,
+		inputTokens: 0,
+		cachedInputTokens: 0,
+		outputTokens: 0,
+		reasoningTokens: 0
+	};
+}
+//#endregion
+export { TopLogprobCapabilityCache as a, emptyUsage as i, addUsage as n, callVerifier as r, RequestLimiter as t };
+
+//# sourceMappingURL=caller-BXrUTAoL.js.map

package/lib/caller.js

@@ -1,2 +1,2 @@
-import { i as emptyUsage, n as addUsage, r as callVerifier, t as RequestLimiter } from "./caller-CGlgZ-Su.js";
+import { i as emptyUsage, n as addUsage, r as callVerifier, t as RequestLimiter } from "./caller-BXrUTAoL.js";
 export { RequestLimiter, addUsage, callVerifier, emptyUsage };