dsh-llm-verifier 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -15
- package/lib/client.js +39 -0
- package/lib/index.js +9 -0
- package/lib/types/client.js +2 -2
- package/lib/types/config.d.ts +2 -0
- package/lib/types/config.js +2 -1
- package/lib/types/index.js +6 -4
- package/package.json +22 -22
- package/src/client.tsx +6 -2
- package/src/config.ts +4 -1
- package/src/index.ts +5 -4
- package/lib/caller-BgqCctCh.js +0 -324
package/README.md
CHANGED
|
@@ -17,29 +17,50 @@
|
|
|
17
17
|
|
|
18
18
|
## 安装与启用 (Installation & Usage)
|
|
19
19
|
|
|
20
|
-
### 1. 使用
|
|
20
|
+
### 1. 使用 `dsh plugin` 安装
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
DeepSeek Harness（DSH）通过 profile 独立管理各个运行环境的插件依赖。请使用 `dsh plugin` 命令将插件安装至目标 profile（如 `web`）：
|
|
23
23
|
|
|
24
24
|
```bash
|
|
25
|
-
#
|
|
25
|
+
# 方式 A：从 npm 官方 Registry 安装（推荐）
|
|
26
26
|
dsh plugin --profile web add dsh-llm-verifier
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
> [!NOTE]
|
|
30
|
+
> `dsh plugin add` 安装成功后，DSH 会自动识别包内的 `dsh.bundle` 声明并完成插件层自动对齐（Reconcile），**无需手动修改任何配置文件**。
|
|
31
|
+
|
|
32
|
+
### 2. 启动与配置
|
|
33
|
+
|
|
34
|
+
启动 DSH Web 客户端：
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
dsh web
|
|
38
|
+
# 或
|
|
39
|
+
dsh --profile web
|
|
40
|
+
```
|
|
30
41
|
|
|
31
|
-
|
|
32
|
-
- **自定义 Patch Overlay（可选）**：若在自定义 Profile 清单中手动声明，可在 `cordis.patch.yml` 中配置：
|
|
33
|
-
```yaml
|
|
34
|
-
- insert:
|
|
35
|
-
- id: llm-verifier
|
|
36
|
-
name: 'dsh-llm-verifier'
|
|
37
|
-
```
|
|
38
|
-
- **配置面板**：启动 DSH 后打开 **`设置 → LLM Verifier`** 即可可视化配置裁判所使用的 Provider、Model、推理强度、并发限制与缓存策略。
|
|
42
|
+
启动后进入前端界面，打开 **`设置 → LLM Verifier`** 即可可视化配置裁判所使用的 Provider、Model、推理强度（Reasoning Effort）、最大并发与缓存策略。
|
|
39
43
|
|
|
40
|
-
### 3.
|
|
44
|
+
### 3. 常用管理命令
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# 更新插件至最新版本
|
|
48
|
+
dsh plugin --profile web update dsh-llm-verifier
|
|
41
49
|
|
|
42
|
-
|
|
50
|
+
# 卸载插件
|
|
51
|
+
dsh plugin --profile web remove dsh-llm-verifier
|
|
52
|
+
|
|
53
|
+
# 查看当前 Profile 已安装的插件与依赖列表
|
|
54
|
+
dsh plugin --profile web list
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### 4. 作为独立库引用（可选）
|
|
58
|
+
|
|
59
|
+
如果你在其它 TypeScript / JavaScript 项目中需要复用核心评分标尺与锦标赛算法，可直接作为普通 npm 依赖安装并引入：
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pnpm add dsh-llm-verifier
|
|
63
|
+
```
|
|
43
64
|
|
|
44
65
|
```typescript
|
|
45
66
|
import {
|
|
@@ -143,6 +164,7 @@ flowchart LR
|
|
|
143
164
|
|
|
144
165
|
| 配置项 | 说明 |
|
|
145
166
|
|---|---|
|
|
167
|
+
| **启用工具 (Enabled)** | 是否允许 Agent 调用 verifier 工具；关闭后调用会立即返回错误，不产生任何模型请求（设置页「工具开关」按钮可切换） |
|
|
146
168
|
| **供应商 (Provider)** | 从 DSH 当前已配置且可路由的 Provider 列表中选择 |
|
|
147
169
|
| **模型 (Model)** | 从所选 Provider 的模型目录中指定具体裁判模型 |
|
|
148
170
|
| **推理强度 (Reasoning Effort)** | 使用 Adapter 为该模型声明的思考强度，或保留模型默认值 |
|
|
@@ -215,4 +237,4 @@ software is provided “AS IS”, without warranty. See the upstream LICENSE
|
|
|
215
237
|
for the complete legal text.
|
|
216
238
|
```
|
|
217
239
|
|
|
218
|
-
衷心感谢上游项目的作者及贡献者开源了细粒度 A–T 奖励机制、Token 对数概率期望打分、任务进度验证、Bradley–Terry 软胜率模型以及 Probabilistic Pivot Tournament 锦标赛算法。本 DSH 插件将这一系列优秀成果移植并深度整合至 TypeScript 与 DeepSeek Harness 生态中；本插件不对上述理论算法声明原创所有权。
|
|
240
|
+
衷心感谢上游项目的作者及贡献者开源了细粒度 A–T 奖励机制、Token 对数概率期望打分、任务进度验证、Bradley–Terry 软胜率模型以及 Probabilistic Pivot Tournament 锦标赛算法。本 DSH 插件将这一系列优秀成果移植并深度整合至 TypeScript 与 DeepSeek Harness 生态中；本插件不对上述理论算法声明原创所有权。
|
package/lib/client.js
CHANGED
|
@@ -56,6 +56,7 @@ window.__ModuleLoader__.load({
|
|
|
56
56
|
function values(view) {
|
|
57
57
|
const v = record(view.value);
|
|
58
58
|
return {
|
|
59
|
+
enabled: v.enabled !== false,
|
|
59
60
|
provider: String(v.provider ?? ""),
|
|
60
61
|
model: String(v.model ?? ""),
|
|
61
62
|
...typeof v.reasoningEffort === "string" ? { reasoningEffort: v.reasoningEffort } : {},
|
|
@@ -178,6 +179,44 @@ window.__ModuleLoader__.load({
|
|
|
178
179
|
},
|
|
179
180
|
children: "选择任意已在 DSH「模型」页配置并启用的模型作为独立裁判。设置实时生效。"
|
|
180
181
|
})] }),
|
|
182
|
+
/* @__PURE__ */ (0, react_jsx_runtime.jsxs)("div", {
|
|
183
|
+
style: card,
|
|
184
|
+
children: [
|
|
185
|
+
/* @__PURE__ */ (0, react_jsx_runtime.jsxs)("div", {
|
|
186
|
+
style: sectionTitle,
|
|
187
|
+
children: [/* @__PURE__ */ (0, react_jsx_runtime.jsx)(_deepseek_ai_dsh_client_ui_primitives.StateDot, { state: draft.enabled ? "done" : "error" }), /* @__PURE__ */ (0, react_jsx_runtime.jsx)("strong", { children: "工具开关" })]
|
|
188
|
+
}),
|
|
189
|
+
/* @__PURE__ */ (0, react_jsx_runtime.jsxs)("div", {
|
|
190
|
+
style: row,
|
|
191
|
+
children: [/* @__PURE__ */ (0, react_jsx_runtime.jsx)(Label, {
|
|
192
|
+
title: "启用 Verifier 工具",
|
|
193
|
+
help: draft.enabled ? "四个 verifier 工具可被 Agent 调用，每次调用会向裁判模型发起请求。" : "已停用：Agent 调用 verifier 工具会立即返回错误，不产生任何模型请求。"
|
|
194
|
+
}), /* @__PURE__ */ (0, react_jsx_runtime.jsx)("button", {
|
|
195
|
+
onClick: () => patch("enabled", !draft.enabled),
|
|
196
|
+
style: {
|
|
197
|
+
width: 80,
|
|
198
|
+
height: 32,
|
|
199
|
+
borderRadius: 16,
|
|
200
|
+
border: "1px solid var(--dsw-alias-border-l2, rgba(255,255,255,0.16))",
|
|
201
|
+
background: draft.enabled ? "var(--dsw-success, #2f9e5b)" : "var(--dsw-surface-sunken)",
|
|
202
|
+
color: "#fff",
|
|
203
|
+
cursor: "pointer",
|
|
204
|
+
fontSize: 13,
|
|
205
|
+
fontWeight: 600
|
|
206
|
+
},
|
|
207
|
+
children: draft.enabled ? "已启用" : "已停用"
|
|
208
|
+
})]
|
|
209
|
+
}),
|
|
210
|
+
!draft.enabled && /* @__PURE__ */ (0, react_jsx_runtime.jsx)("div", {
|
|
211
|
+
style: {
|
|
212
|
+
padding: "0 0 14px",
|
|
213
|
+
fontSize: 12,
|
|
214
|
+
color: "var(--dsw-state-warn-primary, #d9a441)"
|
|
215
|
+
},
|
|
216
|
+
children: "⚠ 当前停用 verifier_compare / verifier_select / verifier_track / verifier_current_session 四个工具"
|
|
217
|
+
})
|
|
218
|
+
]
|
|
219
|
+
}),
|
|
181
220
|
/* @__PURE__ */ (0, react_jsx_runtime.jsxs)("div", {
|
|
182
221
|
style: card,
|
|
183
222
|
children: [
|
package/lib/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
|
9
9
|
//#region src/config.ts
|
|
10
10
|
const VERIFIER_SETTINGS_NAMESPACE = settingsNamespace("llm-verifier");
|
|
11
11
|
const Config = z.object({
|
|
12
|
+
enabled: z.boolean().default(true),
|
|
12
13
|
provider: z.string().default("deepseek-official"),
|
|
13
14
|
model: z.string().default("deepseek-v4-flash"),
|
|
14
15
|
reasoningEffort: z.string(),
|
|
@@ -44,6 +45,7 @@ function resolveConfig(config = {}) {
|
|
|
44
45
|
if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every((value) => Number.isFinite(value) && value >= 0)) throw new Error("llm-verifier: estimated token prices must be finite non-negative numbers");
|
|
45
46
|
const reasoningEffort = config.reasoningEffort?.trim();
|
|
46
47
|
return {
|
|
48
|
+
enabled: config.enabled ?? true,
|
|
47
49
|
provider,
|
|
48
50
|
model,
|
|
49
51
|
...reasoningEffort ? { reasoningEffort } : {},
|
|
@@ -673,6 +675,9 @@ function apply(ctx, config = {}) {
|
|
|
673
675
|
provider: selected.provider,
|
|
674
676
|
model: selected.model
|
|
675
677
|
});
|
|
678
|
+
const requireEnabled = () => {
|
|
679
|
+
if (!current().enabled) throw new Error("llm-verifier: verifier tools are disabled — enable them in Settings → LLM Verifier");
|
|
680
|
+
};
|
|
676
681
|
ctx.tools.register(defineTool({
|
|
677
682
|
name: "verifier_compare",
|
|
678
683
|
description: "Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.",
|
|
@@ -740,6 +745,7 @@ function apply(ctx, config = {}) {
|
|
|
740
745
|
},
|
|
741
746
|
timeoutMs: entry.timeoutMs * 20,
|
|
742
747
|
async execute(args, exec) {
|
|
748
|
+
requireEnabled();
|
|
743
749
|
const { verifier, selected } = await engine();
|
|
744
750
|
return {
|
|
745
751
|
...await verifier.compare({
|
|
@@ -825,6 +831,7 @@ function apply(ctx, config = {}) {
|
|
|
825
831
|
},
|
|
826
832
|
timeoutMs: entry.timeoutMs * 100,
|
|
827
833
|
async execute(args, exec) {
|
|
834
|
+
requireEnabled();
|
|
828
835
|
const { verifier, selected } = await engine();
|
|
829
836
|
return {
|
|
830
837
|
...await verifier.select({
|
|
@@ -901,6 +908,7 @@ function apply(ctx, config = {}) {
|
|
|
901
908
|
},
|
|
902
909
|
timeoutMs: entry.timeoutMs * 20,
|
|
903
910
|
async execute(args, exec) {
|
|
911
|
+
requireEnabled();
|
|
904
912
|
const { verifier, selected } = await engine();
|
|
905
913
|
return {
|
|
906
914
|
...await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, "repeats"), exec.signal, await images(args.images, exec.signal)),
|
|
@@ -986,6 +994,7 @@ function apply(ctx, config = {}) {
|
|
|
986
994
|
},
|
|
987
995
|
timeoutMs: entry.timeoutMs * 20,
|
|
988
996
|
async execute(args, exec) {
|
|
997
|
+
requireEnabled();
|
|
989
998
|
const agent = exec.agent ?? ctx.agents.currentInitiator();
|
|
990
999
|
if (agent === void 0) throw new Error("llm-verifier: verifier_current_session requires an agent-owned tool call");
|
|
991
1000
|
const extracted = await extractSession(agent, async (ref) => {
|
package/lib/types/client.js
CHANGED
|
@@ -8,7 +8,7 @@ const sectionTitle = { display: 'flex', gap: 10, alignItems: 'center', padding:
|
|
|
8
8
|
const row = { display: 'grid', gridTemplateColumns: 'minmax(150px, 1fr) minmax(220px, 1.4fr)', gap: 18, alignItems: 'center', padding: '14px 0', borderBottom: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' };
|
|
9
9
|
const selectStyle = { width: '100%', minHeight: 38, padding: '0 12px', borderRadius: 10, color: 'var(--dsw-text-primary)', background: 'var(--dsw-surface-sunken)', border: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' };
|
|
10
10
|
function record(value) { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : {}; }
|
|
11
|
-
function values(view) { const v = record(view.value); return { provider: String(v.provider ?? ''), model: String(v.model ?? ''), ...(typeof v.reasoningEffort === 'string' ? { reasoningEffort: v.reasoningEffort } : {}), maxTokens: Number(v.maxTokens ?? 32768), maxConcurrency: Number(v.maxConcurrency ?? 8), maxRetries: Number(v.maxRetries ?? 3), timeoutMs: Number(v.timeoutMs ?? 300000), cacheMaxEntries: Number(v.cacheMaxEntries ?? 10000), estimatedInputUsdPerMillion: Number(v.estimatedInputUsdPerMillion ?? 0), estimatedOutputUsdPerMillion: Number(v.estimatedOutputUsdPerMillion ?? 0) }; }
|
|
11
|
+
function values(view) { const v = record(view.value); return { enabled: v.enabled !== false, provider: String(v.provider ?? ''), model: String(v.model ?? ''), ...(typeof v.reasoningEffort === 'string' ? { reasoningEffort: v.reasoningEffort } : {}), maxTokens: Number(v.maxTokens ?? 32768), maxConcurrency: Number(v.maxConcurrency ?? 8), maxRetries: Number(v.maxRetries ?? 3), timeoutMs: Number(v.timeoutMs ?? 300000), cacheMaxEntries: Number(v.cacheMaxEntries ?? 10000), estimatedInputUsdPerMillion: Number(v.estimatedInputUsdPerMillion ?? 0), estimatedOutputUsdPerMillion: Number(v.estimatedOutputUsdPerMillion ?? 0) }; }
|
|
12
12
|
function message(error) { return error instanceof Error ? error.message : String(error); }
|
|
13
13
|
function Label({ title, help }) { return _jsxs("div", { children: [_jsx("div", { style: { fontWeight: 600 }, children: title }), _jsx("div", { style: { fontSize: 12, color: 'var(--dsw-text-secondary)', marginTop: 3 }, children: help })] }); }
|
|
14
14
|
function VerifierSettings({ api }) {
|
|
@@ -59,7 +59,7 @@ function VerifierSettings({ api }) {
|
|
|
59
59
|
if (!loaded || !draft)
|
|
60
60
|
return _jsxs("div", { style: shell, children: [_jsx("h2", { children: "LLM Verifier" }), _jsx("p", { children: error ?? '正在读取 DSH 模型和设置…' }), error && _jsx(Button, { onClick: () => void load(), children: "\u91CD\u8BD5" })] });
|
|
61
61
|
const numeric = (key, min = 0) => _jsx(Input, { type: "number", min: min, value: String(draft[key]), onChange: e => patch(key, Number(e.target.value)) });
|
|
62
|
-
return _jsxs("div", { style: shell, children: [_jsxs("div", { children: [_jsx("h2", { style: { margin: '0 0 6px' }, children: "LLM Verifier" }), _jsx("p", { style: { margin: 0, color: 'var(--dsw-text-secondary)' }, children: "\u9009\u62E9\u4EFB\u610F\u5DF2\u5728 DSH\u300C\u6A21\u578B\u300D\u9875\u914D\u7F6E\u5E76\u542F\u7528\u7684\u6A21\u578B\u4F5C\u4E3A\u72EC\u7ACB\u88C1\u5224\u3002\u8BBE\u7F6E\u5B9E\u65F6\u751F\u6548\u3002" })] }), _jsxs("div", { style: card, children: [_jsxs("div", { style: sectionTitle, children: [_jsx(StateDot, { state: "done" }), _jsx("strong", { children: "\u88C1\u5224\u6A21\u578B" })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u4F9B\u5E94\u5546", help: "\u53EA\u663E\u793A\u5F53\u524D DSH \u4E2D\u53EF\u8DEF\u7531\u7684\u4F9B\u5E94\u5546" }), _jsx("select", { style: selectStyle, value: draft.provider, onChange: e => { const provider = e.target.value; const first = loaded.groups.find(g => g.id === provider)?.models[0]; setDraft({ ...draft, provider, ...(first ? { model: first.id, reasoningEffort: first.reasoning?.defaultEffort } : {}) }); }, children: loaded.groups.map(g => _jsxs("option", { value: g.id, children: [g.name, " \u00B7 ", g.id] }, g.id)) })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6A21\u578B", help: "\u6A21\u578B\u76EE\u5F55\u6765\u81EA DSH adapter\uFF0C\u9009\u62E9\u7ED3\u679C\u4F1A\u6301\u4E45\u5316" }), _jsx("select", { style: selectStyle, value: draft.model, onChange: e => { const model = e.target.value; const found = models.find(m => m.id === model); setDraft({ ...draft, model, ...(found?.reasoning?.defaultEffort ? { reasoningEffort: found.reasoning.defaultEffort } : { reasoningEffort: undefined }) }); }, children: models.map(m => _jsxs("option", { value: m.id, children: [m.name, " \u00B7 ", m.id] }, m.id)) })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u63A8\u7406\u5F3A\u5EA6", help: "\u7531\u6240\u9009\u6A21\u578B adapter \u58F0\u660E\uFF1B\u7559\u7A7A\u4F7F\u7528\u6A21\u578B\u9ED8\u8BA4\u503C" }), _jsxs("select", { style: selectStyle, value: draft.reasoningEffort ?? '', onChange: e => patch('reasoningEffort', e.target.value || undefined), children: [_jsx("option", { value: "", children: "\u6A21\u578B\u9ED8\u8BA4" }), efforts.map(e => _jsx("option", { value: e.id, children: e.name }, e.id))] })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u8F93\u51FA Token", help: "\u6BCF\u4E2A\u88C1\u5224\u8BF7\u6C42\u7684\u8F93\u51FA\u4E0A\u9650" }), numeric('maxTokens', 1)] })] }), _jsxs("div", { style: card, children: [_jsx("div", { style: sectionTitle, children: _jsx("strong", { children: "\u6267\u884C\u63A7\u5236" }) }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u5E76\u53D1", help: "\u6240\u6709 verifier \u5DE5\u5177\u5171\u4EAB\u7684\u8BF7\u6C42\u5E76\u53D1\u4E0A\u9650" }), numeric('maxConcurrency', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u591A\u91CD\u8BD5", help: "\u77ED\u6682\u7F51\u7EDC\u3001\u9650\u6D41\u548C\u670D\u52A1\u7AEF\u9519\u8BEF\u7684\u91CD\u8BD5\u6B21\u6570" }), numeric('maxRetries', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8BF7\u6C42\u8D85\u65F6\uFF08\u6BEB\u79D2\uFF09", help: "\u5355\u4E2A\u6A21\u578B\u8BF7\u6C42\u7684\u8D85\u65F6\u65F6\u95F4" }), numeric('timeoutMs', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u7F13\u5B58\u6761\u76EE\u4E0A\u9650", help: "\u6301\u4E45\u8BC4\u5206\u7F13\u5B58\u4FDD\u7559\u7684\u6700\u5927\u6761\u76EE\u6570" }), numeric('cacheMaxEntries', 1)] })] }), _jsxs("div", { style: card, children: [_jsx("div", { style: sectionTitle, children: _jsx("strong", { children: "\u8D39\u7528\u4F30\u7B97\uFF08\u6BCF\u767E\u4E07 Token\uFF0CUSD\uFF09" }) }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8F93\u5165\u4EF7\u683C", help: "\u4EC5\u7528\u4E8E\u7ED3\u679C\u4E2D\u7684 estimatedCostUsd" }), numeric('estimatedInputUsdPerMillion', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8F93\u51FA\u4EF7\u683C", help: "\u4EC5\u7528\u4E8E\u7ED3\u679C\u4E2D\u7684 estimatedCostUsd" }), numeric('estimatedOutputUsdPerMillion', 0)] })] }), loaded.failures.length > 0 && _jsxs("div", { style: { ...card, borderColor: 'var(--dsw-alias-state-warn-primary, #d9a441)', paddingBottom: 16 }, children: [_jsx("strong", { children: "\u90E8\u5206\u6A21\u578B\u76EE\u5F55\u8BFB\u53D6\u5931\u8D25" }), loaded.failures.map(x => _jsx("div", { children: x }, x))] }), error && _jsx("div", { style: { color: 'var(--dsw-danger)' }, children: error }), saved && _jsx("div", { style: { color: 'var(--dsw-success)' }, children: "\u5DF2\u4FDD\u5B58\uFF0C\u540E\u7EED verifier \u8C03\u7528\u5C06\u4F7F\u7528\u65B0\u8BBE\u7F6E\u3002" }), _jsxs("div", { style: { display: 'flex', gap: 10 }, children: [_jsx(Button, { disabled: busy || !loaded.writable, onClick: () => void save(), children: busy ? '保存中…' : '保存设置' }), _jsx(Button, { variant: "outline", disabled: busy, onClick: () => void load(), children: "\u91CD\u65B0\u8F7D\u5165" })] })] });
|
|
62
|
+
return _jsxs("div", { style: shell, children: [_jsxs("div", { children: [_jsx("h2", { style: { margin: '0 0 6px' }, children: "LLM Verifier" }), _jsx("p", { style: { margin: 0, color: 'var(--dsw-text-secondary)' }, children: "\u9009\u62E9\u4EFB\u610F\u5DF2\u5728 DSH\u300C\u6A21\u578B\u300D\u9875\u914D\u7F6E\u5E76\u542F\u7528\u7684\u6A21\u578B\u4F5C\u4E3A\u72EC\u7ACB\u88C1\u5224\u3002\u8BBE\u7F6E\u5B9E\u65F6\u751F\u6548\u3002" })] }), _jsxs("div", { style: card, children: [_jsxs("div", { style: sectionTitle, children: [_jsx(StateDot, { state: draft.enabled ? 'done' : 'error' }), _jsx("strong", { children: "\u5DE5\u5177\u5F00\u5173" })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u542F\u7528 Verifier \u5DE5\u5177", help: draft.enabled ? '四个 verifier 工具可被 Agent 调用，每次调用会向裁判模型发起请求。' : '已停用：Agent 调用 verifier 工具会立即返回错误，不产生任何模型请求。' }), _jsx("button", { onClick: () => patch('enabled', !draft.enabled), style: { width: 80, height: 32, borderRadius: 16, border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,0.16))', background: draft.enabled ? 'var(--dsw-success, #2f9e5b)' : 'var(--dsw-surface-sunken)', color: '#fff', cursor: 'pointer', fontSize: 13, fontWeight: 600 }, children: draft.enabled ? '已启用' : '已停用' })] }), !draft.enabled && _jsx("div", { style: { padding: '0 0 14px', fontSize: 12, color: 'var(--dsw-state-warn-primary, #d9a441)' }, children: "\u26A0 \u5F53\u524D\u505C\u7528 verifier_compare / verifier_select / verifier_track / verifier_current_session \u56DB\u4E2A\u5DE5\u5177" })] }), _jsxs("div", { style: card, children: [_jsxs("div", { style: sectionTitle, children: [_jsx(StateDot, { state: "done" }), _jsx("strong", { children: "\u88C1\u5224\u6A21\u578B" })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u4F9B\u5E94\u5546", help: "\u53EA\u663E\u793A\u5F53\u524D DSH \u4E2D\u53EF\u8DEF\u7531\u7684\u4F9B\u5E94\u5546" }), _jsx("select", { style: selectStyle, value: draft.provider, onChange: e => { const provider = e.target.value; const first = loaded.groups.find(g => g.id === provider)?.models[0]; setDraft({ ...draft, provider, ...(first ? { model: first.id, reasoningEffort: first.reasoning?.defaultEffort } : {}) }); }, children: loaded.groups.map(g => _jsxs("option", { value: g.id, children: [g.name, " \u00B7 ", g.id] }, g.id)) })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6A21\u578B", help: "\u6A21\u578B\u76EE\u5F55\u6765\u81EA DSH adapter\uFF0C\u9009\u62E9\u7ED3\u679C\u4F1A\u6301\u4E45\u5316" }), _jsx("select", { style: selectStyle, value: draft.model, onChange: e => { const model = e.target.value; const found = models.find(m => m.id === model); setDraft({ ...draft, model, ...(found?.reasoning?.defaultEffort ? { reasoningEffort: found.reasoning.defaultEffort } : { reasoningEffort: undefined }) }); }, children: models.map(m => _jsxs("option", { value: m.id, children: [m.name, " \u00B7 ", m.id] }, m.id)) })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u63A8\u7406\u5F3A\u5EA6", help: "\u7531\u6240\u9009\u6A21\u578B adapter \u58F0\u660E\uFF1B\u7559\u7A7A\u4F7F\u7528\u6A21\u578B\u9ED8\u8BA4\u503C" }), _jsxs("select", { style: selectStyle, value: draft.reasoningEffort ?? '', onChange: e => patch('reasoningEffort', e.target.value || undefined), children: [_jsx("option", { value: "", children: "\u6A21\u578B\u9ED8\u8BA4" }), efforts.map(e => _jsx("option", { value: e.id, children: e.name }, e.id))] })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u8F93\u51FA Token", help: "\u6BCF\u4E2A\u88C1\u5224\u8BF7\u6C42\u7684\u8F93\u51FA\u4E0A\u9650" }), numeric('maxTokens', 1)] })] }), _jsxs("div", { style: card, children: [_jsx("div", { style: sectionTitle, children: _jsx("strong", { children: "\u6267\u884C\u63A7\u5236" }) }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u5E76\u53D1", help: "\u6240\u6709 verifier \u5DE5\u5177\u5171\u4EAB\u7684\u8BF7\u6C42\u5E76\u53D1\u4E0A\u9650" }), numeric('maxConcurrency', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u591A\u91CD\u8BD5", help: "\u77ED\u6682\u7F51\u7EDC\u3001\u9650\u6D41\u548C\u670D\u52A1\u7AEF\u9519\u8BEF\u7684\u91CD\u8BD5\u6B21\u6570" }), numeric('maxRetries', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8BF7\u6C42\u8D85\u65F6\uFF08\u6BEB\u79D2\uFF09", help: "\u5355\u4E2A\u6A21\u578B\u8BF7\u6C42\u7684\u8D85\u65F6\u65F6\u95F4" }), numeric('timeoutMs', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u7F13\u5B58\u6761\u76EE\u4E0A\u9650", help: "\u6301\u4E45\u8BC4\u5206\u7F13\u5B58\u4FDD\u7559\u7684\u6700\u5927\u6761\u76EE\u6570" }), numeric('cacheMaxEntries', 1)] })] }), _jsxs("div", { style: card, children: [_jsx("div", { style: sectionTitle, children: _jsx("strong", { children: "\u8D39\u7528\u4F30\u7B97\uFF08\u6BCF\u767E\u4E07 Token\uFF0CUSD\uFF09" }) }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8F93\u5165\u4EF7\u683C", help: "\u4EC5\u7528\u4E8E\u7ED3\u679C\u4E2D\u7684 estimatedCostUsd" }), numeric('estimatedInputUsdPerMillion', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8F93\u51FA\u4EF7\u683C", help: "\u4EC5\u7528\u4E8E\u7ED3\u679C\u4E2D\u7684 estimatedCostUsd" }), numeric('estimatedOutputUsdPerMillion', 0)] })] }), loaded.failures.length > 0 && _jsxs("div", { style: { ...card, borderColor: 'var(--dsw-alias-state-warn-primary, #d9a441)', paddingBottom: 16 }, children: [_jsx("strong", { children: "\u90E8\u5206\u6A21\u578B\u76EE\u5F55\u8BFB\u53D6\u5931\u8D25" }), loaded.failures.map(x => _jsx("div", { children: x }, x))] }), error && _jsx("div", { style: { color: 'var(--dsw-danger)' }, children: error }), saved && _jsx("div", { style: { color: 'var(--dsw-success)' }, children: "\u5DF2\u4FDD\u5B58\uFF0C\u540E\u7EED verifier \u8C03\u7528\u5C06\u4F7F\u7528\u65B0\u8BBE\u7F6E\u3002" }), _jsxs("div", { style: { display: 'flex', gap: 10 }, children: [_jsx(Button, { disabled: busy || !loaded.writable, onClick: () => void save(), children: busy ? '保存中…' : '保存设置' }), _jsx(Button, { variant: "outline", disabled: busy, onClick: () => void load(), children: "\u91CD\u65B0\u8F7D\u5165" })] })] });
|
|
63
63
|
}
|
|
64
64
|
export const inject = ['slots', 'connection'];
|
|
65
65
|
export function apply(ctx) { const connection = ctx.get('connection'); ctx.slots.inject('settings.section', () => ctx.slots.register({ name: 'settings.section', id: 'llm-verifier', order: 35, label: 'LLM Verifier', inject: () => ({ api: connection.api }) }, VerifierSettings)); }
|
package/lib/types/config.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { Context } from '@deepseek-ai/cordis';
|
|
|
2
2
|
import z from 'schemastery';
|
|
3
3
|
export declare const VERIFIER_SETTINGS_NAMESPACE: import("@deepseek-ai/dsh-settings").SettingsNamespace;
|
|
4
4
|
export interface Config {
|
|
5
|
+
enabled?: boolean;
|
|
5
6
|
provider?: string;
|
|
6
7
|
model?: string;
|
|
7
8
|
reasoningEffort?: string;
|
|
@@ -16,6 +17,7 @@ export interface Config {
|
|
|
16
17
|
estimatedOutputUsdPerMillion?: number;
|
|
17
18
|
}
|
|
18
19
|
export interface ResolvedConfig {
|
|
20
|
+
enabled: boolean;
|
|
19
21
|
provider: string;
|
|
20
22
|
model: string;
|
|
21
23
|
reasoningEffort?: string;
|
package/lib/types/config.js
CHANGED
|
@@ -2,6 +2,7 @@ import { installSettingsSection, settingsNamespace } from '@deepseek-ai/dsh-sett
|
|
|
2
2
|
import z from 'schemastery';
|
|
3
3
|
export const VERIFIER_SETTINGS_NAMESPACE = settingsNamespace('llm-verifier');
|
|
4
4
|
export const Config = z.object({
|
|
5
|
+
enabled: z.boolean().default(true),
|
|
5
6
|
provider: z.string().default('deepseek-official'),
|
|
6
7
|
model: z.string().default('deepseek-v4-flash'),
|
|
7
8
|
reasoningEffort: z.string(),
|
|
@@ -43,7 +44,7 @@ export function resolveConfig(config = {}) {
|
|
|
43
44
|
if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every(value => Number.isFinite(value) && value >= 0))
|
|
44
45
|
throw new Error('llm-verifier: estimated token prices must be finite non-negative numbers');
|
|
45
46
|
const reasoningEffort = config.reasoningEffort?.trim();
|
|
46
|
-
return { provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values };
|
|
47
|
+
return { enabled: config.enabled ?? true, provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values };
|
|
47
48
|
}
|
|
48
49
|
export function installVerifierSettings(ctx, entry, onChange) {
|
|
49
50
|
let source = () => entry;
|
package/lib/types/index.js
CHANGED
|
@@ -33,10 +33,12 @@ export function apply(ctx, config = {}) {
|
|
|
33
33
|
};
|
|
34
34
|
const images = (values, signal) => loadVerifierImages(values, signal);
|
|
35
35
|
const route = (selected) => ({ provider: selected.provider, model: selected.model });
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
ctx.tools.register(defineTool({ name: '
|
|
39
|
-
ctx.tools.register(defineTool({ name: '
|
|
36
|
+
const requireEnabled = () => { if (!current().enabled)
|
|
37
|
+
throw new Error('llm-verifier: verifier tools are disabled — enable them in Settings → LLM Verifier'); };
|
|
38
|
+
ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: args.problem, candidateA: args.candidate_a, candidateB: args.candidate_b, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) }; } }));
|
|
39
|
+
ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Uses the configured DSH verifier model and the O(Nk) Probabilistic Pivot Tournament.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { requireEnabled(); const { verifier, selected } = await engine(); const result = await verifier.select({ problem: args.problem, candidates: args.candidates, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), pivots: positive(args.pivots, 2, 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) }; } }));
|
|
40
|
+
ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Do not use for a single completed answer or invent checkpoints that were not supplied by the task history. Trusts observed output rather than narration.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const { verifier, selected } = await engine(); const result = await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { ...result, ...route(selected) }; } }));
|
|
41
|
+
ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Use autonomously near the end of a non-trivial coding, debugging, migration, deployment, or operations task when independent completion verification would materially reduce risk and the session contains real tool evidence. Do not use for routine conversation, simple factual answers, or every turn. Extracts the current DSH session, applies secret redaction, bounds and truncation, then sends the evidence to the verifier model selected in Settings.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = exec.agent ?? ctx.agents.currentInitiator(); if (agent === undefined)
|
|
40
42
|
throw new Error('llm-verifier: verifier_current_session requires an agent-owned tool call'); const extracted = await extractSession(agent, async (ref) => { const stored = await ctx.attachments.readImage(ref, exec.signal); return { data: stored.data, mediaType: stored.ref.mediaType }; }, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars }); const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(args.repeats, 2, 'repeats'), images: extracted.images }, exec.signal); return { sessionId: extracted.sessionId, problem: extracted.problem, score: result.scoreA, baselineScore: result.scoreB, winner: result.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: result.calls, stats: result.stats, ...route(selected) }; } }));
|
|
41
43
|
}
|
|
42
44
|
//# sourceMappingURL=index.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-llm-verifier",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Configurable DSH-native LLM verifier with a Web settings page",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -59,40 +59,40 @@
|
|
|
59
59
|
},
|
|
60
60
|
"peerDependencies": {
|
|
61
61
|
"@deepseek-ai/cordis": "^4.0.1",
|
|
62
|
-
"@deepseek-ai/dsh-tools": "^0.1.0-rc.
|
|
63
|
-
"@deepseek-ai/dsh-agent": "^0.1.0-rc.
|
|
64
|
-
"@deepseek-ai/dsh-session": "^0.1.0-rc.
|
|
65
|
-
"@deepseek-ai/dsh-llm": "^0.1.0-rc.
|
|
66
|
-
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.
|
|
67
|
-
"@deepseek-ai/dsh-settings": "^0.1.0-rc.
|
|
68
|
-
"@deepseek-ai/dsh-credentials": "^0.1.0-rc.
|
|
62
|
+
"@deepseek-ai/dsh-tools": "^0.1.0-rc.8",
|
|
63
|
+
"@deepseek-ai/dsh-agent": "^0.1.0-rc.8",
|
|
64
|
+
"@deepseek-ai/dsh-session": "^0.1.0-rc.8",
|
|
65
|
+
"@deepseek-ai/dsh-llm": "^0.1.0-rc.8",
|
|
66
|
+
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.8",
|
|
67
|
+
"@deepseek-ai/dsh-settings": "^0.1.0-rc.8",
|
|
68
|
+
"@deepseek-ai/dsh-credentials": "^0.1.0-rc.8"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
71
|
"schemastery": "^3.18.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@deepseek-ai/cordis": "^4.0.1",
|
|
75
|
-
"@deepseek-ai/dsh-tools": "^0.1.0-rc.
|
|
76
|
-
"@deepseek-ai/dsh-agent": "^0.1.0-rc.
|
|
77
|
-
"@deepseek-ai/dsh-session": "^0.1.0-rc.
|
|
78
|
-
"@deepseek-ai/dsh-llm": "^0.1.0-rc.
|
|
79
|
-
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.
|
|
75
|
+
"@deepseek-ai/dsh-tools": "^0.1.0-rc.8",
|
|
76
|
+
"@deepseek-ai/dsh-agent": "^0.1.0-rc.8",
|
|
77
|
+
"@deepseek-ai/dsh-session": "^0.1.0-rc.8",
|
|
78
|
+
"@deepseek-ai/dsh-llm": "^0.1.0-rc.8",
|
|
79
|
+
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.8",
|
|
80
80
|
"@types/node": "^22.20.0",
|
|
81
81
|
"tsdown": "0.22.2",
|
|
82
82
|
"typescript": "~5.7.2",
|
|
83
83
|
"vitest": "^3.0.0",
|
|
84
|
-
"@deepseek-ai/dsh-settings": "^0.1.0-rc.
|
|
85
|
-
"@deepseek-ai/dsh-api-remotes": "^0.1.0-rc.
|
|
86
|
-
"@deepseek-ai/dsh-client-connection": "^0.1.0-rc.
|
|
87
|
-
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.
|
|
88
|
-
"@deepseek-ai/dsh-client-ui-settings": "^0.1.0-rc.
|
|
89
|
-
"@deepseek-ai/dsh-client-ui-slots": "^0.1.0-rc.
|
|
90
|
-
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.
|
|
84
|
+
"@deepseek-ai/dsh-settings": "^0.1.0-rc.8",
|
|
85
|
+
"@deepseek-ai/dsh-api-remotes": "^0.1.0-rc.8",
|
|
86
|
+
"@deepseek-ai/dsh-client-connection": "^0.1.0-rc.8",
|
|
87
|
+
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.8",
|
|
88
|
+
"@deepseek-ai/dsh-client-ui-settings": "^0.1.0-rc.8",
|
|
89
|
+
"@deepseek-ai/dsh-client-ui-slots": "^0.1.0-rc.8",
|
|
90
|
+
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.8",
|
|
91
91
|
"@deepseek-ai/dsh-client-web-react": "^0.1.0-rc.7",
|
|
92
|
-
"@deepseek-ai/dsh-client-locale": "^0.1.0-rc.
|
|
92
|
+
"@deepseek-ai/dsh-client-locale": "^0.1.0-rc.8",
|
|
93
93
|
"react": "^18.2.0",
|
|
94
94
|
"@types/react": "~18.3.1",
|
|
95
|
-
"@deepseek-ai/dsh-credentials": "^0.1.0-rc.
|
|
95
|
+
"@deepseek-ai/dsh-credentials": "^0.1.0-rc.8"
|
|
96
96
|
},
|
|
97
97
|
"files": [
|
|
98
98
|
"lib/**/*.js",
|
package/src/client.tsx
CHANGED
|
@@ -6,7 +6,7 @@ import { Button, Input, StateDot } from '@deepseek-ai/dsh-client-ui-primitives'
|
|
|
6
6
|
import { useEffect, useMemo, useState } from 'react'
|
|
7
7
|
|
|
8
8
|
const NS = 'llm-verifier'
|
|
9
|
-
interface Values { provider: string; model: string; reasoningEffort?: string; maxTokens: number; maxConcurrency: number; maxRetries: number; timeoutMs: number; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number }
|
|
9
|
+
interface Values { enabled: boolean; provider: string; model: string; reasoningEffort?: string; maxTokens: number; maxConcurrency: number; maxRetries: number; timeoutMs: number; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number }
|
|
10
10
|
interface Loaded { groups: ModelProviderGroup[]; settings: SettingsNamespaceView; writable: boolean; failures: string[] }
|
|
11
11
|
const shell: React.CSSProperties = { display: 'flex', flexDirection: 'column', gap: 18, padding: '8px 4px 32px', color: 'var(--dsw-text-primary)' }
|
|
12
12
|
const card: React.CSSProperties = { display: 'flex', flexDirection: 'column', gap: 0, padding: '16px 16px 0', border: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))', borderRadius: 12, background: 'var(--dsw-alias-bg-module, rgba(20, 31, 57, 0.42))', overflow: 'hidden' }
|
|
@@ -14,7 +14,7 @@ const sectionTitle: React.CSSProperties = { display: 'flex', gap: 10, alignItems
|
|
|
14
14
|
const row: React.CSSProperties = { display: 'grid', gridTemplateColumns: 'minmax(150px, 1fr) minmax(220px, 1.4fr)', gap: 18, alignItems: 'center', padding: '14px 0', borderBottom: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' }
|
|
15
15
|
const selectStyle: React.CSSProperties = { width: '100%', minHeight: 38, padding: '0 12px', borderRadius: 10, color: 'var(--dsw-text-primary)', background: 'var(--dsw-surface-sunken)', border: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' }
|
|
16
16
|
function record(value: unknown): Record<string, unknown> { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {} }
|
|
17
|
-
function values(view: SettingsNamespaceView): Values { const v=record(view.value); return { provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),timeoutMs:Number(v.timeoutMs??300000),cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0) } }
|
|
17
|
+
function values(view: SettingsNamespaceView): Values { const v=record(view.value); return { enabled:v.enabled!==false,provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),timeoutMs:Number(v.timeoutMs??300000),cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0) } }
|
|
18
18
|
function message(error: unknown): string { return error instanceof Error ? error.message : String(error) }
|
|
19
19
|
function Label({title,help}:{title:string;help:string}) { return <div><div style={{fontWeight:600}}>{title}</div><div style={{fontSize:12,color:'var(--dsw-text-secondary)',marginTop:3}}>{help}</div></div> }
|
|
20
20
|
|
|
@@ -30,6 +30,10 @@ function VerifierSettings({ api }:{api:any}) {
|
|
|
30
30
|
const numeric=(key:keyof Values,min=0)=><Input type="number" min={min} value={String(draft[key])} onChange={e=>patch(key,Number(e.target.value) as never)} />
|
|
31
31
|
return <div style={shell}>
|
|
32
32
|
<div><h2 style={{margin:'0 0 6px'}}>LLM Verifier</h2><p style={{margin:0,color:'var(--dsw-text-secondary)'}}>选择任意已在 DSH「模型」页配置并启用的模型作为独立裁判。设置实时生效。</p></div>
|
|
33
|
+
<div style={card}><div style={sectionTitle}><StateDot state={draft.enabled?'done':'error'}/><strong>工具开关</strong></div>
|
|
34
|
+
<div style={row}><Label title="启用 Verifier 工具" help={draft.enabled?'四个 verifier 工具可被 Agent 调用，每次调用会向裁判模型发起请求。':'已停用：Agent 调用 verifier 工具会立即返回错误，不产生任何模型请求。'}/><button onClick={()=>patch('enabled',!draft.enabled)} style={{width:80,height:32,borderRadius:16,border:'1px solid var(--dsw-alias-border-l2, rgba(255,255,255,0.16))',background:draft.enabled?'var(--dsw-success, #2f9e5b)':'var(--dsw-surface-sunken)',color:'#fff',cursor:'pointer',fontSize:13,fontWeight:600}}>{draft.enabled?'已启用':'已停用'}</button></div>
|
|
35
|
+
{!draft.enabled&&<div style={{padding:'0 0 14px',fontSize:12,color:'var(--dsw-state-warn-primary, #d9a441)'}}>⚠ 当前停用 verifier_compare / verifier_select / verifier_track / verifier_current_session 四个工具</div>}
|
|
36
|
+
</div>
|
|
33
37
|
<div style={card}><div style={sectionTitle}><StateDot state="done"/><strong>裁判模型</strong></div>
|
|
34
38
|
<div style={row}><Label title="供应商" help="只显示当前 DSH 中可路由的供应商"/><select style={selectStyle} value={draft.provider} onChange={e=>{const provider=e.target.value;const first=loaded.groups.find(g=>g.id===provider)?.models[0];setDraft({...draft,provider,...(first?{model:first.id,reasoningEffort:first.reasoning?.defaultEffort}:{})})}}>{loaded.groups.map(g=><option key={g.id} value={g.id}>{g.name} · {g.id}</option>)}</select></div>
|
|
35
39
|
<div style={row}><Label title="模型" help="模型目录来自 DSH adapter，选择结果会持久化"/><select style={selectStyle} value={draft.model} onChange={e=>{const model=e.target.value;const found=models.find(m=>m.id===model);setDraft({...draft,model,...(found?.reasoning?.defaultEffort?{reasoningEffort:found.reasoning.defaultEffort}:{reasoningEffort:undefined})})}}>{models.map(m=><option key={m.id} value={m.id}>{m.name} · {m.id}</option>)}</select></div>
|
package/src/config.ts
CHANGED
|
@@ -5,6 +5,7 @@ import z from 'schemastery'
|
|
|
5
5
|
export const VERIFIER_SETTINGS_NAMESPACE = settingsNamespace('llm-verifier')
|
|
6
6
|
|
|
7
7
|
export interface Config {
|
|
8
|
+
enabled?: boolean
|
|
8
9
|
provider?: string
|
|
9
10
|
model?: string
|
|
10
11
|
reasoningEffort?: string
|
|
@@ -20,6 +21,7 @@ export interface Config {
|
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
export interface ResolvedConfig {
|
|
24
|
+
enabled: boolean
|
|
23
25
|
provider: string
|
|
24
26
|
model: string
|
|
25
27
|
reasoningEffort?: string
|
|
@@ -35,6 +37,7 @@ export interface ResolvedConfig {
|
|
|
35
37
|
}
|
|
36
38
|
|
|
37
39
|
export const Config: z<Config> = z.object({
|
|
40
|
+
enabled: z.boolean().default(true),
|
|
38
41
|
provider: z.string().default('deepseek-official'),
|
|
39
42
|
model: z.string().default('deepseek-v4-flash'),
|
|
40
43
|
reasoningEffort: z.string(),
|
|
@@ -70,7 +73,7 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
|
|
|
70
73
|
const estimatedOutputUsdPerMillion = config.estimatedOutputUsdPerMillion ?? 0
|
|
71
74
|
if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every(value => Number.isFinite(value) && value >= 0)) throw new Error('llm-verifier: estimated token prices must be finite non-negative numbers')
|
|
72
75
|
const reasoningEffort = config.reasoningEffort?.trim()
|
|
73
|
-
return { provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values }
|
|
76
|
+
return { enabled: config.enabled ?? true, provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values }
|
|
74
77
|
}
|
|
75
78
|
|
|
76
79
|
export function installVerifierSettings(ctx: Context, entry: ResolvedConfig, onChange: () => void): () => ResolvedConfig {
|
package/src/index.ts
CHANGED
|
@@ -37,12 +37,13 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
37
37
|
}
|
|
38
38
|
const images = (values: readonly string[] | undefined, signal: AbortSignal) => loadVerifierImages(values, signal)
|
|
39
39
|
const route = (selected: { provider: string; model: string }) => ({ provider: selected.provider, model: selected.model })
|
|
40
|
+
const requireEnabled = () => { if (!current().enabled) throw new Error('llm-verifier: verifier tools are disabled — enable them in Settings → LLM Verifier') }
|
|
40
41
|
|
|
41
|
-
ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: args.problem, candidateA: args.candidate_a, candidateB: args.candidate_b, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) } } }))
|
|
42
|
+
ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: args.problem, candidateA: args.candidate_a, candidateB: args.candidate_b, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) } } }))
|
|
42
43
|
|
|
43
|
-
ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Uses the configured DSH verifier model and the O(Nk) Probabilistic Pivot Tournament.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { const { verifier, selected } = await engine(); const result = await verifier.select({ problem: args.problem, candidates: args.candidates, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), pivots: positive(args.pivots, 2, 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) } } }))
|
|
44
|
+
ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Uses the configured DSH verifier model and the O(Nk) Probabilistic Pivot Tournament.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { requireEnabled(); const { verifier, selected } = await engine(); const result = await verifier.select({ problem: args.problem, candidates: args.candidates, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), pivots: positive(args.pivots, 2, 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { ...result, ...route(selected) } } }))
|
|
44
45
|
|
|
45
|
-
ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Do not use for a single completed answer or invent checkpoints that were not supplied by the task history. Trusts observed output rather than narration.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { const { verifier, selected } = await engine(); const result = await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { ...result, ...route(selected) } } }))
|
|
46
|
+
ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Do not use for a single completed answer or invent checkpoints that were not supplied by the task history. Trusts observed output rather than narration.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const { verifier, selected } = await engine(); const result = await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { ...result, ...route(selected) } } }))
|
|
46
47
|
|
|
47
|
-
ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Use autonomously near the end of a non-trivial coding, debugging, migration, deployment, or operations task when independent completion verification would materially reduce risk and the session contains real tool evidence. Do not use for routine conversation, simple factual answers, or every turn. Extracts the current DSH session, applies secret redaction, bounds and truncation, then sends the evidence to the verifier model selected in Settings.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { const agent = exec.agent ?? ctx.agents.currentInitiator(); if (agent === undefined) throw new Error('llm-verifier: verifier_current_session requires an agent-owned tool call'); const extracted = await extractSession(agent, async (ref: ImageAttachmentRef) => { const stored = await ctx.attachments.readImage(ref, exec.signal); return { data: stored.data, mediaType: stored.ref.mediaType } }, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars }); const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(args.repeats, 2, 'repeats'), images: extracted.images }, exec.signal); return { sessionId: extracted.sessionId, problem: extracted.problem, score: result.scoreA, baselineScore: result.scoreB, winner: result.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: result.calls, stats: result.stats, ...route(selected) } } }))
|
|
48
|
+
ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Use autonomously near the end of a non-trivial coding, debugging, migration, deployment, or operations task when independent completion verification would materially reduce risk and the session contains real tool evidence. Do not use for routine conversation, simple factual answers, or every turn. Extracts the current DSH session, applies secret redaction, bounds and truncation, then sends the evidence to the verifier model selected in Settings.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = exec.agent ?? ctx.agents.currentInitiator(); if (agent === undefined) throw new Error('llm-verifier: verifier_current_session requires an agent-owned tool call'); const extracted = await extractSession(agent, async (ref: ImageAttachmentRef) => { const stored = await ctx.attachments.readImage(ref, exec.signal); return { data: stored.data, mediaType: stored.ref.mediaType } }, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars }); const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(args.repeats, 2, 'repeats'), images: extracted.images }, exec.signal); return { sessionId: extracted.sessionId, problem: extracted.problem, score: result.scoreA, baselineScore: result.scoreB, winner: result.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: result.calls, stats: result.stats, ...route(selected) } } }))
|
|
48
49
|
}
|
package/lib/caller-BgqCctCh.js
DELETED
|
@@ -1,324 +0,0 @@
|
|
|
1
|
-
import { settingsNamespace } from "@deepseek-ai/dsh-settings";
|
|
2
|
-
import { BlockAssembler, ReasoningEffortId, createUserMessage, deepFreeze } from "@deepseek-ai/dsh-llm";
|
|
3
|
-
import { credentialRef } from "@deepseek-ai/dsh-credentials";
|
|
4
|
-
//#region src/top-logprobs.ts
|
|
5
|
-
var TopLogprobsUnsupportedError = class extends Error {
|
|
6
|
-
constructor(message) {
|
|
7
|
-
super(message);
|
|
8
|
-
this.name = "TopLogprobsUnsupportedError";
|
|
9
|
-
}
|
|
10
|
-
};
|
|
11
|
-
function object(value) {
|
|
12
|
-
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
13
|
-
}
|
|
14
|
-
function text(value) {
|
|
15
|
-
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
16
|
-
}
|
|
17
|
-
function endpoint(baseURL) {
|
|
18
|
-
return baseURL.replace(/\/+$/, "") + "/chat/completions";
|
|
19
|
-
}
|
|
20
|
-
function dataUrl(image) {
|
|
21
|
-
return "data:" + image.mediaType + ";base64," + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).toString("base64");
|
|
22
|
-
}
|
|
23
|
-
async function credential(ctx, name) {
|
|
24
|
-
if (!name) return void 0;
|
|
25
|
-
return (await ctx.get("credentials")?.resolve(credentialRef(name)))?.value;
|
|
26
|
-
}
|
|
27
|
-
async function resolveTopLogprobRoute(ctx, provider) {
|
|
28
|
-
const settings = ctx.get("settings");
|
|
29
|
-
if (!settings) return void 0;
|
|
30
|
-
if (provider === "deepseek-official") {
|
|
31
|
-
const value = object(settings.get(settingsNamespace("llm-deepseek"))) ?? {};
|
|
32
|
-
const apiKey = await credential(ctx, text(value.apiKeyEnv) ?? "DEEPSEEK_API_KEY");
|
|
33
|
-
if (!apiKey) return void 0;
|
|
34
|
-
return {
|
|
35
|
-
baseURL: text(value.baseURL) ?? "https://api.deepseek.com",
|
|
36
|
-
apiKey,
|
|
37
|
-
deepSeekThinking: true
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
const profile = object(object(object(settings.get(settingsNamespace("llm-pi-ai")))?.providers)?.[provider]);
|
|
41
|
-
if (!profile || profile.api !== "openai-completions") return void 0;
|
|
42
|
-
const baseURL = text(profile.baseURL);
|
|
43
|
-
if (!baseURL || !/^https:\/\//i.test(baseURL)) return void 0;
|
|
44
|
-
const apiKey = await credential(ctx, text(profile.apiKeyEnv));
|
|
45
|
-
const rawHeaders = object(profile.headers);
|
|
46
|
-
const headers = rawHeaders === void 0 ? void 0 : Object.fromEntries(Object.entries(rawHeaders).filter((entry) => typeof entry[1] === "string"));
|
|
47
|
-
return {
|
|
48
|
-
baseURL,
|
|
49
|
-
...apiKey ? { apiKey } : {},
|
|
50
|
-
...headers ? { headers } : {},
|
|
51
|
-
deepSeekThinking: false
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
async function callTopLogprobs(route, model, prompt, maxTokens, reasoningEffort, signal, images) {
|
|
55
|
-
const content = images?.length ? [{
|
|
56
|
-
type: "text",
|
|
57
|
-
text: prompt
|
|
58
|
-
}, ...images.map((image) => ({
|
|
59
|
-
type: "image_url",
|
|
60
|
-
image_url: { url: dataUrl(image) }
|
|
61
|
-
}))] : prompt;
|
|
62
|
-
const thinking = route.deepSeekThinking && reasoningEffort ? reasoningEffort === "off" ? { thinking: { type: "disabled" } } : {
|
|
63
|
-
thinking: { type: "enabled" },
|
|
64
|
-
reasoning_effort: reasoningEffort
|
|
65
|
-
} : {};
|
|
66
|
-
const response = await fetch(endpoint(route.baseURL), {
|
|
67
|
-
method: "POST",
|
|
68
|
-
redirect: "error",
|
|
69
|
-
signal,
|
|
70
|
-
headers: {
|
|
71
|
-
"content-type": "application/json",
|
|
72
|
-
...route.apiKey ? { authorization: "Bearer " + route.apiKey } : {},
|
|
73
|
-
...route.headers
|
|
74
|
-
},
|
|
75
|
-
body: JSON.stringify({
|
|
76
|
-
model,
|
|
77
|
-
messages: [{
|
|
78
|
-
role: "user",
|
|
79
|
-
content
|
|
80
|
-
}],
|
|
81
|
-
max_tokens: maxTokens,
|
|
82
|
-
temperature: 1,
|
|
83
|
-
logprobs: true,
|
|
84
|
-
top_logprobs: 20,
|
|
85
|
-
...thinking
|
|
86
|
-
})
|
|
87
|
-
});
|
|
88
|
-
const raw = await response.text();
|
|
89
|
-
if (!response.ok) {
|
|
90
|
-
const excerpt = raw.slice(0, 1e3);
|
|
91
|
-
if ([
|
|
92
|
-
400,
|
|
93
|
-
404,
|
|
94
|
-
405,
|
|
95
|
-
415,
|
|
96
|
-
422
|
|
97
|
-
].includes(response.status) && /logprob|unsupported|unknown|invalid|not support/i.test(excerpt)) throw new TopLogprobsUnsupportedError("provider rejected top_logprobs: HTTP " + response.status + " " + excerpt);
|
|
98
|
-
throw new Error("llm-verifier: top_logprobs request failed with HTTP " + response.status + ": " + excerpt);
|
|
99
|
-
}
|
|
100
|
-
let body;
|
|
101
|
-
try {
|
|
102
|
-
body = object(JSON.parse(raw)) ?? {};
|
|
103
|
-
} catch {
|
|
104
|
-
throw new Error("llm-verifier: top_logprobs endpoint returned invalid JSON");
|
|
105
|
-
}
|
|
106
|
-
const choice = object((Array.isArray(body.choices) ? body.choices : [])[0]);
|
|
107
|
-
const message = object(choice?.message);
|
|
108
|
-
const answer = typeof message?.content === "string" ? message.content : "";
|
|
109
|
-
const logprobs = object(choice?.logprobs);
|
|
110
|
-
const rows = Array.isArray(logprobs?.content) ? logprobs.content : [];
|
|
111
|
-
if (!rows.length) throw new TopLogprobsUnsupportedError("provider returned no token logprobs");
|
|
112
|
-
const tokens = [];
|
|
113
|
-
const positions = [];
|
|
114
|
-
for (const rawRow of rows) {
|
|
115
|
-
const row = object(rawRow) ?? {};
|
|
116
|
-
const token = typeof row.token === "string" ? row.token : "";
|
|
117
|
-
tokens.push(token);
|
|
118
|
-
const alternatives = (Array.isArray(row.top_logprobs) ? row.top_logprobs : []).flatMap((value) => {
|
|
119
|
-
const item = object(value);
|
|
120
|
-
return item && typeof item.token === "string" && typeof item.logprob === "number" ? [{
|
|
121
|
-
token: item.token,
|
|
122
|
-
logprob: item.logprob
|
|
123
|
-
}] : [];
|
|
124
|
-
});
|
|
125
|
-
if (!alternatives.length && typeof row.logprob === "number") alternatives.push({
|
|
126
|
-
token,
|
|
127
|
-
logprob: row.logprob
|
|
128
|
-
});
|
|
129
|
-
positions.push(alternatives);
|
|
130
|
-
}
|
|
131
|
-
const rawUsage = object(body.usage) ?? {};
|
|
132
|
-
const promptDetails = object(rawUsage.prompt_tokens_details) ?? {};
|
|
133
|
-
const completionDetails = object(rawUsage.completion_tokens_details) ?? {};
|
|
134
|
-
const cached = Number(rawUsage.prompt_cache_hit_tokens ?? promptDetails.cached_tokens ?? 0) || 0;
|
|
135
|
-
const input = Number(rawUsage.prompt_tokens ?? 0) || 0;
|
|
136
|
-
return {
|
|
137
|
-
text: answer,
|
|
138
|
-
tokens,
|
|
139
|
-
positions,
|
|
140
|
-
scoringMode: "top-logprobs",
|
|
141
|
-
usage: {
|
|
142
|
-
calls: 1,
|
|
143
|
-
attempts: 1,
|
|
144
|
-
retries: 0,
|
|
145
|
-
inputTokens: Math.max(0, input - cached),
|
|
146
|
-
cachedInputTokens: cached,
|
|
147
|
-
outputTokens: Number(rawUsage.completion_tokens ?? 0) || 0,
|
|
148
|
-
reasoningTokens: Number(completionDetails.reasoning_tokens ?? 0) || 0
|
|
149
|
-
}
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
|
-
var TopLogprobCapabilityCache = class {
|
|
153
|
-
unsupported = /* @__PURE__ */ new Set();
|
|
154
|
-
isUnsupported(provider, model) {
|
|
155
|
-
return this.unsupported.has(provider + "\0" + model);
|
|
156
|
-
}
|
|
157
|
-
markUnsupported(provider, model) {
|
|
158
|
-
this.unsupported.add(provider + "\0" + model);
|
|
159
|
-
}
|
|
160
|
-
};
|
|
161
|
-
//#endregion
|
|
162
|
-
//#region src/caller.ts
|
|
163
|
-
function failureMessage(finish) {
|
|
164
|
-
if (finish.kind === "error" || finish.kind === "aborted") return finish.failure.message;
|
|
165
|
-
if (finish.kind === "max-tokens") return "verifier response reached max tokens before completing its answer";
|
|
166
|
-
}
|
|
167
|
-
async function delay(ms, signal) {
|
|
168
|
-
if (signal?.aborted) throw signal.reason;
|
|
169
|
-
await new Promise((resolve, reject) => {
|
|
170
|
-
const timer = setTimeout(resolve, ms);
|
|
171
|
-
const abort = () => {
|
|
172
|
-
clearTimeout(timer);
|
|
173
|
-
reject(signal?.reason);
|
|
174
|
-
};
|
|
175
|
-
signal?.addEventListener("abort", abort, { once: true });
|
|
176
|
-
});
|
|
177
|
-
}
|
|
178
|
-
function usage(attempts, value = {}) {
|
|
179
|
-
return {
|
|
180
|
-
calls: 1,
|
|
181
|
-
attempts,
|
|
182
|
-
retries: attempts - 1,
|
|
183
|
-
inputTokens: value.inputTokens ?? 0,
|
|
184
|
-
cachedInputTokens: (value.cacheReadTokens ?? 0) + (value.cacheWriteTokens ?? 0),
|
|
185
|
-
outputTokens: value.outputTokens ?? 0,
|
|
186
|
-
reasoningTokens: value.reasoningTokens ?? 0
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
async function callExplicitTag(config, prompt, signal, images) {
|
|
190
|
-
let attempt = 0;
|
|
191
|
-
while (true) {
|
|
192
|
-
attempt += 1;
|
|
193
|
-
const controller = new AbortController();
|
|
194
|
-
const timeout = setTimeout(() => controller.abort(/* @__PURE__ */ new Error("llm-verifier: request timed out")), config.timeoutMs);
|
|
195
|
-
const abort = () => controller.abort(signal?.reason);
|
|
196
|
-
signal?.addEventListener("abort", abort, { once: true });
|
|
197
|
-
try {
|
|
198
|
-
const content = [{
|
|
199
|
-
type: "text",
|
|
200
|
-
text: prompt
|
|
201
|
-
}];
|
|
202
|
-
for (const image of images ?? []) {
|
|
203
|
-
const ref = await config.attachments.saveImage({
|
|
204
|
-
data: image.data,
|
|
205
|
-
mediaType: image.mediaType
|
|
206
|
-
});
|
|
207
|
-
content.push({
|
|
208
|
-
type: "image",
|
|
209
|
-
attachment: ref
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
const messages = [createUserMessage({
|
|
213
|
-
content,
|
|
214
|
-
source: {
|
|
215
|
-
kind: "plugin",
|
|
216
|
-
plugin: "dsh-llm-verifier"
|
|
217
|
-
}
|
|
218
|
-
})];
|
|
219
|
-
const assembler = new BlockAssembler();
|
|
220
|
-
const options = deepFreeze({
|
|
221
|
-
provider: config.provider,
|
|
222
|
-
model: config.model,
|
|
223
|
-
...config.reasoningEffort ? { reasoningEffort: ReasoningEffortId(config.reasoningEffort) } : {},
|
|
224
|
-
messages,
|
|
225
|
-
maxTokens: config.maxTokens,
|
|
226
|
-
temperature: 1,
|
|
227
|
-
signal: controller.signal
|
|
228
|
-
});
|
|
229
|
-
for await (const chunk of config.llm.stream(options)) assembler.push(chunk);
|
|
230
|
-
const failed = failureMessage(assembler.finish);
|
|
231
|
-
if (failed !== void 0) throw new Error("llm-verifier: model call failed: " + failed);
|
|
232
|
-
const text = assembler.blocks().filter((block) => block.type === "text").map((block) => block.text).join("");
|
|
233
|
-
if (!text.trim()) throw new Error("llm-verifier: selected DSH model produced no text");
|
|
234
|
-
return {
|
|
235
|
-
text,
|
|
236
|
-
tokens: [],
|
|
237
|
-
positions: [],
|
|
238
|
-
scoringMode: "explicit-tag",
|
|
239
|
-
usage: usage(attempt, assembler.usage)
|
|
240
|
-
};
|
|
241
|
-
} catch (error) {
|
|
242
|
-
if (signal?.aborted) throw signal.reason;
|
|
243
|
-
if (attempt > config.maxRetries || !(error instanceof Error) || !/rate|quota|timeout|timed out|temporar|network|fetch|socket|5dd/i.test(error.message)) throw error;
|
|
244
|
-
await delay(Math.min(3e4, config.retryBaseDelayMs * 2 ** (attempt - 1) * (.8 + Math.random() * .4)), signal);
|
|
245
|
-
} finally {
|
|
246
|
-
clearTimeout(timeout);
|
|
247
|
-
signal?.removeEventListener("abort", abort);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
var RequestLimiter = class {
|
|
252
|
-
limit;
|
|
253
|
-
active = 0;
|
|
254
|
-
queue = [];
|
|
255
|
-
constructor(limit) {
|
|
256
|
-
this.limit = limit;
|
|
257
|
-
}
|
|
258
|
-
async run(operation, signal) {
|
|
259
|
-
if (this.active >= this.limit) await new Promise((resolve, reject) => {
|
|
260
|
-
const enter = () => {
|
|
261
|
-
signal?.removeEventListener("abort", abort);
|
|
262
|
-
resolve();
|
|
263
|
-
};
|
|
264
|
-
const abort = () => {
|
|
265
|
-
const index = this.queue.indexOf(enter);
|
|
266
|
-
if (index >= 0) this.queue.splice(index, 1);
|
|
267
|
-
reject(signal?.reason);
|
|
268
|
-
};
|
|
269
|
-
this.queue.push(enter);
|
|
270
|
-
signal?.addEventListener("abort", abort, { once: true });
|
|
271
|
-
});
|
|
272
|
-
if (signal?.aborted) throw signal.reason;
|
|
273
|
-
this.active += 1;
|
|
274
|
-
try {
|
|
275
|
-
return await operation();
|
|
276
|
-
} finally {
|
|
277
|
-
this.active -= 1;
|
|
278
|
-
this.queue.shift()?.();
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
};
|
|
282
|
-
async function callAutomatic(config, prompt, signal, images) {
|
|
283
|
-
if (!config.topLogprobCapabilities.isUnsupported(config.provider, config.model)) {
|
|
284
|
-
const route = await resolveTopLogprobRoute(config.ctx, config.provider);
|
|
285
|
-
if (route !== void 0) try {
|
|
286
|
-
return await callTopLogprobs(route, config.model, prompt, config.maxTokens, config.reasoningEffort, signal, images);
|
|
287
|
-
} catch (error) {
|
|
288
|
-
if (!(error instanceof TopLogprobsUnsupportedError)) throw error;
|
|
289
|
-
config.topLogprobCapabilities.markUnsupported(config.provider, config.model);
|
|
290
|
-
}
|
|
291
|
-
else config.topLogprobCapabilities.markUnsupported(config.provider, config.model);
|
|
292
|
-
}
|
|
293
|
-
return callExplicitTag(config, prompt, signal, images);
|
|
294
|
-
}
|
|
295
|
-
async function callVerifier(config, prompt, signal, images) {
|
|
296
|
-
const invoke = () => callAutomatic(config, prompt, signal, images);
|
|
297
|
-
return config.limiter === void 0 ? invoke() : config.limiter.run(invoke, signal);
|
|
298
|
-
}
|
|
299
|
-
function addUsage(target, source) {
|
|
300
|
-
for (const key of [
|
|
301
|
-
"calls",
|
|
302
|
-
"attempts",
|
|
303
|
-
"retries",
|
|
304
|
-
"inputTokens",
|
|
305
|
-
"cachedInputTokens",
|
|
306
|
-
"outputTokens",
|
|
307
|
-
"reasoningTokens"
|
|
308
|
-
]) target[key] += source[key];
|
|
309
|
-
}
|
|
310
|
-
function emptyUsage() {
|
|
311
|
-
return {
|
|
312
|
-
calls: 0,
|
|
313
|
-
attempts: 0,
|
|
314
|
-
retries: 0,
|
|
315
|
-
inputTokens: 0,
|
|
316
|
-
cachedInputTokens: 0,
|
|
317
|
-
outputTokens: 0,
|
|
318
|
-
reasoningTokens: 0
|
|
319
|
-
};
|
|
320
|
-
}
|
|
321
|
-
//#endregion
|
|
322
|
-
export { TopLogprobCapabilityCache as a, emptyUsage as i, addUsage as n, callVerifier as r, RequestLimiter as t };
|
|
323
|
-
|
|
324
|
-
//# sourceMappingURL=caller-BgqCctCh.js.map
|