dsh-llm-verifier 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -6
- package/lib/caller-BXrUTAoL.js +324 -0
- package/lib/caller.js +1 -1
- package/lib/client.js +275 -114
- package/lib/index.js +301 -53
- package/lib/types/auto.d.ts +26 -0
- package/lib/types/auto.js +86 -0
- package/lib/types/client.js +16 -11
- package/lib/types/config.d.ts +15 -0
- package/lib/types/config.js +21 -1
- package/lib/types/index.d.ts +1 -0
- package/lib/types/index.js +42 -10
- package/package.json +1 -1
- package/src/auto.test.ts +69 -0
- package/src/auto.ts +113 -0
- package/src/client.tsx +57 -27
- package/src/config.ts +34 -1
- package/src/index.ts +45 -12
package/lib/types/client.js
CHANGED
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
|
2
|
-
import { Button, IconDataOutline16, IconRefreshOutline16, Input
|
|
2
|
+
import { Button, IconDataOutline16, IconRefreshOutline16, Input } from '@deepseek-ai/dsh-client-ui-primitives';
|
|
3
3
|
import { useEffect, useMemo, useState } from 'react';
|
|
4
4
|
const NS = 'llm-verifier';
|
|
5
|
-
const shell = { display: 'flex', flexDirection: 'column', gap:
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
5
|
+
const shell = { width: '100%', maxWidth: 720, display: 'flex', flexDirection: 'column', gap: 12, padding: '0 0 32px', color: 'var(--dsw-alias-label-primary)' };
|
|
6
|
+
const settingsHeading = { margin: 0, fontSize: 16, fontWeight: 500, lineHeight: '24px', color: 'var(--dsw-alias-label-primary)' };
|
|
7
|
+
const settingsIntro = { margin: 0, fontSize: 14, lineHeight: '22px', color: 'var(--dsw-alias-label-tertiary)' };
|
|
8
|
+
const group = { width: '100%', display: 'flex', flexDirection: 'column', borderTop: '1px solid var(--dsw-alias-border-l2)' };
|
|
9
|
+
const groupTitle = { margin: 0, padding: '18px 0 8px', fontSize: 14, fontWeight: 500, lineHeight: '22px', color: 'var(--dsw-alias-label-primary)' };
|
|
10
|
+
const row = { minHeight: 64, display: 'grid', gridTemplateColumns: 'minmax(180px, 1fr) minmax(230px, 288px)', gap: 24, alignItems: 'center', padding: '12px 0', borderBottom: '1px solid var(--dsw-alias-border-l2)' };
|
|
11
|
+
const selectStyle = { boxSizing: 'border-box', width: '100%', height: 36, padding: '0 34px 0 12px', borderRadius: 8, color: 'var(--dsw-alias-label-primary)', background: 'var(--dsw-alias-bg-input)', border: '1px solid var(--dsw-alias-border-l2)', font: 'inherit', fontSize: 14, lineHeight: '22px', outline: 'none' };
|
|
12
|
+
const toggleStyle = (enabled) => ({ position: 'relative', justifySelf: 'end', width: 40, height: 22, padding: 0, border: 0, borderRadius: 999, cursor: 'pointer', transition: 'background .15s ease', background: enabled ? 'var(--dsw-alias-state-success-primary)' : 'var(--dsw-alias-bg-input)' });
|
|
13
|
+
const toggleThumbStyle = (enabled) => ({ position: 'absolute', top: 3, left: enabled ? 21 : 3, width: 16, height: 16, borderRadius: '50%', background: 'var(--dsw-static-neutral-00, #fff)', boxShadow: '0 1px 3px rgba(0,0,0,.28)', transition: 'left .15s ease' });
|
|
10
14
|
const dashboardCard = { border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.13))', background: 'color-mix(in srgb, var(--dsw-alias-bg-module, #171925) 88%, transparent)', borderRadius: 16, boxShadow: '0 12px 36px rgba(0,0,0,.12)' };
|
|
11
15
|
const muted = { color: 'var(--dsw-text-secondary)', fontSize: 12 };
|
|
12
16
|
const toolLabels = { verifier_compare: '两项对比', verifier_select: '多项优选', verifier_track: '进度跟踪', verifier_current_session: '会话验收' };
|
|
13
17
|
const toolColors = { verifier_compare: '#4f8cff', verifier_select: '#8b6df6', verifier_track: '#2fc5c9', verifier_current_session: '#f5a524' };
|
|
14
18
|
function record(value) { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : {}; }
|
|
15
|
-
function values(view) { const v = record(view.value); return { enabled: v.enabled !== false, provider: String(v.provider ?? ''), model: String(v.model ?? ''), ...(typeof v.reasoningEffort === 'string' ? { reasoningEffort: v.reasoningEffort } : {}), maxTokens: Number(v.maxTokens ?? 32768), maxConcurrency: Number(v.maxConcurrency ?? 8), maxRetries: Number(v.maxRetries ?? 3), timeoutMs: Number(v.timeoutMs ?? 300000), cacheMaxEntries: Number(v.cacheMaxEntries ?? 10000), estimatedInputUsdPerMillion: Number(v.estimatedInputUsdPerMillion ?? 0), estimatedOutputUsdPerMillion: Number(v.estimatedOutputUsdPerMillion ?? 0) }; }
|
|
19
|
+
function values(view) { const v = record(view.value); const mode = v.autoVerifyMode === 'manual' || v.autoVerifyMode === 'strict' ? v.autoVerifyMode : 'smart'; return { enabled: v.enabled !== false, autoVerifyMode: mode, autoVerifyThreshold: Number(v.autoVerifyThreshold ?? .65), autoVerifyRepeats: Number(v.autoVerifyRepeats ?? 1), autoVerifyMinToolCalls: Number(v.autoVerifyMinToolCalls ?? 3), autoVerifyMaxChars: Number(v.autoVerifyMaxChars ?? 80000), autoVerifyMaxPerTask: Number(v.autoVerifyMaxPerTask ?? 2), autoVerifyMaxPerSession: Number(v.autoVerifyMaxPerSession ?? 8), provider: String(v.provider ?? ''), model: String(v.model ?? ''), ...(typeof v.reasoningEffort === 'string' ? { reasoningEffort: v.reasoningEffort } : {}), maxTokens: Number(v.maxTokens ?? 32768), maxConcurrency: Number(v.maxConcurrency ?? 8), maxRetries: Number(v.maxRetries ?? 3), timeoutMs: Number(v.timeoutMs ?? 300000), cacheMaxEntries: Number(v.cacheMaxEntries ?? 10000), estimatedInputUsdPerMillion: Number(v.estimatedInputUsdPerMillion ?? 0), estimatedOutputUsdPerMillion: Number(v.estimatedOutputUsdPerMillion ?? 0) }; }
|
|
16
20
|
function message(error) { return error instanceof Error ? error.message : String(error); }
|
|
17
|
-
function Label({ title, help }) { return _jsxs("div", { children: [_jsx("div", { style: { fontWeight:
|
|
21
|
+
function Label({ title, help }) { return _jsxs("div", { style: { minWidth: 0 }, children: [_jsx("div", { style: { fontSize: 14, fontWeight: 400, lineHeight: '22px', color: 'var(--dsw-alias-label-primary)' }, children: title }), _jsx("div", { style: { fontSize: 12, lineHeight: '18px', color: 'var(--dsw-alias-label-tertiary)', marginTop: 2 }, children: help })] }); }
|
|
22
|
+
function GroupTitle({ children }) { return _jsx("h3", { style: groupTitle, children: children }); }
|
|
18
23
|
function compact(value) { return new Intl.NumberFormat('zh-CN', { notation: value >= 10_000 ? 'compact' : 'standard', maximumFractionDigits: value >= 1_000 ? 1 : 0 }).format(value); }
|
|
19
24
|
function money(value) { return '$' + value.toLocaleString('en-US', { minimumFractionDigits: value < 0.01 ? 4 : 2, maximumFractionDigits: value < 0.01 ? 4 : 2 }); }
|
|
20
25
|
function duration(value) { if (value < 1000)
|
|
@@ -69,9 +74,9 @@ function VerifierSettings({ api }) {
|
|
|
69
74
|
setBusy(false);
|
|
70
75
|
} };
|
|
71
76
|
if (!loaded || !draft)
|
|
72
|
-
return _jsxs("div", { style: shell, children: [_jsx("h2", { children: "LLM Verifier" }), _jsx("p", { children: error ?? '正在读取 DSH 模型和设置…' }), error && _jsx(Button, { onClick: () => void load(), children: "\u91CD\u8BD5" })] });
|
|
73
|
-
const numeric = (key, min = 0) => _jsx(Input, { type: "number", min: min, value: String(draft[key]), onChange: e => patch(key, Number(e.target.value)) });
|
|
74
|
-
return _jsxs("div", { style: shell, children: [
|
|
77
|
+
return _jsxs("div", { style: shell, children: [_jsx("h2", { style: settingsHeading, children: "LLM Verifier" }), _jsx("p", { style: settingsIntro, children: error ?? '正在读取 DSH 模型和设置…' }), error && _jsx("div", { children: _jsx(Button, { variant: "outline", onClick: () => void load(), children: "\u91CD\u8BD5" }) })] });
|
|
78
|
+
const numeric = (key, min = 0) => _jsx(Input, { style: { width: '100%', height: 36, borderRadius: 8 }, type: "number", min: min, value: String(draft[key]), onChange: e => patch(key, Number(e.target.value)) });
|
|
79
|
+
return _jsxs("div", { style: shell, children: [_jsx("h2", { style: settingsHeading, children: "LLM Verifier" }), _jsx("p", { style: settingsIntro, children: "\u9009\u62E9\u5DF2\u5728 DSH\u300C\u6A21\u578B\u300D\u4E2D\u914D\u7F6E\u7684\u6A21\u578B\u4F5C\u4E3A\u72EC\u7ACB\u88C1\u5224\u3002\u4FEE\u6539\u540E\u70B9\u51FB\u9875\u9762\u5E95\u90E8\u7684\u4FDD\u5B58\u6309\u94AE\u751F\u6548\u3002" }), _jsxs("section", { style: group, children: [_jsx(GroupTitle, { children: "\u5DE5\u5177" }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u542F\u7528 Verifier \u5DE5\u5177", help: draft.enabled ? '允许 Agent 调用四个 verifier 工具并向裁判模型发起请求。' : '停用后，所有 verifier 工具都会立即返回错误。' }), _jsx("button", { type: "button", role: "switch", "aria-checked": draft.enabled, "aria-label": "\u542F\u7528 Verifier \u5DE5\u5177", onClick: () => patch('enabled', !draft.enabled), style: toggleStyle(draft.enabled), children: _jsx("span", { style: toggleThumbStyle(draft.enabled) }) })] }), !draft.enabled && _jsx("p", { style: { margin: '8px 0 0', fontSize: 12, lineHeight: '18px', color: 'var(--dsw-alias-state-warn-label)' }, children: "verifier_compare\u3001verifier_select\u3001verifier_track \u548C verifier_current_session \u5F53\u524D\u4E0D\u53EF\u7528\u3002" })] }), _jsxs("section", { style: group, children: [_jsx(GroupTitle, { children: "\u81EA\u52A8\u9A8C\u6536" }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8C03\u7528\u7B56\u7565", help: "\u624B\u52A8\u4EC5\u66B4\u9732\u5DE5\u5177\uFF1B\u667A\u80FD\u5728\u590D\u6742\u4E14\u6709\u771F\u5B9E\u8BC1\u636E\u7684\u4EFB\u52A1\u7ED3\u675F\u524D\u9A8C\u6536\uFF1B\u4E25\u683C\u5BF9\u4EFB\u4F55\u5DF2\u5B8C\u6210\u7684\u5199\u5165\u3001\u6267\u884C\u6216\u8FDC\u7A0B\u64CD\u4F5C\u8FDB\u884C\u9A8C\u6536\u3002" }), _jsxs("select", { style: selectStyle, value: draft.autoVerifyMode, onChange: e => patch('autoVerifyMode', e.target.value), children: [_jsx("option", { value: "manual", children: "\u624B\u52A8" }), _jsx("option", { value: "smart", children: "\u667A\u80FD\uFF08\u63A8\u8350\uFF09" }), _jsx("option", { value: "strict", children: "\u4E25\u683C" })] })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u901A\u8FC7\u9608\u503C", help: "\u4F1A\u8BDD\u8BC1\u636E\u5206\u6570\u8FBE\u5230\u8BE5\u503C\u4E14\u80DC\u8FC7\u7A7A\u5DE5\u4F5C\u57FA\u7EBF\u624D\u5141\u8BB8\u7ED3\u675F\uFF1B\u8303\u56F4 0\u20131\u3002" }), numeric('autoVerifyThreshold', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u81EA\u52A8\u8BC4\u4F30\u8F6E\u6B21", help: "\u6BCF\u9879\u6807\u51C6\u7684\u81EA\u52A8\u8BC4\u5206\u91CD\u590D\u6B21\u6570\uFF1B1 \u4E3A\u4F4E\u6210\u672C\u521D\u7B5B\u3002" }), numeric('autoVerifyRepeats', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u667A\u80FD\u6A21\u5F0F\u6700\u5C11\u5DE5\u5177\u8C03\u7528", help: "\u8FBE\u5230\u6B64\u5DE5\u5177\u8BC1\u636E\u6570\u91CF\u4E14\u5305\u542B\u5199\u5165/\u6267\u884C\u7C7B\u64CD\u4F5C\u65F6\u624D\u81EA\u52A8\u9A8C\u6536\u3002" }), numeric('autoVerifyMinToolCalls', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u8BC1\u636E\u5B57\u7B26", help: "\u53D1\u9001\u7ED9\u88C1\u5224\u524D\u4FDD\u7559\u7684\u6700\u8FD1\u4F1A\u8BDD\u8BC1\u636E\u5B57\u7B26\u6570\u3002" }), numeric('autoVerifyMaxChars', 1000)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6BCF\u4EFB\u52A1\u6700\u591A\u9A8C\u6536", help: "\u4F4E\u5206\u53CD\u9988\u540E\u5141\u8BB8\u518D\u6B21\u9A8C\u6536\u7684\u6B21\u6570\u4E0A\u9650\uFF0C\u9632\u6B62\u5FAA\u73AF\u3002" }), numeric('autoVerifyMaxPerTask', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6BCF\u4F1A\u8BDD\u6700\u591A\u9A8C\u6536", help: "\u540C\u4E00\u4F1A\u8BDD\u4E2D\u7684\u81EA\u52A8\u9A8C\u6536\u603B\u9884\u7B97\u3002" }), numeric('autoVerifyMaxPerSession', 1)] }), draft.autoVerifyMode !== 'manual' && _jsx("p", { style: { margin: '8px 0 0', fontSize: 12, lineHeight: '18px', color: 'var(--dsw-alias-state-warn-label)' }, children: "\u81EA\u52A8\u9A8C\u6536\u4F1A\u5C06\u8131\u654F\u540E\u7684\u4EFB\u52A1\u3001Assistant \u8F68\u8FF9\u4E0E\u771F\u5B9E\u5DE5\u5177\u8F93\u51FA\u53D1\u9001\u7ED9\u6240\u9009\u88C1\u5224\u6A21\u578B\uFF1B\u672A\u901A\u8FC7\u65F6\u4F1A\u81EA\u52A8\u8981\u6C42 Agent \u4FEE\u590D\u5E76\u91CD\u65B0\u9A8C\u8BC1\u3002" })] }), _jsxs("section", { style: group, children: [_jsx(GroupTitle, { children: "\u88C1\u5224\u6A21\u578B" }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u4F9B\u5E94\u5546", help: "\u5F53\u524D DSH \u4E2D\u53EF\u8DEF\u7531\u7684\u6A21\u578B\u4F9B\u5E94\u5546\u3002" }), _jsx("select", { style: selectStyle, value: draft.provider, onChange: e => { const provider = e.target.value; const first = loaded.groups.find(g => g.id === provider)?.models[0]; setDraft({ ...draft, provider, ...(first ? { model: first.id, reasoningEffort: first.reasoning?.defaultEffort } : {}) }); }, children: loaded.groups.map(g => _jsxs("option", { value: g.id, children: [g.name, " \u00B7 ", g.id] }, g.id)) })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6A21\u578B", help: "\u7528\u4F5C\u88C1\u5224\u7684\u5177\u4F53\u6A21\u578B\u3002" }), _jsx("select", { style: selectStyle, value: draft.model, onChange: e => { const model = e.target.value; const found = models.find(m => m.id === model); setDraft({ ...draft, model, ...(found?.reasoning?.defaultEffort ? { reasoningEffort: found.reasoning.defaultEffort } : { reasoningEffort: undefined }) }); }, children: models.map(m => _jsxs("option", { value: m.id, children: [m.name, " \u00B7 ", m.id] }, m.id)) })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u63A8\u7406\u5F3A\u5EA6", help: "\u7559\u7A7A\u65F6\u4F7F\u7528\u6240\u9009\u6A21\u578B\u7684\u9ED8\u8BA4\u503C\u3002" }), _jsxs("select", { style: selectStyle, value: draft.reasoningEffort ?? '', onChange: e => patch('reasoningEffort', e.target.value || undefined), children: [_jsx("option", { value: "", children: "\u6A21\u578B\u9ED8\u8BA4" }), efforts.map(e => _jsx("option", { value: e.id, children: e.name }, e.id))] })] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u8F93\u51FA Token", help: "\u5355\u6B21\u88C1\u5224\u8BF7\u6C42\u5141\u8BB8\u751F\u6210\u7684\u6700\u5927 Token \u6570\u3002" }), numeric('maxTokens', 1)] })] }), _jsxs("section", { style: group, children: [_jsx(GroupTitle, { children: "\u6267\u884C" }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u5927\u5E76\u53D1", help: "\u6240\u6709 verifier \u5DE5\u5177\u5171\u4EAB\u7684\u8BF7\u6C42\u5E76\u53D1\u4E0A\u9650\u3002" }), numeric('maxConcurrency', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u6700\u591A\u91CD\u8BD5", help: "\u9650\u6D41\u3001\u8D85\u65F6\u548C\u77ED\u6682\u7F51\u7EDC\u9519\u8BEF\u7684\u91CD\u8BD5\u6B21\u6570\u3002" }), numeric('maxRetries', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8BF7\u6C42\u8D85\u65F6", help: "\u5355\u6B21\u6A21\u578B\u8BF7\u6C42\u7684\u8D85\u65F6\u65F6\u95F4\uFF0C\u5355\u4F4D\u4E3A\u6BEB\u79D2\u3002" }), numeric('timeoutMs', 1)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u7F13\u5B58\u6761\u76EE\u4E0A\u9650", help: "\u672C\u5730\u6301\u4E45\u8BC4\u5206\u7F13\u5B58\u4FDD\u7559\u7684\u6700\u5927\u6761\u76EE\u6570\u3002" }), numeric('cacheMaxEntries', 1)] })] }), _jsxs("section", { style: group, children: [_jsx(GroupTitle, { children: "\u8D39\u7528\u4F30\u7B97" }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8F93\u5165\u4EF7\u683C", help: "\u6BCF\u767E\u4E07\u8F93\u5165 Token \u7684\u7F8E\u5143\u4EF7\u683C\uFF0C\u4EC5\u7528\u4E8E\u7EDF\u8BA1\u4F30\u7B97\u3002" }), numeric('estimatedInputUsdPerMillion', 0)] }), _jsxs("div", { style: row, children: [_jsx(Label, { title: "\u8F93\u51FA\u4EF7\u683C", help: "\u6BCF\u767E\u4E07\u8F93\u51FA Token \u7684\u7F8E\u5143\u4EF7\u683C\uFF0C\u4EC5\u7528\u4E8E\u7EDF\u8BA1\u4F30\u7B97\u3002" }), numeric('estimatedOutputUsdPerMillion', 0)] })] }), loaded.failures.length > 0 && _jsxs("div", { style: { padding: '10px 12px', borderRadius: 8, background: 'var(--dsw-alias-state-warn-bg)', color: 'var(--dsw-alias-state-warn-label)', fontSize: 12, lineHeight: '18px' }, children: [_jsx("div", { style: { fontWeight: 500, marginBottom: 3 }, children: "\u90E8\u5206\u6A21\u578B\u76EE\u5F55\u8BFB\u53D6\u5931\u8D25" }), loaded.failures.map(x => _jsx("div", { children: x }, x))] }), error && _jsx("p", { style: { margin: 0, fontSize: 12, lineHeight: '18px', color: 'var(--dsw-alias-state-error-primary)' }, children: error }), saved && _jsx("p", { style: { margin: 0, fontSize: 12, lineHeight: '18px', color: 'var(--dsw-alias-state-success-primary)' }, children: "\u8BBE\u7F6E\u5DF2\u4FDD\u5B58\u3002" }), _jsxs("div", { style: { display: 'flex', justifyContent: 'flex-end', gap: 8, paddingTop: 4 }, children: [_jsx(Button, { variant: "outline", disabled: busy, onClick: () => void load(), children: "\u91CD\u65B0\u8F7D\u5165" }), _jsx(Button, { variant: "primary", disabled: busy || !loaded.writable, onClick: () => void save(), children: busy ? '保存中…' : '保存设置' })] })] });
|
|
75
80
|
}
|
|
76
81
|
function Metric({ label, value, note, accent }) {
|
|
77
82
|
return _jsxs("div", { style: { ...dashboardCard, padding: '16px 18px', minWidth: 0 }, children: [_jsx("div", { style: muted, children: label }), _jsx("div", { style: { fontSize: 25, fontWeight: 750, lineHeight: 1.2, margin: '7px 0 5px', color: accent }, children: value }), _jsx("div", { style: { ...muted, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }, children: note })] });
|
package/lib/types/config.d.ts
CHANGED
|
@@ -1,8 +1,16 @@
|
|
|
1
1
|
import type { Context } from '@deepseek-ai/cordis';
|
|
2
2
|
import z from 'schemastery';
|
|
3
3
|
export declare const VERIFIER_SETTINGS_NAMESPACE: import("@deepseek-ai/dsh-settings").SettingsNamespace;
|
|
4
|
+
export type AutoVerifyMode = 'manual' | 'smart' | 'strict';
|
|
4
5
|
export interface Config {
|
|
5
6
|
enabled?: boolean;
|
|
7
|
+
autoVerifyMode?: AutoVerifyMode;
|
|
8
|
+
autoVerifyThreshold?: number;
|
|
9
|
+
autoVerifyRepeats?: number;
|
|
10
|
+
autoVerifyMinToolCalls?: number;
|
|
11
|
+
autoVerifyMaxChars?: number;
|
|
12
|
+
autoVerifyMaxPerTask?: number;
|
|
13
|
+
autoVerifyMaxPerSession?: number;
|
|
6
14
|
provider?: string;
|
|
7
15
|
model?: string;
|
|
8
16
|
reasoningEffort?: string;
|
|
@@ -18,6 +26,13 @@ export interface Config {
|
|
|
18
26
|
}
|
|
19
27
|
export interface ResolvedConfig {
|
|
20
28
|
enabled: boolean;
|
|
29
|
+
autoVerifyMode: AutoVerifyMode;
|
|
30
|
+
autoVerifyThreshold: number;
|
|
31
|
+
autoVerifyRepeats: number;
|
|
32
|
+
autoVerifyMinToolCalls: number;
|
|
33
|
+
autoVerifyMaxChars: number;
|
|
34
|
+
autoVerifyMaxPerTask: number;
|
|
35
|
+
autoVerifyMaxPerSession: number;
|
|
21
36
|
provider: string;
|
|
22
37
|
model: string;
|
|
23
38
|
reasoningEffort?: string;
|
package/lib/types/config.js
CHANGED
|
@@ -3,6 +3,13 @@ import z from 'schemastery';
|
|
|
3
3
|
export const VERIFIER_SETTINGS_NAMESPACE = settingsNamespace('llm-verifier');
|
|
4
4
|
export const Config = z.object({
|
|
5
5
|
enabled: z.boolean().default(true),
|
|
6
|
+
autoVerifyMode: z.union(['manual', 'smart', 'strict']).default('smart'),
|
|
7
|
+
autoVerifyThreshold: z.number().min(0).max(1).default(0.65),
|
|
8
|
+
autoVerifyRepeats: z.number().step(1).min(1).default(1),
|
|
9
|
+
autoVerifyMinToolCalls: z.number().step(1).min(1).default(3),
|
|
10
|
+
autoVerifyMaxChars: z.number().step(1).min(1000).default(80000),
|
|
11
|
+
autoVerifyMaxPerTask: z.number().step(1).min(1).default(2),
|
|
12
|
+
autoVerifyMaxPerSession: z.number().step(1).min(1).default(8),
|
|
6
13
|
provider: z.string().default('deepseek-official'),
|
|
7
14
|
model: z.string().default('deepseek-v4-flash'),
|
|
8
15
|
reasoningEffort: z.string(),
|
|
@@ -23,7 +30,18 @@ export function resolveConfig(config = {}) {
|
|
|
23
30
|
throw new Error('llm-verifier: provider must be non-empty');
|
|
24
31
|
if (!model)
|
|
25
32
|
throw new Error('llm-verifier: model must be non-empty');
|
|
33
|
+
const autoVerifyMode = config.autoVerifyMode ?? 'smart';
|
|
34
|
+
if (!['manual', 'smart', 'strict'].includes(autoVerifyMode))
|
|
35
|
+
throw new Error('llm-verifier: autoVerifyMode must be manual, smart, or strict');
|
|
36
|
+
const autoVerifyThreshold = config.autoVerifyThreshold ?? 0.65;
|
|
37
|
+
if (!Number.isFinite(autoVerifyThreshold) || autoVerifyThreshold < 0 || autoVerifyThreshold > 1)
|
|
38
|
+
throw new Error('llm-verifier: autoVerifyThreshold must be between 0 and 1');
|
|
26
39
|
const values = {
|
|
40
|
+
autoVerifyRepeats: config.autoVerifyRepeats ?? 1,
|
|
41
|
+
autoVerifyMinToolCalls: config.autoVerifyMinToolCalls ?? 3,
|
|
42
|
+
autoVerifyMaxChars: config.autoVerifyMaxChars ?? 80000,
|
|
43
|
+
autoVerifyMaxPerTask: config.autoVerifyMaxPerTask ?? 2,
|
|
44
|
+
autoVerifyMaxPerSession: config.autoVerifyMaxPerSession ?? 8,
|
|
27
45
|
maxTokens: config.maxTokens ?? 32768,
|
|
28
46
|
timeoutMs: config.timeoutMs ?? 300000,
|
|
29
47
|
maxConcurrency: config.maxConcurrency ?? 8,
|
|
@@ -33,6 +51,8 @@ export function resolveConfig(config = {}) {
|
|
|
33
51
|
for (const [name, value] of Object.entries(values))
|
|
34
52
|
if (!Number.isSafeInteger(value) || value <= 0)
|
|
35
53
|
throw new Error('llm-verifier: ' + name + ' must be a positive safe integer');
|
|
54
|
+
if (values.autoVerifyMaxChars < 1000)
|
|
55
|
+
throw new Error('llm-verifier: autoVerifyMaxChars must be at least 1000');
|
|
36
56
|
const maxRetries = config.maxRetries ?? 3;
|
|
37
57
|
if (!Number.isSafeInteger(maxRetries) || maxRetries < 0)
|
|
38
58
|
throw new Error('llm-verifier: maxRetries must be a non-negative safe integer');
|
|
@@ -44,7 +64,7 @@ export function resolveConfig(config = {}) {
|
|
|
44
64
|
if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every(value => Number.isFinite(value) && value >= 0))
|
|
45
65
|
throw new Error('llm-verifier: estimated token prices must be finite non-negative numbers');
|
|
46
66
|
const reasoningEffort = config.reasoningEffort?.trim();
|
|
47
|
-
return { enabled: config.enabled ?? true, provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values };
|
|
67
|
+
return { enabled: config.enabled ?? true, autoVerifyMode, autoVerifyThreshold, provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values };
|
|
48
68
|
}
|
|
49
69
|
export function installVerifierSettings(ctx, entry, onChange) {
|
|
50
70
|
let source = () => entry;
|
package/lib/types/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export * from './core.ts';
|
|
|
7
7
|
export * from './engine.ts';
|
|
8
8
|
export * from './cache.ts';
|
|
9
9
|
export * from './statistics.ts';
|
|
10
|
+
export * from './auto.ts';
|
|
10
11
|
export { callVerifier, RequestLimiter, type VerifierClientConfig, type VerifierImage, type UsageStats, type VerifierCompletion } from './caller.ts';
|
|
11
12
|
export declare function apply(ctx: Context, config?: Config): void;
|
|
12
13
|
//# sourceMappingURL=index.d.ts.map
|
package/lib/types/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
2
|
+
import { createUserMessage } from '@deepseek-ai/dsh-llm';
|
|
2
3
|
import { Config, installVerifierSettings, resolveConfig } from "./config.js";
|
|
3
4
|
import { RequestLimiter } from "./caller.js";
|
|
4
5
|
import { TopLogprobCapabilityCache } from "./top-logprobs.js";
|
|
@@ -6,6 +7,7 @@ import { ScoreCache, resolveCacheFile } from "./cache.js";
|
|
|
6
7
|
import { VerifierEngine, normalizeCriteria } from "./engine.js";
|
|
7
8
|
import { loadVerifierImages } from "./images.js";
|
|
8
9
|
import { extractSession } from "./session.js";
|
|
10
|
+
import { AutoVerificationBudget, analyzeAutoTask, automaticFeedback } from "./auto.js";
|
|
9
11
|
import { StatisticsStore, emptyRunStats, errorDetails, resolveStatisticsFile } from "./statistics.js";
|
|
10
12
|
export const name = 'llm-verifier';
|
|
11
13
|
export const inject = ['tools', 'agents', 'attachments', 'llm', 'connection'];
|
|
@@ -14,6 +16,7 @@ export * from "./core.js";
|
|
|
14
16
|
export * from "./engine.js";
|
|
15
17
|
export * from "./cache.js";
|
|
16
18
|
export * from "./statistics.js";
|
|
19
|
+
export * from "./auto.js";
|
|
17
20
|
export { callVerifier, RequestLimiter } from "./caller.js";
|
|
18
21
|
const criterionSchema = { type: 'object', additionalProperties: false, properties: { id: { type: 'string', required: true }, name: { type: 'string', required: true }, description: { type: 'string', required: true } } };
|
|
19
22
|
const statsSchema = { type: 'object', additionalProperties: false, properties: { calls: { type: 'integer', required: true }, attempts: { type: 'integer', required: true }, retries: { type: 'integer', required: true }, inputTokens: { type: 'integer', required: true }, cachedInputTokens: { type: 'integer', required: true }, outputTokens: { type: 'integer', required: true }, reasoningTokens: { type: 'integer', required: true }, cacheHits: { type: 'integer', required: true }, cacheMisses: { type: 'integer', required: true }, estimatedCostUsd: { type: 'number', required: true }, topLogprobScores: { type: 'integer', required: true }, explicitTagScores: { type: 'integer', required: true } } };
|
|
@@ -29,6 +32,7 @@ function statsFrom(value) { if (typeof value !== 'object' || value === null || !
|
|
|
29
32
|
function rpcSuccess(value) { return { ok: true, value }; }
|
|
30
33
|
function rpcFailure(message) { return { ok: false, error: { code: 'bad-request', message, details: { issues: [] } } }; }
|
|
31
34
|
export function apply(ctx, config = {}) {
|
|
35
|
+
const services = ctx;
|
|
32
36
|
const entry = resolveConfig(config);
|
|
33
37
|
let limiter = new RequestLimiter(entry.maxConcurrency);
|
|
34
38
|
const current = installVerifierSettings(ctx, entry, () => { limiter = new RequestLimiter(current().maxConcurrency); });
|
|
@@ -36,32 +40,40 @@ export function apply(ctx, config = {}) {
|
|
|
36
40
|
const cache = new ScoreCache(cacheFile, entry.cacheMaxEntries);
|
|
37
41
|
const statistics = new StatisticsStore(resolveStatisticsFile(cacheFile));
|
|
38
42
|
const topLogprobCapabilities = new TopLogprobCapabilityCache();
|
|
43
|
+
const autoBudget = new AutoVerificationBudget();
|
|
39
44
|
const engine = async () => {
|
|
40
45
|
const selected = current();
|
|
41
46
|
await ctx.llm.resolveCallConfig({ provider: selected.provider, model: selected.model, ...(selected.reasoningEffort ? { reasoningEffort: selected.reasoningEffort } : {}), maxTokens: selected.maxTokens });
|
|
42
|
-
return { verifier: new VerifierEngine({ ...selected, ctx, llm: ctx.llm, attachments:
|
|
47
|
+
return { verifier: new VerifierEngine({ ...selected, ctx, llm: ctx.llm, attachments: services.attachments, topLogprobCapabilities, limiter }, selected.maxConcurrency, cache, { input: selected.estimatedInputUsdPerMillion, output: selected.estimatedOutputUsdPerMillion }), selected };
|
|
43
48
|
};
|
|
44
49
|
const images = (values, signal) => loadVerifierImages(values, signal);
|
|
45
50
|
const route = (selected) => ({ provider: selected.provider, model: selected.model });
|
|
46
51
|
const requireEnabled = () => { if (!current().enabled)
|
|
47
52
|
throw new Error('llm-verifier: verifier tools are disabled — enable them in Settings → LLM Verifier'); };
|
|
48
|
-
const record = async (toolName,
|
|
53
|
+
const record = async (toolName, agent, operation) => {
|
|
49
54
|
const startedAt = Date.now();
|
|
50
55
|
let selected = current();
|
|
51
56
|
try {
|
|
52
57
|
const completed = await operation();
|
|
53
58
|
selected = completed.selected;
|
|
54
59
|
const value = { ...completed.result, ...route(selected) };
|
|
55
|
-
await statistics.record({ toolName, ...(
|
|
60
|
+
await statistics.record({ toolName, ...(agent ? { sessionId: String(agent.id) } : {}), startedAt, success: true, provider: selected.provider, model: selected.model, stats: statsFrom(value) }).catch(() => { });
|
|
56
61
|
return value;
|
|
57
62
|
}
|
|
58
63
|
catch (error) {
|
|
59
64
|
const details = errorDetails(error);
|
|
60
|
-
await statistics.record({ toolName, ...(
|
|
65
|
+
await statistics.record({ toolName, ...(agent ? { sessionId: String(agent.id) } : {}), startedAt, success: false, ...details, provider: selected.provider, model: selected.model, stats: emptyRunStats() }).catch(() => { });
|
|
61
66
|
throw error;
|
|
62
67
|
}
|
|
63
68
|
};
|
|
64
|
-
|
|
69
|
+
const verifySession = async (agent, options, signal) => record('verifier_current_session', agent, async () => {
|
|
70
|
+
const extracted = await extractSession(agent, async (ref) => { const stored = await services.attachments.readImage(ref, signal); return { data: stored.data, mediaType: stored.ref.mediaType }; }, { fromSeq: options.fromSeq, toSeq: options.toSeq, includeAssistantText: options.includeAssistantText, redactPatterns: options.redactPatterns, maxChars: options.maxChars });
|
|
71
|
+
const { verifier, selected } = await engine();
|
|
72
|
+
const compared = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(options.repeats, 2, 'repeats'), images: extracted.images }, signal);
|
|
73
|
+
const result = { sessionId: extracted.sessionId, problem: extracted.problem, score: compared.scoreA, baselineScore: compared.scoreB, winner: compared.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: compared.calls, stats: compared.stats };
|
|
74
|
+
return { result, selected };
|
|
75
|
+
});
|
|
76
|
+
ctx.effect(() => services.connection.rpc.handle('/llm-verifier', async (endpoint, payload) => {
|
|
65
77
|
if (endpoint !== 'statistics')
|
|
66
78
|
return rpcFailure('unknown llm-verifier endpoint');
|
|
67
79
|
if (typeof payload !== 'object' || payload === null)
|
|
@@ -76,10 +88,30 @@ export function apply(ctx, config = {}) {
|
|
|
76
88
|
return rpcFailure(error instanceof Error ? error.message : String(error));
|
|
77
89
|
}
|
|
78
90
|
}, { authority: 'loopback' }), 'llm-verifier: statistics rpc');
|
|
79
|
-
ctx.
|
|
80
|
-
ctx.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
91
|
+
ctx.on('agent/disposed', ({ agent }) => { autoBudget.release(agent); });
|
|
92
|
+
ctx.on('agent/turn-stopping', async ({ agent, signal }) => {
|
|
93
|
+
const selected = current();
|
|
94
|
+
if (!selected.enabled || selected.autoVerifyMode === 'manual' || signal.aborted)
|
|
95
|
+
return;
|
|
96
|
+
const evidence = analyzeAutoTask(agent.session.events, { mode: selected.autoVerifyMode, minToolCalls: selected.autoVerifyMinToolCalls, maxPerTask: selected.autoVerifyMaxPerTask, maxPerSession: selected.autoVerifyMaxPerSession });
|
|
97
|
+
if (!autoBudget.claim(agent, evidence, { mode: selected.autoVerifyMode, minToolCalls: selected.autoVerifyMinToolCalls, maxPerTask: selected.autoVerifyMaxPerTask, maxPerSession: selected.autoVerifyMaxPerSession }))
|
|
98
|
+
return;
|
|
99
|
+
try {
|
|
100
|
+
const result = await verifySession(agent, { fromSeq: evidence.taskStartSeq, includeAssistantText: true, maxChars: selected.autoVerifyMaxChars, repeats: selected.autoVerifyRepeats }, signal);
|
|
101
|
+
const passed = result.winner === 'A' && result.score >= selected.autoVerifyThreshold;
|
|
102
|
+
if (!passed)
|
|
103
|
+
agent.steer(createUserMessage({ content: [{ type: 'text', text: automaticFeedback(result.score, result.baselineScore, result.winner, selected.autoVerifyThreshold) }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }));
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
ctx.logger.warn('llm-verifier automatic session verification failed: ' + (error instanceof Error ? error.message : String(error)));
|
|
107
|
+
if (selected.autoVerifyMode === 'strict' && !signal.aborted)
|
|
108
|
+
agent.steer(createUserMessage({ content: [{ type: 'text', text: '[Automatic verifier gate]\nThe required strict verification could not run: ' + (error instanceof Error ? error.message : String(error)) + '\nInspect the task evidence and run a directly relevant verification command before concluding.' }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }));
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); return record('verifier_compare', exec.agent, async () => { const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: args.problem, candidateA: args.candidate_a, candidateB: args.candidate_b, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { result, selected }; }); } }));
|
|
112
|
+
ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Deterministic orchestrators should call this directly once they have three or more real candidates.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { requireEnabled(); return record('verifier_select', exec.agent, async () => { const { verifier, selected } = await engine(); const result = await verifier.select({ problem: args.problem, candidates: args.candidates, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), pivots: positive(args.pivots, 2, 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { result, selected }; }); } }));
|
|
113
|
+
ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Deterministic goal/workflow orchestrators should call this directly when real checkpoints already exist. Do not use for a single completed answer or invent checkpoints.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); return record('verifier_track', exec.agent, async () => { const { verifier, selected } = await engine(); const result = await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { result, selected }; }); } }));
|
|
114
|
+
ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Explicitly verify the current DSH session. Smart/strict policy can also invoke this gate automatically at the turn-stopping lifecycle boundary after consequential work with real tool evidence. Extracts the session, applies redaction and bounds, then sends the evidence to the configured verifier model.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = exec.agent ?? ctx.agents.currentInitiator(); if (agent === undefined)
|
|
115
|
+
throw new Error('llm-verifier: verifier_current_session requires an agent-owned tool call'); return verifySession(agent, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars, repeats: args.repeats }, exec.signal); } }));
|
|
84
116
|
}
|
|
85
117
|
//# sourceMappingURL=index.js.map
|
package/package.json
CHANGED
package/src/auto.test.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import { Session } from '@deepseek-ai/dsh-session'
|
|
3
|
+
import { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
|
|
4
|
+
import { AutoVerificationBudget, analyzeAutoTask, automaticFeedback } from './auto.ts'
|
|
5
|
+
|
|
6
|
+
function taskSession() {
|
|
7
|
+
const session = Session.create('session-00000000-0000-4000-8000-000000000009' as never)
|
|
8
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Implement and test the feature' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
9
|
+
return session
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function call(session: ReturnType<typeof taskSession>, name: string, id: string) {
|
|
13
|
+
session.append('tool/call', { turn: 1, step: 1, callId: id as never, name, arguments: '{}' })
|
|
14
|
+
session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: id as never, content: [{ type: 'text', text: 'ok' }], isError: false }) }, { surfaceOp: 'append' })
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const smart = { mode: 'smart' as const, minToolCalls: 3, maxPerTask: 2, maxPerSession: 8 }
|
|
18
|
+
|
|
19
|
+
describe('automatic verification policy', () => {
|
|
20
|
+
it('requires consequential work and enough evidence in smart mode', () => {
|
|
21
|
+
const session = taskSession()
|
|
22
|
+
call(session, 'read', 'one')
|
|
23
|
+
call(session, 'grep', 'two')
|
|
24
|
+
expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'no-consequential-work' })
|
|
25
|
+
call(session, 'edit', 'three')
|
|
26
|
+
expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: true, toolCalls: 3, consequentialToolCalls: 1 })
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
it('strict mode accepts one completed consequential call', () => {
|
|
30
|
+
const session = taskSession()
|
|
31
|
+
call(session, 'ssh_exec', 'one')
|
|
32
|
+
expect(analyzeAutoTask(session.events, { ...smart, mode: 'strict' })).toMatchObject({ eligible: true, reason: 'strict-eligible' })
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
it('manual verification suppresses the automatic gate', () => {
|
|
36
|
+
const session = taskSession()
|
|
37
|
+
call(session, 'edit', 'one')
|
|
38
|
+
call(session, 'pwsh', 'two')
|
|
39
|
+
call(session, 'verifier_current_session', 'three')
|
|
40
|
+
expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'already-verified', hasManualSessionVerification: true })
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
it('resets per-task attempts for a new direct user message and enforces the session cap', () => {
|
|
44
|
+
const session = taskSession()
|
|
45
|
+
call(session, 'edit', 'one')
|
|
46
|
+
call(session, 'read', 'two')
|
|
47
|
+
call(session, 'pwsh', 'three')
|
|
48
|
+
const agent = { id: session.id, session } as never
|
|
49
|
+
const budget = new AutoVerificationBudget()
|
|
50
|
+
const policy = { ...smart, maxPerTask: 1, maxPerSession: 2 }
|
|
51
|
+
expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(true)
|
|
52
|
+
expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(false)
|
|
53
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Do another task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
54
|
+
call(session, 'write', 'four')
|
|
55
|
+
call(session, 'read', 'five')
|
|
56
|
+
call(session, 'pwsh', 'six')
|
|
57
|
+
expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(true)
|
|
58
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Third task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
59
|
+
call(session, 'edit', 'seven')
|
|
60
|
+
call(session, 'read', 'eight')
|
|
61
|
+
call(session, 'pwsh', 'nine')
|
|
62
|
+
expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(false)
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
it('builds actionable low-score feedback', () => {
|
|
66
|
+
expect(automaticFeedback(0.42, 0.31, 'A', 0.65)).toContain('42.0%')
|
|
67
|
+
expect(automaticFeedback(0.42, 0.31, 'A', 0.65)).toContain('verification command')
|
|
68
|
+
})
|
|
69
|
+
})
|
package/src/auto.ts
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
2
|
+
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
3
|
+
|
|
4
|
+
export type AutoVerifyMode = 'manual' | 'smart' | 'strict'
|
|
5
|
+
|
|
6
|
+
export interface AutoVerifyPolicy {
|
|
7
|
+
mode: AutoVerifyMode
|
|
8
|
+
minToolCalls: number
|
|
9
|
+
maxPerTask: number
|
|
10
|
+
maxPerSession: number
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface AutoTaskEvidence {
|
|
14
|
+
taskStartSeq: number
|
|
15
|
+
toolCalls: number
|
|
16
|
+
completedToolResults: number
|
|
17
|
+
consequentialToolCalls: number
|
|
18
|
+
hasManualSessionVerification: boolean
|
|
19
|
+
eligible: boolean
|
|
20
|
+
reason: string
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const PASSIVE_TOOLS = new Set([
|
|
24
|
+
'read', 'read_image', 'glob', 'grep', 'web_search', 'ssh_list', 'job_list',
|
|
25
|
+
'job_output', 'list_agents', 'get_goal', 'skill', 'mcp__codegraph__codegraph_explore',
|
|
26
|
+
])
|
|
27
|
+
const VERIFIER_TOOLS = new Set([
|
|
28
|
+
'verifier_compare', 'verifier_select', 'verifier_track', 'verifier_current_session',
|
|
29
|
+
])
|
|
30
|
+
const CONSEQUENTIAL_TOOLS = new Set([
|
|
31
|
+
'edit', 'write', 'pwsh', 'bash', 'run_code', 'codex_image_generate',
|
|
32
|
+
'ssh_exec', 'ssh_upload', 'ssh_download', 'ssh_tunnel', 'ssh_cluster',
|
|
33
|
+
'job_kill', 'workbench_session_delete', 'create_goal', 'update_goal',
|
|
34
|
+
])
|
|
35
|
+
|
|
36
|
+
function latestDirectUserSeq(events: readonly SessionEvent[]): number | undefined {
|
|
37
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
38
|
+
const event = events[index]
|
|
39
|
+
if (event?.type === 'user/message' && event.data.source.kind === 'user') return event.seq
|
|
40
|
+
}
|
|
41
|
+
return undefined
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function isConsequential(name: string): boolean {
|
|
45
|
+
if (CONSEQUENTIAL_TOOLS.has(name)) return true
|
|
46
|
+
if (PASSIVE_TOOLS.has(name) || VERIFIER_TOOLS.has(name)) return false
|
|
47
|
+
return /(?:edit|write|patch|apply|deploy|upload|delete|remove|kill|exec|shell|command|migration|database|tunnel|cluster)/iu.test(name)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function analyzeAutoTask(events: readonly SessionEvent[], policy: AutoVerifyPolicy): AutoTaskEvidence {
|
|
51
|
+
const taskStartSeq = latestDirectUserSeq(events)
|
|
52
|
+
if (taskStartSeq === undefined) return { taskStartSeq: 0, toolCalls: 0, completedToolResults: 0, consequentialToolCalls: 0, hasManualSessionVerification: false, eligible: false, reason: 'no-direct-user-task' }
|
|
53
|
+
|
|
54
|
+
const relevant = events.filter(event => event.seq >= taskStartSeq)
|
|
55
|
+
const calls = relevant.filter((event): event is SessionEvent<'tool/call'> => event.type === 'tool/call')
|
|
56
|
+
const toolCalls = calls.filter(event => !VERIFIER_TOOLS.has(event.data.name)).length
|
|
57
|
+
const completedToolResults = relevant.filter(event => event.type === 'tool/result' && event.data.error === undefined && event.data.message.content.every(block => block.isError !== true)).length
|
|
58
|
+
const consequentialToolCalls = calls.filter(event => isConsequential(event.data.name)).length
|
|
59
|
+
const hasManualSessionVerification = calls.some(event => event.data.name === 'verifier_current_session')
|
|
60
|
+
|
|
61
|
+
if (policy.mode === 'manual') return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'manual-mode' }
|
|
62
|
+
if (hasManualSessionVerification) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'already-verified' }
|
|
63
|
+
if (consequentialToolCalls === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'no-consequential-work' }
|
|
64
|
+
if (completedToolResults === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'no-completed-evidence' }
|
|
65
|
+
if (policy.mode === 'smart' && toolCalls < policy.minToolCalls) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'insufficient-tool-evidence' }
|
|
66
|
+
return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: true, reason: policy.mode + '-eligible' }
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface SessionBudget {
|
|
70
|
+
taskStartSeq: number
|
|
71
|
+
taskAttempts: number
|
|
72
|
+
sessionAttempts: number
|
|
73
|
+
lastEvaluatedSeq: number
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export class AutoVerificationBudget {
|
|
77
|
+
private readonly states = new Map<string, SessionBudget>()
|
|
78
|
+
|
|
79
|
+
claim(agent: Agent, evidence: AutoTaskEvidence, policy: AutoVerifyPolicy): boolean {
|
|
80
|
+
if (!evidence.eligible) return false
|
|
81
|
+
const id = String(agent.id)
|
|
82
|
+
const previous = this.states.get(id)
|
|
83
|
+
const state: SessionBudget = previous ?? { taskStartSeq: evidence.taskStartSeq, taskAttempts: 0, sessionAttempts: 0, lastEvaluatedSeq: -1 }
|
|
84
|
+
if (state.taskStartSeq !== evidence.taskStartSeq) {
|
|
85
|
+
state.taskStartSeq = evidence.taskStartSeq
|
|
86
|
+
state.taskAttempts = 0
|
|
87
|
+
state.lastEvaluatedSeq = -1
|
|
88
|
+
}
|
|
89
|
+
const lastSeq = agent.session.events.at(-1)?.seq ?? -1
|
|
90
|
+
if (lastSeq <= state.lastEvaluatedSeq || state.taskAttempts >= policy.maxPerTask || state.sessionAttempts >= policy.maxPerSession) {
|
|
91
|
+
this.states.set(id, state)
|
|
92
|
+
return false
|
|
93
|
+
}
|
|
94
|
+
state.taskAttempts += 1
|
|
95
|
+
state.sessionAttempts += 1
|
|
96
|
+
state.lastEvaluatedSeq = lastSeq
|
|
97
|
+
this.states.set(id, state)
|
|
98
|
+
return true
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
release(agent: Agent): void {
|
|
102
|
+
this.states.delete(String(agent.id))
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function automaticFeedback(score: number, baselineScore: number, winner: 'A' | 'B' | 'tie', threshold: number): string {
|
|
107
|
+
const percent = (value: number) => (value * 100).toFixed(1) + '%'
|
|
108
|
+
return [
|
|
109
|
+
'[Automatic verifier gate]',
|
|
110
|
+
`The independent verifier did not clear this task for completion: evidence score ${percent(score)}, baseline ${percent(baselineScore)}, verdict ${winner}, required ${percent(threshold)}.`,
|
|
111
|
+
'Re-open the task requirements, inspect the actual tool outputs for unresolved errors or missing proof, make any necessary corrections, and run a directly relevant verification command before concluding. Do not merely restate that the task is complete.',
|
|
112
|
+
].join('\n')
|
|
113
|
+
}
|