dsh-llm-verifier 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -6
- package/lib/caller-BXrUTAoL.js +324 -0
- package/lib/caller.js +1 -1
- package/lib/client.js +275 -114
- package/lib/index.js +301 -53
- package/lib/types/auto.d.ts +26 -0
- package/lib/types/auto.js +86 -0
- package/lib/types/client.js +16 -11
- package/lib/types/config.d.ts +15 -0
- package/lib/types/config.js +21 -1
- package/lib/types/index.d.ts +1 -0
- package/lib/types/index.js +42 -10
- package/package.json +1 -1
- package/src/auto.test.ts +69 -0
- package/src/auto.ts +113 -0
- package/src/client.tsx +57 -27
- package/src/config.ts +34 -1
- package/src/index.ts +45 -12
package/src/client.tsx
CHANGED
|
@@ -3,11 +3,11 @@ import type { ModelProviderGroup, SettingsNamespaceView } from '@deepseek-ai/dsh
|
|
|
3
3
|
import type {} from '@deepseek-ai/dsh-client-ui-settings/client'
|
|
4
4
|
import type {} from '@deepseek-ai/dsh-client-ui-conversation/client'
|
|
5
5
|
import type {} from '@deepseek-ai/dsh-client-ui-slots'
|
|
6
|
-
import { Button, IconDataOutline16, IconRefreshOutline16, Input
|
|
6
|
+
import { Button, IconDataOutline16, IconRefreshOutline16, Input } from '@deepseek-ai/dsh-client-ui-primitives'
|
|
7
7
|
import { useEffect, useMemo, useState } from 'react'
|
|
8
8
|
|
|
9
9
|
const NS = 'llm-verifier'
|
|
10
|
-
interface Values { enabled: boolean; provider: string; model: string; reasoningEffort?: string; maxTokens: number; maxConcurrency: number; maxRetries: number; timeoutMs: number; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number }
|
|
10
|
+
interface Values { enabled: boolean; autoVerifyMode: 'manual'|'smart'|'strict'; autoVerifyThreshold: number; autoVerifyRepeats: number; autoVerifyMinToolCalls: number; autoVerifyMaxChars: number; autoVerifyMaxPerTask: number; autoVerifyMaxPerSession: number; provider: string; model: string; reasoningEffort?: string; maxTokens: number; maxConcurrency: number; maxRetries: number; timeoutMs: number; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number }
|
|
11
11
|
interface Loaded { groups: ModelProviderGroup[]; settings: SettingsNamespaceView; writable: boolean; failures: string[] }
|
|
12
12
|
interface RunStats { calls: number; attempts: number; retries: number; inputTokens: number; cachedInputTokens: number; outputTokens: number; reasoningTokens: number; cacheHits: number; cacheMisses: number; estimatedCostUsd: number; topLogprobScores: number; explicitTagScores: number }
|
|
13
13
|
interface InvocationRecord { id: string; toolName: string; sessionId?: string; startedAt: number; finishedAt: number; durationMs: number; success: boolean; errorName?: string; errorMessage?: string; provider: string; model: string; stats: RunStats }
|
|
@@ -18,20 +18,25 @@ interface Totals extends RunStats { invocations: number; successes: number; fail
|
|
|
18
18
|
interface StatisticsOverview { generatedAt: number; fromMs: number; toMs: number; sessionId?: string; totals: Totals; daily: DailyStatistics[]; tools: ToolStatistics[]; models: ModelStatistics[]; recent: InvocationRecord[] }
|
|
19
19
|
interface StatisticsPageProps { sessionId: string; rpc: { call(channel: string, endpoint: string, payload: unknown, signal?: AbortSignal): Promise<{ ok: boolean; value?: unknown; error?: { message: string } }> } }
|
|
20
20
|
|
|
21
|
-
const shell: React.CSSProperties = { display: 'flex', flexDirection: 'column', gap:
|
|
22
|
-
const
|
|
23
|
-
const
|
|
24
|
-
const
|
|
25
|
-
const
|
|
21
|
+
const shell: React.CSSProperties = { width: '100%', maxWidth: 720, display: 'flex', flexDirection: 'column', gap: 12, padding: '0 0 32px', color: 'var(--dsw-alias-label-primary)' }
|
|
22
|
+
const settingsHeading: React.CSSProperties = { margin: 0, fontSize: 16, fontWeight: 500, lineHeight: '24px', color: 'var(--dsw-alias-label-primary)' }
|
|
23
|
+
const settingsIntro: React.CSSProperties = { margin: 0, fontSize: 14, lineHeight: '22px', color: 'var(--dsw-alias-label-tertiary)' }
|
|
24
|
+
const group: React.CSSProperties = { width: '100%', display: 'flex', flexDirection: 'column', borderTop: '1px solid var(--dsw-alias-border-l2)' }
|
|
25
|
+
const groupTitle: React.CSSProperties = { margin: 0, padding: '18px 0 8px', fontSize: 14, fontWeight: 500, lineHeight: '22px', color: 'var(--dsw-alias-label-primary)' }
|
|
26
|
+
const row: React.CSSProperties = { minHeight: 64, display: 'grid', gridTemplateColumns: 'minmax(180px, 1fr) minmax(230px, 288px)', gap: 24, alignItems: 'center', padding: '12px 0', borderBottom: '1px solid var(--dsw-alias-border-l2)' }
|
|
27
|
+
const selectStyle: React.CSSProperties = { boxSizing: 'border-box', width: '100%', height: 36, padding: '0 34px 0 12px', borderRadius: 8, color: 'var(--dsw-alias-label-primary)', background: 'var(--dsw-alias-bg-input)', border: '1px solid var(--dsw-alias-border-l2)', font: 'inherit', fontSize: 14, lineHeight: '22px', outline: 'none' }
|
|
28
|
+
const toggleStyle = (enabled: boolean): React.CSSProperties => ({ position: 'relative', justifySelf: 'end', width: 40, height: 22, padding: 0, border: 0, borderRadius: 999, cursor: 'pointer', transition: 'background .15s ease', background: enabled ? 'var(--dsw-alias-state-success-primary)' : 'var(--dsw-alias-bg-input)' })
|
|
29
|
+
const toggleThumbStyle = (enabled: boolean): React.CSSProperties => ({ position: 'absolute', top: 3, left: enabled ? 21 : 3, width: 16, height: 16, borderRadius: '50%', background: 'var(--dsw-static-neutral-00, #fff)', boxShadow: '0 1px 3px rgba(0,0,0,.28)', transition: 'left .15s ease' })
|
|
26
30
|
const dashboardCard: React.CSSProperties = { border: '1px solid var(--dsw-alias-border-l2, rgba(255,255,255,.13))', background: 'color-mix(in srgb, var(--dsw-alias-bg-module, #171925) 88%, transparent)', borderRadius: 16, boxShadow: '0 12px 36px rgba(0,0,0,.12)' }
|
|
27
31
|
const muted: React.CSSProperties = { color: 'var(--dsw-text-secondary)', fontSize: 12 }
|
|
28
32
|
const toolLabels: Record<string, string> = { verifier_compare: '两项对比', verifier_select: '多项优选', verifier_track: '进度跟踪', verifier_current_session: '会话验收' }
|
|
29
33
|
const toolColors: Record<string, string> = { verifier_compare: '#4f8cff', verifier_select: '#8b6df6', verifier_track: '#2fc5c9', verifier_current_session: '#f5a524' }
|
|
30
34
|
|
|
31
35
|
function record(value: unknown): Record<string, unknown> { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {} }
|
|
32
|
-
function values(view: SettingsNamespaceView): Values { const v=record(view.value); return { enabled:v.enabled!==false,provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),timeoutMs:Number(v.timeoutMs??300000),cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0) } }
|
|
36
|
+
function values(view: SettingsNamespaceView): Values { const v=record(view.value); const mode=v.autoVerifyMode==='manual'||v.autoVerifyMode==='strict'?v.autoVerifyMode:'smart'; return { enabled:v.enabled!==false,autoVerifyMode:mode,autoVerifyThreshold:Number(v.autoVerifyThreshold??.65),autoVerifyRepeats:Number(v.autoVerifyRepeats??1),autoVerifyMinToolCalls:Number(v.autoVerifyMinToolCalls??3),autoVerifyMaxChars:Number(v.autoVerifyMaxChars??80000),autoVerifyMaxPerTask:Number(v.autoVerifyMaxPerTask??2),autoVerifyMaxPerSession:Number(v.autoVerifyMaxPerSession??8),provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),timeoutMs:Number(v.timeoutMs??300000),cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0) } }
|
|
33
37
|
function message(error: unknown): string { return error instanceof Error ? error.message : String(error) }
|
|
34
|
-
function Label({title,help}:{title:string;help:string}) { return <div><div style={{fontWeight:
|
|
38
|
+
function Label({title,help}:{title:string;help:string}) { return <div style={{minWidth:0}}><div style={{fontSize:14,fontWeight:400,lineHeight:'22px',color:'var(--dsw-alias-label-primary)'}}>{title}</div><div style={{fontSize:12,lineHeight:'18px',color:'var(--dsw-alias-label-tertiary)',marginTop:2}}>{help}</div></div> }
|
|
39
|
+
function GroupTitle({children}:{children:React.ReactNode}) { return <h3 style={groupTitle}>{children}</h3> }
|
|
35
40
|
function compact(value: number): string { return new Intl.NumberFormat('zh-CN', { notation: value >= 10_000 ? 'compact' : 'standard', maximumFractionDigits: value >= 1_000 ? 1 : 0 }).format(value) }
|
|
36
41
|
function money(value: number): string { return '$' + value.toLocaleString('en-US', { minimumFractionDigits: value < 0.01 ? 4 : 2, maximumFractionDigits: value < 0.01 ? 4 : 2 }) }
|
|
37
42
|
function duration(value: number): string { if (value < 1000) return Math.round(value) + ' ms'; if (value < 60_000) return (value / 1000).toFixed(1) + ' s'; return (value / 60_000).toFixed(1) + ' min' }
|
|
@@ -47,25 +52,50 @@ function VerifierSettings({ api }:{api:any}) {
|
|
|
47
52
|
const selected=models.find(m=>m.id===draft?.model); const efforts=selected?.reasoning?.efforts??[]
|
|
48
53
|
const patch=<K extends keyof Values>(key:K,value:Values[K])=>setDraft(v=>v?{...v,[key]:value}:v)
|
|
49
54
|
const save=async()=>{if(!loaded||!draft)return;setBusy(true);setSaved(false);setError(null);try{const section={...record(loaded.settings.user),...draft};if(!draft.reasoningEffort)delete section.reasoningEffort;const res=await api.settings.update({ns:NS,patch:section,expectedRevision:loaded.settings.revision});if(!res.result.ok)throw new Error(res.result.error.message);setLoaded(v=>v?{...v,settings:res.result.value}:v);setDraft(values(res.result.value));setSaved(true)}catch(e){setError(message(e))}finally{setBusy(false)}}
|
|
50
|
-
if(!loaded||!draft)return <div style={shell}><h2>LLM Verifier</h2><p>{error??'正在读取 DSH 模型和设置…'}</p>{error&&<Button onClick={()=>void load()}>重试</Button>}</div>
|
|
51
|
-
const numeric=(key:keyof Values,min=0)=><Input type="number" min={min} value={String(draft[key])} onChange={e=>patch(key,Number(e.target.value) as never)} />
|
|
55
|
+
if(!loaded||!draft)return <div style={shell}><h2 style={settingsHeading}>LLM Verifier</h2><p style={settingsIntro}>{error??'正在读取 DSH 模型和设置…'}</p>{error&&<div><Button variant="outline" onClick={()=>void load()}>重试</Button></div>}</div>
|
|
56
|
+
const numeric=(key:keyof Values,min=0)=><Input style={{width:'100%',height:36,borderRadius:8}} type="number" min={min} value={String(draft[key])} onChange={e=>patch(key,Number(e.target.value) as never)} />
|
|
52
57
|
return <div style={shell}>
|
|
53
|
-
<
|
|
54
|
-
<
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
<div style={row}><Label title="
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
58
|
+
<h2 style={settingsHeading}>LLM Verifier</h2>
|
|
59
|
+
<p style={settingsIntro}>选择已在 DSH「模型」中配置的模型作为独立裁判。修改后点击页面底部的保存按钮生效。</p>
|
|
60
|
+
|
|
61
|
+
<section style={group}><GroupTitle>工具</GroupTitle>
|
|
62
|
+
<div style={row}><Label title="启用 Verifier 工具" help={draft.enabled?'允许 Agent 调用四个 verifier 工具并向裁判模型发起请求。':'停用后，所有 verifier 工具都会立即返回错误。'}/><button type="button" role="switch" aria-checked={draft.enabled} aria-label="启用 Verifier 工具" onClick={()=>patch('enabled',!draft.enabled)} style={toggleStyle(draft.enabled)}><span style={toggleThumbStyle(draft.enabled)}/></button></div>
|
|
63
|
+
{!draft.enabled&&<p style={{margin:'8px 0 0',fontSize:12,lineHeight:'18px',color:'var(--dsw-alias-state-warn-label)'}}>verifier_compare、verifier_select、verifier_track 和 verifier_current_session 当前不可用。</p>}
|
|
64
|
+
</section>
|
|
65
|
+
|
|
66
|
+
<section style={group}><GroupTitle>自动验收</GroupTitle>
|
|
67
|
+
<div style={row}><Label title="调用策略" help="手动仅暴露工具；智能在复杂且有真实证据的任务结束前验收；严格对任何已完成的写入、执行或远程操作进行验收。"/><select style={selectStyle} value={draft.autoVerifyMode} onChange={e=>patch('autoVerifyMode',e.target.value as Values['autoVerifyMode'])}><option value="manual">手动</option><option value="smart">智能（推荐）</option><option value="strict">严格</option></select></div>
|
|
68
|
+
<div style={row}><Label title="通过阈值" help="会话证据分数达到该值且胜过空工作基线才允许结束；范围 0–1。"/>{numeric('autoVerifyThreshold',0)}</div>
|
|
69
|
+
<div style={row}><Label title="自动评估轮次" help="每项标准的自动评分重复次数；1 为低成本初筛。"/>{numeric('autoVerifyRepeats',1)}</div>
|
|
70
|
+
<div style={row}><Label title="智能模式最少工具调用" help="达到此工具证据数量且包含写入/执行类操作时才自动验收。"/>{numeric('autoVerifyMinToolCalls',1)}</div>
|
|
71
|
+
<div style={row}><Label title="最大证据字符" help="发送给裁判前保留的最近会话证据字符数。"/>{numeric('autoVerifyMaxChars',1000)}</div>
|
|
72
|
+
<div style={row}><Label title="每任务最多验收" help="低分反馈后允许再次验收的次数上限，防止循环。"/>{numeric('autoVerifyMaxPerTask',1)}</div>
|
|
73
|
+
<div style={row}><Label title="每会话最多验收" help="同一会话中的自动验收总预算。"/>{numeric('autoVerifyMaxPerSession',1)}</div>
|
|
74
|
+
{draft.autoVerifyMode!=='manual'&&<p style={{margin:'8px 0 0',fontSize:12,lineHeight:'18px',color:'var(--dsw-alias-state-warn-label)'}}>自动验收会将脱敏后的任务、Assistant 轨迹与真实工具输出发送给所选裁判模型；未通过时会自动要求 Agent 修复并重新验证。</p>}
|
|
75
|
+
</section>
|
|
76
|
+
|
|
77
|
+
<section style={group}><GroupTitle>裁判模型</GroupTitle>
|
|
78
|
+
<div style={row}><Label title="供应商" help="当前 DSH 中可路由的模型供应商。"/><select style={selectStyle} value={draft.provider} onChange={e=>{const provider=e.target.value;const first=loaded.groups.find(g=>g.id===provider)?.models[0];setDraft({...draft,provider,...(first?{model:first.id,reasoningEffort:first.reasoning?.defaultEffort}:{})})}}>{loaded.groups.map(g=><option key={g.id} value={g.id}>{g.name} · {g.id}</option>)}</select></div>
|
|
79
|
+
<div style={row}><Label title="模型" help="用作裁判的具体模型。"/><select style={selectStyle} value={draft.model} onChange={e=>{const model=e.target.value;const found=models.find(m=>m.id===model);setDraft({...draft,model,...(found?.reasoning?.defaultEffort?{reasoningEffort:found.reasoning.defaultEffort}:{reasoningEffort:undefined})})}}>{models.map(m=><option key={m.id} value={m.id}>{m.name} · {m.id}</option>)}</select></div>
|
|
80
|
+
<div style={row}><Label title="推理强度" help="留空时使用所选模型的默认值。"/><select style={selectStyle} value={draft.reasoningEffort??''} onChange={e=>patch('reasoningEffort',e.target.value||undefined)}><option value="">模型默认</option>{efforts.map(e=><option key={e.id} value={e.id}>{e.name}</option>)}</select></div>
|
|
81
|
+
<div style={row}><Label title="最大输出 Token" help="单次裁判请求允许生成的最大 Token 数。"/>{numeric('maxTokens',1)}</div>
|
|
82
|
+
</section>
|
|
83
|
+
|
|
84
|
+
<section style={group}><GroupTitle>执行</GroupTitle>
|
|
85
|
+
<div style={row}><Label title="最大并发" help="所有 verifier 工具共享的请求并发上限。"/>{numeric('maxConcurrency',1)}</div>
|
|
86
|
+
<div style={row}><Label title="最多重试" help="限流、超时和短暂网络错误的重试次数。"/>{numeric('maxRetries',0)}</div>
|
|
87
|
+
<div style={row}><Label title="请求超时" help="单次模型请求的超时时间，单位为毫秒。"/>{numeric('timeoutMs',1)}</div>
|
|
88
|
+
<div style={row}><Label title="缓存条目上限" help="本地持久评分缓存保留的最大条目数。"/>{numeric('cacheMaxEntries',1)}</div>
|
|
89
|
+
</section>
|
|
90
|
+
|
|
91
|
+
<section style={group}><GroupTitle>费用估算</GroupTitle>
|
|
92
|
+
<div style={row}><Label title="输入价格" help="每百万输入 Token 的美元价格，仅用于统计估算。"/>{numeric('estimatedInputUsdPerMillion',0)}</div>
|
|
93
|
+
<div style={row}><Label title="输出价格" help="每百万输出 Token 的美元价格，仅用于统计估算。"/>{numeric('estimatedOutputUsdPerMillion',0)}</div>
|
|
94
|
+
</section>
|
|
95
|
+
|
|
96
|
+
{loaded.failures.length>0&&<div style={{padding:'10px 12px',borderRadius:8,background:'var(--dsw-alias-state-warn-bg)',color:'var(--dsw-alias-state-warn-label)',fontSize:12,lineHeight:'18px'}}><div style={{fontWeight:500,marginBottom:3}}>部分模型目录读取失败</div>{loaded.failures.map(x=><div key={x}>{x}</div>)}</div>}
|
|
97
|
+
{error&&<p style={{margin:0,fontSize:12,lineHeight:'18px',color:'var(--dsw-alias-state-error-primary)'}}>{error}</p>}{saved&&<p style={{margin:0,fontSize:12,lineHeight:'18px',color:'var(--dsw-alias-state-success-primary)'}}>设置已保存。</p>}
|
|
98
|
+
<div style={{display:'flex',justifyContent:'flex-end',gap:8,paddingTop:4}}><Button variant="outline" disabled={busy} onClick={()=>void load()}>重新载入</Button><Button variant="primary" disabled={busy||!loaded.writable} onClick={()=>void save()}>{busy?'保存中…':'保存设置'}</Button></div>
|
|
69
99
|
</div>
|
|
70
100
|
}
|
|
71
101
|
|
package/src/config.ts
CHANGED
|
@@ -4,8 +4,17 @@ import z from 'schemastery'
|
|
|
4
4
|
|
|
5
5
|
export const VERIFIER_SETTINGS_NAMESPACE = settingsNamespace('llm-verifier')
|
|
6
6
|
|
|
7
|
+
export type AutoVerifyMode = 'manual' | 'smart' | 'strict'
|
|
8
|
+
|
|
7
9
|
export interface Config {
|
|
8
10
|
enabled?: boolean
|
|
11
|
+
autoVerifyMode?: AutoVerifyMode
|
|
12
|
+
autoVerifyThreshold?: number
|
|
13
|
+
autoVerifyRepeats?: number
|
|
14
|
+
autoVerifyMinToolCalls?: number
|
|
15
|
+
autoVerifyMaxChars?: number
|
|
16
|
+
autoVerifyMaxPerTask?: number
|
|
17
|
+
autoVerifyMaxPerSession?: number
|
|
9
18
|
provider?: string
|
|
10
19
|
model?: string
|
|
11
20
|
reasoningEffort?: string
|
|
@@ -22,6 +31,13 @@ export interface Config {
|
|
|
22
31
|
|
|
23
32
|
export interface ResolvedConfig {
|
|
24
33
|
enabled: boolean
|
|
34
|
+
autoVerifyMode: AutoVerifyMode
|
|
35
|
+
autoVerifyThreshold: number
|
|
36
|
+
autoVerifyRepeats: number
|
|
37
|
+
autoVerifyMinToolCalls: number
|
|
38
|
+
autoVerifyMaxChars: number
|
|
39
|
+
autoVerifyMaxPerTask: number
|
|
40
|
+
autoVerifyMaxPerSession: number
|
|
25
41
|
provider: string
|
|
26
42
|
model: string
|
|
27
43
|
reasoningEffort?: string
|
|
@@ -38,6 +54,13 @@ export interface ResolvedConfig {
|
|
|
38
54
|
|
|
39
55
|
export const Config: z<Config> = z.object({
|
|
40
56
|
enabled: z.boolean().default(true),
|
|
57
|
+
autoVerifyMode: z.union(['manual', 'smart', 'strict']).default('smart'),
|
|
58
|
+
autoVerifyThreshold: z.number().min(0).max(1).default(0.65),
|
|
59
|
+
autoVerifyRepeats: z.number().step(1).min(1).default(1),
|
|
60
|
+
autoVerifyMinToolCalls: z.number().step(1).min(1).default(3),
|
|
61
|
+
autoVerifyMaxChars: z.number().step(1).min(1000).default(80000),
|
|
62
|
+
autoVerifyMaxPerTask: z.number().step(1).min(1).default(2),
|
|
63
|
+
autoVerifyMaxPerSession: z.number().step(1).min(1).default(8),
|
|
41
64
|
provider: z.string().default('deepseek-official'),
|
|
42
65
|
model: z.string().default('deepseek-v4-flash'),
|
|
43
66
|
reasoningEffort: z.string(),
|
|
@@ -57,7 +80,16 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
|
|
|
57
80
|
const model = (config.model ?? 'deepseek-v4-flash').trim()
|
|
58
81
|
if (!provider) throw new Error('llm-verifier: provider must be non-empty')
|
|
59
82
|
if (!model) throw new Error('llm-verifier: model must be non-empty')
|
|
83
|
+
const autoVerifyMode = config.autoVerifyMode ?? 'smart'
|
|
84
|
+
if (!['manual', 'smart', 'strict'].includes(autoVerifyMode)) throw new Error('llm-verifier: autoVerifyMode must be manual, smart, or strict')
|
|
85
|
+
const autoVerifyThreshold = config.autoVerifyThreshold ?? 0.65
|
|
86
|
+
if (!Number.isFinite(autoVerifyThreshold) || autoVerifyThreshold < 0 || autoVerifyThreshold > 1) throw new Error('llm-verifier: autoVerifyThreshold must be between 0 and 1')
|
|
60
87
|
const values = {
|
|
88
|
+
autoVerifyRepeats: config.autoVerifyRepeats ?? 1,
|
|
89
|
+
autoVerifyMinToolCalls: config.autoVerifyMinToolCalls ?? 3,
|
|
90
|
+
autoVerifyMaxChars: config.autoVerifyMaxChars ?? 80000,
|
|
91
|
+
autoVerifyMaxPerTask: config.autoVerifyMaxPerTask ?? 2,
|
|
92
|
+
autoVerifyMaxPerSession: config.autoVerifyMaxPerSession ?? 8,
|
|
61
93
|
maxTokens: config.maxTokens ?? 32768,
|
|
62
94
|
timeoutMs: config.timeoutMs ?? 300000,
|
|
63
95
|
maxConcurrency: config.maxConcurrency ?? 8,
|
|
@@ -65,6 +97,7 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
|
|
|
65
97
|
cacheMaxEntries: config.cacheMaxEntries ?? 10000,
|
|
66
98
|
}
|
|
67
99
|
for (const [name, value] of Object.entries(values)) if (!Number.isSafeInteger(value) || value <= 0) throw new Error('llm-verifier: ' + name + ' must be a positive safe integer')
|
|
100
|
+
if (values.autoVerifyMaxChars < 1000) throw new Error('llm-verifier: autoVerifyMaxChars must be at least 1000')
|
|
68
101
|
const maxRetries = config.maxRetries ?? 3
|
|
69
102
|
if (!Number.isSafeInteger(maxRetries) || maxRetries < 0) throw new Error('llm-verifier: maxRetries must be a non-negative safe integer')
|
|
70
103
|
const cacheDir = (config.cacheDir ?? '.dsh-verifier-cache').trim()
|
|
@@ -73,7 +106,7 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
|
|
|
73
106
|
const estimatedOutputUsdPerMillion = config.estimatedOutputUsdPerMillion ?? 0
|
|
74
107
|
if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every(value => Number.isFinite(value) && value >= 0)) throw new Error('llm-verifier: estimated token prices must be finite non-negative numbers')
|
|
75
108
|
const reasoningEffort = config.reasoningEffort?.trim()
|
|
76
|
-
return { enabled: config.enabled ?? true, provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values }
|
|
109
|
+
return { enabled: config.enabled ?? true, autoVerifyMode, autoVerifyThreshold, provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values }
|
|
77
110
|
}
|
|
78
111
|
|
|
79
112
|
export function installVerifierSettings(ctx: Context, entry: ResolvedConfig, onChange: () => void): () => ResolvedConfig {
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { Context } from '@deepseek-ai/cordis'
|
|
2
|
-
import type {} from '@deepseek-ai/dsh-client-connection'
|
|
2
|
+
import type { HostConnectionHandle } from '@deepseek-ai/dsh-client-connection'
|
|
3
3
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
4
|
-
import
|
|
4
|
+
import { createUserMessage } from '@deepseek-ai/dsh-llm'
|
|
5
|
+
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
6
|
+
import type { AttachmentStore, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
5
7
|
import { Config, installVerifierSettings, resolveConfig } from './config.ts'
|
|
6
8
|
import { RequestLimiter } from './caller.ts'
|
|
7
9
|
import { TopLogprobCapabilityCache } from './top-logprobs.ts'
|
|
@@ -9,7 +11,8 @@ import { ScoreCache, resolveCacheFile } from './cache.ts'
|
|
|
9
11
|
import { VerifierEngine, normalizeCriteria, type RunStats } from './engine.ts'
|
|
10
12
|
import { loadVerifierImages } from './images.ts'
|
|
11
13
|
import { extractSession } from './session.ts'
|
|
12
|
-
import {
|
|
14
|
+
import { AutoVerificationBudget, analyzeAutoTask, automaticFeedback } from './auto.ts'
|
|
15
|
+
import { StatisticsStore, emptyRunStats, errorDetails, resolveStatisticsFile, type StatisticsOverview, type VerifierToolName } from './statistics.ts'
|
|
13
16
|
|
|
14
17
|
export const name = 'llm-verifier'
|
|
15
18
|
export const inject = ['tools', 'agents', 'attachments', 'llm', 'connection']
|
|
@@ -18,6 +21,7 @@ export * from './core.ts'
|
|
|
18
21
|
export * from './engine.ts'
|
|
19
22
|
export * from './cache.ts'
|
|
20
23
|
export * from './statistics.ts'
|
|
24
|
+
export * from './auto.ts'
|
|
21
25
|
export { callVerifier, RequestLimiter, type VerifierClientConfig, type VerifierImage, type UsageStats, type VerifierCompletion } from './caller.ts'
|
|
22
26
|
|
|
23
27
|
const criterionSchema = { type: 'object' as const, additionalProperties: false, properties: { id: { type: 'string' as const, required: true as const }, name: { type: 'string' as const, required: true as const }, description: { type: 'string' as const, required: true as const } } }
|
|
@@ -31,7 +35,11 @@ function statsFrom(value: unknown): RunStats { if (typeof value !== 'object' ||
|
|
|
31
35
|
function rpcSuccess<T>(value: T) { return { ok: true as const, value } }
|
|
32
36
|
function rpcFailure(message: string) { return { ok: false as const, error: { code: 'bad-request' as const, message, details: { issues: [] } } } }
|
|
33
37
|
|
|
38
|
+
interface SessionVerificationOptions { fromSeq?: number; toSeq?: number; includeAssistantText?: boolean; redactPatterns?: readonly string[]; maxChars?: number; repeats?: number }
|
|
39
|
+
interface SessionVerificationResult { sessionId: string; problem: string; score: number; baselineScore: number; winner: 'A' | 'B' | 'tie'; fromSeq: number; toSeq: number; omittedCharacters: number; calls: number; stats: RunStats }
|
|
40
|
+
|
|
34
41
|
export function apply(ctx: Context, config: Config = {}): void {
|
|
42
|
+
const services = ctx as Context & { attachments: AttachmentStore; connection: HostConnectionHandle }
|
|
35
43
|
const entry = resolveConfig(config)
|
|
36
44
|
let limiter = new RequestLimiter(entry.maxConcurrency)
|
|
37
45
|
const current = installVerifierSettings(ctx, entry, () => { limiter = new RequestLimiter(current().maxConcurrency) })
|
|
@@ -39,30 +47,39 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
39
47
|
const cache = new ScoreCache(cacheFile, entry.cacheMaxEntries)
|
|
40
48
|
const statistics = new StatisticsStore(resolveStatisticsFile(cacheFile))
|
|
41
49
|
const topLogprobCapabilities = new TopLogprobCapabilityCache()
|
|
50
|
+
const autoBudget = new AutoVerificationBudget()
|
|
42
51
|
const engine = async () => {
|
|
43
52
|
const selected = current()
|
|
44
53
|
await ctx.llm.resolveCallConfig({ provider: selected.provider, model: selected.model, ...(selected.reasoningEffort ? { reasoningEffort: selected.reasoningEffort as never } : {}), maxTokens: selected.maxTokens })
|
|
45
|
-
return { verifier: new VerifierEngine({ ...selected, ctx, llm: ctx.llm, attachments:
|
|
54
|
+
return { verifier: new VerifierEngine({ ...selected, ctx, llm: ctx.llm, attachments: services.attachments, topLogprobCapabilities, limiter }, selected.maxConcurrency, cache, { input: selected.estimatedInputUsdPerMillion, output: selected.estimatedOutputUsdPerMillion }), selected }
|
|
46
55
|
}
|
|
47
56
|
const images = (values: readonly string[] | undefined, signal: AbortSignal) => loadVerifierImages(values, signal)
|
|
48
57
|
const route = (selected: { provider: string; model: string }) => ({ provider: selected.provider, model: selected.model })
|
|
49
58
|
const requireEnabled = () => { if (!current().enabled) throw new Error('llm-verifier: verifier tools are disabled — enable them in Settings → LLM Verifier') }
|
|
50
|
-
const record = async <T>(toolName: VerifierToolName,
|
|
59
|
+
const record = async <T>(toolName: VerifierToolName, agent: { id: string } | undefined, operation: () => Promise<{ result: T; selected: { provider: string; model: string } }>): Promise<T & { provider: string; model: string }> => {
|
|
51
60
|
const startedAt = Date.now()
|
|
52
61
|
let selected: { provider: string; model: string } = current()
|
|
53
62
|
try {
|
|
54
63
|
const completed = await operation()
|
|
55
64
|
selected = completed.selected
|
|
56
65
|
const value = { ...completed.result as T & object, ...route(selected) } as T & { provider: string; model: string }
|
|
57
|
-
await statistics.record({ toolName, ...(
|
|
66
|
+
await statistics.record({ toolName, ...(agent ? { sessionId: String(agent.id) } : {}), startedAt, success: true, provider: selected.provider, model: selected.model, stats: statsFrom(value) }).catch(() => {})
|
|
58
67
|
return value
|
|
59
68
|
} catch (error) {
|
|
60
69
|
const details = errorDetails(error)
|
|
61
|
-
await statistics.record({ toolName, ...(
|
|
70
|
+
await statistics.record({ toolName, ...(agent ? { sessionId: String(agent.id) } : {}), startedAt, success: false, ...details, provider: selected.provider, model: selected.model, stats: emptyRunStats() }).catch(() => {})
|
|
62
71
|
throw error
|
|
63
72
|
}
|
|
64
73
|
}
|
|
65
|
-
|
|
74
|
+
const verifySession = async (agent: Agent, options: SessionVerificationOptions, signal: AbortSignal): Promise<SessionVerificationResult & { provider: string; model: string }> => record('verifier_current_session', agent, async () => {
|
|
75
|
+
const extracted = await extractSession(agent, async (ref: ImageAttachmentRef) => { const stored = await services.attachments.readImage(ref, signal); return { data: stored.data, mediaType: stored.ref.mediaType } }, { fromSeq: options.fromSeq, toSeq: options.toSeq, includeAssistantText: options.includeAssistantText, redactPatterns: options.redactPatterns, maxChars: options.maxChars })
|
|
76
|
+
const { verifier, selected } = await engine()
|
|
77
|
+
const compared = await verifier.compare({ problem: extracted.problem, candidateA: extracted.trace, candidateB: '(No useful work or verification was performed.)', repeats: positive(options.repeats, 2, 'repeats'), images: extracted.images }, signal)
|
|
78
|
+
const result: SessionVerificationResult = { sessionId: extracted.sessionId, problem: extracted.problem, score: compared.scoreA, baselineScore: compared.scoreB, winner: compared.winner, fromSeq: extracted.fromSeq, toSeq: extracted.toSeq, omittedCharacters: extracted.omittedCharacters, calls: compared.calls, stats: compared.stats }
|
|
79
|
+
return { result, selected }
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
ctx.effect(() => services.connection.rpc.handle('/llm-verifier', async (endpoint: string, payload: unknown) => {
|
|
66
83
|
if (endpoint !== 'statistics') return rpcFailure('unknown llm-verifier endpoint')
|
|
67
84
|
if (typeof payload !== 'object' || payload === null) return rpcFailure('statistics payload must be an object')
|
|
68
85
|
const row = payload as Record<string, unknown>
|
|
@@ -73,11 +90,27 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
73
90
|
} catch (error) { return rpcFailure(error instanceof Error ? error.message : String(error)) }
|
|
74
91
|
}, { authority: 'loopback' }), 'llm-verifier: statistics rpc')
|
|
75
92
|
|
|
76
|
-
ctx.
|
|
93
|
+
ctx.on('agent/disposed', ({ agent }) => { autoBudget.release(agent) })
|
|
94
|
+
ctx.on('agent/turn-stopping', async ({ agent, signal }) => {
|
|
95
|
+
const selected = current()
|
|
96
|
+
if (!selected.enabled || selected.autoVerifyMode === 'manual' || signal.aborted) return
|
|
97
|
+
const evidence = analyzeAutoTask(agent.session.events, { mode: selected.autoVerifyMode, minToolCalls: selected.autoVerifyMinToolCalls, maxPerTask: selected.autoVerifyMaxPerTask, maxPerSession: selected.autoVerifyMaxPerSession })
|
|
98
|
+
if (!autoBudget.claim(agent, evidence, { mode: selected.autoVerifyMode, minToolCalls: selected.autoVerifyMinToolCalls, maxPerTask: selected.autoVerifyMaxPerTask, maxPerSession: selected.autoVerifyMaxPerSession })) return
|
|
99
|
+
try {
|
|
100
|
+
const result = await verifySession(agent, { fromSeq: evidence.taskStartSeq, includeAssistantText: true, maxChars: selected.autoVerifyMaxChars, repeats: selected.autoVerifyRepeats }, signal)
|
|
101
|
+
const passed = result.winner === 'A' && result.score >= selected.autoVerifyThreshold
|
|
102
|
+
if (!passed) agent.steer(createUserMessage({ content: [{ type: 'text', text: automaticFeedback(result.score, result.baselineScore, result.winner, selected.autoVerifyThreshold) }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }))
|
|
103
|
+
} catch (error) {
|
|
104
|
+
ctx.logger.warn('llm-verifier automatic session verification failed: ' + (error instanceof Error ? error.message : String(error)))
|
|
105
|
+
if (selected.autoVerifyMode === 'strict' && !signal.aborted) agent.steer(createUserMessage({ content: [{ type: 'text', text: '[Automatic verifier gate]\nThe required strict verification could not run: ' + (error instanceof Error ? error.message : String(error)) + '\nInspect the task evidence and run a directly relevant verification command before concluding.' }], source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } }))
|
|
106
|
+
}
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
ctx.tools.register(defineTool({ name: 'verifier_compare', description: 'Use autonomously when exactly two substantive answers, patches, plans, or execution trajectories need an independent evidence-based comparison and the choice is consequential or uncertain. Do not use for trivial deterministic questions or when there is only one candidate. Uses the verifier model selected in DSH Settings, with top-logprob A–T expectations when supported and explicit-tag fallback otherwise.', parameters: { problem: { type: 'string', required: true }, candidate_a: { type: 'string', required: true }, candidate_b: { type: 'string', required: true }, ...commonParams }, output: { schema: { type: 'object', additionalProperties: false, properties: { scoreA: { type: 'number', required: true }, scoreB: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, criteria: { type: 'array', items: criterionResultSchema, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); return record('verifier_compare', exec.agent, async () => { const { verifier, selected } = await engine(); const result = await verifier.compare({ problem: args.problem, candidateA: args.candidate_a, candidateB: args.candidate_b, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), images: await images(args.images, exec.signal) }, exec.signal); return { result, selected } }) } }))
|
|
77
110
|
|
|
78
|
-
ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool.
|
|
111
|
+
ctx.tools.register(defineTool({ name: 'verifier_select', description: 'Use autonomously when three or more substantive candidate answers, patches, plans, or trajectories must be ranked and an independent choice is valuable. Use verifier_compare for exactly two candidates; do not generate extra candidates merely to invoke this tool. Deterministic orchestrators should call this directly once they have three or more real candidates.', parameters: { problem: { type: 'string', required: true }, candidates: { type: 'array', items: { type: 'string' }, required: true }, ...commonParams, pivots: { type: 'integer' }, seed: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { index: { type: 'integer', required: true }, best: { type: 'string', required: true }, scores: { type: 'array', items: { type: 'number' }, required: true }, ranking: { type: 'array', items: { type: 'integer' }, required: true }, pivots: { type: 'array', items: { type: 'integer' }, required: true }, comparisons: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 100, async execute(args, exec) { requireEnabled(); return record('verifier_select', exec.agent, async () => { const { verifier, selected } = await engine(); const result = await verifier.select({ problem: args.problem, candidates: args.candidates, criteria: normalizeCriteria(args.criteria), repeats: positive(args.repeats, 2, 'repeats'), pivots: positive(args.pivots, 2, 'pivots'), seed: args.seed ?? 0, images: await images(args.images, exec.signal) }, exec.signal); return { result, selected } }) } }))
|
|
79
112
|
|
|
80
|
-
ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement.
|
|
113
|
+
ctx.tools.register(defineTool({ name: 'verifier_track', description: 'Use autonomously for a genuinely multi-step task when progress at explicit checkpoints is uncertain or needs evidence-based measurement. Deterministic goal/workflow orchestrators should call this directly when real checkpoints already exist. Do not use for a single completed answer or invent checkpoints.', parameters: { problem: { type: 'string', required: true }, steps: { type: 'array', items: { type: 'string' }, required: true }, checkpoints: { type: 'array', items: { type: 'integer' }, required: true }, repeats: commonParams.repeats, images: commonParams.images }, output: { schema: { type: 'object', additionalProperties: false, properties: { scores: { type: 'array', items: { type: 'number' }, required: true }, perRepeat: { type: 'array', items: { type: 'array', items: { type: 'number' } }, required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); return record('verifier_track', exec.agent, async () => { const { verifier, selected } = await engine(); const result = await verifier.track(args.problem, args.steps, args.checkpoints, positive(args.repeats, 2, 'repeats'), exec.signal, await images(args.images, exec.signal)); return { result, selected } }) } }))
|
|
81
114
|
|
|
82
|
-
ctx.tools.register(defineTool({ name: 'verifier_current_session', description: '
|
|
115
|
+
ctx.tools.register(defineTool({ name: 'verifier_current_session', description: 'Explicitly verify the current DSH session. Smart/strict policy can also invoke this gate automatically at the turn-stopping lifecycle boundary after consequential work with real tool evidence. Extracts the session, applies redaction and bounds, then sends the evidence to the configured verifier model.', parameters: { from_seq: { type: 'integer' }, to_seq: { type: 'integer' }, include_assistant_text: { type: 'boolean' }, redact_patterns: { type: 'array', items: { type: 'string' } }, max_chars: { type: 'integer' }, repeats: { type: 'integer' } }, output: { schema: { type: 'object', additionalProperties: false, properties: { sessionId: { type: 'string', required: true }, problem: { type: 'string', required: true }, score: { type: 'number', required: true }, baselineScore: { type: 'number', required: true }, winner: { type: 'string', enum: ['A', 'B', 'tie'], required: true }, fromSeq: { type: 'integer', required: true }, toSeq: { type: 'integer', required: true }, omittedCharacters: { type: 'integer', required: true }, calls: { type: 'integer', required: true }, stats: { ...statsSchema, required: true }, provider: { type: 'string', required: true }, model: { type: 'string', required: true } } }, render: (_args, value) => renderJson(value) }, timeoutMs: entry.timeoutMs * 20, async execute(args, exec) { requireEnabled(); const agent = exec.agent ?? ctx.agents.currentInitiator(); if (agent === undefined) throw new Error('llm-verifier: verifier_current_session requires an agent-owned tool call'); return verifySession(agent, { fromSeq: args.from_seq, toSeq: args.to_seq, includeAssistantText: args.include_assistant_text, redactPatterns: args.redact_patterns, maxChars: args.max_chars, repeats: args.repeats }, exec.signal) } }))
|
|
83
116
|
}
|