dsh-llm-verifier 0.1.0

package/lib/types/top-logprobs.js

@@ -0,0 +1,97 @@
+import { credentialRef } from '@deepseek-ai/dsh-credentials';
+import { settingsNamespace } from '@deepseek-ai/dsh-settings';
+export class TopLogprobsUnsupportedError extends Error {
+    constructor(message) { super(message); this.name = 'TopLogprobsUnsupportedError'; }
+}
+function object(value) { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : undefined; }
+function text(value) { return typeof value === 'string' && value.trim() ? value.trim() : undefined; }
+function endpoint(baseURL) { return baseURL.replace(/\/+$/, '') + '/chat/completions'; }
+function dataUrl(image) { return 'data:' + image.mediaType + ';base64,' + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).toString('base64'); }
+async function credential(ctx, name) {
+    if (!name)
+        return undefined;
+    const provider = ctx.get('credentials');
+    return (await provider?.resolve(credentialRef(name)))?.value;
+}
+export async function resolveTopLogprobRoute(ctx, provider) {
+    const settings = ctx.get('settings');
+    if (provider === 'deepseek-official') {
+        const value = settings ? object(settings.get(settingsNamespace('llm-deepseek'))) ?? {} : {};
+        const apiKeyEnv = text(value.apiKeyEnv) ?? 'DEEPSEEK_API_KEY';
+        const apiKey = await credential(ctx, apiKeyEnv);
+        if (!apiKey)
+            return undefined;
+        return { baseURL: text(value.baseURL) ?? 'https://api.deepseek.com', apiKey, deepSeekThinking: true };
+    }
+    if (!settings)
+        return undefined;
+    const root = object(settings.get(settingsNamespace('llm-pi-ai')));
+    const profiles = object(root?.providers);
+    const profile = object(profiles?.[provider]);
+    // Only explicitly OpenAI-compatible profiles are safe to serialize directly.
+    // Other DSH adapters keep their private protocol and use the explicit-tag fallback.
+    if (!profile || profile.api !== 'openai-completions')
+        return undefined;
+    const baseURL = text(profile.baseURL);
+    if (!baseURL || !/^https:\/\//i.test(baseURL))
+        return undefined;
+    const apiKey = await credential(ctx, text(profile.apiKeyEnv));
+    const rawHeaders = object(profile.headers);
+    const headers = rawHeaders === undefined ? undefined : Object.fromEntries(Object.entries(rawHeaders).filter((entry) => typeof entry[1] === 'string'));
+    return { baseURL, ...(apiKey ? { apiKey } : {}), ...(headers ? { headers } : {}), deepSeekThinking: false };
+}
+export async function callTopLogprobs(route, model, prompt, maxTokens, reasoningEffort, signal, images) {
+    const content = images?.length ? [{ type: 'text', text: prompt }, ...images.map(image => ({ type: 'image_url', image_url: { url: dataUrl(image) } }))] : prompt;
+    const thinking = route.deepSeekThinking && reasoningEffort ? reasoningEffort === 'off' ? { thinking: { type: 'disabled' } } : { thinking: { type: 'enabled' }, reasoning_effort: reasoningEffort } : {};
+    const response = await fetch(endpoint(route.baseURL), {
+        method: 'POST', redirect: 'error', signal,
+        headers: { 'content-type': 'application/json', ...(route.apiKey ? { authorization: 'Bearer ' + route.apiKey } : {}), ...route.headers },
+        body: JSON.stringify({ model, messages: [{ role: 'user', content }], max_tokens: maxTokens, temperature: 1, logprobs: true, top_logprobs: 20, ...thinking }),
+    });
+    const raw = await response.text();
+    if (!response.ok) {
+        const excerpt = raw.slice(0, 1000);
+        if ([400, 404, 405, 415, 422].includes(response.status) && /logprob|top_logprobs|unsupported|unknown (?:field|parameter)|unrecognized (?:field|parameter)|not support/i.test(excerpt))
+            throw new TopLogprobsUnsupportedError('provider rejected top_logprobs: HTTP ' + response.status + ' ' + excerpt);
+        throw new Error('llm-verifier: top_logprobs request failed with HTTP ' + response.status + ': ' + excerpt);
+    }
+    let body;
+    try {
+        body = object(JSON.parse(raw)) ?? {};
+    }
+    catch {
+        throw new Error('llm-verifier: top_logprobs endpoint returned invalid JSON');
+    }
+    const choices = Array.isArray(body.choices) ? body.choices : [];
+    const choice = object(choices[0]);
+    const message = object(choice?.message);
+    const answer = typeof message?.content === 'string' ? message.content : '';
+    const logprobs = object(choice?.logprobs);
+    const rows = Array.isArray(logprobs?.content) ? logprobs.content : [];
+    if (!rows.length)
+        throw new TopLogprobsUnsupportedError('provider returned no token logprobs');
+    const tokens = [];
+    const positions = [];
+    for (const rawRow of rows) {
+        const row = object(rawRow) ?? {};
+        const token = typeof row.token === 'string' ? row.token : '';
+        tokens.push(token);
+        const top = Array.isArray(row.top_logprobs) ? row.top_logprobs : [];
+        const alternatives = top.flatMap(value => { const item = object(value); return item && typeof item.token === 'string' && typeof item.logprob === 'number' ? [{ token: item.token, logprob: item.logprob }] : []; });
+        if (!alternatives.length && typeof row.logprob === 'number')
+            alternatives.push({ token, logprob: row.logprob });
+        positions.push(alternatives);
+    }
+    const rawUsage = object(body.usage) ?? {};
+    const promptDetails = object(rawUsage.prompt_tokens_details) ?? {};
+    const completionDetails = object(rawUsage.completion_tokens_details) ?? {};
+    const cached = Number(rawUsage.prompt_cache_hit_tokens ?? promptDetails.cached_tokens ?? 0) || 0;
+    const input = Number(rawUsage.prompt_tokens ?? 0) || 0;
+    return { text: answer, tokens, positions, scoringMode: 'top-logprobs', usage: { calls: 1, attempts: 1, retries: 0, inputTokens: Math.max(0, input - cached), cachedInputTokens: cached, outputTokens: Number(rawUsage.completion_tokens ?? 0) || 0, reasoningTokens: Number(completionDetails.reasoning_tokens ?? 0) || 0 } };
+}
+export class TopLogprobCapabilityCache {
+    unsupported = new Set();
+    isUnsupported(provider, model) { return this.unsupported.has(provider + '\0' + model); }
+    markUnsupported(provider, model) { this.unsupported.add(provider + '\0' + model); }
+}
+//# sourceMappingURL=top-logprobs.js.map

package/package.json

@@ -0,0 +1,110 @@
+{
+  "name": "dsh-llm-verifier",
+  "version": "0.1.0",
+  "description": "Configurable DSH-native LLM verifier with a Web settings page",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Aa728848/dsh-llm-verifier.git"
+  },
+  "homepage": "https://github.com/Aa728848/dsh-llm-verifier#readme",
+  "bugs": {
+    "url": "https://github.com/Aa728848/dsh-llm-verifier/issues"
+  },
+  "keywords": [
+    "deepseek",
+    "dsh",
+    "verifier",
+    "llm",
+    "cordis",
+    "plugin"
+  ],
+  "author": "eddyskywalker",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./core": {
+      "types": "./lib/types/core.d.ts",
+      "default": "./lib/core.js"
+    },
+    "./caller": {
+      "types": "./lib/types/caller.d.ts",
+      "default": "./lib/caller.js"
+    },
+    "./package.json": "./package.json",
+    "./client": {
+      "types": "./lib/types/client.d.ts",
+      "default": "./lib/client.js"
+    }
+  },
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    },
+    "client": {
+      "inject": [
+        "@deepseek-ai/dsh-client-connection",
+        "@deepseek-ai/dsh-client-runtime",
+        "@deepseek-ai/dsh-client-ui-slots",
+        "@deepseek-ai/dsh-client-ui-settings",
+        "@deepseek-ai/dsh-client-locale",
+        "@deepseek-ai/dsh-api-remotes"
+      ],
+      "platform": "web"
+    }
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-tools": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-agent": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-session": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-llm": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-attachment": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-settings": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-credentials": "^0.1.0-rc.7"
+  },
+  "dependencies": {
+    "schemastery": "^3.18.0"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-tools": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-agent": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-session": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-llm": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-attachment": "^0.1.0-rc.7",
+    "@types/node": "^22.20.0",
+    "tsdown": "0.22.2",
+    "typescript": "~5.7.2",
+    "vitest": "^3.0.0",
+    "@deepseek-ai/dsh-settings": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-api-remotes": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-connection": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-ui-settings": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-ui-slots": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-web-react": "^0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-locale": "^0.1.0-rc.7",
+    "react": "^18.2.0",
+    "@types/react": "~18.3.1",
+    "@deepseek-ai/dsh-credentials": "^0.1.0-rc.7"
+  },
+  "files": [
+    "lib/**/*.js",
+    "lib/**/*.d.ts",
+    "src",
+    "cordis.patch.yml",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsdown",
+    "typecheck": "tsc --noEmit -p tsconfig.build.json",
+    "test": "vitest run"
+  },
+  "license": "MIT"
+}

package/src/cache.ts

@@ -0,0 +1,88 @@
+import { createHash } from 'node:crypto'
+import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
+import { dirname, isAbsolute, join, resolve } from 'node:path'
+import type { UsageStats } from './caller.ts'
+
+export interface CachedPairScore {
+  scoreA: number
+  scoreB: number
+  usage: UsageStats
+  scoringMode: 'top-logprobs' | 'explicit-tag'
+  createdAt: number
+}
+
+interface CacheDocument {
+  version: 1
+  entries: Record<string, CachedPairScore>
+}
+
+export function stableHash(value: unknown): string {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex')
+}
+
+export function resolveCacheFile(cacheDir: string, cwd = process.cwd()): string {
+  const root = isAbsolute(cacheDir) ? cacheDir : resolve(cwd, cacheDir)
+  return join(root, 'scores-v1.json')
+}
+
+export class ScoreCache {
+  private readonly file: string
+  private readonly maxEntries: number
+  private loaded = false
+  private entries = new Map<string, CachedPairScore>()
+  private readonly inflight = new Map<string, Promise<CachedPairScore>>()
+  private writing: Promise<void> = Promise.resolve()
+
+  constructor(file: string, maxEntries: number) {
+    this.file = file
+    this.maxEntries = maxEntries
+  }
+
+  async load(): Promise<void> {
+    if (this.loaded) return
+    this.loaded = true
+    try {
+      const document = JSON.parse(await readFile(this.file, 'utf8')) as CacheDocument
+      if (document.version !== 1 || typeof document.entries !== 'object' || document.entries === null) return
+      this.entries = new Map(Object.entries(document.entries).map(([key, value]) => [key, { ...value, scoringMode: value.scoringMode ?? 'explicit-tag' }]))
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
+    }
+  }
+
+  async getOrCreate(key: string, create: () => Promise<CachedPairScore>): Promise<{ value: CachedPairScore; hit: boolean }> {
+    await this.load()
+    const cached = this.entries.get(key)
+    if (cached !== undefined) return { value: cached, hit: true }
+    const existing = this.inflight.get(key)
+    if (existing !== undefined) return { value: await existing, hit: true }
+    const pending = create()
+    this.inflight.set(key, pending)
+    try {
+      const value = await pending
+      this.entries.set(key, value)
+      this.trim()
+      await this.persist()
+      return { value, hit: false }
+    } finally {
+      this.inflight.delete(key)
+    }
+  }
+
+  private trim(): void {
+    if (this.entries.size <= this.maxEntries) return
+    const sorted = [...this.entries].sort((a, b) => a[1].createdAt - b[1].createdAt)
+    for (let index = 0; index < sorted.length - this.maxEntries; index += 1) this.entries.delete(sorted[index]![0])
+  }
+
+  private async persist(): Promise<void> {
+    const snapshot: CacheDocument = { version: 1, entries: Object.fromEntries(this.entries) }
+    this.writing = this.writing.then(async () => {
+      await mkdir(dirname(this.file), { recursive: true })
+      const temporary = this.file + '.tmp-' + process.pid
+      await writeFile(temporary, JSON.stringify(snapshot), 'utf8')
+      try { await rename(temporary, this.file) } catch (error) { await unlink(temporary).catch(() => {}); throw error }
+    })
+    await this.writing
+  }
+}

package/src/caller.test.ts

@@ -0,0 +1,34 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { callVerifier } from './caller.ts'
+import { TopLogprobCapabilityCache } from './top-logprobs.ts'
+
+function chunks(text = '<score_A> A </score_A>') { return [{ type: 'block-start', index: 0, blockType: 'text' }, { type: 'text-delta', index: 0, text }, { type: 'block-end', index: 0, block: { type: 'text', text } }, { type: 'usage', usage: { inputTokens: 7, cacheReadTokens: 3, outputTokens: 4, reasoningTokens: 2 } }, { type: 'finish', reason: { kind: 'stop' } }] as any[] }
+function ctx(settingsValue?: unknown) { return { get(name: string) { if (name === 'settings' && settingsValue !== undefined) return { get: () => settingsValue }; if (name === 'credentials') return { resolve: async () => ({ value: 'secret' }) }; return undefined } } as any }
+function config(stream: (options: any) => AsyncIterable<any>, saveImage = vi.fn(), context = ctx()) { return { ctx: context, llm: { stream } as any, attachments: { saveImage } as any, topLogprobCapabilities: new TopLogprobCapabilityCache(), provider: 'openai', model: 'gpt-5', reasoningEffort: 'high', maxTokens: 100, timeoutMs: 1000, maxRetries: 2, retryBaseDelayMs: 1 } }
+async function* streamOf(items: any[]) { for (const item of items) yield item }
+afterEach(() => vi.unstubAllGlobals())
+
+describe('automatic verifier scoring', () => {
+  it('falls back to explicit A-T tags when the route has no safe logprob transport', async () => {
+    let seen: any
+    const result = await callVerifier(config(async function* (options) { seen = options; yield* streamOf(chunks()) }), 'prompt')
+    expect(seen.provider).toBe('openai'); expect(result.scoringMode).toBe('explicit-tag')
+  })
+  it('uses top-logprob distributions on an explicit OpenAI-compatible route', async () => {
+    const body = { choices: [{ message: { content: '<score_A> A </score_A>' }, logprobs: { content: [{ token: '<score_A>', logprob: 0, top_logprobs: [] }, { token: 'A', logprob: -0.1, top_logprobs: [{ token: 'A', logprob: Math.log(0.7) }, { token: 'T', logprob: Math.log(0.3) }] }] } }], usage: { prompt_tokens: 10, completion_tokens: 2 } }
+    vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(body), { status: 200 })))
+    let streamed = false
+    const context = ctx({ providers: { openai: { api: 'openai-completions', baseURL: 'https://example.test/v1', apiKeyEnv: 'OPENAI_API_KEY' } } })
+    const result = await callVerifier(config(async function* () { streamed = true; yield* streamOf(chunks()) }, vi.fn(), context), 'prompt')
+    expect(result.scoringMode).toBe('top-logprobs'); expect(result.positions[1]?.length).toBe(2); expect(streamed).toBe(false)
+  })
+  it('remembers a provider logprob rejection and falls back through DSH', async () => {
+    const fetcher = vi.fn(async () => new Response('{"error":{"message":"logprobs unsupported"}}', { status: 400 }))
+    vi.stubGlobal('fetch', fetcher)
+    const context = ctx({ providers: { openai: { api: 'openai-completions', baseURL: 'https://example.test/v1' } } })
+    const cfg = config(async function* () { yield* streamOf(chunks()) }, vi.fn(), context)
+    expect((await callVerifier(cfg, 'prompt')).scoringMode).toBe('explicit-tag')
+    expect((await callVerifier(cfg, 'prompt2')).scoringMode).toBe('explicit-tag')
+    expect(fetcher).toHaveBeenCalledTimes(1)
+  })
+})

package/src/caller.ts

@@ -0,0 +1,138 @@
+import { BlockAssembler, ReasoningEffortId, createUserMessage, deepFreeze, type ContentBlock, type FinishReason, type LlmRuntime } from '@deepseek-ai/dsh-llm'
+import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
+import type { CompletionLogprobs } from './core.ts'
+import { TopLogprobCapabilityCache, TopLogprobsUnsupportedError, callTopLogprobs, resolveTopLogprobRoute } from './top-logprobs.ts'
+import type { Context } from '@deepseek-ai/cordis'
+
+export interface VerifierClientConfig {
+  ctx: Context
+  llm: LlmRuntime
+  attachments: AttachmentStore
+  topLogprobCapabilities: TopLogprobCapabilityCache
+  provider: string
+  model: string
+  reasoningEffort?: string
+  maxTokens: number
+  timeoutMs: number
+  maxRetries: number
+  retryBaseDelayMs: number
+  limiter?: RequestLimiter
+}
+
+export interface VerifierImage {
+  data: Uint8Array
+  mediaType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif'
+}
+
+export interface UsageStats {
+  calls: number
+  attempts: number
+  retries: number
+  inputTokens: number
+  cachedInputTokens: number
+  outputTokens: number
+  reasoningTokens: number
+}
+
+export type ScoringMode = 'top-logprobs' | 'explicit-tag'
+export interface VerifierCompletion extends CompletionLogprobs { usage: UsageStats; scoringMode: ScoringMode }
+
+function failureMessage(finish: FinishReason): string | undefined {
+  if (finish.kind === 'error' || finish.kind === 'aborted') return finish.failure.message
+  if (finish.kind === 'max-tokens') return 'verifier response reached max tokens before completing its answer'
+  return undefined
+}
+
+async function delay(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) throw signal.reason
+  await new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(resolve, ms)
+    const abort = () => { clearTimeout(timer); reject(signal?.reason) }
+    signal?.addEventListener('abort', abort, { once: true })
+  })
+}
+
+function usage(attempts: number, value = {} as { inputTokens?: number; outputTokens?: number; cacheReadTokens?: number; cacheWriteTokens?: number; reasoningTokens?: number }): UsageStats {
+  return { calls: 1, attempts, retries: attempts - 1, inputTokens: value.inputTokens ?? 0, cachedInputTokens: (value.cacheReadTokens ?? 0) + (value.cacheWriteTokens ?? 0), outputTokens: value.outputTokens ?? 0, reasoningTokens: value.reasoningTokens ?? 0 }
+}
+
+async function callExplicitTag(config: VerifierClientConfig, prompt: string, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<VerifierCompletion> {
+  let attempt = 0
+  while (true) {
+    attempt += 1
+    const controller = new AbortController()
+    const timeout = setTimeout(() => controller.abort(new Error('llm-verifier: request timed out')), config.timeoutMs)
+    const abort = () => controller.abort(signal?.reason)
+    signal?.addEventListener('abort', abort, { once: true })
+    try {
+      const content: ContentBlock[] = [{ type: 'text', text: prompt }]
+      for (const image of images ?? []) {
+        const ref = await config.attachments.saveImage({ data: image.data, mediaType: image.mediaType })
+        content.push({ type: 'image', attachment: ref })
+      }
+      const messages = [createUserMessage({ content, source: { kind: 'plugin', plugin: 'dsh-llm-verifier' } })]
+      const assembler = new BlockAssembler()
+      const options = deepFreeze({
+        provider: config.provider,
+        model: config.model,
+        ...(config.reasoningEffort ? { reasoningEffort: ReasoningEffortId(config.reasoningEffort) } : {}),
+        messages,
+        maxTokens: config.maxTokens,
+        temperature: 1,
+        signal: controller.signal,
+      })
+      for await (const chunk of config.llm.stream(options)) assembler.push(chunk)
+      const failed = failureMessage(assembler.finish)
+      if (failed !== undefined) throw new Error('llm-verifier: model call failed: ' + failed)
+      const text = assembler.blocks().filter((block): block is Extract<ContentBlock, { type: 'text' }> => block.type === 'text').map(block => block.text).join('')
+      if (!text.trim()) throw new Error('llm-verifier: selected DSH model produced no text')
+      // DSH adapters expose provider-neutral text/usage but not top-logprob candidates.
+      // extractScore() therefore uses the model's explicit final A–T tags.
+      return { text, tokens: [], positions: [], scoringMode: 'explicit-tag', usage: usage(attempt, assembler.usage) }
+    } catch (error) {
+      if (signal?.aborted) throw signal.reason
+      if (attempt > config.maxRetries || !(error instanceof Error) || !/rate|quota|timeout|timed out|temporar|network|fetch|socket|5dd/i.test(error.message)) throw error
+      await delay(Math.min(30000, config.retryBaseDelayMs * 2 ** (attempt - 1) * (0.8 + Math.random() * 0.4)), signal)
+    } finally {
+      clearTimeout(timeout)
+      signal?.removeEventListener('abort', abort)
+    }
+  }
+}
+
+export class RequestLimiter {
+  private active = 0
+  private readonly queue: Array<() => void> = []
+  constructor(readonly limit: number) {}
+  async run<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T> {
+    if (this.active >= this.limit) await new Promise<void>((resolve, reject) => {
+      const enter = () => { signal?.removeEventListener('abort', abort); resolve() }
+      const abort = () => { const index = this.queue.indexOf(enter); if (index >= 0) this.queue.splice(index, 1); reject(signal?.reason) }
+      this.queue.push(enter); signal?.addEventListener('abort', abort, { once: true })
+    })
+    if (signal?.aborted) throw signal.reason
+    this.active += 1
+    try { return await operation() } finally { this.active -= 1; this.queue.shift()?.() }
+  }
+}
+
+async function callAutomatic(config: VerifierClientConfig, prompt: string, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<VerifierCompletion> {
+  if (!config.topLogprobCapabilities.isUnsupported(config.provider, config.model)) {
+    const route = await resolveTopLogprobRoute(config.ctx, config.provider)
+    if (route !== undefined) {
+      try { return await callTopLogprobs(route, config.model, prompt, config.maxTokens, config.reasoningEffort, signal, images) }
+      catch (error) {
+        if (!(error instanceof TopLogprobsUnsupportedError)) throw error
+        config.topLogprobCapabilities.markUnsupported(config.provider, config.model)
+      }
+    } else config.topLogprobCapabilities.markUnsupported(config.provider, config.model)
+  }
+  return callExplicitTag(config, prompt, signal, images)
+}
+
+export async function callVerifier(config: VerifierClientConfig, prompt: string, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<VerifierCompletion> {
+  const invoke = () => callAutomatic(config, prompt, signal, images)
+  return config.limiter === undefined ? invoke() : config.limiter.run(invoke, signal)
+}
+export function addUsage(target: UsageStats, source: UsageStats): void { for (const key of ['calls', 'attempts', 'retries', 'inputTokens', 'cachedInputTokens', 'outputTokens', 'reasoningTokens'] as const) target[key] += source[key] }
+export function emptyUsage(): UsageStats { return { calls: 0, attempts: 0, retries: 0, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningTokens: 0 } }

package/src/client.tsx

@@ -0,0 +1,48 @@
+import type { ClientContext } from '@deepseek-ai/dsh-client-runtime/client'
+import type { ModelProviderGroup, SettingsNamespaceView } from '@deepseek-ai/dsh-api-remotes/client'
+import type {} from '@deepseek-ai/dsh-client-ui-settings/client'
+import type {} from '@deepseek-ai/dsh-client-ui-slots'
+import { Button, Input, StateDot } from '@deepseek-ai/dsh-client-ui-primitives'
+import { useEffect, useMemo, useState } from 'react'
+
+const NS = 'llm-verifier'
+interface Values { provider: string; model: string; reasoningEffort?: string; maxTokens: number; maxConcurrency: number; maxRetries: number; timeoutMs: number; cacheMaxEntries: number; estimatedInputUsdPerMillion: number; estimatedOutputUsdPerMillion: number }
+interface Loaded { groups: ModelProviderGroup[]; settings: SettingsNamespaceView; writable: boolean; failures: string[] }
+const shell: React.CSSProperties = { display: 'flex', flexDirection: 'column', gap: 18, padding: '8px 4px 32px', color: 'var(--dsw-text-primary)' }
+const card: React.CSSProperties = { display: 'flex', flexDirection: 'column', gap: 0, padding: '16px 16px 0', border: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))', borderRadius: 12, background: 'var(--dsw-alias-bg-module, rgba(20, 31, 57, 0.42))', overflow: 'hidden' }
+const sectionTitle: React.CSSProperties = { display: 'flex', gap: 10, alignItems: 'center', padding: '0 0 12px', borderBottom: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' }
+const row: React.CSSProperties = { display: 'grid', gridTemplateColumns: 'minmax(150px, 1fr) minmax(220px, 1.4fr)', gap: 18, alignItems: 'center', padding: '14px 0', borderBottom: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' }
+const selectStyle: React.CSSProperties = { width: '100%', minHeight: 38, padding: '0 12px', borderRadius: 10, color: 'var(--dsw-text-primary)', background: 'var(--dsw-surface-sunken)', border: '1px solid var(--dsw-alias-border-l2, rgba(255, 255, 255, 0.16))' }
+function record(value: unknown): Record<string, unknown> { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {} }
+function values(view: SettingsNamespaceView): Values { const v=record(view.value); return { provider:String(v.provider??''),model:String(v.model??''),...(typeof v.reasoningEffort==='string'?{reasoningEffort:v.reasoningEffort}:{}),maxTokens:Number(v.maxTokens??32768),maxConcurrency:Number(v.maxConcurrency??8),maxRetries:Number(v.maxRetries??3),timeoutMs:Number(v.timeoutMs??300000),cacheMaxEntries:Number(v.cacheMaxEntries??10000),estimatedInputUsdPerMillion:Number(v.estimatedInputUsdPerMillion??0),estimatedOutputUsdPerMillion:Number(v.estimatedOutputUsdPerMillion??0) } }
+function message(error: unknown): string { return error instanceof Error ? error.message : String(error) }
+function Label({title,help}:{title:string;help:string}) { return <div><div style={{fontWeight:600}}>{title}</div><div style={{fontSize:12,color:'var(--dsw-text-secondary)',marginTop:3}}>{help}</div></div> }
+
+function VerifierSettings({ api }:{api:any}) {
+  const [loaded,setLoaded]=useState<Loaded|null>(null); const [draft,setDraft]=useState<Values|null>(null); const [busy,setBusy]=useState(false); const [error,setError]=useState<string|null>(null); const [saved,setSaved]=useState(false)
+  const load=async()=>{setError(null);try{const [m,s]=await Promise.all([api.llm.models({}),api.settings.describe({})]);if(!m.result.ok)throw new Error(m.result.error.message);if(!s.result.ok)throw new Error(s.result.error.message);const view=s.result.value.namespaces.find((x:SettingsNamespaceView)=>x.ns===NS);if(!view)throw new Error('Verifier settings namespace is not registered. Restart the DSH host.');const next={groups:m.result.value.groups,settings:view,writable:s.result.value.writable,failures:m.result.value.failures.map((f:any)=>f.name+': '+f.message)};setLoaded(next);setDraft(values(view))}catch(e){setError(message(e))}}
+  useEffect(()=>{void load()},[])
+  const models=useMemo(()=>loaded?.groups.find(g=>g.id===draft?.provider)?.models??[],[loaded,draft?.provider])
+  const selected=models.find(m=>m.id===draft?.model); const efforts=selected?.reasoning?.efforts??[]
+  const patch=<K extends keyof Values>(key:K,value:Values[K])=>setDraft(v=>v?{...v,[key]:value}:v)
+  const save=async()=>{if(!loaded||!draft)return;setBusy(true);setSaved(false);setError(null);try{const section={...record(loaded.settings.user),...draft};if(!draft.reasoningEffort)delete section.reasoningEffort;const res=await api.settings.update({ns:NS,patch:section,expectedRevision:loaded.settings.revision});if(!res.result.ok)throw new Error(res.result.error.message);setLoaded(v=>v?{...v,settings:res.result.value}:v);setDraft(values(res.result.value));setSaved(true)}catch(e){setError(message(e))}finally{setBusy(false)}}
+  if(!loaded||!draft)return <div style={shell}><h2>LLM Verifier</h2><p>{error??'正在读取 DSH 模型和设置…'}</p>{error&&<Button onClick={()=>void load()}>重试</Button>}</div>
+  const numeric=(key:keyof Values,min=0)=><Input type="number" min={min} value={String(draft[key])} onChange={e=>patch(key,Number(e.target.value) as never)} />
+  return <div style={shell}>
+    <div><h2 style={{margin:'0 0 6px'}}>LLM Verifier</h2><p style={{margin:0,color:'var(--dsw-text-secondary)'}}>选择任意已在 DSH「模型」页配置并启用的模型作为独立裁判。设置实时生效。</p></div>
+    <div style={card}><div style={sectionTitle}><StateDot state="done"/><strong>裁判模型</strong></div>
+      <div style={row}><Label title="供应商" help="只显示当前 DSH 中可路由的供应商"/><select style={selectStyle} value={draft.provider} onChange={e=>{const provider=e.target.value;const first=loaded.groups.find(g=>g.id===provider)?.models[0];setDraft({...draft,provider,...(first?{model:first.id,reasoningEffort:first.reasoning?.defaultEffort}:{})})}}>{loaded.groups.map(g=><option key={g.id} value={g.id}>{g.name} · {g.id}</option>)}</select></div>
+      <div style={row}><Label title="模型" help="模型目录来自 DSH adapter，选择结果会持久化"/><select style={selectStyle} value={draft.model} onChange={e=>{const model=e.target.value;const found=models.find(m=>m.id===model);setDraft({...draft,model,...(found?.reasoning?.defaultEffort?{reasoningEffort:found.reasoning.defaultEffort}:{reasoningEffort:undefined})})}}>{models.map(m=><option key={m.id} value={m.id}>{m.name} · {m.id}</option>)}</select></div>
+      <div style={row}><Label title="推理强度" help="由所选模型 adapter 声明；留空使用模型默认值"/><select style={selectStyle} value={draft.reasoningEffort??''} onChange={e=>patch('reasoningEffort',e.target.value||undefined)}><option value="">模型默认</option>{efforts.map(e=><option key={e.id} value={e.id}>{e.name}</option>)}</select></div>
+      <div style={row}><Label title="最大输出 Token" help="每个裁判请求的输出上限"/>{numeric('maxTokens',1)}</div>
+    </div>
+    <div style={card}><div style={sectionTitle}><strong>执行控制</strong></div><div style={row}><Label title="最大并发" help="所有 verifier 工具共享的请求并发上限"/>{numeric('maxConcurrency',1)}</div><div style={row}><Label title="最多重试" help="短暂网络、限流和服务端错误的重试次数"/>{numeric('maxRetries',0)}</div><div style={row}><Label title="请求超时（毫秒）" help="单个模型请求的超时时间"/>{numeric('timeoutMs',1)}</div><div style={row}><Label title="缓存条目上限" help="持久评分缓存保留的最大条目数"/>{numeric('cacheMaxEntries',1)}</div></div>
+    <div style={card}><div style={sectionTitle}><strong>费用估算（每百万 Token，USD）</strong></div><div style={row}><Label title="输入价格" help="仅用于结果中的 estimatedCostUsd"/>{numeric('estimatedInputUsdPerMillion',0)}</div><div style={row}><Label title="输出价格" help="仅用于结果中的 estimatedCostUsd"/>{numeric('estimatedOutputUsdPerMillion',0)}</div></div>
+    {loaded.failures.length>0&&<div style={{...card,borderColor:'var(--dsw-alias-state-warn-primary, #d9a441)',paddingBottom:16}}><strong>部分模型目录读取失败</strong>{loaded.failures.map(x=><div key={x}>{x}</div>)}</div>}
+    {error&&<div style={{color:'var(--dsw-danger)'}}>{error}</div>}{saved&&<div style={{color:'var(--dsw-success)'}}>已保存，后续 verifier 调用将使用新设置。</div>}
+    <div style={{display:'flex',gap:10}}><Button disabled={busy||!loaded.writable} onClick={()=>void save()}>{busy?'保存中…':'保存设置'}</Button><Button variant="outline" disabled={busy} onClick={()=>void load()}>重新载入</Button></div>
+  </div>
+}
+
+export const inject=['slots','connection']
+export function apply(ctx:ClientContext):void { const connection=ctx.get('connection') as any; ctx.slots.inject('settings.section',()=>ctx.slots.register({name:'settings.section',id:'llm-verifier',order:35,label:'LLM Verifier',inject:()=>({api:connection.api})},VerifierSettings as never)) }

package/src/config.ts

@@ -0,0 +1,84 @@
+import type { Context } from '@deepseek-ai/cordis'
+import { installSettingsSection, settingsNamespace } from '@deepseek-ai/dsh-settings'
+import z from 'schemastery'
+
+export const VERIFIER_SETTINGS_NAMESPACE = settingsNamespace('llm-verifier')
+
+export interface Config {
+  provider?: string
+  model?: string
+  reasoningEffort?: string
+  maxTokens?: number
+  timeoutMs?: number
+  maxConcurrency?: number
+  maxRetries?: number
+  retryBaseDelayMs?: number
+  cacheDir?: string
+  cacheMaxEntries?: number
+  estimatedInputUsdPerMillion?: number
+  estimatedOutputUsdPerMillion?: number
+}
+
+export interface ResolvedConfig {
+  provider: string
+  model: string
+  reasoningEffort?: string
+  maxTokens: number
+  timeoutMs: number
+  maxConcurrency: number
+  maxRetries: number
+  retryBaseDelayMs: number
+  cacheDir: string
+  cacheMaxEntries: number
+  estimatedInputUsdPerMillion: number
+  estimatedOutputUsdPerMillion: number
+}
+
+export const Config: z<Config> = z.object({
+  provider: z.string().default('deepseek-official'),
+  model: z.string().default('deepseek-v4-flash'),
+  reasoningEffort: z.string(),
+  maxTokens: z.number().step(1).min(1).default(32768),
+  timeoutMs: z.number().step(1).min(1).default(300000),
+  maxConcurrency: z.number().step(1).min(1).default(8),
+  maxRetries: z.number().step(1).min(0).default(3),
+  retryBaseDelayMs: z.number().step(1).min(1).default(500),
+  cacheDir: z.string().default('.dsh-verifier-cache'),
+  cacheMaxEntries: z.number().step(1).min(1).default(10000),
+  estimatedInputUsdPerMillion: z.number().min(0).default(0),
+  estimatedOutputUsdPerMillion: z.number().min(0).default(0),
+})
+
+export function resolveConfig(config: Config = {}): ResolvedConfig {
+  const provider = (config.provider ?? 'deepseek-official').trim()
+  const model = (config.model ?? 'deepseek-v4-flash').trim()
+  if (!provider) throw new Error('llm-verifier: provider must be non-empty')
+  if (!model) throw new Error('llm-verifier: model must be non-empty')
+  const values = {
+    maxTokens: config.maxTokens ?? 32768,
+    timeoutMs: config.timeoutMs ?? 300000,
+    maxConcurrency: config.maxConcurrency ?? 8,
+    retryBaseDelayMs: config.retryBaseDelayMs ?? 500,
+    cacheMaxEntries: config.cacheMaxEntries ?? 10000,
+  }
+  for (const [name, value] of Object.entries(values)) if (!Number.isSafeInteger(value) || value <= 0) throw new Error('llm-verifier: ' + name + ' must be a positive safe integer')
+  const maxRetries = config.maxRetries ?? 3
+  if (!Number.isSafeInteger(maxRetries) || maxRetries < 0) throw new Error('llm-verifier: maxRetries must be a non-negative safe integer')
+  const cacheDir = (config.cacheDir ?? '.dsh-verifier-cache').trim()
+  if (!cacheDir) throw new Error('llm-verifier: cacheDir must be non-empty')
+  const estimatedInputUsdPerMillion = config.estimatedInputUsdPerMillion ?? 0
+  const estimatedOutputUsdPerMillion = config.estimatedOutputUsdPerMillion ?? 0
+  if (![estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion].every(value => Number.isFinite(value) && value >= 0)) throw new Error('llm-verifier: estimated token prices must be finite non-negative numbers')
+  const reasoningEffort = config.reasoningEffort?.trim()
+  return { provider, model, ...(reasoningEffort ? { reasoningEffort } : {}), maxRetries, cacheDir, estimatedInputUsdPerMillion, estimatedOutputUsdPerMillion, ...values }
+}
+
+export function installVerifierSettings(ctx: Context, entry: ResolvedConfig, onChange: () => void): () => ResolvedConfig {
+  let source = () => entry
+  installSettingsSection(ctx, VERIFIER_SETTINGS_NAMESPACE, Config as z<ResolvedConfig>, entry, {
+    setSource(current) { source = current },
+    onChange,
+    validate(value) { resolveConfig(value) },
+  })
+  return () => resolveConfig(source())
+}