dsh-llm-verifier 0.1.0

package/src/session.ts

@@ -0,0 +1,75 @@
+import type { ContentBlock } from '@deepseek-ai/dsh-llm'
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+import type { Agent } from '@deepseek-ai/dsh-agent'
+import type { VerifierImage } from './caller.ts'
+
+export interface SessionExtractOptions {
+  fromSeq?: number
+  toSeq?: number
+  includeAssistantText?: boolean
+  redactPatterns?: readonly string[]
+  maxChars?: number
+}
+
+export interface SessionExtraction {
+  problem: string
+  trace: string
+  images: VerifierImage[]
+  sessionId: string
+  fromSeq: number
+  toSeq: number
+  omittedCharacters: number
+}
+
+function textOf(blocks: readonly ContentBlock[]): string {
+  const parts: string[] = []
+  for (const block of blocks) {
+    if (block.type === 'text') parts.push(block.text)
+    else if (block.type === 'reasoning') parts.push('[Reasoning] ' + block.text)
+    else if (block.type === 'tool-call') parts.push('[Tool Call] ' + block.name + ' ' + block.arguments)
+    else if (block.type === 'tool-result') parts.push('[Tool Result] ' + textOf(block.content))
+  }
+  return parts.join('\n')
+}
+
+function redact(text: string, patterns: readonly string[]): string {
+  let result = text
+  for (const pattern of patterns) {
+    let regex: RegExp
+    try { regex = new RegExp(pattern, 'giu') } catch { throw new Error('llm-verifier: invalid redact pattern: ' + pattern) }
+    result = result.replace(regex, '[REDACTED]')
+  }
+  return result
+}
+
+export async function extractSession(agent: Agent, loadImage: (ref: Extract<ContentBlock, { type: 'image' }>['attachment']) => Promise<VerifierImage>, options: SessionExtractOptions = {}): Promise<SessionExtraction> {
+  const all = agent.session.events as readonly SessionEvent[]
+  const from = options.fromSeq ?? 0
+  const to = options.toSeq ?? Number.MAX_SAFE_INTEGER
+  const events = all.filter(event => event.seq >= from && event.seq <= to)
+  const defaultPatterns = ['Bearer\s+[A-Za-z0-9._~+\/=-]+', '(?:api[_-]?key|token|password|secret)\s*[:=]\s*[^\s,;]+']
+  const patterns = [...defaultPatterns, ...(options.redactPatterns ?? [])]
+  let problem = ''
+  const trace: string[] = []
+  const images: VerifierImage[] = []
+  for (const event of events) {
+    if (event.type === 'user/message') {
+      if (event.data.source.kind !== 'user') continue
+      const text = textOf(event.data.content)
+      if (!problem && text.trim()) problem = text.trim()
+      for (const block of event.data.content) if (block.type === 'image') images.push(await loadImage(block.attachment))
+      trace.push('--- User seq ' + event.seq + ' ---\n' + text)
+    } else if (event.type === 'assistant/message' && options.includeAssistantText !== false) {
+      trace.push('--- Assistant turn ' + event.data.turn + ' step ' + event.data.step + ' ---\n' + textOf(event.data.message.content))
+    } else if (event.type === 'tool/call') {
+      trace.push('--- Tool Call turn ' + event.data.turn + ' step ' + event.data.step + ' ---\n[Command] ' + event.data.name + ' ' + event.data.arguments)
+    } else if (event.type === 'tool/result') {
+      trace.push('--- Tool Result turn ' + event.data.turn + ' step ' + event.data.step + ' ---\n[Output] ' + textOf(event.data.message.content))
+    }
+  }
+  const raw = redact(trace.join('\n\n'), patterns)
+  const maxChars = options.maxChars ?? 200000
+  const omittedCharacters = Math.max(0, raw.length - maxChars)
+  const bounded = omittedCharacters ? '[Earlier trace truncated: ' + omittedCharacters + ' characters omitted]\n' + raw.slice(-maxChars) : raw
+  return { problem: redact(problem, patterns), trace: bounded, images, sessionId: String(agent.id), fromSeq: events[0]?.seq ?? from, toSeq: events.at(-1)?.seq ?? from, omittedCharacters }
+}

package/src/top-logprobs.ts

@@ -0,0 +1,97 @@
+import type { Context } from '@deepseek-ai/cordis'
+import { credentialRef } from '@deepseek-ai/dsh-credentials'
+import { settingsNamespace } from '@deepseek-ai/dsh-settings'
+import type { CompletionLogprobs, TokenAlternative } from './core.ts'
+import type { UsageStats, VerifierImage } from './caller.ts'
+
+export interface TopLogprobRoute {
+  baseURL: string
+  apiKey?: string
+  headers?: Record<string, string>
+  deepSeekThinking: boolean
+}
+export interface TopLogprobCompletion extends CompletionLogprobs { usage: UsageStats; scoringMode: 'top-logprobs' }
+export class TopLogprobsUnsupportedError extends Error { constructor(message: string) { super(message); this.name = 'TopLogprobsUnsupportedError' } }
+
+function object(value: unknown): Record<string, unknown> | undefined { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : undefined }
+function text(value: unknown): string | undefined { return typeof value === 'string' && value.trim() ? value.trim() : undefined }
+function endpoint(baseURL: string): string { return baseURL.replace(/\/+$/, '') + '/chat/completions' }
+function dataUrl(image: VerifierImage): string { return 'data:' + image.mediaType + ';base64,' + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).toString('base64') }
+
+async function credential(ctx: Context, name: string | undefined): Promise<string | undefined> {
+  if (!name) return undefined
+  const provider = ctx.get('credentials')
+  return (await provider?.resolve(credentialRef(name)))?.value
+}
+
+export async function resolveTopLogprobRoute(ctx: Context, provider: string): Promise<TopLogprobRoute | undefined> {
+  const settings = ctx.get('settings')
+  if (provider === 'deepseek-official') {
+    const value = settings ? object(settings.get(settingsNamespace('llm-deepseek'))) ?? {} : {}
+    const apiKeyEnv = text(value.apiKeyEnv) ?? 'DEEPSEEK_API_KEY'
+    const apiKey = await credential(ctx, apiKeyEnv)
+    if (!apiKey) return undefined
+    return { baseURL: text(value.baseURL) ?? 'https://api.deepseek.com', apiKey, deepSeekThinking: true }
+  }
+  if (!settings) return undefined
+  const root = object(settings.get(settingsNamespace('llm-pi-ai')))
+  const profiles = object(root?.providers)
+  const profile = object(profiles?.[provider])
+  // Only explicitly OpenAI-compatible profiles are safe to serialize directly.
+  // Other DSH adapters keep their private protocol and use the explicit-tag fallback.
+  if (!profile || profile.api !== 'openai-completions') return undefined
+  const baseURL = text(profile.baseURL)
+  if (!baseURL || !/^https:\/\//i.test(baseURL)) return undefined
+  const apiKey = await credential(ctx, text(profile.apiKeyEnv))
+  const rawHeaders = object(profile.headers)
+  const headers = rawHeaders === undefined ? undefined : Object.fromEntries(Object.entries(rawHeaders).filter((entry): entry is [string, string] => typeof entry[1] === 'string'))
+  return { baseURL, ...(apiKey ? { apiKey } : {}), ...(headers ? { headers } : {}), deepSeekThinking: false }
+}
+
+export async function callTopLogprobs(route: TopLogprobRoute, model: string, prompt: string, maxTokens: number, reasoningEffort: string | undefined, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<TopLogprobCompletion> {
+  const content: string | Record<string, unknown>[] = images?.length ? [{ type: 'text', text: prompt }, ...images.map(image => ({ type: 'image_url', image_url: { url: dataUrl(image) } }))] : prompt
+  const thinking = route.deepSeekThinking && reasoningEffort ? reasoningEffort === 'off' ? { thinking: { type: 'disabled' } } : { thinking: { type: 'enabled' }, reasoning_effort: reasoningEffort } : {}
+  const response = await fetch(endpoint(route.baseURL), {
+    method: 'POST', redirect: 'error', signal,
+    headers: { 'content-type': 'application/json', ...(route.apiKey ? { authorization: 'Bearer ' + route.apiKey } : {}), ...route.headers },
+    body: JSON.stringify({ model, messages: [{ role: 'user', content }], max_tokens: maxTokens, temperature: 1, logprobs: true, top_logprobs: 20, ...thinking }),
+  })
+  const raw = await response.text()
+  if (!response.ok) {
+    const excerpt = raw.slice(0, 1000)
+    if ([400, 404, 405, 415, 422].includes(response.status) && /logprob|top_logprobs|unsupported|unknown (?:field|parameter)|unrecognized (?:field|parameter)|not support/i.test(excerpt)) throw new TopLogprobsUnsupportedError('provider rejected top_logprobs: HTTP ' + response.status + ' ' + excerpt)
+    throw new Error('llm-verifier: top_logprobs request failed with HTTP ' + response.status + ': ' + excerpt)
+  }
+  let body: Record<string, unknown>
+  try { body = object(JSON.parse(raw)) ?? {} } catch { throw new Error('llm-verifier: top_logprobs endpoint returned invalid JSON') }
+  const choices = Array.isArray(body.choices) ? body.choices : []
+  const choice = object(choices[0])
+  const message = object(choice?.message)
+  const answer = typeof message?.content === 'string' ? message.content : ''
+  const logprobs = object(choice?.logprobs)
+  const rows = Array.isArray(logprobs?.content) ? logprobs.content : []
+  if (!rows.length) throw new TopLogprobsUnsupportedError('provider returned no token logprobs')
+  const tokens: string[] = []
+  const positions: TokenAlternative[][] = []
+  for (const rawRow of rows) {
+    const row = object(rawRow) ?? {}
+    const token = typeof row.token === 'string' ? row.token : ''
+    tokens.push(token)
+    const top = Array.isArray(row.top_logprobs) ? row.top_logprobs : []
+    const alternatives = top.flatMap(value => { const item = object(value); return item && typeof item.token === 'string' && typeof item.logprob === 'number' ? [{ token: item.token, logprob: item.logprob }] : [] })
+    if (!alternatives.length && typeof row.logprob === 'number') alternatives.push({ token, logprob: row.logprob })
+    positions.push(alternatives)
+  }
+  const rawUsage = object(body.usage) ?? {}
+  const promptDetails = object(rawUsage.prompt_tokens_details) ?? {}
+  const completionDetails = object(rawUsage.completion_tokens_details) ?? {}
+  const cached = Number(rawUsage.prompt_cache_hit_tokens ?? promptDetails.cached_tokens ?? 0) || 0
+  const input = Number(rawUsage.prompt_tokens ?? 0) || 0
+  return { text: answer, tokens, positions, scoringMode: 'top-logprobs', usage: { calls: 1, attempts: 1, retries: 0, inputTokens: Math.max(0, input - cached), cachedInputTokens: cached, outputTokens: Number(rawUsage.completion_tokens ?? 0) || 0, reasoningTokens: Number(completionDetails.reasoning_tokens ?? 0) || 0 } }
+}
+
+export class TopLogprobCapabilityCache {
+  private readonly unsupported = new Set<string>()
+  isUnsupported(provider: string, model: string): boolean { return this.unsupported.has(provider + '\0' + model) }
+  markUnsupported(provider: string, model: string): void { this.unsupported.add(provider + '\0' + model) }
+}