dsh-llm-verifier 0.1.13 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/README.md +328 -301
- package/cordis.patch.yml +4 -4
- package/lib/caller-BWwSVNZf.js.map +1 -0
- package/lib/client.js +176 -14
- package/lib/client.js.map +1 -0
- package/lib/core.js.map +1 -0
- package/lib/index.js +400 -51
- package/lib/index.js.map +1 -0
- package/lib/types/auto.d.ts.map +1 -0
- package/lib/types/auto.js +8 -2
- package/lib/types/auto.js.map +1 -0
- package/lib/types/cache.d.ts.map +1 -0
- package/lib/types/cache.js.map +1 -0
- package/lib/types/caller.d.ts.map +1 -0
- package/lib/types/caller.js.map +1 -0
- package/lib/types/client-i18n.d.ts +27 -0
- package/lib/types/client-i18n.d.ts.map +1 -0
- package/lib/types/client-i18n.js +18 -0
- package/lib/types/client-i18n.js.map +1 -0
- package/lib/types/client.d.ts +15 -2
- package/lib/types/client.d.ts.map +1 -0
- package/lib/types/client.js +137 -10
- package/lib/types/client.js.map +1 -0
- package/lib/types/config.d.ts +4 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +5 -3
- package/lib/types/config.js.map +1 -0
- package/lib/types/core.d.ts.map +1 -0
- package/lib/types/core.js.map +1 -0
- package/lib/types/engine.d.ts.map +1 -0
- package/lib/types/engine.js.map +1 -0
- package/lib/types/images.d.ts.map +1 -0
- package/lib/types/images.js.map +1 -0
- package/lib/types/index.d.ts +2 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +162 -9
- package/lib/types/index.js.map +1 -0
- package/lib/types/plan-gate.d.ts +27 -0
- package/lib/types/plan-gate.d.ts.map +1 -0
- package/lib/types/plan-gate.js +60 -0
- package/lib/types/plan-gate.js.map +1 -0
- package/lib/types/router.d.ts +28 -32
- package/lib/types/router.d.ts.map +1 -0
- package/lib/types/router.js +41 -9
- package/lib/types/router.js.map +1 -0
- package/lib/types/session.d.ts +11 -0
- package/lib/types/session.d.ts.map +1 -0
- package/lib/types/session.js +41 -11
- package/lib/types/session.js.map +1 -0
- package/lib/types/statistics.d.ts.map +1 -0
- package/lib/types/statistics.js.map +1 -0
- package/lib/types/team-gate.d.ts +20 -0
- package/lib/types/team-gate.d.ts.map +1 -0
- package/lib/types/team-gate.js +59 -0
- package/lib/types/team-gate.js.map +1 -0
- package/lib/types/top-logprobs.d.ts.map +1 -0
- package/lib/types/top-logprobs.js.map +1 -0
- package/lib/types/topic-storage.d.ts.map +1 -0
- package/lib/types/topic-storage.js.map +1 -0
- package/package.json +114 -112
- package/src/auto.test.ts +95 -86
- package/src/auto.ts +140 -133
- package/src/cache.ts +115 -115
- package/src/caller.test.ts +34 -34
- package/src/caller.ts +151 -151
- package/src/client-i18n.ts +18 -0
- package/src/client.tsx +383 -213
- package/src/config.ts +189 -183
- package/src/core.test.ts +71 -71
- package/src/core.ts +205 -205
- package/src/engine.test.ts +136 -136
- package/src/engine.ts +140 -140
- package/src/images.ts +33 -33
- package/src/index.ts +394 -247
- package/src/parity.test.ts +85 -85
- package/src/plan-gate.test.ts +56 -0
- package/src/plan-gate.ts +66 -0
- package/src/router.test.ts +107 -89
- package/src/router.ts +361 -309
- package/src/session.test.ts +71 -36
- package/src/session.ts +132 -103
- package/src/statistics.test.ts +80 -80
- package/src/statistics.ts +328 -328
- package/src/team-gate.test.ts +92 -0
- package/src/team-gate.ts +80 -0
- package/src/top-logprobs.test.ts +84 -84
- package/src/top-logprobs.ts +153 -153
- package/src/topic-storage.test.ts +31 -31
- package/src/topic-storage.ts +42 -42
package/src/top-logprobs.ts
CHANGED
|
@@ -1,153 +1,153 @@
|
|
|
1
|
-
import type { Context } from '@deepseek-ai/cordis'
|
|
2
|
-
import { credentialRef } from '@deepseek-ai/dsh-credentials'
|
|
3
|
-
import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
|
|
4
|
-
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
|
5
|
-
import type { CompletionLogprobs, TokenAlternative } from './core.ts'
|
|
6
|
-
import type { UsageStats, VerifierImage } from './caller.ts'
|
|
7
|
-
|
|
8
|
-
export interface TopLogprobRoute {
|
|
9
|
-
baseURL: string
|
|
10
|
-
apiKey?: string
|
|
11
|
-
headers?: Record<string, string>
|
|
12
|
-
deepSeekThinking: boolean
|
|
13
|
-
}
|
|
14
|
-
export interface TopLogprobCompletion extends CompletionLogprobs { usage: UsageStats; scoringMode: 'top-logprobs' }
|
|
15
|
-
export class TopLogprobsUnsupportedError extends Error { constructor(message: string) { super(message); this.name = 'TopLogprobsUnsupportedError' } }
|
|
16
|
-
|
|
17
|
-
function object(value: unknown): Record<string, unknown> | undefined { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : undefined }
|
|
18
|
-
function text(value: unknown): string | undefined { return typeof value === 'string' && value.trim() ? value.trim() : undefined }
|
|
19
|
-
function endpoint(baseURL: string): string { return baseURL.replace(/\/+$/, '') + '/chat/completions' }
|
|
20
|
-
function dataUrl(image: VerifierImage): string { return 'data:' + image.mediaType + ';base64,' + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).toString('base64') }
|
|
21
|
-
|
|
22
|
-
async function credential(ctx: Context, name: string | undefined): Promise<string | undefined> {
|
|
23
|
-
if (!name) return undefined
|
|
24
|
-
const provider = ctx.get('credentials')
|
|
25
|
-
return (await provider?.resolve(credentialRef(name)))?.value
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export async function resolveTopLogprobRoute(ctx: Context, provider: string): Promise<TopLogprobRoute | undefined> {
|
|
29
|
-
const settings = ctx.get('settings')
|
|
30
|
-
if (provider === 'deepseek-official') {
|
|
31
|
-
const value = settings ? object(settings.get('llm-deepseek' as never)) ?? {} : {}
|
|
32
|
-
const apiKeyEnv = text(value.apiKeyEnv) ?? 'DEEPSEEK_API_KEY'
|
|
33
|
-
const apiKey = await credential(ctx, apiKeyEnv)
|
|
34
|
-
if (!apiKey) return undefined
|
|
35
|
-
return { baseURL: text(value.baseURL) ?? 'https://api.deepseek.com', apiKey, deepSeekThinking: true }
|
|
36
|
-
}
|
|
37
|
-
if (!settings) return undefined
|
|
38
|
-
const root = object(settings.get('llm-pi-ai' as never))
|
|
39
|
-
const profiles = object(root?.providers)
|
|
40
|
-
const profile = object(profiles?.[provider])
|
|
41
|
-
// Only explicitly OpenAI-compatible profiles are safe to serialize directly.
|
|
42
|
-
// Other DSH adapters keep their private protocol and use the explicit-tag fallback.
|
|
43
|
-
if (!profile || profile.api !== 'openai-completions') return undefined
|
|
44
|
-
const baseURL = text(profile.baseURL)
|
|
45
|
-
if (!baseURL || !/^https:\/\//i.test(baseURL)) return undefined
|
|
46
|
-
const apiKey = await credential(ctx, text(profile.apiKeyEnv))
|
|
47
|
-
const rawHeaders = object(profile.headers)
|
|
48
|
-
const headers = rawHeaders === undefined ? undefined : Object.fromEntries(Object.entries(rawHeaders).filter((entry): entry is [string, string] => typeof entry[1] === 'string'))
|
|
49
|
-
return { baseURL, ...(apiKey ? { apiKey } : {}), ...(headers ? { headers } : {}), deepSeekThinking: false }
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export async function callTopLogprobs(route: TopLogprobRoute, model: string, prompt: string, maxTokens: number, reasoningEffort: string | undefined, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<TopLogprobCompletion> {
|
|
53
|
-
const content: string | Record<string, unknown>[] = images?.length ? [{ type: 'text', text: prompt }, ...images.map(image => ({ type: 'image_url', image_url: { url: dataUrl(image) } }))] : prompt
|
|
54
|
-
const thinking = route.deepSeekThinking && reasoningEffort ? reasoningEffort === 'off' ? { thinking: { type: 'disabled' } } : { thinking: { type: 'enabled' }, reasoning_effort: reasoningEffort } : {}
|
|
55
|
-
const response = await fetch(endpoint(route.baseURL), {
|
|
56
|
-
method: 'POST', redirect: 'error', signal,
|
|
57
|
-
headers: { 'content-type': 'application/json', ...(route.apiKey ? { authorization: 'Bearer ' + route.apiKey } : {}), ...route.headers },
|
|
58
|
-
body: JSON.stringify({ model, messages: [{ role: 'user', content }], max_tokens: maxTokens, temperature: 1, logprobs: true, top_logprobs: 20, ...thinking }),
|
|
59
|
-
})
|
|
60
|
-
const raw = await response.text()
|
|
61
|
-
if (!response.ok) {
|
|
62
|
-
const excerpt = raw.slice(0, 1000)
|
|
63
|
-
if ([400, 404, 405, 415, 422].includes(response.status) && /logprob|top_logprobs|unsupported|unknown (?:field|parameter)|unrecognized (?:field|parameter)|not support/i.test(excerpt)) throw new TopLogprobsUnsupportedError('provider rejected top_logprobs: HTTP ' + response.status + ' ' + excerpt)
|
|
64
|
-
throw new Error('llm-verifier: top_logprobs request failed with HTTP ' + response.status + ': ' + excerpt)
|
|
65
|
-
}
|
|
66
|
-
let body: Record<string, unknown>
|
|
67
|
-
try { body = object(JSON.parse(raw)) ?? {} } catch { throw new Error('llm-verifier: top_logprobs endpoint returned invalid JSON') }
|
|
68
|
-
const choices = Array.isArray(body.choices) ? body.choices : []
|
|
69
|
-
const choice = object(choices[0])
|
|
70
|
-
const message = object(choice?.message)
|
|
71
|
-
const answer = typeof message?.content === 'string' ? message.content : ''
|
|
72
|
-
const logprobs = object(choice?.logprobs)
|
|
73
|
-
const rows = Array.isArray(logprobs?.content) ? logprobs.content : []
|
|
74
|
-
if (!rows.length) throw new TopLogprobsUnsupportedError('provider returned no token logprobs')
|
|
75
|
-
const tokens: string[] = []
|
|
76
|
-
const positions: TokenAlternative[][] = []
|
|
77
|
-
for (const rawRow of rows) {
|
|
78
|
-
const row = object(rawRow) ?? {}
|
|
79
|
-
const token = typeof row.token === 'string' ? row.token : ''
|
|
80
|
-
tokens.push(token)
|
|
81
|
-
const top = Array.isArray(row.top_logprobs) ? row.top_logprobs : []
|
|
82
|
-
const alternatives = top.flatMap(value => { const item = object(value); return item && typeof item.token === 'string' && typeof item.logprob === 'number' ? [{ token: item.token, logprob: item.logprob }] : [] })
|
|
83
|
-
if (!alternatives.length && typeof row.logprob === 'number') alternatives.push({ token, logprob: row.logprob })
|
|
84
|
-
positions.push(alternatives)
|
|
85
|
-
}
|
|
86
|
-
const rawUsage = object(body.usage) ?? {}
|
|
87
|
-
const promptDetails = object(rawUsage.prompt_tokens_details) ?? {}
|
|
88
|
-
const completionDetails = object(rawUsage.completion_tokens_details) ?? {}
|
|
89
|
-
const cached = Number(rawUsage.prompt_cache_hit_tokens ?? promptDetails.cached_tokens ?? 0) || 0
|
|
90
|
-
const input = Number(rawUsage.prompt_tokens ?? 0) || 0
|
|
91
|
-
return { text: answer, tokens, positions, scoringMode: 'top-logprobs', usage: { calls: 1, attempts: 1, retries: 0, inputTokens: Math.max(0, input - cached), cachedInputTokens: cached, outputTokens: Number(rawUsage.completion_tokens ?? 0) || 0, reasoningTokens: Number(completionDetails.reasoning_tokens ?? 0) || 0 } }
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/** Marks older than this are dropped on hydration so a provider that later gains logprobs support is re-probed. */
|
|
95
|
-
export const CAPABILITY_TTL_MS = 24 * 60 * 60 * 1000
|
|
96
|
-
|
|
97
|
-
interface CapabilityDocument { version: 1; entries: Record<string, number> }
|
|
98
|
-
|
|
99
|
-
/** Resolves the capability memory file beside the score cache inside the topic verifier directory. */
|
|
100
|
-
export function resolveCapabilityFile(cacheDir: string, cwd = process.cwd()): string {
|
|
101
|
-
const root = isAbsolute(cacheDir) ? cacheDir : resolve(cwd, cacheDir)
|
|
102
|
-
return join(root, 'capabilities-v1.json')
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
export class TopLogprobCapabilityCache {
|
|
106
|
-
private readonly unsupported = new Map<string, number>()
|
|
107
|
-
private loaded = false
|
|
108
|
-
private hydrating: Promise<void> | undefined
|
|
109
|
-
private writing: Promise<void> = Promise.resolve()
|
|
110
|
-
|
|
111
|
-
constructor(private readonly file?: string, private readonly now: () => number = Date.now) {}
|
|
112
|
-
|
|
113
|
-
isUnsupported(provider: string, model: string): boolean { return this.unsupported.has(provider + '\0' + model) }
|
|
114
|
-
|
|
115
|
-
/** Hydrates persisted marks once; in-process marks always win over file contents. */
|
|
116
|
-
async ensureLoaded(): Promise<void> {
|
|
117
|
-
if (this.loaded || this.file === undefined) return
|
|
118
|
-
this.hydrating ??= (async () => {
|
|
119
|
-
try {
|
|
120
|
-
const document = JSON.parse(await readFile(this.file!, 'utf8')) as CapabilityDocument
|
|
121
|
-
const entries = document !== null && typeof document === 'object' && document.version === 1 && typeof document.entries === 'object' && document.entries !== null ? document.entries : {}
|
|
122
|
-
const now = this.now()
|
|
123
|
-
for (const [key, markedAt] of Object.entries(entries)) {
|
|
124
|
-
if (typeof markedAt !== 'number' || !Number.isFinite(markedAt) || markedAt < 0 || now - markedAt > CAPABILITY_TTL_MS) continue
|
|
125
|
-
// Max-merge: never let an older persisted stamp clobber a fresher in-process probe.
|
|
126
|
-
const existing = this.unsupported.get(key)
|
|
127
|
-
if (existing === undefined || existing < markedAt) this.unsupported.set(key, markedAt)
|
|
128
|
-
}
|
|
129
|
-
} catch { /* a missing or unreadable file simply starts empty */ }
|
|
130
|
-
this.loaded = true
|
|
131
|
-
})()
|
|
132
|
-
await this.hydrating
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
markUnsupported(provider: string, model: string): void {
|
|
136
|
-
this.unsupported.set(provider + '\0' + model, this.now())
|
|
137
|
-
if (this.file === undefined) return
|
|
138
|
-
// Serialize behind hydration so an early mark never clobbers not-yet-loaded entries.
|
|
139
|
-
this.writing = this.writing
|
|
140
|
-
.then(() => this.ensureLoaded())
|
|
141
|
-
.then(async () => {
|
|
142
|
-
const snapshot: CapabilityDocument = { version: 1, entries: Object.fromEntries(this.unsupported) }
|
|
143
|
-
await mkdir(dirname(this.file!), { recursive: true })
|
|
144
|
-
const temporary = this.file! + '.tmp-' + process.pid
|
|
145
|
-
await writeFile(temporary, JSON.stringify(snapshot), 'utf8')
|
|
146
|
-
try { await rename(temporary, this.file!) } catch (error) { await unlink(temporary).catch(() => {}); throw error }
|
|
147
|
-
})
|
|
148
|
-
.catch(() => { /* capability memory is best-effort; the next mark rewrites the file */ })
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/** Resolves once the trailing persistence attempt settles; exposed for tests. */
|
|
152
|
-
flush(): Promise<void> { return this.writing }
|
|
153
|
-
}
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
2
|
+
import { credentialRef } from '@deepseek-ai/dsh-credentials'
|
|
3
|
+
import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
|
5
|
+
import type { CompletionLogprobs, TokenAlternative } from './core.ts'
|
|
6
|
+
import type { UsageStats, VerifierImage } from './caller.ts'
|
|
7
|
+
|
|
8
|
+
export interface TopLogprobRoute {
|
|
9
|
+
baseURL: string
|
|
10
|
+
apiKey?: string
|
|
11
|
+
headers?: Record<string, string>
|
|
12
|
+
deepSeekThinking: boolean
|
|
13
|
+
}
|
|
14
|
+
export interface TopLogprobCompletion extends CompletionLogprobs { usage: UsageStats; scoringMode: 'top-logprobs' }
|
|
15
|
+
export class TopLogprobsUnsupportedError extends Error { constructor(message: string) { super(message); this.name = 'TopLogprobsUnsupportedError' } }
|
|
16
|
+
|
|
17
|
+
function object(value: unknown): Record<string, unknown> | undefined { return typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : undefined }
|
|
18
|
+
function text(value: unknown): string | undefined { return typeof value === 'string' && value.trim() ? value.trim() : undefined }
|
|
19
|
+
function endpoint(baseURL: string): string { return baseURL.replace(/\/+$/, '') + '/chat/completions' }
|
|
20
|
+
function dataUrl(image: VerifierImage): string { return 'data:' + image.mediaType + ';base64,' + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).toString('base64') }
|
|
21
|
+
|
|
22
|
+
async function credential(ctx: Context, name: string | undefined): Promise<string | undefined> {
|
|
23
|
+
if (!name) return undefined
|
|
24
|
+
const provider = ctx.get('credentials')
|
|
25
|
+
return (await provider?.resolve(credentialRef(name)))?.value
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function resolveTopLogprobRoute(ctx: Context, provider: string): Promise<TopLogprobRoute | undefined> {
|
|
29
|
+
const settings = ctx.get('settings')
|
|
30
|
+
if (provider === 'deepseek-official') {
|
|
31
|
+
const value = settings ? object(settings.get('llm-deepseek' as never)) ?? {} : {}
|
|
32
|
+
const apiKeyEnv = text(value.apiKeyEnv) ?? 'DEEPSEEK_API_KEY'
|
|
33
|
+
const apiKey = await credential(ctx, apiKeyEnv)
|
|
34
|
+
if (!apiKey) return undefined
|
|
35
|
+
return { baseURL: text(value.baseURL) ?? 'https://api.deepseek.com', apiKey, deepSeekThinking: true }
|
|
36
|
+
}
|
|
37
|
+
if (!settings) return undefined
|
|
38
|
+
const root = object(settings.get('llm-pi-ai' as never))
|
|
39
|
+
const profiles = object(root?.providers)
|
|
40
|
+
const profile = object(profiles?.[provider])
|
|
41
|
+
// Only explicitly OpenAI-compatible profiles are safe to serialize directly.
|
|
42
|
+
// Other DSH adapters keep their private protocol and use the explicit-tag fallback.
|
|
43
|
+
if (!profile || profile.api !== 'openai-completions') return undefined
|
|
44
|
+
const baseURL = text(profile.baseURL)
|
|
45
|
+
if (!baseURL || !/^https:\/\//i.test(baseURL)) return undefined
|
|
46
|
+
const apiKey = await credential(ctx, text(profile.apiKeyEnv))
|
|
47
|
+
const rawHeaders = object(profile.headers)
|
|
48
|
+
const headers = rawHeaders === undefined ? undefined : Object.fromEntries(Object.entries(rawHeaders).filter((entry): entry is [string, string] => typeof entry[1] === 'string'))
|
|
49
|
+
return { baseURL, ...(apiKey ? { apiKey } : {}), ...(headers ? { headers } : {}), deepSeekThinking: false }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export async function callTopLogprobs(route: TopLogprobRoute, model: string, prompt: string, maxTokens: number, reasoningEffort: string | undefined, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<TopLogprobCompletion> {
|
|
53
|
+
const content: string | Record<string, unknown>[] = images?.length ? [{ type: 'text', text: prompt }, ...images.map(image => ({ type: 'image_url', image_url: { url: dataUrl(image) } }))] : prompt
|
|
54
|
+
const thinking = route.deepSeekThinking && reasoningEffort ? reasoningEffort === 'off' ? { thinking: { type: 'disabled' } } : { thinking: { type: 'enabled' }, reasoning_effort: reasoningEffort } : {}
|
|
55
|
+
const response = await fetch(endpoint(route.baseURL), {
|
|
56
|
+
method: 'POST', redirect: 'error', signal,
|
|
57
|
+
headers: { 'content-type': 'application/json', ...(route.apiKey ? { authorization: 'Bearer ' + route.apiKey } : {}), ...route.headers },
|
|
58
|
+
body: JSON.stringify({ model, messages: [{ role: 'user', content }], max_tokens: maxTokens, temperature: 1, logprobs: true, top_logprobs: 20, ...thinking }),
|
|
59
|
+
})
|
|
60
|
+
const raw = await response.text()
|
|
61
|
+
if (!response.ok) {
|
|
62
|
+
const excerpt = raw.slice(0, 1000)
|
|
63
|
+
if ([400, 404, 405, 415, 422].includes(response.status) && /logprob|top_logprobs|unsupported|unknown (?:field|parameter)|unrecognized (?:field|parameter)|not support/i.test(excerpt)) throw new TopLogprobsUnsupportedError('provider rejected top_logprobs: HTTP ' + response.status + ' ' + excerpt)
|
|
64
|
+
throw new Error('llm-verifier: top_logprobs request failed with HTTP ' + response.status + ': ' + excerpt)
|
|
65
|
+
}
|
|
66
|
+
let body: Record<string, unknown>
|
|
67
|
+
try { body = object(JSON.parse(raw)) ?? {} } catch { throw new Error('llm-verifier: top_logprobs endpoint returned invalid JSON') }
|
|
68
|
+
const choices = Array.isArray(body.choices) ? body.choices : []
|
|
69
|
+
const choice = object(choices[0])
|
|
70
|
+
const message = object(choice?.message)
|
|
71
|
+
const answer = typeof message?.content === 'string' ? message.content : ''
|
|
72
|
+
const logprobs = object(choice?.logprobs)
|
|
73
|
+
const rows = Array.isArray(logprobs?.content) ? logprobs.content : []
|
|
74
|
+
if (!rows.length) throw new TopLogprobsUnsupportedError('provider returned no token logprobs')
|
|
75
|
+
const tokens: string[] = []
|
|
76
|
+
const positions: TokenAlternative[][] = []
|
|
77
|
+
for (const rawRow of rows) {
|
|
78
|
+
const row = object(rawRow) ?? {}
|
|
79
|
+
const token = typeof row.token === 'string' ? row.token : ''
|
|
80
|
+
tokens.push(token)
|
|
81
|
+
const top = Array.isArray(row.top_logprobs) ? row.top_logprobs : []
|
|
82
|
+
const alternatives = top.flatMap(value => { const item = object(value); return item && typeof item.token === 'string' && typeof item.logprob === 'number' ? [{ token: item.token, logprob: item.logprob }] : [] })
|
|
83
|
+
if (!alternatives.length && typeof row.logprob === 'number') alternatives.push({ token, logprob: row.logprob })
|
|
84
|
+
positions.push(alternatives)
|
|
85
|
+
}
|
|
86
|
+
const rawUsage = object(body.usage) ?? {}
|
|
87
|
+
const promptDetails = object(rawUsage.prompt_tokens_details) ?? {}
|
|
88
|
+
const completionDetails = object(rawUsage.completion_tokens_details) ?? {}
|
|
89
|
+
const cached = Number(rawUsage.prompt_cache_hit_tokens ?? promptDetails.cached_tokens ?? 0) || 0
|
|
90
|
+
const input = Number(rawUsage.prompt_tokens ?? 0) || 0
|
|
91
|
+
return { text: answer, tokens, positions, scoringMode: 'top-logprobs', usage: { calls: 1, attempts: 1, retries: 0, inputTokens: Math.max(0, input - cached), cachedInputTokens: cached, outputTokens: Number(rawUsage.completion_tokens ?? 0) || 0, reasoningTokens: Number(completionDetails.reasoning_tokens ?? 0) || 0 } }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Marks older than this are dropped on hydration so a provider that later gains logprobs support is re-probed. */
|
|
95
|
+
export const CAPABILITY_TTL_MS = 24 * 60 * 60 * 1000
|
|
96
|
+
|
|
97
|
+
interface CapabilityDocument { version: 1; entries: Record<string, number> }
|
|
98
|
+
|
|
99
|
+
/** Resolves the capability memory file beside the score cache inside the topic verifier directory. */
|
|
100
|
+
export function resolveCapabilityFile(cacheDir: string, cwd = process.cwd()): string {
|
|
101
|
+
const root = isAbsolute(cacheDir) ? cacheDir : resolve(cwd, cacheDir)
|
|
102
|
+
return join(root, 'capabilities-v1.json')
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export class TopLogprobCapabilityCache {
|
|
106
|
+
private readonly unsupported = new Map<string, number>()
|
|
107
|
+
private loaded = false
|
|
108
|
+
private hydrating: Promise<void> | undefined
|
|
109
|
+
private writing: Promise<void> = Promise.resolve()
|
|
110
|
+
|
|
111
|
+
constructor(private readonly file?: string, private readonly now: () => number = Date.now) {}
|
|
112
|
+
|
|
113
|
+
isUnsupported(provider: string, model: string): boolean { return this.unsupported.has(provider + '\0' + model) }
|
|
114
|
+
|
|
115
|
+
/** Hydrates persisted marks once; in-process marks always win over file contents. */
|
|
116
|
+
async ensureLoaded(): Promise<void> {
|
|
117
|
+
if (this.loaded || this.file === undefined) return
|
|
118
|
+
this.hydrating ??= (async () => {
|
|
119
|
+
try {
|
|
120
|
+
const document = JSON.parse(await readFile(this.file!, 'utf8')) as CapabilityDocument
|
|
121
|
+
const entries = document !== null && typeof document === 'object' && document.version === 1 && typeof document.entries === 'object' && document.entries !== null ? document.entries : {}
|
|
122
|
+
const now = this.now()
|
|
123
|
+
for (const [key, markedAt] of Object.entries(entries)) {
|
|
124
|
+
if (typeof markedAt !== 'number' || !Number.isFinite(markedAt) || markedAt < 0 || now - markedAt > CAPABILITY_TTL_MS) continue
|
|
125
|
+
// Max-merge: never let an older persisted stamp clobber a fresher in-process probe.
|
|
126
|
+
const existing = this.unsupported.get(key)
|
|
127
|
+
if (existing === undefined || existing < markedAt) this.unsupported.set(key, markedAt)
|
|
128
|
+
}
|
|
129
|
+
} catch { /* a missing or unreadable file simply starts empty */ }
|
|
130
|
+
this.loaded = true
|
|
131
|
+
})()
|
|
132
|
+
await this.hydrating
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
markUnsupported(provider: string, model: string): void {
|
|
136
|
+
this.unsupported.set(provider + '\0' + model, this.now())
|
|
137
|
+
if (this.file === undefined) return
|
|
138
|
+
// Serialize behind hydration so an early mark never clobbers not-yet-loaded entries.
|
|
139
|
+
this.writing = this.writing
|
|
140
|
+
.then(() => this.ensureLoaded())
|
|
141
|
+
.then(async () => {
|
|
142
|
+
const snapshot: CapabilityDocument = { version: 1, entries: Object.fromEntries(this.unsupported) }
|
|
143
|
+
await mkdir(dirname(this.file!), { recursive: true })
|
|
144
|
+
const temporary = this.file! + '.tmp-' + process.pid
|
|
145
|
+
await writeFile(temporary, JSON.stringify(snapshot), 'utf8')
|
|
146
|
+
try { await rename(temporary, this.file!) } catch (error) { await unlink(temporary).catch(() => {}); throw error }
|
|
147
|
+
})
|
|
148
|
+
.catch(() => { /* capability memory is best-effort; the next mark rewrites the file */ })
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Resolves once the trailing persistence attempt settles; exposed for tests. */
|
|
152
|
+
flush(): Promise<void> { return this.writing }
|
|
153
|
+
}
|
|
@@ -1,31 +1,31 @@
|
|
|
1
|
-
import { dirname, join } from 'node:path'
|
|
2
|
-
import { describe, expect, it } from 'vitest'
|
|
3
|
-
import type { SessionHeader } from '@deepseek-ai/dsh-session'
|
|
4
|
-
import { resolveTopicDataDir } from './topic-storage.ts'
|
|
5
|
-
|
|
6
|
-
const header = { id: 'session-test', cwd: 'C:\\workspace', createdAt: 1 } as unknown as SessionHeader
|
|
7
|
-
|
|
8
|
-
describe('resolveTopicDataDir', () => {
|
|
9
|
-
it('places verifier files beside the session artifact', () => {
|
|
10
|
-
const artifact = join('C:\\Users\\eddy\\.dsh\\sessions', 'project', 'session-test', 'session.jsonl.zstd')
|
|
11
|
-
const result = resolveTopicDataDir({ locate: () => ({ path: artifact }) }, header, 'verifier')
|
|
12
|
-
expect(result).toBe(join(dirname(artifact), 'verifier'))
|
|
13
|
-
})
|
|
14
|
-
|
|
15
|
-
it('rejects paths that escape the topic directory', () => {
|
|
16
|
-
const locator = { locate: () => ({ path: join('C:\\data', 'session-test', 'session.jsonl') }) }
|
|
17
|
-
expect(() => resolveTopicDataDir(locator, header, '..\\shared')).toThrow(/inside the topic directory/u)
|
|
18
|
-
expect(() => resolveTopicDataDir(locator, header, 'C:\\shared')).toThrow(/relative/u)
|
|
19
|
-
})
|
|
20
|
-
|
|
21
|
-
it('fails closed when the backend cannot locate a topic artifact', () => {
|
|
22
|
-
expect(() => resolveTopicDataDir({ locate: () => undefined }, header, 'verifier')).toThrow(/does not expose/u)
|
|
23
|
-
expect(() => resolveTopicDataDir({}, header, 'verifier')).toThrow(/does not expose/u)
|
|
24
|
-
})
|
|
25
|
-
|
|
26
|
-
it('falls back to root-based path when locate is missing', () => {
|
|
27
|
-
const result = resolveTopicDataDir({ root: 'C:\\data\\sessions' }, header, 'verifier')
|
|
28
|
-
expect(result).toBe(join('C:\\data\\sessions', 'session-test', 'verifier'))
|
|
29
|
-
})
|
|
30
|
-
})
|
|
31
|
-
|
|
1
|
+
import { dirname, join } from 'node:path'
|
|
2
|
+
import { describe, expect, it } from 'vitest'
|
|
3
|
+
import type { SessionHeader } from '@deepseek-ai/dsh-session'
|
|
4
|
+
import { resolveTopicDataDir } from './topic-storage.ts'
|
|
5
|
+
|
|
6
|
+
const header = { id: 'session-test', cwd: 'C:\\workspace', createdAt: 1 } as unknown as SessionHeader
|
|
7
|
+
|
|
8
|
+
describe('resolveTopicDataDir', () => {
|
|
9
|
+
it('places verifier files beside the session artifact', () => {
|
|
10
|
+
const artifact = join('C:\\Users\\eddy\\.dsh\\sessions', 'project', 'session-test', 'session.jsonl.zstd')
|
|
11
|
+
const result = resolveTopicDataDir({ locate: () => ({ path: artifact }) }, header, 'verifier')
|
|
12
|
+
expect(result).toBe(join(dirname(artifact), 'verifier'))
|
|
13
|
+
})
|
|
14
|
+
|
|
15
|
+
it('rejects paths that escape the topic directory', () => {
|
|
16
|
+
const locator = { locate: () => ({ path: join('C:\\data', 'session-test', 'session.jsonl') }) }
|
|
17
|
+
expect(() => resolveTopicDataDir(locator, header, '..\\shared')).toThrow(/inside the topic directory/u)
|
|
18
|
+
expect(() => resolveTopicDataDir(locator, header, 'C:\\shared')).toThrow(/relative/u)
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
it('fails closed when the backend cannot locate a topic artifact', () => {
|
|
22
|
+
expect(() => resolveTopicDataDir({ locate: () => undefined }, header, 'verifier')).toThrow(/does not expose/u)
|
|
23
|
+
expect(() => resolveTopicDataDir({}, header, 'verifier')).toThrow(/does not expose/u)
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
it('falls back to root-based path when locate is missing', () => {
|
|
27
|
+
const result = resolveTopicDataDir({ root: 'C:\\data\\sessions' }, header, 'verifier')
|
|
28
|
+
expect(result).toBe(join('C:\\data\\sessions', 'session-test', 'verifier'))
|
|
29
|
+
})
|
|
30
|
+
})
|
|
31
|
+
|
package/src/topic-storage.ts
CHANGED
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
import { dirname, isAbsolute, relative, resolve } from 'node:path'
|
|
2
|
-
import type { SessionHeader } from '@deepseek-ai/dsh-session'
|
|
3
|
-
|
|
4
|
-
export interface SessionArtifactLocator {
|
|
5
|
-
locate?(meta: SessionHeader): { readonly path: string } | undefined
|
|
6
|
-
root?: string
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
function escapes(root: string, target: string): boolean {
|
|
10
|
-
const child = relative(root, target)
|
|
11
|
-
return child === '..' || child.startsWith('..\\') || child.startsWith('../') || isAbsolute(child)
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Resolve verifier sidecars beneath the persistence backend's per-session
|
|
16
|
-
* directory. The JSONL backend deliberately owns this directory for
|
|
17
|
-
* session-local artifacts, so permanent session deletion removes these files
|
|
18
|
-
* together with the conversation log.
|
|
19
|
-
*/
|
|
20
|
-
export function resolveTopicDataDir(locator: SessionArtifactLocator, header: SessionHeader, cacheDir: string): string {
|
|
21
|
-
if (isAbsolute(cacheDir)) throw new Error('llm-verifier: cacheDir must be relative so verifier data stays inside its topic directory')
|
|
22
|
-
|
|
23
|
-
let topicDir: string | undefined
|
|
24
|
-
if (typeof locator?.locate === 'function') {
|
|
25
|
-
const location = locator.locate(header)
|
|
26
|
-
if (location && typeof location.path === 'string') {
|
|
27
|
-
topicDir = dirname(location.path)
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
if (!topicDir && typeof locator?.root === 'string' && header?.id) {
|
|
32
|
-
topicDir = resolve(locator.root, String(header.id))
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
if (topicDir === undefined) {
|
|
36
|
-
throw new Error('llm-verifier: the active session persistence backend does not expose a per-session artifact directory')
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
const target = resolve(topicDir, cacheDir)
|
|
40
|
-
if (escapes(topicDir, target)) throw new Error('llm-verifier: cacheDir must stay inside the topic directory')
|
|
41
|
-
return target
|
|
42
|
-
}
|
|
1
|
+
import { dirname, isAbsolute, relative, resolve } from 'node:path'
|
|
2
|
+
import type { SessionHeader } from '@deepseek-ai/dsh-session'
|
|
3
|
+
|
|
4
|
+
export interface SessionArtifactLocator {
|
|
5
|
+
locate?(meta: SessionHeader): { readonly path: string } | undefined
|
|
6
|
+
root?: string
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function escapes(root: string, target: string): boolean {
|
|
10
|
+
const child = relative(root, target)
|
|
11
|
+
return child === '..' || child.startsWith('..\\') || child.startsWith('../') || isAbsolute(child)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Resolve verifier sidecars beneath the persistence backend's per-session
|
|
16
|
+
* directory. The JSONL backend deliberately owns this directory for
|
|
17
|
+
* session-local artifacts, so permanent session deletion removes these files
|
|
18
|
+
* together with the conversation log.
|
|
19
|
+
*/
|
|
20
|
+
export function resolveTopicDataDir(locator: SessionArtifactLocator, header: SessionHeader, cacheDir: string): string {
|
|
21
|
+
if (isAbsolute(cacheDir)) throw new Error('llm-verifier: cacheDir must be relative so verifier data stays inside its topic directory')
|
|
22
|
+
|
|
23
|
+
let topicDir: string | undefined
|
|
24
|
+
if (typeof locator?.locate === 'function') {
|
|
25
|
+
const location = locator.locate(header)
|
|
26
|
+
if (location && typeof location.path === 'string') {
|
|
27
|
+
topicDir = dirname(location.path)
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (!topicDir && typeof locator?.root === 'string' && header?.id) {
|
|
32
|
+
topicDir = resolve(locator.root, String(header.id))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (topicDir === undefined) {
|
|
36
|
+
throw new Error('llm-verifier: the active session persistence backend does not expose a per-session artifact directory')
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const target = resolve(topicDir, cacheDir)
|
|
40
|
+
if (escapes(topicDir, target)) throw new Error('llm-verifier: cacheDir must stay inside the topic directory')
|
|
41
|
+
return target
|
|
42
|
+
}
|