dsh-llm-verifier 0.1.10 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +1 -1
- package/lib/index.js +8 -4
- package/lib/types/client.d.ts +3 -1
- package/lib/types/client.js +1 -1
- package/lib/types/index.js +4 -1
- package/lib/types/topic-storage.d.ts +2 -1
- package/lib/types/topic-storage.js +13 -4
- package/package.json +9 -9
- package/src/client.tsx +2 -2
- package/src/index.ts +5 -2
- package/src/statistics.test.ts +12 -0
- package/src/topic-storage.test.ts +7 -0
- package/src/topic-storage.ts +19 -4
package/lib/client.js
CHANGED
|
@@ -534,7 +534,7 @@ window.__ModuleLoader__.load({
|
|
|
534
534
|
groups: m.value.groups,
|
|
535
535
|
settings: view,
|
|
536
536
|
writable: s.value.writable,
|
|
537
|
-
failures: m.value.failures.map((f) => f.provider + ": " + f.message)
|
|
537
|
+
failures: m.value.failures.map((f) => (f.id ?? f.provider ?? f.name ?? "unknown") + ": " + f.message)
|
|
538
538
|
};
|
|
539
539
|
setLoaded(next);
|
|
540
540
|
setDraft(values(view));
|
package/lib/index.js
CHANGED
|
@@ -1605,10 +1605,14 @@ function escapes(root, target) {
|
|
|
1605
1605
|
* together with the conversation log.
|
|
1606
1606
|
*/
|
|
1607
1607
|
function resolveTopicDataDir(locator, header, cacheDir) {
|
|
1608
|
-
const location = locator.locate(header);
|
|
1609
|
-
if (location === void 0) throw new Error("llm-verifier: the active session persistence backend does not expose a per-session artifact directory");
|
|
1610
1608
|
if (isAbsolute(cacheDir)) throw new Error("llm-verifier: cacheDir must be relative so verifier data stays inside its topic directory");
|
|
1611
|
-
|
|
1609
|
+
let topicDir;
|
|
1610
|
+
if (typeof locator?.locate === "function") {
|
|
1611
|
+
const location = locator.locate(header);
|
|
1612
|
+
if (location && typeof location.path === "string") topicDir = dirname(location.path);
|
|
1613
|
+
}
|
|
1614
|
+
if (!topicDir && typeof locator?.root === "string" && header?.id) topicDir = resolve(locator.root, String(header.id));
|
|
1615
|
+
if (topicDir === void 0) throw new Error("llm-verifier: the active session persistence backend does not expose a per-session artifact directory");
|
|
1612
1616
|
const target = resolve(topicDir, cacheDir);
|
|
1613
1617
|
if (escapes(topicDir, target)) throw new Error("llm-verifier: cacheDir must stay inside the topic directory");
|
|
1614
1618
|
return target;
|
|
@@ -2019,7 +2023,7 @@ function apply(ctx, config = {}) {
|
|
|
2019
2023
|
...sessionId ? { sessionId } : {}
|
|
2020
2024
|
};
|
|
2021
2025
|
try {
|
|
2022
|
-
const headers = (await services.sessionPersistence.list()).filter((header) => sessionId === void 0 || String(header.id) === sessionId);
|
|
2026
|
+
const headers = (await services.sessionPersistence.list()).map((item) => item && typeof item === "object" && "header" in item ? item.header : item).filter((header) => header !== void 0 && (sessionId === void 0 || String(header.id) === sessionId));
|
|
2023
2027
|
return rpcSuccess(mergeStatisticsOverviews(await Promise.all(headers.map((header) => topic(header).statistics.overview(query))), query));
|
|
2024
2028
|
} catch (error) {
|
|
2025
2029
|
return rpcFailure(error instanceof Error ? error.message : String(error));
|
package/lib/types/client.d.ts
CHANGED
package/lib/types/client.js
CHANGED
|
@@ -40,7 +40,7 @@ export function VerifierSettings({ remote }) {
|
|
|
40
40
|
const view = s.value.namespaces.find((x) => x.ns === NS);
|
|
41
41
|
if (!view)
|
|
42
42
|
throw new Error(t['settings.nsUnregistered']);
|
|
43
|
-
const next = { groups: m.value.groups, settings: view, writable: s.value.writable, failures: m.value.failures.map((f) => f.provider + ': ' + f.message) };
|
|
43
|
+
const next = { groups: m.value.groups, settings: view, writable: s.value.writable, failures: m.value.failures.map((f) => (f.id ?? f.provider ?? f.name ?? 'unknown') + ': ' + f.message) };
|
|
44
44
|
setLoaded(next);
|
|
45
45
|
setDraft(values(view));
|
|
46
46
|
}
|
package/lib/types/index.js
CHANGED
|
@@ -125,7 +125,10 @@ export function apply(ctx, config = {}) {
|
|
|
125
125
|
const sessionId = typeof row.sessionId === 'string' && row.sessionId.length > 0 ? row.sessionId : undefined;
|
|
126
126
|
const query = { fromMs: numberField(row.fromMs, Number.NaN), toMs: numberField(row.toMs, Number.NaN), timezoneOffsetMinutes: numberField(row.timezoneOffsetMinutes, 0), recentLimit: numberField(row.recentLimit, 40), ...(sessionId ? { sessionId } : {}) };
|
|
127
127
|
try {
|
|
128
|
-
const
|
|
128
|
+
const items = await services.sessionPersistence.list();
|
|
129
|
+
const headers = items
|
|
130
|
+
.map(item => item && typeof item === 'object' && 'header' in item ? item.header : item)
|
|
131
|
+
.filter(header => header !== undefined && (sessionId === undefined || String(header.id) === sessionId));
|
|
129
132
|
const overviews = await Promise.all(headers.map(header => topic(header).statistics.overview(query)));
|
|
130
133
|
const value = mergeStatisticsOverviews(overviews, query);
|
|
131
134
|
return rpcSuccess(value);
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { SessionHeader } from '@deepseek-ai/dsh-session';
|
|
2
2
|
export interface SessionArtifactLocator {
|
|
3
|
-
locate(meta: SessionHeader): {
|
|
3
|
+
locate?(meta: SessionHeader): {
|
|
4
4
|
readonly path: string;
|
|
5
5
|
} | undefined;
|
|
6
|
+
root?: string;
|
|
6
7
|
}
|
|
7
8
|
/**
|
|
8
9
|
* Resolve verifier sidecars beneath the persistence backend's per-session
|
|
@@ -10,12 +10,21 @@ function escapes(root, target) {
|
|
|
10
10
|
* together with the conversation log.
|
|
11
11
|
*/
|
|
12
12
|
export function resolveTopicDataDir(locator, header, cacheDir) {
|
|
13
|
-
const location = locator.locate(header);
|
|
14
|
-
if (location === undefined)
|
|
15
|
-
throw new Error('llm-verifier: the active session persistence backend does not expose a per-session artifact directory');
|
|
16
13
|
if (isAbsolute(cacheDir))
|
|
17
14
|
throw new Error('llm-verifier: cacheDir must be relative so verifier data stays inside its topic directory');
|
|
18
|
-
|
|
15
|
+
let topicDir;
|
|
16
|
+
if (typeof locator?.locate === 'function') {
|
|
17
|
+
const location = locator.locate(header);
|
|
18
|
+
if (location && typeof location.path === 'string') {
|
|
19
|
+
topicDir = dirname(location.path);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
if (!topicDir && typeof locator?.root === 'string' && header?.id) {
|
|
23
|
+
topicDir = resolve(locator.root, String(header.id));
|
|
24
|
+
}
|
|
25
|
+
if (topicDir === undefined) {
|
|
26
|
+
throw new Error('llm-verifier: the active session persistence backend does not expose a per-session artifact directory');
|
|
27
|
+
}
|
|
19
28
|
const target = resolve(topicDir, cacheDir);
|
|
20
29
|
if (escapes(topicDir, target))
|
|
21
30
|
throw new Error('llm-verifier: cacheDir must stay inside the topic directory');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-llm-verifier",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13",
|
|
4
4
|
"description": "Configurable DSH-native LLM verifier with a Web settings page",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -60,14 +60,14 @@
|
|
|
60
60
|
},
|
|
61
61
|
"peerDependencies": {
|
|
62
62
|
"@deepseek-ai/cordis": "^4.0.1 || ^4.0.2",
|
|
63
|
-
"@deepseek-ai/dsh-tools": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
64
|
-
"@deepseek-ai/dsh-agent": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
65
|
-
"@deepseek-ai/dsh-session": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
66
|
-
"@deepseek-ai/dsh-llm": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
67
|
-
"@deepseek-ai/dsh-attachment": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
68
|
-
"@deepseek-ai/dsh-settings": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
69
|
-
"@deepseek-ai/dsh-credentials": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2",
|
|
70
|
-
"@deepseek-ai/dsh-client-connection": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2"
|
|
63
|
+
"@deepseek-ai/dsh-tools": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
64
|
+
"@deepseek-ai/dsh-agent": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
65
|
+
"@deepseek-ai/dsh-session": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
66
|
+
"@deepseek-ai/dsh-llm": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
67
|
+
"@deepseek-ai/dsh-attachment": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
68
|
+
"@deepseek-ai/dsh-settings": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
69
|
+
"@deepseek-ai/dsh-credentials": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2",
|
|
70
|
+
"@deepseek-ai/dsh-client-connection": "^0.1.1-rc.2 || ^0.1.2-alpha.1 || ^0.1.2-alpha.2 || ^0.1.2-alpha.3 || ^0.1.2-alpha.4 || ^0.1.2-alpha.5 || ^0.1.2-rc.1 || ~0.1.2"
|
|
71
71
|
},
|
|
72
72
|
"dependencies": {
|
|
73
73
|
"schemastery": "^3.18.0"
|
package/src/client.tsx
CHANGED
|
@@ -26,7 +26,7 @@ interface VerifierRemote {
|
|
|
26
26
|
session: {
|
|
27
27
|
modelCatalog(): Promise<{
|
|
28
28
|
ok: boolean
|
|
29
|
-
value: { groups: readonly ModelProviderGroup[]; failures: readonly { provider
|
|
29
|
+
value: { groups: readonly ModelProviderGroup[]; failures: readonly { id?: string; provider?: string; name?: string; message: string }[] }
|
|
30
30
|
error: { message: string }
|
|
31
31
|
}>
|
|
32
32
|
}
|
|
@@ -81,7 +81,7 @@ export function VerifierSettings({ remote }: VerifierSettingsProps) {
|
|
|
81
81
|
const lang = useLanguage()
|
|
82
82
|
const t = dictionaries[lang]
|
|
83
83
|
const [loaded,setLoaded]=useState<Loaded|null>(null); const [draft,setDraft]=useState<Values|null>(null); const [busy,setBusy]=useState(false); const [error,setError]=useState<string|null>(null); const [saved,setSaved]=useState(false)
|
|
84
|
-
const load=async()=>{setError(null);try{const [m,s]=await Promise.all([remote.session.modelCatalog(),remote.settings.describe()]);if(!m.ok)throw new Error(m.error.message);if(!s.ok)throw new Error(s.error.message);const view=s.value.namespaces.find((x:SettingsNamespaceView)=>x.ns===NS);if(!view)throw new Error(t['settings.nsUnregistered']);const next={groups:m.value.groups,settings:view,writable:s.value.writable,failures:m.value.failures.map((f: { provider
|
|
84
|
+
const load=async()=>{setError(null);try{const [m,s]=await Promise.all([remote.session.modelCatalog(),remote.settings.describe()]);if(!m.ok)throw new Error(m.error.message);if(!s.ok)throw new Error(s.error.message);const view=s.value.namespaces.find((x:SettingsNamespaceView)=>x.ns===NS);if(!view)throw new Error(t['settings.nsUnregistered']);const next={groups:m.value.groups,settings:view,writable:s.value.writable,failures:m.value.failures.map((f: { id?: string; provider?: string; name?: string; message: string })=>(f.id??f.provider??f.name??'unknown')+': '+f.message)};setLoaded(next);setDraft(values(view))}catch(e){setError(message(e))}}
|
|
85
85
|
useEffect(()=>{void load()},[])
|
|
86
86
|
const models=useMemo(()=>loaded?.groups.find(g=>g.id===draft?.provider)?.models??[],[loaded,draft?.provider])
|
|
87
87
|
const selected=models.find(m=>m.id===draft?.model); const efforts=selected?.reasoning?.efforts??[]
|
package/src/index.ts
CHANGED
|
@@ -44,7 +44,7 @@ interface SessionVerificationOptions { fromSeq?: number; toSeq?: number; include
|
|
|
44
44
|
interface SessionVerificationResult { sessionId: string; problem: string; score: number; baselineScore: number; winner: 'A' | 'B' | 'tie'; fromSeq: number; toSeq: number; omittedCharacters: number; calls: number; stats: RunStats }
|
|
45
45
|
|
|
46
46
|
export function apply(ctx: Context, config: Config = {}): void {
|
|
47
|
-
const services = ctx as Context & { attachments: AttachmentStore; connection: HostConnectionHandle; sessionPersistence: SessionArtifactLocator & { list(signal?: AbortSignal): Promise<
|
|
47
|
+
const services = ctx as Context & { attachments: AttachmentStore; connection: HostConnectionHandle; sessionPersistence: SessionArtifactLocator & { list(signal?: AbortSignal): Promise<readonly unknown[]> } }
|
|
48
48
|
const entry = resolveConfig(config)
|
|
49
49
|
let limiter = new RequestLimiter(entry.maxConcurrency)
|
|
50
50
|
const current = installVerifierSettings(ctx, entry, () => { limiter = new RequestLimiter(current().maxConcurrency) })
|
|
@@ -128,7 +128,10 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
128
128
|
const sessionId = typeof row.sessionId === 'string' && row.sessionId.length > 0 ? row.sessionId : undefined
|
|
129
129
|
const query: StatisticsQuery = { fromMs: numberField(row.fromMs, Number.NaN), toMs: numberField(row.toMs, Number.NaN), timezoneOffsetMinutes: numberField(row.timezoneOffsetMinutes, 0), recentLimit: numberField(row.recentLimit, 40), ...(sessionId ? { sessionId } : {}) }
|
|
130
130
|
try {
|
|
131
|
-
const
|
|
131
|
+
const items = await services.sessionPersistence.list()
|
|
132
|
+
const headers: SessionHeader[] = items
|
|
133
|
+
.map(item => item && typeof item === 'object' && 'header' in item ? (item as { header: SessionHeader }).header : item as SessionHeader)
|
|
134
|
+
.filter(header => header !== undefined && (sessionId === undefined || String(header.id) === sessionId))
|
|
132
135
|
const overviews = await Promise.all(headers.map(header => topic(header).statistics.overview(query)))
|
|
133
136
|
const value: StatisticsOverview = mergeStatisticsOverviews(overviews, query)
|
|
134
137
|
return rpcSuccess(value)
|
package/src/statistics.test.ts
CHANGED
|
@@ -65,4 +65,16 @@ describe('StatisticsStore', () => {
|
|
|
65
65
|
const result = await store.overview({ fromMs: 0, toMs: 10 })
|
|
66
66
|
expect(result.recent[0]?.errorMessage).toHaveLength(500)
|
|
67
67
|
})
|
|
68
|
+
|
|
69
|
+
it('normalizes legacy and modern sessionPersistence.list output formats', () => {
|
|
70
|
+
const legacyHeaders = [{ id: 'sess-1', cwd: 'C:\\test' }]
|
|
71
|
+
const modernSnapshots = [{ header: { id: 'sess-2', cwd: 'C:\\test' }, revision: 'rev-1' }]
|
|
72
|
+
const normalize = (items: readonly unknown[]) =>
|
|
73
|
+
items
|
|
74
|
+
.map(item => item && typeof item === 'object' && 'header' in item ? (item as { header: { id: string } }).header : item as { id: string })
|
|
75
|
+
.filter(header => header !== undefined)
|
|
76
|
+
|
|
77
|
+
expect(normalize(legacyHeaders).map(h => h.id)).toEqual(['sess-1'])
|
|
78
|
+
expect(normalize(modernSnapshots).map(h => h.id)).toEqual(['sess-2'])
|
|
79
|
+
})
|
|
68
80
|
})
|
|
@@ -20,5 +20,12 @@ describe('resolveTopicDataDir', () => {
|
|
|
20
20
|
|
|
21
21
|
it('fails closed when the backend cannot locate a topic artifact', () => {
|
|
22
22
|
expect(() => resolveTopicDataDir({ locate: () => undefined }, header, 'verifier')).toThrow(/does not expose/u)
|
|
23
|
+
expect(() => resolveTopicDataDir({}, header, 'verifier')).toThrow(/does not expose/u)
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
it('falls back to root-based path when locate is missing', () => {
|
|
27
|
+
const result = resolveTopicDataDir({ root: 'C:\\data\\sessions' }, header, 'verifier')
|
|
28
|
+
expect(result).toBe(join('C:\\data\\sessions', 'session-test', 'verifier'))
|
|
23
29
|
})
|
|
24
30
|
})
|
|
31
|
+
|
package/src/topic-storage.ts
CHANGED
|
@@ -2,7 +2,8 @@ import { dirname, isAbsolute, relative, resolve } from 'node:path'
|
|
|
2
2
|
import type { SessionHeader } from '@deepseek-ai/dsh-session'
|
|
3
3
|
|
|
4
4
|
export interface SessionArtifactLocator {
|
|
5
|
-
locate(meta: SessionHeader): { readonly path: string } | undefined
|
|
5
|
+
locate?(meta: SessionHeader): { readonly path: string } | undefined
|
|
6
|
+
root?: string
|
|
6
7
|
}
|
|
7
8
|
|
|
8
9
|
function escapes(root: string, target: string): boolean {
|
|
@@ -17,10 +18,24 @@ function escapes(root: string, target: string): boolean {
|
|
|
17
18
|
* together with the conversation log.
|
|
18
19
|
*/
|
|
19
20
|
export function resolveTopicDataDir(locator: SessionArtifactLocator, header: SessionHeader, cacheDir: string): string {
|
|
20
|
-
const location = locator.locate(header)
|
|
21
|
-
if (location === undefined) throw new Error('llm-verifier: the active session persistence backend does not expose a per-session artifact directory')
|
|
22
21
|
if (isAbsolute(cacheDir)) throw new Error('llm-verifier: cacheDir must be relative so verifier data stays inside its topic directory')
|
|
23
|
-
|
|
22
|
+
|
|
23
|
+
let topicDir: string | undefined
|
|
24
|
+
if (typeof locator?.locate === 'function') {
|
|
25
|
+
const location = locator.locate(header)
|
|
26
|
+
if (location && typeof location.path === 'string') {
|
|
27
|
+
topicDir = dirname(location.path)
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (!topicDir && typeof locator?.root === 'string' && header?.id) {
|
|
32
|
+
topicDir = resolve(locator.root, String(header.id))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (topicDir === undefined) {
|
|
36
|
+
throw new Error('llm-verifier: the active session persistence backend does not expose a per-session artifact directory')
|
|
37
|
+
}
|
|
38
|
+
|
|
24
39
|
const target = resolve(topicDir, cacheDir)
|
|
25
40
|
if (escapes(topicDir, target)) throw new Error('llm-verifier: cacheDir must stay inside the topic directory')
|
|
26
41
|
return target
|