dsh-llm-verifier 0.1.13 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/README.md +328 -301
- package/cordis.patch.yml +4 -4
- package/lib/caller-BWwSVNZf.js.map +1 -0
- package/lib/client.js +176 -14
- package/lib/client.js.map +1 -0
- package/lib/core.js.map +1 -0
- package/lib/index.js +400 -51
- package/lib/index.js.map +1 -0
- package/lib/types/auto.d.ts.map +1 -0
- package/lib/types/auto.js +8 -2
- package/lib/types/auto.js.map +1 -0
- package/lib/types/cache.d.ts.map +1 -0
- package/lib/types/cache.js.map +1 -0
- package/lib/types/caller.d.ts.map +1 -0
- package/lib/types/caller.js.map +1 -0
- package/lib/types/client-i18n.d.ts +27 -0
- package/lib/types/client-i18n.d.ts.map +1 -0
- package/lib/types/client-i18n.js +18 -0
- package/lib/types/client-i18n.js.map +1 -0
- package/lib/types/client.d.ts +15 -2
- package/lib/types/client.d.ts.map +1 -0
- package/lib/types/client.js +137 -10
- package/lib/types/client.js.map +1 -0
- package/lib/types/config.d.ts +4 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +5 -3
- package/lib/types/config.js.map +1 -0
- package/lib/types/core.d.ts.map +1 -0
- package/lib/types/core.js.map +1 -0
- package/lib/types/engine.d.ts.map +1 -0
- package/lib/types/engine.js.map +1 -0
- package/lib/types/images.d.ts.map +1 -0
- package/lib/types/images.js.map +1 -0
- package/lib/types/index.d.ts +2 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +162 -9
- package/lib/types/index.js.map +1 -0
- package/lib/types/plan-gate.d.ts +27 -0
- package/lib/types/plan-gate.d.ts.map +1 -0
- package/lib/types/plan-gate.js +60 -0
- package/lib/types/plan-gate.js.map +1 -0
- package/lib/types/router.d.ts +28 -32
- package/lib/types/router.d.ts.map +1 -0
- package/lib/types/router.js +41 -9
- package/lib/types/router.js.map +1 -0
- package/lib/types/session.d.ts +11 -0
- package/lib/types/session.d.ts.map +1 -0
- package/lib/types/session.js +41 -11
- package/lib/types/session.js.map +1 -0
- package/lib/types/statistics.d.ts.map +1 -0
- package/lib/types/statistics.js.map +1 -0
- package/lib/types/team-gate.d.ts +20 -0
- package/lib/types/team-gate.d.ts.map +1 -0
- package/lib/types/team-gate.js +59 -0
- package/lib/types/team-gate.js.map +1 -0
- package/lib/types/top-logprobs.d.ts.map +1 -0
- package/lib/types/top-logprobs.js.map +1 -0
- package/lib/types/topic-storage.d.ts.map +1 -0
- package/lib/types/topic-storage.js.map +1 -0
- package/package.json +114 -112
- package/src/auto.test.ts +95 -86
- package/src/auto.ts +140 -133
- package/src/cache.ts +115 -115
- package/src/caller.test.ts +34 -34
- package/src/caller.ts +151 -151
- package/src/client-i18n.ts +18 -0
- package/src/client.tsx +383 -213
- package/src/config.ts +189 -183
- package/src/core.test.ts +71 -71
- package/src/core.ts +205 -205
- package/src/engine.test.ts +136 -136
- package/src/engine.ts +140 -140
- package/src/images.ts +33 -33
- package/src/index.ts +394 -247
- package/src/parity.test.ts +85 -85
- package/src/plan-gate.test.ts +56 -0
- package/src/plan-gate.ts +66 -0
- package/src/router.test.ts +107 -89
- package/src/router.ts +361 -309
- package/src/session.test.ts +71 -36
- package/src/session.ts +132 -103
- package/src/statistics.test.ts +80 -80
- package/src/statistics.ts +328 -328
- package/src/team-gate.test.ts +92 -0
- package/src/team-gate.ts +80 -0
- package/src/top-logprobs.test.ts +84 -84
- package/src/top-logprobs.ts +153 -153
- package/src/topic-storage.test.ts +31 -31
- package/src/topic-storage.ts +42 -42
package/src/session.test.ts
CHANGED
|
@@ -1,36 +1,71 @@
|
|
|
1
|
-
import { describe, expect, it } from 'vitest'
|
|
2
|
-
import { Session } from '@deepseek-ai/dsh-session'
|
|
3
|
-
import { createUserMessage, createAssistantMessage, createToolResultMessage } from '@deepseek-ai/dsh-llm'
|
|
4
|
-
import { extractSession } from './session.ts'
|
|
5
|
-
|
|
6
|
-
describe('current session extraction', () => {
|
|
7
|
-
it('keeps direct evidence, skips plugin instructions, and redacts secrets', async () => {
|
|
8
|
-
const session = Session.create('session-00000000-0000-4000-8000-000000000001' as never)
|
|
9
|
-
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Fix task token = abc123 Bearer live-secret' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
10
|
-
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'hidden plugin instruction' }], source: { kind: 'plugin', plugin: 'test' } }), { surfaceOp: 'append' })
|
|
11
|
-
session.append('assistant/message', { turn: 1, step: 1, message: createAssistantMessage({ content: [{ type: 'text', text: 'running checks' }], source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' } }) }, { surfaceOp: 'append' })
|
|
12
|
-
session.append('tool/call', { turn: 1, step: 1, callId: 'call-1' as never, name: 'pwsh', arguments: '{"command":"test"}' })
|
|
13
|
-
session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'call-1' as never, content: [{ type: 'text', text: 'exit 0 password=hunter2' }], isError: false }) }, { surfaceOp: 'append' })
|
|
14
|
-
const agent = { id: session.id, session } as never
|
|
15
|
-
const result = await extractSession(agent, async () => { throw new Error('no image expected') })
|
|
16
|
-
expect(result.problem).toContain('Fix task')
|
|
17
|
-
expect(result.problem).toContain('[REDACTED]')
|
|
18
|
-
expect(result.problem).not.toContain('abc123')
|
|
19
|
-
expect(result.problem).not.toContain('live-secret')
|
|
20
|
-
expect(result.trace).not.toContain('hidden plugin instruction')
|
|
21
|
-
expect(result.trace).toContain('exit 0')
|
|
22
|
-
expect(result.trace).not.toContain('hunter2')
|
|
23
|
-
})
|
|
24
|
-
|
|
25
|
-
it('extracts PTC mode tool/code-dispatch events and redacts content', async () => {
|
|
26
|
-
const session = Session.create('session-00000000-0000-4000-8000-000000000002' as never)
|
|
27
|
-
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Run PTC task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
28
|
-
session.append('tool/code-dispatch', { subCallId: 'code-1' as never, name: 'edit', arguments: { file: 'src/main.ts' }, isError: false, content: [{ type: 'text', text: 'updated file api_key=secret-value' }] })
|
|
29
|
-
const agent = { id: session.id, session } as never
|
|
30
|
-
const result = await extractSession(agent, async () => { throw new Error('no image expected') })
|
|
31
|
-
expect(result.trace).toContain('--- Code Dispatch edit')
|
|
32
|
-
expect(result.trace).toContain('src/main.ts')
|
|
33
|
-
expect(result.trace).toContain('[REDACTED]')
|
|
34
|
-
expect(result.trace).not.toContain('secret-value')
|
|
35
|
-
})
|
|
36
|
-
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
import { Session } from '@deepseek-ai/dsh-session'
|
|
3
|
+
import { createUserMessage, createAssistantMessage, createToolResultMessage } from '@deepseek-ai/dsh-llm'
|
|
4
|
+
import { extractSession, sessionEvents } from './session.ts'
|
|
5
|
+
|
|
6
|
+
describe('current session extraction', () => {
|
|
7
|
+
it('keeps direct evidence, skips plugin instructions, and redacts secrets', async () => {
|
|
8
|
+
const session = Session.create('session-00000000-0000-4000-8000-000000000001' as never)
|
|
9
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Fix task token = abc123 Bearer live-secret' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
10
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'hidden plugin instruction' }], source: { kind: 'plugin', plugin: 'test' } }), { surfaceOp: 'append' })
|
|
11
|
+
session.append('assistant/message', { turn: 1, step: 1, message: createAssistantMessage({ content: [{ type: 'text', text: 'running checks' }], source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' } }) }, { surfaceOp: 'append' })
|
|
12
|
+
session.append('tool/call', { turn: 1, step: 1, callId: 'call-1' as never, name: 'pwsh', arguments: '{"command":"test"}' })
|
|
13
|
+
session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'call-1' as never, content: [{ type: 'text', text: 'exit 0 password=hunter2' }], isError: false }) }, { surfaceOp: 'append' })
|
|
14
|
+
const agent = { id: session.id, session } as never
|
|
15
|
+
const result = await extractSession(agent, async () => { throw new Error('no image expected') })
|
|
16
|
+
expect(result.problem).toContain('Fix task')
|
|
17
|
+
expect(result.problem).toContain('[REDACTED]')
|
|
18
|
+
expect(result.problem).not.toContain('abc123')
|
|
19
|
+
expect(result.problem).not.toContain('live-secret')
|
|
20
|
+
expect(result.trace).not.toContain('hidden plugin instruction')
|
|
21
|
+
expect(result.trace).toContain('exit 0')
|
|
22
|
+
expect(result.trace).not.toContain('hunter2')
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
it('extracts PTC mode tool/code-dispatch events and redacts content', async () => {
|
|
26
|
+
const session = Session.create('session-00000000-0000-4000-8000-000000000002' as never)
|
|
27
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Run PTC task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
28
|
+
session.append('tool/code-dispatch', { subCallId: 'code-1' as never, name: 'edit', arguments: { file: 'src/main.ts' }, isError: false, content: [{ type: 'text', text: 'updated file api_key=secret-value' }] })
|
|
29
|
+
const agent = { id: session.id, session } as never
|
|
30
|
+
const result = await extractSession(agent, async () => { throw new Error('no image expected') })
|
|
31
|
+
expect(result.trace).toContain('--- Code Dispatch edit')
|
|
32
|
+
expect(result.trace).toContain('src/main.ts')
|
|
33
|
+
expect(result.trace).toContain('[REDACTED]')
|
|
34
|
+
expect(result.trace).not.toContain('secret-value')
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
it('extracts Session V3 tool/ptc-dispatch, file content blocks, and team messages', async () => {
|
|
38
|
+
const session = Session.create('session-00000000-0000-4000-8000-000000000003' as never)
|
|
39
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Team task instructions' }, { type: 'file' as never, path: 'src/config.ts' } as never], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
|
40
|
+
session.append('tool/ptc-dispatch' as never, { subCallId: 'ptc-1', name: 'run_test', arguments: { target: 'unit' }, isError: false, content: [{ type: 'text', text: 'all tests passed' }] } as never)
|
|
41
|
+
session.append('team/message/queued' as never, { message: { senderName: 'Alice', content: [{ type: 'text', text: 'Reviewed PR #123' }] } } as never)
|
|
42
|
+
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Follow-up from Bob' }], source: { kind: 'team-message' as never } as never }), { surfaceOp: 'append' })
|
|
43
|
+
const agent = { id: session.id, session } as never
|
|
44
|
+
const result = await extractSession(agent, async () => { throw new Error('no image expected') })
|
|
45
|
+
expect(result.trace).toContain('[File] src/config.ts')
|
|
46
|
+
expect(result.trace).toContain('--- PTC Dispatch run_test')
|
|
47
|
+
expect(result.trace).toContain('all tests passed')
|
|
48
|
+
expect(result.trace).toContain('--- Team Message Queued from Alice')
|
|
49
|
+
expect(result.trace).toContain('Reviewed PR #123')
|
|
50
|
+
expect(result.trace).toContain('--- Team Message seq')
|
|
51
|
+
expect(result.trace).toContain('Follow-up from Bob')
|
|
52
|
+
})
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
describe('session event accessor', () => {
|
|
56
|
+
it('reads the 0.1.5 snapshotEvents() shape', () => {
|
|
57
|
+
const events = [{ seq: 0, type: 'user/message' }]
|
|
58
|
+
expect(sessionEvents({ snapshotEvents: () => events as never })).toBe(events)
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('falls back to the legacy events array', () => {
|
|
62
|
+
const events = [{ seq: 0, type: 'user/message' }]
|
|
63
|
+
expect(sessionEvents({ events: events as never })).toBe(events)
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
it('returns an empty log instead of throwing on an unknown session shape', () => {
|
|
67
|
+
expect(sessionEvents({})).toEqual([])
|
|
68
|
+
expect(sessionEvents(undefined)).toEqual([])
|
|
69
|
+
expect(sessionEvents({ events: undefined })).toEqual([])
|
|
70
|
+
})
|
|
71
|
+
})
|
package/src/session.ts
CHANGED
|
@@ -1,103 +1,132 @@
|
|
|
1
|
-
import type { ContentBlock } from '@deepseek-ai/dsh-llm'
|
|
2
|
-
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
3
|
-
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
4
|
-
import type { VerifierImage } from './caller.ts'
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
1
|
+
import type { ContentBlock } from '@deepseek-ai/dsh-llm'
|
|
2
|
+
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
3
|
+
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
4
|
+
import type { VerifierImage } from './caller.ts'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Read one session's event log.
|
|
8
|
+
*
|
|
9
|
+
* DSH 0.1.5 replaced the `events` array with `snapshotEvents()`; older hosts
|
|
10
|
+
* expose the array directly. Both shapes are accepted so the plugin keeps
|
|
11
|
+
* working across the versions it declares support for.
|
|
12
|
+
* @param session - Agent session, or any object exposing one of the two shapes.
|
|
13
|
+
* @returns The session's events in log order, or an empty array.
|
|
14
|
+
*/
|
|
15
|
+
export function sessionEvents(session: unknown): readonly SessionEvent[] {
|
|
16
|
+
const candidate = session as { snapshotEvents?: () => readonly SessionEvent[]; events?: readonly SessionEvent[] } | undefined
|
|
17
|
+
if (typeof candidate?.snapshotEvents === 'function') return candidate.snapshotEvents()
|
|
18
|
+
return candidate?.events ?? []
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface SessionExtractOptions {
|
|
22
|
+
fromSeq?: number
|
|
23
|
+
toSeq?: number
|
|
24
|
+
includeAssistantText?: boolean
|
|
25
|
+
redactPatterns?: readonly string[]
|
|
26
|
+
maxChars?: number
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface SessionExtraction {
|
|
30
|
+
problem: string
|
|
31
|
+
trace: string
|
|
32
|
+
images: VerifierImage[]
|
|
33
|
+
sessionId: string
|
|
34
|
+
fromSeq: number
|
|
35
|
+
toSeq: number
|
|
36
|
+
omittedCharacters: number
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function textOf(blocks: readonly ContentBlock[]): string {
|
|
40
|
+
const parts: string[] = []
|
|
41
|
+
for (const block of blocks) {
|
|
42
|
+
const blockType = (block as { type: string }).type
|
|
43
|
+
if (blockType === 'text') parts.push((block as Extract<ContentBlock, { type: 'text' }>).text)
|
|
44
|
+
else if (blockType === 'reasoning') parts.push('[Reasoning] ' + (block as Extract<ContentBlock, { type: 'reasoning' }>).text)
|
|
45
|
+
else if (blockType === 'tool-call') parts.push('[Tool Call] ' + (block as Extract<ContentBlock, { type: 'tool-call' }>).name + ' ' + (block as Extract<ContentBlock, { type: 'tool-call' }>).arguments)
|
|
46
|
+
else if (blockType === 'tool-result') parts.push('[Tool Result] ' + textOf((block as Extract<ContentBlock, { type: 'tool-result' }>).content))
|
|
47
|
+
else if (blockType === 'file') {
|
|
48
|
+
const fileData = block as unknown as { path?: string; filename?: string; title?: string }
|
|
49
|
+
parts.push('[File] ' + (fileData.path ?? fileData.filename ?? fileData.title ?? 'attachment'))
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return parts.join('\n')
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export const DEFAULT_REDACT_PATTERNS = [
|
|
56
|
+
'Bearer\\s+[A-Za-z0-9._~+\\/=-]+',
|
|
57
|
+
"(?:api[_-]?key|token|password|secret)\\s*[=:]\\s*[\"']?[^\\s,\"';}]+",
|
|
58
|
+
] as const
|
|
59
|
+
|
|
60
|
+
function validateRedactPattern(pattern: string): void {
|
|
61
|
+
if (pattern.length === 0 || pattern.length > 500) throw new Error('llm-verifier: redact patterns must contain 1-500 characters')
|
|
62
|
+
// Reject common nested/unbounded constructs that can cause catastrophic backtracking.
|
|
63
|
+
if (/\([^)]*[+*][^)]*\)[+*{]|\.\*[+*{]|\.\+[+*{]/u.test(pattern)) throw new Error('llm-verifier: unsafe redact pattern')
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function redactText(text: string, patterns: readonly string[] = DEFAULT_REDACT_PATTERNS): string {
|
|
67
|
+
let result = text
|
|
68
|
+
for (const pattern of patterns) {
|
|
69
|
+
validateRedactPattern(pattern)
|
|
70
|
+
let regex: RegExp
|
|
71
|
+
try { regex = new RegExp(pattern, 'giu') } catch { throw new Error('llm-verifier: invalid redact pattern: ' + pattern) }
|
|
72
|
+
result = result.replace(regex, '[REDACTED]')
|
|
73
|
+
}
|
|
74
|
+
return result
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function sanitizeVerifierText(text: string, maxChars: number, patterns: readonly string[] = DEFAULT_REDACT_PATTERNS): string {
|
|
78
|
+
if (!Number.isSafeInteger(maxChars) || maxChars < 1) throw new Error('llm-verifier: sanitizer maxChars must be a positive integer')
|
|
79
|
+
const redacted = redactText(text, patterns).trim()
|
|
80
|
+
return redacted.length <= maxChars ? redacted : redacted.slice(0, maxChars) + '\n[Truncated ' + (redacted.length - maxChars) + ' characters]'
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export async function extractSession(agent: Agent, loadImage: (ref: Extract<ContentBlock, { type: 'image' }>['attachment']) => Promise<VerifierImage>, options: SessionExtractOptions = {}): Promise<SessionExtraction> {
|
|
84
|
+
const all = sessionEvents(agent.session)
|
|
85
|
+
const from = options.fromSeq ?? 0
|
|
86
|
+
const to = options.toSeq ?? Number.MAX_SAFE_INTEGER
|
|
87
|
+
const events = all.filter(event => event.seq >= from && event.seq <= to)
|
|
88
|
+
const patterns = [...DEFAULT_REDACT_PATTERNS, ...(options.redactPatterns ?? [])]
|
|
89
|
+
let problem = ''
|
|
90
|
+
const trace: string[] = []
|
|
91
|
+
const images: VerifierImage[] = []
|
|
92
|
+
for (const rawEvent of events) {
|
|
93
|
+
const event = rawEvent as unknown as { type: string; seq: number; data: any }
|
|
94
|
+
if (event.type === 'user/message') {
|
|
95
|
+
const sourceKind = event.data?.source?.kind as string
|
|
96
|
+
if (sourceKind !== 'user' && sourceKind !== 'team-message') continue
|
|
97
|
+
const text = textOf(event.data.content)
|
|
98
|
+
if (!problem && sourceKind === 'user' && text.trim()) problem = text.trim()
|
|
99
|
+
for (const block of event.data.content) if (block.type === 'image') images.push(await loadImage(block.attachment))
|
|
100
|
+
const tag = sourceKind === 'team-message' ? 'Team Message' : 'User'
|
|
101
|
+
trace.push('--- ' + tag + ' seq ' + event.seq + ' ---\n' + text)
|
|
102
|
+
} else if (event.type === 'assistant/message' && options.includeAssistantText !== false) {
|
|
103
|
+
trace.push('--- Assistant turn ' + event.data.turn + ' step ' + event.data.step + ' ---\n' + textOf(event.data.message.content))
|
|
104
|
+
} else if (event.type === 'tool/call') {
|
|
105
|
+
trace.push('--- Tool Call turn ' + event.data.turn + ' step ' + event.data.step + ' ---\n[Command] ' + event.data.name + ' ' + event.data.arguments)
|
|
106
|
+
} else if (event.type === 'tool/result') {
|
|
107
|
+
trace.push('--- Tool Result turn ' + event.data.turn + ' step ' + event.data.step + ' ---\n[Output] ' + textOf(event.data.message.content))
|
|
108
|
+
} else if (event.type === 'tool/ptc-dispatch' || event.type === 'tool/code-dispatch') {
|
|
109
|
+
const data = event.data as { name: string; arguments?: unknown; isError?: boolean; content?: readonly ContentBlock[] }
|
|
110
|
+
const status = data.isError ? ' [Error]' : ''
|
|
111
|
+
const args = data.arguments !== undefined ? ' ' + (typeof data.arguments === 'string' ? data.arguments : JSON.stringify(data.arguments)) : ''
|
|
112
|
+
const content = Array.isArray(data.content) ? textOf(data.content) : ''
|
|
113
|
+
if (Array.isArray(data.content)) {
|
|
114
|
+
for (const block of data.content) {
|
|
115
|
+
if (block.type === 'image') images.push(await loadImage(block.attachment))
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
const label = event.type === 'tool/ptc-dispatch' ? 'PTC Dispatch ' : 'Code Dispatch '
|
|
119
|
+
trace.push('--- ' + label + data.name + status + ' seq ' + event.seq + ' ---\n[Command] ' + data.name + args + '\n[Output] ' + content)
|
|
120
|
+
} else if (event.type === 'team/message/queued') {
|
|
121
|
+
const data = event.data as { message?: { senderName?: string; targetId?: string; content?: readonly ContentBlock[] } }
|
|
122
|
+
const sender = data?.message?.senderName ?? 'Teammate'
|
|
123
|
+
const content = Array.isArray(data?.message?.content) ? textOf(data.message!.content) : ''
|
|
124
|
+
trace.push('--- Team Message Queued from ' + sender + ' seq ' + event.seq + ' ---\n' + content)
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const raw = redactText(trace.join('\n\n'), patterns)
|
|
128
|
+
const maxChars = options.maxChars ?? 200000
|
|
129
|
+
const omittedCharacters = Math.max(0, raw.length - maxChars)
|
|
130
|
+
const bounded = omittedCharacters ? '[Earlier trace truncated: ' + omittedCharacters + ' characters omitted]\n' + raw.slice(-maxChars) : raw
|
|
131
|
+
return { problem: redactText(problem, patterns), trace: bounded, images, sessionId: String(agent.id), fromSeq: events[0]?.seq ?? from, toSeq: events.at(-1)?.seq ?? from, omittedCharacters }
|
|
132
|
+
}
|
package/src/statistics.test.ts
CHANGED
|
@@ -1,80 +1,80 @@
|
|
|
1
|
-
import { mkdtemp, readFile } from 'node:fs/promises'
|
|
2
|
-
import { join } from 'node:path'
|
|
3
|
-
import { tmpdir } from 'node:os'
|
|
4
|
-
import { describe, expect, it } from 'vitest'
|
|
5
|
-
import { StatisticsStore, emptyRunStats, mergeStatisticsOverviews } from './statistics.ts'
|
|
6
|
-
|
|
7
|
-
function stats(overrides: Partial<ReturnType<typeof emptyRunStats>> = {}) {
|
|
8
|
-
return { ...emptyRunStats(), ...overrides }
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
describe('StatisticsStore', () => {
|
|
12
|
-
it('records and aggregates invocations by day, tool, model, and session', async () => {
|
|
13
|
-
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
14
|
-
const file = join(root, 'statistics.json')
|
|
15
|
-
const store = new StatisticsStore(file, 100)
|
|
16
|
-
const first = Date.UTC(2026, 7, 19, 23, 30)
|
|
17
|
-
const second = Date.UTC(2026, 7, 20, 1, 30)
|
|
18
|
-
await store.record({ toolName: 'verifier_compare', sessionId: 'one', startedAt: first, finishedAt: first + 250, success: true, provider: 'p', model: 'm', stats: stats({ calls: 2, attempts: 2, inputTokens: 100, cachedInputTokens: 20, outputTokens: 10, cacheHits: 1, cacheMisses: 1, estimatedCostUsd: 0.12, topLogprobScores: 2 }) })
|
|
19
|
-
await store.record({ toolName: 'verifier_track', sessionId: 'two', startedAt: second, finishedAt: second + 750, success: false, errorName: 'Error', errorMessage: 'failed', provider: 'p', model: 'm2', stats: stats({ calls: 1, attempts: 2, retries: 1, inputTokens: 50, outputTokens: 5, cacheMisses: 1, explicitTagScores: 1 }) })
|
|
20
|
-
|
|
21
|
-
const all = await store.overview({ fromMs: first - 1, toMs: second + 10_000, timezoneOffsetMinutes: -120 })
|
|
22
|
-
expect(all.totals).toMatchObject({ invocations: 2, successes: 1, failures: 1, calls: 3, attempts: 4, retries: 1, tokens: 185, cacheHits: 1, cacheMisses: 2 })
|
|
23
|
-
expect(all.totals.successRate).toBe(0.5)
|
|
24
|
-
expect(all.totals.averageDurationMs).toBe(500)
|
|
25
|
-
expect(all.daily).toHaveLength(1)
|
|
26
|
-
expect(all.daily[0]).toMatchObject({ date: '2026-08-20', invocations: 2, calls: 3, tokens: 185 })
|
|
27
|
-
expect(all.tools.map(row => [row.toolName, row.invocations])).toEqual([['verifier_compare', 1], ['verifier_track', 1]])
|
|
28
|
-
expect(all.models.map(row => row.model)).toEqual(['m', 'm2'])
|
|
29
|
-
|
|
30
|
-
const one = await store.overview({ fromMs: first - 1, toMs: second + 10_000, sessionId: 'one' })
|
|
31
|
-
expect(one.totals.invocations).toBe(1)
|
|
32
|
-
expect(one.recent[0]?.sessionId).toBe('one')
|
|
33
|
-
expect(JSON.parse(await readFile(file, 'utf8')).version).toBe(1)
|
|
34
|
-
})
|
|
35
|
-
|
|
36
|
-
it('serializes concurrent writes and keeps only the configured tail', async () => {
|
|
37
|
-
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
38
|
-
const store = new StatisticsStore(join(root, 'statistics.json'), 2)
|
|
39
|
-
await Promise.all([0, 1, 2].map(index => store.record({ toolName: 'verifier_select', startedAt: 1_000 + index, success: true, provider: 'p', model: 'm', stats: stats() })))
|
|
40
|
-
const result = await store.overview({ fromMs: 0, toMs: 10_000 })
|
|
41
|
-
expect(result.totals.invocations).toBe(2)
|
|
42
|
-
expect(result.recent.map(record => record.startedAt)).toEqual([1_002, 1_001])
|
|
43
|
-
})
|
|
44
|
-
|
|
45
|
-
it('merges independently persisted topic summaries', async () => {
|
|
46
|
-
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
47
|
-
const first = new StatisticsStore(join(root, 'one.json'))
|
|
48
|
-
const second = new StatisticsStore(join(root, 'two.json'))
|
|
49
|
-
await first.record({ toolName: 'verifier_compare', sessionId: 'one', startedAt: 1, finishedAt: 11, success: true, provider: 'p', model: 'm', stats: stats() })
|
|
50
|
-
await second.record({ toolName: 'verifier_track', sessionId: 'two', startedAt: 2, finishedAt: 22, success: false, provider: 'p', model: 'm', stats: stats() })
|
|
51
|
-
const query = { fromMs: 0, toMs: 100, recentLimit: 10 }
|
|
52
|
-
const merged = mergeStatisticsOverviews(await Promise.all([first.overview(query), second.overview(query)]), query)
|
|
53
|
-
expect(merged.totals.invocations).toBe(2)
|
|
54
|
-
expect(merged.totals.successes).toBe(1)
|
|
55
|
-
expect(merged.totals.averageDurationMs).toBe(15)
|
|
56
|
-
expect(merged.tools.map(row => row.toolName)).toEqual(['verifier_compare', 'verifier_track'])
|
|
57
|
-
expect(merged.models).toHaveLength(1)
|
|
58
|
-
expect(merged.recent.map(row => row.sessionId)).toEqual(['two', 'one'])
|
|
59
|
-
})
|
|
60
|
-
|
|
61
|
-
it('bounds persisted error messages', async () => {
|
|
62
|
-
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
63
|
-
const store = new StatisticsStore(join(root, 'statistics.json'))
|
|
64
|
-
await store.record({ toolName: 'verifier_current_session', startedAt: 1, success: false, errorName: 'Failure', errorMessage: 'x'.repeat(1_000), provider: 'p', model: 'm', stats: stats() })
|
|
65
|
-
const result = await store.overview({ fromMs: 0, toMs: 10 })
|
|
66
|
-
expect(result.recent[0]?.errorMessage).toHaveLength(500)
|
|
67
|
-
})
|
|
68
|
-
|
|
69
|
-
it('normalizes legacy and modern sessionPersistence.list output formats', () => {
|
|
70
|
-
const legacyHeaders = [{ id: 'sess-1', cwd: 'C:\\test' }]
|
|
71
|
-
const modernSnapshots = [{ header: { id: 'sess-2', cwd: 'C:\\test' }, revision: 'rev-1' }]
|
|
72
|
-
const normalize = (items: readonly unknown[]) =>
|
|
73
|
-
items
|
|
74
|
-
.map(item => item && typeof item === 'object' && 'header' in item ? (item as { header: { id: string } }).header : item as { id: string })
|
|
75
|
-
.filter(header => header !== undefined)
|
|
76
|
-
|
|
77
|
-
expect(normalize(legacyHeaders).map(h => h.id)).toEqual(['sess-1'])
|
|
78
|
-
expect(normalize(modernSnapshots).map(h => h.id)).toEqual(['sess-2'])
|
|
79
|
-
})
|
|
80
|
-
})
|
|
1
|
+
import { mkdtemp, readFile } from 'node:fs/promises'
|
|
2
|
+
import { join } from 'node:path'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { describe, expect, it } from 'vitest'
|
|
5
|
+
import { StatisticsStore, emptyRunStats, mergeStatisticsOverviews } from './statistics.ts'
|
|
6
|
+
|
|
7
|
+
function stats(overrides: Partial<ReturnType<typeof emptyRunStats>> = {}) {
|
|
8
|
+
return { ...emptyRunStats(), ...overrides }
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
describe('StatisticsStore', () => {
|
|
12
|
+
it('records and aggregates invocations by day, tool, model, and session', async () => {
|
|
13
|
+
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
14
|
+
const file = join(root, 'statistics.json')
|
|
15
|
+
const store = new StatisticsStore(file, 100)
|
|
16
|
+
const first = Date.UTC(2026, 7, 19, 23, 30)
|
|
17
|
+
const second = Date.UTC(2026, 7, 20, 1, 30)
|
|
18
|
+
await store.record({ toolName: 'verifier_compare', sessionId: 'one', startedAt: first, finishedAt: first + 250, success: true, provider: 'p', model: 'm', stats: stats({ calls: 2, attempts: 2, inputTokens: 100, cachedInputTokens: 20, outputTokens: 10, cacheHits: 1, cacheMisses: 1, estimatedCostUsd: 0.12, topLogprobScores: 2 }) })
|
|
19
|
+
await store.record({ toolName: 'verifier_track', sessionId: 'two', startedAt: second, finishedAt: second + 750, success: false, errorName: 'Error', errorMessage: 'failed', provider: 'p', model: 'm2', stats: stats({ calls: 1, attempts: 2, retries: 1, inputTokens: 50, outputTokens: 5, cacheMisses: 1, explicitTagScores: 1 }) })
|
|
20
|
+
|
|
21
|
+
const all = await store.overview({ fromMs: first - 1, toMs: second + 10_000, timezoneOffsetMinutes: -120 })
|
|
22
|
+
expect(all.totals).toMatchObject({ invocations: 2, successes: 1, failures: 1, calls: 3, attempts: 4, retries: 1, tokens: 185, cacheHits: 1, cacheMisses: 2 })
|
|
23
|
+
expect(all.totals.successRate).toBe(0.5)
|
|
24
|
+
expect(all.totals.averageDurationMs).toBe(500)
|
|
25
|
+
expect(all.daily).toHaveLength(1)
|
|
26
|
+
expect(all.daily[0]).toMatchObject({ date: '2026-08-20', invocations: 2, calls: 3, tokens: 185 })
|
|
27
|
+
expect(all.tools.map(row => [row.toolName, row.invocations])).toEqual([['verifier_compare', 1], ['verifier_track', 1]])
|
|
28
|
+
expect(all.models.map(row => row.model)).toEqual(['m', 'm2'])
|
|
29
|
+
|
|
30
|
+
const one = await store.overview({ fromMs: first - 1, toMs: second + 10_000, sessionId: 'one' })
|
|
31
|
+
expect(one.totals.invocations).toBe(1)
|
|
32
|
+
expect(one.recent[0]?.sessionId).toBe('one')
|
|
33
|
+
expect(JSON.parse(await readFile(file, 'utf8')).version).toBe(1)
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
it('serializes concurrent writes and keeps only the configured tail', async () => {
|
|
37
|
+
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
38
|
+
const store = new StatisticsStore(join(root, 'statistics.json'), 2)
|
|
39
|
+
await Promise.all([0, 1, 2].map(index => store.record({ toolName: 'verifier_select', startedAt: 1_000 + index, success: true, provider: 'p', model: 'm', stats: stats() })))
|
|
40
|
+
const result = await store.overview({ fromMs: 0, toMs: 10_000 })
|
|
41
|
+
expect(result.totals.invocations).toBe(2)
|
|
42
|
+
expect(result.recent.map(record => record.startedAt)).toEqual([1_002, 1_001])
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
it('merges independently persisted topic summaries', async () => {
|
|
46
|
+
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
47
|
+
const first = new StatisticsStore(join(root, 'one.json'))
|
|
48
|
+
const second = new StatisticsStore(join(root, 'two.json'))
|
|
49
|
+
await first.record({ toolName: 'verifier_compare', sessionId: 'one', startedAt: 1, finishedAt: 11, success: true, provider: 'p', model: 'm', stats: stats() })
|
|
50
|
+
await second.record({ toolName: 'verifier_track', sessionId: 'two', startedAt: 2, finishedAt: 22, success: false, provider: 'p', model: 'm', stats: stats() })
|
|
51
|
+
const query = { fromMs: 0, toMs: 100, recentLimit: 10 }
|
|
52
|
+
const merged = mergeStatisticsOverviews(await Promise.all([first.overview(query), second.overview(query)]), query)
|
|
53
|
+
expect(merged.totals.invocations).toBe(2)
|
|
54
|
+
expect(merged.totals.successes).toBe(1)
|
|
55
|
+
expect(merged.totals.averageDurationMs).toBe(15)
|
|
56
|
+
expect(merged.tools.map(row => row.toolName)).toEqual(['verifier_compare', 'verifier_track'])
|
|
57
|
+
expect(merged.models).toHaveLength(1)
|
|
58
|
+
expect(merged.recent.map(row => row.sessionId)).toEqual(['two', 'one'])
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('bounds persisted error messages', async () => {
|
|
62
|
+
const root = await mkdtemp(join(tmpdir(), 'dsh-verifier-statistics-'))
|
|
63
|
+
const store = new StatisticsStore(join(root, 'statistics.json'))
|
|
64
|
+
await store.record({ toolName: 'verifier_current_session', startedAt: 1, success: false, errorName: 'Failure', errorMessage: 'x'.repeat(1_000), provider: 'p', model: 'm', stats: stats() })
|
|
65
|
+
const result = await store.overview({ fromMs: 0, toMs: 10 })
|
|
66
|
+
expect(result.recent[0]?.errorMessage).toHaveLength(500)
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
it('normalizes legacy and modern sessionPersistence.list output formats', () => {
|
|
70
|
+
const legacyHeaders = [{ id: 'sess-1', cwd: 'C:\\test' }]
|
|
71
|
+
const modernSnapshots = [{ header: { id: 'sess-2', cwd: 'C:\\test' }, revision: 'rev-1' }]
|
|
72
|
+
const normalize = (items: readonly unknown[]) =>
|
|
73
|
+
items
|
|
74
|
+
.map(item => item && typeof item === 'object' && 'header' in item ? (item as { header: { id: string } }).header : item as { id: string })
|
|
75
|
+
.filter(header => header !== undefined)
|
|
76
|
+
|
|
77
|
+
expect(normalize(legacyHeaders).map(h => h.id)).toEqual(['sess-1'])
|
|
78
|
+
expect(normalize(modernSnapshots).map(h => h.id)).toEqual(['sess-2'])
|
|
79
|
+
})
|
|
80
|
+
})
|