dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
package/lib/random.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Small deterministic helpers shared by scoring and reporting. */
|
|
2
|
+
|
|
3
|
+
/** FNV-1a 32-bit hash; used to derive reproducible bootstrap seeds. */
|
|
4
|
+
export function hashSeed(text) {
|
|
5
|
+
let hash = 2166136261
|
|
6
|
+
const value = typeof text === 'string' ? text : String(text ?? '')
|
|
7
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
8
|
+
hash ^= value.charCodeAt(i)
|
|
9
|
+
hash = Math.imul(hash, 16777619)
|
|
10
|
+
}
|
|
11
|
+
return hash >>> 0
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Stable short hex hash for a diff, so the ledger can identify it without storing it. */
|
|
15
|
+
export function hashHex(text) {
|
|
16
|
+
return hashSeed(text).toString(16).padStart(8, '0')
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Mulberry32 PRNG: deterministic, tiny, good enough for bootstrap resampling. */
|
|
20
|
+
export function mulberry32(seed) {
|
|
21
|
+
let a = (Number.isFinite(seed) ? seed : 1) >>> 0
|
|
22
|
+
return () => {
|
|
23
|
+
a = (a + 0x6d2b79f5) >>> 0
|
|
24
|
+
let t = a
|
|
25
|
+
t = Math.imul(t ^ (t >>> 15), t | 1)
|
|
26
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
|
|
27
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
|
28
|
+
}
|
|
29
|
+
}
|
package/lib/recorder.js
ADDED
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
import { stat as statFile } from 'node:fs/promises'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import { checkReferences } from './integrity.js'
|
|
4
|
+
import { isCorrection, staticProfile } from './score.js'
|
|
5
|
+
|
|
6
|
+
const MAX_TRACKED_SESSIONS = 128
|
|
7
|
+
|
|
8
|
+
/** Best-effort text extraction from a user message of unknown shape. */
|
|
9
|
+
export function messageText(message) {
|
|
10
|
+
if (!message || typeof message !== 'object') return ''
|
|
11
|
+
if (typeof message.content === 'string') return message.content
|
|
12
|
+
if (Array.isArray(message.content)) {
|
|
13
|
+
return message.content
|
|
14
|
+
.map((block) => {
|
|
15
|
+
if (!block || typeof block !== 'object') return ''
|
|
16
|
+
if (typeof block.text === 'string') return block.text
|
|
17
|
+
if (typeof block.value === 'string') return block.value
|
|
18
|
+
return ''
|
|
19
|
+
})
|
|
20
|
+
.filter(Boolean)
|
|
21
|
+
.join('\n')
|
|
22
|
+
}
|
|
23
|
+
if (typeof message.text === 'string') return message.text
|
|
24
|
+
return ''
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function skillNameOf(args) {
|
|
28
|
+
if (!args || typeof args !== 'object') return null
|
|
29
|
+
const candidate = args.name ?? args.skill ?? args.skillName
|
|
30
|
+
return typeof candidate === 'string' && candidate.length > 0 ? candidate : null
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* MCP tools reach the model as `mcp__<server>__<tool>`, so the telemetry can
|
|
35
|
+
* split one skill's window by MCP server without any extra configuration.
|
|
36
|
+
*/
|
|
37
|
+
export function mcpOf(toolName) {
|
|
38
|
+
if (typeof toolName !== 'string' || !toolName.startsWith('mcp__')) return null
|
|
39
|
+
const rest = toolName.slice('mcp__'.length)
|
|
40
|
+
const cut = rest.indexOf('__')
|
|
41
|
+
if (cut <= 0) return { server: rest, tool: '' }
|
|
42
|
+
return { server: rest.slice(0, cut), tool: rest.slice(cut + 2) }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function errorNameOf(result) {
|
|
46
|
+
const info = result?.error?.info
|
|
47
|
+
if (info && typeof info.name === 'string' && info.name.length > 0) return info.name
|
|
48
|
+
return 'ToolError'
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Session origin, so short-lived subagent runs can be judged differently. */
|
|
52
|
+
function originOf(agent) {
|
|
53
|
+
try {
|
|
54
|
+
const origin = agent && agent.session && agent.session.header ? agent.session.header.origin : null
|
|
55
|
+
return typeof origin === 'string' && origin.length > 0 ? origin : null
|
|
56
|
+
} catch {
|
|
57
|
+
return null
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function nowMs() {
|
|
62
|
+
return Date.now()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Live telemetry recorder.
|
|
67
|
+
*
|
|
68
|
+
* A "window" is the stretch of work that follows one `skill` load inside one
|
|
69
|
+
* session. The window stays open across the turn boundary on purpose: the
|
|
70
|
+
* evidence that a skill failed to help is usually the *next* user message ("no,
|
|
71
|
+
* that's wrong"), so the window must survive until that message arrives.
|
|
72
|
+
*
|
|
73
|
+
* Each window is persisted when the turn ends and re-persisted when it finally
|
|
74
|
+
* closes (with the correction attached). Readers keep the last record per
|
|
75
|
+
* windowId, so a re-persist is an update, not a duplicate.
|
|
76
|
+
*/
|
|
77
|
+
export function createRecorder(ctx, config, { store, resolveSkill }) {
|
|
78
|
+
const starts = new Map() // callId -> ms timestamp
|
|
79
|
+
const steps = new Map() // sessionId -> current loop step number
|
|
80
|
+
const turns = new Map() // sessionId -> current turn number
|
|
81
|
+
const stepStats = new Map() // sessionId -> accumulator for the current step
|
|
82
|
+
// A user-typed `/skill-name` is injected as a user message on every following
|
|
83
|
+
// step, so the message id is the dedupe key for that load path.
|
|
84
|
+
const seenInvocations = new Set()
|
|
85
|
+
// A session can have several windows open at once: parallel `skill` loads in
|
|
86
|
+
// one loop step are concurrent use, so their follow-up work counts for each of
|
|
87
|
+
// them instead of being handed to whichever loaded last.
|
|
88
|
+
const sessions = new Map() // sessionId -> { windows: [] }
|
|
89
|
+
|
|
90
|
+
const windowLimit = Math.max(1, Number(config.windowToolCalls ?? 12))
|
|
91
|
+
const ttlMs = Math.max(60_000, Number(config.windowTtlMs ?? 30 * 60_000))
|
|
92
|
+
|
|
93
|
+
function stateFor(sessionId) {
|
|
94
|
+
const existing = sessions.get(sessionId)
|
|
95
|
+
if (existing) {
|
|
96
|
+
existing.lastAt = nowMs()
|
|
97
|
+
return existing
|
|
98
|
+
}
|
|
99
|
+
if (sessions.size >= MAX_TRACKED_SESSIONS) {
|
|
100
|
+
// Evict least-recently-active, not least-recently-created: a root turn
|
|
101
|
+
// that just loaded a skill must not lose its window to a burst of
|
|
102
|
+
// short-lived subagent sessions.
|
|
103
|
+
let oldest = null
|
|
104
|
+
let oldestAt = Infinity
|
|
105
|
+
for (const [id, candidate] of sessions) {
|
|
106
|
+
const at = candidate.lastAt ?? 0
|
|
107
|
+
if (at < oldestAt) {
|
|
108
|
+
oldestAt = at
|
|
109
|
+
oldest = id
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (oldest !== null) {
|
|
113
|
+
const stale = sessions.get(oldest)
|
|
114
|
+
if (stale) closeWindows(stale, 'evicted')
|
|
115
|
+
flushStep(oldest, 'evicted')
|
|
116
|
+
sessions.delete(oldest)
|
|
117
|
+
steps.delete(oldest)
|
|
118
|
+
turns.delete(oldest)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const state = { windows: [], lastAt: nowMs() }
|
|
122
|
+
sessions.set(sessionId, state)
|
|
123
|
+
return state
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function persist(w, closeReason) {
|
|
127
|
+
if (!w) return
|
|
128
|
+
w.flushed = true
|
|
129
|
+
void store.append({
|
|
130
|
+
t: 'window',
|
|
131
|
+
v: 1,
|
|
132
|
+
at: nowMs(),
|
|
133
|
+
windowId: w.windowId,
|
|
134
|
+
sessionId: w.sessionId,
|
|
135
|
+
skill: w.skill,
|
|
136
|
+
toolCalls: w.toolCalls,
|
|
137
|
+
errors: w.errors,
|
|
138
|
+
errorNames: w.errorNames.slice(0, 12),
|
|
139
|
+
loopErrors: w.loopErrors,
|
|
140
|
+
llmRetries: w.llmRetries,
|
|
141
|
+
lastLoopError: w.lastLoopError,
|
|
142
|
+
mcp: w.mcp,
|
|
143
|
+
tokens: w.tokens,
|
|
144
|
+
corrections: w.corrections.map((text) => text.slice(0, 200)),
|
|
145
|
+
otherSkills: [...w.otherSkills],
|
|
146
|
+
origin: w.origin,
|
|
147
|
+
step: w.step,
|
|
148
|
+
turnEnded: w.turnEnded,
|
|
149
|
+
truncated: w.truncated,
|
|
150
|
+
closeReason,
|
|
151
|
+
})
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function closeWindows(state, reason, except) {
|
|
155
|
+
if (!state || state.windows.length === 0) return
|
|
156
|
+
const closing = state.windows.filter((w) => w !== except)
|
|
157
|
+
state.windows = except ? [except] : []
|
|
158
|
+
for (const w of closing) persist(w, reason)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function openWindow(state, skill, sessionId, callId, step, origin) {
|
|
162
|
+
const current = state.windows[state.windows.length - 1]
|
|
163
|
+
if (current) {
|
|
164
|
+
// Two loads inside one loop step run concurrently: that is parallel use,
|
|
165
|
+
// not the model abandoning one skill for another. Only a load in a later
|
|
166
|
+
// step disagrees with what the previous skill was doing.
|
|
167
|
+
const sameStep = step !== undefined && current.step === step
|
|
168
|
+
if (sameStep) {
|
|
169
|
+
// keep the sibling window open; the same follow-up work serves both
|
|
170
|
+
} else {
|
|
171
|
+
for (const w of state.windows) if (w.skill !== skill) w.otherSkills.push(skill)
|
|
172
|
+
closeWindows(state, 'superseded')
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
state.windows.push({
|
|
176
|
+
windowId: `${sessionId}:${callId}`,
|
|
177
|
+
sessionId,
|
|
178
|
+
skill,
|
|
179
|
+
toolCalls: 0,
|
|
180
|
+
errors: 0,
|
|
181
|
+
errorNames: [],
|
|
182
|
+
loopErrors: 0,
|
|
183
|
+
llmRetries: 0,
|
|
184
|
+
lastLoopError: null,
|
|
185
|
+
mcp: Object.create(null),
|
|
186
|
+
tokens: { input: 0, output: 0, cacheRead: 0, calls: 0 },
|
|
187
|
+
corrections: [],
|
|
188
|
+
otherSkills: [],
|
|
189
|
+
turnEnded: false,
|
|
190
|
+
truncated: false,
|
|
191
|
+
step,
|
|
192
|
+
origin,
|
|
193
|
+
openedAt: nowMs(),
|
|
194
|
+
lastActivityAt: nowMs(),
|
|
195
|
+
flushed: false,
|
|
196
|
+
})
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Static authoring quality cannot be resolved at report time: the skill
|
|
201
|
+
* catalog is layered per agent scope, and by then no agent is available. So
|
|
202
|
+
* resolve it once, at load time, while `exec.agent` is still in hand.
|
|
203
|
+
*/
|
|
204
|
+
async function captureStatic(skill, agent, signal, fallbackContent) {
|
|
205
|
+
try {
|
|
206
|
+
if (typeof resolveSkill !== 'function') return
|
|
207
|
+
let resolved = null
|
|
208
|
+
try {
|
|
209
|
+
resolved = await resolveSkill(skill, agent, signal)
|
|
210
|
+
} catch {
|
|
211
|
+
resolved = null
|
|
212
|
+
}
|
|
213
|
+
const value = typeof fallbackContent === 'string' ? { content: fallbackContent } : null
|
|
214
|
+
const definition =
|
|
215
|
+
resolved?.definition ?? (value && typeof value.content === 'string' ? { content: value.content } : undefined)
|
|
216
|
+
const summary =
|
|
217
|
+
resolved?.summary ??
|
|
218
|
+
(value && typeof value.content === 'string'
|
|
219
|
+
? { name: skill, description: '', whenToUse: '', source: value.provider ?? null }
|
|
220
|
+
: undefined)
|
|
221
|
+
// Reference integrity needs the filesystem view, so it is resolved here,
|
|
222
|
+
// while the skill directory is still known, and handed to the pure profile.
|
|
223
|
+
let integrity = null
|
|
224
|
+
try {
|
|
225
|
+
const baseDir =
|
|
226
|
+
(summary && summary.resourceBase && summary.resourceBase.kind === 'directory' && summary.resourceBase.path) ||
|
|
227
|
+
(summary && typeof summary.path === 'string' ? path.dirname(summary.path) : null)
|
|
228
|
+
if (baseDir && definition) {
|
|
229
|
+
integrity = await checkReferences({ baseDir, content: definition.content, stat: statFile })
|
|
230
|
+
}
|
|
231
|
+
} catch {
|
|
232
|
+
integrity = null
|
|
233
|
+
}
|
|
234
|
+
const profile = staticProfile(summary, definition, { integrity })
|
|
235
|
+
if (!profile) return
|
|
236
|
+
await store.append({
|
|
237
|
+
t: 'static',
|
|
238
|
+
v: 1,
|
|
239
|
+
at: nowMs(),
|
|
240
|
+
skill,
|
|
241
|
+
source: profile.source,
|
|
242
|
+
static: profile,
|
|
243
|
+
})
|
|
244
|
+
} catch {
|
|
245
|
+
// static capture is optional
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function flushStep(sessionId, closeReason = 'next-step') {
|
|
250
|
+
const step = stepStats.get(sessionId)
|
|
251
|
+
if (!step) return
|
|
252
|
+
stepStats.delete(sessionId)
|
|
253
|
+
const active =
|
|
254
|
+
step.toolCalls > 0 || step.errors > 0 || step.loopErrors > 0 || step.skills.length > 0 || step.tokens.calls > 0
|
|
255
|
+
if (!active) return
|
|
256
|
+
void store.append({
|
|
257
|
+
t: 'step',
|
|
258
|
+
v: 1,
|
|
259
|
+
at: nowMs(),
|
|
260
|
+
sessionId,
|
|
261
|
+
turn: step.turn,
|
|
262
|
+
step: step.step,
|
|
263
|
+
startedAt: step.startedAt,
|
|
264
|
+
durationMs: Math.max(0, nowMs() - step.startedAt),
|
|
265
|
+
toolCalls: step.toolCalls,
|
|
266
|
+
errors: step.errors,
|
|
267
|
+
errorNames: step.errorNames.slice(0, 8),
|
|
268
|
+
mcpCalls: step.mcpCalls,
|
|
269
|
+
mcpErrors: step.mcpErrors,
|
|
270
|
+
loopErrors: step.loopErrors,
|
|
271
|
+
skills: [...step.skills],
|
|
272
|
+
tokens: step.tokens,
|
|
273
|
+
closeReason,
|
|
274
|
+
})
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function beginStep(sessionId, turn, stepNumber) {
|
|
278
|
+
flushStep(sessionId, 'next-step')
|
|
279
|
+
stepStats.set(sessionId, {
|
|
280
|
+
startedAt: nowMs(),
|
|
281
|
+
sessionId,
|
|
282
|
+
turn: Number.isFinite(Number(turn)) ? Number(turn) : null,
|
|
283
|
+
step: Number.isFinite(Number(stepNumber)) ? Number(stepNumber) : null,
|
|
284
|
+
toolCalls: 0,
|
|
285
|
+
errors: 0,
|
|
286
|
+
errorNames: [],
|
|
287
|
+
mcpCalls: 0,
|
|
288
|
+
mcpErrors: 0,
|
|
289
|
+
loopErrors: 0,
|
|
290
|
+
skills: [],
|
|
291
|
+
tokens: { input: 0, output: 0, cacheRead: 0, calls: 0 },
|
|
292
|
+
})
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const offs = []
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* The `skill` tool is not the only way a skill gets loaded: a user-typed
|
|
299
|
+
* `/skill-name` is resolved by the harness and injected as a user message with
|
|
300
|
+
* `source: { kind: 'skill-invocation', name }` — no tool call, so it would
|
|
301
|
+
* otherwise be invisible. The message repeats on later steps, hence the id
|
|
302
|
+
* dedupe.
|
|
303
|
+
*/
|
|
304
|
+
function recordInvocations(sessionId, messages, agent) {
|
|
305
|
+
if (!Array.isArray(messages)) return
|
|
306
|
+
for (const message of messages) {
|
|
307
|
+
if (!message || typeof message !== 'object') continue
|
|
308
|
+
const source = message.source
|
|
309
|
+
if (!source || source.kind !== 'skill-invocation') continue
|
|
310
|
+
const skill = typeof source.name === 'string' && source.name.length > 0 ? source.name : null
|
|
311
|
+
if (!skill) continue
|
|
312
|
+
const id = typeof message.id === 'string' ? message.id : `${skill}:${sessionId}`
|
|
313
|
+
const seenKey = `${sessionId}|${id}`
|
|
314
|
+
if (seenInvocations.has(seenKey)) continue
|
|
315
|
+
seenInvocations.add(seenKey)
|
|
316
|
+
if (seenInvocations.size > 4000) seenInvocations.clear()
|
|
317
|
+
const state = stateFor(sessionId)
|
|
318
|
+
const windowId = `${sessionId}:${id}`
|
|
319
|
+
void store.append({
|
|
320
|
+
t: 'skill',
|
|
321
|
+
v: 1,
|
|
322
|
+
at: nowMs(),
|
|
323
|
+
sessionId,
|
|
324
|
+
callId: id,
|
|
325
|
+
skill,
|
|
326
|
+
ok: true,
|
|
327
|
+
via: 'invocation',
|
|
328
|
+
origin: originOf(agent),
|
|
329
|
+
turn: turns.get(sessionId),
|
|
330
|
+
windowId,
|
|
331
|
+
})
|
|
332
|
+
openWindow(state, skill, sessionId, id, steps.get(sessionId), originOf(agent))
|
|
333
|
+
void captureStatic(skill, agent, undefined, undefined)
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// The loop step is the only way to tell "loaded A and B together" (parallel
|
|
338
|
+
// work in one step) from "loaded A, then switched to B". Pass the waterfall
|
|
339
|
+
// through untouched.
|
|
340
|
+
offs.push(
|
|
341
|
+
ctx.on('agent/pre-step', (payload, next) => {
|
|
342
|
+
try {
|
|
343
|
+
if (payload && payload.agent) {
|
|
344
|
+
const sessionId = String(payload.agent.id)
|
|
345
|
+
steps.set(sessionId, payload.step)
|
|
346
|
+
if (typeof payload.turn === 'number') turns.set(sessionId, payload.turn)
|
|
347
|
+
beginStep(sessionId, payload.turn ?? turns.get(sessionId), payload.step)
|
|
348
|
+
recordInvocations(sessionId, payload.messages, payload.agent)
|
|
349
|
+
}
|
|
350
|
+
} catch {
|
|
351
|
+
// never interfere with the loop
|
|
352
|
+
}
|
|
353
|
+
return next()
|
|
354
|
+
}),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
// The loop can fail on its own (a broken step, a failed model request, an
|
|
358
|
+
// aborted tool batch). That is evidence about the skill that was loaded right
|
|
359
|
+
// before it, so it lands inside the open windows instead of being lost.
|
|
360
|
+
offs.push(
|
|
361
|
+
ctx.on('agent/error', (payload) => {
|
|
362
|
+
try {
|
|
363
|
+
const sessionId = payload && payload.agent ? String(payload.agent.id) : null
|
|
364
|
+
if (!sessionId) return
|
|
365
|
+
const state = sessions.get(sessionId)
|
|
366
|
+
const step = stepStats.get(sessionId)
|
|
367
|
+
if (step) step.loopErrors += 1
|
|
368
|
+
if (!state || state.windows.length === 0) return
|
|
369
|
+
const error = payload.error
|
|
370
|
+
const label =
|
|
371
|
+
error instanceof Error
|
|
372
|
+
? error.name || 'Error'
|
|
373
|
+
: error && typeof error === 'object' && typeof error.code === 'string'
|
|
374
|
+
? error.code
|
|
375
|
+
: 'AgentError'
|
|
376
|
+
for (const w of state.windows) {
|
|
377
|
+
w.loopErrors += 1
|
|
378
|
+
w.lastLoopError = label
|
|
379
|
+
}
|
|
380
|
+
} catch {
|
|
381
|
+
// ignore
|
|
382
|
+
}
|
|
383
|
+
}),
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
offs.push(
|
|
387
|
+
ctx.on('agent/request-error', (payload, next) => {
|
|
388
|
+
try {
|
|
389
|
+
const sessionId = payload && payload.agent ? String(payload.agent.id) : null
|
|
390
|
+
if (sessionId) {
|
|
391
|
+
const state = sessions.get(sessionId)
|
|
392
|
+
if (state && state.windows.length > 0) {
|
|
393
|
+
for (const w of state.windows) w.llmRetries += 1
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
} catch {
|
|
397
|
+
// ignore
|
|
398
|
+
}
|
|
399
|
+
return next()
|
|
400
|
+
}),
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
// Start timing on skill loads only; pass the waterfall through untouched.
|
|
404
|
+
offs.push(
|
|
405
|
+
ctx.on('tools/pre-execute', (exec, next) => {
|
|
406
|
+
try {
|
|
407
|
+
if (exec && exec.callId !== undefined) {
|
|
408
|
+
starts.set(exec.callId, nowMs())
|
|
409
|
+
if (starts.size > 512) starts.delete(starts.keys().next().value)
|
|
410
|
+
}
|
|
411
|
+
} catch {
|
|
412
|
+
// never interfere with execution
|
|
413
|
+
}
|
|
414
|
+
return next()
|
|
415
|
+
}),
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
offs.push(
|
|
419
|
+
ctx.on('tools/result', (exec, result) => {
|
|
420
|
+
try {
|
|
421
|
+
if (!exec || !exec.agent) return
|
|
422
|
+
const sessionId = String(exec.agent.id)
|
|
423
|
+
const state = stateFor(sessionId)
|
|
424
|
+
const step = stepStats.get(sessionId)
|
|
425
|
+
const mcpCall = mcpOf(exec.name)
|
|
426
|
+
if (step) {
|
|
427
|
+
step.toolCalls += 1
|
|
428
|
+
if (result?.isError === true) {
|
|
429
|
+
step.errors += 1
|
|
430
|
+
if (step.errorNames.length < 8) step.errorNames.push(errorNameOf(result))
|
|
431
|
+
}
|
|
432
|
+
if (mcpCall) {
|
|
433
|
+
step.mcpCalls += 1
|
|
434
|
+
if (result?.isError === true) step.mcpErrors += 1
|
|
435
|
+
}
|
|
436
|
+
if (exec.name === 'skill') {
|
|
437
|
+
const loaded = skillNameOf(exec.arguments)
|
|
438
|
+
if (loaded && !step.skills.includes(loaded)) step.skills.push(loaded)
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
const newest = state.windows[state.windows.length - 1]
|
|
442
|
+
if (newest && nowMs() - newest.lastActivityAt > ttlMs) closeWindows(state, 'stale')
|
|
443
|
+
|
|
444
|
+
if (exec.name === 'skill') {
|
|
445
|
+
const skill = skillNameOf(exec.arguments)
|
|
446
|
+
const started = starts.get(exec.callId)
|
|
447
|
+
starts.delete(exec.callId)
|
|
448
|
+
if (!skill) return
|
|
449
|
+
const ok = result?.isError !== true
|
|
450
|
+
const windowId = `${sessionId}:${exec.callId}`
|
|
451
|
+
void store.append({
|
|
452
|
+
t: 'skill',
|
|
453
|
+
v: 1,
|
|
454
|
+
at: nowMs(),
|
|
455
|
+
sessionId,
|
|
456
|
+
callId: exec.callId,
|
|
457
|
+
skill,
|
|
458
|
+
ok,
|
|
459
|
+
loadMs: started === undefined ? undefined : nowMs() - started,
|
|
460
|
+
error: ok ? undefined : String(result?.error?.message ?? 'load failed').slice(0, 300),
|
|
461
|
+
origin: originOf(exec.agent),
|
|
462
|
+
turn: turns.get(sessionId),
|
|
463
|
+
windowId,
|
|
464
|
+
})
|
|
465
|
+
openWindow(state, skill, sessionId, exec.callId, steps.get(sessionId), originOf(exec.agent))
|
|
466
|
+
if (ok)
|
|
467
|
+
void captureStatic(
|
|
468
|
+
skill,
|
|
469
|
+
exec.agent,
|
|
470
|
+
exec.signal,
|
|
471
|
+
result?.value && typeof result.value.content === 'string' ? result.value.content : undefined,
|
|
472
|
+
)
|
|
473
|
+
return
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (state.windows.length === 0) return
|
|
477
|
+
const current = steps.get(sessionId)
|
|
478
|
+
// Count the call for every window still open in this step: a parallel
|
|
479
|
+
// batch of skill loads is one piece of work, not one winner.
|
|
480
|
+
const targets = state.windows.filter((w) => current === undefined || w.step === undefined || w.step === current)
|
|
481
|
+
const startedAt = starts.get(exec.callId)
|
|
482
|
+
if (exec.callId !== undefined) starts.delete(exec.callId)
|
|
483
|
+
const elapsed = startedAt === undefined ? undefined : nowMs() - startedAt
|
|
484
|
+
const mcp = mcpOf(exec.name)
|
|
485
|
+
for (const w of targets.length > 0 ? targets : state.windows) {
|
|
486
|
+
w.lastActivityAt = nowMs()
|
|
487
|
+
if (state) state.lastAt = nowMs()
|
|
488
|
+
if (mcp) {
|
|
489
|
+
if (!Object.hasOwn(w.mcp, mcp.server)) {
|
|
490
|
+
w.mcp[mcp.server] = { calls: 0, errors: 0, ms: 0, samples: [], tools: Object.create(null) }
|
|
491
|
+
}
|
|
492
|
+
const bucket = w.mcp[mcp.server]
|
|
493
|
+
bucket.calls += 1
|
|
494
|
+
if (result?.isError === true) bucket.errors += 1
|
|
495
|
+
if (elapsed !== undefined) {
|
|
496
|
+
bucket.ms += elapsed
|
|
497
|
+
// keep a bounded latency sample so the report can show p50/p95,
|
|
498
|
+
// not just a mean that one slow call can distort
|
|
499
|
+
if (!Array.isArray(bucket.samples)) bucket.samples = []
|
|
500
|
+
if (bucket.samples.length < 40) bucket.samples.push(elapsed)
|
|
501
|
+
}
|
|
502
|
+
if (mcp.tool) {
|
|
503
|
+
const tool = Object.hasOwn(bucket.tools, mcp.tool)
|
|
504
|
+
? bucket.tools[mcp.tool]
|
|
505
|
+
: { calls: 0, errors: 0, ms: 0, samples: [] }
|
|
506
|
+
tool.calls += 1
|
|
507
|
+
if (result?.isError === true) tool.errors += 1
|
|
508
|
+
if (elapsed !== undefined) {
|
|
509
|
+
tool.ms = (tool.ms ?? 0) + elapsed
|
|
510
|
+
if (!Array.isArray(tool.samples)) tool.samples = []
|
|
511
|
+
if (tool.samples.length < 20) tool.samples.push(elapsed)
|
|
512
|
+
}
|
|
513
|
+
bucket.tools[mcp.tool] = tool
|
|
514
|
+
}
|
|
515
|
+
w.mcp[mcp.server] = bucket
|
|
516
|
+
}
|
|
517
|
+
if (w.toolCalls >= windowLimit) {
|
|
518
|
+
w.truncated = true
|
|
519
|
+
continue
|
|
520
|
+
}
|
|
521
|
+
w.toolCalls += 1
|
|
522
|
+
if (result?.isError === true) {
|
|
523
|
+
w.errors += 1
|
|
524
|
+
if (w.errorNames.length < 12) w.errorNames.push(errorNameOf(result))
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
} catch {
|
|
528
|
+
// telemetry must never break tool execution
|
|
529
|
+
}
|
|
530
|
+
}),
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
// The next user message is the strongest signal about the last run: either it
|
|
534
|
+
// is a correction (the skill failed) or it is a normal follow-up (it worked).
|
|
535
|
+
offs.push(
|
|
536
|
+
ctx.on('agent/inbox/claimed', (payload) => {
|
|
537
|
+
try {
|
|
538
|
+
const sessionId = payload?.agent ? String(payload.agent.id) : null
|
|
539
|
+
if (!sessionId) return
|
|
540
|
+
const state = sessions.get(sessionId)
|
|
541
|
+
if (!state || state.windows.length === 0) return
|
|
542
|
+
const newest = state.windows[state.windows.length - 1]
|
|
543
|
+
if (nowMs() - newest.lastActivityAt > ttlMs) {
|
|
544
|
+
closeWindows(state, 'stale')
|
|
545
|
+
return
|
|
546
|
+
}
|
|
547
|
+
const text = messageText(payload.message)
|
|
548
|
+
if (text.length > 0 && isCorrection(text)) {
|
|
549
|
+
for (const w of state.windows) w.corrections.push(text)
|
|
550
|
+
}
|
|
551
|
+
closeWindows(state, 'user-turn')
|
|
552
|
+
} catch {
|
|
553
|
+
// ignore
|
|
554
|
+
}
|
|
555
|
+
}),
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
// Real provider usage rides on every committed assistant message, so token
|
|
559
|
+
// cost is measured instead of estimated. Usage belongs to the model call that
|
|
560
|
+
// answered the current step, which is exactly the work an open window caused.
|
|
561
|
+
offs.push(
|
|
562
|
+
ctx.on('session/event', (session, event) => {
|
|
563
|
+
try {
|
|
564
|
+
if (config.countTokens === false || !event || event.type !== 'assistant/message') return
|
|
565
|
+
const usage = event.data && event.data.usage
|
|
566
|
+
if (!usage || typeof usage !== 'object') return
|
|
567
|
+
const sessionId = session && session.id ? String(session.id) : null
|
|
568
|
+
if (!sessionId) return
|
|
569
|
+
const state = sessions.get(sessionId)
|
|
570
|
+
const step = stepStats.get(sessionId)
|
|
571
|
+
if (step) {
|
|
572
|
+
step.tokens.input += Number(usage.inputTokens) || 0
|
|
573
|
+
step.tokens.output += Number(usage.outputTokens) || 0
|
|
574
|
+
step.tokens.cacheRead += Number(usage.cacheReadTokens) || 0
|
|
575
|
+
step.tokens.calls += 1
|
|
576
|
+
}
|
|
577
|
+
if (!state || state.windows.length === 0) return
|
|
578
|
+
for (const w of state.windows) {
|
|
579
|
+
const bucket = w.tokens ?? { input: 0, output: 0, cacheRead: 0, calls: 0 }
|
|
580
|
+
bucket.input += Number(usage.inputTokens) || 0
|
|
581
|
+
bucket.output += Number(usage.outputTokens) || 0
|
|
582
|
+
bucket.cacheRead += Number(usage.cacheReadTokens) || 0
|
|
583
|
+
bucket.calls += 1
|
|
584
|
+
w.tokens = bucket
|
|
585
|
+
}
|
|
586
|
+
} catch {
|
|
587
|
+
// token accounting is best effort
|
|
588
|
+
}
|
|
589
|
+
}),
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
// Persist at turn end, but keep observing for the follow-up message.
|
|
593
|
+
offs.push(
|
|
594
|
+
ctx.on('agent/turn-stopping', (payload) => {
|
|
595
|
+
try {
|
|
596
|
+
const sessionId = payload?.agent ? String(payload.agent.id) : null
|
|
597
|
+
if (!sessionId) return
|
|
598
|
+
const state = sessions.get(sessionId)
|
|
599
|
+
if (!state || state.windows.length === 0) return
|
|
600
|
+
for (const w of state.windows) {
|
|
601
|
+
w.turnEnded = true
|
|
602
|
+
persist(w, 'turn-end')
|
|
603
|
+
}
|
|
604
|
+
flushStep(sessionId, 'turn-end')
|
|
605
|
+
} catch {
|
|
606
|
+
// ignore
|
|
607
|
+
}
|
|
608
|
+
}),
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
return {
|
|
612
|
+
/** Await every write queued so far (used by tests and by the snapshot flush). */
|
|
613
|
+
flush() {
|
|
614
|
+
return store.queue
|
|
615
|
+
},
|
|
616
|
+
async dispose() {
|
|
617
|
+
for (const sessionId of [...stepStats.keys()]) flushStep(sessionId, 'dispose')
|
|
618
|
+
for (const state of sessions.values()) closeWindows(state, 'dispose')
|
|
619
|
+
sessions.clear()
|
|
620
|
+
steps.clear()
|
|
621
|
+
turns.clear()
|
|
622
|
+
seenInvocations.clear()
|
|
623
|
+
// let every queued write land before the caller tears the store down
|
|
624
|
+
await store.queue
|
|
625
|
+
for (const off of offs) {
|
|
626
|
+
try {
|
|
627
|
+
off()
|
|
628
|
+
} catch {
|
|
629
|
+
// ignore
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
},
|
|
633
|
+
}
|
|
634
|
+
}
|