dsh-skill-eval 0.0.0-stage → 1.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +718 -0
  2. package/LICENSE +21 -0
  3. package/README.md +207 -2
  4. package/README.zh.md +164 -0
  5. package/client.js +2791 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +82 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +468 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1173 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +557 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +141 -0
  26. package/lib/html.js +17 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +644 -0
  32. package/lib/render.js +580 -0
  33. package/lib/report.js +689 -0
  34. package/lib/score.js +565 -0
  35. package/lib/settings.js +415 -0
  36. package/lib/snapshot.js +250 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +240 -0
  39. package/lib/validity.js +125 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +227 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +234 -0
  47. package/scripts/lib.mjs +87 -0
  48. package/scripts/verify-live.mjs +178 -0
package/lib/random.js ADDED
@@ -0,0 +1,29 @@
1
+ /** Small deterministic helpers shared by scoring and reporting. */
2
+
3
+ /** FNV-1a 32-bit hash; used to derive reproducible bootstrap seeds. */
4
+ export function hashSeed(text) {
5
+ let hash = 2166136261
6
+ const value = typeof text === 'string' ? text : String(text ?? '')
7
+ for (let i = 0; i < value.length; i += 1) {
8
+ hash ^= value.charCodeAt(i)
9
+ hash = Math.imul(hash, 16777619)
10
+ }
11
+ return hash >>> 0
12
+ }
13
+
14
+ /** Stable short hex hash for a diff, so the ledger can identify it without storing it. */
15
+ export function hashHex(text) {
16
+ return hashSeed(text).toString(16).padStart(8, '0')
17
+ }
18
+
19
+ /** Mulberry32 PRNG: deterministic, tiny, good enough for bootstrap resampling. */
20
+ export function mulberry32(seed) {
21
+ let a = (Number.isFinite(seed) ? seed : 1) >>> 0
22
+ return () => {
23
+ a = (a + 0x6d2b79f5) >>> 0
24
+ let t = a
25
+ t = Math.imul(t ^ (t >>> 15), t | 1)
26
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
27
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296
28
+ }
29
+ }
@@ -0,0 +1,644 @@
1
+ import { stat as statFile } from 'node:fs/promises'
2
+ import path from 'node:path'
3
+ import { checkReferences } from './integrity.js'
4
+ import { isCorrection, staticProfile } from './score.js'
5
+
6
+ const MAX_TRACKED_SESSIONS = 128
7
+
8
+ /** Best-effort text extraction from a user message of unknown shape. */
9
+ export function messageText(message) {
10
+ if (!message || typeof message !== 'object') return ''
11
+ if (typeof message.content === 'string') return message.content
12
+ if (Array.isArray(message.content)) {
13
+ return message.content
14
+ .map((block) => {
15
+ if (!block || typeof block !== 'object') return ''
16
+ if (typeof block.text === 'string') return block.text
17
+ if (typeof block.value === 'string') return block.value
18
+ return ''
19
+ })
20
+ .filter(Boolean)
21
+ .join('\n')
22
+ }
23
+ if (typeof message.text === 'string') return message.text
24
+ return ''
25
+ }
26
+
27
+ function skillNameOf(args) {
28
+ if (!args || typeof args !== 'object') return null
29
+ const candidate = args.name ?? args.skill ?? args.skillName
30
+ return typeof candidate === 'string' && candidate.length > 0 ? candidate : null
31
+ }
32
+
33
+ /**
34
+ * MCP tools reach the model as `mcp__<server>__<tool>`, so the telemetry can
35
+ * split one skill's window by MCP server without any extra configuration.
36
+ */
37
+ export function mcpOf(toolName) {
38
+ if (typeof toolName !== 'string' || !toolName.startsWith('mcp__')) return null
39
+ const rest = toolName.slice('mcp__'.length)
40
+ const cut = rest.indexOf('__')
41
+ if (cut <= 0) return { server: rest, tool: '' }
42
+ return { server: rest.slice(0, cut), tool: rest.slice(cut + 2) }
43
+ }
44
+
45
+ function errorNameOf(result) {
46
+ const info = result?.error?.info
47
+ if (info && typeof info.name === 'string' && info.name.length > 0) return info.name
48
+ return 'ToolError'
49
+ }
50
+
51
+ /** Session origin, so short-lived subagent runs can be judged differently. */
52
+ function originOf(agent) {
53
+ try {
54
+ const origin = agent && agent.session && agent.session.header ? agent.session.header.origin : null
55
+ return typeof origin === 'string' && origin.length > 0 ? origin : null
56
+ } catch {
57
+ return null
58
+ }
59
+ }
60
+
61
+ function nowMs() {
62
+ return Date.now()
63
+ }
64
+
65
+ /**
66
+ * Live telemetry recorder.
67
+ *
68
+ * A "window" is the stretch of work that follows one `skill` load inside one
69
+ * session. The window stays open across the turn boundary on purpose: the
70
+ * evidence that a skill failed to help is usually the *next* user message ("no,
71
+ * that's wrong"), so the window must survive until that message arrives.
72
+ *
73
+ * Each window is persisted when the turn ends and re-persisted when it finally
74
+ * closes (with the correction attached). Readers keep the last record per
75
+ * windowId, so a re-persist is an update, not a duplicate.
76
+ */
77
+ export function createRecorder(ctx, config, { store, resolveSkill }) {
78
+ const starts = new Map() // callId -> ms timestamp
79
+ const steps = new Map() // sessionId -> current loop step number
80
+ const turns = new Map() // sessionId -> current turn number
81
+ const stepStats = new Map() // sessionId -> accumulator for the current step
82
+ // A user-typed `/skill-name` is injected as a user message on every following
83
+ // step, so the message id is the dedupe key for that load path.
84
+ const seenInvocations = new Set()
85
+ // A session can have several windows open at once: parallel `skill` loads in
86
+ // one loop step are concurrent use, so their follow-up work counts for each of
87
+ // them instead of being handed to whichever loaded last.
88
+ const sessions = new Map() // sessionId -> { windows: [] }
89
+
90
+ const windowLimit = Math.max(1, Number(config.windowToolCalls ?? 12))
91
+ // Read live: the settings form advertises windowTtlMs as applying immediately.
92
+ const ttlMs = () => Math.max(60_000, Number(config.windowTtlMs ?? 30 * 60_000))
93
+
94
+ function stateFor(sessionId) {
95
+ const existing = sessions.get(sessionId)
96
+ if (existing) {
97
+ existing.lastAt = nowMs()
98
+ return existing
99
+ }
100
+ if (sessions.size >= MAX_TRACKED_SESSIONS) {
101
+ // Evict least-recently-active, not least-recently-created: a root turn
102
+ // that just loaded a skill must not lose its window to a burst of
103
+ // short-lived subagent sessions.
104
+ let oldest = null
105
+ let oldestAt = Infinity
106
+ for (const [id, candidate] of sessions) {
107
+ const at = candidate.lastAt ?? 0
108
+ if (at < oldestAt) {
109
+ oldestAt = at
110
+ oldest = id
111
+ }
112
+ }
113
+ if (oldest !== null) {
114
+ const stale = sessions.get(oldest)
115
+ if (stale) closeWindows(stale, 'evicted')
116
+ flushStep(oldest, 'evicted')
117
+ sessions.delete(oldest)
118
+ steps.delete(oldest)
119
+ turns.delete(oldest)
120
+ }
121
+ }
122
+ const state = { windows: [], lastAt: nowMs() }
123
+ sessions.set(sessionId, state)
124
+ return state
125
+ }
126
+
127
+ function persist(w, closeReason) {
128
+ if (!w) return
129
+ w.flushed = true
130
+ void store.append({
131
+ t: 'window',
132
+ v: 1,
133
+ at: nowMs(),
134
+ windowId: w.windowId,
135
+ sessionId: w.sessionId,
136
+ skill: w.skill,
137
+ toolCalls: w.toolCalls,
138
+ errors: w.errors,
139
+ errorNames: w.errorNames.slice(0, 12),
140
+ loopErrors: w.loopErrors,
141
+ llmRetries: w.llmRetries,
142
+ lastLoopError: w.lastLoopError,
143
+ mcp: w.mcp,
144
+ tokens: w.tokens,
145
+ corrections: w.corrections.map((text) => text.slice(0, 200)),
146
+ otherSkills: [...w.otherSkills],
147
+ origin: w.origin,
148
+ step: w.step,
149
+ turnEnded: w.turnEnded,
150
+ truncated: w.truncated,
151
+ closeReason,
152
+ })
153
+ }
154
+
155
+ function closeWindows(state, reason, except) {
156
+ if (!state || state.windows.length === 0) return
157
+ const closing = state.windows.filter((w) => w !== except)
158
+ state.windows = except ? [except] : []
159
+ for (const w of closing) persist(w, reason)
160
+ }
161
+
162
+ function openWindow(state, skill, sessionId, callId, step, origin) {
163
+ const current = state.windows[state.windows.length - 1]
164
+ if (current) {
165
+ // Two loads inside one loop step run concurrently: that is parallel use,
166
+ // not the model abandoning one skill for another. Only a load in a later
167
+ // step disagrees with what the previous skill was doing.
168
+ const sameStep = step !== undefined && current.step === step
169
+ if (sameStep) {
170
+ // keep the sibling window open; the same follow-up work serves both
171
+ } else {
172
+ for (const w of state.windows) if (w.skill !== skill) w.otherSkills.push(skill)
173
+ closeWindows(state, 'superseded')
174
+ }
175
+ }
176
+ state.windows.push({
177
+ windowId: `${sessionId}:${callId}`,
178
+ sessionId,
179
+ skill,
180
+ toolCalls: 0,
181
+ errors: 0,
182
+ errorNames: [],
183
+ loopErrors: 0,
184
+ llmRetries: 0,
185
+ lastLoopError: null,
186
+ mcp: Object.create(null),
187
+ tokens: { input: 0, output: 0, cacheRead: 0, calls: 0 },
188
+ corrections: [],
189
+ otherSkills: [],
190
+ turnEnded: false,
191
+ truncated: false,
192
+ step,
193
+ origin,
194
+ openedAt: nowMs(),
195
+ lastActivityAt: nowMs(),
196
+ flushed: false,
197
+ })
198
+ }
199
+
200
+ /**
201
+ * Static authoring quality cannot be resolved at report time: the skill
202
+ * catalog is layered per agent scope, and by then no agent is available. So
203
+ * resolve it once, at load time, while `exec.agent` is still in hand.
204
+ */
205
+ async function captureStatic(skill, agent, signal, fallbackContent) {
206
+ try {
207
+ if (typeof resolveSkill !== 'function') return
208
+ let resolved = null
209
+ try {
210
+ resolved = await resolveSkill(skill, agent, signal)
211
+ } catch {
212
+ resolved = null
213
+ }
214
+ const value = typeof fallbackContent === 'string' ? { content: fallbackContent } : null
215
+ const definition =
216
+ resolved?.definition ?? (value && typeof value.content === 'string' ? { content: value.content } : undefined)
217
+ const summary =
218
+ resolved?.summary ??
219
+ (value && typeof value.content === 'string'
220
+ ? { name: skill, description: '', whenToUse: '', source: value.provider ?? null }
221
+ : undefined)
222
+ // Reference integrity needs the filesystem view, so it is resolved here,
223
+ // while the skill directory is still known, and handed to the pure profile.
224
+ let integrity = null
225
+ try {
226
+ const baseDir =
227
+ (summary && summary.resourceBase && summary.resourceBase.kind === 'directory' && summary.resourceBase.path) ||
228
+ (summary && typeof summary.path === 'string' ? path.dirname(summary.path) : null)
229
+ if (baseDir && definition) {
230
+ integrity = await checkReferences({ baseDir, content: definition.content, stat: statFile })
231
+ }
232
+ } catch {
233
+ integrity = null
234
+ }
235
+ const profile = staticProfile(summary, definition, { integrity })
236
+ if (!profile) return
237
+ await store.append({
238
+ t: 'static',
239
+ v: 1,
240
+ at: nowMs(),
241
+ skill,
242
+ source: profile.source,
243
+ static: profile,
244
+ })
245
+ } catch {
246
+ // static capture is optional
247
+ }
248
+ }
249
+
250
+ function flushStep(sessionId, closeReason = 'next-step') {
251
+ const step = stepStats.get(sessionId)
252
+ if (!step) return
253
+ stepStats.delete(sessionId)
254
+ const active =
255
+ step.toolCalls > 0 || step.errors > 0 || step.loopErrors > 0 || step.skills.length > 0 || step.tokens.calls > 0
256
+ if (!active) return
257
+ void store.append({
258
+ t: 'step',
259
+ v: 1,
260
+ at: nowMs(),
261
+ sessionId,
262
+ turn: step.turn,
263
+ step: step.step,
264
+ startedAt: step.startedAt,
265
+ durationMs: Math.max(0, nowMs() - step.startedAt),
266
+ toolCalls: step.toolCalls,
267
+ errors: step.errors,
268
+ errorNames: step.errorNames.slice(0, 8),
269
+ mcpCalls: step.mcpCalls,
270
+ mcpErrors: step.mcpErrors,
271
+ loopErrors: step.loopErrors,
272
+ skills: [...step.skills],
273
+ tokens: step.tokens,
274
+ closeReason,
275
+ })
276
+ }
277
+
278
+ function beginStep(sessionId, turn, stepNumber) {
279
+ flushStep(sessionId, 'next-step')
280
+ stepStats.set(sessionId, {
281
+ startedAt: nowMs(),
282
+ sessionId,
283
+ turn: Number.isFinite(Number(turn)) ? Number(turn) : null,
284
+ step: Number.isFinite(Number(stepNumber)) ? Number(stepNumber) : null,
285
+ toolCalls: 0,
286
+ errors: 0,
287
+ errorNames: [],
288
+ mcpCalls: 0,
289
+ mcpErrors: 0,
290
+ loopErrors: 0,
291
+ skills: [],
292
+ tokens: { input: 0, output: 0, cacheRead: 0, calls: 0 },
293
+ })
294
+ }
295
+
296
+ const offs = []
297
+
298
+ /**
299
+ * The `skill` tool is not the only way a skill gets loaded: a user-typed
300
+ * `/skill-name` is resolved by the harness and injected as a user message with
301
+ * `source: { kind: 'skill-invocation', name }` — no tool call, so it would
302
+ * otherwise be invisible. The message repeats on later steps, hence the id
303
+ * dedupe.
304
+ */
305
+ function recordInvocations(sessionId, messages, agent) {
306
+ if (!Array.isArray(messages)) return
307
+ for (const message of messages) {
308
+ if (!message || typeof message !== 'object') continue
309
+ const source = message.source
310
+ if (!source || source.kind !== 'skill-invocation') continue
311
+ const skill = typeof source.name === 'string' && source.name.length > 0 ? source.name : null
312
+ if (!skill) continue
313
+ const id = typeof message.id === 'string' ? message.id : `${skill}:${sessionId}`
314
+ const seenKey = `${sessionId}|${id}`
315
+ if (seenInvocations.has(seenKey)) continue
316
+ seenInvocations.add(seenKey)
317
+ // Evict the oldest keys instead of clearing everything: a wholesale
318
+ // reset would let a repeated invocation message be recorded twice.
319
+ if (seenInvocations.size > 4000) {
320
+ for (let i = 0; i < 500; i += 1) {
321
+ const oldest = seenInvocations.values().next().value
322
+ if (oldest === undefined) break
323
+ seenInvocations.delete(oldest)
324
+ }
325
+ }
326
+ const state = stateFor(sessionId)
327
+ const windowId = `${sessionId}:${id}`
328
+ void store.append({
329
+ t: 'skill',
330
+ v: 1,
331
+ at: nowMs(),
332
+ sessionId,
333
+ callId: id,
334
+ skill,
335
+ ok: true,
336
+ via: 'invocation',
337
+ origin: originOf(agent),
338
+ turn: turns.get(sessionId),
339
+ windowId,
340
+ })
341
+ openWindow(state, skill, sessionId, id, steps.get(sessionId), originOf(agent))
342
+ void captureStatic(skill, agent, undefined, undefined)
343
+ }
344
+ }
345
+
346
+ // The loop step is the only way to tell "loaded A and B together" (parallel
347
+ // work in one step) from "loaded A, then switched to B". Pass the waterfall
348
+ // through untouched.
349
+ offs.push(
350
+ ctx.on('agent/pre-step', (payload, next) => {
351
+ try {
352
+ if (payload && payload.agent) {
353
+ const sessionId = String(payload.agent.id)
354
+ steps.set(sessionId, payload.step)
355
+ if (typeof payload.turn === 'number') turns.set(sessionId, payload.turn)
356
+ beginStep(sessionId, payload.turn ?? turns.get(sessionId), payload.step)
357
+ recordInvocations(sessionId, payload.messages, payload.agent)
358
+ }
359
+ } catch {
360
+ // never interfere with the loop
361
+ }
362
+ return next()
363
+ }),
364
+ )
365
+
366
+ // The loop can fail on its own (a broken step, a failed model request, an
367
+ // aborted tool batch). That is evidence about the skill that was loaded right
368
+ // before it, so it lands inside the open windows instead of being lost.
369
+ offs.push(
370
+ ctx.on('agent/error', (payload) => {
371
+ try {
372
+ const sessionId = payload && payload.agent ? String(payload.agent.id) : null
373
+ if (!sessionId) return
374
+ const state = sessions.get(sessionId)
375
+ const step = stepStats.get(sessionId)
376
+ if (step) step.loopErrors += 1
377
+ if (!state || state.windows.length === 0) return
378
+ const error = payload.error
379
+ const label =
380
+ error instanceof Error
381
+ ? error.name || 'Error'
382
+ : error && typeof error === 'object' && typeof error.code === 'string'
383
+ ? error.code
384
+ : 'AgentError'
385
+ for (const w of state.windows) {
386
+ w.loopErrors += 1
387
+ w.lastLoopError = label
388
+ }
389
+ } catch {
390
+ // ignore
391
+ }
392
+ }),
393
+ )
394
+
395
+ offs.push(
396
+ ctx.on('agent/request-error', (payload, next) => {
397
+ try {
398
+ const sessionId = payload && payload.agent ? String(payload.agent.id) : null
399
+ if (sessionId) {
400
+ const state = sessions.get(sessionId)
401
+ if (state && state.windows.length > 0) {
402
+ for (const w of state.windows) w.llmRetries += 1
403
+ }
404
+ }
405
+ } catch {
406
+ // ignore
407
+ }
408
+ return next()
409
+ }),
410
+ )
411
+
412
+ // Start timing on skill loads only; pass the waterfall through untouched.
413
+ offs.push(
414
+ ctx.on('tools/pre-execute', (exec, next) => {
415
+ try {
416
+ if (exec && exec.callId !== undefined) {
417
+ starts.set(exec.callId, nowMs())
418
+ if (starts.size > 512) starts.delete(starts.keys().next().value)
419
+ }
420
+ } catch {
421
+ // never interfere with execution
422
+ }
423
+ return next()
424
+ }),
425
+ )
426
+
427
+ offs.push(
428
+ ctx.on('tools/result', (exec, result) => {
429
+ try {
430
+ if (!exec || !exec.agent) return
431
+ const sessionId = String(exec.agent.id)
432
+ const state = stateFor(sessionId)
433
+ const step = stepStats.get(sessionId)
434
+ const mcpCall = mcpOf(exec.name)
435
+ if (step) {
436
+ step.toolCalls += 1
437
+ if (result?.isError === true) {
438
+ step.errors += 1
439
+ if (step.errorNames.length < 8) step.errorNames.push(errorNameOf(result))
440
+ }
441
+ if (mcpCall) {
442
+ step.mcpCalls += 1
443
+ if (result?.isError === true) step.mcpErrors += 1
444
+ }
445
+ if (exec.name === 'skill') {
446
+ const loaded = skillNameOf(exec.arguments)
447
+ if (loaded && !step.skills.includes(loaded)) step.skills.push(loaded)
448
+ }
449
+ }
450
+ const newest = state.windows[state.windows.length - 1]
451
+ if (newest && nowMs() - newest.lastActivityAt > ttlMs()) closeWindows(state, 'stale')
452
+
453
+ if (exec.name === 'skill') {
454
+ const skill = skillNameOf(exec.arguments)
455
+ const started = starts.get(exec.callId)
456
+ starts.delete(exec.callId)
457
+ if (!skill) return
458
+ const ok = result?.isError !== true
459
+ const windowId = `${sessionId}:${exec.callId}`
460
+ void store.append({
461
+ t: 'skill',
462
+ v: 1,
463
+ at: nowMs(),
464
+ sessionId,
465
+ callId: exec.callId,
466
+ skill,
467
+ ok,
468
+ loadMs: started === undefined ? undefined : nowMs() - started,
469
+ error: ok ? undefined : String(result?.error?.message ?? 'load failed').slice(0, 300),
470
+ origin: originOf(exec.agent),
471
+ turn: turns.get(sessionId),
472
+ windowId,
473
+ })
474
+ if (ok) {
475
+ openWindow(state, skill, sessionId, exec.callId, steps.get(sessionId), originOf(exec.agent))
476
+ void captureStatic(
477
+ skill,
478
+ exec.agent,
479
+ exec.signal,
480
+ result?.value && typeof result.value.content === 'string' ? result.value.content : undefined,
481
+ )
482
+ }
483
+ return
484
+ }
485
+
486
+ if (state.windows.length === 0) return
487
+ const current = steps.get(sessionId)
488
+ // Count the call for every window still open in this step: a parallel
489
+ // batch of skill loads is one piece of work, not one winner.
490
+ const targets = state.windows.filter((w) => current === undefined || w.step === undefined || w.step === current)
491
+ const startedAt = starts.get(exec.callId)
492
+ if (exec.callId !== undefined) starts.delete(exec.callId)
493
+ const elapsed = startedAt === undefined ? undefined : nowMs() - startedAt
494
+ const mcp = mcpOf(exec.name)
495
+ for (const w of targets.length > 0 ? targets : state.windows) {
496
+ w.lastActivityAt = nowMs()
497
+ if (state) state.lastAt = nowMs()
498
+ if (mcp) {
499
+ if (!Object.hasOwn(w.mcp, mcp.server)) {
500
+ w.mcp[mcp.server] = { calls: 0, errors: 0, ms: 0, samples: [], tools: Object.create(null) }
501
+ }
502
+ const bucket = w.mcp[mcp.server]
503
+ bucket.calls += 1
504
+ if (result?.isError === true) bucket.errors += 1
505
+ if (elapsed !== undefined) {
506
+ bucket.ms += elapsed
507
+ // keep a bounded latency sample so the report can show p50/p95,
508
+ // not just a mean that one slow call can distort
509
+ if (!Array.isArray(bucket.samples)) bucket.samples = []
510
+ if (bucket.samples.length < 40) bucket.samples.push(elapsed)
511
+ }
512
+ if (mcp.tool) {
513
+ const tool = Object.hasOwn(bucket.tools, mcp.tool)
514
+ ? bucket.tools[mcp.tool]
515
+ : { calls: 0, errors: 0, ms: 0, samples: [] }
516
+ tool.calls += 1
517
+ if (result?.isError === true) tool.errors += 1
518
+ if (elapsed !== undefined) {
519
+ tool.ms = (tool.ms ?? 0) + elapsed
520
+ if (!Array.isArray(tool.samples)) tool.samples = []
521
+ if (tool.samples.length < 20) tool.samples.push(elapsed)
522
+ }
523
+ bucket.tools[mcp.tool] = tool
524
+ }
525
+ w.mcp[mcp.server] = bucket
526
+ }
527
+ if (w.toolCalls >= windowLimit) {
528
+ w.truncated = true
529
+ continue
530
+ }
531
+ w.toolCalls += 1
532
+ if (result?.isError === true) {
533
+ w.errors += 1
534
+ if (w.errorNames.length < 12) w.errorNames.push(errorNameOf(result))
535
+ }
536
+ }
537
+ } catch {
538
+ // telemetry must never break tool execution
539
+ }
540
+ }),
541
+ )
542
+
543
+ // The next user message is the strongest signal about the last run: either it
544
+ // is a correction (the skill failed) or it is a normal follow-up (it worked).
545
+ offs.push(
546
+ ctx.on('agent/inbox/claimed', (payload) => {
547
+ try {
548
+ const sessionId = payload?.agent ? String(payload.agent.id) : null
549
+ if (!sessionId) return
550
+ const state = sessions.get(sessionId)
551
+ if (!state || state.windows.length === 0) return
552
+ const newest = state.windows[state.windows.length - 1]
553
+ if (nowMs() - newest.lastActivityAt > ttlMs()) {
554
+ closeWindows(state, 'stale')
555
+ return
556
+ }
557
+ const text = messageText(payload.message)
558
+ if (text.length > 0 && isCorrection(text)) {
559
+ for (const w of state.windows) w.corrections.push(text)
560
+ }
561
+ closeWindows(state, 'user-turn')
562
+ } catch {
563
+ // ignore
564
+ }
565
+ }),
566
+ )
567
+
568
+ // Real provider usage rides on every committed assistant message, so token
569
+ // cost is measured instead of estimated. Usage belongs to the model call that
570
+ // answered the current step, which is exactly the work an open window caused.
571
+ offs.push(
572
+ ctx.on('session/event', (session, event) => {
573
+ try {
574
+ if (config.countTokens === false || !event || event.type !== 'assistant/message') return
575
+ const usage = event.data && event.data.usage
576
+ if (!usage || typeof usage !== 'object') return
577
+ const sessionId = session && session.id ? String(session.id) : null
578
+ if (!sessionId) return
579
+ const state = sessions.get(sessionId)
580
+ const step = stepStats.get(sessionId)
581
+ if (step) {
582
+ step.tokens.input += Number(usage.inputTokens) || 0
583
+ step.tokens.output += Number(usage.outputTokens) || 0
584
+ step.tokens.cacheRead += Number(usage.cacheReadTokens) || 0
585
+ step.tokens.calls += 1
586
+ }
587
+ if (!state || state.windows.length === 0) return
588
+ for (const w of state.windows) {
589
+ const bucket = w.tokens ?? { input: 0, output: 0, cacheRead: 0, calls: 0 }
590
+ bucket.input += Number(usage.inputTokens) || 0
591
+ bucket.output += Number(usage.outputTokens) || 0
592
+ bucket.cacheRead += Number(usage.cacheReadTokens) || 0
593
+ bucket.calls += 1
594
+ w.tokens = bucket
595
+ }
596
+ } catch {
597
+ // token accounting is best effort
598
+ }
599
+ }),
600
+ )
601
+
602
+ // Persist at turn end, but keep observing for the follow-up message.
603
+ offs.push(
604
+ ctx.on('agent/turn-stopping', (payload) => {
605
+ try {
606
+ const sessionId = payload?.agent ? String(payload.agent.id) : null
607
+ if (!sessionId) return
608
+ const state = sessions.get(sessionId)
609
+ if (!state || state.windows.length === 0) return
610
+ for (const w of state.windows) {
611
+ w.turnEnded = true
612
+ persist(w, 'turn-end')
613
+ }
614
+ flushStep(sessionId, 'turn-end')
615
+ } catch {
616
+ // ignore
617
+ }
618
+ }),
619
+ )
620
+
621
+ return {
622
+ /** Await every write queued so far (used by tests and by the snapshot flush). */
623
+ flush() {
624
+ return store.queue
625
+ },
626
+ async dispose() {
627
+ for (const sessionId of [...stepStats.keys()]) flushStep(sessionId, 'dispose')
628
+ for (const state of sessions.values()) closeWindows(state, 'dispose')
629
+ sessions.clear()
630
+ steps.clear()
631
+ turns.clear()
632
+ seenInvocations.clear()
633
+ // let every queued write land before the caller tears the store down
634
+ await store.queue
635
+ for (const off of offs) {
636
+ try {
637
+ off()
638
+ } catch {
639
+ // ignore
640
+ }
641
+ }
642
+ },
643
+ }
644
+ }