dsh-skill-eval 0.0.0-stage → 1.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +675 -0
  2. package/LICENSE +21 -0
  3. package/README.md +200 -2
  4. package/README.zh.md +157 -0
  5. package/client.js +2637 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +64 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +463 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1111 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +547 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +140 -0
  26. package/lib/html.js +8 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +634 -0
  32. package/lib/render.js +582 -0
  33. package/lib/report.js +646 -0
  34. package/lib/score.js +525 -0
  35. package/lib/settings.js +396 -0
  36. package/lib/snapshot.js +166 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +186 -0
  39. package/lib/validity.js +115 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +226 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +199 -0
  47. package/scripts/lib.mjs +52 -0
  48. package/scripts/verify-live.mjs +168 -0
package/lib/random.js ADDED
@@ -0,0 +1,29 @@
1
+ /** Small deterministic helpers shared by scoring and reporting. */
2
+
3
+ /** FNV-1a 32-bit hash; used to derive reproducible bootstrap seeds. */
4
+ export function hashSeed(text) {
5
+ let hash = 2166136261
6
+ const value = typeof text === 'string' ? text : String(text ?? '')
7
+ for (let i = 0; i < value.length; i += 1) {
8
+ hash ^= value.charCodeAt(i)
9
+ hash = Math.imul(hash, 16777619)
10
+ }
11
+ return hash >>> 0
12
+ }
13
+
14
+ /** Stable short hex hash for a diff, so the ledger can identify it without storing it. */
15
+ export function hashHex(text) {
16
+ return hashSeed(text).toString(16).padStart(8, '0')
17
+ }
18
+
19
+ /** Mulberry32 PRNG: deterministic, tiny, good enough for bootstrap resampling. */
20
+ export function mulberry32(seed) {
21
+ let a = (Number.isFinite(seed) ? seed : 1) >>> 0
22
+ return () => {
23
+ a = (a + 0x6d2b79f5) >>> 0
24
+ let t = a
25
+ t = Math.imul(t ^ (t >>> 15), t | 1)
26
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
27
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296
28
+ }
29
+ }
@@ -0,0 +1,634 @@
1
+ import { stat as statFile } from 'node:fs/promises'
2
+ import path from 'node:path'
3
+ import { checkReferences } from './integrity.js'
4
+ import { isCorrection, staticProfile } from './score.js'
5
+
6
+ const MAX_TRACKED_SESSIONS = 128
7
+
8
+ /** Best-effort text extraction from a user message of unknown shape. */
9
+ export function messageText(message) {
10
+ if (!message || typeof message !== 'object') return ''
11
+ if (typeof message.content === 'string') return message.content
12
+ if (Array.isArray(message.content)) {
13
+ return message.content
14
+ .map((block) => {
15
+ if (!block || typeof block !== 'object') return ''
16
+ if (typeof block.text === 'string') return block.text
17
+ if (typeof block.value === 'string') return block.value
18
+ return ''
19
+ })
20
+ .filter(Boolean)
21
+ .join('\n')
22
+ }
23
+ if (typeof message.text === 'string') return message.text
24
+ return ''
25
+ }
26
+
27
+ function skillNameOf(args) {
28
+ if (!args || typeof args !== 'object') return null
29
+ const candidate = args.name ?? args.skill ?? args.skillName
30
+ return typeof candidate === 'string' && candidate.length > 0 ? candidate : null
31
+ }
32
+
33
+ /**
34
+ * MCP tools reach the model as `mcp__<server>__<tool>`, so the telemetry can
35
+ * split one skill's window by MCP server without any extra configuration.
36
+ */
37
+ export function mcpOf(toolName) {
38
+ if (typeof toolName !== 'string' || !toolName.startsWith('mcp__')) return null
39
+ const rest = toolName.slice('mcp__'.length)
40
+ const cut = rest.indexOf('__')
41
+ if (cut <= 0) return { server: rest, tool: '' }
42
+ return { server: rest.slice(0, cut), tool: rest.slice(cut + 2) }
43
+ }
44
+
45
+ function errorNameOf(result) {
46
+ const info = result?.error?.info
47
+ if (info && typeof info.name === 'string' && info.name.length > 0) return info.name
48
+ return 'ToolError'
49
+ }
50
+
51
+ /** Session origin, so short-lived subagent runs can be judged differently. */
52
+ function originOf(agent) {
53
+ try {
54
+ const origin = agent && agent.session && agent.session.header ? agent.session.header.origin : null
55
+ return typeof origin === 'string' && origin.length > 0 ? origin : null
56
+ } catch {
57
+ return null
58
+ }
59
+ }
60
+
61
+ function nowMs() {
62
+ return Date.now()
63
+ }
64
+
65
+ /**
66
+ * Live telemetry recorder.
67
+ *
68
+ * A "window" is the stretch of work that follows one `skill` load inside one
69
+ * session. The window stays open across the turn boundary on purpose: the
70
+ * evidence that a skill failed to help is usually the *next* user message ("no,
71
+ * that's wrong"), so the window must survive until that message arrives.
72
+ *
73
+ * Each window is persisted when the turn ends and re-persisted when it finally
74
+ * closes (with the correction attached). Readers keep the last record per
75
+ * windowId, so a re-persist is an update, not a duplicate.
76
+ */
77
+ export function createRecorder(ctx, config, { store, resolveSkill }) {
78
+ const starts = new Map() // callId -> ms timestamp
79
+ const steps = new Map() // sessionId -> current loop step number
80
+ const turns = new Map() // sessionId -> current turn number
81
+ const stepStats = new Map() // sessionId -> accumulator for the current step
82
+ // A user-typed `/skill-name` is injected as a user message on every following
83
+ // step, so the message id is the dedupe key for that load path.
84
+ const seenInvocations = new Set()
85
+ // A session can have several windows open at once: parallel `skill` loads in
86
+ // one loop step are concurrent use, so their follow-up work counts for each of
87
+ // them instead of being handed to whichever loaded last.
88
+ const sessions = new Map() // sessionId -> { windows: [] }
89
+
90
+ const windowLimit = Math.max(1, Number(config.windowToolCalls ?? 12))
91
+ const ttlMs = Math.max(60_000, Number(config.windowTtlMs ?? 30 * 60_000))
92
+
93
+ function stateFor(sessionId) {
94
+ const existing = sessions.get(sessionId)
95
+ if (existing) {
96
+ existing.lastAt = nowMs()
97
+ return existing
98
+ }
99
+ if (sessions.size >= MAX_TRACKED_SESSIONS) {
100
+ // Evict least-recently-active, not least-recently-created: a root turn
101
+ // that just loaded a skill must not lose its window to a burst of
102
+ // short-lived subagent sessions.
103
+ let oldest = null
104
+ let oldestAt = Infinity
105
+ for (const [id, candidate] of sessions) {
106
+ const at = candidate.lastAt ?? 0
107
+ if (at < oldestAt) {
108
+ oldestAt = at
109
+ oldest = id
110
+ }
111
+ }
112
+ if (oldest !== null) {
113
+ const stale = sessions.get(oldest)
114
+ if (stale) closeWindows(stale, 'evicted')
115
+ flushStep(oldest, 'evicted')
116
+ sessions.delete(oldest)
117
+ steps.delete(oldest)
118
+ turns.delete(oldest)
119
+ }
120
+ }
121
+ const state = { windows: [], lastAt: nowMs() }
122
+ sessions.set(sessionId, state)
123
+ return state
124
+ }
125
+
126
+ function persist(w, closeReason) {
127
+ if (!w) return
128
+ w.flushed = true
129
+ void store.append({
130
+ t: 'window',
131
+ v: 1,
132
+ at: nowMs(),
133
+ windowId: w.windowId,
134
+ sessionId: w.sessionId,
135
+ skill: w.skill,
136
+ toolCalls: w.toolCalls,
137
+ errors: w.errors,
138
+ errorNames: w.errorNames.slice(0, 12),
139
+ loopErrors: w.loopErrors,
140
+ llmRetries: w.llmRetries,
141
+ lastLoopError: w.lastLoopError,
142
+ mcp: w.mcp,
143
+ tokens: w.tokens,
144
+ corrections: w.corrections.map((text) => text.slice(0, 200)),
145
+ otherSkills: [...w.otherSkills],
146
+ origin: w.origin,
147
+ step: w.step,
148
+ turnEnded: w.turnEnded,
149
+ truncated: w.truncated,
150
+ closeReason,
151
+ })
152
+ }
153
+
154
+ function closeWindows(state, reason, except) {
155
+ if (!state || state.windows.length === 0) return
156
+ const closing = state.windows.filter((w) => w !== except)
157
+ state.windows = except ? [except] : []
158
+ for (const w of closing) persist(w, reason)
159
+ }
160
+
161
+ function openWindow(state, skill, sessionId, callId, step, origin) {
162
+ const current = state.windows[state.windows.length - 1]
163
+ if (current) {
164
+ // Two loads inside one loop step run concurrently: that is parallel use,
165
+ // not the model abandoning one skill for another. Only a load in a later
166
+ // step disagrees with what the previous skill was doing.
167
+ const sameStep = step !== undefined && current.step === step
168
+ if (sameStep) {
169
+ // keep the sibling window open; the same follow-up work serves both
170
+ } else {
171
+ for (const w of state.windows) if (w.skill !== skill) w.otherSkills.push(skill)
172
+ closeWindows(state, 'superseded')
173
+ }
174
+ }
175
+ state.windows.push({
176
+ windowId: `${sessionId}:${callId}`,
177
+ sessionId,
178
+ skill,
179
+ toolCalls: 0,
180
+ errors: 0,
181
+ errorNames: [],
182
+ loopErrors: 0,
183
+ llmRetries: 0,
184
+ lastLoopError: null,
185
+ mcp: Object.create(null),
186
+ tokens: { input: 0, output: 0, cacheRead: 0, calls: 0 },
187
+ corrections: [],
188
+ otherSkills: [],
189
+ turnEnded: false,
190
+ truncated: false,
191
+ step,
192
+ origin,
193
+ openedAt: nowMs(),
194
+ lastActivityAt: nowMs(),
195
+ flushed: false,
196
+ })
197
+ }
198
+
199
+ /**
200
+ * Static authoring quality cannot be resolved at report time: the skill
201
+ * catalog is layered per agent scope, and by then no agent is available. So
202
+ * resolve it once, at load time, while `exec.agent` is still in hand.
203
+ */
204
+ async function captureStatic(skill, agent, signal, fallbackContent) {
205
+ try {
206
+ if (typeof resolveSkill !== 'function') return
207
+ let resolved = null
208
+ try {
209
+ resolved = await resolveSkill(skill, agent, signal)
210
+ } catch {
211
+ resolved = null
212
+ }
213
+ const value = typeof fallbackContent === 'string' ? { content: fallbackContent } : null
214
+ const definition =
215
+ resolved?.definition ?? (value && typeof value.content === 'string' ? { content: value.content } : undefined)
216
+ const summary =
217
+ resolved?.summary ??
218
+ (value && typeof value.content === 'string'
219
+ ? { name: skill, description: '', whenToUse: '', source: value.provider ?? null }
220
+ : undefined)
221
+ // Reference integrity needs the filesystem view, so it is resolved here,
222
+ // while the skill directory is still known, and handed to the pure profile.
223
+ let integrity = null
224
+ try {
225
+ const baseDir =
226
+ (summary && summary.resourceBase && summary.resourceBase.kind === 'directory' && summary.resourceBase.path) ||
227
+ (summary && typeof summary.path === 'string' ? path.dirname(summary.path) : null)
228
+ if (baseDir && definition) {
229
+ integrity = await checkReferences({ baseDir, content: definition.content, stat: statFile })
230
+ }
231
+ } catch {
232
+ integrity = null
233
+ }
234
+ const profile = staticProfile(summary, definition, { integrity })
235
+ if (!profile) return
236
+ await store.append({
237
+ t: 'static',
238
+ v: 1,
239
+ at: nowMs(),
240
+ skill,
241
+ source: profile.source,
242
+ static: profile,
243
+ })
244
+ } catch {
245
+ // static capture is optional
246
+ }
247
+ }
248
+
249
+ function flushStep(sessionId, closeReason = 'next-step') {
250
+ const step = stepStats.get(sessionId)
251
+ if (!step) return
252
+ stepStats.delete(sessionId)
253
+ const active =
254
+ step.toolCalls > 0 || step.errors > 0 || step.loopErrors > 0 || step.skills.length > 0 || step.tokens.calls > 0
255
+ if (!active) return
256
+ void store.append({
257
+ t: 'step',
258
+ v: 1,
259
+ at: nowMs(),
260
+ sessionId,
261
+ turn: step.turn,
262
+ step: step.step,
263
+ startedAt: step.startedAt,
264
+ durationMs: Math.max(0, nowMs() - step.startedAt),
265
+ toolCalls: step.toolCalls,
266
+ errors: step.errors,
267
+ errorNames: step.errorNames.slice(0, 8),
268
+ mcpCalls: step.mcpCalls,
269
+ mcpErrors: step.mcpErrors,
270
+ loopErrors: step.loopErrors,
271
+ skills: [...step.skills],
272
+ tokens: step.tokens,
273
+ closeReason,
274
+ })
275
+ }
276
+
277
+ function beginStep(sessionId, turn, stepNumber) {
278
+ flushStep(sessionId, 'next-step')
279
+ stepStats.set(sessionId, {
280
+ startedAt: nowMs(),
281
+ sessionId,
282
+ turn: Number.isFinite(Number(turn)) ? Number(turn) : null,
283
+ step: Number.isFinite(Number(stepNumber)) ? Number(stepNumber) : null,
284
+ toolCalls: 0,
285
+ errors: 0,
286
+ errorNames: [],
287
+ mcpCalls: 0,
288
+ mcpErrors: 0,
289
+ loopErrors: 0,
290
+ skills: [],
291
+ tokens: { input: 0, output: 0, cacheRead: 0, calls: 0 },
292
+ })
293
+ }
294
+
295
+ const offs = []
296
+
297
+ /**
298
+ * The `skill` tool is not the only way a skill gets loaded: a user-typed
299
+ * `/skill-name` is resolved by the harness and injected as a user message with
300
+ * `source: { kind: 'skill-invocation', name }` — no tool call, so it would
301
+ * otherwise be invisible. The message repeats on later steps, hence the id
302
+ * dedupe.
303
+ */
304
+ function recordInvocations(sessionId, messages, agent) {
305
+ if (!Array.isArray(messages)) return
306
+ for (const message of messages) {
307
+ if (!message || typeof message !== 'object') continue
308
+ const source = message.source
309
+ if (!source || source.kind !== 'skill-invocation') continue
310
+ const skill = typeof source.name === 'string' && source.name.length > 0 ? source.name : null
311
+ if (!skill) continue
312
+ const id = typeof message.id === 'string' ? message.id : `${skill}:${sessionId}`
313
+ const seenKey = `${sessionId}|${id}`
314
+ if (seenInvocations.has(seenKey)) continue
315
+ seenInvocations.add(seenKey)
316
+ if (seenInvocations.size > 4000) seenInvocations.clear()
317
+ const state = stateFor(sessionId)
318
+ const windowId = `${sessionId}:${id}`
319
+ void store.append({
320
+ t: 'skill',
321
+ v: 1,
322
+ at: nowMs(),
323
+ sessionId,
324
+ callId: id,
325
+ skill,
326
+ ok: true,
327
+ via: 'invocation',
328
+ origin: originOf(agent),
329
+ turn: turns.get(sessionId),
330
+ windowId,
331
+ })
332
+ openWindow(state, skill, sessionId, id, steps.get(sessionId), originOf(agent))
333
+ void captureStatic(skill, agent, undefined, undefined)
334
+ }
335
+ }
336
+
337
+ // The loop step is the only way to tell "loaded A and B together" (parallel
338
+ // work in one step) from "loaded A, then switched to B". Pass the waterfall
339
+ // through untouched.
340
+ offs.push(
341
+ ctx.on('agent/pre-step', (payload, next) => {
342
+ try {
343
+ if (payload && payload.agent) {
344
+ const sessionId = String(payload.agent.id)
345
+ steps.set(sessionId, payload.step)
346
+ if (typeof payload.turn === 'number') turns.set(sessionId, payload.turn)
347
+ beginStep(sessionId, payload.turn ?? turns.get(sessionId), payload.step)
348
+ recordInvocations(sessionId, payload.messages, payload.agent)
349
+ }
350
+ } catch {
351
+ // never interfere with the loop
352
+ }
353
+ return next()
354
+ }),
355
+ )
356
+
357
+ // The loop can fail on its own (a broken step, a failed model request, an
358
+ // aborted tool batch). That is evidence about the skill that was loaded right
359
+ // before it, so it lands inside the open windows instead of being lost.
360
+ offs.push(
361
+ ctx.on('agent/error', (payload) => {
362
+ try {
363
+ const sessionId = payload && payload.agent ? String(payload.agent.id) : null
364
+ if (!sessionId) return
365
+ const state = sessions.get(sessionId)
366
+ const step = stepStats.get(sessionId)
367
+ if (step) step.loopErrors += 1
368
+ if (!state || state.windows.length === 0) return
369
+ const error = payload.error
370
+ const label =
371
+ error instanceof Error
372
+ ? error.name || 'Error'
373
+ : error && typeof error === 'object' && typeof error.code === 'string'
374
+ ? error.code
375
+ : 'AgentError'
376
+ for (const w of state.windows) {
377
+ w.loopErrors += 1
378
+ w.lastLoopError = label
379
+ }
380
+ } catch {
381
+ // ignore
382
+ }
383
+ }),
384
+ )
385
+
386
+ offs.push(
387
+ ctx.on('agent/request-error', (payload, next) => {
388
+ try {
389
+ const sessionId = payload && payload.agent ? String(payload.agent.id) : null
390
+ if (sessionId) {
391
+ const state = sessions.get(sessionId)
392
+ if (state && state.windows.length > 0) {
393
+ for (const w of state.windows) w.llmRetries += 1
394
+ }
395
+ }
396
+ } catch {
397
+ // ignore
398
+ }
399
+ return next()
400
+ }),
401
+ )
402
+
403
+ // Start timing on skill loads only; pass the waterfall through untouched.
404
+ offs.push(
405
+ ctx.on('tools/pre-execute', (exec, next) => {
406
+ try {
407
+ if (exec && exec.callId !== undefined) {
408
+ starts.set(exec.callId, nowMs())
409
+ if (starts.size > 512) starts.delete(starts.keys().next().value)
410
+ }
411
+ } catch {
412
+ // never interfere with execution
413
+ }
414
+ return next()
415
+ }),
416
+ )
417
+
418
+ offs.push(
419
+ ctx.on('tools/result', (exec, result) => {
420
+ try {
421
+ if (!exec || !exec.agent) return
422
+ const sessionId = String(exec.agent.id)
423
+ const state = stateFor(sessionId)
424
+ const step = stepStats.get(sessionId)
425
+ const mcpCall = mcpOf(exec.name)
426
+ if (step) {
427
+ step.toolCalls += 1
428
+ if (result?.isError === true) {
429
+ step.errors += 1
430
+ if (step.errorNames.length < 8) step.errorNames.push(errorNameOf(result))
431
+ }
432
+ if (mcpCall) {
433
+ step.mcpCalls += 1
434
+ if (result?.isError === true) step.mcpErrors += 1
435
+ }
436
+ if (exec.name === 'skill') {
437
+ const loaded = skillNameOf(exec.arguments)
438
+ if (loaded && !step.skills.includes(loaded)) step.skills.push(loaded)
439
+ }
440
+ }
441
+ const newest = state.windows[state.windows.length - 1]
442
+ if (newest && nowMs() - newest.lastActivityAt > ttlMs) closeWindows(state, 'stale')
443
+
444
+ if (exec.name === 'skill') {
445
+ const skill = skillNameOf(exec.arguments)
446
+ const started = starts.get(exec.callId)
447
+ starts.delete(exec.callId)
448
+ if (!skill) return
449
+ const ok = result?.isError !== true
450
+ const windowId = `${sessionId}:${exec.callId}`
451
+ void store.append({
452
+ t: 'skill',
453
+ v: 1,
454
+ at: nowMs(),
455
+ sessionId,
456
+ callId: exec.callId,
457
+ skill,
458
+ ok,
459
+ loadMs: started === undefined ? undefined : nowMs() - started,
460
+ error: ok ? undefined : String(result?.error?.message ?? 'load failed').slice(0, 300),
461
+ origin: originOf(exec.agent),
462
+ turn: turns.get(sessionId),
463
+ windowId,
464
+ })
465
+ openWindow(state, skill, sessionId, exec.callId, steps.get(sessionId), originOf(exec.agent))
466
+ if (ok)
467
+ void captureStatic(
468
+ skill,
469
+ exec.agent,
470
+ exec.signal,
471
+ result?.value && typeof result.value.content === 'string' ? result.value.content : undefined,
472
+ )
473
+ return
474
+ }
475
+
476
+ if (state.windows.length === 0) return
477
+ const current = steps.get(sessionId)
478
+ // Count the call for every window still open in this step: a parallel
479
+ // batch of skill loads is one piece of work, not one winner.
480
+ const targets = state.windows.filter((w) => current === undefined || w.step === undefined || w.step === current)
481
+ const startedAt = starts.get(exec.callId)
482
+ if (exec.callId !== undefined) starts.delete(exec.callId)
483
+ const elapsed = startedAt === undefined ? undefined : nowMs() - startedAt
484
+ const mcp = mcpOf(exec.name)
485
+ for (const w of targets.length > 0 ? targets : state.windows) {
486
+ w.lastActivityAt = nowMs()
487
+ if (state) state.lastAt = nowMs()
488
+ if (mcp) {
489
+ if (!Object.hasOwn(w.mcp, mcp.server)) {
490
+ w.mcp[mcp.server] = { calls: 0, errors: 0, ms: 0, samples: [], tools: Object.create(null) }
491
+ }
492
+ const bucket = w.mcp[mcp.server]
493
+ bucket.calls += 1
494
+ if (result?.isError === true) bucket.errors += 1
495
+ if (elapsed !== undefined) {
496
+ bucket.ms += elapsed
497
+ // keep a bounded latency sample so the report can show p50/p95,
498
+ // not just a mean that one slow call can distort
499
+ if (!Array.isArray(bucket.samples)) bucket.samples = []
500
+ if (bucket.samples.length < 40) bucket.samples.push(elapsed)
501
+ }
502
+ if (mcp.tool) {
503
+ const tool = Object.hasOwn(bucket.tools, mcp.tool)
504
+ ? bucket.tools[mcp.tool]
505
+ : { calls: 0, errors: 0, ms: 0, samples: [] }
506
+ tool.calls += 1
507
+ if (result?.isError === true) tool.errors += 1
508
+ if (elapsed !== undefined) {
509
+ tool.ms = (tool.ms ?? 0) + elapsed
510
+ if (!Array.isArray(tool.samples)) tool.samples = []
511
+ if (tool.samples.length < 20) tool.samples.push(elapsed)
512
+ }
513
+ bucket.tools[mcp.tool] = tool
514
+ }
515
+ w.mcp[mcp.server] = bucket
516
+ }
517
+ if (w.toolCalls >= windowLimit) {
518
+ w.truncated = true
519
+ continue
520
+ }
521
+ w.toolCalls += 1
522
+ if (result?.isError === true) {
523
+ w.errors += 1
524
+ if (w.errorNames.length < 12) w.errorNames.push(errorNameOf(result))
525
+ }
526
+ }
527
+ } catch {
528
+ // telemetry must never break tool execution
529
+ }
530
+ }),
531
+ )
532
+
533
+ // The next user message is the strongest signal about the last run: either it
534
+ // is a correction (the skill failed) or it is a normal follow-up (it worked).
535
+ offs.push(
536
+ ctx.on('agent/inbox/claimed', (payload) => {
537
+ try {
538
+ const sessionId = payload?.agent ? String(payload.agent.id) : null
539
+ if (!sessionId) return
540
+ const state = sessions.get(sessionId)
541
+ if (!state || state.windows.length === 0) return
542
+ const newest = state.windows[state.windows.length - 1]
543
+ if (nowMs() - newest.lastActivityAt > ttlMs) {
544
+ closeWindows(state, 'stale')
545
+ return
546
+ }
547
+ const text = messageText(payload.message)
548
+ if (text.length > 0 && isCorrection(text)) {
549
+ for (const w of state.windows) w.corrections.push(text)
550
+ }
551
+ closeWindows(state, 'user-turn')
552
+ } catch {
553
+ // ignore
554
+ }
555
+ }),
556
+ )
557
+
558
+ // Real provider usage rides on every committed assistant message, so token
559
+ // cost is measured instead of estimated. Usage belongs to the model call that
560
+ // answered the current step, which is exactly the work an open window caused.
561
+ offs.push(
562
+ ctx.on('session/event', (session, event) => {
563
+ try {
564
+ if (config.countTokens === false || !event || event.type !== 'assistant/message') return
565
+ const usage = event.data && event.data.usage
566
+ if (!usage || typeof usage !== 'object') return
567
+ const sessionId = session && session.id ? String(session.id) : null
568
+ if (!sessionId) return
569
+ const state = sessions.get(sessionId)
570
+ const step = stepStats.get(sessionId)
571
+ if (step) {
572
+ step.tokens.input += Number(usage.inputTokens) || 0
573
+ step.tokens.output += Number(usage.outputTokens) || 0
574
+ step.tokens.cacheRead += Number(usage.cacheReadTokens) || 0
575
+ step.tokens.calls += 1
576
+ }
577
+ if (!state || state.windows.length === 0) return
578
+ for (const w of state.windows) {
579
+ const bucket = w.tokens ?? { input: 0, output: 0, cacheRead: 0, calls: 0 }
580
+ bucket.input += Number(usage.inputTokens) || 0
581
+ bucket.output += Number(usage.outputTokens) || 0
582
+ bucket.cacheRead += Number(usage.cacheReadTokens) || 0
583
+ bucket.calls += 1
584
+ w.tokens = bucket
585
+ }
586
+ } catch {
587
+ // token accounting is best effort
588
+ }
589
+ }),
590
+ )
591
+
592
+ // Persist at turn end, but keep observing for the follow-up message.
593
+ offs.push(
594
+ ctx.on('agent/turn-stopping', (payload) => {
595
+ try {
596
+ const sessionId = payload?.agent ? String(payload.agent.id) : null
597
+ if (!sessionId) return
598
+ const state = sessions.get(sessionId)
599
+ if (!state || state.windows.length === 0) return
600
+ for (const w of state.windows) {
601
+ w.turnEnded = true
602
+ persist(w, 'turn-end')
603
+ }
604
+ flushStep(sessionId, 'turn-end')
605
+ } catch {
606
+ // ignore
607
+ }
608
+ }),
609
+ )
610
+
611
+ return {
612
+ /** Await every write queued so far (used by tests and by the snapshot flush). */
613
+ flush() {
614
+ return store.queue
615
+ },
616
+ async dispose() {
617
+ for (const sessionId of [...stepStats.keys()]) flushStep(sessionId, 'dispose')
618
+ for (const state of sessions.values()) closeWindows(state, 'dispose')
619
+ sessions.clear()
620
+ steps.clear()
621
+ turns.clear()
622
+ seenInvocations.clear()
623
+ // let every queued write land before the caller tears the store down
624
+ await store.queue
625
+ for (const off of offs) {
626
+ try {
627
+ off()
628
+ } catch {
629
+ // ignore
630
+ }
631
+ }
632
+ },
633
+ }
634
+ }