dsh-skill-eval 0.0.0-stage → 1.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +675 -0
  2. package/LICENSE +21 -0
  3. package/README.md +200 -2
  4. package/README.zh.md +157 -0
  5. package/client.js +2637 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +64 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +463 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1111 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +547 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +140 -0
  26. package/lib/html.js +8 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +634 -0
  32. package/lib/render.js +582 -0
  33. package/lib/report.js +646 -0
  34. package/lib/score.js +525 -0
  35. package/lib/settings.js +396 -0
  36. package/lib/snapshot.js +166 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +186 -0
  39. package/lib/validity.js +115 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +226 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +199 -0
  47. package/scripts/lib.mjs +52 -0
  48. package/scripts/verify-live.mjs +168 -0
package/lib/report.js ADDED
@@ -0,0 +1,646 @@
1
+ import { EventStore, resolveDataDir, foldSamples } from './store.js'
2
+ import { DEFAULT_THRESHOLDS, DEFAULT_WEIGHTS, aggregate, gradeOf, scoreSkill, staticProfile } from './score.js'
3
+ import { buildAdvice, buildMcpAdvice, topAdvice } from './advice.js'
4
+ import { buildValidity } from './validity.js'
5
+ import { buildSteps, summarizeSteps } from './steps.js'
6
+ import { hashSeed, mulberry32 } from './random.js'
7
+
8
+ /**
9
+ * Bootstrap 95% interval over the recorded loads.
10
+ *
11
+ * The score is a point estimate over a handful of runs, so "88" and "88 ± 9"
12
+ * are very different claims. Resampling the loads (with replacement) and
13
+ * re-scoring gives an honest interval without assuming a distribution.
14
+ */
15
+ export function bootstrapCI(samples, compute, options = {}) {
16
+ const resamples = Number.isFinite(options.resamples) ? options.resamples : 200
17
+ const seed = Number.isFinite(options.seed) ? options.seed : 1
18
+ if (!Array.isArray(samples) || samples.length < 2 || resamples <= 0) return null
19
+ const random = mulberry32(seed)
20
+ const totals = []
21
+ for (let i = 0; i < resamples; i += 1) {
22
+ const draw = []
23
+ for (let j = 0; j < samples.length; j += 1) draw.push(samples[Math.floor(random() * samples.length)])
24
+ const value = compute(draw)
25
+ if (typeof value === 'number' && Number.isFinite(value)) totals.push(value)
26
+ }
27
+ if (totals.length < 20) return null
28
+ totals.sort((a, b) => a - b)
29
+ const at = (p) => totals[Math.min(totals.length - 1, Math.max(0, Math.round(p * (totals.length - 1))))]
30
+ return {
31
+ level: options.level ?? 0.95,
32
+ low: Math.round(at(0.025) * 10) / 10,
33
+ high: Math.round(at(0.975) * 10) / 10,
34
+ resamples: totals.length,
35
+ }
36
+ }
37
+
38
+ /**
39
+ * Two skills whose intervals overlap are not ordered by the data. Reporting a
40
+ * rank without saying that is how a "top fix" list turns into a wild goose
41
+ * chase, so the report carries an explicit ranking note.
42
+ */
43
+ export function rankingOf(entries) {
44
+ const scored = (entries ?? []).filter(
45
+ (entry) => entry && entry.score && entry.score.ci && typeof entry.score.total === 'number',
46
+ )
47
+ if (scored.length < 2) return { significant: true, note: null, ties: [] }
48
+ const ties = []
49
+ for (let i = 1; i < scored.length; i += 1) {
50
+ const a = scored[i - 1]
51
+ const b = scored[i]
52
+ const overlap = a.score.ci.low <= b.score.ci.high && b.score.ci.low <= a.score.ci.high
53
+ const close = Math.abs(a.score.total - b.score.total) <= 10
54
+ if (overlap && close) ties.push({ a: a.skill, b: b.skill, aTotal: a.score.total, bTotal: b.score.total })
55
+ }
56
+ if (ties.length === 0) return { significant: true, note: null, ties: [] }
57
+ const first = ties[0]
58
+ return {
59
+ significant: false,
60
+ ties,
61
+ note: `rank is not significant: "${first.a}" (${first.aTotal}) and "${first.b}" (${first.bTotal}) have overlapping 95% intervals — treat them as a tie and fix the evidence, not the order`,
62
+ noteZh: `排名不显著:“${first.a}”(${first.aTotal})与“${first.b}”(${first.bTotal})的 95% 区间重叠 —— 当作平手,去补证据,不要纠结顺序`,
63
+ }
64
+ }
65
+
66
+ export const EDIT_DIMENSIONS = ['static', 'reliability', 'effectiveness', 'efficiency', 'mcp', 'lifecycle']
67
+
68
+ /**
69
+ * RRSI's annealed edit budget: early iterations may carry a bundle of edits,
70
+ * later ones shrink to a single attributable change. Here "iteration" is the
71
+ * number of edits this skill already had accepted.
72
+ */
73
+ export function annealedBudget(accepted, { max = 4, min = 1, horizon = 6 } = {}) {
74
+ const hi = Math.max(1, Math.trunc(max))
75
+ const lo = Math.max(1, Math.min(hi, Math.trunc(min)))
76
+ const steps = Math.max(1, Math.trunc(horizon))
77
+ const progress = Math.min(1, Math.max(0, Number(accepted) || 0) / steps)
78
+ const value = Math.round(hi - (hi - lo) * ((1 - Math.cos(Math.PI * progress)) / 2))
79
+ return Math.max(lo, Math.min(hi, value))
80
+ }
81
+
82
+ /**
83
+ * Directional exploration: if the last edits all targeted one dimension, the
84
+ * next change should exercise a dimension the run has not touched.
85
+ */
86
+ export function explorationDirective(edits, { window = 2 } = {}) {
87
+ const recent = (edits ?? []).filter((entry) => entry && typeof entry.dimension === 'string')
88
+ if (recent.length < Math.max(2, window)) return null
89
+ const tail = recent.slice(-Math.max(2, window))
90
+ const stuck = tail[0].dimension
91
+ if (!tail.every((entry) => entry.dimension === stuck)) return null
92
+ const used = new Map()
93
+ for (const entry of recent) used.set(entry.dimension, (used.get(entry.dimension) ?? 0) + 1)
94
+ const suggestion = EDIT_DIMENSIONS.filter((dimension) => dimension !== stuck).sort(
95
+ (a, b) => (used.get(a) ?? 0) - (used.get(b) ?? 0),
96
+ )[0]
97
+ return {
98
+ stuck,
99
+ suggestion,
100
+ reason: `last ${tail.length} reviewed edit(s) all targeted "${stuck}"`,
101
+ }
102
+ }
103
+
104
+ /** Accept any positive relative weights (0-1 or percent) and normalise them. */
105
+ export function normalizeWeights(input) {
106
+ const raw = { ...DEFAULT_WEIGHTS, ...(input && typeof input === 'object' ? input : {}) }
107
+ const cleaned = {}
108
+ let sum = 0
109
+ for (const key of Object.keys(DEFAULT_WEIGHTS)) {
110
+ const value = Number(raw[key])
111
+ const weight = Number.isFinite(value) && value > 0 ? value : 0
112
+ cleaned[key] = weight
113
+ sum += weight
114
+ }
115
+ if (!(sum > 0)) return { ...DEFAULT_WEIGHTS }
116
+ const out = {}
117
+ for (const key of Object.keys(cleaned)) out[key] = Math.round((cleaned[key] / sum) * 1000) / 1000
118
+ return out
119
+ }
120
+
121
+ export function resolveThresholds(input) {
122
+ const raw = { ...DEFAULT_THRESHOLDS, ...(input && typeof input === 'object' ? input : {}) }
123
+ const pick = (key, fallback) => {
124
+ const value = Number(raw[key])
125
+ return Number.isFinite(value) && value > 0 && value <= 100 ? value : fallback
126
+ }
127
+ const thresholds = {
128
+ excellent: pick('excellent', DEFAULT_THRESHOLDS.excellent),
129
+ good: pick('good', DEFAULT_THRESHOLDS.good),
130
+ pass: pick('pass', DEFAULT_THRESHOLDS.pass),
131
+ }
132
+ // Sort the three floors rather than clamping only the ends: {90, 50, 70} must
133
+ // become A90/B70/C50, not collapse B and C onto the same value.
134
+ const bands = [thresholds.excellent, thresholds.good, thresholds.pass].sort((a, b) => b - a)
135
+ thresholds.excellent = bands[0]
136
+ thresholds.good = bands[1]
137
+ thresholds.pass = bands[2]
138
+ return thresholds
139
+ }
140
+
141
+ /**
142
+ * Ties the append-only store to the live skill catalog.
143
+ *
144
+ * The store is the only durable state; everything here is derived on demand.
145
+ * Static profiles are cached and invalidated on `skills/change`.
146
+ */
147
+ export function createAnalyzer(ctx, config) {
148
+ const store = new EventStore(resolveDataDir(config))
149
+ const staticCache = new Map()
150
+ let reportCache = null
151
+
152
+ const invalidate = () => {
153
+ staticCache.clear()
154
+ reportCache = null
155
+ }
156
+ ctx.on('skills/change', invalidate)
157
+
158
+ /**
159
+ * Prefer the live catalog (fresh after an edit) and fall back to the static
160
+ * profile captured at load time, which is the only path that knows the
161
+ * agent-scoped layer the skill was resolved from.
162
+ */
163
+ async function catalogStatic(name) {
164
+ if (staticCache.has(name)) return staticCache.get(name)
165
+ let profile = null
166
+ for (const options of [{}, { cwd: process.cwd() }]) {
167
+ try {
168
+ const summary = (await ctx.skills.list(options)).find((s) => s.name === name)
169
+ if (!summary) continue
170
+ const definition = await ctx.skills.get(name, options)
171
+ profile = staticProfile(summary, definition)
172
+ break
173
+ } catch {
174
+ profile = null
175
+ }
176
+ }
177
+ staticCache.set(name, profile)
178
+ return profile
179
+ }
180
+
181
+ function recordedStatic(records, name) {
182
+ let latest = null
183
+ for (const record of records) {
184
+ if (!record || record.t !== 'static' || record.skill !== name || !record.static) continue
185
+ if (!latest || (Number(record.at) || 0) >= (Number(latest.at) || 0)) latest = record
186
+ }
187
+ if (!latest || !latest.static || typeof latest.static !== 'object' || Array.isArray(latest.static)) return null
188
+ // A hand-edited or older store may hold any shape here; normalise it so one
189
+ // bad record cannot crash advice/report rendering for every skill.
190
+ const text = (value) =>
191
+ typeof value === 'string' ? value : value === null || value === undefined ? '' : String(value)
192
+ const num = (value, fallback = 0) => {
193
+ const n = Number(value)
194
+ return Number.isFinite(n) && n >= 0 ? n : fallback
195
+ }
196
+ const check = (entry) =>
197
+ entry && typeof entry === 'object'
198
+ ? { ...entry, fraction: num(entry.fraction), points: num(entry.points), weight: num(entry.weight) }
199
+ : null
200
+ const checks = (Array.isArray(latest.static.checks) ? latest.static.checks : []).map(check).filter(Boolean)
201
+ const failedChecks = (
202
+ Array.isArray(latest.static.failedChecks) ? latest.static.failedChecks : checks.filter((c) => c.fraction < 1)
203
+ )
204
+ .map(check)
205
+ .filter(Boolean)
206
+ const integrity =
207
+ latest.static.integrity && typeof latest.static.integrity === 'object'
208
+ ? {
209
+ checked: num(latest.static.integrity.checked),
210
+ missing: Array.isArray(latest.static.integrity.missing)
211
+ ? latest.static.integrity.missing.filter((item) => typeof item === 'string')
212
+ : [],
213
+ }
214
+ : null
215
+ return {
216
+ name: text(latest.static.name),
217
+ description: text(latest.static.description),
218
+ whenToUse: text(latest.static.whenToUse),
219
+ bodyLines: num(latest.static.bodyLines),
220
+ sectionCount: num(latest.static.sectionCount),
221
+ source: latest.static.source ?? null,
222
+ provider: latest.static.provider ?? null,
223
+ path: latest.static.path ?? null,
224
+ resolved: latest.static.resolved === true,
225
+ integrity,
226
+ checks,
227
+ score: Number.isFinite(Number(latest.static.score)) ? Number(latest.static.score) : null,
228
+ failedChecks,
229
+ capturedAt: latest.at,
230
+ origin: 'recorded',
231
+ }
232
+ }
233
+
234
+ async function resolveStatic(name, records = []) {
235
+ const profile = (await catalogStatic(name)) ?? recordedStatic(records, name)
236
+ if (!profile || typeof profile !== 'object' || !('content' in profile)) return profile
237
+ // old static records embedded the whole body; never surface it again
238
+ const { content: _body, ...rest } = profile
239
+ return rest
240
+ }
241
+
242
+ async function build(options = {}) {
243
+ // The panel polls and each turn strip asks once; reuse the last report while
244
+ // the store has not changed (EventStore.version) and settings are unchanged.
245
+ const stamp = await store.stamp()
246
+ if (!options.skill && reportCache && reportCache.stamp === stamp && Date.now() - reportCache.at < 2000) {
247
+ return reportCache.report
248
+ }
249
+ const records = await store.readAll()
250
+ const folded = foldSamples(records)
251
+ const weights = normalizeWeights(config.weights)
252
+ const thresholds = resolveThresholds(config.thresholds)
253
+ const minSamples = config.minSamples ?? 3
254
+
255
+ const editsChronological = records.filter((record) => record && record.t === 'edit')
256
+ const edits = [...editsChronological].reverse().slice(0, 30)
257
+ const entries = []
258
+ for (const group of folded) {
259
+ if (options.skill && options.skill !== group.skill) continue
260
+ const cacheReadWeight = config.cacheReadWeight ?? 0
261
+ const tokenBudgetPerLoad = config.tokenBudgetPerLoad ?? 20000
262
+ const stats = aggregate(group.samples, { cacheReadWeight })
263
+ const sp = await resolveStatic(group.skill, records)
264
+ // The point estimate and every bootstrap draw must use the same scoring
265
+ // options, or the interval can exclude the score it belongs to.
266
+ const scoreSamples = (samples) =>
267
+ scoreSkill({
268
+ stats: aggregate(samples, { cacheReadWeight }),
269
+ static: sp,
270
+ weights,
271
+ thresholds,
272
+ minSamples,
273
+ tokenBudgetPerLoad,
274
+ })
275
+ const score = scoreSamples(group.samples)
276
+ score.ci = bootstrapCI(group.samples, (draw) => scoreSamples(draw).total, {
277
+ resamples: config.ciResamples ?? 200,
278
+ seed: hashSeed(group.skill),
279
+ })
280
+ // Last few per-load scores for the panel sparkline. One sample is scored
281
+ // with the same options as the aggregate; capped so the payload stays small.
282
+ const trend = group.samples
283
+ .slice(-12)
284
+ .map((sample) => {
285
+ const point = scoreSamples([sample])
286
+ return Number.isFinite(point.total)
287
+ ? {
288
+ at: Number.isFinite(Number(sample.at)) ? Number(sample.at) : null,
289
+ total: Math.round(point.total * 10) / 10,
290
+ }
291
+ : null
292
+ })
293
+ .filter(Boolean)
294
+ const windows = group.samples.map((sample) => sample.window).filter(Boolean)
295
+ const advice = buildAdvice({
296
+ stats,
297
+ static: sp,
298
+ score,
299
+ minSamples,
300
+ windows,
301
+ gating: config.adviceNoiseFloor !== false,
302
+ tokenBudgetPerLoad: config.tokenBudgetPerLoad ?? 20000,
303
+ pruneMinLoads: config.pruneMinLoads ?? 6,
304
+ pruneMaxScore: config.pruneMaxScore ?? 60,
305
+ })
306
+ const skillEdits = editsChronological.filter((record) => record.skill === group.skill)
307
+ const accepted = skillEdits.filter((record) => record.verdict === 'accept').length
308
+ const adviceBudget = annealedBudget(accepted, {
309
+ max: config.editBudgetMax ?? 4,
310
+ min: config.editBudgetMin ?? 1,
311
+ horizon: config.editBudgetHorizon ?? 6,
312
+ })
313
+ const exploration = explorationDirective(skillEdits, { window: config.noveltyWindow ?? 2 })
314
+ entries.push({
315
+ skill: group.skill,
316
+ adviceBudget,
317
+ exploration,
318
+ source: sp?.source ?? null,
319
+ firstAt: group.firstAt,
320
+ lastAt: group.lastAt,
321
+ stats,
322
+ staticQuality: sp,
323
+ score,
324
+ trend,
325
+ advice,
326
+ topAdvice: topAdvice(advice, Math.min(config.maxAdvice ?? 3, adviceBudget)),
327
+ })
328
+ }
329
+ entries.sort((a, b) => a.score.total - b.score.total)
330
+ const skillScores = new Map(entries.map((entry) => [entry.skill, entry.score.total]))
331
+ const ranking = rankingOf(entries)
332
+ const mcpRows = mcpServers(entries)
333
+ const mcpAdvice = buildMcpAdvice(mcpRows, {
334
+ errorRate: typeof config.mcpErrorRatePercent === 'number' ? config.mcpErrorRatePercent / 100 : undefined,
335
+ toolErrorRate:
336
+ typeof config.mcpToolErrorRatePercent === 'number' ? config.mcpToolErrorRatePercent / 100 : undefined,
337
+ p95Ms: typeof config.mcpP95Ms === 'number' ? config.mcpP95Ms : undefined,
338
+ toolP95Ms: typeof config.mcpToolP95Ms === 'number' ? config.mcpToolP95Ms : undefined,
339
+ })
340
+
341
+ const stepLog = buildSteps(records, { thresholds, limit: 800 })
342
+ const stepView = summarizeSteps(stepLog, { limit: 12 })
343
+ const report = {
344
+ generatedAt: new Date().toISOString(),
345
+ dataDir: store.dir,
346
+ mcpServers: mcpRows,
347
+ mcpAdvice,
348
+ // The alert line the board draws on the latency chart must be the same
349
+ // threshold the advice uses, otherwise the chart celebrates a server the
350
+ // advice calls slow (or the reverse).
351
+ mcp: {
352
+ p95Ms: typeof config.mcpP95Ms === 'number' ? config.mcpP95Ms : null,
353
+ errorRatePercent: typeof config.mcpErrorRatePercent === 'number' ? config.mcpErrorRatePercent : null,
354
+ },
355
+ runs: runsOf(records, skillScores, thresholds, minSamples, config.ciResamples ?? 200),
356
+ turnLoads: turnLoads(records),
357
+ steps: stepLog.slice(-400),
358
+ stepTurns: stepView.turns,
359
+ stepOverall: stepView.overall,
360
+ edits,
361
+ validity: buildValidity(
362
+ {
363
+ skills: entries,
364
+ edits,
365
+ scoring: { minSamples: config.minSamples ?? 3, resamples: config.ciResamples ?? 200 },
366
+ ranking,
367
+ },
368
+ { minSamples: config.minSamples ?? 3 },
369
+ ),
370
+ ranking,
371
+ scoring: {
372
+ weights,
373
+ weightsPercent: Object.fromEntries(
374
+ Object.entries(weights).map(([key, value]) => [key, Math.round(value * 100)]),
375
+ ),
376
+ thresholds,
377
+ minSamples,
378
+ editBudgetMax: config.editBudgetMax ?? 4,
379
+ customizable: true,
380
+ },
381
+ totals: {
382
+ invocations: records.filter((r) => r && r.t === 'skill').length,
383
+ skills: entries.length,
384
+ closed: entries.reduce((n, e) => n + e.stats.closed, 0),
385
+ open: entries.reduce((n, e) => n + e.stats.open, 0),
386
+ storeError: store.failed,
387
+ },
388
+ skills: entries,
389
+ }
390
+ if (!options.skill) reportCache = { stamp, at: Date.now(), report }
391
+ return report
392
+ }
393
+
394
+ /** One row per MCP server, merged across every skill window. */
395
+ function mcpServers(entries) {
396
+ const merged = new Map()
397
+ for (const entry of entries) {
398
+ for (const [server, bucket] of Object.entries(entry.stats.mcp ?? {})) {
399
+ const row = merged.get(server) ?? {
400
+ server,
401
+ calls: 0,
402
+ errors: 0,
403
+ ms: 0,
404
+ samples: [],
405
+ tools: Object.create(null),
406
+ }
407
+ row.calls += Number(bucket.calls) || 0
408
+ row.errors += Number(bucket.errors) || 0
409
+ row.ms += Number(bucket.ms) || 0
410
+ if (Array.isArray(bucket.samples)) {
411
+ for (const sample of bucket.samples) {
412
+ if (typeof sample === 'number' && row.samples.length < 400) row.samples.push(sample)
413
+ }
414
+ }
415
+ for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
416
+ const t = Object.hasOwn(row.tools, tool) ? row.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
417
+ t.calls += Number(stats.calls) || 0
418
+ t.errors += Number(stats.errors) || 0
419
+ t.ms = (t.ms ?? 0) + (Number(stats.ms) || 0)
420
+ if (Array.isArray(stats.samples)) {
421
+ for (const sample of stats.samples) {
422
+ if (typeof sample === 'number' && t.samples.length < 60) t.samples.push(sample)
423
+ }
424
+ }
425
+ row.tools[tool] = t
426
+ }
427
+ merged.set(server, row)
428
+ }
429
+ }
430
+ const rows = [...merged.values()].map((row) => {
431
+ const errorRate = row.calls > 0 ? row.errors / row.calls : 0
432
+ const avgMs = row.calls > 0 ? row.ms / row.calls : null
433
+ const samples = Array.isArray(row.samples) ? [...row.samples].sort((a, b) => a - b) : []
434
+ const percentile = (p) => {
435
+ if (samples.length === 0) return null
436
+ const index = Math.min(samples.length - 1, Math.max(0, Math.round((p / 100) * (samples.length - 1))))
437
+ return samples[index]
438
+ }
439
+ let health = 100 - 70 * errorRate
440
+ if (avgMs !== null && avgMs > 1500) health -= Math.min(20, (avgMs - 1500) / 100)
441
+ health = Math.max(0, Math.min(100, health))
442
+ const toolPercentile = (samples, p) => {
443
+ if (!Array.isArray(samples) || samples.length === 0) return null
444
+ const sorted = [...samples].sort((a, b) => a - b)
445
+ const index = Math.min(sorted.length - 1, Math.max(0, Math.round((p / 100) * (sorted.length - 1))))
446
+ return sorted[index]
447
+ }
448
+ const tools = Object.entries(row.tools)
449
+ .map(([tool, stats]) => ({
450
+ tool,
451
+ calls: stats.calls,
452
+ errors: stats.errors,
453
+ errorRate: stats.calls > 0 ? stats.errors / stats.calls : 0,
454
+ avgMs: stats.calls > 0 && stats.ms ? Math.round(stats.ms / stats.calls) : null,
455
+ p50Ms: toolPercentile(stats.samples, 50),
456
+ p95Ms: toolPercentile(stats.samples, 95),
457
+ }))
458
+ .sort((a, b) => b.calls - a.calls)
459
+ return {
460
+ server: row.server,
461
+ calls: row.calls,
462
+ errors: row.errors,
463
+ errorRate,
464
+ avgMs: avgMs === null ? null : Math.round(avgMs),
465
+ p50Ms: percentile(50),
466
+ p95Ms: percentile(95),
467
+ samples: samples.length,
468
+ health: Math.round(health * 10) / 10,
469
+ grade: gradeOf(health),
470
+ tools,
471
+ }
472
+ })
473
+ rows.sort((a, b) => a.health - b.health)
474
+ return rows
475
+ }
476
+
477
+ function clamp01(value) {
478
+ return Math.max(0, Math.min(100, value))
479
+ }
480
+
481
+ /**
482
+ * Score one run (session) with its own dimensions: the quality of the skills
483
+ * it actually used, how stable the execution was, whether the user accepted
484
+ * the result, and what it cost. Weights differ from the skill score because a
485
+ * run is a sample of work, not an artifact.
486
+ */
487
+ function scoreRun(run, skillScores, thresholds, minSamples) {
488
+ const skillValues = run.skills.map((name) => skillScores.get(name)).filter((value) => typeof value === 'number')
489
+ const quality = skillValues.length > 0 ? skillValues.reduce((a, b) => a + b, 0) / skillValues.length : null
490
+ const windows = run.closed
491
+ const correctionRate = windows > 0 ? run.corrections / windows : null
492
+ const errRate = run.toolCalls > 0 ? run.errors / run.toolCalls : null
493
+ const loopRate = windows > 0 ? Math.min(1, run.loopErrors / windows) : null
494
+ const outcome = correctionRate === null ? null : clamp01(100 - 45 * correctionRate)
495
+ const stability =
496
+ errRate === null && loopRate === null ? null : clamp01(100 - 60 * (errRate ?? 0) - 40 * (loopRate ?? 0))
497
+ const callsPerLoad = run.loads > 0 ? run.toolCalls / run.loads : null
498
+ // A load with no execution behind it is cheap but empty: judge it down
499
+ // instead of rewarding "nothing happened" with full efficiency.
500
+ let efficiency = callsPerLoad === null ? null : clamp01(100 - Math.max(0, callsPerLoad - 8) * 6)
501
+ if (efficiency !== null && callsPerLoad < 1) efficiency = Math.min(efficiency, 45)
502
+ const dims = { quality, stability, outcome, efficiency }
503
+ const weights = { quality: 0.4, stability: 0.3, outcome: 0.2, efficiency: 0.1 }
504
+ let weighted = 0
505
+ let weightSum = 0
506
+ for (const [key, value] of Object.entries(dims)) {
507
+ if (typeof value !== 'number' || !Number.isFinite(value)) continue
508
+ weighted += value * weights[key]
509
+ weightSum += weights[key]
510
+ }
511
+ const total = weightSum > 0 ? Math.round((weighted / weightSum) * 10) / 10 : null
512
+ return {
513
+ total,
514
+ grade: gradeOf(total, thresholds),
515
+ confidence:
516
+ run.loads === 0 ? 'none' : run.loads < minSamples ? 'low' : run.loads < minSamples * 3 ? 'medium' : 'high',
517
+ provisional: run.loads < minSamples,
518
+ dimensions: dims,
519
+ }
520
+ }
521
+
522
+ /**
523
+ * Skill loads grouped by the loop turn that produced them, so a client surface
524
+ * can render the result under the exact completed turn instead of guessing.
525
+ */
526
+ function turnLoads(records) {
527
+ const byTurn = new Map()
528
+ for (const record of records) {
529
+ if (!record || record.t !== 'skill' || typeof record.sessionId !== 'string') continue
530
+ if (typeof record.turn !== 'number' || typeof record.skill !== 'string') continue
531
+ const key = `${record.sessionId}#${record.turn}`
532
+ const entry = byTurn.get(key) ?? {
533
+ sessionId: record.sessionId,
534
+ turn: record.turn,
535
+ skills: [],
536
+ firstAt: record.at,
537
+ lastAt: record.at,
538
+ }
539
+ if (!entry.skills.includes(record.skill)) entry.skills.push(record.skill)
540
+ entry.lastAt = Math.max(entry.lastAt ?? record.at, record.at)
541
+ byTurn.set(key, entry)
542
+ }
543
+ return [...byTurn.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0)).slice(0, 100)
544
+ }
545
+
546
+ /** One row per session: what that run actually did, and how it went. */
547
+ function runsOf(records, skillScores, thresholds, minSamples, ciResamples) {
548
+ const bySession = new Map()
549
+ const windowsBySession = new Map()
550
+ const touch = (sessionId) => {
551
+ if (typeof sessionId !== 'string' || sessionId.length === 0) return null
552
+ let run = bySession.get(sessionId)
553
+ if (!run) {
554
+ run = {
555
+ sessionId,
556
+ firstAt: null,
557
+ lastAt: null,
558
+ loads: 0,
559
+ closed: 0,
560
+ toolCalls: 0,
561
+ errors: 0,
562
+ corrections: 0,
563
+ mcpCalls: 0,
564
+ mcpErrors: 0,
565
+ tokens: 0,
566
+ loopErrors: 0,
567
+ skills: [],
568
+ score: null,
569
+ }
570
+ bySession.set(sessionId, run)
571
+ }
572
+ return run
573
+ }
574
+ for (const record of records) {
575
+ if (!record || typeof record !== 'object') continue
576
+ if (typeof record.sessionId !== 'string' || record.sessionId.length === 0) continue
577
+ if (record.t === 'skill') {
578
+ const run = touch(record.sessionId)
579
+ if (!run) continue
580
+ run.loads += 1
581
+ if (record.skill && !run.skills.includes(record.skill)) run.skills.push(record.skill)
582
+ run.firstAt = run.firstAt === null ? record.at : Math.min(run.firstAt, record.at)
583
+ run.lastAt = run.lastAt === null ? record.at : Math.max(run.lastAt, record.at)
584
+ } else if (record.t === 'window') {
585
+ touch(record.sessionId)
586
+ const bucket = windowsBySession.get(record.sessionId) ?? new Map()
587
+ // One window is persisted at turn end and again when it closes: keep the
588
+ // latest record per windowId so run counters and a bootstrap draw count
589
+ // it once.
590
+ bucket.set(record.windowId ?? `${record.at}:${bucket.size}`, record)
591
+ windowsBySession.set(record.sessionId, bucket)
592
+ }
593
+ }
594
+ // Sum the deduplicated, latest-per-window records, not every cumulative
595
+ // snapshot the recorder wrote.
596
+ for (const [sessionId, run] of bySession) {
597
+ const windows = [...(windowsBySession.get(sessionId) ?? new Map()).values()]
598
+ run.closed = windows.length
599
+ for (const window of windows) {
600
+ run.toolCalls += Number(window.toolCalls) || 0
601
+ run.errors += Number(window.errors) || 0
602
+ run.loopErrors += Number(window.loopErrors) || 0
603
+ run.corrections += Array.isArray(window.corrections) ? window.corrections.length : 0
604
+ const windowTokens = window.tokens
605
+ if (windowTokens && typeof windowTokens === 'object') {
606
+ run.tokens += (Number(windowTokens.input) || 0) + (Number(windowTokens.output) || 0)
607
+ }
608
+ for (const bucket of Object.values(window.mcp ?? {})) {
609
+ run.mcpCalls += Number(bucket?.calls) || 0
610
+ run.mcpErrors += Number(bucket?.errors) || 0
611
+ }
612
+ if (typeof window.at === 'number') {
613
+ run.lastAt = run.lastAt === null ? window.at : Math.max(run.lastAt, window.at)
614
+ }
615
+ }
616
+ }
617
+ const runs = [...bySession.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0)).slice(0, 20)
618
+ for (const run of runs) {
619
+ run.score = scoreRun(run, skillScores, thresholds, minSamples)
620
+ const windows = [...(windowsBySession.get(run.sessionId) ?? new Map()).values()]
621
+ run.score.ci = bootstrapCI(
622
+ windows,
623
+ (draw) => {
624
+ const summed = {
625
+ ...run,
626
+ closed: draw.length,
627
+ toolCalls: draw.reduce((a, w) => a + (w.toolCalls ?? 0), 0),
628
+ errors: draw.reduce((a, w) => a + (w.errors ?? 0), 0),
629
+ loopErrors: draw.reduce((a, w) => a + (w.loopErrors ?? 0), 0),
630
+ corrections: draw.reduce((a, w) => a + (Array.isArray(w.corrections) ? w.corrections.length : 0), 0),
631
+ }
632
+ return scoreRun(summed, skillScores, thresholds, minSamples).total
633
+ },
634
+ { resamples: ciResamples ?? 0, seed: hashSeed(run.sessionId) },
635
+ )
636
+ }
637
+ return runs
638
+ }
639
+
640
+ async function buildOne(name) {
641
+ const report = await build({ skill: name })
642
+ return report.skills[0] ?? null
643
+ }
644
+
645
+ return { build, buildOne, store, invalidate, resolveStatic }
646
+ }