dsh-skill-eval 0.0.0-stage → 1.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +718 -0
  2. package/LICENSE +21 -0
  3. package/README.md +207 -2
  4. package/README.zh.md +164 -0
  5. package/client.js +2791 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +82 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +468 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1173 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +557 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +141 -0
  26. package/lib/html.js +17 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +644 -0
  32. package/lib/render.js +580 -0
  33. package/lib/report.js +689 -0
  34. package/lib/score.js +565 -0
  35. package/lib/settings.js +415 -0
  36. package/lib/snapshot.js +250 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +240 -0
  39. package/lib/validity.js +125 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +227 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +234 -0
  47. package/scripts/lib.mjs +87 -0
  48. package/scripts/verify-live.mjs +178 -0
package/lib/report.js ADDED
@@ -0,0 +1,689 @@
1
+ import { EventStore, resolveDataDir, foldSamples } from './store.js'
2
+ import { DEFAULT_THRESHOLDS, DEFAULT_WEIGHTS, aggregate, gradeOf, scoreSkill, staticProfile } from './score.js'
3
+ import { buildAdvice, buildMcpAdvice, topAdvice } from './advice.js'
4
+ import { buildValidity } from './validity.js'
5
+ import { buildSteps, summarizeSteps } from './steps.js'
6
+ import { hashSeed, mulberry32 } from './random.js'
7
+
8
+ /**
9
+ * Bootstrap 95% interval over the recorded loads.
10
+ *
11
+ * The score is a point estimate over a handful of runs, so "88" and "88 ± 9"
12
+ * are very different claims. Resampling the loads (with replacement) and
13
+ * re-scoring gives an honest interval without assuming a distribution.
14
+ */
15
+ export function bootstrapCI(samples, compute, options = {}) {
16
+ const resamples = Number.isFinite(options.resamples) ? options.resamples : 200
17
+ const seed = Number.isFinite(options.seed) ? options.seed : 1
18
+ if (!Array.isArray(samples) || samples.length < 2 || resamples <= 0) return null
19
+ const random = mulberry32(seed)
20
+ const totals = []
21
+ for (let i = 0; i < resamples; i += 1) {
22
+ const draw = []
23
+ for (let j = 0; j < samples.length; j += 1) draw.push(samples[Math.floor(random() * samples.length)])
24
+ const value = compute(draw)
25
+ if (typeof value === 'number' && Number.isFinite(value)) totals.push(value)
26
+ }
27
+ if (totals.length < 20) return null
28
+ totals.sort((a, b) => a - b)
29
+ const at = (p) => totals[Math.min(totals.length - 1, Math.max(0, Math.round(p * (totals.length - 1))))]
30
+ return {
31
+ level: options.level ?? 0.95,
32
+ low: Math.round(at(0.025) * 10) / 10,
33
+ high: Math.round(at(0.975) * 10) / 10,
34
+ resamples: totals.length,
35
+ }
36
+ }
37
+
38
+ /**
39
+ * Two skills whose intervals overlap are not ordered by the data. Reporting a
40
+ * rank without saying that is how a "top fix" list turns into a wild goose
41
+ * chase, so the report carries an explicit ranking note.
42
+ */
43
+ export function rankingOf(entries) {
44
+ const scored = (entries ?? []).filter(
45
+ (entry) => entry && entry.score && entry.score.ci && typeof entry.score.total === 'number',
46
+ )
47
+ // Fewer than two intervals means nothing was compared; claiming a
48
+ // significant ranking here would be a fabricated statistic.
49
+ if (scored.length < 2)
50
+ return { significant: null, note: 'not enough scored skills with 95% intervals to rank them', ties: [] }
51
+ const ties = []
52
+ for (let i = 1; i < scored.length; i += 1) {
53
+ const a = scored[i - 1]
54
+ const b = scored[i]
55
+ const overlap = a.score.ci.low <= b.score.ci.high && b.score.ci.low <= a.score.ci.high
56
+ const close = Math.abs(a.score.total - b.score.total) <= 10
57
+ if (overlap && close) ties.push({ a: a.skill, b: b.skill, aTotal: a.score.total, bTotal: b.score.total })
58
+ }
59
+ if (ties.length === 0) return { significant: true, note: null, ties: [] }
60
+ const first = ties[0]
61
+ return {
62
+ significant: false,
63
+ ties,
64
+ note: `rank is not significant: "${first.a}" (${first.aTotal}) and "${first.b}" (${first.bTotal}) have overlapping 95% intervals — treat them as a tie and fix the evidence, not the order`,
65
+ noteZh: `排名不显著:“${first.a}”(${first.aTotal})与“${first.b}”(${first.bTotal})的 95% 区间重叠 —— 当作平手,去补证据,不要纠结顺序`,
66
+ }
67
+ }
68
+
69
+ export const EDIT_DIMENSIONS = ['static', 'reliability', 'effectiveness', 'efficiency', 'mcp', 'lifecycle']
70
+
71
+ /**
72
+ * RRSI's annealed edit budget: early iterations may carry a bundle of edits,
73
+ * later ones shrink to a single attributable change. Here "iteration" is the
74
+ * number of edits this skill already had accepted.
75
+ */
76
+ export function annealedBudget(accepted, { max = 4, min = 1, horizon = 6 } = {}) {
77
+ const hi = Math.max(1, Math.trunc(max))
78
+ const lo = Math.max(1, Math.min(hi, Math.trunc(min)))
79
+ const steps = Math.max(1, Math.trunc(horizon))
80
+ const progress = Math.min(1, Math.max(0, Number(accepted) || 0) / steps)
81
+ const value = Math.round(hi - (hi - lo) * ((1 - Math.cos(Math.PI * progress)) / 2))
82
+ return Math.max(lo, Math.min(hi, value))
83
+ }
84
+
85
+ /**
86
+ * Directional exploration: if the last edits all targeted one dimension, the
87
+ * next change should exercise a dimension the run has not touched.
88
+ */
89
+ export function explorationDirective(edits, { window = 2 } = {}) {
90
+ const recent = (edits ?? []).filter((entry) => entry && typeof entry.dimension === 'string')
91
+ if (recent.length < Math.max(2, window)) return null
92
+ const tail = recent.slice(-Math.max(2, window))
93
+ const stuck = tail[0].dimension
94
+ if (!tail.every((entry) => entry.dimension === stuck)) return null
95
+ const used = new Map()
96
+ for (const entry of recent) used.set(entry.dimension, (used.get(entry.dimension) ?? 0) + 1)
97
+ const suggestion = EDIT_DIMENSIONS.filter((dimension) => dimension !== stuck).sort(
98
+ (a, b) => (used.get(a) ?? 0) - (used.get(b) ?? 0),
99
+ )[0]
100
+ return {
101
+ stuck,
102
+ suggestion,
103
+ reason: `last ${tail.length} reviewed edit(s) all targeted "${stuck}"`,
104
+ }
105
+ }
106
+
107
+ /** Accept any positive relative weights (0-1 or percent) and normalise them. */
108
+ export function normalizeWeights(input) {
109
+ const raw = { ...DEFAULT_WEIGHTS, ...(input && typeof input === 'object' ? input : {}) }
110
+ const cleaned = {}
111
+ let sum = 0
112
+ for (const key of Object.keys(DEFAULT_WEIGHTS)) {
113
+ const value = Number(raw[key])
114
+ const weight = Number.isFinite(value) && value > 0 ? value : 0
115
+ cleaned[key] = weight
116
+ sum += weight
117
+ }
118
+ if (!(sum > 0)) return { ...DEFAULT_WEIGHTS }
119
+ const out = {}
120
+ for (const key of Object.keys(cleaned)) out[key] = Math.round((cleaned[key] / sum) * 1000) / 1000
121
+ return out
122
+ }
123
+
124
+ /**
125
+ * Display weights as integers that always sum to 100. Independent rounding can
126
+ * show 29+29+29+14=101 for a legal 2/2/2/1 configuration, which then fails the
127
+ * release check; largest-remainder keeps the displayed total honest.
128
+ */
129
+ export function percentWeights(weights) {
130
+ const keys = Object.keys(weights ?? {})
131
+ if (keys.length === 0) return {}
132
+ const exact = keys.map((key) => Math.max(0, Number(weights[key]) || 0) * 100)
133
+ const base = exact.map((value) => Math.floor(value))
134
+ let remainder = 100 - base.reduce((sum, value) => sum + value, 0)
135
+ const order = exact
136
+ .map((value, index) => ({ index, frac: value - Math.floor(value) }))
137
+ .sort((a, b) => b.frac - a.frac)
138
+ for (const item of order) {
139
+ if (remainder <= 0) break
140
+ base[item.index] += 1
141
+ remainder -= 1
142
+ }
143
+ while (remainder > 0) {
144
+ let largest = 0
145
+ for (let i = 1; i < base.length; i += 1) if (base[i] > base[largest]) largest = i
146
+ base[largest] += 1
147
+ remainder -= 1
148
+ }
149
+ return Object.fromEntries(keys.map((key, index) => [key, base[index]]))
150
+ }
151
+
152
+ export function resolveThresholds(input) {
153
+ const raw = { ...DEFAULT_THRESHOLDS, ...(input && typeof input === 'object' ? input : {}) }
154
+ const pick = (key, fallback) => {
155
+ const value = Number(raw[key])
156
+ return Number.isFinite(value) && value > 0 && value <= 100 ? value : fallback
157
+ }
158
+ const thresholds = {
159
+ excellent: pick('excellent', DEFAULT_THRESHOLDS.excellent),
160
+ good: pick('good', DEFAULT_THRESHOLDS.good),
161
+ pass: pick('pass', DEFAULT_THRESHOLDS.pass),
162
+ }
163
+ // Sort the three floors rather than clamping only the ends: {90, 50, 70} must
164
+ // become A90/B70/C50, not collapse B and C onto the same value.
165
+ const bands = [thresholds.excellent, thresholds.good, thresholds.pass].sort((a, b) => b - a)
166
+ thresholds.excellent = bands[0]
167
+ thresholds.good = bands[1]
168
+ thresholds.pass = bands[2]
169
+ return thresholds
170
+ }
171
+
172
+ /**
173
+ * Ties the append-only store to the live skill catalog.
174
+ *
175
+ * The store is the only durable state; everything here is derived on demand.
176
+ * Static profiles are cached and invalidated on `skills/change`.
177
+ */
178
+ export function createAnalyzer(ctx, config) {
179
+ const store = new EventStore(resolveDataDir(config))
180
+ const staticCache = new Map()
181
+ let reportCache = null
182
+
183
+ const invalidate = () => {
184
+ staticCache.clear()
185
+ reportCache = null
186
+ }
187
+ const offSkillsChange = ctx.on('skills/change', invalidate)
188
+
189
+ /**
190
+ * Prefer the live catalog (fresh after an edit) and fall back to the static
191
+ * profile captured at load time, which is the only path that knows the
192
+ * agent-scoped layer the skill was resolved from.
193
+ */
194
+ async function catalogStatic(name) {
195
+ if (staticCache.has(name)) return staticCache.get(name)
196
+ let profile = null
197
+ for (const options of [{}, { cwd: process.cwd() }]) {
198
+ try {
199
+ const summary = (await ctx.skills.list(options)).find((s) => s.name === name)
200
+ if (!summary) continue
201
+ const definition = await ctx.skills.get(name, options)
202
+ profile = staticProfile(summary, definition)
203
+ break
204
+ } catch {
205
+ profile = null
206
+ }
207
+ }
208
+ staticCache.set(name, profile)
209
+ return profile
210
+ }
211
+
212
+ function recordedStatic(records, name) {
213
+ let latest = null
214
+ for (const record of records) {
215
+ if (!record || record.t !== 'static' || record.skill !== name || !record.static) continue
216
+ if (!latest || (Number(record.at) || 0) >= (Number(latest.at) || 0)) latest = record
217
+ }
218
+ if (!latest || !latest.static || typeof latest.static !== 'object' || Array.isArray(latest.static)) return null
219
+ // A hand-edited or older store may hold any shape here; normalise it so one
220
+ // bad record cannot crash advice/report rendering for every skill.
221
+ const text = (value) =>
222
+ typeof value === 'string' ? value : value === null || value === undefined ? '' : String(value)
223
+ const num = (value, fallback = 0) => {
224
+ const n = Number(value)
225
+ return Number.isFinite(n) && n >= 0 ? n : fallback
226
+ }
227
+ const check = (entry) =>
228
+ entry && typeof entry === 'object'
229
+ ? { ...entry, fraction: num(entry.fraction), points: num(entry.points), weight: num(entry.weight) }
230
+ : null
231
+ const checks = (Array.isArray(latest.static.checks) ? latest.static.checks : []).map(check).filter(Boolean)
232
+ const failedChecks = (
233
+ Array.isArray(latest.static.failedChecks) ? latest.static.failedChecks : checks.filter((c) => c.fraction < 1)
234
+ )
235
+ .map(check)
236
+ .filter(Boolean)
237
+ const integrity =
238
+ latest.static.integrity && typeof latest.static.integrity === 'object'
239
+ ? {
240
+ checked: num(latest.static.integrity.checked),
241
+ missing: Array.isArray(latest.static.integrity.missing)
242
+ ? latest.static.integrity.missing.filter((item) => typeof item === 'string')
243
+ : [],
244
+ }
245
+ : null
246
+ return {
247
+ name: text(latest.static.name),
248
+ description: text(latest.static.description),
249
+ whenToUse: text(latest.static.whenToUse),
250
+ bodyLines: num(latest.static.bodyLines),
251
+ sectionCount: num(latest.static.sectionCount),
252
+ source: latest.static.source ?? null,
253
+ provider: latest.static.provider ?? null,
254
+ path: latest.static.path ?? null,
255
+ resolved: latest.static.resolved === true,
256
+ integrity,
257
+ checks,
258
+ score: Number.isFinite(Number(latest.static.score)) ? Number(latest.static.score) : null,
259
+ failedChecks,
260
+ capturedAt: latest.at,
261
+ origin: 'recorded',
262
+ }
263
+ }
264
+
265
+ async function resolveStatic(name, records = []) {
266
+ const profile = (await catalogStatic(name)) ?? recordedStatic(records, name)
267
+ if (!profile || typeof profile !== 'object' || !('content' in profile)) return profile
268
+ // old static records embedded the whole body; never surface it again
269
+ const { content: _body, ...rest } = profile
270
+ return rest
271
+ }
272
+
273
+ async function build(options = {}) {
274
+ // The panel polls and each turn strip asks once; reuse the last report while
275
+ // the store has not changed (EventStore.version) and settings are unchanged.
276
+ const stamp = await store.stamp()
277
+ if (!options.skill && reportCache && reportCache.stamp === stamp && Date.now() - reportCache.at < 2000) {
278
+ return reportCache.report
279
+ }
280
+ const records = await store.readAll()
281
+ const folded = foldSamples(records)
282
+ const weights = normalizeWeights(config.weights)
283
+ const thresholds = resolveThresholds(config.thresholds)
284
+ const minSamples = config.minSamples ?? 3
285
+
286
+ const editsChronological = records.filter((record) => record && record.t === 'edit')
287
+ const edits = [...editsChronological].reverse().slice(0, 30)
288
+ const entries = []
289
+ for (const group of folded) {
290
+ if (options.skill && options.skill !== group.skill) continue
291
+ const cacheReadWeight = config.cacheReadWeight ?? 0
292
+ const tokenBudgetPerLoad = config.tokenBudgetPerLoad ?? 20000
293
+ const stats = aggregate(group.samples, { cacheReadWeight })
294
+ const sp = await resolveStatic(group.skill, records)
295
+ // The point estimate and every bootstrap draw must use the same scoring
296
+ // options, or the interval can exclude the score it belongs to.
297
+ const scoreSamples = (samples) =>
298
+ scoreSkill({
299
+ stats: aggregate(samples, { cacheReadWeight }),
300
+ static: sp,
301
+ weights,
302
+ thresholds,
303
+ minSamples,
304
+ tokenBudgetPerLoad,
305
+ })
306
+ const score = scoreSamples(group.samples)
307
+ score.ci = bootstrapCI(group.samples, (draw) => scoreSamples(draw).total, {
308
+ resamples: config.ciResamples ?? 200,
309
+ seed: hashSeed(group.skill),
310
+ })
311
+ // Last few per-load scores for the panel sparkline. One sample is scored
312
+ // with the same options as the aggregate; capped so the payload stays small.
313
+ const trend = group.samples
314
+ .slice(-12)
315
+ .map((sample) => {
316
+ const point = scoreSamples([sample])
317
+ return Number.isFinite(point.total)
318
+ ? {
319
+ at: Number.isFinite(Number(sample.at)) ? Number(sample.at) : null,
320
+ total: Math.round(point.total * 10) / 10,
321
+ }
322
+ : null
323
+ })
324
+ .filter(Boolean)
325
+ const windows = group.samples.map((sample) => sample.window).filter(Boolean)
326
+ const advice = buildAdvice({
327
+ stats,
328
+ static: sp,
329
+ score,
330
+ minSamples,
331
+ windows,
332
+ gating: config.adviceNoiseFloor !== false,
333
+ tokenBudgetPerLoad: config.tokenBudgetPerLoad ?? 20000,
334
+ pruneMinLoads: config.pruneMinLoads ?? 6,
335
+ pruneMaxScore: config.pruneMaxScore ?? 60,
336
+ })
337
+ const skillEdits = editsChronological.filter((record) => record.skill === group.skill)
338
+ const accepted = skillEdits.filter((record) => record.verdict === 'accept').length
339
+ const adviceBudget = annealedBudget(accepted, {
340
+ max: config.editBudgetMax ?? 4,
341
+ min: config.editBudgetMin ?? 1,
342
+ horizon: config.editBudgetHorizon ?? 6,
343
+ })
344
+ const exploration = explorationDirective(skillEdits, { window: config.noveltyWindow ?? 2 })
345
+ entries.push({
346
+ skill: group.skill,
347
+ adviceBudget,
348
+ exploration,
349
+ source: sp?.source ?? null,
350
+ firstAt: group.firstAt,
351
+ lastAt: group.lastAt,
352
+ stats,
353
+ staticQuality: sp,
354
+ score,
355
+ trend,
356
+ advice,
357
+ topAdvice: topAdvice(advice, Math.min(config.maxAdvice ?? 3, adviceBudget)),
358
+ })
359
+ }
360
+ entries.sort((a, b) => a.score.total - b.score.total)
361
+ const skillScores = new Map(entries.map((entry) => [entry.skill, entry.score.total]))
362
+ const ranking = rankingOf(entries)
363
+ const mcpRows = mcpServers(entries)
364
+ const mcpAdvice = buildMcpAdvice(mcpRows, {
365
+ errorRate: typeof config.mcpErrorRatePercent === 'number' ? config.mcpErrorRatePercent / 100 : undefined,
366
+ toolErrorRate:
367
+ typeof config.mcpToolErrorRatePercent === 'number' ? config.mcpToolErrorRatePercent / 100 : undefined,
368
+ p95Ms: typeof config.mcpP95Ms === 'number' ? config.mcpP95Ms : undefined,
369
+ toolP95Ms: typeof config.mcpToolP95Ms === 'number' ? config.mcpToolP95Ms : undefined,
370
+ })
371
+
372
+ const stepLog = buildSteps(records, { thresholds, limit: 800 })
373
+ const stepView = summarizeSteps(stepLog, { limit: 12 })
374
+ const report = {
375
+ generatedAt: new Date().toISOString(),
376
+ dataDir: store.dir,
377
+ mcpServers: mcpRows,
378
+ mcpAdvice,
379
+ // The alert line the board draws on the latency chart must be the same
380
+ // threshold the advice uses, otherwise the chart celebrates a server the
381
+ // advice calls slow (or the reverse).
382
+ mcp: {
383
+ p95Ms: typeof config.mcpP95Ms === 'number' ? config.mcpP95Ms : null,
384
+ errorRatePercent: typeof config.mcpErrorRatePercent === 'number' ? config.mcpErrorRatePercent : null,
385
+ },
386
+ runs: runsOf(records, skillScores, thresholds, minSamples, config.ciResamples ?? 200),
387
+ turnLoads: turnLoads(records),
388
+ steps: stepLog.slice(-400),
389
+ stepTurns: stepView.turns,
390
+ stepOverall: stepView.overall,
391
+ edits,
392
+ validity: buildValidity(
393
+ {
394
+ skills: entries,
395
+ edits,
396
+ scoring: { minSamples: config.minSamples ?? 3, resamples: config.ciResamples ?? 200 },
397
+ ranking,
398
+ },
399
+ { minSamples: config.minSamples ?? 3 },
400
+ ),
401
+ ranking,
402
+ scoring: {
403
+ weights,
404
+ weightsPercent: percentWeights(weights),
405
+ thresholds,
406
+ minSamples,
407
+ editBudgetMax: config.editBudgetMax ?? 4,
408
+ customizable: true,
409
+ },
410
+ totals: {
411
+ invocations: records.filter((r) => r && r.t === 'skill').length,
412
+ skills: entries.length,
413
+ closed: entries.reduce((n, e) => n + e.stats.closed, 0),
414
+ open: entries.reduce((n, e) => n + e.stats.open, 0),
415
+ storeError: store.failed,
416
+ },
417
+ skills: entries,
418
+ }
419
+ if (!options.skill) reportCache = { stamp, at: Date.now(), report }
420
+ return report
421
+ }
422
+
423
+ /** One row per MCP server, merged across every skill window. */
424
+ function mcpServers(entries) {
425
+ const merged = new Map()
426
+ for (const entry of entries) {
427
+ for (const [server, bucket] of Object.entries(entry.stats.mcp ?? {})) {
428
+ const row = merged.get(server) ?? {
429
+ server,
430
+ calls: 0,
431
+ errors: 0,
432
+ ms: 0,
433
+ samples: [],
434
+ tools: Object.create(null),
435
+ }
436
+ row.calls += Number(bucket.calls) || 0
437
+ row.errors += Number(bucket.errors) || 0
438
+ row.ms += Number(bucket.ms) || 0
439
+ if (Array.isArray(bucket.samples)) {
440
+ for (const sample of bucket.samples) {
441
+ if (typeof sample === 'number' && row.samples.length < 400) row.samples.push(sample)
442
+ }
443
+ }
444
+ for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
445
+ if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
446
+ const t = Object.hasOwn(row.tools, tool) ? row.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
447
+ t.calls += Number(stats.calls) || 0
448
+ t.errors += Number(stats.errors) || 0
449
+ t.ms = (t.ms ?? 0) + (Number(stats.ms) || 0)
450
+ if (Array.isArray(stats.samples)) {
451
+ for (const sample of stats.samples) {
452
+ if (typeof sample === 'number' && t.samples.length < 60) t.samples.push(sample)
453
+ }
454
+ }
455
+ row.tools[tool] = t
456
+ }
457
+ merged.set(server, row)
458
+ }
459
+ }
460
+ const rows = [...merged.values()].map((row) => {
461
+ const errorRate = row.calls > 0 ? row.errors / row.calls : 0
462
+ const avgMs = row.calls > 0 ? row.ms / row.calls : null
463
+ const samples = Array.isArray(row.samples) ? [...row.samples].sort((a, b) => a - b) : []
464
+ const percentile = (p) => {
465
+ if (samples.length === 0) return null
466
+ const index = Math.min(samples.length - 1, Math.max(0, Math.round((p / 100) * (samples.length - 1))))
467
+ return samples[index]
468
+ }
469
+ let health = 100 - 70 * errorRate
470
+ if (avgMs !== null && avgMs > 1500) health -= Math.min(20, (avgMs - 1500) / 100)
471
+ health = Math.max(0, Math.min(100, health))
472
+ const toolPercentile = (samples, p) => {
473
+ if (!Array.isArray(samples) || samples.length === 0) return null
474
+ const sorted = [...samples].sort((a, b) => a - b)
475
+ const index = Math.min(sorted.length - 1, Math.max(0, Math.round((p / 100) * (sorted.length - 1))))
476
+ return sorted[index]
477
+ }
478
+ const tools = Object.entries(row.tools)
479
+ .map(([tool, stats]) => ({
480
+ tool,
481
+ calls: stats.calls,
482
+ errors: stats.errors,
483
+ errorRate: stats.calls > 0 ? stats.errors / stats.calls : 0,
484
+ avgMs: stats.calls > 0 && stats.ms ? Math.round(stats.ms / stats.calls) : null,
485
+ p50Ms: toolPercentile(stats.samples, 50),
486
+ p95Ms: toolPercentile(stats.samples, 95),
487
+ }))
488
+ .sort((a, b) => b.calls - a.calls)
489
+ return {
490
+ server: row.server,
491
+ calls: row.calls,
492
+ errors: row.errors,
493
+ errorRate,
494
+ avgMs: avgMs === null ? null : Math.round(avgMs),
495
+ p50Ms: percentile(50),
496
+ p95Ms: percentile(95),
497
+ samples: samples.length,
498
+ health: Math.round(health * 10) / 10,
499
+ grade: gradeOf(health),
500
+ tools,
501
+ }
502
+ })
503
+ rows.sort((a, b) => a.health - b.health)
504
+ return rows
505
+ }
506
+
507
+ function clamp01(value) {
508
+ return Math.max(0, Math.min(100, value))
509
+ }
510
+
511
+ /**
512
+ * Score one run (session) with its own dimensions: the quality of the skills
513
+ * it actually used, how stable the execution was, whether the user accepted
514
+ * the result, and what it cost. Weights differ from the skill score because a
515
+ * run is a sample of work, not an artifact.
516
+ */
517
+ function scoreRun(run, skillScores, thresholds, minSamples) {
518
+ const skillValues = run.skills.map((name) => skillScores.get(name)).filter((value) => typeof value === 'number')
519
+ const quality = skillValues.length > 0 ? skillValues.reduce((a, b) => a + b, 0) / skillValues.length : null
520
+ const windows = run.closed
521
+ const correctionRate = windows > 0 ? run.corrections / windows : null
522
+ const errRate = run.toolCalls > 0 ? run.errors / run.toolCalls : null
523
+ const loopRate = windows > 0 ? Math.min(1, run.loopErrors / windows) : null
524
+ const outcome = correctionRate === null ? null : clamp01(100 - 45 * correctionRate)
525
+ const stability =
526
+ errRate === null && loopRate === null ? null : clamp01(100 - 60 * (errRate ?? 0) - 40 * (loopRate ?? 0))
527
+ const callsPerLoad = run.loads > 0 ? run.toolCalls / run.loads : null
528
+ // A load with no execution behind it is cheap but empty: judge it down
529
+ // instead of rewarding "nothing happened" with full efficiency.
530
+ let efficiency = callsPerLoad === null ? null : clamp01(100 - Math.max(0, callsPerLoad - 8) * 6)
531
+ if (efficiency !== null && callsPerLoad < 1) efficiency = Math.min(efficiency, 45)
532
+ const dims = { quality, stability, outcome, efficiency }
533
+ const weights = { quality: 0.4, stability: 0.3, outcome: 0.2, efficiency: 0.1 }
534
+ let weighted = 0
535
+ let weightSum = 0
536
+ for (const [key, value] of Object.entries(dims)) {
537
+ if (typeof value !== 'number' || !Number.isFinite(value)) continue
538
+ weighted += value * weights[key]
539
+ weightSum += weights[key]
540
+ }
541
+ const total = weightSum > 0 ? Math.round((weighted / weightSum) * 10) / 10 : null
542
+ return {
543
+ total,
544
+ grade: gradeOf(total, thresholds),
545
+ confidence:
546
+ run.loads === 0 ? 'none' : run.loads < minSamples ? 'low' : run.loads < minSamples * 3 ? 'medium' : 'high',
547
+ provisional: run.loads < minSamples,
548
+ dimensions: dims,
549
+ }
550
+ }
551
+
552
+ /**
553
+ * Skill loads grouped by the loop turn that produced them, so a client surface
554
+ * can render the result under the exact completed turn instead of guessing.
555
+ */
556
+ function turnLoads(records) {
557
+ const byTurn = new Map()
558
+ for (const record of records) {
559
+ if (!record || record.t !== 'skill' || typeof record.sessionId !== 'string') continue
560
+ if (typeof record.turn !== 'number' || typeof record.skill !== 'string') continue
561
+ const key = `${record.sessionId}#${record.turn}`
562
+ const entry = byTurn.get(key) ?? {
563
+ sessionId: record.sessionId,
564
+ turn: record.turn,
565
+ skills: [],
566
+ firstAt: record.at,
567
+ lastAt: record.at,
568
+ }
569
+ if (!entry.skills.includes(record.skill)) entry.skills.push(record.skill)
570
+ entry.lastAt = Math.max(entry.lastAt ?? record.at, record.at)
571
+ byTurn.set(key, entry)
572
+ }
573
+ return [...byTurn.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0)).slice(0, 100)
574
+ }
575
+
576
+ /** One row per session: what that run actually did, and how it went. */
577
+ function runsOf(records, skillScores, thresholds, minSamples, ciResamples) {
578
+ const bySession = new Map()
579
+ const windowsBySession = new Map()
580
+ const touch = (sessionId) => {
581
+ if (typeof sessionId !== 'string' || sessionId.length === 0) return null
582
+ let run = bySession.get(sessionId)
583
+ if (!run) {
584
+ run = {
585
+ sessionId,
586
+ firstAt: null,
587
+ lastAt: null,
588
+ loads: 0,
589
+ closed: 0,
590
+ toolCalls: 0,
591
+ errors: 0,
592
+ corrections: 0,
593
+ mcpCalls: 0,
594
+ mcpErrors: 0,
595
+ tokens: 0,
596
+ loopErrors: 0,
597
+ skills: [],
598
+ score: null,
599
+ }
600
+ bySession.set(sessionId, run)
601
+ }
602
+ return run
603
+ }
604
+ for (const record of records) {
605
+ if (!record || typeof record !== 'object') continue
606
+ if (typeof record.sessionId !== 'string' || record.sessionId.length === 0) continue
607
+ if (record.t === 'skill') {
608
+ const run = touch(record.sessionId)
609
+ if (!run) continue
610
+ run.loads += 1
611
+ if (record.skill && !run.skills.includes(record.skill)) run.skills.push(record.skill)
612
+ run.firstAt = run.firstAt === null ? record.at : Math.min(run.firstAt, record.at)
613
+ run.lastAt = run.lastAt === null ? record.at : Math.max(run.lastAt, record.at)
614
+ } else if (record.t === 'window') {
615
+ touch(record.sessionId)
616
+ const bucket = windowsBySession.get(record.sessionId) ?? new Map()
617
+ // One window is persisted at turn end and again when it closes: keep the
618
+ // latest record per windowId so run counters and a bootstrap draw count
619
+ // it once.
620
+ bucket.set(record.windowId ?? `${record.at}:${bucket.size}`, record)
621
+ windowsBySession.set(record.sessionId, bucket)
622
+ }
623
+ }
624
+ // Sum the deduplicated, latest-per-window records, not every cumulative
625
+ // snapshot the recorder wrote.
626
+ for (const [sessionId, run] of bySession) {
627
+ const windows = [...(windowsBySession.get(sessionId) ?? new Map()).values()]
628
+ run.closed = windows.length
629
+ for (const window of windows) {
630
+ run.toolCalls += Number(window.toolCalls) || 0
631
+ run.errors += Number(window.errors) || 0
632
+ run.loopErrors += Number(window.loopErrors) || 0
633
+ run.corrections += Array.isArray(window.corrections) ? window.corrections.length : 0
634
+ const windowTokens = window.tokens
635
+ if (windowTokens && typeof windowTokens === 'object') {
636
+ run.tokens += (Number(windowTokens.input) || 0) + (Number(windowTokens.output) || 0)
637
+ }
638
+ for (const bucket of Object.values(window.mcp ?? {})) {
639
+ run.mcpCalls += Number(bucket?.calls) || 0
640
+ run.mcpErrors += Number(bucket?.errors) || 0
641
+ }
642
+ if (typeof window.at === 'number') {
643
+ run.lastAt = run.lastAt === null ? window.at : Math.max(run.lastAt, window.at)
644
+ }
645
+ }
646
+ }
647
+ const runs = [...bySession.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0)).slice(0, 20)
648
+ for (const run of runs) {
649
+ run.score = scoreRun(run, skillScores, thresholds, minSamples)
650
+ const windows = [...(windowsBySession.get(run.sessionId) ?? new Map()).values()]
651
+ run.score.ci = bootstrapCI(
652
+ windows,
653
+ (draw) => {
654
+ const summed = {
655
+ ...run,
656
+ closed: draw.length,
657
+ toolCalls: draw.reduce((a, w) => a + (w.toolCalls ?? 0), 0),
658
+ errors: draw.reduce((a, w) => a + (w.errors ?? 0), 0),
659
+ loopErrors: draw.reduce((a, w) => a + (w.loopErrors ?? 0), 0),
660
+ corrections: draw.reduce((a, w) => a + (Array.isArray(w.corrections) ? w.corrections.length : 0), 0),
661
+ }
662
+ return scoreRun(summed, skillScores, thresholds, minSamples).total
663
+ },
664
+ { resamples: ciResamples ?? 0, seed: hashSeed(run.sessionId) },
665
+ )
666
+ }
667
+ return runs
668
+ }
669
+
670
+ async function buildOne(name) {
671
+ const report = await build({ skill: name })
672
+ return report.skills[0] ?? null
673
+ }
674
+
675
+ return {
676
+ build,
677
+ buildOne,
678
+ store,
679
+ invalidate,
680
+ resolveStatic,
681
+ dispose() {
682
+ try {
683
+ if (typeof offSkillsChange === 'function') offSkillsChange()
684
+ } catch {
685
+ // best effort teardown
686
+ }
687
+ },
688
+ }
689
+ }