dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
package/lib/report.js
ADDED
|
@@ -0,0 +1,646 @@
|
|
|
1
|
+
import { EventStore, resolveDataDir, foldSamples } from './store.js'
|
|
2
|
+
import { DEFAULT_THRESHOLDS, DEFAULT_WEIGHTS, aggregate, gradeOf, scoreSkill, staticProfile } from './score.js'
|
|
3
|
+
import { buildAdvice, buildMcpAdvice, topAdvice } from './advice.js'
|
|
4
|
+
import { buildValidity } from './validity.js'
|
|
5
|
+
import { buildSteps, summarizeSteps } from './steps.js'
|
|
6
|
+
import { hashSeed, mulberry32 } from './random.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Bootstrap 95% interval over the recorded loads.
|
|
10
|
+
*
|
|
11
|
+
* The score is a point estimate over a handful of runs, so "88" and "88 ± 9"
|
|
12
|
+
* are very different claims. Resampling the loads (with replacement) and
|
|
13
|
+
* re-scoring gives an honest interval without assuming a distribution.
|
|
14
|
+
*/
|
|
15
|
+
export function bootstrapCI(samples, compute, options = {}) {
|
|
16
|
+
const resamples = Number.isFinite(options.resamples) ? options.resamples : 200
|
|
17
|
+
const seed = Number.isFinite(options.seed) ? options.seed : 1
|
|
18
|
+
if (!Array.isArray(samples) || samples.length < 2 || resamples <= 0) return null
|
|
19
|
+
const random = mulberry32(seed)
|
|
20
|
+
const totals = []
|
|
21
|
+
for (let i = 0; i < resamples; i += 1) {
|
|
22
|
+
const draw = []
|
|
23
|
+
for (let j = 0; j < samples.length; j += 1) draw.push(samples[Math.floor(random() * samples.length)])
|
|
24
|
+
const value = compute(draw)
|
|
25
|
+
if (typeof value === 'number' && Number.isFinite(value)) totals.push(value)
|
|
26
|
+
}
|
|
27
|
+
if (totals.length < 20) return null
|
|
28
|
+
totals.sort((a, b) => a - b)
|
|
29
|
+
const at = (p) => totals[Math.min(totals.length - 1, Math.max(0, Math.round(p * (totals.length - 1))))]
|
|
30
|
+
return {
|
|
31
|
+
level: options.level ?? 0.95,
|
|
32
|
+
low: Math.round(at(0.025) * 10) / 10,
|
|
33
|
+
high: Math.round(at(0.975) * 10) / 10,
|
|
34
|
+
resamples: totals.length,
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Two skills whose intervals overlap are not ordered by the data. Reporting a
|
|
40
|
+
* rank without saying that is how a "top fix" list turns into a wild goose
|
|
41
|
+
* chase, so the report carries an explicit ranking note.
|
|
42
|
+
*/
|
|
43
|
+
export function rankingOf(entries) {
|
|
44
|
+
const scored = (entries ?? []).filter(
|
|
45
|
+
(entry) => entry && entry.score && entry.score.ci && typeof entry.score.total === 'number',
|
|
46
|
+
)
|
|
47
|
+
if (scored.length < 2) return { significant: true, note: null, ties: [] }
|
|
48
|
+
const ties = []
|
|
49
|
+
for (let i = 1; i < scored.length; i += 1) {
|
|
50
|
+
const a = scored[i - 1]
|
|
51
|
+
const b = scored[i]
|
|
52
|
+
const overlap = a.score.ci.low <= b.score.ci.high && b.score.ci.low <= a.score.ci.high
|
|
53
|
+
const close = Math.abs(a.score.total - b.score.total) <= 10
|
|
54
|
+
if (overlap && close) ties.push({ a: a.skill, b: b.skill, aTotal: a.score.total, bTotal: b.score.total })
|
|
55
|
+
}
|
|
56
|
+
if (ties.length === 0) return { significant: true, note: null, ties: [] }
|
|
57
|
+
const first = ties[0]
|
|
58
|
+
return {
|
|
59
|
+
significant: false,
|
|
60
|
+
ties,
|
|
61
|
+
note: `rank is not significant: "${first.a}" (${first.aTotal}) and "${first.b}" (${first.bTotal}) have overlapping 95% intervals — treat them as a tie and fix the evidence, not the order`,
|
|
62
|
+
noteZh: `排名不显著:“${first.a}”(${first.aTotal})与“${first.b}”(${first.bTotal})的 95% 区间重叠 —— 当作平手,去补证据,不要纠结顺序`,
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export const EDIT_DIMENSIONS = ['static', 'reliability', 'effectiveness', 'efficiency', 'mcp', 'lifecycle']
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* RRSI's annealed edit budget: early iterations may carry a bundle of edits,
|
|
70
|
+
* later ones shrink to a single attributable change. Here "iteration" is the
|
|
71
|
+
* number of edits this skill already had accepted.
|
|
72
|
+
*/
|
|
73
|
+
export function annealedBudget(accepted, { max = 4, min = 1, horizon = 6 } = {}) {
|
|
74
|
+
const hi = Math.max(1, Math.trunc(max))
|
|
75
|
+
const lo = Math.max(1, Math.min(hi, Math.trunc(min)))
|
|
76
|
+
const steps = Math.max(1, Math.trunc(horizon))
|
|
77
|
+
const progress = Math.min(1, Math.max(0, Number(accepted) || 0) / steps)
|
|
78
|
+
const value = Math.round(hi - (hi - lo) * ((1 - Math.cos(Math.PI * progress)) / 2))
|
|
79
|
+
return Math.max(lo, Math.min(hi, value))
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Directional exploration: if the last edits all targeted one dimension, the
|
|
84
|
+
* next change should exercise a dimension the run has not touched.
|
|
85
|
+
*/
|
|
86
|
+
export function explorationDirective(edits, { window = 2 } = {}) {
|
|
87
|
+
const recent = (edits ?? []).filter((entry) => entry && typeof entry.dimension === 'string')
|
|
88
|
+
if (recent.length < Math.max(2, window)) return null
|
|
89
|
+
const tail = recent.slice(-Math.max(2, window))
|
|
90
|
+
const stuck = tail[0].dimension
|
|
91
|
+
if (!tail.every((entry) => entry.dimension === stuck)) return null
|
|
92
|
+
const used = new Map()
|
|
93
|
+
for (const entry of recent) used.set(entry.dimension, (used.get(entry.dimension) ?? 0) + 1)
|
|
94
|
+
const suggestion = EDIT_DIMENSIONS.filter((dimension) => dimension !== stuck).sort(
|
|
95
|
+
(a, b) => (used.get(a) ?? 0) - (used.get(b) ?? 0),
|
|
96
|
+
)[0]
|
|
97
|
+
return {
|
|
98
|
+
stuck,
|
|
99
|
+
suggestion,
|
|
100
|
+
reason: `last ${tail.length} reviewed edit(s) all targeted "${stuck}"`,
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Accept any positive relative weights (0-1 or percent) and normalise them. */
|
|
105
|
+
export function normalizeWeights(input) {
|
|
106
|
+
const raw = { ...DEFAULT_WEIGHTS, ...(input && typeof input === 'object' ? input : {}) }
|
|
107
|
+
const cleaned = {}
|
|
108
|
+
let sum = 0
|
|
109
|
+
for (const key of Object.keys(DEFAULT_WEIGHTS)) {
|
|
110
|
+
const value = Number(raw[key])
|
|
111
|
+
const weight = Number.isFinite(value) && value > 0 ? value : 0
|
|
112
|
+
cleaned[key] = weight
|
|
113
|
+
sum += weight
|
|
114
|
+
}
|
|
115
|
+
if (!(sum > 0)) return { ...DEFAULT_WEIGHTS }
|
|
116
|
+
const out = {}
|
|
117
|
+
for (const key of Object.keys(cleaned)) out[key] = Math.round((cleaned[key] / sum) * 1000) / 1000
|
|
118
|
+
return out
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function resolveThresholds(input) {
|
|
122
|
+
const raw = { ...DEFAULT_THRESHOLDS, ...(input && typeof input === 'object' ? input : {}) }
|
|
123
|
+
const pick = (key, fallback) => {
|
|
124
|
+
const value = Number(raw[key])
|
|
125
|
+
return Number.isFinite(value) && value > 0 && value <= 100 ? value : fallback
|
|
126
|
+
}
|
|
127
|
+
const thresholds = {
|
|
128
|
+
excellent: pick('excellent', DEFAULT_THRESHOLDS.excellent),
|
|
129
|
+
good: pick('good', DEFAULT_THRESHOLDS.good),
|
|
130
|
+
pass: pick('pass', DEFAULT_THRESHOLDS.pass),
|
|
131
|
+
}
|
|
132
|
+
// Sort the three floors rather than clamping only the ends: {90, 50, 70} must
|
|
133
|
+
// become A90/B70/C50, not collapse B and C onto the same value.
|
|
134
|
+
const bands = [thresholds.excellent, thresholds.good, thresholds.pass].sort((a, b) => b - a)
|
|
135
|
+
thresholds.excellent = bands[0]
|
|
136
|
+
thresholds.good = bands[1]
|
|
137
|
+
thresholds.pass = bands[2]
|
|
138
|
+
return thresholds
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Ties the append-only store to the live skill catalog.
|
|
143
|
+
*
|
|
144
|
+
* The store is the only durable state; everything here is derived on demand.
|
|
145
|
+
* Static profiles are cached and invalidated on `skills/change`.
|
|
146
|
+
*/
|
|
147
|
+
export function createAnalyzer(ctx, config) {
|
|
148
|
+
const store = new EventStore(resolveDataDir(config))
|
|
149
|
+
const staticCache = new Map()
|
|
150
|
+
let reportCache = null
|
|
151
|
+
|
|
152
|
+
const invalidate = () => {
|
|
153
|
+
staticCache.clear()
|
|
154
|
+
reportCache = null
|
|
155
|
+
}
|
|
156
|
+
ctx.on('skills/change', invalidate)
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Prefer the live catalog (fresh after an edit) and fall back to the static
|
|
160
|
+
* profile captured at load time, which is the only path that knows the
|
|
161
|
+
* agent-scoped layer the skill was resolved from.
|
|
162
|
+
*/
|
|
163
|
+
async function catalogStatic(name) {
|
|
164
|
+
if (staticCache.has(name)) return staticCache.get(name)
|
|
165
|
+
let profile = null
|
|
166
|
+
for (const options of [{}, { cwd: process.cwd() }]) {
|
|
167
|
+
try {
|
|
168
|
+
const summary = (await ctx.skills.list(options)).find((s) => s.name === name)
|
|
169
|
+
if (!summary) continue
|
|
170
|
+
const definition = await ctx.skills.get(name, options)
|
|
171
|
+
profile = staticProfile(summary, definition)
|
|
172
|
+
break
|
|
173
|
+
} catch {
|
|
174
|
+
profile = null
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
staticCache.set(name, profile)
|
|
178
|
+
return profile
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function recordedStatic(records, name) {
|
|
182
|
+
let latest = null
|
|
183
|
+
for (const record of records) {
|
|
184
|
+
if (!record || record.t !== 'static' || record.skill !== name || !record.static) continue
|
|
185
|
+
if (!latest || (Number(record.at) || 0) >= (Number(latest.at) || 0)) latest = record
|
|
186
|
+
}
|
|
187
|
+
if (!latest || !latest.static || typeof latest.static !== 'object' || Array.isArray(latest.static)) return null
|
|
188
|
+
// A hand-edited or older store may hold any shape here; normalise it so one
|
|
189
|
+
// bad record cannot crash advice/report rendering for every skill.
|
|
190
|
+
const text = (value) =>
|
|
191
|
+
typeof value === 'string' ? value : value === null || value === undefined ? '' : String(value)
|
|
192
|
+
const num = (value, fallback = 0) => {
|
|
193
|
+
const n = Number(value)
|
|
194
|
+
return Number.isFinite(n) && n >= 0 ? n : fallback
|
|
195
|
+
}
|
|
196
|
+
const check = (entry) =>
|
|
197
|
+
entry && typeof entry === 'object'
|
|
198
|
+
? { ...entry, fraction: num(entry.fraction), points: num(entry.points), weight: num(entry.weight) }
|
|
199
|
+
: null
|
|
200
|
+
const checks = (Array.isArray(latest.static.checks) ? latest.static.checks : []).map(check).filter(Boolean)
|
|
201
|
+
const failedChecks = (
|
|
202
|
+
Array.isArray(latest.static.failedChecks) ? latest.static.failedChecks : checks.filter((c) => c.fraction < 1)
|
|
203
|
+
)
|
|
204
|
+
.map(check)
|
|
205
|
+
.filter(Boolean)
|
|
206
|
+
const integrity =
|
|
207
|
+
latest.static.integrity && typeof latest.static.integrity === 'object'
|
|
208
|
+
? {
|
|
209
|
+
checked: num(latest.static.integrity.checked),
|
|
210
|
+
missing: Array.isArray(latest.static.integrity.missing)
|
|
211
|
+
? latest.static.integrity.missing.filter((item) => typeof item === 'string')
|
|
212
|
+
: [],
|
|
213
|
+
}
|
|
214
|
+
: null
|
|
215
|
+
return {
|
|
216
|
+
name: text(latest.static.name),
|
|
217
|
+
description: text(latest.static.description),
|
|
218
|
+
whenToUse: text(latest.static.whenToUse),
|
|
219
|
+
bodyLines: num(latest.static.bodyLines),
|
|
220
|
+
sectionCount: num(latest.static.sectionCount),
|
|
221
|
+
source: latest.static.source ?? null,
|
|
222
|
+
provider: latest.static.provider ?? null,
|
|
223
|
+
path: latest.static.path ?? null,
|
|
224
|
+
resolved: latest.static.resolved === true,
|
|
225
|
+
integrity,
|
|
226
|
+
checks,
|
|
227
|
+
score: Number.isFinite(Number(latest.static.score)) ? Number(latest.static.score) : null,
|
|
228
|
+
failedChecks,
|
|
229
|
+
capturedAt: latest.at,
|
|
230
|
+
origin: 'recorded',
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function resolveStatic(name, records = []) {
|
|
235
|
+
const profile = (await catalogStatic(name)) ?? recordedStatic(records, name)
|
|
236
|
+
if (!profile || typeof profile !== 'object' || !('content' in profile)) return profile
|
|
237
|
+
// old static records embedded the whole body; never surface it again
|
|
238
|
+
const { content: _body, ...rest } = profile
|
|
239
|
+
return rest
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async function build(options = {}) {
|
|
243
|
+
// The panel polls and each turn strip asks once; reuse the last report while
|
|
244
|
+
// the store has not changed (EventStore.version) and settings are unchanged.
|
|
245
|
+
const stamp = await store.stamp()
|
|
246
|
+
if (!options.skill && reportCache && reportCache.stamp === stamp && Date.now() - reportCache.at < 2000) {
|
|
247
|
+
return reportCache.report
|
|
248
|
+
}
|
|
249
|
+
const records = await store.readAll()
|
|
250
|
+
const folded = foldSamples(records)
|
|
251
|
+
const weights = normalizeWeights(config.weights)
|
|
252
|
+
const thresholds = resolveThresholds(config.thresholds)
|
|
253
|
+
const minSamples = config.minSamples ?? 3
|
|
254
|
+
|
|
255
|
+
const editsChronological = records.filter((record) => record && record.t === 'edit')
|
|
256
|
+
const edits = [...editsChronological].reverse().slice(0, 30)
|
|
257
|
+
const entries = []
|
|
258
|
+
for (const group of folded) {
|
|
259
|
+
if (options.skill && options.skill !== group.skill) continue
|
|
260
|
+
const cacheReadWeight = config.cacheReadWeight ?? 0
|
|
261
|
+
const tokenBudgetPerLoad = config.tokenBudgetPerLoad ?? 20000
|
|
262
|
+
const stats = aggregate(group.samples, { cacheReadWeight })
|
|
263
|
+
const sp = await resolveStatic(group.skill, records)
|
|
264
|
+
// The point estimate and every bootstrap draw must use the same scoring
|
|
265
|
+
// options, or the interval can exclude the score it belongs to.
|
|
266
|
+
const scoreSamples = (samples) =>
|
|
267
|
+
scoreSkill({
|
|
268
|
+
stats: aggregate(samples, { cacheReadWeight }),
|
|
269
|
+
static: sp,
|
|
270
|
+
weights,
|
|
271
|
+
thresholds,
|
|
272
|
+
minSamples,
|
|
273
|
+
tokenBudgetPerLoad,
|
|
274
|
+
})
|
|
275
|
+
const score = scoreSamples(group.samples)
|
|
276
|
+
score.ci = bootstrapCI(group.samples, (draw) => scoreSamples(draw).total, {
|
|
277
|
+
resamples: config.ciResamples ?? 200,
|
|
278
|
+
seed: hashSeed(group.skill),
|
|
279
|
+
})
|
|
280
|
+
// Last few per-load scores for the panel sparkline. One sample is scored
|
|
281
|
+
// with the same options as the aggregate; capped so the payload stays small.
|
|
282
|
+
const trend = group.samples
|
|
283
|
+
.slice(-12)
|
|
284
|
+
.map((sample) => {
|
|
285
|
+
const point = scoreSamples([sample])
|
|
286
|
+
return Number.isFinite(point.total)
|
|
287
|
+
? {
|
|
288
|
+
at: Number.isFinite(Number(sample.at)) ? Number(sample.at) : null,
|
|
289
|
+
total: Math.round(point.total * 10) / 10,
|
|
290
|
+
}
|
|
291
|
+
: null
|
|
292
|
+
})
|
|
293
|
+
.filter(Boolean)
|
|
294
|
+
const windows = group.samples.map((sample) => sample.window).filter(Boolean)
|
|
295
|
+
const advice = buildAdvice({
|
|
296
|
+
stats,
|
|
297
|
+
static: sp,
|
|
298
|
+
score,
|
|
299
|
+
minSamples,
|
|
300
|
+
windows,
|
|
301
|
+
gating: config.adviceNoiseFloor !== false,
|
|
302
|
+
tokenBudgetPerLoad: config.tokenBudgetPerLoad ?? 20000,
|
|
303
|
+
pruneMinLoads: config.pruneMinLoads ?? 6,
|
|
304
|
+
pruneMaxScore: config.pruneMaxScore ?? 60,
|
|
305
|
+
})
|
|
306
|
+
const skillEdits = editsChronological.filter((record) => record.skill === group.skill)
|
|
307
|
+
const accepted = skillEdits.filter((record) => record.verdict === 'accept').length
|
|
308
|
+
const adviceBudget = annealedBudget(accepted, {
|
|
309
|
+
max: config.editBudgetMax ?? 4,
|
|
310
|
+
min: config.editBudgetMin ?? 1,
|
|
311
|
+
horizon: config.editBudgetHorizon ?? 6,
|
|
312
|
+
})
|
|
313
|
+
const exploration = explorationDirective(skillEdits, { window: config.noveltyWindow ?? 2 })
|
|
314
|
+
entries.push({
|
|
315
|
+
skill: group.skill,
|
|
316
|
+
adviceBudget,
|
|
317
|
+
exploration,
|
|
318
|
+
source: sp?.source ?? null,
|
|
319
|
+
firstAt: group.firstAt,
|
|
320
|
+
lastAt: group.lastAt,
|
|
321
|
+
stats,
|
|
322
|
+
staticQuality: sp,
|
|
323
|
+
score,
|
|
324
|
+
trend,
|
|
325
|
+
advice,
|
|
326
|
+
topAdvice: topAdvice(advice, Math.min(config.maxAdvice ?? 3, adviceBudget)),
|
|
327
|
+
})
|
|
328
|
+
}
|
|
329
|
+
entries.sort((a, b) => a.score.total - b.score.total)
|
|
330
|
+
const skillScores = new Map(entries.map((entry) => [entry.skill, entry.score.total]))
|
|
331
|
+
const ranking = rankingOf(entries)
|
|
332
|
+
const mcpRows = mcpServers(entries)
|
|
333
|
+
const mcpAdvice = buildMcpAdvice(mcpRows, {
|
|
334
|
+
errorRate: typeof config.mcpErrorRatePercent === 'number' ? config.mcpErrorRatePercent / 100 : undefined,
|
|
335
|
+
toolErrorRate:
|
|
336
|
+
typeof config.mcpToolErrorRatePercent === 'number' ? config.mcpToolErrorRatePercent / 100 : undefined,
|
|
337
|
+
p95Ms: typeof config.mcpP95Ms === 'number' ? config.mcpP95Ms : undefined,
|
|
338
|
+
toolP95Ms: typeof config.mcpToolP95Ms === 'number' ? config.mcpToolP95Ms : undefined,
|
|
339
|
+
})
|
|
340
|
+
|
|
341
|
+
const stepLog = buildSteps(records, { thresholds, limit: 800 })
|
|
342
|
+
const stepView = summarizeSteps(stepLog, { limit: 12 })
|
|
343
|
+
const report = {
|
|
344
|
+
generatedAt: new Date().toISOString(),
|
|
345
|
+
dataDir: store.dir,
|
|
346
|
+
mcpServers: mcpRows,
|
|
347
|
+
mcpAdvice,
|
|
348
|
+
// The alert line the board draws on the latency chart must be the same
|
|
349
|
+
// threshold the advice uses, otherwise the chart celebrates a server the
|
|
350
|
+
// advice calls slow (or the reverse).
|
|
351
|
+
mcp: {
|
|
352
|
+
p95Ms: typeof config.mcpP95Ms === 'number' ? config.mcpP95Ms : null,
|
|
353
|
+
errorRatePercent: typeof config.mcpErrorRatePercent === 'number' ? config.mcpErrorRatePercent : null,
|
|
354
|
+
},
|
|
355
|
+
runs: runsOf(records, skillScores, thresholds, minSamples, config.ciResamples ?? 200),
|
|
356
|
+
turnLoads: turnLoads(records),
|
|
357
|
+
steps: stepLog.slice(-400),
|
|
358
|
+
stepTurns: stepView.turns,
|
|
359
|
+
stepOverall: stepView.overall,
|
|
360
|
+
edits,
|
|
361
|
+
validity: buildValidity(
|
|
362
|
+
{
|
|
363
|
+
skills: entries,
|
|
364
|
+
edits,
|
|
365
|
+
scoring: { minSamples: config.minSamples ?? 3, resamples: config.ciResamples ?? 200 },
|
|
366
|
+
ranking,
|
|
367
|
+
},
|
|
368
|
+
{ minSamples: config.minSamples ?? 3 },
|
|
369
|
+
),
|
|
370
|
+
ranking,
|
|
371
|
+
scoring: {
|
|
372
|
+
weights,
|
|
373
|
+
weightsPercent: Object.fromEntries(
|
|
374
|
+
Object.entries(weights).map(([key, value]) => [key, Math.round(value * 100)]),
|
|
375
|
+
),
|
|
376
|
+
thresholds,
|
|
377
|
+
minSamples,
|
|
378
|
+
editBudgetMax: config.editBudgetMax ?? 4,
|
|
379
|
+
customizable: true,
|
|
380
|
+
},
|
|
381
|
+
totals: {
|
|
382
|
+
invocations: records.filter((r) => r && r.t === 'skill').length,
|
|
383
|
+
skills: entries.length,
|
|
384
|
+
closed: entries.reduce((n, e) => n + e.stats.closed, 0),
|
|
385
|
+
open: entries.reduce((n, e) => n + e.stats.open, 0),
|
|
386
|
+
storeError: store.failed,
|
|
387
|
+
},
|
|
388
|
+
skills: entries,
|
|
389
|
+
}
|
|
390
|
+
if (!options.skill) reportCache = { stamp, at: Date.now(), report }
|
|
391
|
+
return report
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** One row per MCP server, merged across every skill window. */
|
|
395
|
+
function mcpServers(entries) {
|
|
396
|
+
const merged = new Map()
|
|
397
|
+
for (const entry of entries) {
|
|
398
|
+
for (const [server, bucket] of Object.entries(entry.stats.mcp ?? {})) {
|
|
399
|
+
const row = merged.get(server) ?? {
|
|
400
|
+
server,
|
|
401
|
+
calls: 0,
|
|
402
|
+
errors: 0,
|
|
403
|
+
ms: 0,
|
|
404
|
+
samples: [],
|
|
405
|
+
tools: Object.create(null),
|
|
406
|
+
}
|
|
407
|
+
row.calls += Number(bucket.calls) || 0
|
|
408
|
+
row.errors += Number(bucket.errors) || 0
|
|
409
|
+
row.ms += Number(bucket.ms) || 0
|
|
410
|
+
if (Array.isArray(bucket.samples)) {
|
|
411
|
+
for (const sample of bucket.samples) {
|
|
412
|
+
if (typeof sample === 'number' && row.samples.length < 400) row.samples.push(sample)
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
|
|
416
|
+
const t = Object.hasOwn(row.tools, tool) ? row.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
|
|
417
|
+
t.calls += Number(stats.calls) || 0
|
|
418
|
+
t.errors += Number(stats.errors) || 0
|
|
419
|
+
t.ms = (t.ms ?? 0) + (Number(stats.ms) || 0)
|
|
420
|
+
if (Array.isArray(stats.samples)) {
|
|
421
|
+
for (const sample of stats.samples) {
|
|
422
|
+
if (typeof sample === 'number' && t.samples.length < 60) t.samples.push(sample)
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
row.tools[tool] = t
|
|
426
|
+
}
|
|
427
|
+
merged.set(server, row)
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
const rows = [...merged.values()].map((row) => {
|
|
431
|
+
const errorRate = row.calls > 0 ? row.errors / row.calls : 0
|
|
432
|
+
const avgMs = row.calls > 0 ? row.ms / row.calls : null
|
|
433
|
+
const samples = Array.isArray(row.samples) ? [...row.samples].sort((a, b) => a - b) : []
|
|
434
|
+
const percentile = (p) => {
|
|
435
|
+
if (samples.length === 0) return null
|
|
436
|
+
const index = Math.min(samples.length - 1, Math.max(0, Math.round((p / 100) * (samples.length - 1))))
|
|
437
|
+
return samples[index]
|
|
438
|
+
}
|
|
439
|
+
let health = 100 - 70 * errorRate
|
|
440
|
+
if (avgMs !== null && avgMs > 1500) health -= Math.min(20, (avgMs - 1500) / 100)
|
|
441
|
+
health = Math.max(0, Math.min(100, health))
|
|
442
|
+
const toolPercentile = (samples, p) => {
|
|
443
|
+
if (!Array.isArray(samples) || samples.length === 0) return null
|
|
444
|
+
const sorted = [...samples].sort((a, b) => a - b)
|
|
445
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.round((p / 100) * (sorted.length - 1))))
|
|
446
|
+
return sorted[index]
|
|
447
|
+
}
|
|
448
|
+
const tools = Object.entries(row.tools)
|
|
449
|
+
.map(([tool, stats]) => ({
|
|
450
|
+
tool,
|
|
451
|
+
calls: stats.calls,
|
|
452
|
+
errors: stats.errors,
|
|
453
|
+
errorRate: stats.calls > 0 ? stats.errors / stats.calls : 0,
|
|
454
|
+
avgMs: stats.calls > 0 && stats.ms ? Math.round(stats.ms / stats.calls) : null,
|
|
455
|
+
p50Ms: toolPercentile(stats.samples, 50),
|
|
456
|
+
p95Ms: toolPercentile(stats.samples, 95),
|
|
457
|
+
}))
|
|
458
|
+
.sort((a, b) => b.calls - a.calls)
|
|
459
|
+
return {
|
|
460
|
+
server: row.server,
|
|
461
|
+
calls: row.calls,
|
|
462
|
+
errors: row.errors,
|
|
463
|
+
errorRate,
|
|
464
|
+
avgMs: avgMs === null ? null : Math.round(avgMs),
|
|
465
|
+
p50Ms: percentile(50),
|
|
466
|
+
p95Ms: percentile(95),
|
|
467
|
+
samples: samples.length,
|
|
468
|
+
health: Math.round(health * 10) / 10,
|
|
469
|
+
grade: gradeOf(health),
|
|
470
|
+
tools,
|
|
471
|
+
}
|
|
472
|
+
})
|
|
473
|
+
rows.sort((a, b) => a.health - b.health)
|
|
474
|
+
return rows
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function clamp01(value) {
|
|
478
|
+
return Math.max(0, Math.min(100, value))
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Score one run (session) with its own dimensions: the quality of the skills
|
|
483
|
+
* it actually used, how stable the execution was, whether the user accepted
|
|
484
|
+
* the result, and what it cost. Weights differ from the skill score because a
|
|
485
|
+
* run is a sample of work, not an artifact.
|
|
486
|
+
*/
|
|
487
|
+
function scoreRun(run, skillScores, thresholds, minSamples) {
|
|
488
|
+
const skillValues = run.skills.map((name) => skillScores.get(name)).filter((value) => typeof value === 'number')
|
|
489
|
+
const quality = skillValues.length > 0 ? skillValues.reduce((a, b) => a + b, 0) / skillValues.length : null
|
|
490
|
+
const windows = run.closed
|
|
491
|
+
const correctionRate = windows > 0 ? run.corrections / windows : null
|
|
492
|
+
const errRate = run.toolCalls > 0 ? run.errors / run.toolCalls : null
|
|
493
|
+
const loopRate = windows > 0 ? Math.min(1, run.loopErrors / windows) : null
|
|
494
|
+
const outcome = correctionRate === null ? null : clamp01(100 - 45 * correctionRate)
|
|
495
|
+
const stability =
|
|
496
|
+
errRate === null && loopRate === null ? null : clamp01(100 - 60 * (errRate ?? 0) - 40 * (loopRate ?? 0))
|
|
497
|
+
const callsPerLoad = run.loads > 0 ? run.toolCalls / run.loads : null
|
|
498
|
+
// A load with no execution behind it is cheap but empty: judge it down
|
|
499
|
+
// instead of rewarding "nothing happened" with full efficiency.
|
|
500
|
+
let efficiency = callsPerLoad === null ? null : clamp01(100 - Math.max(0, callsPerLoad - 8) * 6)
|
|
501
|
+
if (efficiency !== null && callsPerLoad < 1) efficiency = Math.min(efficiency, 45)
|
|
502
|
+
const dims = { quality, stability, outcome, efficiency }
|
|
503
|
+
const weights = { quality: 0.4, stability: 0.3, outcome: 0.2, efficiency: 0.1 }
|
|
504
|
+
let weighted = 0
|
|
505
|
+
let weightSum = 0
|
|
506
|
+
for (const [key, value] of Object.entries(dims)) {
|
|
507
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) continue
|
|
508
|
+
weighted += value * weights[key]
|
|
509
|
+
weightSum += weights[key]
|
|
510
|
+
}
|
|
511
|
+
const total = weightSum > 0 ? Math.round((weighted / weightSum) * 10) / 10 : null
|
|
512
|
+
return {
|
|
513
|
+
total,
|
|
514
|
+
grade: gradeOf(total, thresholds),
|
|
515
|
+
confidence:
|
|
516
|
+
run.loads === 0 ? 'none' : run.loads < minSamples ? 'low' : run.loads < minSamples * 3 ? 'medium' : 'high',
|
|
517
|
+
provisional: run.loads < minSamples,
|
|
518
|
+
dimensions: dims,
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* Skill loads grouped by the loop turn that produced them, so a client surface
|
|
524
|
+
* can render the result under the exact completed turn instead of guessing.
|
|
525
|
+
*/
|
|
526
|
+
function turnLoads(records) {
|
|
527
|
+
const byTurn = new Map()
|
|
528
|
+
for (const record of records) {
|
|
529
|
+
if (!record || record.t !== 'skill' || typeof record.sessionId !== 'string') continue
|
|
530
|
+
if (typeof record.turn !== 'number' || typeof record.skill !== 'string') continue
|
|
531
|
+
const key = `${record.sessionId}#${record.turn}`
|
|
532
|
+
const entry = byTurn.get(key) ?? {
|
|
533
|
+
sessionId: record.sessionId,
|
|
534
|
+
turn: record.turn,
|
|
535
|
+
skills: [],
|
|
536
|
+
firstAt: record.at,
|
|
537
|
+
lastAt: record.at,
|
|
538
|
+
}
|
|
539
|
+
if (!entry.skills.includes(record.skill)) entry.skills.push(record.skill)
|
|
540
|
+
entry.lastAt = Math.max(entry.lastAt ?? record.at, record.at)
|
|
541
|
+
byTurn.set(key, entry)
|
|
542
|
+
}
|
|
543
|
+
return [...byTurn.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0)).slice(0, 100)
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/** One row per session: what that run actually did, and how it went. */
|
|
547
|
+
function runsOf(records, skillScores, thresholds, minSamples, ciResamples) {
|
|
548
|
+
const bySession = new Map()
|
|
549
|
+
const windowsBySession = new Map()
|
|
550
|
+
const touch = (sessionId) => {
|
|
551
|
+
if (typeof sessionId !== 'string' || sessionId.length === 0) return null
|
|
552
|
+
let run = bySession.get(sessionId)
|
|
553
|
+
if (!run) {
|
|
554
|
+
run = {
|
|
555
|
+
sessionId,
|
|
556
|
+
firstAt: null,
|
|
557
|
+
lastAt: null,
|
|
558
|
+
loads: 0,
|
|
559
|
+
closed: 0,
|
|
560
|
+
toolCalls: 0,
|
|
561
|
+
errors: 0,
|
|
562
|
+
corrections: 0,
|
|
563
|
+
mcpCalls: 0,
|
|
564
|
+
mcpErrors: 0,
|
|
565
|
+
tokens: 0,
|
|
566
|
+
loopErrors: 0,
|
|
567
|
+
skills: [],
|
|
568
|
+
score: null,
|
|
569
|
+
}
|
|
570
|
+
bySession.set(sessionId, run)
|
|
571
|
+
}
|
|
572
|
+
return run
|
|
573
|
+
}
|
|
574
|
+
for (const record of records) {
|
|
575
|
+
if (!record || typeof record !== 'object') continue
|
|
576
|
+
if (typeof record.sessionId !== 'string' || record.sessionId.length === 0) continue
|
|
577
|
+
if (record.t === 'skill') {
|
|
578
|
+
const run = touch(record.sessionId)
|
|
579
|
+
if (!run) continue
|
|
580
|
+
run.loads += 1
|
|
581
|
+
if (record.skill && !run.skills.includes(record.skill)) run.skills.push(record.skill)
|
|
582
|
+
run.firstAt = run.firstAt === null ? record.at : Math.min(run.firstAt, record.at)
|
|
583
|
+
run.lastAt = run.lastAt === null ? record.at : Math.max(run.lastAt, record.at)
|
|
584
|
+
} else if (record.t === 'window') {
|
|
585
|
+
touch(record.sessionId)
|
|
586
|
+
const bucket = windowsBySession.get(record.sessionId) ?? new Map()
|
|
587
|
+
// One window is persisted at turn end and again when it closes: keep the
|
|
588
|
+
// latest record per windowId so run counters and a bootstrap draw count
|
|
589
|
+
// it once.
|
|
590
|
+
bucket.set(record.windowId ?? `${record.at}:${bucket.size}`, record)
|
|
591
|
+
windowsBySession.set(record.sessionId, bucket)
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
// Sum the deduplicated, latest-per-window records, not every cumulative
|
|
595
|
+
// snapshot the recorder wrote.
|
|
596
|
+
for (const [sessionId, run] of bySession) {
|
|
597
|
+
const windows = [...(windowsBySession.get(sessionId) ?? new Map()).values()]
|
|
598
|
+
run.closed = windows.length
|
|
599
|
+
for (const window of windows) {
|
|
600
|
+
run.toolCalls += Number(window.toolCalls) || 0
|
|
601
|
+
run.errors += Number(window.errors) || 0
|
|
602
|
+
run.loopErrors += Number(window.loopErrors) || 0
|
|
603
|
+
run.corrections += Array.isArray(window.corrections) ? window.corrections.length : 0
|
|
604
|
+
const windowTokens = window.tokens
|
|
605
|
+
if (windowTokens && typeof windowTokens === 'object') {
|
|
606
|
+
run.tokens += (Number(windowTokens.input) || 0) + (Number(windowTokens.output) || 0)
|
|
607
|
+
}
|
|
608
|
+
for (const bucket of Object.values(window.mcp ?? {})) {
|
|
609
|
+
run.mcpCalls += Number(bucket?.calls) || 0
|
|
610
|
+
run.mcpErrors += Number(bucket?.errors) || 0
|
|
611
|
+
}
|
|
612
|
+
if (typeof window.at === 'number') {
|
|
613
|
+
run.lastAt = run.lastAt === null ? window.at : Math.max(run.lastAt, window.at)
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
const runs = [...bySession.values()].sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0)).slice(0, 20)
|
|
618
|
+
for (const run of runs) {
|
|
619
|
+
run.score = scoreRun(run, skillScores, thresholds, minSamples)
|
|
620
|
+
const windows = [...(windowsBySession.get(run.sessionId) ?? new Map()).values()]
|
|
621
|
+
run.score.ci = bootstrapCI(
|
|
622
|
+
windows,
|
|
623
|
+
(draw) => {
|
|
624
|
+
const summed = {
|
|
625
|
+
...run,
|
|
626
|
+
closed: draw.length,
|
|
627
|
+
toolCalls: draw.reduce((a, w) => a + (w.toolCalls ?? 0), 0),
|
|
628
|
+
errors: draw.reduce((a, w) => a + (w.errors ?? 0), 0),
|
|
629
|
+
loopErrors: draw.reduce((a, w) => a + (w.loopErrors ?? 0), 0),
|
|
630
|
+
corrections: draw.reduce((a, w) => a + (Array.isArray(w.corrections) ? w.corrections.length : 0), 0),
|
|
631
|
+
}
|
|
632
|
+
return scoreRun(summed, skillScores, thresholds, minSamples).total
|
|
633
|
+
},
|
|
634
|
+
{ resamples: ciResamples ?? 0, seed: hashSeed(run.sessionId) },
|
|
635
|
+
)
|
|
636
|
+
}
|
|
637
|
+
return runs
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
async function buildOne(name) {
|
|
641
|
+
const report = await build({ skill: name })
|
|
642
|
+
return report.skills[0] ?? null
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
return { build, buildOne, store, invalidate, resolveStatic }
|
|
646
|
+
}
|