dsh-skill-eval 0.0.0-stage → 1.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +718 -0
- package/LICENSE +21 -0
- package/README.md +207 -2
- package/README.zh.md +164 -0
- package/client.js +2791 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +82 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +468 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1173 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +557 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +141 -0
- package/lib/html.js +17 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +644 -0
- package/lib/render.js +580 -0
- package/lib/report.js +689 -0
- package/lib/score.js +565 -0
- package/lib/settings.js +415 -0
- package/lib/snapshot.js +250 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +240 -0
- package/lib/validity.js +125 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +227 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +234 -0
- package/scripts/lib.mjs +87 -0
- package/scripts/verify-live.mjs +178 -0
package/lib/score.js
ADDED
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure scoring core for dsh-skill-eval.
|
|
3
|
+
*
|
|
4
|
+
* The only import is the deterministic RNG helper: this module must stay
|
|
5
|
+
* unit-testable and safe to load inside the host process without touching any
|
|
6
|
+
* DSH API.
|
|
7
|
+
*
|
|
8
|
+
* The weighting model borrows the *idea* from the public skill-eval tooling
|
|
9
|
+
* (aws-samples/sample-agent-skill-eval: 40/40/20 audit|functional|trigger with an
|
|
10
|
+
* A-F grade band; NVIDIA/SkillEvaluator: multi-dimensional rubric + 70 threshold).
|
|
11
|
+
* Nothing is copied: every signal below is derived from live DSH runtime
|
|
12
|
+
* telemetry that those static-first tools cannot observe.
|
|
13
|
+
*/
|
|
14
|
+
import { mulberry32 } from './random.js'
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_WEIGHTS = Object.freeze({
|
|
17
|
+
static: 0.3,
|
|
18
|
+
reliability: 0.25,
|
|
19
|
+
effectiveness: 0.3,
|
|
20
|
+
efficiency: 0.15,
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
export const DEFAULT_THRESHOLDS = Object.freeze({
|
|
24
|
+
pass: 70,
|
|
25
|
+
good: 80,
|
|
26
|
+
excellent: 90,
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
export function gradeOf(score, thresholds = DEFAULT_THRESHOLDS) {
|
|
30
|
+
if (score === null || score === undefined || score === '') return '?'
|
|
31
|
+
const s = Number(score)
|
|
32
|
+
if (!Number.isFinite(s)) return '?'
|
|
33
|
+
const bands = [
|
|
34
|
+
[thresholds.excellent ?? 90, 'A'],
|
|
35
|
+
[thresholds.good ?? 80, 'B'],
|
|
36
|
+
[thresholds.pass ?? 70, 'C'],
|
|
37
|
+
[60, 'D'],
|
|
38
|
+
[0, 'F'],
|
|
39
|
+
]
|
|
40
|
+
for (const [min, grade] of bands) if (s >= min) return grade
|
|
41
|
+
return 'F'
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Dissatisfaction markers only. Weak words like 重新/再试/again/actually are
|
|
46
|
+
* excluded on purpose: "重新排序这些页面" is a legitimate request, and a false
|
|
47
|
+
* positive would punish a healthy skill.
|
|
48
|
+
*/
|
|
49
|
+
const CORRECTION_PATTERNS = [
|
|
50
|
+
/不对/,
|
|
51
|
+
/不是这个/,
|
|
52
|
+
/不是我要/,
|
|
53
|
+
/错了/,
|
|
54
|
+
/搞错/,
|
|
55
|
+
/重做/,
|
|
56
|
+
/再来一次/,
|
|
57
|
+
/别用/,
|
|
58
|
+
/不要用/,
|
|
59
|
+
/还是不行/,
|
|
60
|
+
/还是不对/,
|
|
61
|
+
/没生效/,
|
|
62
|
+
/不好用/,
|
|
63
|
+
/有问题/,
|
|
64
|
+
/\bwrong\b/i,
|
|
65
|
+
/\bnope\b/i,
|
|
66
|
+
/that'?s not (what|right|correct)/i,
|
|
67
|
+
/\bredo\b/i,
|
|
68
|
+
/\bstart over\b/i,
|
|
69
|
+
/\bstill (broken|failing|wrong|not working)\b/i,
|
|
70
|
+
/\b(try|do it) again\b/i,
|
|
71
|
+
/\b(doesn'?t|didn'?t|won'?t) work/i,
|
|
72
|
+
/\bnot working\b/i,
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Does a user message read as a correction/rejection of what just happened?
|
|
77
|
+
* Deliberately conservative: false positives punish a skill for normal follow-ups.
|
|
78
|
+
*/
|
|
79
|
+
export function isCorrection(text) {
|
|
80
|
+
if (typeof text !== 'string' || text.length === 0) return false
|
|
81
|
+
const t = text.trim().slice(0, 400)
|
|
82
|
+
if (t.length === 0) return false
|
|
83
|
+
return CORRECTION_PATTERNS.some((re) => re.test(t))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function clamp(n, lo, hi) {
|
|
87
|
+
return Math.min(hi, Math.max(lo, n))
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function round(n, digits = 1) {
|
|
91
|
+
if (!Number.isFinite(n)) return null
|
|
92
|
+
const f = 10 ** digits
|
|
93
|
+
return Math.round(n * f) / f
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function median(nums) {
|
|
97
|
+
const xs = nums.filter((n) => Number.isFinite(n)).sort((a, b) => a - b)
|
|
98
|
+
if (xs.length === 0) return null
|
|
99
|
+
const mid = Math.floor(xs.length / 2)
|
|
100
|
+
return xs.length % 2 === 1 ? xs[mid] : (xs[mid - 1] + xs[mid]) / 2
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ---------------------------------------------------------------------------
|
|
104
|
+
// Static (authoring) quality — the "audit" dimension.
|
|
105
|
+
// ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
const STATIC_CHECKS = [
|
|
108
|
+
{
|
|
109
|
+
id: 'desc-length',
|
|
110
|
+
weight: 12,
|
|
111
|
+
label: 'description length',
|
|
112
|
+
evaluate: (s) => {
|
|
113
|
+
const n = (s.description ?? '').length
|
|
114
|
+
if (n >= 60) return 1
|
|
115
|
+
if (n >= 30) return 0.5
|
|
116
|
+
return 0
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
id: 'desc-trigger',
|
|
121
|
+
weight: 12,
|
|
122
|
+
label: 'description states when to use it',
|
|
123
|
+
evaluate: (s) =>
|
|
124
|
+
/use when|when to use|triggers?|适用|适用于|当用户|用于|使用场景/i.test(s.description ?? '') ? 1 : 0,
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
id: 'when-to-use',
|
|
128
|
+
weight: 8,
|
|
129
|
+
label: 'explicit whenToUse field',
|
|
130
|
+
evaluate: (s) => ((s.whenToUse ?? '').trim().length > 0 ? 1 : 0),
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
id: 'body-present',
|
|
134
|
+
weight: 10,
|
|
135
|
+
label: 'body is substantive',
|
|
136
|
+
evaluate: (s) => {
|
|
137
|
+
const n = s.bodyLines ?? 0
|
|
138
|
+
if (n >= 15) return 1
|
|
139
|
+
if (n >= 5) return 0.5
|
|
140
|
+
return 0
|
|
141
|
+
},
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
id: 'body-compact',
|
|
145
|
+
weight: 18,
|
|
146
|
+
label: 'body stays under the 500-line guidance',
|
|
147
|
+
evaluate: (s) => {
|
|
148
|
+
const n = s.bodyLines ?? 0
|
|
149
|
+
if (n === 0) return 0
|
|
150
|
+
if (n <= 500) return 1
|
|
151
|
+
return clamp(1 - (n - 500) / 1000, 0, 1)
|
|
152
|
+
},
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
id: 'sections',
|
|
156
|
+
weight: 10,
|
|
157
|
+
label: 'body is sectioned',
|
|
158
|
+
evaluate: (s) => {
|
|
159
|
+
const n = s.sectionCount ?? 0
|
|
160
|
+
if (n >= 2) return 1
|
|
161
|
+
if (n === 1) return 0.5
|
|
162
|
+
return 0
|
|
163
|
+
},
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
id: 'progressive-disclosure',
|
|
167
|
+
weight: 15,
|
|
168
|
+
label: 'delegates detail to references/ or scripts/',
|
|
169
|
+
evaluate: (s) =>
|
|
170
|
+
/references\/|scripts\/|assets\/|##\s*(Reference|References|参考|参考资料)/i.test(s.content ?? '') ? 1 : 0,
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
id: 'no-placeholder',
|
|
174
|
+
weight: 10,
|
|
175
|
+
label: 'no TODO/TBD placeholders',
|
|
176
|
+
evaluate: (s) => (/\b(TODO|TBD|FIXME|XXX)\b/.test(s.content ?? '') ? 0 : 1),
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
id: 'has-example',
|
|
180
|
+
weight: 5,
|
|
181
|
+
label: 'contains a usage example',
|
|
182
|
+
evaluate: (s) => (((s.content ?? '').match(/```/g)?.length ?? 0) >= 2 ? 1 : 0),
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
id: 'anti-trigger',
|
|
186
|
+
weight: 5,
|
|
187
|
+
label: 'states when NOT to use it',
|
|
188
|
+
evaluate: (s) =>
|
|
189
|
+
/do not use|don'?t use|not for|never use|不(?:要|应|适用)用于|不用于|避免(?:用于|使用)|不要用在/i.test(
|
|
190
|
+
`${s.description ?? ''}\n${s.whenToUse ?? ''}\n${s.content ?? ''}`,
|
|
191
|
+
)
|
|
192
|
+
? 1
|
|
193
|
+
: 0,
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
id: 'skill-name',
|
|
197
|
+
weight: 4,
|
|
198
|
+
label: 'skill id is a kebab-case name',
|
|
199
|
+
evaluate: (s) => (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(s.name ?? '') ? 1 : 0),
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
id: 'reference-integrity',
|
|
203
|
+
weight: 8,
|
|
204
|
+
label: 'every referenced file exists',
|
|
205
|
+
// null = cannot be checked here (no filesystem view); the check is then
|
|
206
|
+
// excluded from the static score instead of pretending to pass.
|
|
207
|
+
evaluate: (s) => {
|
|
208
|
+
const integrity = s.integrity
|
|
209
|
+
if (!integrity || typeof integrity.checked !== 'number') return null
|
|
210
|
+
if (integrity.checked === 0) return 1
|
|
211
|
+
const missing = integrity.missing?.length ?? 0
|
|
212
|
+
return (integrity.checked - missing) / integrity.checked
|
|
213
|
+
},
|
|
214
|
+
},
|
|
215
|
+
]
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Build the static profile from a skill summary + loaded definition.
|
|
219
|
+
* Returns null when the skill cannot be resolved at all (deleted provider etc.).
|
|
220
|
+
*/
|
|
221
|
+
export function staticProfile(summary, definition, options = {}) {
|
|
222
|
+
if (!summary) return null
|
|
223
|
+
const content = typeof definition?.content === 'string' ? definition.content : ''
|
|
224
|
+
const bodyLines = content.length === 0 ? 0 : content.split('\n').length
|
|
225
|
+
const sectionCount = (content.match(/^##\s+\S/gm) ?? []).length
|
|
226
|
+
const view = {
|
|
227
|
+
name: summary.name,
|
|
228
|
+
description: summary.description ?? '',
|
|
229
|
+
whenToUse: summary.whenToUse ?? '',
|
|
230
|
+
content,
|
|
231
|
+
bodyLines,
|
|
232
|
+
sectionCount,
|
|
233
|
+
source: summary.source ?? null,
|
|
234
|
+
provider: summary.provider ?? null,
|
|
235
|
+
path: summary.path ?? definition?.path ?? null,
|
|
236
|
+
resolved: content.length > 0,
|
|
237
|
+
integrity: options.integrity ?? null,
|
|
238
|
+
}
|
|
239
|
+
const raw = STATIC_CHECKS.map((c) => {
|
|
240
|
+
const evaluated = c.evaluate(view)
|
|
241
|
+
if (evaluated === null || evaluated === undefined) {
|
|
242
|
+
return { id: c.id, label: c.label, weight: c.weight, fraction: null, points: 0, skipped: true }
|
|
243
|
+
}
|
|
244
|
+
const fraction = clamp(Number(evaluated) || 0, 0, 1)
|
|
245
|
+
return { id: c.id, label: c.label, weight: c.weight, fraction, points: round(c.weight * fraction, 1) }
|
|
246
|
+
})
|
|
247
|
+
const checks = raw
|
|
248
|
+
const counted = checks.filter((c) => !c.skipped)
|
|
249
|
+
const earned = counted.reduce((a, c) => a + c.weight * c.fraction, 0)
|
|
250
|
+
const totalWeight = counted.reduce((a, c) => a + c.weight, 0)
|
|
251
|
+
// The full body is only needed while the checks run; persisting it would
|
|
252
|
+
// duplicate every SKILL.md into the append-only store and every report JSON.
|
|
253
|
+
const { content: _body, ...persisted } = view
|
|
254
|
+
return {
|
|
255
|
+
...persisted,
|
|
256
|
+
checks,
|
|
257
|
+
score: round((earned / totalWeight) * 100, 1),
|
|
258
|
+
failedChecks: counted.filter((c) => c.fraction < 1),
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// ---------------------------------------------------------------------------
|
|
263
|
+
// Runtime aggregation over invocation samples.
|
|
264
|
+
// ---------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Aggregate closed+open invocation samples for one skill.
|
|
268
|
+
*
|
|
269
|
+
* sample = {
|
|
270
|
+
* ok: boolean, // did the `skill` tool call succeed
|
|
271
|
+
* loadMs?: number, // duration of the load call
|
|
272
|
+
* error?: string, // load error message
|
|
273
|
+
* window?: { // the observation window after a successful load
|
|
274
|
+
* toolCalls: number,
|
|
275
|
+
* errors: number,
|
|
276
|
+
* errorNames: string[],
|
|
277
|
+
* corrections: string[],
|
|
278
|
+
* otherSkills: string[],
|
|
279
|
+
* turnEnded: boolean,
|
|
280
|
+
* truncated: boolean,
|
|
281
|
+
* } | null
|
|
282
|
+
* }
|
|
283
|
+
*/
|
|
284
|
+
export function aggregate(samples, options = {}) {
|
|
285
|
+
const safe = Array.isArray(samples) ? samples.filter(Boolean) : []
|
|
286
|
+
const loads = safe.length
|
|
287
|
+
const loadOk = safe.filter((s) => s.ok).length
|
|
288
|
+
const closed = safe.filter((s) => s.window)
|
|
289
|
+
const windows = closed.map((s) => s.window)
|
|
290
|
+
|
|
291
|
+
const count = (value) => {
|
|
292
|
+
const n = Number(value)
|
|
293
|
+
return Number.isFinite(n) && n > 0 ? n : 0
|
|
294
|
+
}
|
|
295
|
+
const toolCalls = windows.reduce((a, w) => a + count(w.toolCalls), 0)
|
|
296
|
+
const errors = windows.reduce((a, w) => a + count(w.errors), 0)
|
|
297
|
+
const withCorrection = windows.filter((w) => (Array.isArray(w.corrections) ? w.corrections.length : 0) > 0).length
|
|
298
|
+
const withSibling = windows.filter((w) => (Array.isArray(w.otherSkills) ? w.otherSkills.length : 0) > 0).length
|
|
299
|
+
// A subagent that loads a skill and finishes immediately is not the same
|
|
300
|
+
// failure as a root session dropping a skill: judge abandonment on root runs.
|
|
301
|
+
const rootWindows = windows.filter((w) => w.origin !== 'subagent')
|
|
302
|
+
// A window force-closed for memory reasons is censored, not abandoned: it
|
|
303
|
+
// must not turn "we ran out of tracked sessions" into an effectiveness claim.
|
|
304
|
+
const abandonBase = (rootWindows.length > 0 ? rootWindows : windows).filter((w) => w.closeReason !== 'evicted')
|
|
305
|
+
const abandonedRoot = abandonBase.filter((w) => count(w.toolCalls) === 0).length
|
|
306
|
+
|
|
307
|
+
const withLoopError = windows.filter((w) => count(w.loopErrors) > 0).length
|
|
308
|
+
const loopErrors = windows.reduce((a, w) => a + count(w.loopErrors), 0)
|
|
309
|
+
const llmRetries = windows.reduce((a, w) => a + count(w.llmRetries), 0)
|
|
310
|
+
const loopErrorNames = Object.create(null)
|
|
311
|
+
for (const w of windows) {
|
|
312
|
+
if (typeof w.lastLoopError === 'string' && w.lastLoopError) {
|
|
313
|
+
loopErrorNames[w.lastLoopError] = (loopErrorNames[w.lastLoopError] ?? 0) + 1
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// MCP health is independent of which skill was loaded: merge every window's
|
|
318
|
+
// per-server bucket so the report can show a board per MCP server.
|
|
319
|
+
const mcp = Object.create(null)
|
|
320
|
+
for (const w of windows) {
|
|
321
|
+
for (const [server, bucket] of Object.entries(w.mcp ?? {})) {
|
|
322
|
+
if (!bucket || typeof bucket !== 'object') continue
|
|
323
|
+
if (!Object.hasOwn(mcp, server)) {
|
|
324
|
+
mcp[server] = { calls: 0, errors: 0, ms: 0, windows: 0, samples: [], tools: Object.create(null) }
|
|
325
|
+
}
|
|
326
|
+
const entry = mcp[server]
|
|
327
|
+
entry.calls += count(bucket.calls)
|
|
328
|
+
entry.errors += count(bucket.errors)
|
|
329
|
+
entry.ms += count(bucket.ms)
|
|
330
|
+
entry.windows += 1
|
|
331
|
+
if (Array.isArray(bucket.samples)) {
|
|
332
|
+
for (const sample of bucket.samples) {
|
|
333
|
+
if (typeof sample === 'number' && entry.samples.length < 400) entry.samples.push(sample)
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
|
|
337
|
+
if (!stats || typeof stats !== 'object' || Array.isArray(stats)) continue
|
|
338
|
+
const t = Object.hasOwn(entry.tools, tool) ? entry.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
|
|
339
|
+
t.calls += count(stats.calls)
|
|
340
|
+
t.errors += count(stats.errors)
|
|
341
|
+
t.ms = (t.ms ?? 0) + count(stats.ms)
|
|
342
|
+
if (Array.isArray(stats.samples)) {
|
|
343
|
+
for (const sample of stats.samples) {
|
|
344
|
+
if (typeof sample === 'number' && t.samples.length < 60) t.samples.push(sample)
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
entry.tools[tool] = t
|
|
348
|
+
}
|
|
349
|
+
mcp[server] = entry
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const corrections = windows.flatMap((w) =>
|
|
354
|
+
Array.isArray(w.corrections) ? w.corrections.filter((c) => typeof c === 'string') : [],
|
|
355
|
+
)
|
|
356
|
+
const siblings = Object.create(null)
|
|
357
|
+
for (const w of windows) {
|
|
358
|
+
for (const s of Array.isArray(w.otherSkills) ? w.otherSkills : []) {
|
|
359
|
+
if (typeof s === 'string' && s) siblings[s] = (siblings[s] ?? 0) + 1
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
const errorNames = Object.create(null)
|
|
363
|
+
for (const w of windows) {
|
|
364
|
+
for (const e of Array.isArray(w.errorNames) ? w.errorNames : []) {
|
|
365
|
+
if (typeof e === 'string' && e) errorNames[e] = (errorNames[e] ?? 0) + 1
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const tokens = { input: 0, output: 0, cacheRead: 0, calls: 0 }
|
|
370
|
+
for (const w of windows) {
|
|
371
|
+
const t = w.tokens
|
|
372
|
+
if (!t || typeof t !== 'object') continue
|
|
373
|
+
tokens.input += Number(t.input) || 0
|
|
374
|
+
tokens.output += Number(t.output) || 0
|
|
375
|
+
tokens.cacheRead += Number(t.cacheRead) || 0
|
|
376
|
+
tokens.calls += Number(t.calls) || 0
|
|
377
|
+
}
|
|
378
|
+
const cacheWeight = Number.isFinite(options.cacheReadWeight) ? options.cacheReadWeight : 0
|
|
379
|
+
const billable = tokens.input + tokens.output + tokens.cacheRead * cacheWeight
|
|
380
|
+
// Only windows that actually carried model usage count: older windows were
|
|
381
|
+
// recorded before token capture existed and would dilute the per-load cost.
|
|
382
|
+
const tokenWindows = windows.filter((w) => w.tokens && (Number(w.tokens.calls) || 0) > 0).length
|
|
383
|
+
const tokensPerLoad = tokenWindows > 0 && tokens.calls > 0 ? billable / tokenWindows : null
|
|
384
|
+
|
|
385
|
+
return {
|
|
386
|
+
loads,
|
|
387
|
+
closed: closed.length,
|
|
388
|
+
tokens,
|
|
389
|
+
tokenWindows,
|
|
390
|
+
tokensPerLoad: tokensPerLoad === null ? null : Math.round(tokensPerLoad),
|
|
391
|
+
billableTokens: Math.round(billable),
|
|
392
|
+
open: loads - closed.length,
|
|
393
|
+
loadOk,
|
|
394
|
+
loadOkRate: loads > 0 ? loadOk / loads : null,
|
|
395
|
+
loadErrors: safe.map((s) => s.error).filter(Boolean),
|
|
396
|
+
toolCalls,
|
|
397
|
+
errors,
|
|
398
|
+
errRate: toolCalls > 0 ? errors / toolCalls : null,
|
|
399
|
+
mcp,
|
|
400
|
+
loopErrors,
|
|
401
|
+
llmRetries,
|
|
402
|
+
loopErrorRate: windows.length > 0 ? withLoopError / windows.length : null,
|
|
403
|
+
loopErrorNames,
|
|
404
|
+
correctionRate: windows.length > 0 ? withCorrection / windows.length : null,
|
|
405
|
+
abandonRate: abandonBase.length > 0 ? abandonedRoot / abandonBase.length : null,
|
|
406
|
+
abandonBase: abandonBase.length,
|
|
407
|
+
siblingRate: windows.length > 0 ? withSibling / windows.length : null,
|
|
408
|
+
avgCalls: windows.length > 0 ? toolCalls / windows.length : null,
|
|
409
|
+
p50LoadMs: median(safe.map((s) => s.loadMs)),
|
|
410
|
+
corrections: corrections.slice(-5),
|
|
411
|
+
siblings,
|
|
412
|
+
errorNames,
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// ---------------------------------------------------------------------------
|
|
417
|
+
// Scoring.
|
|
418
|
+
// ---------------------------------------------------------------------------
|
|
419
|
+
|
|
420
|
+
function reliabilityScore(stats) {
|
|
421
|
+
if (stats.loads === 0) return null
|
|
422
|
+
const loadOk = stats.loadOkRate ?? 0
|
|
423
|
+
// A 50% post-load tool error rate zeroes the error half.
|
|
424
|
+
const cleanWindow = stats.errRate === null ? 1 : 1 - clamp(stats.errRate * 2, 0, 1)
|
|
425
|
+
// The agent loop itself failing right after a load (a broken step, a failed
|
|
426
|
+
// model request, an aborted batch) is the skill's problem too, but it is a
|
|
427
|
+
// weaker signal than a tool error, so it costs at most 40% of the dimension.
|
|
428
|
+
const loopPenalty = 1 - 0.4 * clamp(stats.loopErrorRate ?? 0, 0, 1)
|
|
429
|
+
// A load failure makes the skill unusable, so it multiplies the whole
|
|
430
|
+
// dimension instead of only the load half: 0% load success = 0 reliability.
|
|
431
|
+
return clamp(100 * loadOk * (0.6 + 0.4 * cleanWindow) * loopPenalty, 0, 100)
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function effectivenessScore(stats) {
|
|
435
|
+
if (stats.closed === 0) return null
|
|
436
|
+
let score = 100
|
|
437
|
+
score -= 45 * (stats.correctionRate ?? 0)
|
|
438
|
+
score -= 25 * (stats.abandonRate ?? 0)
|
|
439
|
+
score -= 20 * (stats.siblingRate ?? 0)
|
|
440
|
+
score -= 10 * clamp(stats.errRate ?? 0, 0, 1)
|
|
441
|
+
return clamp(score, 0, 100)
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function efficiencyScore(stats, tokenBudgetPerLoad = 20000) {
|
|
445
|
+
if (stats.closed === 0) return null
|
|
446
|
+
let score = 100
|
|
447
|
+
const calls = stats.avgCalls ?? 0
|
|
448
|
+
if (calls > 8) score -= Math.min(40, (calls - 8) * 6)
|
|
449
|
+
if (calls < 1) score -= 15
|
|
450
|
+
const ms = stats.p50LoadMs
|
|
451
|
+
if (Number.isFinite(ms) && ms > 2000) score -= Math.min(30, (ms - 2000) / 100)
|
|
452
|
+
// Measured billable tokens per load: double the budget costs 30 points.
|
|
453
|
+
const perLoad = stats.tokensPerLoad
|
|
454
|
+
if (Number.isFinite(perLoad) && perLoad > tokenBudgetPerLoad) {
|
|
455
|
+
score -= Math.min(35, ((perLoad - tokenBudgetPerLoad) / tokenBudgetPerLoad) * 30)
|
|
456
|
+
}
|
|
457
|
+
return clamp(score, 0, 100)
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* Composite score. Dimensions that have no evidence (no samples / unresolvable
|
|
462
|
+
* skill) are dropped and their weight is redistributed over the rest, so a
|
|
463
|
+
* brand-new skill is not punished for having no history.
|
|
464
|
+
*/
|
|
465
|
+
export function scoreSkill({
|
|
466
|
+
stats,
|
|
467
|
+
static: staticDim,
|
|
468
|
+
weights = DEFAULT_WEIGHTS,
|
|
469
|
+
thresholds = DEFAULT_THRESHOLDS,
|
|
470
|
+
minSamples = 3,
|
|
471
|
+
tokenBudgetPerLoad = 20000,
|
|
472
|
+
}) {
|
|
473
|
+
const dims = {
|
|
474
|
+
static: staticDim?.score ?? null,
|
|
475
|
+
reliability: reliabilityScore(stats),
|
|
476
|
+
effectiveness: effectivenessScore(stats),
|
|
477
|
+
efficiency: efficiencyScore(stats, tokenBudgetPerLoad),
|
|
478
|
+
}
|
|
479
|
+
let weighted = 0
|
|
480
|
+
let weightSum = 0
|
|
481
|
+
const used = {}
|
|
482
|
+
for (const key of Object.keys(dims)) {
|
|
483
|
+
const value = dims[key]
|
|
484
|
+
if (value === null || value === undefined || !Number.isFinite(value)) continue
|
|
485
|
+
const w = Number(weights[key] ?? 0)
|
|
486
|
+
if (w <= 0) continue
|
|
487
|
+
weighted += value * w
|
|
488
|
+
weightSum += w
|
|
489
|
+
used[key] = w
|
|
490
|
+
}
|
|
491
|
+
const total = weightSum > 0 ? round(weighted / weightSum, 1) : null
|
|
492
|
+
const confidence =
|
|
493
|
+
stats.loads === 0 ? 'none' : stats.loads < minSamples ? 'low' : stats.loads < minSamples * 3 ? 'medium' : 'high'
|
|
494
|
+
return {
|
|
495
|
+
total,
|
|
496
|
+
grade: gradeOf(total, thresholds),
|
|
497
|
+
provisional: stats.loads < minSamples,
|
|
498
|
+
confidence,
|
|
499
|
+
samples: stats.loads,
|
|
500
|
+
minSamples,
|
|
501
|
+
dimensions: dims,
|
|
502
|
+
effectiveWeights: used,
|
|
503
|
+
coverage: round(weightSum, 2),
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Bootstrap interval for a rate measured over windows (correction rate, error
|
|
509
|
+
* rate, ...). Advice uses it as a noise floor: a rule only keeps its severity
|
|
510
|
+
* when the interval's low end is already past the threshold.
|
|
511
|
+
*/
|
|
512
|
+
/**
|
|
513
|
+
* Bootstrap interval for a call-weighted rate. Advice reports "X% of the tool
|
|
514
|
+
* calls inside the window failed", so its noise floor must resample calls, not
|
|
515
|
+
* the fraction of windows that contain any error: one severe window can
|
|
516
|
+
* otherwise be gated away by nine clean one-call windows.
|
|
517
|
+
*/
|
|
518
|
+
export function bootstrapCallRate(windows, options = {}) {
|
|
519
|
+
const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
|
|
520
|
+
const resamples = Number.isFinite(options.resamples) ? options.resamples : 400
|
|
521
|
+
if (rows.length < 2 || resamples <= 0) return null
|
|
522
|
+
const totalCalls = rows.reduce((sum, w) => sum + (Number(w.toolCalls) || 0), 0)
|
|
523
|
+
const totalErrors = rows.reduce((sum, w) => sum + (Number(w.errors) || 0), 0)
|
|
524
|
+
if (!(totalCalls > 0)) return null
|
|
525
|
+
const random = mulberry32(Number.isFinite(options.seed) ? options.seed : 1)
|
|
526
|
+
const rates = []
|
|
527
|
+
for (let i = 0; i < resamples; i += 1) {
|
|
528
|
+
let calls = 0
|
|
529
|
+
let errors = 0
|
|
530
|
+
for (let j = 0; j < rows.length; j += 1) {
|
|
531
|
+
const w = rows[Math.floor(random() * rows.length)]
|
|
532
|
+
calls += Number(w.toolCalls) || 0
|
|
533
|
+
errors += Number(w.errors) || 0
|
|
534
|
+
}
|
|
535
|
+
if (calls > 0) rates.push(errors / calls)
|
|
536
|
+
}
|
|
537
|
+
if (rates.length === 0) return null
|
|
538
|
+
rates.sort((a, b) => a - b)
|
|
539
|
+
const at = (p) => rates[Math.min(rates.length - 1, Math.max(0, Math.round(p * (rates.length - 1))))]
|
|
540
|
+
return {
|
|
541
|
+
mean: Math.min(1, totalErrors / totalCalls),
|
|
542
|
+
low: at(0.025),
|
|
543
|
+
high: at(0.975),
|
|
544
|
+
resamples: rates.length,
|
|
545
|
+
windows: rows.length,
|
|
546
|
+
calls: totalCalls,
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
export function bootstrapRate(windows, isHit, options = {}) {
|
|
551
|
+
const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
|
|
552
|
+
const resamples = Number.isFinite(options.resamples) ? options.resamples : 400
|
|
553
|
+
if (rows.length < 2 || resamples <= 0) return null
|
|
554
|
+
const random = mulberry32(Number.isFinite(options.seed) ? options.seed : 1)
|
|
555
|
+
const rates = []
|
|
556
|
+
for (let i = 0; i < resamples; i += 1) {
|
|
557
|
+
let hits = 0
|
|
558
|
+
for (let j = 0; j < rows.length; j += 1) if (isHit(rows[Math.floor(random() * rows.length)])) hits += 1
|
|
559
|
+
rates.push(hits / rows.length)
|
|
560
|
+
}
|
|
561
|
+
rates.sort((a, b) => a - b)
|
|
562
|
+
const at = (p) => rates[Math.min(rates.length - 1, Math.max(0, Math.round(p * (rates.length - 1))))]
|
|
563
|
+
const mean = rows.filter(isHit).length / rows.length
|
|
564
|
+
return { mean, low: at(0.025), high: at(0.975), resamples: rates.length, windows: rows.length }
|
|
565
|
+
}
|