dsh-skill-eval 0.0.0-stage → 1.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +675 -0
  2. package/LICENSE +21 -0
  3. package/README.md +200 -2
  4. package/README.zh.md +157 -0
  5. package/client.js +2637 -0
  6. package/cordis.patch.yml +9 -0
  7. package/docs/ACCEPTANCE.md +71 -0
  8. package/docs/AUDIT.md +64 -0
  9. package/docs/CODE-REVIEW.md +49 -0
  10. package/docs/COMPARISON.md +33 -0
  11. package/docs/DETAIL.md +463 -0
  12. package/docs/DEV-RELOAD.md +114 -0
  13. package/docs/ENTRYPOINTS.md +65 -0
  14. package/docs/RELEASE.md +44 -0
  15. package/docs/RRSI-NOTES.md +129 -0
  16. package/docs/SCENARIOS.md +91 -0
  17. package/docs/UI-REVIEW.md +115 -0
  18. package/docs/WORKFLOW.md +108 -0
  19. package/icon.svg +9 -0
  20. package/index.js +1111 -0
  21. package/lib/accept.js +350 -0
  22. package/lib/advice.js +547 -0
  23. package/lib/critic.js +101 -0
  24. package/lib/dev.js +148 -0
  25. package/lib/history.js +140 -0
  26. package/lib/html.js +8 -0
  27. package/lib/inspect.js +142 -0
  28. package/lib/integrity.js +46 -0
  29. package/lib/loop.js +195 -0
  30. package/lib/random.js +29 -0
  31. package/lib/recorder.js +634 -0
  32. package/lib/render.js +582 -0
  33. package/lib/report.js +646 -0
  34. package/lib/score.js +525 -0
  35. package/lib/settings.js +396 -0
  36. package/lib/snapshot.js +166 -0
  37. package/lib/steps.js +141 -0
  38. package/lib/store.js +186 -0
  39. package/lib/validity.js +115 -0
  40. package/locale/en.json +6 -0
  41. package/locale/zh.json +6 -0
  42. package/package.json +96 -4
  43. package/scripts/accept.mjs +226 -0
  44. package/scripts/build-client.mjs +59 -0
  45. package/scripts/build.mjs +81 -0
  46. package/scripts/doctor.mjs +199 -0
  47. package/scripts/lib.mjs +52 -0
  48. package/scripts/verify-live.mjs +168 -0
package/lib/score.js ADDED
@@ -0,0 +1,525 @@
1
+ /**
2
+ * Pure scoring core for dsh-skill-eval.
3
+ *
4
+ * The only import is the deterministic RNG helper: this module must stay
5
+ * unit-testable and safe to load inside the host process without touching any
6
+ * DSH API.
7
+ *
8
+ * The weighting model borrows the *idea* from the public skill-eval tooling
9
+ * (aws-samples/sample-agent-skill-eval: 40/40/20 audit|functional|trigger with an
10
+ * A-F grade band; NVIDIA/SkillEvaluator: multi-dimensional rubric + 70 threshold).
11
+ * Nothing is copied: every signal below is derived from live DSH runtime
12
+ * telemetry that those static-first tools cannot observe.
13
+ */
14
+ import { mulberry32 } from './random.js'
15
+
16
+ export const DEFAULT_WEIGHTS = Object.freeze({
17
+ static: 0.3,
18
+ reliability: 0.25,
19
+ effectiveness: 0.3,
20
+ efficiency: 0.15,
21
+ })
22
+
23
+ export const DEFAULT_THRESHOLDS = Object.freeze({
24
+ pass: 70,
25
+ good: 80,
26
+ excellent: 90,
27
+ })
28
+
29
+ export function gradeOf(score, thresholds = DEFAULT_THRESHOLDS) {
30
+ if (score === null || score === undefined || score === '') return '?'
31
+ const s = Number(score)
32
+ if (!Number.isFinite(s)) return '?'
33
+ const bands = [
34
+ [thresholds.excellent ?? 90, 'A'],
35
+ [thresholds.good ?? 80, 'B'],
36
+ [thresholds.pass ?? 70, 'C'],
37
+ [60, 'D'],
38
+ [0, 'F'],
39
+ ]
40
+ for (const [min, grade] of bands) if (s >= min) return grade
41
+ return 'F'
42
+ }
43
+
44
+ /**
45
+ * Dissatisfaction markers only. Weak words like 重新/再试/again/actually are
46
+ * excluded on purpose: "重新排序这些页面" is a legitimate request, and a false
47
+ * positive would punish a healthy skill.
48
+ */
49
+ const CORRECTION_PATTERNS = [
50
+ /不对/,
51
+ /不是这个/,
52
+ /不是我要/,
53
+ /错了/,
54
+ /搞错/,
55
+ /重做/,
56
+ /再来一次/,
57
+ /别用/,
58
+ /不要用/,
59
+ /还是不行/,
60
+ /还是不对/,
61
+ /没生效/,
62
+ /不好用/,
63
+ /有问题/,
64
+ /\bwrong\b/i,
65
+ /\bnope\b/i,
66
+ /that'?s not (what|right|correct)/i,
67
+ /\bredo\b/i,
68
+ /\bstart over\b/i,
69
+ /\bstill (broken|failing|wrong|not working)\b/i,
70
+ /\b(try|do it) again\b/i,
71
+ /\b(doesn'?t|didn'?t|won'?t) work/i,
72
+ /\bnot working\b/i,
73
+ ]
74
+
75
+ /**
76
+ * Does a user message read as a correction/rejection of what just happened?
77
+ * Deliberately conservative: false positives punish a skill for normal follow-ups.
78
+ */
79
+ export function isCorrection(text) {
80
+ if (typeof text !== 'string' || text.length === 0) return false
81
+ const t = text.trim().slice(0, 400)
82
+ if (t.length === 0) return false
83
+ return CORRECTION_PATTERNS.some((re) => re.test(t))
84
+ }
85
+
86
+ function clamp(n, lo, hi) {
87
+ return Math.min(hi, Math.max(lo, n))
88
+ }
89
+
90
+ function round(n, digits = 1) {
91
+ if (!Number.isFinite(n)) return null
92
+ const f = 10 ** digits
93
+ return Math.round(n * f) / f
94
+ }
95
+
96
+ export function median(nums) {
97
+ const xs = nums.filter((n) => Number.isFinite(n)).sort((a, b) => a - b)
98
+ if (xs.length === 0) return null
99
+ const mid = Math.floor(xs.length / 2)
100
+ return xs.length % 2 === 1 ? xs[mid] : (xs[mid - 1] + xs[mid]) / 2
101
+ }
102
+
103
+ // ---------------------------------------------------------------------------
104
+ // Static (authoring) quality — the "audit" dimension.
105
+ // ---------------------------------------------------------------------------
106
+
107
+ const STATIC_CHECKS = [
108
+ {
109
+ id: 'desc-length',
110
+ weight: 12,
111
+ label: 'description length',
112
+ evaluate: (s) => {
113
+ const n = (s.description ?? '').length
114
+ if (n >= 60) return 1
115
+ if (n >= 30) return 0.5
116
+ return 0
117
+ },
118
+ },
119
+ {
120
+ id: 'desc-trigger',
121
+ weight: 12,
122
+ label: 'description states when to use it',
123
+ evaluate: (s) =>
124
+ /use when|when to use|triggers?|适用|适用于|当用户|用于|使用场景/i.test(s.description ?? '') ? 1 : 0,
125
+ },
126
+ {
127
+ id: 'when-to-use',
128
+ weight: 8,
129
+ label: 'explicit whenToUse field',
130
+ evaluate: (s) => ((s.whenToUse ?? '').trim().length > 0 ? 1 : 0),
131
+ },
132
+ {
133
+ id: 'body-present',
134
+ weight: 10,
135
+ label: 'body is substantive',
136
+ evaluate: (s) => {
137
+ const n = s.bodyLines ?? 0
138
+ if (n >= 15) return 1
139
+ if (n >= 5) return 0.5
140
+ return 0
141
+ },
142
+ },
143
+ {
144
+ id: 'body-compact',
145
+ weight: 18,
146
+ label: 'body stays under the 500-line guidance',
147
+ evaluate: (s) => {
148
+ const n = s.bodyLines ?? 0
149
+ if (n === 0) return 0
150
+ if (n <= 500) return 1
151
+ return clamp(1 - (n - 500) / 1000, 0, 1)
152
+ },
153
+ },
154
+ {
155
+ id: 'sections',
156
+ weight: 10,
157
+ label: 'body is sectioned',
158
+ evaluate: (s) => {
159
+ const n = s.sectionCount ?? 0
160
+ if (n >= 2) return 1
161
+ if (n === 1) return 0.5
162
+ return 0
163
+ },
164
+ },
165
+ {
166
+ id: 'progressive-disclosure',
167
+ weight: 15,
168
+ label: 'delegates detail to references/ or scripts/',
169
+ evaluate: (s) =>
170
+ /references\/|scripts\/|assets\/|##\s*(Reference|References|参考|参考资料)/i.test(s.content ?? '') ? 1 : 0,
171
+ },
172
+ {
173
+ id: 'no-placeholder',
174
+ weight: 10,
175
+ label: 'no TODO/TBD placeholders',
176
+ evaluate: (s) => (/\b(TODO|TBD|FIXME|XXX)\b/.test(s.content ?? '') ? 0 : 1),
177
+ },
178
+ {
179
+ id: 'has-example',
180
+ weight: 5,
181
+ label: 'contains a usage example',
182
+ evaluate: (s) => (((s.content ?? '').match(/```/g)?.length ?? 0) >= 2 ? 1 : 0),
183
+ },
184
+ {
185
+ id: 'anti-trigger',
186
+ weight: 5,
187
+ label: 'states when NOT to use it',
188
+ evaluate: (s) =>
189
+ /do not use|don'?t use|not for|never use|不(?:要|应|适用)用于|不用于|避免(?:用于|使用)|不要用在/i.test(
190
+ `${s.description ?? ''}\n${s.whenToUse ?? ''}\n${s.content ?? ''}`,
191
+ )
192
+ ? 1
193
+ : 0,
194
+ },
195
+ {
196
+ id: 'skill-name',
197
+ weight: 4,
198
+ label: 'skill id is a kebab-case name',
199
+ evaluate: (s) => (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(s.name ?? '') ? 1 : 0),
200
+ },
201
+ {
202
+ id: 'reference-integrity',
203
+ weight: 8,
204
+ label: 'every referenced file exists',
205
+ // null = cannot be checked here (no filesystem view); the check is then
206
+ // excluded from the static score instead of pretending to pass.
207
+ evaluate: (s) => {
208
+ const integrity = s.integrity
209
+ if (!integrity || typeof integrity.checked !== 'number') return null
210
+ if (integrity.checked === 0) return 1
211
+ const missing = integrity.missing?.length ?? 0
212
+ return (integrity.checked - missing) / integrity.checked
213
+ },
214
+ },
215
+ ]
216
+
217
+ /**
218
+ * Build the static profile from a skill summary + loaded definition.
219
+ * Returns null when the skill cannot be resolved at all (deleted provider etc.).
220
+ */
221
+ export function staticProfile(summary, definition, options = {}) {
222
+ if (!summary) return null
223
+ const content = typeof definition?.content === 'string' ? definition.content : ''
224
+ const bodyLines = content.length === 0 ? 0 : content.split('\n').length
225
+ const sectionCount = (content.match(/^##\s+\S/gm) ?? []).length
226
+ const view = {
227
+ name: summary.name,
228
+ description: summary.description ?? '',
229
+ whenToUse: summary.whenToUse ?? '',
230
+ content,
231
+ bodyLines,
232
+ sectionCount,
233
+ source: summary.source ?? null,
234
+ provider: summary.provider ?? null,
235
+ path: summary.path ?? definition?.path ?? null,
236
+ resolved: content.length > 0,
237
+ integrity: options.integrity ?? null,
238
+ }
239
+ const raw = STATIC_CHECKS.map((c) => {
240
+ const evaluated = c.evaluate(view)
241
+ if (evaluated === null || evaluated === undefined) {
242
+ return { id: c.id, label: c.label, weight: c.weight, fraction: null, points: 0, skipped: true }
243
+ }
244
+ const fraction = clamp(Number(evaluated) || 0, 0, 1)
245
+ return { id: c.id, label: c.label, weight: c.weight, fraction, points: round(c.weight * fraction, 1) }
246
+ })
247
+ const checks = raw
248
+ const counted = checks.filter((c) => !c.skipped)
249
+ const earned = counted.reduce((a, c) => a + c.weight * c.fraction, 0)
250
+ const totalWeight = counted.reduce((a, c) => a + c.weight, 0)
251
+ // The full body is only needed while the checks run; persisting it would
252
+ // duplicate every SKILL.md into the append-only store and every report JSON.
253
+ const { content: _body, ...persisted } = view
254
+ return {
255
+ ...persisted,
256
+ checks,
257
+ score: round((earned / totalWeight) * 100, 1),
258
+ failedChecks: counted.filter((c) => c.fraction < 1),
259
+ }
260
+ }
261
+
262
+ // ---------------------------------------------------------------------------
263
+ // Runtime aggregation over invocation samples.
264
+ // ---------------------------------------------------------------------------
265
+
266
+ /**
267
+ * Aggregate closed+open invocation samples for one skill.
268
+ *
269
+ * sample = {
270
+ * ok: boolean, // did the `skill` tool call succeed
271
+ * loadMs?: number, // duration of the load call
272
+ * error?: string, // load error message
273
+ * window?: { // the observation window after a successful load
274
+ * toolCalls: number,
275
+ * errors: number,
276
+ * errorNames: string[],
277
+ * corrections: string[],
278
+ * otherSkills: string[],
279
+ * turnEnded: boolean,
280
+ * truncated: boolean,
281
+ * } | null
282
+ * }
283
+ */
284
+ export function aggregate(samples, options = {}) {
285
+ const safe = Array.isArray(samples) ? samples.filter(Boolean) : []
286
+ const loads = safe.length
287
+ const loadOk = safe.filter((s) => s.ok).length
288
+ const closed = safe.filter((s) => s.window)
289
+ const windows = closed.map((s) => s.window)
290
+
291
+ const count = (value) => {
292
+ const n = Number(value)
293
+ return Number.isFinite(n) && n > 0 ? n : 0
294
+ }
295
+ const toolCalls = windows.reduce((a, w) => a + count(w.toolCalls), 0)
296
+ const errors = windows.reduce((a, w) => a + count(w.errors), 0)
297
+ const withCorrection = windows.filter((w) => (Array.isArray(w.corrections) ? w.corrections.length : 0) > 0).length
298
+ const withSibling = windows.filter((w) => (Array.isArray(w.otherSkills) ? w.otherSkills.length : 0) > 0).length
299
+ // A subagent that loads a skill and finishes immediately is not the same
300
+ // failure as a root session dropping a skill: judge abandonment on root runs.
301
+ const rootWindows = windows.filter((w) => w.origin !== 'subagent')
302
+ // A window force-closed for memory reasons is censored, not abandoned: it
303
+ // must not turn "we ran out of tracked sessions" into an effectiveness claim.
304
+ const abandonBase = (rootWindows.length > 0 ? rootWindows : windows).filter((w) => w.closeReason !== 'evicted')
305
+ const abandonedRoot = abandonBase.filter((w) => count(w.toolCalls) === 0).length
306
+
307
+ const withLoopError = windows.filter((w) => count(w.loopErrors) > 0).length
308
+ const loopErrors = windows.reduce((a, w) => a + count(w.loopErrors), 0)
309
+ const llmRetries = windows.reduce((a, w) => a + count(w.llmRetries), 0)
310
+ const loopErrorNames = Object.create(null)
311
+ for (const w of windows) {
312
+ if (typeof w.lastLoopError === 'string' && w.lastLoopError) {
313
+ loopErrorNames[w.lastLoopError] = (loopErrorNames[w.lastLoopError] ?? 0) + 1
314
+ }
315
+ }
316
+
317
+ // MCP health is independent of which skill was loaded: merge every window's
318
+ // per-server bucket so the report can show a board per MCP server.
319
+ const mcp = Object.create(null)
320
+ for (const w of windows) {
321
+ for (const [server, bucket] of Object.entries(w.mcp ?? {})) {
322
+ if (!bucket || typeof bucket !== 'object') continue
323
+ if (!Object.hasOwn(mcp, server)) {
324
+ mcp[server] = { calls: 0, errors: 0, ms: 0, windows: 0, samples: [], tools: Object.create(null) }
325
+ }
326
+ const entry = mcp[server]
327
+ entry.calls += count(bucket.calls)
328
+ entry.errors += count(bucket.errors)
329
+ entry.ms += count(bucket.ms)
330
+ entry.windows += 1
331
+ if (Array.isArray(bucket.samples)) {
332
+ for (const sample of bucket.samples) {
333
+ if (typeof sample === 'number' && entry.samples.length < 400) entry.samples.push(sample)
334
+ }
335
+ }
336
+ for (const [tool, stats] of Object.entries(bucket.tools ?? {})) {
337
+ const t = Object.hasOwn(entry.tools, tool) ? entry.tools[tool] : { calls: 0, errors: 0, ms: 0, samples: [] }
338
+ t.calls += count(stats.calls)
339
+ t.errors += count(stats.errors)
340
+ t.ms = (t.ms ?? 0) + count(stats.ms)
341
+ if (Array.isArray(stats.samples)) {
342
+ for (const sample of stats.samples) {
343
+ if (typeof sample === 'number' && t.samples.length < 60) t.samples.push(sample)
344
+ }
345
+ }
346
+ entry.tools[tool] = t
347
+ }
348
+ mcp[server] = entry
349
+ }
350
+ }
351
+
352
+ const corrections = windows.flatMap((w) =>
353
+ Array.isArray(w.corrections) ? w.corrections.filter((c) => typeof c === 'string') : [],
354
+ )
355
+ const siblings = Object.create(null)
356
+ for (const w of windows) {
357
+ for (const s of Array.isArray(w.otherSkills) ? w.otherSkills : []) {
358
+ if (typeof s === 'string' && s) siblings[s] = (siblings[s] ?? 0) + 1
359
+ }
360
+ }
361
+ const errorNames = Object.create(null)
362
+ for (const w of windows) {
363
+ for (const e of Array.isArray(w.errorNames) ? w.errorNames : []) {
364
+ if (typeof e === 'string' && e) errorNames[e] = (errorNames[e] ?? 0) + 1
365
+ }
366
+ }
367
+
368
+ const tokens = { input: 0, output: 0, cacheRead: 0, calls: 0 }
369
+ for (const w of windows) {
370
+ const t = w.tokens
371
+ if (!t || typeof t !== 'object') continue
372
+ tokens.input += Number(t.input) || 0
373
+ tokens.output += Number(t.output) || 0
374
+ tokens.cacheRead += Number(t.cacheRead) || 0
375
+ tokens.calls += Number(t.calls) || 0
376
+ }
377
+ const cacheWeight = Number.isFinite(options.cacheReadWeight) ? options.cacheReadWeight : 0
378
+ const billable = tokens.input + tokens.output + tokens.cacheRead * cacheWeight
379
+ // Only windows that actually carried model usage count: older windows were
380
+ // recorded before token capture existed and would dilute the per-load cost.
381
+ const tokenWindows = windows.filter((w) => w.tokens && (Number(w.tokens.calls) || 0) > 0).length
382
+ const tokensPerLoad = tokenWindows > 0 && tokens.calls > 0 ? billable / tokenWindows : null
383
+
384
+ return {
385
+ loads,
386
+ closed: closed.length,
387
+ tokens,
388
+ tokenWindows,
389
+ tokensPerLoad: tokensPerLoad === null ? null : Math.round(tokensPerLoad),
390
+ billableTokens: Math.round(billable),
391
+ open: loads - closed.length,
392
+ loadOk,
393
+ loadOkRate: loads > 0 ? loadOk / loads : null,
394
+ loadErrors: safe.map((s) => s.error).filter(Boolean),
395
+ toolCalls,
396
+ errors,
397
+ errRate: toolCalls > 0 ? errors / toolCalls : null,
398
+ mcp,
399
+ loopErrors,
400
+ llmRetries,
401
+ loopErrorRate: windows.length > 0 ? withLoopError / windows.length : null,
402
+ loopErrorNames,
403
+ correctionRate: windows.length > 0 ? withCorrection / windows.length : null,
404
+ abandonRate: abandonBase.length > 0 ? abandonedRoot / abandonBase.length : null,
405
+ abandonBase: abandonBase.length,
406
+ siblingRate: windows.length > 0 ? withSibling / windows.length : null,
407
+ avgCalls: windows.length > 0 ? toolCalls / windows.length : null,
408
+ p50LoadMs: median(safe.map((s) => s.loadMs)),
409
+ corrections: corrections.slice(-5),
410
+ siblings,
411
+ errorNames,
412
+ }
413
+ }
414
+
415
+ // ---------------------------------------------------------------------------
416
+ // Scoring.
417
+ // ---------------------------------------------------------------------------
418
+
419
+ function reliabilityScore(stats) {
420
+ if (stats.loads === 0) return null
421
+ const loadOk = stats.loadOkRate ?? 0
422
+ // A 50% post-load tool error rate zeroes the error half.
423
+ const cleanWindow = stats.errRate === null ? 1 : 1 - clamp(stats.errRate * 2, 0, 1)
424
+ // The agent loop itself failing right after a load (a broken step, a failed
425
+ // model request, an aborted batch) is the skill's problem too, but it is a
426
+ // weaker signal than a tool error, so it costs at most 40% of the dimension.
427
+ const loopPenalty = 1 - 0.4 * clamp(stats.loopErrorRate ?? 0, 0, 1)
428
+ // A load failure makes the skill unusable, so it multiplies the whole
429
+ // dimension instead of only the load half: 0% load success = 0 reliability.
430
+ return clamp(100 * loadOk * (0.6 + 0.4 * cleanWindow) * loopPenalty, 0, 100)
431
+ }
432
+
433
+ function effectivenessScore(stats) {
434
+ if (stats.closed === 0) return null
435
+ let score = 100
436
+ score -= 45 * (stats.correctionRate ?? 0)
437
+ score -= 25 * (stats.abandonRate ?? 0)
438
+ score -= 20 * (stats.siblingRate ?? 0)
439
+ score -= 10 * clamp(stats.errRate ?? 0, 0, 1)
440
+ return clamp(score, 0, 100)
441
+ }
442
+
443
+ function efficiencyScore(stats, tokenBudgetPerLoad = 20000) {
444
+ if (stats.closed === 0) return null
445
+ let score = 100
446
+ const calls = stats.avgCalls ?? 0
447
+ if (calls > 8) score -= Math.min(40, (calls - 8) * 6)
448
+ if (calls < 1) score -= 15
449
+ const ms = stats.p50LoadMs
450
+ if (Number.isFinite(ms) && ms > 2000) score -= Math.min(30, (ms - 2000) / 100)
451
+ // Measured billable tokens per load: double the budget costs 30 points.
452
+ const perLoad = stats.tokensPerLoad
453
+ if (Number.isFinite(perLoad) && perLoad > tokenBudgetPerLoad) {
454
+ score -= Math.min(35, ((perLoad - tokenBudgetPerLoad) / tokenBudgetPerLoad) * 30)
455
+ }
456
+ return clamp(score, 0, 100)
457
+ }
458
+
459
+ /**
460
+ * Composite score. Dimensions that have no evidence (no samples / unresolvable
461
+ * skill) are dropped and their weight is redistributed over the rest, so a
462
+ * brand-new skill is not punished for having no history.
463
+ */
464
+ export function scoreSkill({
465
+ stats,
466
+ static: staticDim,
467
+ weights = DEFAULT_WEIGHTS,
468
+ thresholds = DEFAULT_THRESHOLDS,
469
+ minSamples = 3,
470
+ tokenBudgetPerLoad = 20000,
471
+ }) {
472
+ const dims = {
473
+ static: staticDim?.score ?? null,
474
+ reliability: reliabilityScore(stats),
475
+ effectiveness: effectivenessScore(stats),
476
+ efficiency: efficiencyScore(stats, tokenBudgetPerLoad),
477
+ }
478
+ let weighted = 0
479
+ let weightSum = 0
480
+ const used = {}
481
+ for (const key of Object.keys(dims)) {
482
+ const value = dims[key]
483
+ if (value === null || value === undefined || !Number.isFinite(value)) continue
484
+ const w = Number(weights[key] ?? 0)
485
+ if (w <= 0) continue
486
+ weighted += value * w
487
+ weightSum += w
488
+ used[key] = w
489
+ }
490
+ const total = weightSum > 0 ? round(weighted / weightSum, 1) : null
491
+ const confidence =
492
+ stats.loads === 0 ? 'none' : stats.loads < minSamples ? 'low' : stats.loads < minSamples * 3 ? 'medium' : 'high'
493
+ return {
494
+ total,
495
+ grade: gradeOf(total, thresholds),
496
+ provisional: stats.loads < minSamples,
497
+ confidence,
498
+ samples: stats.loads,
499
+ dimensions: dims,
500
+ effectiveWeights: used,
501
+ coverage: round(weightSum, 2),
502
+ }
503
+ }
504
+
505
+ /**
506
+ * Bootstrap interval for a rate measured over windows (correction rate, error
507
+ * rate, ...). Advice uses it as a noise floor: a rule only keeps its severity
508
+ * when the interval's low end is already past the threshold.
509
+ */
510
+ export function bootstrapRate(windows, isHit, options = {}) {
511
+ const rows = Array.isArray(windows) ? windows.filter(Boolean) : []
512
+ const resamples = Number.isFinite(options.resamples) ? options.resamples : 400
513
+ if (rows.length < 2 || resamples <= 0) return null
514
+ const random = mulberry32(Number.isFinite(options.seed) ? options.seed : 1)
515
+ const rates = []
516
+ for (let i = 0; i < resamples; i += 1) {
517
+ let hits = 0
518
+ for (let j = 0; j < rows.length; j += 1) if (isHit(rows[Math.floor(random() * rows.length)])) hits += 1
519
+ rates.push(hits / rows.length)
520
+ }
521
+ rates.sort((a, b) => a - b)
522
+ const at = (p) => rates[Math.min(rates.length - 1, Math.max(0, Math.round(p * (rates.length - 1))))]
523
+ const mean = rows.filter(isHit).length / rows.length
524
+ return { mean, low: at(0.025), high: at(0.975), resamples: rates.length, windows: rows.length }
525
+ }