dsh-skill-eval 0.0.0-stage → 1.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +675 -0
- package/LICENSE +21 -0
- package/README.md +200 -2
- package/README.zh.md +157 -0
- package/client.js +2637 -0
- package/cordis.patch.yml +9 -0
- package/docs/ACCEPTANCE.md +71 -0
- package/docs/AUDIT.md +64 -0
- package/docs/CODE-REVIEW.md +49 -0
- package/docs/COMPARISON.md +33 -0
- package/docs/DETAIL.md +463 -0
- package/docs/DEV-RELOAD.md +114 -0
- package/docs/ENTRYPOINTS.md +65 -0
- package/docs/RELEASE.md +44 -0
- package/docs/RRSI-NOTES.md +129 -0
- package/docs/SCENARIOS.md +91 -0
- package/docs/UI-REVIEW.md +115 -0
- package/docs/WORKFLOW.md +108 -0
- package/icon.svg +9 -0
- package/index.js +1111 -0
- package/lib/accept.js +350 -0
- package/lib/advice.js +547 -0
- package/lib/critic.js +101 -0
- package/lib/dev.js +148 -0
- package/lib/history.js +140 -0
- package/lib/html.js +8 -0
- package/lib/inspect.js +142 -0
- package/lib/integrity.js +46 -0
- package/lib/loop.js +195 -0
- package/lib/random.js +29 -0
- package/lib/recorder.js +634 -0
- package/lib/render.js +582 -0
- package/lib/report.js +646 -0
- package/lib/score.js +525 -0
- package/lib/settings.js +396 -0
- package/lib/snapshot.js +166 -0
- package/lib/steps.js +141 -0
- package/lib/store.js +186 -0
- package/lib/validity.js +115 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +96 -4
- package/scripts/accept.mjs +226 -0
- package/scripts/build-client.mjs +59 -0
- package/scripts/build.mjs +81 -0
- package/scripts/doctor.mjs +199 -0
- package/scripts/lib.mjs +52 -0
- package/scripts/verify-live.mjs +168 -0
package/lib/advice.js
ADDED
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rule-based optimization advice. Every rule is anchored to a measured signal
|
|
3
|
+
* (and carries the evidence that triggered it) so the receiving agent can act
|
|
4
|
+
* on it instead of being told "improve the skill".
|
|
5
|
+
*
|
|
6
|
+
* severity: 'critical' | 'warning' | 'info'
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { bootstrapRate } from './score.js'
|
|
10
|
+
|
|
11
|
+
const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 }
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Chinese copy for every rule title/fix. Evidence stays measured data (ids,
|
|
15
|
+
* counts, percentages), so it is deliberately not translated. `{server}` and
|
|
16
|
+
* `{tool}` are filled from the rule item at build time.
|
|
17
|
+
*/
|
|
18
|
+
const ADVICE_ZH = {
|
|
19
|
+
'load-failed': {
|
|
20
|
+
title: '技能在运行时加载失败',
|
|
21
|
+
fix: '复现失败的加载(front-matter 非法、缺 SKILL.md、资源目录不可读),修好技能,让 `skill` 工具能返回内容。',
|
|
22
|
+
},
|
|
23
|
+
'post-load-errors': {
|
|
24
|
+
title: '加载该技能后工具错误激增',
|
|
25
|
+
fix: '把正文假设的命令/路径写全,加一段预检,并给出有记录的回退路径,让模型加载后不再靠猜。',
|
|
26
|
+
},
|
|
27
|
+
'token-cost': {
|
|
28
|
+
title: '加载该技能占用的策略 token 过多',
|
|
29
|
+
fix: '砍掉注入内容(精简正文、把细节移到 references/),或让它在任务更晚的阶段才激活;成本必须由实测收益覆盖。',
|
|
30
|
+
},
|
|
31
|
+
'loop-errors': {
|
|
32
|
+
title: '加载该技能后 agent 循环中断',
|
|
33
|
+
fix: '正文把模型带进了不可达状态(不存在的工具/命令、无法执行的指令)。核对正文点名的每个工具与命令,删掉循环执行不了的步骤。',
|
|
34
|
+
},
|
|
35
|
+
'llm-retries': {
|
|
36
|
+
title: '模型请求持续失败并重试',
|
|
37
|
+
fix: '通常是加载的技能把上下文顶出窗口,或产生格式错误的工具调用;精简正文与引用,或拆分技能。',
|
|
38
|
+
},
|
|
39
|
+
'user-corrections': {
|
|
40
|
+
title: '用户在该技能运行后立即否定结果',
|
|
41
|
+
fix: '正文很可能忽略了已声明的约束。收紧输出契约和"不要做什么"清单,并加一段回答前的自检。',
|
|
42
|
+
},
|
|
43
|
+
'dead-load': {
|
|
44
|
+
title: '加载的技能经常被忽略',
|
|
45
|
+
fix: '让技能首屏变成命令式清单(“第 1 步用工具 Y 做 X”)。只讲背景的正文会被读完就丢。',
|
|
46
|
+
},
|
|
47
|
+
'trigger-collision': {
|
|
48
|
+
title: '触发面与另一个技能重叠',
|
|
49
|
+
fix: '把它和冲突技能的 description/whenToUse 区分开,并加明确的反触发(“不要用于 …”)。',
|
|
50
|
+
},
|
|
51
|
+
'verbose-execution': {
|
|
52
|
+
title: '执行时工具调用过多',
|
|
53
|
+
fix: '把重复步骤收进 scripts/ 写成一个脚本,让技能只调用一次,而不是每次重新推导流程。',
|
|
54
|
+
},
|
|
55
|
+
'slow-load': {
|
|
56
|
+
title: '加载该技能很慢',
|
|
57
|
+
fix: '技能文件可能太大。把示例/参考资料移到 references/,让默认加载保持精简。',
|
|
58
|
+
},
|
|
59
|
+
'prune-candidate': {
|
|
60
|
+
title: '该技能的成本高于收益——删除或合并',
|
|
61
|
+
fix: '删除这个技能,或把它仍然有用的那一步并入真正被使用的技能。持续低分的技能每次加载都在消耗上下文并稀释触发精度。',
|
|
62
|
+
},
|
|
63
|
+
'body-too-long': {
|
|
64
|
+
title: '技能正文超过 500 行的建议上限',
|
|
65
|
+
fix: '拆到 `references/*.md` 按需加载,顶层正文只做路由。',
|
|
66
|
+
},
|
|
67
|
+
'weak-description': {
|
|
68
|
+
title: '描述太短,无法稳定触发',
|
|
69
|
+
fix: '在 description 本身(不只是正文)写清能力、适用场景和操作对象。',
|
|
70
|
+
},
|
|
71
|
+
'no-trigger-language': {
|
|
72
|
+
title: '描述从未说明何时使用该技能',
|
|
73
|
+
fix: '补一个 “Use when …” 分句,写上应当触发它的具体用户话术。',
|
|
74
|
+
},
|
|
75
|
+
'no-progressive-disclosure': {
|
|
76
|
+
title: '所有内容都堆在一个文件里',
|
|
77
|
+
fix: '长参考资料移到 `references/`,可执行步骤移到 `scripts/`,再从正文指向它们。',
|
|
78
|
+
},
|
|
79
|
+
'no-example': {
|
|
80
|
+
title: '没有具体示例',
|
|
81
|
+
fix: '加一个最小可运行示例;模型照抄示例的可靠性远高于遵循散文。',
|
|
82
|
+
},
|
|
83
|
+
'broken-references': {
|
|
84
|
+
title: '技能引用了不存在的文件',
|
|
85
|
+
fix: '补上缺失文件或修正路径。把模型指向不存在的引用会浪费上下文,通常以猜测作答收场。',
|
|
86
|
+
},
|
|
87
|
+
'no-anti-trigger': {
|
|
88
|
+
title: '没有明确的“不适用”边界',
|
|
89
|
+
fix: '加一行:“Do not use for X / 不适用于 Y”。明确的反触发是触发精度的静态一半。',
|
|
90
|
+
},
|
|
91
|
+
'bad-skill-name': {
|
|
92
|
+
title: '技能 id 不是 kebab-case',
|
|
93
|
+
fix: '改成小写单词用连字符连接(agentskills.io 约定);不合规 id 会破坏跨客户端发现。',
|
|
94
|
+
},
|
|
95
|
+
'placeholder-text': {
|
|
96
|
+
title: '正文残留占位符',
|
|
97
|
+
fix: '解决或删除占位符;模型会把它们当成自由发挥的许可。',
|
|
98
|
+
},
|
|
99
|
+
'unresolved-skill': {
|
|
100
|
+
title: '无法解析技能定义',
|
|
101
|
+
fix: '先确认技能仍存在,再依据运行时数字行动。',
|
|
102
|
+
},
|
|
103
|
+
'low-sample': {
|
|
104
|
+
title: '证据还不足',
|
|
105
|
+
fix: '把分数当作临时值;保持插件启用,等更多运行后再读。',
|
|
106
|
+
},
|
|
107
|
+
'mcp-server-errors': {
|
|
108
|
+
title: 'MCP 服务器 “{server}” 的调用失败率偏高',
|
|
109
|
+
fix: '检查服务器进程(鉴权 token、启动命令、版本)以及工具 schema 是否仍然匹配;死掉的服务器会悄悄烧掉整轮运行。',
|
|
110
|
+
},
|
|
111
|
+
'mcp-server-slow': {
|
|
112
|
+
title: 'MCP 服务器 “{server}” 尾部延迟高',
|
|
113
|
+
fix: '优先用该服务器上最便宜的工具、缓存结果,或把调用移出关键路径;p95 3 秒+ 的工具会把一步变成一分钟。',
|
|
114
|
+
},
|
|
115
|
+
'mcp-tool-errors': {
|
|
116
|
+
title: 'MCP 工具 “{server}/{tool}” 不可靠',
|
|
117
|
+
fix: '检查该技能传给它参数;schema 不匹配是反复失败的常见原因。',
|
|
118
|
+
},
|
|
119
|
+
'mcp-tool-slow': {
|
|
120
|
+
title: 'MCP 工具 “{server}/{tool}” 尾部慢',
|
|
121
|
+
fix: '传更窄的参数(过滤、limit)、复用结果,或每轮只调一次;尾部延迟才是把聊天变成等待的原因。',
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function withZhCopy(item) {
|
|
126
|
+
const zh = ADVICE_ZH[item.id]
|
|
127
|
+
if (!zh) return item
|
|
128
|
+
const fill = (text) => text.replaceAll('{server}', item.server ?? '').replaceAll('{tool}', item.tool ?? '')
|
|
129
|
+
return { ...item, titleZh: fill(zh.title), fixZh: fill(zh.fix) }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Pick the display text for one advice item. */
|
|
133
|
+
export function localizeAdvice(item, lang) {
|
|
134
|
+
if (!item || lang !== 'zh' || !item.titleZh) return item
|
|
135
|
+
return { ...item, title: item.titleZh, fix: item.fixZh ?? item.fix }
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Localize every advice list in a report (used by the HTML board). */
|
|
139
|
+
export function localizeReport(report, lang) {
|
|
140
|
+
if (!report || lang !== 'zh') return report
|
|
141
|
+
const loc = (item) => localizeAdvice(item, lang)
|
|
142
|
+
return {
|
|
143
|
+
...report,
|
|
144
|
+
skills: (report.skills ?? []).map((skill) => ({
|
|
145
|
+
...skill,
|
|
146
|
+
advice: (skill.advice ?? []).map(loc),
|
|
147
|
+
topAdvice: (skill.topAdvice ?? []).map(loc),
|
|
148
|
+
})),
|
|
149
|
+
mcpAdvice: (report.mcpAdvice ?? []).map(loc),
|
|
150
|
+
ranking: report.ranking
|
|
151
|
+
? { ...report.ranking, note: report.ranking.noteZh ?? report.ranking.note }
|
|
152
|
+
: report.ranking,
|
|
153
|
+
validity: report.validity
|
|
154
|
+
? {
|
|
155
|
+
...report.validity,
|
|
156
|
+
canSay: report.validity.canSayZh ?? report.validity.canSay,
|
|
157
|
+
cannotSay: report.validity.cannotSayZh ?? report.validity.cannotSay,
|
|
158
|
+
}
|
|
159
|
+
: report.validity,
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function pct(x) {
|
|
164
|
+
return x === null || x === undefined ? 'n/a' : `${Math.round(x * 100)}%`
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function num(x, digits = 1) {
|
|
168
|
+
return x === null || x === undefined ? 'n/a' : Number(x).toFixed(digits)
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function topKeys(map, limit = 3) {
|
|
172
|
+
return Object.entries(map ?? {})
|
|
173
|
+
.sort((a, b) => b[1] - a[1])
|
|
174
|
+
.slice(0, limit)
|
|
175
|
+
.map(([k, v]) => `${k}×${v}`)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Noise-floor gate: keep the rule's severity only when the measured rate is
|
|
180
|
+
* already outside the bootstrap interval at that threshold. Otherwise report it
|
|
181
|
+
* as info ("not outside noise yet") instead of asking for a fix on noise.
|
|
182
|
+
*/
|
|
183
|
+
function gated(windows, isHit, threshold, resamples) {
|
|
184
|
+
// No window-level evidence: keep the historical behaviour (report the mean).
|
|
185
|
+
if (!Array.isArray(windows) || windows.length === 0) return { level: 'strong', ci: null }
|
|
186
|
+
const ci = bootstrapRate(windows, isHit, { resamples, seed: 17 })
|
|
187
|
+
if (!ci) return { level: 'mean', ci: null }
|
|
188
|
+
if (ci.low > threshold) return { level: 'strong', ci }
|
|
189
|
+
if (ci.mean > threshold) return { level: 'weak', ci }
|
|
190
|
+
return { level: 'none', ci }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function noiseNote(ci, threshold, level) {
|
|
194
|
+
if (!ci) return ''
|
|
195
|
+
const base = ` (95% interval ${Math.round(ci.low * 100)}-${Math.round(ci.high * 100)}%, threshold ${Math.round(
|
|
196
|
+
threshold * 100,
|
|
197
|
+
)}%`
|
|
198
|
+
return level === 'strong' ? `${base})` : `${base} — not outside noise yet)`
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export function buildAdvice({
|
|
202
|
+
stats,
|
|
203
|
+
static: sp,
|
|
204
|
+
score,
|
|
205
|
+
minSamples = 3,
|
|
206
|
+
windows,
|
|
207
|
+
gating = true,
|
|
208
|
+
tokenBudgetPerLoad = 20000,
|
|
209
|
+
pruneMinLoads = 6,
|
|
210
|
+
pruneMaxScore = 60,
|
|
211
|
+
}) {
|
|
212
|
+
const out = []
|
|
213
|
+
const gate = (isHit, threshold) => (gating ? gated(windows, isHit, threshold, 400) : { level: 'strong', ci: null })
|
|
214
|
+
|
|
215
|
+
// ---- runtime evidence rules -------------------------------------------
|
|
216
|
+
|
|
217
|
+
if (stats.loadErrors.length > 0) {
|
|
218
|
+
out.push({
|
|
219
|
+
id: 'load-failed',
|
|
220
|
+
severity: 'critical',
|
|
221
|
+
dimension: 'reliability',
|
|
222
|
+
title: 'Skill load fails at runtime',
|
|
223
|
+
evidence: `${stats.loads - stats.loadOk}/${stats.loads} loads returned an error: ${stats.loadErrors
|
|
224
|
+
.slice(-3)
|
|
225
|
+
.join(' | ')}`,
|
|
226
|
+
fix: 'Reproduce the failing load (invalid front-matter, missing SKILL.md, unreadable resource base) and repair the skill so the `skill` tool returns content.',
|
|
227
|
+
})
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const errGate = gate((w) => (w.errors || 0) > 0, 0.25)
|
|
231
|
+
if (stats.errRate !== null && stats.errRate > 0.25 && errGate.level !== 'none') {
|
|
232
|
+
out.push({
|
|
233
|
+
id: 'post-load-errors',
|
|
234
|
+
severity: errGate.level === 'strong' ? 'critical' : 'info',
|
|
235
|
+
dimension: 'reliability',
|
|
236
|
+
title: 'Tool errors spike after this skill is loaded',
|
|
237
|
+
evidence: `${pct(stats.errRate)} of the ${stats.toolCalls} tool calls inside the window failed${noiseNote(errGate.ci, 0.25, errGate.level)}${
|
|
238
|
+
Object.keys(stats.errorNames).length ? ` (${topKeys(stats.errorNames).join(', ')})` : ''
|
|
239
|
+
}`,
|
|
240
|
+
fix: 'Add the exact commands/paths the body assumes, a preflight check section, and a documented fallback path, so the model stops guessing after loading.',
|
|
241
|
+
})
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
if (stats.tokensPerLoad !== null && stats.tokensPerLoad > tokenBudgetPerLoad * 1.25) {
|
|
245
|
+
out.push({
|
|
246
|
+
id: 'token-cost',
|
|
247
|
+
severity: 'warning',
|
|
248
|
+
dimension: 'efficiency',
|
|
249
|
+
title: 'Loading this skill is expensive in policy tokens',
|
|
250
|
+
evidence: `about ${Math.round(stats.tokensPerLoad).toLocaleString()} billable tokens per load across ${
|
|
251
|
+
(stats.tokens && stats.tokens.calls) || 0
|
|
252
|
+
} model call(s) (budget ${Math.round(tokenBudgetPerLoad).toLocaleString()})`,
|
|
253
|
+
fix: 'Cut what the skill injects (trim the body, move detail to references/) or make it activate later in the task; cost must be paid for by measured gain.',
|
|
254
|
+
})
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
if (stats.loopErrorRate !== null && stats.loopErrorRate > 0.2) {
|
|
258
|
+
out.push({
|
|
259
|
+
id: 'loop-errors',
|
|
260
|
+
severity: 'critical',
|
|
261
|
+
dimension: 'reliability',
|
|
262
|
+
title: 'The agent loop breaks after this skill is loaded',
|
|
263
|
+
evidence: `${pct(stats.loopErrorRate)} of runs hit an agent-loop error after the load${
|
|
264
|
+
Object.keys(stats.loopErrorNames ?? {}).length ? ` (${topKeys(stats.loopErrorNames).join(', ')})` : ''
|
|
265
|
+
}`,
|
|
266
|
+
fix: 'The body is steering the model into an unreachable state (a tool or command that does not exist, an instruction that cannot be followed). Check every tool/command the body names and remove steps the loop cannot execute.',
|
|
267
|
+
})
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
if (stats.llmRetries > 0 && stats.closed > 0 && stats.llmRetries / stats.closed >= 2) {
|
|
271
|
+
out.push({
|
|
272
|
+
id: 'llm-retries',
|
|
273
|
+
severity: 'warning',
|
|
274
|
+
dimension: 'reliability',
|
|
275
|
+
title: 'Model requests keep failing and retrying inside the run',
|
|
276
|
+
evidence: `${stats.llmRetries} retried model request(s) across ${stats.closed} run(s)`,
|
|
277
|
+
fix: 'Usually the loaded skill pushes the context over the window or emits a malformed tool call; trim the body and its references, or split the skill.',
|
|
278
|
+
})
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const correctionGate = gate((w) => (w.corrections?.length ?? 0) > 0, 0.3)
|
|
282
|
+
if (stats.correctionRate !== null && stats.correctionRate > 0.3 && correctionGate.level !== 'none') {
|
|
283
|
+
out.push({
|
|
284
|
+
id: 'user-corrections',
|
|
285
|
+
severity: correctionGate.level === 'strong' ? 'critical' : 'info',
|
|
286
|
+
dimension: 'effectiveness',
|
|
287
|
+
title: 'The user rejects the result right after the skill runs',
|
|
288
|
+
evidence: `${pct(stats.correctionRate)} of runs were followed by a correcting message (e.g. “${
|
|
289
|
+
stats.corrections[0] ?? ''
|
|
290
|
+
}”)${noiseNote(correctionGate.ci, 0.3, correctionGate.level)}`,
|
|
291
|
+
fix: 'The body likely ignores stated constraints. Tighten the output contract and the "do not" list; add a short self-check the model must run before answering.',
|
|
292
|
+
})
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const abandonGate = gate((w) => (w.toolCalls || 0) === 0, 0.3)
|
|
296
|
+
if (stats.abandonRate !== null && stats.abandonRate > 0.3 && abandonGate.level !== 'none') {
|
|
297
|
+
out.push({
|
|
298
|
+
id: 'dead-load',
|
|
299
|
+
severity: abandonGate.level === 'strong' ? 'warning' : 'info',
|
|
300
|
+
dimension: 'effectiveness',
|
|
301
|
+
title: 'Loaded skills are often ignored',
|
|
302
|
+
evidence: `${pct(stats.abandonRate)} of runs made zero tool calls after the load and produced no follow-through${noiseNote(abandonGate.ci, 0.3, abandonGate.level)}`,
|
|
303
|
+
fix: 'Make the first screen of the skill an imperative checklist ("step 1 do X with tool Y"). A body that only explains background gets read and dropped.',
|
|
304
|
+
})
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
const siblingGate = gate((w) => (w.otherSkills?.length ?? 0) > 0, 0.3)
|
|
308
|
+
if (stats.siblingRate !== null && stats.siblingRate > 0.3 && siblingGate.level !== 'none') {
|
|
309
|
+
out.push({
|
|
310
|
+
id: 'trigger-collision',
|
|
311
|
+
severity: siblingGate.level === 'strong' ? 'warning' : 'info',
|
|
312
|
+
dimension: 'effectiveness',
|
|
313
|
+
title: 'Trigger surface overlaps another skill',
|
|
314
|
+
evidence: `${pct(stats.siblingRate)} of runs loaded another skill in the same window: ${topKeys(
|
|
315
|
+
stats.siblings,
|
|
316
|
+
).join(', ')}${noiseNote(siblingGate.ci, 0.3, siblingGate.level)}`,
|
|
317
|
+
fix: 'Differentiate the description/whenToUse from the colliding skill and add explicit anti-triggers ("do not use for …").',
|
|
318
|
+
})
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if (stats.avgCalls !== null && stats.avgCalls > 8) {
|
|
322
|
+
out.push({
|
|
323
|
+
id: 'verbose-execution',
|
|
324
|
+
severity: 'info',
|
|
325
|
+
dimension: 'efficiency',
|
|
326
|
+
title: 'Execution is tool-call heavy',
|
|
327
|
+
evidence: `average ${num(stats.avgCalls)} tool calls per run after load`,
|
|
328
|
+
fix: 'Bundle the repetitive steps into a script under `scripts/` and have the skill call it once instead of describing a procedure to re-derive each run.',
|
|
329
|
+
})
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if (stats.p50LoadMs !== null && stats.p50LoadMs > 2000) {
|
|
333
|
+
out.push({
|
|
334
|
+
id: 'slow-load',
|
|
335
|
+
severity: 'info',
|
|
336
|
+
dimension: 'efficiency',
|
|
337
|
+
title: 'Loading the skill is slow',
|
|
338
|
+
evidence: `median load ${num(stats.p50LoadMs, 0)} ms`,
|
|
339
|
+
fix: 'The skill file is probably very large. Move examples/reference material into `references/` so the default load stays small.',
|
|
340
|
+
})
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// ---- pruning (RRSI: components that stop earning their place are removed) ----
|
|
344
|
+
const total = typeof score?.total === 'number' ? score.total : null
|
|
345
|
+
const ci = score?.ci ?? null
|
|
346
|
+
if (total !== null && stats.loads >= pruneMinLoads && total < pruneMaxScore && (!ci || ci.high < pruneMaxScore + 5)) {
|
|
347
|
+
out.push({
|
|
348
|
+
id: 'prune-candidate',
|
|
349
|
+
severity: 'critical',
|
|
350
|
+
dimension: 'lifecycle',
|
|
351
|
+
title: 'This skill costs more than it pays back — prune or merge it',
|
|
352
|
+
evidence: `${stats.loads} load(s), score ${total}/100${ci ? ` (95% interval ${ci.low}-${ci.high})` : ''}, corrections ${pct(
|
|
353
|
+
stats.correctionRate,
|
|
354
|
+
)}, ignored ${pct(stats.abandonRate)}`,
|
|
355
|
+
fix: 'Delete the skill, or fold the one step that still helps into the skill that actually gets used. Keeping a consistently low-scoring skill spends context on every load and dilutes trigger precision.',
|
|
356
|
+
})
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// ---- authoring (static) rules ----------------------------------------
|
|
360
|
+
|
|
361
|
+
if (sp) {
|
|
362
|
+
for (const check of sp.failedChecks ?? []) {
|
|
363
|
+
if (check.id === 'body-compact' && check.fraction <= 0.5) {
|
|
364
|
+
out.push({
|
|
365
|
+
id: 'body-too-long',
|
|
366
|
+
severity: 'warning',
|
|
367
|
+
dimension: 'static',
|
|
368
|
+
title: 'Skill body exceeds the 500-line guidance',
|
|
369
|
+
evidence: `${sp.bodyLines} lines of body`,
|
|
370
|
+
fix: 'Split into `references/*.md` loaded on demand and keep the top-level body as a router.',
|
|
371
|
+
})
|
|
372
|
+
}
|
|
373
|
+
if (check.id === 'desc-length' && check.fraction < 1) {
|
|
374
|
+
out.push({
|
|
375
|
+
id: 'weak-description',
|
|
376
|
+
severity: 'warning',
|
|
377
|
+
dimension: 'static',
|
|
378
|
+
title: 'Description is too short to trigger reliably',
|
|
379
|
+
evidence: `description is ${String(sp.description ?? '').length} chars`,
|
|
380
|
+
fix: 'State the capability, the situation that calls for it, and the objects it operates on — in the description itself, not only in the body.',
|
|
381
|
+
})
|
|
382
|
+
}
|
|
383
|
+
if (check.id === 'desc-trigger' && check.fraction < 1) {
|
|
384
|
+
out.push({
|
|
385
|
+
id: 'no-trigger-language',
|
|
386
|
+
severity: 'warning',
|
|
387
|
+
dimension: 'static',
|
|
388
|
+
title: 'Description never says when to use the skill',
|
|
389
|
+
evidence: `description: “${String(sp.description ?? '').slice(0, 120)}”`,
|
|
390
|
+
fix: 'Add a "Use when …" clause with the concrete user phrasings that should activate it.',
|
|
391
|
+
})
|
|
392
|
+
}
|
|
393
|
+
if (check.id === 'progressive-disclosure' && check.fraction < 1) {
|
|
394
|
+
out.push({
|
|
395
|
+
id: 'no-progressive-disclosure',
|
|
396
|
+
severity: 'info',
|
|
397
|
+
dimension: 'static',
|
|
398
|
+
title: 'Everything lives in one file',
|
|
399
|
+
evidence: 'no `references/`, `scripts/` or `assets/` usage detected',
|
|
400
|
+
fix: 'Move long reference material to `references/` and executable steps to `scripts/`, then point at them from the body.',
|
|
401
|
+
})
|
|
402
|
+
}
|
|
403
|
+
if (check.id === 'has-example' && check.fraction < 1) {
|
|
404
|
+
out.push({
|
|
405
|
+
id: 'no-example',
|
|
406
|
+
severity: 'info',
|
|
407
|
+
dimension: 'static',
|
|
408
|
+
title: 'No concrete example',
|
|
409
|
+
evidence: 'no fenced code block in the body',
|
|
410
|
+
fix: 'Add one minimal worked example; models copy examples far more reliably than they follow prose.',
|
|
411
|
+
})
|
|
412
|
+
}
|
|
413
|
+
if (check.id === 'reference-integrity' && check.fraction < 1) {
|
|
414
|
+
const missing = (sp.integrity && sp.integrity.missing) || []
|
|
415
|
+
out.push({
|
|
416
|
+
id: 'broken-references',
|
|
417
|
+
severity: 'critical',
|
|
418
|
+
dimension: 'static',
|
|
419
|
+
title: 'The skill references files that do not exist',
|
|
420
|
+
evidence: `missing: ${missing.slice(0, 5).join(', ') || 'referenced paths not found'} (${sp.integrity?.checked ?? 0} referenced)`,
|
|
421
|
+
fix: 'Create the missing files or fix the paths. A body that sends the model to a nonexistent reference wastes the scene and usually ends in a guessed answer.',
|
|
422
|
+
})
|
|
423
|
+
}
|
|
424
|
+
if (check.id === 'anti-trigger' && check.fraction < 1) {
|
|
425
|
+
out.push({
|
|
426
|
+
id: 'no-anti-trigger',
|
|
427
|
+
severity: 'info',
|
|
428
|
+
dimension: 'static',
|
|
429
|
+
title: 'No explicit "not for this" boundary',
|
|
430
|
+
evidence: 'description and body never state when the skill should not be used',
|
|
431
|
+
fix: 'Add one line: "Do not use for X / 不适用于 Y". Explicit anti-triggers are the static half of trigger precision.',
|
|
432
|
+
})
|
|
433
|
+
}
|
|
434
|
+
if (check.id === 'skill-name' && check.fraction < 1) {
|
|
435
|
+
out.push({
|
|
436
|
+
id: 'bad-skill-name',
|
|
437
|
+
severity: 'warning',
|
|
438
|
+
dimension: 'static',
|
|
439
|
+
title: 'Skill id is not kebab-case',
|
|
440
|
+
evidence: `id: ${sp.name}`,
|
|
441
|
+
fix: 'Rename to lowercase words joined by hyphens (agentskills.io contract); non-conforming ids break cross-client discovery.',
|
|
442
|
+
})
|
|
443
|
+
}
|
|
444
|
+
if (check.id === 'no-placeholder' && check.fraction < 1) {
|
|
445
|
+
out.push({
|
|
446
|
+
id: 'placeholder-text',
|
|
447
|
+
severity: 'warning',
|
|
448
|
+
dimension: 'static',
|
|
449
|
+
title: 'Placeholder text left in the body',
|
|
450
|
+
evidence: 'TODO/TBD/FIXME/XXX present',
|
|
451
|
+
fix: 'Resolve or delete placeholders; a model treats them as licence to improvise.',
|
|
452
|
+
})
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
} else {
|
|
456
|
+
out.push({
|
|
457
|
+
id: 'unresolved-skill',
|
|
458
|
+
severity: 'info',
|
|
459
|
+
dimension: 'static',
|
|
460
|
+
title: 'Skill definition could not be resolved',
|
|
461
|
+
evidence: 'the skill is not in the current catalog (removed, renamed, or provider offline)',
|
|
462
|
+
fix: 'Confirm the skill still exists before acting on the runtime numbers.',
|
|
463
|
+
})
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
if (stats.loads < minSamples) {
|
|
467
|
+
out.push({
|
|
468
|
+
id: 'low-sample',
|
|
469
|
+
severity: 'info',
|
|
470
|
+
dimension: 'confidence',
|
|
471
|
+
title: 'Not enough evidence yet',
|
|
472
|
+
evidence: `${stats.loads} recorded load(s), ${minSamples} needed for a stable score`,
|
|
473
|
+
fix: 'Treat the score as provisional; keep the plugin enabled and re-read after more runs.',
|
|
474
|
+
})
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
const withZh = out.map(withZhCopy)
|
|
478
|
+
withZh.sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity])
|
|
479
|
+
return withZh
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
export function topAdvice(all, limit = 5) {
|
|
483
|
+
return [...all].sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]).slice(0, limit)
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* MCP servers are shared infrastructure, so their health does not belong to one
|
|
488
|
+
* skill's advice list. These rules run over the aggregated MCP board.
|
|
489
|
+
*/
|
|
490
|
+
export function buildMcpAdvice(servers, options = {}) {
|
|
491
|
+
const errorRateLimit = typeof options.errorRate === 'number' ? options.errorRate : 0.2
|
|
492
|
+
const p95LimitMs = typeof options.p95Ms === 'number' ? options.p95Ms : 3000
|
|
493
|
+
const toolErrorRateLimit = typeof options.toolErrorRate === 'number' ? options.toolErrorRate : 0.3
|
|
494
|
+
const toolP95LimitMs = typeof options.toolP95Ms === 'number' ? options.toolP95Ms : 5000
|
|
495
|
+
const out = []
|
|
496
|
+
for (const row of servers ?? []) {
|
|
497
|
+
if (!row || typeof row !== 'object') continue
|
|
498
|
+
const calls = row.calls ?? 0
|
|
499
|
+
const errors = row.errors ?? 0
|
|
500
|
+
const errorRate = typeof row.errorRate === 'number' ? row.errorRate : calls > 0 ? errors / calls : 0
|
|
501
|
+
if (calls > 0 && errorRate > errorRateLimit) {
|
|
502
|
+
out.push({
|
|
503
|
+
id: 'mcp-server-errors',
|
|
504
|
+
severity: 'critical',
|
|
505
|
+
server: row.server,
|
|
506
|
+
title: `MCP server "${row.server}" fails ${pct(errorRate)} of its calls`,
|
|
507
|
+
evidence: `${errors}/${calls} calls failed`,
|
|
508
|
+
fix: 'Check the server process (auth token, startup command, version) and whether the tool schema still matches; a dead server silently burns the whole run.',
|
|
509
|
+
})
|
|
510
|
+
}
|
|
511
|
+
if (row.p95Ms !== null && row.p95Ms !== undefined && row.p95Ms > p95LimitMs) {
|
|
512
|
+
out.push({
|
|
513
|
+
id: 'mcp-server-slow',
|
|
514
|
+
severity: 'warning',
|
|
515
|
+
server: row.server,
|
|
516
|
+
title: `MCP server "${row.server}" is slow at the tail`,
|
|
517
|
+
evidence: `p50 ${row.p50Ms ?? '—'}ms · p95 ${row.p95Ms}ms`,
|
|
518
|
+
fix: 'Prefer the cheapest tool on this server, cache results, or move the call off the critical path; a 3s+p95 tool turns one step into a minute.',
|
|
519
|
+
})
|
|
520
|
+
}
|
|
521
|
+
for (const tool of row.tools ?? []) {
|
|
522
|
+
const toolCalls = tool.calls ?? 0
|
|
523
|
+
const toolErrors = tool.errors ?? 0
|
|
524
|
+
if (toolCalls >= 2 && toolErrors / toolCalls > toolErrorRateLimit) {
|
|
525
|
+
out.push({
|
|
526
|
+
id: 'mcp-tool-errors',
|
|
527
|
+
severity: 'warning',
|
|
528
|
+
server: row.server,
|
|
529
|
+
title: `MCP tool "${row.server}/${tool.tool}" is unreliable`,
|
|
530
|
+
evidence: `${toolErrors}/${toolCalls} calls failed`,
|
|
531
|
+
fix: 'Check the arguments this skill passes to that tool; a schema mismatch is the usual cause of repeated failures.',
|
|
532
|
+
})
|
|
533
|
+
}
|
|
534
|
+
if (toolP95LimitMs > 0 && typeof tool.p95Ms === 'number' && tool.p95Ms > toolP95LimitMs && toolCalls >= 2) {
|
|
535
|
+
out.push({
|
|
536
|
+
id: 'mcp-tool-slow',
|
|
537
|
+
severity: 'warning',
|
|
538
|
+
server: row.server,
|
|
539
|
+
title: `MCP tool "${row.server}/${tool.tool}" has a slow tail`,
|
|
540
|
+
evidence: `p50 ${tool.p50Ms ?? '—'}ms · p95 ${tool.p95Ms}ms over ${toolCalls} call(s)`,
|
|
541
|
+
fix: 'Pass narrower arguments (filters, limits), reuse the result, or call it once per run instead of per step; tail latency is what turns a chat into a wait.',
|
|
542
|
+
})
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
return out.map(withZhCopy).sort((a, b) => (a.severity === b.severity ? 0 : a.severity === 'critical' ? -1 : 1))
|
|
547
|
+
}
|
package/lib/critic.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Leakage critic for proposed skill edits (borrowed from RRSI's selection side).
|
|
3
|
+
*
|
|
4
|
+
* RRSI screens a candidate harness edit *before* it is scored: task names,
|
|
5
|
+
* entities, answers and benchmark-specific logic are rejected so the evolver
|
|
6
|
+
* cannot memorise the evaluation set. A skill body has the same failure mode —
|
|
7
|
+
* a fix that hardcodes yesterday's task is not a reusable mechanism.
|
|
8
|
+
*
|
|
9
|
+
* This module is pure: it classifies a proposed diff/hypothesis and returns a
|
|
10
|
+
* verdict. Nothing is applied here.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const RULES = [
|
|
14
|
+
{
|
|
15
|
+
code: 'hardcoded-answer',
|
|
16
|
+
severity: 'reject',
|
|
17
|
+
label: 'hardcoded answer or expected output',
|
|
18
|
+
pattern: /\b(expected output|correct answer|the answer is|答案(?:是|为)|正确结果)\b/i,
|
|
19
|
+
detail: 'a skill must derive its output, not embed one',
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
code: 'task-specific-id',
|
|
23
|
+
severity: 'reject',
|
|
24
|
+
label: 'task-specific identifier',
|
|
25
|
+
pattern: /\b(task[-_ ]?id|benchmark|eval[-_ ]?set|case[-_ ]?#?\d+|task\s*#\d+)\b/i,
|
|
26
|
+
detail: 'references a single evaluation instance',
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
code: 'hardcoded-date',
|
|
30
|
+
severity: 'warn',
|
|
31
|
+
label: 'hardcoded date or version window',
|
|
32
|
+
pattern: /\b(20\d{2}-\d{2}-\d{2}|before 20\d{2}|after 20\d{2}|until 20\d{2})\b/,
|
|
33
|
+
detail: 'date windows rot; keep them in config instead of the skill body',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
code: 'user-specific-path',
|
|
37
|
+
severity: 'warn',
|
|
38
|
+
label: 'absolute machine-specific path',
|
|
39
|
+
pattern: /(?:\/Users\/[^/\s]+\/|\/home\/[^/\s]+\/|[A-Za-z]:\\Users\\)/,
|
|
40
|
+
detail: 'a shared skill must not depend on one machine layout',
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
code: 'benchmark-name',
|
|
44
|
+
severity: 'reject',
|
|
45
|
+
label: 'named benchmark or leaderboard',
|
|
46
|
+
pattern: /\b(swe-?bench|terminal-?bench|harvey|gdpval|apex-?agents|jobbench|frontier-?eng|engdesign)\b/i,
|
|
47
|
+
detail: 'optimising for a named benchmark is leakage, not capability',
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
code: 'silent-overfit',
|
|
51
|
+
severity: 'warn',
|
|
52
|
+
label: 'special-casing on input shape',
|
|
53
|
+
pattern: /\b(if (the )?(input|request|file) (is|looks like|matches)|when (the )?(input|file) is named)\b/i,
|
|
54
|
+
detail: 'branching on the exact input shape generalises poorly',
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
code: 'no-hypothesis',
|
|
58
|
+
severity: 'warn',
|
|
59
|
+
label: 'no falsifiable hypothesis',
|
|
60
|
+
pattern: null,
|
|
61
|
+
detail: 'state what should improve and how it will be measured (RRSI ledger)',
|
|
62
|
+
},
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
export const LEAK_RULES = RULES
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Classify one proposed skill edit.
|
|
69
|
+
*
|
|
70
|
+
* @param {{skill?: string, diff?: string, hypothesis?: string}} input
|
|
71
|
+
* @returns {{verdict: 'accept'|'warn'|'reject', reasons: Array<{code:string,severity:string,label:string,detail:string}>}}
|
|
72
|
+
*/
|
|
73
|
+
export function reviewSkillEdit(input = {}) {
|
|
74
|
+
const text = [input.diff, input.hypothesis].filter((part) => typeof part === 'string' && part.length > 0).join('\n')
|
|
75
|
+
const reasons = []
|
|
76
|
+
if (text.length === 0) {
|
|
77
|
+
reasons.push({
|
|
78
|
+
code: 'empty-edit',
|
|
79
|
+
severity: 'reject',
|
|
80
|
+
label: 'nothing to review',
|
|
81
|
+
detail: 'pass the proposed diff or at least a hypothesis',
|
|
82
|
+
})
|
|
83
|
+
}
|
|
84
|
+
for (const rule of RULES) {
|
|
85
|
+
if (rule.code === 'no-hypothesis') {
|
|
86
|
+
if (typeof input.hypothesis !== 'string' || input.hypothesis.trim().length === 0) {
|
|
87
|
+
reasons.push({ code: rule.code, severity: rule.severity, label: rule.label, detail: rule.detail })
|
|
88
|
+
}
|
|
89
|
+
continue
|
|
90
|
+
}
|
|
91
|
+
if (rule.pattern && rule.pattern.test(text)) {
|
|
92
|
+
reasons.push({ code: rule.code, severity: rule.severity, label: rule.label, detail: rule.detail })
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const verdict = reasons.some((reason) => reason.severity === 'reject')
|
|
96
|
+
? 'reject'
|
|
97
|
+
: reasons.length > 0
|
|
98
|
+
? 'warn'
|
|
99
|
+
: 'accept'
|
|
100
|
+
return { verdict, reasons }
|
|
101
|
+
}
|