efaimo 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +160 -4
- package/README.md +89 -31
- package/SKILL.md +13 -1
- package/dist/check/check.js +14 -2
- package/dist/cli.js +95 -9
- package/dist/clients/introspect.js +10 -2
- package/dist/clients/rawprobe.d.ts +21 -5
- package/dist/clients/rawprobe.js +101 -43
- package/dist/clients/stateless.js +17 -14
- package/dist/core/engine.js +20 -2
- package/dist/core/grade.js +7 -1
- package/dist/core/types.d.ts +22 -3
- package/dist/reporters/markdown.js +16 -4
- package/dist/reporters/pretty.js +38 -25
- package/dist/rules/injection.d.ts +1 -2
- package/dist/rules/injection.js +7 -11
- package/dist/rules/mcp/index.d.ts +14 -0
- package/dist/rules/mcp/index.js +89 -25
- package/dist/rules/skill/index.js +5 -3
- package/dist/skills/parse.js +39 -3
- package/dist/targets/clientConfigs.d.ts +7 -0
- package/dist/targets/clientConfigs.js +4 -1
- package/dist/targets/repoScan.js +0 -0
- package/dist/testing/anthropicRunner.js +2 -0
- package/dist/testing/harness.d.ts +22 -0
- package/dist/testing/harness.js +98 -13
- package/dist/testing/openaiRunner.js +15 -1
- package/dist/testing/stats.d.ts +30 -0
- package/dist/testing/stats.js +89 -0
- package/dist/util/dotenv.d.ts +9 -1
- package/dist/util/dotenv.js +11 -1
- package/dist/util/safeText.d.ts +8 -0
- package/dist/util/safeText.js +48 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/weigh/weigh.js +3 -1
- package/dist/weigh/window.d.ts +26 -0
- package/dist/weigh/window.js +44 -0
- package/docs/METHODOLOGY.md +70 -0
- package/docs/RULES.md +31 -25
- package/examples/contoso-crm-import/SKILL.md +21 -0
- package/examples/csv-cleanup/SKILL.md +17 -0
- package/examples/scenario.crm.yaml +22 -0
- package/examples/scenario.example.yaml +17 -0
- package/package.json +4 -2
package/dist/reporters/pretty.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import pc from "picocolors";
|
|
2
|
+
import { formatWindowShare } from "../weigh/window.js";
|
|
2
3
|
import { sortFindings } from "../core/grade.js";
|
|
3
4
|
import { VERSION } from "../version.js";
|
|
5
|
+
import { safeText } from "../util/safeText.js";
|
|
4
6
|
const DOCS_RULES = "https://github.com/efaimo-ai/efaimo/blob/main/docs/RULES.md";
|
|
5
7
|
let colorOn = true;
|
|
6
8
|
export function setColor(on) {
|
|
@@ -31,18 +33,18 @@ function n(x) {
|
|
|
31
33
|
return x.toLocaleString("en-US");
|
|
32
34
|
}
|
|
33
35
|
function renderFinding(lines, f) {
|
|
34
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${f.message}`);
|
|
36
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
35
37
|
if (f.detail) {
|
|
36
|
-
for (const d of f.detail.split("\n"))
|
|
38
|
+
for (const d of safeText(f.detail).split("\n"))
|
|
37
39
|
lines.push(paint(pc.dim, ` ${d}`));
|
|
38
40
|
}
|
|
39
41
|
if (f.fixHint)
|
|
40
|
-
lines.push(paint(pc.dim, ` fix: ${f.fixHint}`));
|
|
42
|
+
lines.push(paint(pc.dim, ` fix: ${safeText(f.fixHint)}`));
|
|
41
43
|
}
|
|
42
44
|
export function renderCheckPretty(report) {
|
|
43
45
|
const lines = [];
|
|
44
46
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
45
|
-
lines.push(`check ${report.surface} ${paint(pc.bold, report.target)}`);
|
|
47
|
+
lines.push(`check ${report.surface} ${paint(pc.bold, safeText(report.target))}`);
|
|
46
48
|
const g = report.grade;
|
|
47
49
|
const qualityLabel = report.readiness ? "quality: " : "";
|
|
48
50
|
lines.push(`grade ${gradeColor(g.letter, `${g.letter} (${g.score})`)} ${qualityLabel}` +
|
|
@@ -61,7 +63,7 @@ export function renderCheckPretty(report) {
|
|
|
61
63
|
if (r.findings.length) {
|
|
62
64
|
lines.push(paint(pc.bold, `2026-07-28 readiness`) +
|
|
63
65
|
` ${r.findings.length} item${r.findings.length === 1 ? "" : "s"} to migrate ` +
|
|
64
|
-
paint(pc.dim, "(a migration diff, not graded
|
|
66
|
+
paint(pc.dim, "(a migration diff, not graded)"));
|
|
65
67
|
for (const f of r.findings)
|
|
66
68
|
renderFinding(lines, f);
|
|
67
69
|
}
|
|
@@ -71,7 +73,7 @@ export function renderCheckPretty(report) {
|
|
|
71
73
|
}
|
|
72
74
|
lines.push("");
|
|
73
75
|
for (const note of report.notes)
|
|
74
|
-
lines.push(paint(pc.dim, `note: ${note}`));
|
|
76
|
+
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
75
77
|
lines.push(paint(pc.dim, `rules: ${DOCS_RULES}`));
|
|
76
78
|
return lines.join("\n");
|
|
77
79
|
}
|
|
@@ -98,19 +100,22 @@ export function renderTestReportPretty(r) {
|
|
|
98
100
|
`test ${paint(pc.bold, r.scenario)}`,
|
|
99
101
|
`skill ${r.skill} model ${r.model}`,
|
|
100
102
|
"",
|
|
101
|
-
` with skill ${r.withSkill.passes}/${r.withSkill.
|
|
102
|
-
` without skill ${r.withoutSkill.passes}/${r.withoutSkill.
|
|
103
|
+
` with skill ${r.withSkill.passes}/${r.withSkill.scored} pass (${r.withSkill.passRate.toFixed(0)}%)`,
|
|
104
|
+
` without skill ${r.withoutSkill.passes}/${r.withoutSkill.scored} pass (${r.withoutSkill.passRate.toFixed(0)}%)`,
|
|
103
105
|
` delta ${paint(pc.bold, `${sign}${r.deltaPoints} points`)} ${paint(verdictColor, r.verdict)}`,
|
|
106
|
+
Number.isFinite(r.ci.lo)
|
|
107
|
+
? ` significance p = ${r.p < 0.0001 ? "<0.0001" : r.p.toFixed(4)} 95% CI ${r.ci.lo >= 0 ? "+" : ""}${r.ci.lo} to ${r.ci.hi >= 0 ? "+" : ""}${r.ci.hi} points`
|
|
108
|
+
: ` significance not computable: an arm produced no scoreable trial`,
|
|
104
109
|
"",
|
|
105
110
|
];
|
|
106
111
|
for (const n of r.notes)
|
|
107
|
-
lines.push(paint(pc.dim, `note: ${n}`));
|
|
112
|
+
lines.push(paint(pc.dim, `note: ${safeText(n)}`));
|
|
108
113
|
return lines.join("\n");
|
|
109
114
|
}
|
|
110
115
|
export function renderSkillSetPretty(res) {
|
|
111
116
|
const lines = [];
|
|
112
117
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
113
|
-
lines.push(`check skills ${paint(pc.bold, res.label)} ${res.perSkill.length} skill${res.perSkill.length === 1 ? "" : "s"}`);
|
|
118
|
+
lines.push(`check skills ${paint(pc.bold, safeText(res.label))} ${res.perSkill.length} skill${res.perSkill.length === 1 ? "" : "s"}`);
|
|
114
119
|
const dist = { A: 0, B: 0, C: 0, D: 0, F: 0 };
|
|
115
120
|
for (const s of res.perSkill)
|
|
116
121
|
dist[s.report.grade.letter]++;
|
|
@@ -125,43 +130,42 @@ export function renderSkillSetPretty(res) {
|
|
|
125
130
|
for (const s of [...res.perSkill].sort((a, b) => a.report.grade.score - b.report.grade.score)) {
|
|
126
131
|
const g = s.report.grade;
|
|
127
132
|
const c = s.report.counts;
|
|
128
|
-
lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${s.name.padEnd(nameW)} ` +
|
|
133
|
+
lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${safeText(s.name).padEnd(nameW)} ` +
|
|
129
134
|
`${paint(pc.red, String(c.error))}e ${paint(pc.yellow, String(c.warn))}w ${paint(pc.cyan, String(c.info))}i`);
|
|
130
135
|
}
|
|
131
136
|
if (res.setFindings.length) {
|
|
132
137
|
lines.push("");
|
|
133
138
|
lines.push(paint(pc.bold, "across the set"));
|
|
134
139
|
for (const f of sortFindings(res.setFindings)) {
|
|
135
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${f.message}`);
|
|
140
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
136
141
|
}
|
|
137
142
|
}
|
|
138
143
|
const flagged = res.perSkill.filter((s) => s.report.findings.length);
|
|
139
144
|
if (flagged.length) {
|
|
140
145
|
lines.push("");
|
|
141
146
|
for (const s of flagged) {
|
|
142
|
-
lines.push(paint(pc.bold, `${s.name} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
|
|
147
|
+
lines.push(paint(pc.bold, `${safeText(s.name)} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
|
|
143
148
|
for (const f of s.report.findings) {
|
|
144
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${f.message}`);
|
|
149
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
145
150
|
if (f.fixHint)
|
|
146
|
-
lines.push(paint(pc.dim, ` fix: ${f.fixHint}`));
|
|
151
|
+
lines.push(paint(pc.dim, ` fix: ${safeText(f.fixHint)}`));
|
|
147
152
|
}
|
|
148
153
|
}
|
|
149
154
|
}
|
|
150
155
|
lines.push("");
|
|
151
|
-
lines.push(paint(pc.dim, `${res.perSkill.length} skills under ${res.root}`));
|
|
156
|
+
lines.push(paint(pc.dim, `${res.perSkill.length} skills under ${safeText(res.root)}`));
|
|
152
157
|
lines.push(paint(pc.dim, `rules: ${DOCS_RULES}`));
|
|
153
158
|
return lines.join("\n");
|
|
154
159
|
}
|
|
155
160
|
export function renderServerWeighPretty(w) {
|
|
156
161
|
const lines = [];
|
|
157
162
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
158
|
-
lines.push(`weigh mcp ${paint(pc.bold, w.label)}`);
|
|
163
|
+
lines.push(`weigh mcp ${paint(pc.bold, safeText(w.label))}`);
|
|
159
164
|
lines.push(`tools ${w.toolCount} resources ${w.resourceCount} prompts ${w.promptCount}`);
|
|
160
165
|
lines.push("");
|
|
161
166
|
lines.push("context cost of tool definitions (o200k tokens, estimated)");
|
|
162
|
-
const pct = ((w.totals.claudeStyle / 200000) * 100).toFixed(1);
|
|
163
167
|
lines.push(` raw JSON ${n(w.totals.rawJson).padStart(8)}`);
|
|
164
|
-
lines.push(` Claude-style ${n(w.totals.claudeStyle).padStart(8)} (
|
|
168
|
+
lines.push(` Claude-style ${n(w.totals.claudeStyle).padStart(8)} (${formatWindowShare(w.totals.claudeStyle)})`);
|
|
165
169
|
lines.push(` OpenAI tools ${n(w.totals.openaiTools).padStart(8)}`);
|
|
166
170
|
if (w.instructionsTokens > 0)
|
|
167
171
|
lines.push(` server instructions ${n(w.instructionsTokens).padStart(4)}`);
|
|
@@ -171,8 +175,17 @@ export function renderServerWeighPretty(w) {
|
|
|
171
175
|
if (w.perTool.length) {
|
|
172
176
|
lines.push("");
|
|
173
177
|
lines.push("heaviest tools (Claude-style)");
|
|
174
|
-
|
|
175
|
-
|
|
178
|
+
// Derived from the rows actually printed, not a fixed 28. Against the
|
|
179
|
+
// reference server - the target in --help, on /commands and in every
|
|
180
|
+
// launch draft - "trigger-long-running-operation" is 30 characters, ate
|
|
181
|
+
// the pad, and shifted the token column two places right for that one row.
|
|
182
|
+
// A tool whose pitch is careful measurement was printing a crooked number
|
|
183
|
+
// column in its own flagship demo. Capped so one pathological name cannot
|
|
184
|
+
// push the numbers off a terminal.
|
|
185
|
+
const shown = w.perTool.slice(0, 8);
|
|
186
|
+
const toolW = Math.min(38, Math.max(20, ...shown.map((t) => safeText(t.name).length)));
|
|
187
|
+
for (const [i, t] of shown.entries()) {
|
|
188
|
+
lines.push(` ${String(i + 1).padStart(2)}. ${safeText(t.name).padEnd(toolW)} ${n(t.tokens.claudeStyle).padStart(7)} desc ${n(t.descriptionTokens)} | schema ${n(t.schemaTokens)}`);
|
|
176
189
|
}
|
|
177
190
|
if (w.perTool.length > 8)
|
|
178
191
|
lines.push(paint(pc.dim, ` (+${w.perTool.length - 8} more)`));
|
|
@@ -182,23 +195,23 @@ export function renderServerWeighPretty(w) {
|
|
|
182
195
|
}
|
|
183
196
|
lines.push("");
|
|
184
197
|
for (const note of w.notes)
|
|
185
|
-
lines.push(paint(pc.dim, `note: ${note}`));
|
|
198
|
+
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
186
199
|
return lines.join("\n");
|
|
187
200
|
}
|
|
188
201
|
export function renderSkillWeighPretty(w) {
|
|
189
202
|
const lines = [];
|
|
190
203
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
191
|
-
lines.push(`weigh skills ${paint(pc.bold, w.label)} ${w.perSkill.length} skill${w.perSkill.length === 1 ? "" : "s"}`);
|
|
204
|
+
lines.push(`weigh skills ${paint(pc.bold, safeText(w.label))} ${w.perSkill.length} skill${w.perSkill.length === 1 ? "" : "s"}`);
|
|
192
205
|
lines.push("");
|
|
193
206
|
lines.push(` ${"skill".padEnd(28)} ${"metadata".padStart(8)} ${"body".padStart(9)} ${"lines".padStart(6)} refs`);
|
|
194
207
|
for (const s of w.perSkill) {
|
|
195
|
-
lines.push(` ${s.name.padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
|
|
208
|
+
lines.push(` ${safeText(s.name).padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
|
|
196
209
|
}
|
|
197
210
|
lines.push("");
|
|
198
211
|
lines.push(`totals: metadata ${paint(pc.bold, n(w.totals.metadata))} (always loaded) | body ${n(w.totals.body)} (on trigger) | referenced ${n(w.totals.refFiles)} (on demand)`);
|
|
199
212
|
lines.push("");
|
|
200
213
|
for (const note of w.notes)
|
|
201
|
-
lines.push(paint(pc.dim, `note: ${note}`));
|
|
214
|
+
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
202
215
|
return lines.join("\n");
|
|
203
216
|
}
|
|
204
217
|
export function renderDiffPretty(d, opts) {
|
package/dist/rules/injection.js
CHANGED
|
@@ -9,19 +9,16 @@ export const INJECTION_PATTERNS = [
|
|
|
9
9
|
{
|
|
10
10
|
key: "override-instructions",
|
|
11
11
|
re: /ignore\s+(?:all\s+|any\s+)?(?:previous|prior|earlier|above)\s+(?:instructions|messages|rules|prompts)/i,
|
|
12
|
-
severity: "error",
|
|
13
12
|
label: "attempts to override prior instructions",
|
|
14
13
|
},
|
|
15
14
|
{
|
|
16
15
|
key: "hide-from-user",
|
|
17
16
|
re: /do\s+not\s+(?:tell|reveal|mention|inform|show|disclose)\b[\s\S]{0,40}\b(?:user|human|operator)/i,
|
|
18
|
-
severity: "error",
|
|
19
17
|
label: "instructs the agent to hide behavior from the user",
|
|
20
18
|
},
|
|
21
19
|
{
|
|
22
20
|
key: "act-without-consent",
|
|
23
21
|
re: /without\s+(?:telling|asking|notifying|informing)\s+(?:the\s+)?(?:user|human)/i,
|
|
24
|
-
severity: "error",
|
|
25
22
|
label: "instructs the agent to act without user consent",
|
|
26
23
|
},
|
|
27
24
|
{
|
|
@@ -29,51 +26,43 @@ export const INJECTION_PATTERNS = [
|
|
|
29
26
|
// "post" and "token" dropped: they match API documentation ("Token counting
|
|
30
27
|
// POST /v1/...") far more than exfiltration.
|
|
31
28
|
re: /\b(?:api[_-]?key|secret|password|credential)s?\b[\s\S]{0,40}\b(?:send|forward|exfiltrat|upload|leak|transmit|include\s+it)\b/i,
|
|
32
|
-
severity: "error",
|
|
33
29
|
label: "references sending credentials or secrets somewhere",
|
|
34
30
|
},
|
|
35
31
|
{
|
|
36
32
|
key: "sensitive-file-read",
|
|
37
33
|
re: /\b(?:read|open|cat|print|upload)\b[\s\S]{0,40}(?:\.env\b|\.ssh\b|id_rsa|credentials|\.aws\b|\.npmrc\b)/i,
|
|
38
|
-
severity: "error",
|
|
39
34
|
label: "references reading sensitive local files",
|
|
40
35
|
},
|
|
41
36
|
{
|
|
42
37
|
key: "disable-safety",
|
|
43
38
|
re: /\b(?:disable|bypass|ignore|remove)\s+(?:safety|guardrails?|filters?|restrictions?|policy|policies)\b/i,
|
|
44
|
-
severity: "error",
|
|
45
39
|
label: "asks to bypass safety or policy",
|
|
46
40
|
},
|
|
47
41
|
{
|
|
48
42
|
key: "cross-tool-steering",
|
|
49
43
|
re: /before\s+(?:using|calling|running)\b[^.?!]{0,40}?\btools?\b[\s,]{1,4}(?:always\s+|first\s+|please\s+)?(?:call|run|use|fetch|invoke)\b/i,
|
|
50
|
-
severity: "warn",
|
|
51
44
|
label: "steers the agent to call another tool first (cross-tool steering)",
|
|
52
45
|
},
|
|
53
46
|
{
|
|
54
47
|
key: "templated-exfil-url",
|
|
55
48
|
// Bounded runs (no adjacent unbounded same-class quantifiers) to avoid ReDoS.
|
|
56
49
|
re: /https?:\/\/[^\s"'<>]{1,150}[?&][^\s"'<>]{0,150}(?:\$\{[^}]{1,60}\}|\{\{[^}]{1,60}\}\}|\{[^}]{1,60}\})/,
|
|
57
|
-
severity: "warn",
|
|
58
50
|
label: "URL with a templated query parameter (possible exfiltration channel)",
|
|
59
51
|
},
|
|
60
52
|
{
|
|
61
53
|
key: "zero-width-chars",
|
|
62
54
|
// eslint-disable-next-line no-control-regex
|
|
63
55
|
re: /[]/,
|
|
64
|
-
severity: "warn",
|
|
65
56
|
label: "zero-width or invisible characters present",
|
|
66
57
|
},
|
|
67
58
|
{
|
|
68
59
|
key: "hidden-html-comment",
|
|
69
60
|
re: /<!--[^>]{0,240}\b(?:must|always|never|call|run|fetch|send|ignore)\b[^>]{0,240}-->/i,
|
|
70
|
-
severity: "warn",
|
|
71
61
|
label: "HTML comment containing imperative instructions (hidden from rendered view)",
|
|
72
62
|
},
|
|
73
63
|
{
|
|
74
64
|
key: "persona-override",
|
|
75
65
|
re: /\byou\s+are\s+now\b|\bpretend\s+to\s+be\b|\bnew\s+system\s+prompt\b/i,
|
|
76
|
-
severity: "warn",
|
|
77
66
|
label: "persona or system-prompt override language",
|
|
78
67
|
},
|
|
79
68
|
];
|
|
@@ -93,7 +82,14 @@ export function scanTextForInjection(text, opts) {
|
|
|
93
82
|
// Deliberately info: these are shallow heuristics an attacker evades
|
|
94
83
|
// trivially. Surfacing them must never move a grade or fail CI, or a
|
|
95
84
|
// clean report would read as a security pass (it is not one).
|
|
85
|
+
//
|
|
86
|
+
// `graded: false` is what makes the first half of that true. It used
|
|
87
|
+
// to be a comment only, while gradeFindings charged every info finding
|
|
88
|
+
// a point, so ten heuristic hits quietly cost a whole letter -- the
|
|
89
|
+
// exact "clean report reads as a security verdict" failure this
|
|
90
|
+
// comment was written to prevent, running in reverse.
|
|
96
91
|
severity: "info",
|
|
92
|
+
graded: false,
|
|
97
93
|
title: "possible injection pattern (heuristic)",
|
|
98
94
|
message: `${opts.where}: ${p.label}`,
|
|
99
95
|
target: opts.where,
|
|
@@ -1,4 +1,18 @@
|
|
|
1
1
|
import type { McpRule } from "../../core/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Does this tool name read as destructive?
|
|
4
|
+
*
|
|
5
|
+
* This was a single `\b(delete|remove|...)\b` regex, and `_` is a word
|
|
6
|
+
* character, so `\bdelete\b` has no boundary inside `delete_file` - nor inside
|
|
7
|
+
* `deleteFile`. Every conventional MCP tool name escaped it: delete_file,
|
|
8
|
+
* deleteFile, drop_table, removeUser, kill_process, wipe_db all returned
|
|
9
|
+
* false, while only a bare `delete` or `delete-file` matched. E123 is
|
|
10
|
+
* documented in docs/RULES.md as catching "a destructive-looking tool lacking
|
|
11
|
+
* destructiveHint" and it could not fire on essentially any real server.
|
|
12
|
+
*
|
|
13
|
+
* Split on separators AND on camelCase boundaries, then test each segment.
|
|
14
|
+
*/
|
|
15
|
+
export declare function readsDestructive(name: string): boolean;
|
|
2
16
|
export declare const MCP_RULES: McpRule[];
|
|
3
17
|
/** E101-E118 are 2026-07-28 readiness rules: reported as a migration diff, not graded. */
|
|
4
18
|
export declare function isReadinessRuleId(id: string): boolean;
|
package/dist/rules/mcp/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { scanTextForInjection } from "../injection.js";
|
|
2
2
|
import { looksLikeInitGate } from "../../clients/rawprobe.js";
|
|
3
3
|
import { formatTokens, truncate } from "../../util/misc.js";
|
|
4
|
-
|
|
4
|
+
import { formatWindowShare } from "../../weigh/window.js";
|
|
5
|
+
const SPEC = "2026-07-28";
|
|
5
6
|
function isOutcome(x) {
|
|
6
7
|
return !!x && typeof x === "object" && "ok" in x;
|
|
7
8
|
}
|
|
@@ -25,7 +26,7 @@ const REPO_MATCH_RULES = [
|
|
|
25
26
|
category: "sampling",
|
|
26
27
|
severity: "warn",
|
|
27
28
|
title: "uses deprecated Sampling",
|
|
28
|
-
message: (c) => `source references sampling/createMessage (${matchStr(c)}); Sampling is deprecated in ${
|
|
29
|
+
message: (c) => `source references sampling/createMessage (${matchStr(c)}); Sampling is deprecated in ${SPEC} (SEP-2577)`,
|
|
29
30
|
fixHint: "integrate directly with an LLM provider API instead of MCP Sampling",
|
|
30
31
|
},
|
|
31
32
|
{
|
|
@@ -33,7 +34,7 @@ const REPO_MATCH_RULES = [
|
|
|
33
34
|
category: "roots",
|
|
34
35
|
severity: "warn",
|
|
35
36
|
title: "uses deprecated Roots",
|
|
36
|
-
message: (c) => `source references roots/list (${matchStr(c)}); Roots is deprecated in ${
|
|
37
|
+
message: (c) => `source references roots/list (${matchStr(c)}); Roots is deprecated in ${SPEC} (SEP-2577)`,
|
|
37
38
|
fixHint: "pass directories or files via tool parameters, resource URIs, or server configuration",
|
|
38
39
|
},
|
|
39
40
|
{
|
|
@@ -41,7 +42,7 @@ const REPO_MATCH_RULES = [
|
|
|
41
42
|
category: "sse-resume",
|
|
42
43
|
severity: "info",
|
|
43
44
|
title: "relies on removed SSE resumability",
|
|
44
|
-
message: (c) => `source references Last-Event-ID/resumability (${matchStr(c)}); stream resumability is removed in ${
|
|
45
|
+
message: (c) => `source references Last-Event-ID/resumability (${matchStr(c)}); stream resumability is removed in ${SPEC}`,
|
|
45
46
|
fixHint: "persist cross-call state behind server-minted handles passed as tool arguments",
|
|
46
47
|
},
|
|
47
48
|
{
|
|
@@ -49,7 +50,7 @@ const REPO_MATCH_RULES = [
|
|
|
49
50
|
category: "elicitation",
|
|
50
51
|
severity: "warn",
|
|
51
52
|
title: "uses legacy elicitation",
|
|
52
|
-
message: (c) => `source references elicitation/create (${matchStr(c)}); replaced in ${
|
|
53
|
+
message: (c) => `source references elicitation/create (${matchStr(c)}); replaced in ${SPEC} by MRTR results (resultType "input_required" with inputRequests)`,
|
|
53
54
|
fixHint: "return input_required results and correlate retries via requestState (SEP-2322)",
|
|
54
55
|
},
|
|
55
56
|
{
|
|
@@ -57,21 +58,21 @@ const REPO_MATCH_RULES = [
|
|
|
57
58
|
category: "session-state",
|
|
58
59
|
severity: "info",
|
|
59
60
|
title: "possible in-process session state",
|
|
60
|
-
message: (c) => `source shows session-state patterns (${matchStr(c)}); ${
|
|
61
|
+
message: (c) => `source shows session-state patterns (${matchStr(c)}); ${SPEC} statelessness expects server-minted handles passed via tool arguments`,
|
|
61
62
|
},
|
|
62
63
|
{
|
|
63
64
|
id: "E114",
|
|
64
65
|
category: "ping",
|
|
65
66
|
severity: "info",
|
|
66
67
|
title: "uses removed ping",
|
|
67
|
-
message: (c) => `source references the ping utility (${matchStr(c)}); ping is removed in ${
|
|
68
|
+
message: (c) => `source references the ping utility (${matchStr(c)}); ping is removed in ${SPEC}`,
|
|
68
69
|
},
|
|
69
70
|
{
|
|
70
71
|
id: "E115",
|
|
71
72
|
category: "subscribe",
|
|
72
73
|
severity: "info",
|
|
73
74
|
title: "uses replaced resource subscriptions",
|
|
74
|
-
message: (c) => `source references resources/subscribe (${matchStr(c)}); replaced by subscriptions/listen in ${
|
|
75
|
+
message: (c) => `source references resources/subscribe (${matchStr(c)}); replaced by subscriptions/listen in ${SPEC}`,
|
|
75
76
|
},
|
|
76
77
|
];
|
|
77
78
|
const repoMatchRules = REPO_MATCH_RULES.map((spec) => ({
|
|
@@ -105,10 +106,14 @@ const e101 = {
|
|
|
105
106
|
ruleId: "E101",
|
|
106
107
|
severity: "warn",
|
|
107
108
|
title: "legacy SDK generation",
|
|
108
|
-
message: `depends on ${s.package}${s.range ? `@${s.range}` : ""} (pre-${
|
|
109
|
+
message: `depends on ${s.package}${s.range ? `@${s.range}` : ""} (pre-${SPEC} line)`,
|
|
109
110
|
detail: s.language === "ts"
|
|
110
|
-
?
|
|
111
|
-
|
|
111
|
+
? // No maturity label here on purpose. This said "2.x beta since
|
|
112
|
+
// 2026-07" and stayed saying it after both packages cut 2.0.0 on
|
|
113
|
+
// 2026-07-27, which told people to migrate onto a beta that was
|
|
114
|
+
// not one. The package name and major line are the durable facts.
|
|
115
|
+
`the ${SPEC} revision ships as the new @modelcontextprotocol/server package (2.x)`
|
|
116
|
+
: `the ${SPEC} revision ships as mcp 2.x on PyPI`,
|
|
112
117
|
fixHint: "upgrade to the 2.x SDK line to get stateless transport and MRTR support",
|
|
113
118
|
});
|
|
114
119
|
}
|
|
@@ -126,7 +131,7 @@ const e104 = {
|
|
|
126
131
|
ruleId: "E104",
|
|
127
132
|
severity: "warn",
|
|
128
133
|
title: "uses deprecated MCP Logging",
|
|
129
|
-
message: `server declares the logging capability; MCP Logging is deprecated in ${
|
|
134
|
+
message: `server declares the logging capability; MCP Logging is deprecated in ${SPEC} (SEP-2577) and logging/setLevel is removed`,
|
|
130
135
|
fixHint: "log to stderr (stdio) or use OpenTelemetry; per-request level arrives via _meta io.modelcontextprotocol/logLevel",
|
|
131
136
|
},
|
|
132
137
|
];
|
|
@@ -135,7 +140,7 @@ const e104 = {
|
|
|
135
140
|
ruleId: "E104",
|
|
136
141
|
severity: "warn",
|
|
137
142
|
title: "uses deprecated MCP Logging",
|
|
138
|
-
message: `source references MCP logging APIs (${matchStr(count)}); deprecated in ${
|
|
143
|
+
message: `source references MCP logging APIs (${matchStr(count)}); deprecated in ${SPEC} (SEP-2577)`,
|
|
139
144
|
detail,
|
|
140
145
|
fixHint: "log to stderr (stdio) or use OpenTelemetry",
|
|
141
146
|
}));
|
|
@@ -156,11 +161,28 @@ const e105 = {
|
|
|
156
161
|
severity: "warn",
|
|
157
162
|
title: "requires the removed initialize handshake",
|
|
158
163
|
message: `the server rejected a bare stateless tools/list as not initialized (${bare.errorMessage ?? `code ${bare.errorCode}`})`,
|
|
159
|
-
detail: `${
|
|
164
|
+
detail: `${SPEC} removes initialize and sessions; stateless servers answer bare requests carrying version info in _meta. Whether an answering server is fully ${SPEC}-conformant is judged separately (E107 resultType, E118 cache fields, E106 server/discover).`,
|
|
160
165
|
fixHint: "upgrade to a 2.x SDK, or accept requests without a prior initialize",
|
|
161
166
|
},
|
|
162
167
|
];
|
|
163
168
|
}
|
|
169
|
+
if (bare.kind === "exit") {
|
|
170
|
+
// A crash on the bare request is not a timeout and not a clean answer: the
|
|
171
|
+
// server did not serve a stateless tools/list, so a 2026-07-28 client
|
|
172
|
+
// cannot talk to it. Left unhandled, this outcome produced NO finding, so
|
|
173
|
+
// a crashing server looked one item MORE migrated than one that answers
|
|
174
|
+
// with a not-initialized error.
|
|
175
|
+
return [
|
|
176
|
+
{
|
|
177
|
+
ruleId: "E105",
|
|
178
|
+
severity: "warn",
|
|
179
|
+
title: "requires the removed initialize handshake",
|
|
180
|
+
message: `the server process exited on a bare stateless tools/list (${bare.errorMessage ?? "no reply"}), so it does not answer requests without a prior initialize`,
|
|
181
|
+
detail: `${SPEC} removes initialize and sessions; a server that exits on a bare request cannot serve a ${SPEC} client. A server that answers, even with an error, is judged by E106/E107/E118 instead.`,
|
|
182
|
+
fixHint: "upgrade to a 2.x SDK, or answer requests without a prior initialize instead of exiting",
|
|
183
|
+
},
|
|
184
|
+
];
|
|
185
|
+
}
|
|
164
186
|
if (bare.kind === "timeout") {
|
|
165
187
|
return [
|
|
166
188
|
{
|
|
@@ -189,7 +211,7 @@ const e106 = {
|
|
|
189
211
|
severity: "warn",
|
|
190
212
|
title: "server/discover not implemented",
|
|
191
213
|
message: `server/discover is not implemented (${d.errorMessage ?? "method not found"})`,
|
|
192
|
-
detail: `${
|
|
214
|
+
detail: `${SPEC} servers MUST implement server/discover (SEP-2575) to advertise versions, capabilities, and identity; clients also use it as the back-compat probe.`,
|
|
193
215
|
fixHint: "the 2.x SDKs implement server/discover for you",
|
|
194
216
|
},
|
|
195
217
|
];
|
|
@@ -207,8 +229,8 @@ const e107 = {
|
|
|
207
229
|
ruleId: "E107",
|
|
208
230
|
severity: "info",
|
|
209
231
|
title: "results missing resultType",
|
|
210
|
-
message: `results do not carry the resultType field required in ${
|
|
211
|
-
detail:
|
|
232
|
+
message: `results do not carry the resultType field required in ${SPEC} ("complete" | "input_required")`,
|
|
233
|
+
detail: `${SPEC} requires resultType on every result (SEP-2322); on list results the value must be "complete". ${SPEC} clients treat missing resultType from earlier-protocol servers as "complete", so this is informational until you upgrade.`,
|
|
212
234
|
},
|
|
213
235
|
];
|
|
214
236
|
},
|
|
@@ -256,7 +278,7 @@ const e112 = {
|
|
|
256
278
|
ruleId: "E112",
|
|
257
279
|
severity: "warn",
|
|
258
280
|
title: "nondeterministic tools/list order",
|
|
259
|
-
message: "tool order differed across two fresh connections; deterministic ordering enables prompt-cache hits in
|
|
281
|
+
message: "tool order differed across two fresh connections; deterministic ordering enables prompt-cache hits in stateless hosts",
|
|
260
282
|
fixHint: "sort tools stably before returning them",
|
|
261
283
|
},
|
|
262
284
|
];
|
|
@@ -322,14 +344,26 @@ const e118 = {
|
|
|
322
344
|
title: "missing cache fields (ttlMs, cacheScope)",
|
|
323
345
|
surface: "mcp",
|
|
324
346
|
check(ctx) {
|
|
325
|
-
|
|
347
|
+
// Two measured surfaces, not one. server/discover only became cacheable in
|
|
348
|
+
// the spec that published on 2026-07-28 (DiscoverResult extends
|
|
349
|
+
// CacheableResult); under the locked RC it extended plain Result, so a
|
|
350
|
+
// server that migrated against the RC can pass tools/list and still miss
|
|
351
|
+
// this. Undefined means the surface was never measured, which is not a
|
|
352
|
+
// finding: E106 already owns "server/discover did not answer".
|
|
353
|
+
const surfaces = [];
|
|
354
|
+
if (ctx.probes?.cacheFieldsPresent === false)
|
|
355
|
+
surfaces.push("tools/list");
|
|
356
|
+
if (ctx.probes?.discoverCacheFieldsPresent === false)
|
|
357
|
+
surfaces.push("server/discover");
|
|
358
|
+
if (!surfaces.length)
|
|
326
359
|
return [];
|
|
360
|
+
const subject = surfaces.length > 1 ? `${surfaces.join(" and ")} results omit` : `${surfaces[0]} result omits`;
|
|
327
361
|
return [
|
|
328
362
|
{
|
|
329
363
|
ruleId: "E118",
|
|
330
364
|
severity: "warn",
|
|
331
365
|
title: "missing cache fields (ttlMs, cacheScope)",
|
|
332
|
-
message:
|
|
366
|
+
message: `${subject} ttlMs and/or cacheScope, which ${SPEC} requires on list and resource-read results (SEP-2549, CacheableResult)`,
|
|
333
367
|
detail: "required on tools/list, prompts/list, resources/list, resources/read, resources/templates/list, and server/discover; cacheScope is \"public\" or \"private\"",
|
|
334
368
|
fixHint: "return ttlMs and cacheScope on these results so clients can cache and stop polling; the 2.x SDKs add them for you",
|
|
335
369
|
},
|
|
@@ -421,7 +455,34 @@ const e122 = {
|
|
|
421
455
|
return capList(findings, 5).shown;
|
|
422
456
|
},
|
|
423
457
|
};
|
|
424
|
-
|
|
458
|
+
// "format" was in this list and is dropped. It only ever meant "format a
|
|
459
|
+
// disk" in the author's head, but `format_date`, `format_output` and
|
|
460
|
+
// `format_json` are among the most common tool names there are, and once the
|
|
461
|
+
// matcher below started working the word turned into pure noise. Nobody could
|
|
462
|
+
// have found that while the rule was firing on nothing.
|
|
463
|
+
const DESTRUCTIVE_WORDS = new Set([
|
|
464
|
+
"delete", "remove", "drop", "purge", "destroy", "overwrite",
|
|
465
|
+
"truncate", "erase", "wipe", "revoke", "reset", "deploy", "kill",
|
|
466
|
+
]);
|
|
467
|
+
/**
|
|
468
|
+
* Does this tool name read as destructive?
|
|
469
|
+
*
|
|
470
|
+
* This was a single `\b(delete|remove|...)\b` regex, and `_` is a word
|
|
471
|
+
* character, so `\bdelete\b` has no boundary inside `delete_file` - nor inside
|
|
472
|
+
* `deleteFile`. Every conventional MCP tool name escaped it: delete_file,
|
|
473
|
+
* deleteFile, drop_table, removeUser, kill_process, wipe_db all returned
|
|
474
|
+
* false, while only a bare `delete` or `delete-file` matched. E123 is
|
|
475
|
+
* documented in docs/RULES.md as catching "a destructive-looking tool lacking
|
|
476
|
+
* destructiveHint" and it could not fire on essentially any real server.
|
|
477
|
+
*
|
|
478
|
+
* Split on separators AND on camelCase boundaries, then test each segment.
|
|
479
|
+
*/
|
|
480
|
+
export function readsDestructive(name) {
|
|
481
|
+
return name
|
|
482
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
483
|
+
.split(/[^A-Za-z0-9]+/)
|
|
484
|
+
.some((seg) => DESTRUCTIVE_WORDS.has(seg.toLowerCase()));
|
|
485
|
+
}
|
|
425
486
|
const e123 = {
|
|
426
487
|
id: "E123",
|
|
427
488
|
title: "missing tool annotations",
|
|
@@ -441,7 +502,7 @@ const e123 = {
|
|
|
441
502
|
fixHint: "annotations let hosts gate confirmations and parallelize safely",
|
|
442
503
|
});
|
|
443
504
|
}
|
|
444
|
-
const risky = tools.filter((t) =>
|
|
505
|
+
const risky = tools.filter((t) => readsDestructive(t.name) && !(t.annotations && "destructiveHint" in t.annotations));
|
|
445
506
|
for (const t of capList(risky, 5).shown) {
|
|
446
507
|
findings.push({
|
|
447
508
|
ruleId: "E123",
|
|
@@ -482,7 +543,10 @@ const e124 = {
|
|
|
482
543
|
}
|
|
483
544
|
}
|
|
484
545
|
if (totalParams > 5 && undocumented / totalParams > 0.3) {
|
|
485
|
-
|
|
546
|
+
// Prepend, not append: capList keeps the first 5, and on the worst schemas
|
|
547
|
+
// (5+ oversized enums) an appended summary was exactly the line that got
|
|
548
|
+
// dropped. The aggregate is the headline, so it leads.
|
|
549
|
+
findings.unshift({
|
|
486
550
|
ruleId: "E124",
|
|
487
551
|
severity: "warn",
|
|
488
552
|
title: "schema issues",
|
|
@@ -586,7 +650,7 @@ const e128 = {
|
|
|
586
650
|
ruleId: "E128",
|
|
587
651
|
severity: "warn",
|
|
588
652
|
title: "total context tax",
|
|
589
|
-
message: `tool definitions alone are ~${formatTokens(total)} tokens (
|
|
653
|
+
message: `tool definitions alone are ~${formatTokens(total)} tokens (${formatWindowShare(total)}, estimated) before any work happens`,
|
|
590
654
|
fixHint: "trim descriptions/schemas, split the server, or rely on deferred tool loading",
|
|
591
655
|
},
|
|
592
656
|
];
|
|
@@ -597,7 +661,7 @@ const e128 = {
|
|
|
597
661
|
ruleId: "E128",
|
|
598
662
|
severity: "warn",
|
|
599
663
|
title: "total context tax",
|
|
600
|
-
message: `tool definitions are ~${formatTokens(total)} tokens (
|
|
664
|
+
message: `tool definitions are ~${formatTokens(total)} tokens (${formatWindowShare(total)})`,
|
|
601
665
|
fixHint: "run \`efaimo weigh\` for the per-tool breakdown",
|
|
602
666
|
},
|
|
603
667
|
];
|
|
@@ -16,8 +16,10 @@ const s101 = {
|
|
|
16
16
|
const push = (severity, message, fixHint) => findings.push({ ruleId: "S101", severity, title: "frontmatter does not match the spec", message, target: label, fixHint });
|
|
17
17
|
if (skill.parseError) {
|
|
18
18
|
push("error", `${skill.file}: ${skill.parseError}`, "SKILL.md must start with a --- YAML frontmatter block");
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
// No early return on empty frontmatter. Returning here after one error let
|
|
20
|
+
// a skill with NO frontmatter grade higher than one with BROKEN frontmatter
|
|
21
|
+
// (the required-field checks below never ran), so total absence outscored a
|
|
22
|
+
// typo by two errors. Absent fields ARE missing fields; report them too.
|
|
21
23
|
}
|
|
22
24
|
const name = skill.frontmatter.name;
|
|
23
25
|
if (typeof name !== "string" || !name.trim()) {
|
|
@@ -254,7 +256,7 @@ const s106 = {
|
|
|
254
256
|
fixHint: "agents will fail mid-task when they try to open it",
|
|
255
257
|
});
|
|
256
258
|
}
|
|
257
|
-
else if (ref.
|
|
259
|
+
else if (ref.escapes) {
|
|
258
260
|
findings.push({
|
|
259
261
|
ruleId: "S106",
|
|
260
262
|
severity: "warn",
|
package/dist/skills/parse.js
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import YAML from "yaml";
|
|
4
|
-
import { walkFiles } from "../util/misc.js";
|
|
4
|
+
import { walkFiles, readTextSafe } from "../util/misc.js";
|
|
5
|
+
/**
|
|
6
|
+
* Cap on a single SKILL.md. Referenced files were already capped at 512KB by
|
|
7
|
+
* readTextSafe while the body itself went through an uncapped readFileSync:
|
|
8
|
+
* the safer path had the limit and the main one did not. A 35MB SKILL.md
|
|
9
|
+
* tokenized to 7.2 million tokens at 367MB resident, and auditing skills you
|
|
10
|
+
* have NOT installed yet is the stated use case, so this is untrusted input
|
|
11
|
+
* by construction. 2MB is far above any real skill (the largest in the graded
|
|
12
|
+
* corpus is a few tens of KB) and far below a memory problem.
|
|
13
|
+
*/
|
|
14
|
+
const MAX_SKILL_BYTES = 2 * 1024 * 1024;
|
|
5
15
|
const MD_LINK_RE = /\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
|
|
6
16
|
// Only backticked strings that look like a real intra-skill path: a directory
|
|
7
17
|
// separator, no glob star. A bare filename in prose (`package.json`, `*.py`) is
|
|
@@ -21,7 +31,15 @@ export function parseSkillFile(file) {
|
|
|
21
31
|
};
|
|
22
32
|
let raw;
|
|
23
33
|
try {
|
|
24
|
-
|
|
34
|
+
// Capped, like the referenced files this same module already caps at 512KB
|
|
35
|
+
// via readTextSafe. The body went through plain readFileSync with no limit
|
|
36
|
+
// while the SAFER path had a cap, and the stated use case is auditing
|
|
37
|
+
// skills BEFORE you install them, which is untrusted input by definition.
|
|
38
|
+
// A 35MB SKILL.md tokenized to 7.2M tokens and 367MB resident.
|
|
39
|
+
const capped = readTextSafe(file, MAX_SKILL_BYTES);
|
|
40
|
+
if (capped === undefined)
|
|
41
|
+
throw new Error("unreadable");
|
|
42
|
+
raw = capped;
|
|
25
43
|
}
|
|
26
44
|
catch (e) {
|
|
27
45
|
return { ...base, parseError: `cannot read file: ${e.message}` };
|
|
@@ -79,7 +97,25 @@ export function parseSkillFile(file) {
|
|
|
79
97
|
continue;
|
|
80
98
|
seen.add(cleaned);
|
|
81
99
|
const resolved = path.resolve(dir, cleaned);
|
|
82
|
-
|
|
100
|
+
// Does this reference stay inside the skill? Absolute paths, URLs and
|
|
101
|
+
// globs are already rejected above; `../` was not, and weigh.ts then
|
|
102
|
+
// READ every referenced file to count its tokens. A skill fetched from
|
|
103
|
+
// anywhere could point at ../../../.ssh/id_rsa and get back an
|
|
104
|
+
// existence-and-size oracle, published if --md goes to a CI job summary.
|
|
105
|
+
// S106 already knew this was wrong and warned about it - but only after
|
|
106
|
+
// the read had happened, and `weigh` does not run S106 at all.
|
|
107
|
+
//
|
|
108
|
+
// Recorded here rather than filtered, so S106 can still report it and
|
|
109
|
+
// the reader still learns the reference exists. weigh skips the read.
|
|
110
|
+
const rel = path.relative(dir, resolved);
|
|
111
|
+
const escapes = rel.startsWith("..") || path.isAbsolute(rel);
|
|
112
|
+
base.referencedPaths.push({
|
|
113
|
+
raw: cleaned,
|
|
114
|
+
resolved,
|
|
115
|
+
exists: fs.existsSync(resolved),
|
|
116
|
+
escapes,
|
|
117
|
+
source,
|
|
118
|
+
});
|
|
83
119
|
}
|
|
84
120
|
}
|
|
85
121
|
for (const f of walkFiles(dir, { maxDepth: 4 })) {
|