efaimo 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -2
- package/README.md +88 -30
- package/SKILL.md +13 -1
- package/dist/check/check.js +6 -2
- package/dist/cli.js +78 -4
- package/dist/clients/rawprobe.d.ts +21 -5
- package/dist/clients/rawprobe.js +101 -43
- package/dist/clients/stateless.js +17 -14
- package/dist/core/engine.js +20 -2
- package/dist/core/grade.js +7 -1
- package/dist/core/types.d.ts +22 -3
- package/dist/reporters/markdown.js +16 -4
- package/dist/reporters/pretty.js +36 -25
- package/dist/rules/injection.js +7 -0
- package/dist/rules/mcp/index.d.ts +14 -0
- package/dist/rules/mcp/index.js +68 -24
- package/dist/skills/parse.js +39 -3
- package/dist/targets/clientConfigs.d.ts +7 -0
- package/dist/targets/clientConfigs.js +4 -1
- package/dist/testing/anthropicRunner.js +2 -0
- package/dist/testing/harness.d.ts +21 -0
- package/dist/testing/harness.js +62 -11
- package/dist/testing/openaiRunner.js +8 -1
- package/dist/testing/stats.d.ts +32 -0
- package/dist/testing/stats.js +91 -0
- package/dist/util/dotenv.d.ts +9 -1
- package/dist/util/dotenv.js +11 -1
- package/dist/util/safeText.d.ts +8 -0
- package/dist/util/safeText.js +48 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/weigh/weigh.js +3 -1
- package/dist/weigh/window.d.ts +26 -0
- package/dist/weigh/window.js +44 -0
- package/docs/METHODOLOGY.md +70 -0
- package/docs/RULES.md +29 -23
- package/examples/contoso-crm-import/SKILL.md +21 -0
- package/examples/csv-cleanup/SKILL.md +17 -0
- package/examples/scenario.crm.yaml +22 -0
- package/examples/scenario.example.yaml +17 -0
- package/package.json +3 -1
package/dist/clients/rawprobe.js
CHANGED
|
@@ -8,12 +8,19 @@ import { VERSION } from "../version.js";
|
|
|
8
8
|
* stateless requests, server/discover) need raw transport access.
|
|
9
9
|
* Every probe fails soft into a { skipped } marker.
|
|
10
10
|
*/
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* The MCP revision efaimo audits against. It was called RC_VERSION until
|
|
13
|
+
* 2026-08-01, by which point the revision had been published for four days and
|
|
14
|
+
* the name had been lying for all of them. scripts/spec-drift.mjs reads this
|
|
15
|
+
* declaration by name, so renaming it again means updating that regex, which
|
|
16
|
+
* fails loudly rather than silently watching nothing.
|
|
17
|
+
*/
|
|
18
|
+
export const SPEC_VERSION = "2026-07-28";
|
|
12
19
|
const LEGACY_VERSION = "2025-06-18";
|
|
13
|
-
export function
|
|
20
|
+
export function specMeta() {
|
|
14
21
|
return {
|
|
15
22
|
_meta: {
|
|
16
|
-
"io.modelcontextprotocol/protocolVersion":
|
|
23
|
+
"io.modelcontextprotocol/protocolVersion": SPEC_VERSION,
|
|
17
24
|
"io.modelcontextprotocol/clientCapabilities": {},
|
|
18
25
|
},
|
|
19
26
|
};
|
|
@@ -36,11 +43,20 @@ function toOutcome(reply, timeoutMessage) {
|
|
|
36
43
|
* inconclusive; a crash is its own signal. This keeps E105 from false-flagging a
|
|
37
44
|
* healthy but slow server, the worst verdict a readiness checker can emit.
|
|
38
45
|
*
|
|
39
|
-
* Detection is by message, not by error code
|
|
46
|
+
* Detection is by message, not by error code, and the spec that published on
|
|
47
|
+
* 2026-07-28 is the argument for why. The codes SDKs use here are
|
|
40
48
|
* implementation-defined (TS StreamableHTTP sends -32000 for both "Server not
|
|
41
49
|
* initialized" and "Mcp-Session-Id header is required", -32001 for "Session not
|
|
42
|
-
* found"),
|
|
43
|
-
*
|
|
50
|
+
* found"), and the MCP-defined ones MOVED between the locked RC and the
|
|
51
|
+
* published spec: MissingRequiredClientCapability went -32003 to -32021 and
|
|
52
|
+
* UnsupportedProtocolVersion went -32004 to -32022, and HeaderMismatch moved
|
|
53
|
+
* the same way, -32001 to -32020. This comment said HeaderMismatch was "added
|
|
54
|
+
* alongside them", which was wrong: the RC already defined it at -32001 in the
|
|
55
|
+
* transport prose, and what the published revision added was its entry in the
|
|
56
|
+
* schema. The distinction matters to anyone still returning -32001.
|
|
57
|
+
* -32002 (resource not found) is retired outright,
|
|
58
|
+
* replaced by -32602 and promised never to be reused. Matching on numbers
|
|
59
|
+
* would have rotted here on 2026-07-28; matching on messages did not.
|
|
44
60
|
*/
|
|
45
61
|
export function looksLikeInitGate(o) {
|
|
46
62
|
if (o.kind !== "error")
|
|
@@ -192,11 +208,60 @@ function hasCacheFields(result) {
|
|
|
192
208
|
return (Object.prototype.hasOwnProperty.call(result, "ttlMs") &&
|
|
193
209
|
Object.prototype.hasOwnProperty.call(result, "cacheScope"));
|
|
194
210
|
}
|
|
211
|
+
/** JSON-RPC's own "no such method", unchanged by the 2026-07-28 revision. */
|
|
212
|
+
const METHOD_NOT_FOUND = -32601;
|
|
213
|
+
/**
|
|
214
|
+
* One reading of a server/discover answer, because there are three call sites
|
|
215
|
+
* (stdio, the bare stateless HTTP path, the legacy HTTP path) and they had
|
|
216
|
+
* already drifted: the same four branches were spelled three ways, and only
|
|
217
|
+
* two of them handled a reply that carried neither a result nor an error.
|
|
218
|
+
*
|
|
219
|
+
* Returns undefined when there is nothing to read, so each caller keeps its own
|
|
220
|
+
* fallback: stdio calls that "no answer", while the bare HTTP path deliberately
|
|
221
|
+
* leaves the verdict unset so the legacy branch can still try after initialize.
|
|
222
|
+
*/
|
|
223
|
+
function classifyDiscover(reply) {
|
|
224
|
+
if (reply?.result)
|
|
225
|
+
return { verdict: { supported: true }, cacheFields: hasCacheFields(reply.result) };
|
|
226
|
+
if (reply?.error) {
|
|
227
|
+
const detail = `${reply.error.code} ${reply.error.message}`;
|
|
228
|
+
return reply.error.code === METHOD_NOT_FOUND
|
|
229
|
+
? { verdict: { supported: false, errorMessage: detail } }
|
|
230
|
+
: { verdict: { skipped: `inconclusive (${detail})` } };
|
|
231
|
+
}
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
195
234
|
function toolOrder(result) {
|
|
196
235
|
const tools = result?.tools ?? [];
|
|
197
236
|
return tools.map((x) => x?.name).filter((n) => typeof n === "string");
|
|
198
237
|
}
|
|
238
|
+
/**
|
|
239
|
+
* Kills every spawned server on the way out, including when a probe throws.
|
|
240
|
+
*
|
|
241
|
+
* The body below used to run its cleanup only on the happy path, and the
|
|
242
|
+
* consequence was worse than a leaked process: with no kill(), the child's
|
|
243
|
+
* stdin is never closed, so it never sees EOF and never exits, and its stdio
|
|
244
|
+
* handles keep THIS process's event loop alive. runProbes catches the throw and
|
|
245
|
+
* the report still prints, and then the CLI hangs forever having already said
|
|
246
|
+
* everything it had to say. Measured: killed at 30s with the cleanup removed,
|
|
247
|
+
* 664ms clean exit with it.
|
|
248
|
+
*
|
|
249
|
+
* statelessStdio next door already had this try/finally; this is the same
|
|
250
|
+
* pattern in the file that was missing it. kill() is idempotent, so the early
|
|
251
|
+
* kills inside still free each server as soon as it is done rather than holding
|
|
252
|
+
* them all open until the end.
|
|
253
|
+
*/
|
|
199
254
|
async function probeStdio(target, budget) {
|
|
255
|
+
const sessions = [];
|
|
256
|
+
try {
|
|
257
|
+
return await probeStdioSessions(target, budget, sessions);
|
|
258
|
+
}
|
|
259
|
+
finally {
|
|
260
|
+
for (const s of sessions)
|
|
261
|
+
s.kill();
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
async function probeStdioSessions(target, budget, sessions) {
|
|
200
265
|
const results = {};
|
|
201
266
|
// A responsive server answers the bare request in well under a second, so a
|
|
202
267
|
// generous cap only affects genuinely slow/hung servers (introspect already
|
|
@@ -204,13 +269,12 @@ async function probeStdio(target, budget) {
|
|
|
204
269
|
// honored rather than floored.
|
|
205
270
|
const bareT = Math.min(Math.max(budget, 1), 15000);
|
|
206
271
|
const discoverT = Math.min(Math.max(budget, 1), 10000);
|
|
207
|
-
const sessions = [];
|
|
208
272
|
// A bare stateless tools/list. BOTH legacy (the reference SDK has no init gate)
|
|
209
273
|
// and RC servers answer it, so RC vs legacy is decided by the RESULT shape
|
|
210
274
|
// (resultType, cache fields, server/discover), never by whether it answered.
|
|
211
275
|
const s1 = new StdioSession(target.command, target.args, target.env);
|
|
212
276
|
sessions.push(s1);
|
|
213
|
-
const bare = await s1.request("tools/list",
|
|
277
|
+
const bare = await s1.request("tools/list", specMeta(), bareT);
|
|
214
278
|
const bareOutcome = toOutcome(bare, `no response within ${Math.round(bareT / 1000)}s (the server may require the removed initialize handshake)`);
|
|
215
279
|
results.bareToolsList = bareOutcome;
|
|
216
280
|
let order1;
|
|
@@ -222,18 +286,10 @@ async function probeStdio(target, budget) {
|
|
|
222
286
|
// Probe server/discover only if the server is responsive (answered or errored,
|
|
223
287
|
// not timed out), to avoid a second long wait on an unresponsive server.
|
|
224
288
|
if (!s1.exited && bareOutcome.kind !== "timeout") {
|
|
225
|
-
const d = await s1.request("server/discover",
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
results.serverDiscover = { supported: false, errorMessage: `${d.error.code} ${d.error.message}` };
|
|
230
|
-
}
|
|
231
|
-
else if (d.error) {
|
|
232
|
-
results.serverDiscover = { skipped: `inconclusive (${d.error.code} ${d.error.message})` };
|
|
233
|
-
}
|
|
234
|
-
else {
|
|
235
|
-
results.serverDiscover = { skipped: "no answer to server/discover" };
|
|
236
|
-
}
|
|
289
|
+
const d = classifyDiscover(await s1.request("server/discover", specMeta(), discoverT));
|
|
290
|
+
results.serverDiscover = d?.verdict ?? { skipped: "no answer to server/discover" };
|
|
291
|
+
if (d?.cacheFields !== undefined)
|
|
292
|
+
results.discoverCacheFieldsPresent = d.cacheFields;
|
|
237
293
|
}
|
|
238
294
|
else {
|
|
239
295
|
results.serverDiscover = { skipped: "server did not answer the bare request" };
|
|
@@ -243,7 +299,7 @@ async function probeStdio(target, budget) {
|
|
|
243
299
|
if (order1 && order1.length > 1) {
|
|
244
300
|
const s2 = new StdioSession(target.command, target.args, target.env);
|
|
245
301
|
sessions.push(s2);
|
|
246
|
-
const bare2 = await s2.request("tools/list",
|
|
302
|
+
const bare2 = await s2.request("tools/list", specMeta(), bareT);
|
|
247
303
|
s2.kill();
|
|
248
304
|
if (bare2.result) {
|
|
249
305
|
const order2 = toolOrder(bare2.result);
|
|
@@ -264,8 +320,13 @@ export async function postMessage(url, extraHeaders, msg, sessionId, protocolVer
|
|
|
264
320
|
"content-type": "application/json",
|
|
265
321
|
accept: "application/json, text/event-stream",
|
|
266
322
|
"mcp-protocol-version": protocolVersion,
|
|
267
|
-
// Mcp-Method
|
|
268
|
-
//
|
|
323
|
+
// Mcp-Method mirrors the body's method and is required on every request
|
|
324
|
+
// (SEP-2243). Mcp-Name is NOT required here: the published spec requires
|
|
325
|
+
// it only on tools/call, resources/read and prompts/get, where it carries
|
|
326
|
+
// params.name or params.uri, and efaimo sends none of those. Omitting it
|
|
327
|
+
// is correct rather than a known gap. Sending a wrong value would be
|
|
328
|
+
// worse than sending none: a server MUST reject a header that disagrees
|
|
329
|
+
// with the body, now with a dedicated code (-32020 HeaderMismatch).
|
|
269
330
|
...(method ? { "Mcp-Method": method } : {}),
|
|
270
331
|
...(sessionId ? { "mcp-session-id": sessionId } : {}),
|
|
271
332
|
...(extraHeaders ?? {}),
|
|
@@ -465,7 +526,7 @@ async function probeHttp(target) {
|
|
|
465
526
|
const results = {};
|
|
466
527
|
const url = target.url;
|
|
467
528
|
const headers = target.headers;
|
|
468
|
-
const bare = await postMessage(url, headers, { jsonrpc: "2.0", id: 1, method: "tools/list", params:
|
|
529
|
+
const bare = await postMessage(url, headers, { jsonrpc: "2.0", id: 1, method: "tools/list", params: specMeta() }, undefined, SPEC_VERSION).catch((e) => ({ status: 0, headers: new Headers(), authHeader: undefined, body: undefined, error: e }));
|
|
469
530
|
if (bare.error) {
|
|
470
531
|
const err = bare.error;
|
|
471
532
|
const isTimeout = /timeout|timed out|aborted/i.test(`${err.name} ${err.message}`);
|
|
@@ -490,14 +551,15 @@ async function probeHttp(target) {
|
|
|
490
551
|
// leave E106 (a MUST rule) unmeasured on precisely the servers it targets.
|
|
491
552
|
if (results.bareToolsList && "ok" in results.bareToolsList && results.bareToolsList.ok) {
|
|
492
553
|
try {
|
|
493
|
-
const d = await postMessage(url, headers, { jsonrpc: "2.0", id: 7, method: "server/discover", params:
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
554
|
+
const d = await postMessage(url, headers, { jsonrpc: "2.0", id: 7, method: "server/discover", params: specMeta() }, undefined, SPEC_VERSION);
|
|
555
|
+
// No fallback verdict on purpose: a reply carrying neither result nor
|
|
556
|
+
// error leaves serverDiscover unset so the legacy branch below can still
|
|
557
|
+
// ask, after initialize.
|
|
558
|
+
const c = classifyDiscover(d.body);
|
|
559
|
+
if (c) {
|
|
560
|
+
results.serverDiscover = c.verdict;
|
|
561
|
+
if (c.cacheFields !== undefined)
|
|
562
|
+
results.discoverCacheFieldsPresent = c.cacheFields;
|
|
501
563
|
}
|
|
502
564
|
}
|
|
503
565
|
catch {
|
|
@@ -507,7 +569,7 @@ async function probeHttp(target) {
|
|
|
507
569
|
const bareTools = toolOrder(bare.body && "result" in bare.body ? bare.body.result : undefined);
|
|
508
570
|
if (bareTools.length > 1) {
|
|
509
571
|
try {
|
|
510
|
-
const again = await postMessage(url, headers, { jsonrpc: "2.0", id: 8, method: "tools/list", params:
|
|
572
|
+
const again = await postMessage(url, headers, { jsonrpc: "2.0", id: 8, method: "tools/list", params: specMeta() }, undefined, SPEC_VERSION);
|
|
511
573
|
const order2 = toolOrder(again.body?.result);
|
|
512
574
|
if (order2.length)
|
|
513
575
|
results.toolsOrderDeterministic = arraysEqual(bareTools, order2);
|
|
@@ -545,16 +607,12 @@ async function probeHttp(target) {
|
|
|
545
607
|
}
|
|
546
608
|
if (results.serverDiscover === undefined) {
|
|
547
609
|
const d = await postMessage(url, headers, { jsonrpc: "2.0", id: 4, method: "server/discover", params: {} }, sessionId);
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
results.serverDiscover = { skipped: `inconclusive (${d.body.error.code} ${d.body.error.message})` };
|
|
555
|
-
}
|
|
556
|
-
else
|
|
557
|
-
results.serverDiscover = { skipped: "no answer to server/discover" };
|
|
610
|
+
const c = classifyDiscover(d.body);
|
|
611
|
+
results.serverDiscover = c?.verdict ?? { skipped: "no answer to server/discover" };
|
|
612
|
+
// ??= for the same reason as the resultType/cacheFields lines above:
|
|
613
|
+
// never clobber a measurement the bare stateless path already took.
|
|
614
|
+
if (c?.cacheFields !== undefined)
|
|
615
|
+
results.discoverCacheFieldsPresent ??= c.cacheFields;
|
|
558
616
|
}
|
|
559
617
|
if (order1 && order1.length > 1 && results.toolsOrderDeterministic === undefined) {
|
|
560
618
|
const init2 = await postMessage(url, headers, {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { SPEC_VERSION, specMeta, postMessage, StdioSession } from "./rawprobe.js";
|
|
2
2
|
/**
|
|
3
3
|
* 2026-07-28 introspection path: bare stateless requests, no initialize.
|
|
4
4
|
* The SDK client enforces the legacy handshake, so a stateless RC server makes
|
|
@@ -8,6 +8,15 @@ import { RC_VERSION, rcMeta, postMessage, StdioSession } from "./rawprobe.js";
|
|
|
8
8
|
* connect error.
|
|
9
9
|
*/
|
|
10
10
|
const MAX_PAGES = 20;
|
|
11
|
+
/**
|
|
12
|
+
* Newest shape first, then the RC's. Reading only the RC's field made efaimo
|
|
13
|
+
* report "identity unknown" for precisely the servers that had finished
|
|
14
|
+
* migrating, which is the worst direction for this error to point.
|
|
15
|
+
*/
|
|
16
|
+
function discoverIdentity(d) {
|
|
17
|
+
const found = d?._meta?.["io.modelcontextprotocol/serverInfo"] ?? d?.serverInfo;
|
|
18
|
+
return found ? { name: found.name, version: found.version, title: found.title } : undefined;
|
|
19
|
+
}
|
|
11
20
|
export async function introspectStateless(target, opts = {}) {
|
|
12
21
|
// The legacy path already burned the caller's timeout once; keep this
|
|
13
22
|
// attempt tight so a genuinely dead server fails in bounded time.
|
|
@@ -26,7 +35,7 @@ async function statelessStdio(target, budget) {
|
|
|
26
35
|
let rawFirst;
|
|
27
36
|
let cursor;
|
|
28
37
|
for (let page = 0; page < MAX_PAGES; page++) {
|
|
29
|
-
const params = cursor ? { ...
|
|
38
|
+
const params = cursor ? { ...specMeta(), cursor } : specMeta();
|
|
30
39
|
const reply = await session.request("tools/list", params, budget);
|
|
31
40
|
if (!reply.result) {
|
|
32
41
|
if (page === 0)
|
|
@@ -40,7 +49,7 @@ async function statelessStdio(target, budget) {
|
|
|
40
49
|
if (!cursor)
|
|
41
50
|
break;
|
|
42
51
|
}
|
|
43
|
-
const discover = await session.request("server/discover",
|
|
52
|
+
const discover = await session.request("server/discover", specMeta(), Math.min(budget, 10000));
|
|
44
53
|
return buildIntrospection(target.label, "stdio", undefined, pages, rawFirst, discover.result);
|
|
45
54
|
}
|
|
46
55
|
finally {
|
|
@@ -52,8 +61,8 @@ async function statelessHttp(target) {
|
|
|
52
61
|
let rawFirst;
|
|
53
62
|
let cursor;
|
|
54
63
|
for (let page = 0; page < MAX_PAGES; page++) {
|
|
55
|
-
const params = cursor ? { ...
|
|
56
|
-
const reply = await postMessage(target.url, target.headers, { jsonrpc: "2.0", id: 100 + page, method: "tools/list", params }, undefined,
|
|
64
|
+
const params = cursor ? { ...specMeta(), cursor } : specMeta();
|
|
65
|
+
const reply = await postMessage(target.url, target.headers, { jsonrpc: "2.0", id: 100 + page, method: "tools/list", params }, undefined, SPEC_VERSION);
|
|
57
66
|
const result = reply.body?.result;
|
|
58
67
|
if (!result) {
|
|
59
68
|
if (page === 0)
|
|
@@ -67,7 +76,7 @@ async function statelessHttp(target) {
|
|
|
67
76
|
if (!cursor)
|
|
68
77
|
break;
|
|
69
78
|
}
|
|
70
|
-
const discover = await postMessage(target.url, target.headers, { jsonrpc: "2.0", id: 99, method: "server/discover", params:
|
|
79
|
+
const discover = await postMessage(target.url, target.headers, { jsonrpc: "2.0", id: 99, method: "server/discover", params: specMeta() }, undefined, SPEC_VERSION).catch(() => undefined);
|
|
71
80
|
return buildIntrospection(target.label, "http", "streamable", pages, rawFirst, discover?.body?.result);
|
|
72
81
|
}
|
|
73
82
|
function buildIntrospection(label, transport, httpTransport, pages, rawFirst, discoverResult) {
|
|
@@ -102,14 +111,8 @@ function buildIntrospection(label, transport, httpTransport, pages, rawFirst, di
|
|
|
102
111
|
targetLabel: label,
|
|
103
112
|
transport,
|
|
104
113
|
httpTransport,
|
|
105
|
-
serverInfo: discover
|
|
106
|
-
|
|
107
|
-
name: discover.serverInfo.name,
|
|
108
|
-
version: discover.serverInfo.version,
|
|
109
|
-
title: discover.serverInfo.title,
|
|
110
|
-
}
|
|
111
|
-
: undefined,
|
|
112
|
-
protocolVersion: discover?.supportedVersions?.includes(RC_VERSION) ? RC_VERSION : undefined,
|
|
114
|
+
serverInfo: discoverIdentity(discover),
|
|
115
|
+
protocolVersion: discover?.supportedVersions?.includes(SPEC_VERSION) ? SPEC_VERSION : undefined,
|
|
113
116
|
instructions: discover?.instructions,
|
|
114
117
|
capabilities: caps,
|
|
115
118
|
tools,
|
package/dist/core/engine.js
CHANGED
|
@@ -6,8 +6,26 @@ export function runRules(rules, ctx) {
|
|
|
6
6
|
try {
|
|
7
7
|
findings.push(...rule.check(ctx));
|
|
8
8
|
}
|
|
9
|
-
catch {
|
|
10
|
-
|
|
9
|
+
catch (err) {
|
|
10
|
+
// Say so. This used to be a bare `catch {}`, which meant a rule that
|
|
11
|
+
// threw was indistinguishable from a rule that passed - and since the
|
|
12
|
+
// grade is 100 minus the findings, a crashing rule silently RAISED the
|
|
13
|
+
// score. That is this project's documented dominant failure mode ("green
|
|
14
|
+
// because it checked nothing") sitting inside the engine every check
|
|
15
|
+
// runs through.
|
|
16
|
+
//
|
|
17
|
+
// Reported ungraded on purpose: a broken rule is our defect, not the
|
|
18
|
+
// target's, so it must not cost the target points. It must not be
|
|
19
|
+
// invisible either.
|
|
20
|
+
const id = rule.id ?? "unknown rule";
|
|
21
|
+
findings.push({
|
|
22
|
+
ruleId: "E000",
|
|
23
|
+
severity: "warn",
|
|
24
|
+
graded: false,
|
|
25
|
+
message: `rule ${id} threw and was skipped; this report is incomplete`,
|
|
26
|
+
detail: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
|
|
27
|
+
fixHint: "please report this at https://github.com/efaimo-ai/efaimo/issues",
|
|
28
|
+
});
|
|
11
29
|
}
|
|
12
30
|
}
|
|
13
31
|
return findings;
|
package/dist/core/grade.js
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
const PENALTY = { error: 15, warn: 5, info: 1 };
|
|
2
2
|
export function gradeFindings(findings) {
|
|
3
3
|
let score = 100;
|
|
4
|
-
|
|
4
|
+
// `graded: false` findings are reported but never scored; see Finding.graded
|
|
5
|
+
// and src/rules/injection.ts. Everything else counts, so a new rule has to
|
|
6
|
+
// opt out on purpose rather than by forgetting to opt in.
|
|
7
|
+
for (const f of findings) {
|
|
8
|
+
if (f.graded === false)
|
|
9
|
+
continue;
|
|
5
10
|
score -= PENALTY[f.severity] ?? 0;
|
|
11
|
+
}
|
|
6
12
|
score = Math.max(0, score);
|
|
7
13
|
const letter = score >= 90 ? "A" : score >= 80 ? "B" : score >= 70 ? "C" : score >= 60 ? "D" : "F";
|
|
8
14
|
return { score, letter };
|
package/dist/core/types.d.ts
CHANGED
|
@@ -8,6 +8,15 @@ export interface Finding {
|
|
|
8
8
|
target?: string;
|
|
9
9
|
detail?: string;
|
|
10
10
|
fixHint?: string;
|
|
11
|
+
/**
|
|
12
|
+
* Set false by findings that must be reported but must not affect the score.
|
|
13
|
+
* Today that is the injection heuristics (E130, S105): they are shallow
|
|
14
|
+
* patterns an attacker evades trivially, and `src/rules/injection.ts` says
|
|
15
|
+
* so in its own comment. Letting them dock the grade made a linter hint
|
|
16
|
+
* behave like a security verdict, and ten of them cost a full letter.
|
|
17
|
+
* Defaults to graded, so a rule opts out deliberately or not at all.
|
|
18
|
+
*/
|
|
19
|
+
graded?: boolean;
|
|
11
20
|
}
|
|
12
21
|
export type Surface = "mcp" | "skill";
|
|
13
22
|
export interface ToolDef {
|
|
@@ -71,8 +80,16 @@ export interface ProbeResults {
|
|
|
71
80
|
};
|
|
72
81
|
/** Whether tools/list result carried the RC-required resultType field. */
|
|
73
82
|
resultTypePresent?: boolean;
|
|
74
|
-
/** Whether tools/list result carried both
|
|
83
|
+
/** Whether tools/list result carried both required ttlMs and cacheScope (SEP-2549). */
|
|
75
84
|
cacheFieldsPresent?: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Same question for the server/discover result, measured separately because
|
|
87
|
+
* the answer changed between the locked RC and the published spec: the RC's
|
|
88
|
+
* DiscoverResult extended plain Result, the published 2026-07-28 extends
|
|
89
|
+
* CacheableResult, so cache fields are required there now and were not then.
|
|
90
|
+
* Undefined when server/discover did not answer at all (E106's job).
|
|
91
|
+
*/
|
|
92
|
+
discoverCacheFieldsPresent?: boolean;
|
|
76
93
|
/** Same tool order across two fresh connections. */
|
|
77
94
|
toolsOrderDeterministic?: boolean;
|
|
78
95
|
/** Non-JSON noise observed on stdout before/between JSON-RPC messages (stdio only). */
|
|
@@ -170,10 +187,12 @@ export interface SkillInfo {
|
|
|
170
187
|
frontmatterRaw: string;
|
|
171
188
|
body: string;
|
|
172
189
|
bodyLines: number;
|
|
190
|
+
/** `escapes` = the reference resolves outside the skill directory. weigh must not read those; S106 reports them. */
|
|
173
191
|
referencedPaths: {
|
|
174
192
|
raw: string;
|
|
175
193
|
resolved: string;
|
|
176
194
|
exists: boolean;
|
|
195
|
+
escapes: boolean;
|
|
177
196
|
source: "link" | "code";
|
|
178
197
|
}[];
|
|
179
198
|
files: {
|
|
@@ -205,8 +224,8 @@ export interface CheckReport {
|
|
|
205
224
|
grade: GradeInfo;
|
|
206
225
|
/**
|
|
207
226
|
* MCP only: 2026-07-28 readiness findings (E101-E118), reported as an
|
|
208
|
-
* ungraded migration diff
|
|
209
|
-
*
|
|
227
|
+
* ungraded migration diff: a migration not yet made is a to-do list, not a
|
|
228
|
+
* quality defect in what shipped.
|
|
210
229
|
*/
|
|
211
230
|
readiness?: {
|
|
212
231
|
findings: Finding[];
|
|
@@ -1,7 +1,19 @@
|
|
|
1
1
|
import { VERSION } from "../version.js";
|
|
2
|
+
import { safeText } from "../util/safeText.js";
|
|
2
3
|
const SEV_LABEL = { error: "🔴 error", warn: "🟡 warn", info: "🔵 info" };
|
|
4
|
+
// Everything through here came from the audited target, and
|
|
5
|
+
// docs/INTEGRATIONS.md recommends piping --md into $GITHUB_STEP_SUMMARY, so a
|
|
6
|
+
// hostile tool name or description renders into a page the reader trusts.
|
|
7
|
+
// Two things were missing: control characters (raw ESC bytes were written
|
|
8
|
+
// through literally) and backticks (most call sites wrap the value in a code
|
|
9
|
+
// span, so one backtick closes it and the rest of the name becomes markup).
|
|
10
|
+
// The backtick is neutralised with a zero-width space rather than a backslash,
|
|
11
|
+
// because a backslash is not an escape inside a code span.
|
|
3
12
|
function esc(s) {
|
|
4
|
-
return s
|
|
13
|
+
return safeText(s)
|
|
14
|
+
.replace(/\|/g, "\\|")
|
|
15
|
+
.replace(/`/g, "`")
|
|
16
|
+
.replace(/\n/g, " ");
|
|
5
17
|
}
|
|
6
18
|
function findingsTable(lines, findings) {
|
|
7
19
|
lines.push("| severity | rule | finding |");
|
|
@@ -14,7 +26,7 @@ export function renderCheckMarkdown(report) {
|
|
|
14
26
|
const lines = [];
|
|
15
27
|
lines.push(`## efaimo check (${report.surface}): grade ${report.grade.letter} (${report.grade.score})`);
|
|
16
28
|
lines.push("");
|
|
17
|
-
lines.push(`target: \`${report.target}\``);
|
|
29
|
+
lines.push(`target: \`${esc(report.target)}\``);
|
|
18
30
|
lines.push("");
|
|
19
31
|
if (!report.findings.length) {
|
|
20
32
|
lines.push(report.readiness ? "no quality findings. clean." : "no findings. clean.");
|
|
@@ -63,7 +75,7 @@ export function renderSkillSetMarkdown(res) {
|
|
|
63
75
|
export function renderWeighMarkdown(w) {
|
|
64
76
|
const lines = [];
|
|
65
77
|
if (w.kind === "mcp") {
|
|
66
|
-
lines.push(`## efaimo weigh: \`${w.label}\``);
|
|
78
|
+
lines.push(`## efaimo weigh: \`${esc(w.label)}\``);
|
|
67
79
|
lines.push("");
|
|
68
80
|
lines.push(`tools ${w.toolCount} | resources ${w.resourceCount} | prompts ${w.promptCount}`);
|
|
69
81
|
lines.push("");
|
|
@@ -88,7 +100,7 @@ export function renderWeighMarkdown(w) {
|
|
|
88
100
|
}
|
|
89
101
|
}
|
|
90
102
|
else {
|
|
91
|
-
lines.push(`## efaimo weigh (skills): \`${w.label}\``);
|
|
103
|
+
lines.push(`## efaimo weigh (skills): \`${esc(w.label)}\``);
|
|
92
104
|
lines.push("");
|
|
93
105
|
lines.push("| skill | metadata (always) | body (on trigger) | lines | referenced files |");
|
|
94
106
|
lines.push("|---|---|---|---|---|");
|
package/dist/reporters/pretty.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import pc from "picocolors";
|
|
2
|
+
import { formatWindowShare } from "../weigh/window.js";
|
|
2
3
|
import { sortFindings } from "../core/grade.js";
|
|
3
4
|
import { VERSION } from "../version.js";
|
|
5
|
+
import { safeText } from "../util/safeText.js";
|
|
4
6
|
const DOCS_RULES = "https://github.com/efaimo-ai/efaimo/blob/main/docs/RULES.md";
|
|
5
7
|
let colorOn = true;
|
|
6
8
|
export function setColor(on) {
|
|
@@ -31,18 +33,18 @@ function n(x) {
|
|
|
31
33
|
return x.toLocaleString("en-US");
|
|
32
34
|
}
|
|
33
35
|
function renderFinding(lines, f) {
|
|
34
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${f.message}`);
|
|
36
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
35
37
|
if (f.detail) {
|
|
36
|
-
for (const d of f.detail.split("\n"))
|
|
38
|
+
for (const d of safeText(f.detail).split("\n"))
|
|
37
39
|
lines.push(paint(pc.dim, ` ${d}`));
|
|
38
40
|
}
|
|
39
41
|
if (f.fixHint)
|
|
40
|
-
lines.push(paint(pc.dim, ` fix: ${f.fixHint}`));
|
|
42
|
+
lines.push(paint(pc.dim, ` fix: ${safeText(f.fixHint)}`));
|
|
41
43
|
}
|
|
42
44
|
export function renderCheckPretty(report) {
|
|
43
45
|
const lines = [];
|
|
44
46
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
45
|
-
lines.push(`check ${report.surface} ${paint(pc.bold, report.target)}`);
|
|
47
|
+
lines.push(`check ${report.surface} ${paint(pc.bold, safeText(report.target))}`);
|
|
46
48
|
const g = report.grade;
|
|
47
49
|
const qualityLabel = report.readiness ? "quality: " : "";
|
|
48
50
|
lines.push(`grade ${gradeColor(g.letter, `${g.letter} (${g.score})`)} ${qualityLabel}` +
|
|
@@ -61,7 +63,7 @@ export function renderCheckPretty(report) {
|
|
|
61
63
|
if (r.findings.length) {
|
|
62
64
|
lines.push(paint(pc.bold, `2026-07-28 readiness`) +
|
|
63
65
|
` ${r.findings.length} item${r.findings.length === 1 ? "" : "s"} to migrate ` +
|
|
64
|
-
paint(pc.dim, "(a migration diff, not graded
|
|
66
|
+
paint(pc.dim, "(a migration diff, not graded)"));
|
|
65
67
|
for (const f of r.findings)
|
|
66
68
|
renderFinding(lines, f);
|
|
67
69
|
}
|
|
@@ -71,7 +73,7 @@ export function renderCheckPretty(report) {
|
|
|
71
73
|
}
|
|
72
74
|
lines.push("");
|
|
73
75
|
for (const note of report.notes)
|
|
74
|
-
lines.push(paint(pc.dim, `note: ${note}`));
|
|
76
|
+
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
75
77
|
lines.push(paint(pc.dim, `rules: ${DOCS_RULES}`));
|
|
76
78
|
return lines.join("\n");
|
|
77
79
|
}
|
|
@@ -98,19 +100,20 @@ export function renderTestReportPretty(r) {
|
|
|
98
100
|
`test ${paint(pc.bold, r.scenario)}`,
|
|
99
101
|
`skill ${r.skill} model ${r.model}`,
|
|
100
102
|
"",
|
|
101
|
-
` with skill ${r.withSkill.passes}/${r.withSkill.
|
|
102
|
-
` without skill ${r.withoutSkill.passes}/${r.withoutSkill.
|
|
103
|
+
` with skill ${r.withSkill.passes}/${r.withSkill.scored} pass (${r.withSkill.passRate.toFixed(0)}%)`,
|
|
104
|
+
` without skill ${r.withoutSkill.passes}/${r.withoutSkill.scored} pass (${r.withoutSkill.passRate.toFixed(0)}%)`,
|
|
103
105
|
` delta ${paint(pc.bold, `${sign}${r.deltaPoints} points`)} ${paint(verdictColor, r.verdict)}`,
|
|
106
|
+
` significance p = ${r.p < 0.0001 ? "<0.0001" : r.p.toFixed(4)} 95% CI ${r.ci.lo >= 0 ? "+" : ""}${r.ci.lo} to ${r.ci.hi >= 0 ? "+" : ""}${r.ci.hi} points`,
|
|
104
107
|
"",
|
|
105
108
|
];
|
|
106
109
|
for (const n of r.notes)
|
|
107
|
-
lines.push(paint(pc.dim, `note: ${n}`));
|
|
110
|
+
lines.push(paint(pc.dim, `note: ${safeText(n)}`));
|
|
108
111
|
return lines.join("\n");
|
|
109
112
|
}
|
|
110
113
|
export function renderSkillSetPretty(res) {
|
|
111
114
|
const lines = [];
|
|
112
115
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
113
|
-
lines.push(`check skills ${paint(pc.bold, res.label)} ${res.perSkill.length} skill${res.perSkill.length === 1 ? "" : "s"}`);
|
|
116
|
+
lines.push(`check skills ${paint(pc.bold, safeText(res.label))} ${res.perSkill.length} skill${res.perSkill.length === 1 ? "" : "s"}`);
|
|
114
117
|
const dist = { A: 0, B: 0, C: 0, D: 0, F: 0 };
|
|
115
118
|
for (const s of res.perSkill)
|
|
116
119
|
dist[s.report.grade.letter]++;
|
|
@@ -125,43 +128,42 @@ export function renderSkillSetPretty(res) {
|
|
|
125
128
|
for (const s of [...res.perSkill].sort((a, b) => a.report.grade.score - b.report.grade.score)) {
|
|
126
129
|
const g = s.report.grade;
|
|
127
130
|
const c = s.report.counts;
|
|
128
|
-
lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${s.name.padEnd(nameW)} ` +
|
|
131
|
+
lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${safeText(s.name).padEnd(nameW)} ` +
|
|
129
132
|
`${paint(pc.red, String(c.error))}e ${paint(pc.yellow, String(c.warn))}w ${paint(pc.cyan, String(c.info))}i`);
|
|
130
133
|
}
|
|
131
134
|
if (res.setFindings.length) {
|
|
132
135
|
lines.push("");
|
|
133
136
|
lines.push(paint(pc.bold, "across the set"));
|
|
134
137
|
for (const f of sortFindings(res.setFindings)) {
|
|
135
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${f.message}`);
|
|
138
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
136
139
|
}
|
|
137
140
|
}
|
|
138
141
|
const flagged = res.perSkill.filter((s) => s.report.findings.length);
|
|
139
142
|
if (flagged.length) {
|
|
140
143
|
lines.push("");
|
|
141
144
|
for (const s of flagged) {
|
|
142
|
-
lines.push(paint(pc.bold, `${s.name} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
|
|
145
|
+
lines.push(paint(pc.bold, `${safeText(s.name)} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
|
|
143
146
|
for (const f of s.report.findings) {
|
|
144
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${f.message}`);
|
|
147
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
145
148
|
if (f.fixHint)
|
|
146
|
-
lines.push(paint(pc.dim, ` fix: ${f.fixHint}`));
|
|
149
|
+
lines.push(paint(pc.dim, ` fix: ${safeText(f.fixHint)}`));
|
|
147
150
|
}
|
|
148
151
|
}
|
|
149
152
|
}
|
|
150
153
|
lines.push("");
|
|
151
|
-
lines.push(paint(pc.dim, `${res.perSkill.length} skills under ${res.root}`));
|
|
154
|
+
lines.push(paint(pc.dim, `${res.perSkill.length} skills under ${safeText(res.root)}`));
|
|
152
155
|
lines.push(paint(pc.dim, `rules: ${DOCS_RULES}`));
|
|
153
156
|
return lines.join("\n");
|
|
154
157
|
}
|
|
155
158
|
export function renderServerWeighPretty(w) {
|
|
156
159
|
const lines = [];
|
|
157
160
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
158
|
-
lines.push(`weigh mcp ${paint(pc.bold, w.label)}`);
|
|
161
|
+
lines.push(`weigh mcp ${paint(pc.bold, safeText(w.label))}`);
|
|
159
162
|
lines.push(`tools ${w.toolCount} resources ${w.resourceCount} prompts ${w.promptCount}`);
|
|
160
163
|
lines.push("");
|
|
161
164
|
lines.push("context cost of tool definitions (o200k tokens, estimated)");
|
|
162
|
-
const pct = ((w.totals.claudeStyle / 200000) * 100).toFixed(1);
|
|
163
165
|
lines.push(` raw JSON ${n(w.totals.rawJson).padStart(8)}`);
|
|
164
|
-
lines.push(` Claude-style ${n(w.totals.claudeStyle).padStart(8)} (
|
|
166
|
+
lines.push(` Claude-style ${n(w.totals.claudeStyle).padStart(8)} (${formatWindowShare(w.totals.claudeStyle)})`);
|
|
165
167
|
lines.push(` OpenAI tools ${n(w.totals.openaiTools).padStart(8)}`);
|
|
166
168
|
if (w.instructionsTokens > 0)
|
|
167
169
|
lines.push(` server instructions ${n(w.instructionsTokens).padStart(4)}`);
|
|
@@ -171,8 +173,17 @@ export function renderServerWeighPretty(w) {
|
|
|
171
173
|
if (w.perTool.length) {
|
|
172
174
|
lines.push("");
|
|
173
175
|
lines.push("heaviest tools (Claude-style)");
|
|
174
|
-
|
|
175
|
-
|
|
176
|
+
// Derived from the rows actually printed, not a fixed 28. Against the
|
|
177
|
+
// reference server - the target in --help, on /commands and in every
|
|
178
|
+
// launch draft - "trigger-long-running-operation" is 30 characters, ate
|
|
179
|
+
// the pad, and shifted the token column two places right for that one row.
|
|
180
|
+
// A tool whose pitch is careful measurement was printing a crooked number
|
|
181
|
+
// column in its own flagship demo. Capped so one pathological name cannot
|
|
182
|
+
// push the numbers off a terminal.
|
|
183
|
+
const shown = w.perTool.slice(0, 8);
|
|
184
|
+
const toolW = Math.min(38, Math.max(20, ...shown.map((t) => safeText(t.name).length)));
|
|
185
|
+
for (const [i, t] of shown.entries()) {
|
|
186
|
+
lines.push(` ${String(i + 1).padStart(2)}. ${safeText(t.name).padEnd(toolW)} ${n(t.tokens.claudeStyle).padStart(7)} desc ${n(t.descriptionTokens)} | schema ${n(t.schemaTokens)}`);
|
|
176
187
|
}
|
|
177
188
|
if (w.perTool.length > 8)
|
|
178
189
|
lines.push(paint(pc.dim, ` (+${w.perTool.length - 8} more)`));
|
|
@@ -182,23 +193,23 @@ export function renderServerWeighPretty(w) {
|
|
|
182
193
|
}
|
|
183
194
|
lines.push("");
|
|
184
195
|
for (const note of w.notes)
|
|
185
|
-
lines.push(paint(pc.dim, `note: ${note}`));
|
|
196
|
+
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
186
197
|
return lines.join("\n");
|
|
187
198
|
}
|
|
188
199
|
export function renderSkillWeighPretty(w) {
|
|
189
200
|
const lines = [];
|
|
190
201
|
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
191
|
-
lines.push(`weigh skills ${paint(pc.bold, w.label)} ${w.perSkill.length} skill${w.perSkill.length === 1 ? "" : "s"}`);
|
|
202
|
+
lines.push(`weigh skills ${paint(pc.bold, safeText(w.label))} ${w.perSkill.length} skill${w.perSkill.length === 1 ? "" : "s"}`);
|
|
192
203
|
lines.push("");
|
|
193
204
|
lines.push(` ${"skill".padEnd(28)} ${"metadata".padStart(8)} ${"body".padStart(9)} ${"lines".padStart(6)} refs`);
|
|
194
205
|
for (const s of w.perSkill) {
|
|
195
|
-
lines.push(` ${s.name.padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
|
|
206
|
+
lines.push(` ${safeText(s.name).padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
|
|
196
207
|
}
|
|
197
208
|
lines.push("");
|
|
198
209
|
lines.push(`totals: metadata ${paint(pc.bold, n(w.totals.metadata))} (always loaded) | body ${n(w.totals.body)} (on trigger) | referenced ${n(w.totals.refFiles)} (on demand)`);
|
|
199
210
|
lines.push("");
|
|
200
211
|
for (const note of w.notes)
|
|
201
|
-
lines.push(paint(pc.dim, `note: ${note}`));
|
|
212
|
+
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
202
213
|
return lines.join("\n");
|
|
203
214
|
}
|
|
204
215
|
export function renderDiffPretty(d, opts) {
|