efaimo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +281 -0
- package/SKILL.md +57 -0
- package/dist/check/check.d.ts +31 -0
- package/dist/check/check.js +77 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +372 -0
- package/dist/clients/introspect.d.ts +8 -0
- package/dist/clients/introspect.js +198 -0
- package/dist/clients/rawprobe.d.ts +69 -0
- package/dist/clients/rawprobe.js +597 -0
- package/dist/clients/stateless.d.ts +7 -0
- package/dist/clients/stateless.js +121 -0
- package/dist/core/engine.d.ts +6 -0
- package/dist/core/engine.js +17 -0
- package/dist/core/grade.d.ts +8 -0
- package/dist/core/grade.js +19 -0
- package/dist/core/types.d.ts +243 -0
- package/dist/core/types.js +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +26 -0
- package/dist/mcp/server.d.ts +3 -0
- package/dist/mcp/server.js +88 -0
- package/dist/reporters/badge.d.ts +12 -0
- package/dist/reporters/badge.js +37 -0
- package/dist/reporters/json.d.ts +1 -0
- package/dist/reporters/json.js +10 -0
- package/dist/reporters/markdown.d.ts +7 -0
- package/dist/reporters/markdown.js +121 -0
- package/dist/reporters/pretty.d.ts +15 -0
- package/dist/reporters/pretty.js +221 -0
- package/dist/rules/injection.d.ts +20 -0
- package/dist/rules/injection.js +106 -0
- package/dist/rules/mcp/index.d.ts +4 -0
- package/dist/rules/mcp/index.js +643 -0
- package/dist/rules/skill/index.d.ts +2 -0
- package/dist/rules/skill/index.js +279 -0
- package/dist/skills/parse.d.ts +4 -0
- package/dist/skills/parse.js +113 -0
- package/dist/targets/clientConfigs.d.ts +17 -0
- package/dist/targets/clientConfigs.js +110 -0
- package/dist/targets/repoScan.d.ts +2 -0
- package/dist/targets/repoScan.js +0 -0
- package/dist/targets/resolve.d.ts +28 -0
- package/dist/targets/resolve.js +48 -0
- package/dist/testing/anthropicRunner.d.ts +6 -0
- package/dist/testing/anthropicRunner.js +33 -0
- package/dist/testing/harness.d.ts +45 -0
- package/dist/testing/harness.js +96 -0
- package/dist/testing/openaiRunner.d.ts +12 -0
- package/dist/testing/openaiRunner.js +40 -0
- package/dist/util/childEnv.d.ts +1 -0
- package/dist/util/childEnv.js +43 -0
- package/dist/util/dotenv.d.ts +12 -0
- package/dist/util/dotenv.js +53 -0
- package/dist/util/misc.d.ts +15 -0
- package/dist/util/misc.js +121 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +2 -0
- package/dist/weigh/anthropicExact.d.ts +11 -0
- package/dist/weigh/anthropicExact.js +46 -0
- package/dist/weigh/diff.d.ts +14 -0
- package/dist/weigh/diff.js +19 -0
- package/dist/weigh/serializers.d.ts +21 -0
- package/dist/weigh/serializers.js +44 -0
- package/dist/weigh/tokens.d.ts +12 -0
- package/dist/weigh/tokens.js +23 -0
- package/dist/weigh/weigh.d.ts +6 -0
- package/dist/weigh/weigh.js +104 -0
- package/docs/INTEGRATIONS.md +119 -0
- package/docs/METHODOLOGY.md +83 -0
- package/docs/RULES.md +76 -0
- package/package.json +77 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { countBySeverity, gradeFindings, sortFindings } from "../core/grade.js";
|
|
3
|
+
import { runMcpRules, runRules } from "../core/engine.js";
|
|
4
|
+
import { isReadinessRuleId } from "../rules/mcp/index.js";
|
|
5
|
+
import { SKILL_RULES } from "../rules/skill/index.js";
|
|
6
|
+
import { introspectServer } from "../clients/introspect.js";
|
|
7
|
+
import { runProbes } from "../clients/rawprobe.js";
|
|
8
|
+
import { scanRepo } from "../targets/repoScan.js";
|
|
9
|
+
import { findSkills } from "../skills/parse.js";
|
|
10
|
+
import { weighServer, weighSkills } from "../weigh/weigh.js";
|
|
11
|
+
import { VERSION } from "../version.js";
|
|
12
|
+
function buildReport(surface, target, findings, notes) {
|
|
13
|
+
// MCP readiness findings (E101-E118) are a migration diff for a spec that is
|
|
14
|
+
// not final until 2026-07-28; they are reported separately and never graded.
|
|
15
|
+
const readinessAll = [];
|
|
16
|
+
const gradedAll = [];
|
|
17
|
+
for (const f of findings) {
|
|
18
|
+
(surface === "mcp" && isReadinessRuleId(f.ruleId) ? readinessAll : gradedAll).push(f);
|
|
19
|
+
}
|
|
20
|
+
const graded = sortFindings(gradedAll);
|
|
21
|
+
const readiness = sortFindings(readinessAll);
|
|
22
|
+
return {
|
|
23
|
+
tool: "efaimo",
|
|
24
|
+
version: VERSION,
|
|
25
|
+
surface,
|
|
26
|
+
target,
|
|
27
|
+
findings: graded,
|
|
28
|
+
counts: countBySeverity(graded),
|
|
29
|
+
grade: gradeFindings(graded),
|
|
30
|
+
...(surface === "mcp" ? { readiness: { findings: readiness, counts: countBySeverity(readiness) } } : {}),
|
|
31
|
+
notes,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export async function checkMcpTarget(target, opts = {}) {
|
|
35
|
+
const intro = await introspectServer(target, { timeoutMs: opts.timeoutMs });
|
|
36
|
+
const weigh = await weighServer(intro, { anthropicApiKey: opts.anthropicApiKey });
|
|
37
|
+
const probes = opts.probe === false ? undefined : await runProbes(target, { timeoutMs: opts.timeoutMs });
|
|
38
|
+
const repo = opts.repoPath ? scanRepo(opts.repoPath) : undefined;
|
|
39
|
+
const findings = runMcpRules({ intro, probes, repo, weigh });
|
|
40
|
+
const notes = [...intro.notes];
|
|
41
|
+
if (opts.probe === false)
|
|
42
|
+
notes.push("readiness probes skipped (--no-probe)");
|
|
43
|
+
if (repo)
|
|
44
|
+
notes.push(`repo scan: ${repo.filesScanned} files in ${repo.root}`);
|
|
45
|
+
return { report: buildReport("mcp", target.label, findings, notes), intro, weigh, probes };
|
|
46
|
+
}
|
|
47
|
+
export function checkMcpRepoOnly(repoPath, label) {
|
|
48
|
+
const repo = scanRepo(repoPath);
|
|
49
|
+
const emptyIntro = {
|
|
50
|
+
targetLabel: label,
|
|
51
|
+
transport: "stdio",
|
|
52
|
+
tools: [],
|
|
53
|
+
resources: [],
|
|
54
|
+
prompts: [],
|
|
55
|
+
notes: [],
|
|
56
|
+
};
|
|
57
|
+
const findings = runMcpRules({ intro: emptyIntro, repo });
|
|
58
|
+
return buildReport("mcp", label, findings, [
|
|
59
|
+
`static repo scan only (${repo.filesScanned} files); run against the live server for transport, probe, and quality checks`,
|
|
60
|
+
]);
|
|
61
|
+
}
|
|
62
|
+
export async function checkSkillSet(pathInput, label) {
|
|
63
|
+
const set = findSkills(pathInput);
|
|
64
|
+
if (!set.skills.length) {
|
|
65
|
+
throw new Error(`no SKILL.md found under "${pathInput}"`);
|
|
66
|
+
}
|
|
67
|
+
const weigh = await weighSkills(set);
|
|
68
|
+
const perSkillRules = SKILL_RULES.filter((r) => r.id !== "S103");
|
|
69
|
+
const setRules = SKILL_RULES.filter((r) => r.id === "S103");
|
|
70
|
+
const perSkill = set.skills.map((skill) => {
|
|
71
|
+
const name = skill.name ?? path.basename(skill.dir);
|
|
72
|
+
const findings = runRules(perSkillRules, { skill, set, weigh });
|
|
73
|
+
return { name, dir: skill.dir, report: buildReport("skill", name, findings, []) };
|
|
74
|
+
});
|
|
75
|
+
const setFindings = set.skills.flatMap((skill) => runRules(setRules, { skill, set, weigh }));
|
|
76
|
+
return { label, root: set.root, perSkill, setFindings, weigh };
|
|
77
|
+
}
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import crossSpawn from "cross-spawn";
|
|
4
|
+
import { Command } from "commander";
|
|
5
|
+
import pc from "picocolors";
|
|
6
|
+
import { VERSION } from "./version.js";
|
|
7
|
+
import { resolveTarget } from "./targets/resolve.js";
|
|
8
|
+
import { loadClientServers, SUPPORTED_CLIENTS } from "./targets/clientConfigs.js";
|
|
9
|
+
import { introspectServer } from "./clients/introspect.js";
|
|
10
|
+
import { weighServer, weighSkills } from "./weigh/weigh.js";
|
|
11
|
+
import { findSkills } from "./skills/parse.js";
|
|
12
|
+
import { diffServerWeigh } from "./weigh/diff.js";
|
|
13
|
+
import { checkMcpRepoOnly, checkMcpTarget, checkSkillSet } from "./check/check.js";
|
|
14
|
+
import { renderCheckPretty, renderDiffPretty, renderScenarioPlan, renderServerWeighPretty, renderSkillSetPretty, renderSkillWeighPretty, renderTestReportPretty, setColor, } from "./reporters/pretty.js";
|
|
15
|
+
import { parseScenario, runScenario } from "./testing/harness.js";
|
|
16
|
+
import { anthropicRunner } from "./testing/anthropicRunner.js";
|
|
17
|
+
import { openaiRunner, providerForModel } from "./testing/openaiRunner.js";
|
|
18
|
+
import { toJsonEnvelope } from "./reporters/json.js";
|
|
19
|
+
import { renderCheckMarkdown, renderDiffMarkdown, renderSkillSetMarkdown, renderWeighMarkdown } from "./reporters/markdown.js";
|
|
20
|
+
import { gradeBadgeSpec, makeBadgeSvg, toShieldsEndpoint, weighBadgeSpec } from "./reporters/badge.js";
|
|
21
|
+
import { loadDotEnv } from "./util/dotenv.js";
|
|
22
|
+
// Load a local .env before any command reads a key. Shell env always wins.
|
|
23
|
+
const dotEnvKeys = new Set(loadDotEnv());
|
|
24
|
+
const program = new Command();
|
|
25
|
+
program
|
|
26
|
+
.name("efaimo")
|
|
27
|
+
.description("Audit what your agent loads: quality and context cost for MCP servers and Agent Skills")
|
|
28
|
+
.version(VERSION)
|
|
29
|
+
.addHelpText("after", `
|
|
30
|
+
examples:
|
|
31
|
+
$ npx efaimo weigh "npx -y @modelcontextprotocol/server-everything"
|
|
32
|
+
$ npx efaimo weigh https://mcp.example.com/mcp
|
|
33
|
+
$ npx efaimo weigh ./my-skill
|
|
34
|
+
$ npx efaimo weigh --client claude-code
|
|
35
|
+
$ npx efaimo check --mcp "npx -y my-mcp-server" # incl. 2026-07-28 readiness
|
|
36
|
+
$ npx efaimo check --skill ./skills/
|
|
37
|
+
$ npx efaimo weigh "npx -y my-server" --out base.json && npx efaimo weigh "npx -y my-server" --diff base.json
|
|
38
|
+
`);
|
|
39
|
+
function collectPairs(sep) {
|
|
40
|
+
return (value, prev = {}) => {
|
|
41
|
+
const idx = value.indexOf(sep);
|
|
42
|
+
if (idx === -1)
|
|
43
|
+
throw new Error(`expected "KEY${sep}VALUE", got "${value}"`);
|
|
44
|
+
return { ...prev, [value.slice(0, idx).trim()]: value.slice(idx + 1).trim() };
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function colorSetup(opts) {
|
|
48
|
+
setColor(opts.color !== false && process.env.NO_COLOR === undefined);
|
|
49
|
+
}
|
|
50
|
+
function fail(message) {
|
|
51
|
+
console.error(pc.red(`error: ${message}`));
|
|
52
|
+
process.exit(2);
|
|
53
|
+
}
|
|
54
|
+
function writeBadge(fileArg, spec) {
|
|
55
|
+
if (!fileArg)
|
|
56
|
+
return;
|
|
57
|
+
const file = typeof fileArg === "string" ? fileArg : "efaimo-badge.svg";
|
|
58
|
+
fs.writeFileSync(file, makeBadgeSvg(spec.label, spec.message, spec.color));
|
|
59
|
+
const jsonFile = file.replace(/\.svg$/i, "") + ".json";
|
|
60
|
+
fs.writeFileSync(jsonFile, toShieldsEndpoint(spec));
|
|
61
|
+
console.error(pc.dim(`badge written: ${file}, ${jsonFile}`));
|
|
62
|
+
}
|
|
63
|
+
function seconds(value, label = "--timeout") {
|
|
64
|
+
const n = Number(value);
|
|
65
|
+
if (!Number.isFinite(n) || n <= 0)
|
|
66
|
+
fail(`${label} must be a positive number of seconds, got "${value}"`);
|
|
67
|
+
return n;
|
|
68
|
+
}
|
|
69
|
+
function parseNumberOpt(value, label) {
|
|
70
|
+
if (value === undefined)
|
|
71
|
+
return undefined;
|
|
72
|
+
const n = Number(value);
|
|
73
|
+
if (!Number.isFinite(n))
|
|
74
|
+
fail(`${label} must be a number, got "${value}"`);
|
|
75
|
+
return n;
|
|
76
|
+
}
|
|
77
|
+
function anthropicKeyFor(opts) {
|
|
78
|
+
if (!opts.anthropic)
|
|
79
|
+
return {};
|
|
80
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
81
|
+
if (!apiKey)
|
|
82
|
+
fail("--anthropic needs ANTHROPIC_API_KEY in the environment or a .env file");
|
|
83
|
+
return { apiKey, model: typeof opts.anthropic === "string" ? opts.anthropic : undefined };
|
|
84
|
+
}
|
|
85
|
+
program
|
|
86
|
+
.command("weigh")
|
|
87
|
+
.description("measure the context cost of MCP tool definitions or Agent Skills")
|
|
88
|
+
.argument("[target]", "stdio command | http(s) URL | skill path")
|
|
89
|
+
.option("--client <name>", `weigh every server configured for a client (${SUPPORTED_CLIENTS.join(", ")})`)
|
|
90
|
+
.option("--stdio", "treat target as a stdio command string")
|
|
91
|
+
.option("--skill", "treat target as a skill path")
|
|
92
|
+
.option("--header <header>", 'HTTP header "Key: Value" (repeatable)', collectPairs(":"))
|
|
93
|
+
.option("--env <pair>", "KEY=VALUE for stdio servers (repeatable)", collectPairs("="))
|
|
94
|
+
.option("--timeout <seconds>", "connect timeout in seconds", "45")
|
|
95
|
+
.option("--json", "print JSON")
|
|
96
|
+
.option("--md", "print Markdown")
|
|
97
|
+
.option("--out <file>", "write the JSON result to a file (usable as a --diff baseline)")
|
|
98
|
+
.option("--diff <baseline>", "compare against a baseline JSON from --out")
|
|
99
|
+
.option("--max-tokens <n>", "exit 1 if the primary total exceeds n tokens")
|
|
100
|
+
.option("--allow-increase <pct>", "with --diff: exit 1 if the increase exceeds pct percent")
|
|
101
|
+
.option("--badge [file]", "write an SVG badge + shields endpoint JSON")
|
|
102
|
+
.option("--anthropic [model]", "also measure exact Claude tokens via the count_tokens API (needs ANTHROPIC_API_KEY)")
|
|
103
|
+
.option("--no-color", "disable colors")
|
|
104
|
+
.action(async (targetArg, opts) => {
|
|
105
|
+
colorSetup(opts);
|
|
106
|
+
const timeoutMs = seconds(opts.timeout) * 1000;
|
|
107
|
+
const maxTokens = parseNumberOpt(opts.maxTokens, "--max-tokens");
|
|
108
|
+
const allowIncreasePct = parseNumberOpt(opts.allowIncrease, "--allow-increase");
|
|
109
|
+
const anthropic = anthropicKeyFor(opts);
|
|
110
|
+
const targets = [];
|
|
111
|
+
if (opts.client) {
|
|
112
|
+
const conf = loadClientServers(opts.client);
|
|
113
|
+
if (!conf.entries.length) {
|
|
114
|
+
fail(`no MCP servers found for client "${opts.client}" (checked: ${[...conf.sources, ...conf.missing].join(", ")})`);
|
|
115
|
+
}
|
|
116
|
+
console.error(pc.dim(`config: ${conf.sources.join(", ")}`));
|
|
117
|
+
targets.push(...conf.entries.map((e) => e.target));
|
|
118
|
+
}
|
|
119
|
+
if (targetArg) {
|
|
120
|
+
targets.push(resolveTarget(targetArg, {
|
|
121
|
+
forceStdio: opts.stdio,
|
|
122
|
+
forceSkill: opts.skill,
|
|
123
|
+
env: opts.env,
|
|
124
|
+
headers: opts.header,
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
127
|
+
if (!targets.length)
|
|
128
|
+
fail("nothing to weigh: pass a target or --client <name> (see --help)");
|
|
129
|
+
const results = [];
|
|
130
|
+
const skipped = [];
|
|
131
|
+
for (const target of targets) {
|
|
132
|
+
if (target.kind === "skillset") {
|
|
133
|
+
const set = findSkills(target.path);
|
|
134
|
+
if (!set.skills.length)
|
|
135
|
+
fail(`no SKILL.md found under "${target.label}"`);
|
|
136
|
+
results.push(await weighSkills(set));
|
|
137
|
+
}
|
|
138
|
+
else if (target.kind === "repo") {
|
|
139
|
+
fail(`"${target.label}" is a source directory; weigh needs a live server (stdio command or URL) or a skill path. For repo checks use: efaimo check --mcp ${target.label}`);
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
console.error(pc.dim(`connecting to ${target.label} ...`));
|
|
143
|
+
try {
|
|
144
|
+
const intro = await introspectServer(target, { timeoutMs });
|
|
145
|
+
results.push(await weighServer(intro, { anthropicApiKey: anthropic.apiKey, anthropicModel: anthropic.model }));
|
|
146
|
+
}
|
|
147
|
+
catch (e) {
|
|
148
|
+
// One broken/auth-gated server must not abort a multi-server run
|
|
149
|
+
// (--client weighs everything an editor loads; some entries need auth).
|
|
150
|
+
if (targets.length === 1)
|
|
151
|
+
throw e;
|
|
152
|
+
const reason = (e instanceof Error ? e.message : String(e)).split("\n")[0].trim();
|
|
153
|
+
skipped.push({ label: target.label, reason });
|
|
154
|
+
console.error(pc.yellow(`skipped ${target.label}: ${reason}`));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (!results.length) {
|
|
159
|
+
fail(`no server could be weighed (${skipped.length} of ${targets.length} failed; see reasons above)`);
|
|
160
|
+
}
|
|
161
|
+
// "Single" means the user asked for exactly one target, not that exactly
|
|
162
|
+
// one target survived: a multi-target run with skips must not silently
|
|
163
|
+
// write/diff a baseline or emit a single-object JSON for whichever server
|
|
164
|
+
// happened to work.
|
|
165
|
+
const single = targets.length === 1 && results.length === 1 ? results[0] : undefined;
|
|
166
|
+
const budgetTotal = results.reduce((s, r) => s + (r.kind === "mcp" ? r.totals.claudeStyle : r.totals.metadata + r.totals.body), 0);
|
|
167
|
+
if (opts.out && single) {
|
|
168
|
+
fs.writeFileSync(opts.out, toJsonEnvelope("weigh", single));
|
|
169
|
+
console.error(pc.dim(`baseline written: ${opts.out}`));
|
|
170
|
+
}
|
|
171
|
+
else if (opts.out) {
|
|
172
|
+
console.error(pc.yellow("--out skipped: it writes a single baseline, but multiple targets were requested"));
|
|
173
|
+
}
|
|
174
|
+
if (opts.json) {
|
|
175
|
+
console.log(toJsonEnvelope("weigh", single ?? results));
|
|
176
|
+
}
|
|
177
|
+
else if (opts.md) {
|
|
178
|
+
console.log(results.map(renderWeighMarkdown).join("\n\n---\n\n"));
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
console.log(results
|
|
182
|
+
.map((r) => (r.kind === "mcp" ? renderServerWeighPretty(r) : renderSkillWeighPretty(r)))
|
|
183
|
+
.join("\n\n"));
|
|
184
|
+
if (results.length > 1) {
|
|
185
|
+
const mcpTotals = results
|
|
186
|
+
.filter((r) => r.kind === "mcp")
|
|
187
|
+
.reduce((s, r) => s + r.totals.claudeStyle, 0);
|
|
188
|
+
if (mcpTotals > 0) {
|
|
189
|
+
console.log(pc.bold(`\ncombined MCP context cost (Claude-style, o200k est.): ${mcpTotals.toLocaleString("en-US")} tokens (~${((mcpTotals / 200000) * 100).toFixed(1)}% of a 200k window)`));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
if (skipped.length) {
|
|
194
|
+
console.error(pc.yellow(`weighed ${results.length} of ${targets.length} servers; skipped: ${skipped.map((s) => s.label).join(", ")}`));
|
|
195
|
+
}
|
|
196
|
+
if (opts.diff) {
|
|
197
|
+
if (!single || single.kind !== "mcp")
|
|
198
|
+
fail("--diff works on a single MCP server result");
|
|
199
|
+
const baselineRaw = JSON.parse(fs.readFileSync(opts.diff, "utf8"));
|
|
200
|
+
const baseline = ("data" in baselineRaw && baselineRaw.data ? baselineRaw.data : baselineRaw);
|
|
201
|
+
if (!baseline?.totals)
|
|
202
|
+
fail(`"${opts.diff}" is not a weigh baseline (write one with --out)`);
|
|
203
|
+
const d = diffServerWeigh(baseline, single);
|
|
204
|
+
console.log("");
|
|
205
|
+
console.log(opts.md ? renderDiffMarkdown(d) : renderDiffPretty(d, { maxTokens, allowIncreasePct }));
|
|
206
|
+
if ((allowIncreasePct !== undefined && d.pct > allowIncreasePct) ||
|
|
207
|
+
(maxTokens !== undefined && d.after > maxTokens)) {
|
|
208
|
+
process.exitCode = 1;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
else if (maxTokens !== undefined && budgetTotal > maxTokens) {
|
|
212
|
+
const scope = single ? "" : " (combined)";
|
|
213
|
+
console.error(pc.red(`budget exceeded${scope}: ${budgetTotal.toLocaleString("en-US")} > --max-tokens ${maxTokens}`));
|
|
214
|
+
process.exitCode = 1;
|
|
215
|
+
}
|
|
216
|
+
if (single)
|
|
217
|
+
writeBadge(opts.badge, weighBadgeSpec(single));
|
|
218
|
+
});
|
|
219
|
+
program
|
|
220
|
+
.command("check")
|
|
221
|
+
.description("audit quality and 2026-07-28 readiness of an MCP server, or lint Agent Skills")
|
|
222
|
+
.argument("[target]", "stdio command | http(s) URL | skill path | repo dir")
|
|
223
|
+
.option("--mcp", "treat target as an MCP server / server repo")
|
|
224
|
+
.option("--skill", "treat target as a skill path")
|
|
225
|
+
.option("--stdio", "treat target as a stdio command string")
|
|
226
|
+
.option("--repo <path>", "additionally scan this source repo for deprecated API usage")
|
|
227
|
+
.option("--no-probe", "skip live readiness probes (bare request, server/discover, ordering)")
|
|
228
|
+
.option("--strict", "exit 1 on warnings too")
|
|
229
|
+
.option("--conformance", "after the audit, run the official MCP conformance suite (http targets)")
|
|
230
|
+
.option("--header <header>", 'HTTP header "Key: Value" (repeatable)', collectPairs(":"))
|
|
231
|
+
.option("--env <pair>", "KEY=VALUE for stdio servers (repeatable)", collectPairs("="))
|
|
232
|
+
.option("--timeout <seconds>", "connect timeout in seconds", "45")
|
|
233
|
+
.option("--json", "print JSON")
|
|
234
|
+
.option("--md", "print Markdown")
|
|
235
|
+
.option("--badge [file]", "write a grade badge SVG + shields endpoint JSON")
|
|
236
|
+
.option("--anthropic [model]", "use exact Claude token counts where relevant")
|
|
237
|
+
.option("--no-color", "disable colors")
|
|
238
|
+
.action(async (targetArg, opts) => {
|
|
239
|
+
colorSetup(opts);
|
|
240
|
+
if (!targetArg)
|
|
241
|
+
fail("nothing to check: pass a target (see --help)");
|
|
242
|
+
if (opts.mcp && opts.skill)
|
|
243
|
+
fail("--mcp and --skill are mutually exclusive");
|
|
244
|
+
const timeoutMs = seconds(opts.timeout) * 1000;
|
|
245
|
+
const anthropic = anthropicKeyFor(opts);
|
|
246
|
+
let target = resolveTarget(targetArg, {
|
|
247
|
+
forceStdio: opts.stdio,
|
|
248
|
+
forceSkill: opts.skill,
|
|
249
|
+
forceRepo: false,
|
|
250
|
+
env: opts.env,
|
|
251
|
+
headers: opts.header,
|
|
252
|
+
});
|
|
253
|
+
if (opts.mcp && target.kind === "skillset") {
|
|
254
|
+
target = { kind: "repo", path: target.path, label: target.label };
|
|
255
|
+
}
|
|
256
|
+
let report;
|
|
257
|
+
let skillSet;
|
|
258
|
+
if (target.kind === "skillset") {
|
|
259
|
+
const res = await checkSkillSet(target.path, target.label);
|
|
260
|
+
// A single skill shows a full report; a set shows per-skill grades.
|
|
261
|
+
if (res.perSkill.length === 1 && res.setFindings.length === 0)
|
|
262
|
+
report = res.perSkill[0].report;
|
|
263
|
+
else
|
|
264
|
+
skillSet = res;
|
|
265
|
+
}
|
|
266
|
+
else if (target.kind === "repo") {
|
|
267
|
+
report = checkMcpRepoOnly(target.path, target.label);
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
console.error(pc.dim(`connecting to ${target.label} ...`));
|
|
271
|
+
const res = await checkMcpTarget(target, {
|
|
272
|
+
timeoutMs,
|
|
273
|
+
probe: opts.probe,
|
|
274
|
+
repoPath: opts.repo,
|
|
275
|
+
anthropicApiKey: anthropic.apiKey,
|
|
276
|
+
});
|
|
277
|
+
report = res.report;
|
|
278
|
+
}
|
|
279
|
+
if (skillSet) {
|
|
280
|
+
if (opts.json)
|
|
281
|
+
console.log(toJsonEnvelope("check", skillSet));
|
|
282
|
+
else if (opts.md)
|
|
283
|
+
console.log(renderSkillSetMarkdown(skillSet));
|
|
284
|
+
else
|
|
285
|
+
console.log(renderSkillSetPretty(skillSet));
|
|
286
|
+
const errs = skillSet.perSkill.reduce((n, s) => n + s.report.counts.error, 0) +
|
|
287
|
+
skillSet.setFindings.filter((f) => f.severity === "error").length;
|
|
288
|
+
const warns = skillSet.perSkill.reduce((n, s) => n + s.report.counts.warn, 0) +
|
|
289
|
+
skillSet.setFindings.filter((f) => f.severity === "warn").length;
|
|
290
|
+
if (errs > 0 || (opts.strict && warns > 0))
|
|
291
|
+
process.exitCode = 1;
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
if (!report)
|
|
295
|
+
return;
|
|
296
|
+
if (opts.json)
|
|
297
|
+
console.log(toJsonEnvelope("check", report));
|
|
298
|
+
else if (opts.md)
|
|
299
|
+
console.log(renderCheckMarkdown(report));
|
|
300
|
+
else
|
|
301
|
+
console.log(renderCheckPretty(report));
|
|
302
|
+
writeBadge(opts.badge, gradeBadgeSpec(report));
|
|
303
|
+
if (opts.conformance) {
|
|
304
|
+
if (target.kind !== "http") {
|
|
305
|
+
console.error(pc.yellow("note: the official conformance suite drives http targets (--url); skipping for this target"));
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
console.error(pc.dim("\nrunning official MCP conformance suite (npx @modelcontextprotocol/conformance)...\n"));
|
|
309
|
+
// cross-spawn resolves npx(.cmd) without a shell, so target.url is passed
|
|
310
|
+
// as a literal argument (never interpreted by cmd.exe).
|
|
311
|
+
const r = crossSpawn.sync("npx", ["-y", "@modelcontextprotocol/conformance", "server", "--url", target.url], {
|
|
312
|
+
stdio: "inherit",
|
|
313
|
+
});
|
|
314
|
+
if (r.status !== 0)
|
|
315
|
+
console.error(pc.yellow(`conformance suite exited with code ${r.status ?? "unknown"}`));
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (report.counts.error > 0 || (opts.strict && report.counts.warn > 0)) {
|
|
319
|
+
process.exitCode = 1;
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
program
|
|
323
|
+
.command("test")
|
|
324
|
+
.description("does a skill actually improve task completion? (experimental A/B outcome harness)")
|
|
325
|
+
.argument("<scenario>", "a scenario YAML file (see examples/scenario.example.yaml)")
|
|
326
|
+
.option("--live", "run for real (spends tokens; Claude models need ANTHROPIC_API_KEY, GPT models need OPENAI_API_KEY)")
|
|
327
|
+
.option("--model <model>", "override the scenario's model (e.g. gpt-4o-mini, claude-sonnet-5)")
|
|
328
|
+
.option("--json", "print JSON")
|
|
329
|
+
.option("--no-color", "disable colors")
|
|
330
|
+
.addHelpText("after", "\nWithout --live this validates the scenario and prints the plan, making no API calls.")
|
|
331
|
+
.action(async (file, opts) => {
|
|
332
|
+
colorSetup(opts);
|
|
333
|
+
const parsed = parseScenario(file);
|
|
334
|
+
const scenario = opts.model ? { ...parsed, model: opts.model } : parsed;
|
|
335
|
+
if (!opts.live) {
|
|
336
|
+
console.log(renderScenarioPlan(scenario));
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
const provider = providerForModel(scenario.model);
|
|
340
|
+
if (provider === "unknown") {
|
|
341
|
+
fail(`model "${scenario.model}" is not supported: efaimo test runs Claude (claude-*) and OpenAI (gpt-*, o*) models`);
|
|
342
|
+
}
|
|
343
|
+
const envVar = provider === "openai" ? "OPENAI_API_KEY" : "ANTHROPIC_API_KEY";
|
|
344
|
+
const apiKey = process.env[envVar];
|
|
345
|
+
if (!apiKey)
|
|
346
|
+
fail(`model "${scenario.model}" needs ${envVar} in the environment or a .env file`);
|
|
347
|
+
if (dotEnvKeys.has(envVar))
|
|
348
|
+
console.error(pc.dim(`using ${envVar} from .env`));
|
|
349
|
+
const runner = provider === "openai" ? openaiRunner(apiKey) : anthropicRunner(apiKey);
|
|
350
|
+
console.error(pc.dim(`running ~${scenario.trials * 4} ${provider} API calls against ${scenario.model} ...`));
|
|
351
|
+
const report = await runScenario(scenario, runner);
|
|
352
|
+
if (opts.json)
|
|
353
|
+
console.log(toJsonEnvelope("test", report));
|
|
354
|
+
else
|
|
355
|
+
console.log(renderTestReportPretty(report));
|
|
356
|
+
if (report.verdict === "hurts")
|
|
357
|
+
process.exitCode = 1;
|
|
358
|
+
});
|
|
359
|
+
program
|
|
360
|
+
.command("mcp")
|
|
361
|
+
.description("run efaimo as a read-only MCP server (exposes the skill checks to an agent over stdio)")
|
|
362
|
+
.addHelpText("after", "\nStarts a stdio MCP server with two read-only tools, efaimo_check_skill and efaimo_weigh_skill,\n" +
|
|
363
|
+
"so an agent can lint or weigh a skill mid-session. It reads files only: no process is spawned,\n" +
|
|
364
|
+
"no socket is opened, and `test` (which spends tokens) is not exposed.")
|
|
365
|
+
.action(async () => {
|
|
366
|
+
const { runMcpServer } = await import("./mcp/server.js");
|
|
367
|
+
await runMcpServer();
|
|
368
|
+
});
|
|
369
|
+
program.parseAsync(process.argv).catch((e) => {
|
|
370
|
+
console.error(pc.red(`error: ${e instanceof Error ? e.message : String(e)}`));
|
|
371
|
+
process.exitCode = 2;
|
|
372
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ServerIntrospection } from "../core/types.js";
|
|
2
|
+
import type { ResolvedTarget } from "../targets/resolve.js";
|
|
3
|
+
export interface IntrospectOptions {
|
|
4
|
+
timeoutMs?: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function introspectServer(target: Extract<ResolvedTarget, {
|
|
7
|
+
kind: "stdio" | "http";
|
|
8
|
+
}>, opts?: IntrospectOptions): Promise<ServerIntrospection>;
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import crossSpawn from "cross-spawn";
|
|
2
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
3
|
+
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
4
|
+
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
5
|
+
import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
|
|
6
|
+
import { introspectStateless } from "./stateless.js";
|
|
7
|
+
import { minimalChildEnv } from "../util/childEnv.js";
|
|
8
|
+
import { truncate, withTimeout } from "../util/misc.js";
|
|
9
|
+
import { VERSION } from "../version.js";
|
|
10
|
+
const MAX_PAGES = 20;
|
|
11
|
+
export async function introspectServer(target, opts = {}) {
|
|
12
|
+
const timeoutMs = opts.timeoutMs ?? 45000;
|
|
13
|
+
const notes = [];
|
|
14
|
+
let stderrTail = "";
|
|
15
|
+
let transport;
|
|
16
|
+
let httpTransport;
|
|
17
|
+
const client = new Client({ name: "efaimo", version: VERSION }, { capabilities: {} });
|
|
18
|
+
if (target.kind === "stdio") {
|
|
19
|
+
const stdio = new StdioClientTransport({
|
|
20
|
+
command: target.command,
|
|
21
|
+
args: target.args,
|
|
22
|
+
env: target.env ? { ...minimalChildEnv(), ...target.env } : undefined,
|
|
23
|
+
stderr: "pipe",
|
|
24
|
+
});
|
|
25
|
+
transport = stdio;
|
|
26
|
+
try {
|
|
27
|
+
await withTimeout(client.connect(stdio), timeoutMs, "connect (stdio)");
|
|
28
|
+
}
|
|
29
|
+
catch (e) {
|
|
30
|
+
await safeClose(client, stdio);
|
|
31
|
+
// A 2026-07-28 stateless server has no initialize, so the SDK handshake
|
|
32
|
+
// fails by design; try bare stateless requests before giving up.
|
|
33
|
+
const rc = await introspectStateless(target, { timeoutMs });
|
|
34
|
+
if (rc)
|
|
35
|
+
return rc;
|
|
36
|
+
throw connectError(e, target.label, stderrTail);
|
|
37
|
+
}
|
|
38
|
+
// The child is spawned by connect(); its stderr stream only exists now.
|
|
39
|
+
const errStream = stdio.stderr;
|
|
40
|
+
errStream?.on("data", (chunk) => {
|
|
41
|
+
if (stderrTail.length < 2048)
|
|
42
|
+
stderrTail += chunk.toString("utf8");
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
const url = new URL(target.url);
|
|
47
|
+
// redirect:"error" so a hostile target cannot bounce an authenticated request
|
|
48
|
+
// to an internal host and replay the user's --header credentials.
|
|
49
|
+
const requestInit = { redirect: "error", ...(target.headers ? { headers: target.headers } : {}) };
|
|
50
|
+
const streamable = new StreamableHTTPClientTransport(url, { requestInit });
|
|
51
|
+
try {
|
|
52
|
+
await withTimeout(client.connect(streamable), timeoutMs, "connect (streamable http)");
|
|
53
|
+
transport = streamable;
|
|
54
|
+
httpTransport = "streamable";
|
|
55
|
+
}
|
|
56
|
+
catch (streamableErr) {
|
|
57
|
+
await safeClose(client, streamable);
|
|
58
|
+
const sse = new SSEClientTransport(url, { requestInit });
|
|
59
|
+
const sseClient = new Client({ name: "efaimo", version: VERSION }, { capabilities: {} });
|
|
60
|
+
try {
|
|
61
|
+
await withTimeout(sseClient.connect(sse), timeoutMs, "connect (legacy sse)");
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
await safeClose(sseClient, sse);
|
|
65
|
+
// Neither SDK transport connected; a stateless RC server rejects the
|
|
66
|
+
// initialize both send. Try bare stateless requests before giving up.
|
|
67
|
+
const rc = await introspectStateless(target, { timeoutMs });
|
|
68
|
+
if (rc)
|
|
69
|
+
return rc;
|
|
70
|
+
throw connectError(streamableErr, target.label, "");
|
|
71
|
+
}
|
|
72
|
+
notes.push("connected via legacy HTTP+SSE transport (deprecated; Streamable HTTP failed)");
|
|
73
|
+
return finishIntrospection(sseClient, sse, target.label, "http", "sse-legacy", notes, timeoutMs);
|
|
74
|
+
}
|
|
75
|
+
return finishIntrospection(client, transport, target.label, "http", httpTransport, notes, timeoutMs);
|
|
76
|
+
}
|
|
77
|
+
if (stderrTail.trim())
|
|
78
|
+
notes.push(`server stderr: ${truncate(stderrTail.trim(), 300)}`);
|
|
79
|
+
return finishIntrospection(client, transport, target.label, "stdio", undefined, notes, timeoutMs);
|
|
80
|
+
}
|
|
81
|
+
async function finishIntrospection(client, transport, label, kind, httpTransport, notes, timeoutMs) {
|
|
82
|
+
try {
|
|
83
|
+
const tools = [];
|
|
84
|
+
let rawToolsListResult;
|
|
85
|
+
let cursor;
|
|
86
|
+
for (let page = 0; page < MAX_PAGES; page++) {
|
|
87
|
+
const res = await withTimeout(client.listTools(cursor ? { cursor } : undefined), timeoutMs, "tools/list");
|
|
88
|
+
if (page === 0)
|
|
89
|
+
rawToolsListResult = res;
|
|
90
|
+
for (const t of res.tools) {
|
|
91
|
+
tools.push({
|
|
92
|
+
name: t.name,
|
|
93
|
+
title: t.title,
|
|
94
|
+
description: t.description,
|
|
95
|
+
inputSchema: t.inputSchema,
|
|
96
|
+
outputSchema: t.outputSchema,
|
|
97
|
+
annotations: t.annotations,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
cursor = res.nextCursor;
|
|
101
|
+
if (!cursor)
|
|
102
|
+
break;
|
|
103
|
+
if (page === MAX_PAGES - 1)
|
|
104
|
+
notes.push(`tools/list pagination stopped at ${MAX_PAGES} pages`);
|
|
105
|
+
}
|
|
106
|
+
const serverInfo = client.getServerVersion();
|
|
107
|
+
const capabilities = client.getServerCapabilities();
|
|
108
|
+
const instructions = client.getInstructions();
|
|
109
|
+
// Only call resources/prompts when the server declares them, and surface a
|
|
110
|
+
// note (rather than silently swallowing) if a declared capability fails.
|
|
111
|
+
const resources = [];
|
|
112
|
+
if (capabilities?.resources) {
|
|
113
|
+
try {
|
|
114
|
+
let rCursor;
|
|
115
|
+
for (let page = 0; page < MAX_PAGES; page++) {
|
|
116
|
+
const res = await withTimeout(client.listResources(rCursor ? { cursor: rCursor } : undefined), timeoutMs, "resources/list");
|
|
117
|
+
for (const r of res.resources) {
|
|
118
|
+
resources.push({ uri: r.uri, name: r.name, description: r.description, mimeType: r.mimeType });
|
|
119
|
+
}
|
|
120
|
+
rCursor = res.nextCursor;
|
|
121
|
+
if (!rCursor)
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
catch (e) {
|
|
126
|
+
notes.push(`resources/list failed: ${e.message}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const prompts = [];
|
|
130
|
+
if (capabilities?.prompts) {
|
|
131
|
+
try {
|
|
132
|
+
let pCursor;
|
|
133
|
+
for (let page = 0; page < MAX_PAGES; page++) {
|
|
134
|
+
const res = await withTimeout(client.listPrompts(pCursor ? { cursor: pCursor } : undefined), timeoutMs, "prompts/list");
|
|
135
|
+
for (const p of res.prompts) {
|
|
136
|
+
prompts.push({ name: p.name, title: p.title, description: p.description });
|
|
137
|
+
}
|
|
138
|
+
pCursor = res.nextCursor;
|
|
139
|
+
if (!pCursor)
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch (e) {
|
|
144
|
+
notes.push(`prompts/list failed: ${e.message}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
targetLabel: label,
|
|
149
|
+
transport: kind,
|
|
150
|
+
httpTransport,
|
|
151
|
+
serverInfo: serverInfo
|
|
152
|
+
? { name: serverInfo.name, version: serverInfo.version, title: serverInfo.title }
|
|
153
|
+
: undefined,
|
|
154
|
+
instructions,
|
|
155
|
+
capabilities,
|
|
156
|
+
tools,
|
|
157
|
+
resources,
|
|
158
|
+
prompts,
|
|
159
|
+
rawToolsListResult,
|
|
160
|
+
notes,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
// On Windows the SDK's close() kills only the cmd.exe wrapper for npx/.cmd
|
|
165
|
+
// servers; tree-kill the child first so the real server is not orphaned.
|
|
166
|
+
const pid = transport.pid;
|
|
167
|
+
if (pid && process.platform === "win32") {
|
|
168
|
+
try {
|
|
169
|
+
crossSpawn.sync("taskkill", ["/PID", String(pid), "/T", "/F"], { stdio: "ignore" });
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
/* ignore */
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
await safeClose(client, transport);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
async function safeClose(client, transport) {
|
|
179
|
+
try {
|
|
180
|
+
await client.close();
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
/* ignore */
|
|
184
|
+
}
|
|
185
|
+
try {
|
|
186
|
+
await transport.close();
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
/* ignore */
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
function connectError(e, label, stderrTail) {
|
|
193
|
+
const base = e instanceof Error ? e.message : String(e);
|
|
194
|
+
const hint = stderrTail.trim()
|
|
195
|
+
? `\n server stderr: ${truncate(stderrTail.trim(), 400)}`
|
|
196
|
+
: "";
|
|
197
|
+
return new Error(`could not connect to "${label}": ${base}${hint}\n hints: check the command/URL, required env vars or auth headers; try --timeout for slow cold starts (npx downloads on first run).`);
|
|
198
|
+
}
|