@yawlabs/mcp-compliance 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +602 -591
- package/dist/{chunk-G5K7CRWU.js → chunk-X5CVUDPW.js} +65 -27
- package/dist/index.js +124 -38
- package/dist/mcp/server.js +12 -4
- package/dist/runner.d.ts +9 -1
- package/dist/runner.js +3 -1
- package/package.json +2 -2
|
@@ -37,12 +37,28 @@ function computeScore(tests) {
|
|
|
37
37
|
const failed = total - passed;
|
|
38
38
|
const requiredTests = tests.filter((t) => t.required);
|
|
39
39
|
const requiredPassed = requiredTests.filter((t) => t.passed).length;
|
|
40
|
-
const requiredScore = requiredTests.length > 0 ? requiredPassed / requiredTests.length * 70 : 70;
|
|
41
40
|
const optionalTests = tests.filter((t) => !t.required);
|
|
42
41
|
const optionalPassed = optionalTests.filter((t) => t.passed).length;
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
let score;
|
|
43
|
+
if (total === 0) {
|
|
44
|
+
score = 0;
|
|
45
|
+
} else if (requiredTests.length === 0) {
|
|
46
|
+
score = Math.round(optionalPassed / optionalTests.length * 100);
|
|
47
|
+
} else if (optionalTests.length === 0) {
|
|
48
|
+
score = Math.round(requiredPassed / requiredTests.length * 100);
|
|
49
|
+
} else {
|
|
50
|
+
score = Math.round(requiredPassed / requiredTests.length * 70 + optionalPassed / optionalTests.length * 30);
|
|
51
|
+
}
|
|
52
|
+
let overall;
|
|
53
|
+
if (total === 0) {
|
|
54
|
+
overall = "fail";
|
|
55
|
+
} else if (requiredPassed < requiredTests.length) {
|
|
56
|
+
overall = "fail";
|
|
57
|
+
} else if (passed === total) {
|
|
58
|
+
overall = "pass";
|
|
59
|
+
} else {
|
|
60
|
+
overall = "partial";
|
|
61
|
+
}
|
|
46
62
|
const categories = {};
|
|
47
63
|
for (const t of tests) {
|
|
48
64
|
if (!categories[t.category]) categories[t.category] = { passed: 0, total: 0 };
|
|
@@ -63,7 +79,7 @@ import { request } from "undici";
|
|
|
63
79
|
|
|
64
80
|
// src/sse.ts
|
|
65
81
|
function parseSSEResponse(text) {
|
|
66
|
-
const lines = text.split(
|
|
82
|
+
const lines = text.split(/\r?\n/);
|
|
67
83
|
let firstJsonRpcResponse = null;
|
|
68
84
|
let currentData = [];
|
|
69
85
|
function flushEvent() {
|
|
@@ -106,7 +122,8 @@ function createHttpTransport(opts) {
|
|
|
106
122
|
function normalizeHeaders(raw) {
|
|
107
123
|
const out = {};
|
|
108
124
|
for (const [k, v] of Object.entries(raw)) {
|
|
109
|
-
if (
|
|
125
|
+
if (v === void 0) continue;
|
|
126
|
+
out[k] = Array.isArray(v) ? v.join(", ") : v;
|
|
110
127
|
}
|
|
111
128
|
return out;
|
|
112
129
|
}
|
|
@@ -197,6 +214,13 @@ function createHttpTransport(opts) {
|
|
|
197
214
|
|
|
198
215
|
// src/transport/stdio.ts
|
|
199
216
|
import { spawn } from "child_process";
|
|
217
|
+
function exitDiagnostic(code, signal) {
|
|
218
|
+
if (signal) return `server terminated by signal ${signal} before completing the request`;
|
|
219
|
+
if (code === 0) {
|
|
220
|
+
return "server exited cleanly (code 0) before completing the request. This usually means the command is a one-shot CLI, not a long-running MCP stdio server. If the server needs a subcommand to start (e.g. `serve`, `mcp`, `start`), include it in the command.";
|
|
221
|
+
}
|
|
222
|
+
return `server crashed with exit code ${code} before completing the request`;
|
|
223
|
+
}
|
|
200
224
|
function createStdioTransport(opts) {
|
|
201
225
|
const { command, args = [], env, cwd, verbose = false } = opts;
|
|
202
226
|
const stderrBufferSize = opts.stderrBufferSize ?? 64 * 1024;
|
|
@@ -236,8 +260,7 @@ function createStdioTransport(opts) {
|
|
|
236
260
|
exited = true;
|
|
237
261
|
exitCode = code;
|
|
238
262
|
if (pending.size > 0) {
|
|
239
|
-
|
|
240
|
-
rejectAllPending(new Error(reason));
|
|
263
|
+
rejectAllPending(new Error(exitDiagnostic(code, signal)));
|
|
241
264
|
}
|
|
242
265
|
});
|
|
243
266
|
child.stdout?.setEncoding("utf8");
|
|
@@ -250,6 +273,11 @@ function createStdioTransport(opts) {
|
|
|
250
273
|
handleLine(line);
|
|
251
274
|
}
|
|
252
275
|
if (stdoutBuffer.length > stdoutBufferSize) {
|
|
276
|
+
stderrBuffer += `[mcp-compliance] stdout buffer exceeded ${stdoutBufferSize} bytes without a newline; discarding buffered data
|
|
277
|
+
`;
|
|
278
|
+
if (stderrBuffer.length > stderrBufferSize) {
|
|
279
|
+
stderrBuffer = stderrBuffer.slice(stderrBuffer.length - stderrBufferSize);
|
|
280
|
+
}
|
|
253
281
|
stdoutBuffer = "";
|
|
254
282
|
}
|
|
255
283
|
});
|
|
@@ -307,7 +335,7 @@ function createStdioTransport(opts) {
|
|
|
307
335
|
}
|
|
308
336
|
}
|
|
309
337
|
if (exited) {
|
|
310
|
-
throw new Error(annotateWithStderr(`stdio transport:
|
|
338
|
+
throw new Error(annotateWithStderr(`stdio transport: ${exitDiagnostic(exitCode, null)}`));
|
|
311
339
|
}
|
|
312
340
|
if (spawnError) throw new Error(annotateWithStderr(`stdio transport: spawn failed \u2014 ${spawnError.message}`));
|
|
313
341
|
const stdin = child.stdin;
|
|
@@ -402,7 +430,7 @@ function createStdioTransport(opts) {
|
|
|
402
430
|
// src/types.ts
|
|
403
431
|
var REPORT_SCHEMA_VERSION = "1";
|
|
404
432
|
var TEST_DEFINITIONS = [
|
|
405
|
-
// ── Transport (13
|
|
433
|
+
// ── Transport (16 tests: 13 HTTP + 3 stdio) ──────────────────────
|
|
406
434
|
{
|
|
407
435
|
id: "transport-post",
|
|
408
436
|
name: "HTTP POST accepted",
|
|
@@ -551,7 +579,7 @@ var TEST_DEFINITIONS = [
|
|
|
551
579
|
recommendation: "Return JSON-RPC error -32601 (Method not found) for unknown methods. Do not exit the process or disconnect \u2014 the client should be able to keep using the session after an error.",
|
|
552
580
|
transports: ["stdio"]
|
|
553
581
|
},
|
|
554
|
-
// ── Lifecycle (
|
|
582
|
+
// ── Lifecycle (21 tests) ─────────────────────────────────────────
|
|
555
583
|
{
|
|
556
584
|
id: "lifecycle-init",
|
|
557
585
|
name: "Initialize handshake",
|
|
@@ -1249,17 +1277,13 @@ var STACK_TRACE_PATTERNS = [
|
|
|
1249
1277
|
// PHP
|
|
1250
1278
|
/panicked\s+at\s+'/i,
|
|
1251
1279
|
// Rust
|
|
1252
|
-
/ENOENT|EACCES|EPERM/,
|
|
1253
|
-
// Node.js system errors
|
|
1254
1280
|
/node_modules\//,
|
|
1255
|
-
// Node.js module paths
|
|
1256
|
-
/\/usr\/local\/|\/home\//,
|
|
1257
|
-
// Unix paths
|
|
1258
|
-
/[A-Z]
|
|
1259
|
-
// Windows paths
|
|
1260
|
-
/
|
|
1261
|
-
// Sensitive terms
|
|
1262
|
-
/jdbc:|mysql:|postgres:|mongodb:/i
|
|
1281
|
+
// Node.js module paths (filesystem layout leak)
|
|
1282
|
+
/\/usr\/local\/|\/home\/|\/root\//,
|
|
1283
|
+
// Unix absolute paths
|
|
1284
|
+
/[A-Z]:\\[\w\s.-]+\\[\w\s.-]+/,
|
|
1285
|
+
// Windows absolute paths (drive + 2+ segments)
|
|
1286
|
+
/jdbc:|mysql:\/\/|postgres(?:ql)?:\/\/|mongodb(?:\+srv)?:\/\//i
|
|
1263
1287
|
// DB connection strings
|
|
1264
1288
|
];
|
|
1265
1289
|
var INTERNAL_IP_PATTERNS = [
|
|
@@ -1278,6 +1302,20 @@ function createIdCounter(start = 0) {
|
|
|
1278
1302
|
let id = start;
|
|
1279
1303
|
return () => ++id;
|
|
1280
1304
|
}
|
|
1305
|
+
function dedupAndCapWarnings(warnings, max) {
|
|
1306
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1307
|
+
const deduped = [];
|
|
1308
|
+
for (const w of warnings) {
|
|
1309
|
+
if (seen.has(w)) continue;
|
|
1310
|
+
seen.add(w);
|
|
1311
|
+
deduped.push(w);
|
|
1312
|
+
}
|
|
1313
|
+
if (deduped.length > max) {
|
|
1314
|
+
const truncated = deduped.length - max;
|
|
1315
|
+
return [...deduped.slice(0, max), `... and ${truncated} more warning(s) suppressed`];
|
|
1316
|
+
}
|
|
1317
|
+
return deduped;
|
|
1318
|
+
}
|
|
1281
1319
|
var STDIO_INCOMPATIBLE_IDS = /* @__PURE__ */ new Set([
|
|
1282
1320
|
// Lifecycle tests that use raw undici for HTTP-specific checks
|
|
1283
1321
|
"lifecycle-string-id",
|
|
@@ -4030,14 +4068,13 @@ async function runComplianceSuite(target, options = {}) {
|
|
|
4030
4068
|
return { passed: true, details: "Unknown method returned JSON-RPC error; subsequent ping succeeded" };
|
|
4031
4069
|
}
|
|
4032
4070
|
);
|
|
4033
|
-
const MAX_WARNINGS = 100;
|
|
4034
|
-
if (warnings.length > MAX_WARNINGS) {
|
|
4035
|
-
const truncated = warnings.length - MAX_WARNINGS;
|
|
4036
|
-
warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
|
|
4037
|
-
}
|
|
4038
4071
|
if (inFlight.size > 0) await drainPool();
|
|
4072
|
+
const MAX_WARNINGS = 50;
|
|
4073
|
+
const capped = dedupAndCapWarnings(warnings, MAX_WARNINGS);
|
|
4074
|
+
warnings.length = 0;
|
|
4075
|
+
warnings.push(...capped);
|
|
4039
4076
|
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
4040
|
-
const badge = generateBadge(displayUrl);
|
|
4077
|
+
const badge = displayUrl.startsWith("stdio:") ? { imageUrl: "", reportUrl: "", markdown: "", html: "" } : generateBadge(displayUrl);
|
|
4041
4078
|
return {
|
|
4042
4079
|
schemaVersion: REPORT_SCHEMA_VERSION,
|
|
4043
4080
|
specVersion: SPEC_VERSION,
|
|
@@ -4075,6 +4112,7 @@ export {
|
|
|
4075
4112
|
TEST_DEFINITIONS,
|
|
4076
4113
|
SPEC_VERSION,
|
|
4077
4114
|
SPEC_BASE,
|
|
4115
|
+
dedupAndCapWarnings,
|
|
4078
4116
|
previewTests,
|
|
4079
4117
|
runComplianceSuite
|
|
4080
4118
|
};
|
package/dist/index.js
CHANGED
|
@@ -16,6 +16,9 @@ var GRADE_COLORS = {
|
|
|
16
16
|
F: "#e05d44"
|
|
17
17
|
};
|
|
18
18
|
var UNTESTED_COLOR = "#9f9f9f";
|
|
19
|
+
function escXml(s) {
|
|
20
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
21
|
+
}
|
|
19
22
|
function renderBadgeSvg(input) {
|
|
20
23
|
let gradeLabel = "unknown";
|
|
21
24
|
let color = UNTESTED_COLOR;
|
|
@@ -27,14 +30,16 @@ function renderBadgeSvg(input) {
|
|
|
27
30
|
title = `MCP Compliant: Grade ${input.grade}${input.score != null ? ` (${input.score}%)` : ""} - tested ${date}`;
|
|
28
31
|
}
|
|
29
32
|
const leftText = "MCP Compliant";
|
|
30
|
-
const rightText = gradeLabel;
|
|
33
|
+
const rightText = escXml(gradeLabel);
|
|
34
|
+
const ariaLabel = `${leftText}: ${escXml(gradeLabel)}`;
|
|
35
|
+
const titleEsc = escXml(title);
|
|
31
36
|
const leftWidth = 95;
|
|
32
37
|
const rightWidth = 40;
|
|
33
38
|
const totalWidth = leftWidth + rightWidth;
|
|
34
39
|
const leftX = leftWidth / 2;
|
|
35
40
|
const rightX = leftWidth + rightWidth / 2;
|
|
36
|
-
return `<svg xmlns="http://www.w3.org/2000/svg" width="${totalWidth}" height="20" role="img" aria-label="${
|
|
37
|
-
<title>${
|
|
41
|
+
return `<svg xmlns="http://www.w3.org/2000/svg" width="${totalWidth}" height="20" role="img" aria-label="${ariaLabel}">
|
|
42
|
+
<title>${titleEsc}</title>
|
|
38
43
|
<linearGradient id="s" x2="0" y2="100%">
|
|
39
44
|
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
|
|
40
45
|
<stop offset="1" stop-opacity=".1"/>
|
|
@@ -64,7 +69,7 @@ import { request } from "undici";
|
|
|
64
69
|
|
|
65
70
|
// src/sse.ts
|
|
66
71
|
function parseSSEResponse(text) {
|
|
67
|
-
const lines = text.split(
|
|
72
|
+
const lines = text.split(/\r?\n/);
|
|
68
73
|
let firstJsonRpcResponse = null;
|
|
69
74
|
let currentData = [];
|
|
70
75
|
function flushEvent() {
|
|
@@ -107,7 +112,8 @@ function createHttpTransport(opts) {
|
|
|
107
112
|
function normalizeHeaders(raw) {
|
|
108
113
|
const out = {};
|
|
109
114
|
for (const [k, v] of Object.entries(raw)) {
|
|
110
|
-
if (
|
|
115
|
+
if (v === void 0) continue;
|
|
116
|
+
out[k] = Array.isArray(v) ? v.join(", ") : v;
|
|
111
117
|
}
|
|
112
118
|
return out;
|
|
113
119
|
}
|
|
@@ -198,6 +204,13 @@ function createHttpTransport(opts) {
|
|
|
198
204
|
|
|
199
205
|
// src/transport/stdio.ts
|
|
200
206
|
import { spawn } from "child_process";
|
|
207
|
+
function exitDiagnostic(code, signal) {
|
|
208
|
+
if (signal) return `server terminated by signal ${signal} before completing the request`;
|
|
209
|
+
if (code === 0) {
|
|
210
|
+
return "server exited cleanly (code 0) before completing the request. This usually means the command is a one-shot CLI, not a long-running MCP stdio server. If the server needs a subcommand to start (e.g. `serve`, `mcp`, `start`), include it in the command.";
|
|
211
|
+
}
|
|
212
|
+
return `server crashed with exit code ${code} before completing the request`;
|
|
213
|
+
}
|
|
201
214
|
function createStdioTransport(opts) {
|
|
202
215
|
const { command, args = [], env, cwd, verbose = false } = opts;
|
|
203
216
|
const stderrBufferSize = opts.stderrBufferSize ?? 64 * 1024;
|
|
@@ -237,8 +250,7 @@ function createStdioTransport(opts) {
|
|
|
237
250
|
exited = true;
|
|
238
251
|
exitCode = code;
|
|
239
252
|
if (pending.size > 0) {
|
|
240
|
-
|
|
241
|
-
rejectAllPending(new Error(reason));
|
|
253
|
+
rejectAllPending(new Error(exitDiagnostic(code, signal)));
|
|
242
254
|
}
|
|
243
255
|
});
|
|
244
256
|
child.stdout?.setEncoding("utf8");
|
|
@@ -251,6 +263,11 @@ function createStdioTransport(opts) {
|
|
|
251
263
|
handleLine(line);
|
|
252
264
|
}
|
|
253
265
|
if (stdoutBuffer.length > stdoutBufferSize) {
|
|
266
|
+
stderrBuffer += `[mcp-compliance] stdout buffer exceeded ${stdoutBufferSize} bytes without a newline; discarding buffered data
|
|
267
|
+
`;
|
|
268
|
+
if (stderrBuffer.length > stderrBufferSize) {
|
|
269
|
+
stderrBuffer = stderrBuffer.slice(stderrBuffer.length - stderrBufferSize);
|
|
270
|
+
}
|
|
254
271
|
stdoutBuffer = "";
|
|
255
272
|
}
|
|
256
273
|
});
|
|
@@ -308,7 +325,7 @@ function createStdioTransport(opts) {
|
|
|
308
325
|
}
|
|
309
326
|
}
|
|
310
327
|
if (exited) {
|
|
311
|
-
throw new Error(annotateWithStderr(`stdio transport:
|
|
328
|
+
throw new Error(annotateWithStderr(`stdio transport: ${exitDiagnostic(exitCode, null)}`));
|
|
312
329
|
}
|
|
313
330
|
if (spawnError) throw new Error(annotateWithStderr(`stdio transport: spawn failed \u2014 ${spawnError.message}`));
|
|
314
331
|
const stdin = child.stdin;
|
|
@@ -579,6 +596,11 @@ function validateTarget(t, source) {
|
|
|
579
596
|
|
|
580
597
|
// src/diff.ts
|
|
581
598
|
function diffReports(baseline, current) {
|
|
599
|
+
if (baseline.specVersion && current.specVersion && baseline.specVersion !== current.specVersion) {
|
|
600
|
+
throw new Error(
|
|
601
|
+
`Spec version mismatch: baseline is ${baseline.specVersion}, current is ${current.specVersion}. Re-run the baseline with this tool version (or downgrade the tool to match) before diffing.`
|
|
602
|
+
);
|
|
603
|
+
}
|
|
582
604
|
const baseById = new Map(baseline.tests.map((t) => [t.id, t]));
|
|
583
605
|
const curById = new Map(current.tests.map((t) => [t.id, t]));
|
|
584
606
|
const regressions = [];
|
|
@@ -676,7 +698,7 @@ function hasRegressions(summary) {
|
|
|
676
698
|
}
|
|
677
699
|
|
|
678
700
|
// src/mcp/server.ts
|
|
679
|
-
import { existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
|
|
701
|
+
import { existsSync as existsSync2, readFileSync as readFileSync2, realpathSync } from "fs";
|
|
680
702
|
import { dirname, join as join2, resolve as resolve2 } from "path";
|
|
681
703
|
import { fileURLToPath } from "url";
|
|
682
704
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
@@ -724,12 +746,28 @@ function computeScore(tests) {
|
|
|
724
746
|
const failed = total - passed;
|
|
725
747
|
const requiredTests = tests.filter((t) => t.required);
|
|
726
748
|
const requiredPassed = requiredTests.filter((t) => t.passed).length;
|
|
727
|
-
const requiredScore = requiredTests.length > 0 ? requiredPassed / requiredTests.length * 70 : 70;
|
|
728
749
|
const optionalTests = tests.filter((t) => !t.required);
|
|
729
750
|
const optionalPassed = optionalTests.filter((t) => t.passed).length;
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
751
|
+
let score;
|
|
752
|
+
if (total === 0) {
|
|
753
|
+
score = 0;
|
|
754
|
+
} else if (requiredTests.length === 0) {
|
|
755
|
+
score = Math.round(optionalPassed / optionalTests.length * 100);
|
|
756
|
+
} else if (optionalTests.length === 0) {
|
|
757
|
+
score = Math.round(requiredPassed / requiredTests.length * 100);
|
|
758
|
+
} else {
|
|
759
|
+
score = Math.round(requiredPassed / requiredTests.length * 70 + optionalPassed / optionalTests.length * 30);
|
|
760
|
+
}
|
|
761
|
+
let overall;
|
|
762
|
+
if (total === 0) {
|
|
763
|
+
overall = "fail";
|
|
764
|
+
} else if (requiredPassed < requiredTests.length) {
|
|
765
|
+
overall = "fail";
|
|
766
|
+
} else if (passed === total) {
|
|
767
|
+
overall = "pass";
|
|
768
|
+
} else {
|
|
769
|
+
overall = "partial";
|
|
770
|
+
}
|
|
733
771
|
const categories = {};
|
|
734
772
|
for (const t of tests) {
|
|
735
773
|
if (!categories[t.category]) categories[t.category] = { passed: 0, total: 0 };
|
|
@@ -748,7 +786,7 @@ function computeScore(tests) {
|
|
|
748
786
|
// src/types.ts
|
|
749
787
|
var REPORT_SCHEMA_VERSION = "1";
|
|
750
788
|
var TEST_DEFINITIONS = [
|
|
751
|
-
// ── Transport (13
|
|
789
|
+
// ── Transport (16 tests: 13 HTTP + 3 stdio) ──────────────────────
|
|
752
790
|
{
|
|
753
791
|
id: "transport-post",
|
|
754
792
|
name: "HTTP POST accepted",
|
|
@@ -897,7 +935,7 @@ var TEST_DEFINITIONS = [
|
|
|
897
935
|
recommendation: "Return JSON-RPC error -32601 (Method not found) for unknown methods. Do not exit the process or disconnect \u2014 the client should be able to keep using the session after an error.",
|
|
898
936
|
transports: ["stdio"]
|
|
899
937
|
},
|
|
900
|
-
// ── Lifecycle (
|
|
938
|
+
// ── Lifecycle (21 tests) ─────────────────────────────────────────
|
|
901
939
|
{
|
|
902
940
|
id: "lifecycle-init",
|
|
903
941
|
name: "Initialize handshake",
|
|
@@ -1595,17 +1633,13 @@ var STACK_TRACE_PATTERNS = [
|
|
|
1595
1633
|
// PHP
|
|
1596
1634
|
/panicked\s+at\s+'/i,
|
|
1597
1635
|
// Rust
|
|
1598
|
-
/ENOENT|EACCES|EPERM/,
|
|
1599
|
-
// Node.js system errors
|
|
1600
1636
|
/node_modules\//,
|
|
1601
|
-
// Node.js module paths
|
|
1602
|
-
/\/usr\/local\/|\/home\//,
|
|
1603
|
-
// Unix paths
|
|
1604
|
-
/[A-Z]
|
|
1605
|
-
// Windows paths
|
|
1606
|
-
/
|
|
1607
|
-
// Sensitive terms
|
|
1608
|
-
/jdbc:|mysql:|postgres:|mongodb:/i
|
|
1637
|
+
// Node.js module paths (filesystem layout leak)
|
|
1638
|
+
/\/usr\/local\/|\/home\/|\/root\//,
|
|
1639
|
+
// Unix absolute paths
|
|
1640
|
+
/[A-Z]:\\[\w\s.-]+\\[\w\s.-]+/,
|
|
1641
|
+
// Windows absolute paths (drive + 2+ segments)
|
|
1642
|
+
/jdbc:|mysql:\/\/|postgres(?:ql)?:\/\/|mongodb(?:\+srv)?:\/\//i
|
|
1609
1643
|
// DB connection strings
|
|
1610
1644
|
];
|
|
1611
1645
|
var INTERNAL_IP_PATTERNS = [
|
|
@@ -1624,6 +1658,20 @@ function createIdCounter(start = 0) {
|
|
|
1624
1658
|
let id = start;
|
|
1625
1659
|
return () => ++id;
|
|
1626
1660
|
}
|
|
1661
|
+
function dedupAndCapWarnings(warnings, max) {
|
|
1662
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1663
|
+
const deduped = [];
|
|
1664
|
+
for (const w of warnings) {
|
|
1665
|
+
if (seen.has(w)) continue;
|
|
1666
|
+
seen.add(w);
|
|
1667
|
+
deduped.push(w);
|
|
1668
|
+
}
|
|
1669
|
+
if (deduped.length > max) {
|
|
1670
|
+
const truncated = deduped.length - max;
|
|
1671
|
+
return [...deduped.slice(0, max), `... and ${truncated} more warning(s) suppressed`];
|
|
1672
|
+
}
|
|
1673
|
+
return deduped;
|
|
1674
|
+
}
|
|
1627
1675
|
var STDIO_INCOMPATIBLE_IDS = /* @__PURE__ */ new Set([
|
|
1628
1676
|
// Lifecycle tests that use raw undici for HTTP-specific checks
|
|
1629
1677
|
"lifecycle-string-id",
|
|
@@ -4376,14 +4424,13 @@ async function runComplianceSuite(target, options = {}) {
|
|
|
4376
4424
|
return { passed: true, details: "Unknown method returned JSON-RPC error; subsequent ping succeeded" };
|
|
4377
4425
|
}
|
|
4378
4426
|
);
|
|
4379
|
-
const MAX_WARNINGS = 100;
|
|
4380
|
-
if (warnings.length > MAX_WARNINGS) {
|
|
4381
|
-
const truncated = warnings.length - MAX_WARNINGS;
|
|
4382
|
-
warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
|
|
4383
|
-
}
|
|
4384
4427
|
if (inFlight.size > 0) await drainPool();
|
|
4428
|
+
const MAX_WARNINGS = 50;
|
|
4429
|
+
const capped = dedupAndCapWarnings(warnings, MAX_WARNINGS);
|
|
4430
|
+
warnings.length = 0;
|
|
4431
|
+
warnings.push(...capped);
|
|
4385
4432
|
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
4386
|
-
const badge = generateBadge(displayUrl);
|
|
4433
|
+
const badge = displayUrl.startsWith("stdio:") ? { imageUrl: "", reportUrl: "", markdown: "", html: "" } : generateBadge(displayUrl);
|
|
4387
4434
|
return {
|
|
4388
4435
|
schemaVersion: REPORT_SCHEMA_VERSION,
|
|
4389
4436
|
specVersion: SPEC_VERSION,
|
|
@@ -4611,8 +4658,16 @@ async function startServer() {
|
|
|
4611
4658
|
const transport = new StdioServerTransport();
|
|
4612
4659
|
await server.connect(transport);
|
|
4613
4660
|
}
|
|
4614
|
-
|
|
4615
|
-
|
|
4661
|
+
function isInvokedDirectly() {
|
|
4662
|
+
const argv1 = process.argv[1];
|
|
4663
|
+
if (!argv1) return false;
|
|
4664
|
+
try {
|
|
4665
|
+
return realpathSync(argv1) === realpathSync(fileURLToPath(import.meta.url));
|
|
4666
|
+
} catch {
|
|
4667
|
+
return false;
|
|
4668
|
+
}
|
|
4669
|
+
}
|
|
4670
|
+
if (isInvokedDirectly()) {
|
|
4616
4671
|
startServer().catch((err) => {
|
|
4617
4672
|
console.error("MCP server error:", err);
|
|
4618
4673
|
process.exit(1);
|
|
@@ -4923,7 +4978,7 @@ function formatGithub(report) {
|
|
|
4923
4978
|
}
|
|
4924
4979
|
function formatMarkdown(report) {
|
|
4925
4980
|
const lines = [];
|
|
4926
|
-
const gradeEmoji = { A: "\u{1F7E2}", B: "\u{
|
|
4981
|
+
const gradeEmoji = { A: "\u{1F7E2}", B: "\u{1F535}", C: "\u{1F7E1}", D: "\u{1F7E0}", F: "\u{1F534}" };
|
|
4927
4982
|
lines.push("# MCP Compliance Report");
|
|
4928
4983
|
lines.push("");
|
|
4929
4984
|
lines.push(
|
|
@@ -5253,6 +5308,21 @@ function resolveTarget(cliTarget, cliExtraArgs, cliOpts, config) {
|
|
|
5253
5308
|
if (config?.target) return config.target;
|
|
5254
5309
|
throw new Error("No target specified. Pass a URL or command, or add 'target' to mcp-compliance.config.json.");
|
|
5255
5310
|
}
|
|
5311
|
+
var PRIVATE_TLD_SUFFIXES = [
|
|
5312
|
+
".local",
|
|
5313
|
+
".localhost",
|
|
5314
|
+
".internal",
|
|
5315
|
+
".corp",
|
|
5316
|
+
".home",
|
|
5317
|
+
".home.arpa",
|
|
5318
|
+
".lan",
|
|
5319
|
+
".intranet",
|
|
5320
|
+
".private",
|
|
5321
|
+
".test",
|
|
5322
|
+
".invalid",
|
|
5323
|
+
".example",
|
|
5324
|
+
".onion"
|
|
5325
|
+
];
|
|
5256
5326
|
function isPrivateHost(urlStr) {
|
|
5257
5327
|
let host;
|
|
5258
5328
|
try {
|
|
@@ -5260,7 +5330,10 @@ function isPrivateHost(urlStr) {
|
|
|
5260
5330
|
} catch {
|
|
5261
5331
|
return false;
|
|
5262
5332
|
}
|
|
5263
|
-
if (host === "localhost"
|
|
5333
|
+
if (host === "localhost") return true;
|
|
5334
|
+
for (const suffix of PRIVATE_TLD_SUFFIXES) {
|
|
5335
|
+
if (host === suffix.slice(1) || host.endsWith(suffix)) return true;
|
|
5336
|
+
}
|
|
5264
5337
|
const v4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
5265
5338
|
if (v4) {
|
|
5266
5339
|
const [a, b] = v4.slice(1).map(Number);
|
|
@@ -5380,7 +5453,9 @@ Testing ${describeTarget(transportTarget)}...
|
|
|
5380
5453
|
skip,
|
|
5381
5454
|
onProgress: verbose ? (testId, passed, details) => {
|
|
5382
5455
|
const icon = passed ? chalk2.green("PASS") : chalk2.red("FAIL");
|
|
5383
|
-
|
|
5456
|
+
const stream = opts.format === "terminal" ? process.stdout : process.stderr;
|
|
5457
|
+
stream.write(` ${icon} ${testId} \u2014 ${details}
|
|
5458
|
+
`);
|
|
5384
5459
|
} : void 0
|
|
5385
5460
|
});
|
|
5386
5461
|
if (verbose && opts.format === "terminal") {
|
|
@@ -5414,6 +5489,16 @@ Badge SVG written to ${opts.output}`));
|
|
|
5414
5489
|
console.error(chalk2.red("\nError: --watch only applies to stdio targets (HTTP servers are remote).\n"));
|
|
5415
5490
|
process.exit(1);
|
|
5416
5491
|
}
|
|
5492
|
+
if (opts.format !== "terminal" && opts.format !== "markdown" && opts.format !== "html") {
|
|
5493
|
+
console.error(
|
|
5494
|
+
chalk2.red(
|
|
5495
|
+
`
|
|
5496
|
+
Error: --watch is incompatible with --format=${opts.format} (multi-run output would be unparseable). Use --format=terminal.
|
|
5497
|
+
`
|
|
5498
|
+
)
|
|
5499
|
+
);
|
|
5500
|
+
process.exit(1);
|
|
5501
|
+
}
|
|
5417
5502
|
await runOnce();
|
|
5418
5503
|
let pending = null;
|
|
5419
5504
|
let running = false;
|
|
@@ -5427,8 +5512,9 @@ Badge SVG written to ${opts.output}`));
|
|
|
5427
5512
|
if (running) return;
|
|
5428
5513
|
running = true;
|
|
5429
5514
|
try {
|
|
5430
|
-
|
|
5515
|
+
process.stderr.write(chalk2.dim(`
|
|
5431
5516
|
[watch] ${f} changed \u2014 re-running...
|
|
5517
|
+
|
|
5432
5518
|
`));
|
|
5433
5519
|
await runOnce();
|
|
5434
5520
|
} catch (err) {
|
|
@@ -5440,7 +5526,7 @@ Badge SVG written to ${opts.output}`));
|
|
|
5440
5526
|
});
|
|
5441
5527
|
process.on("SIGINT", () => {
|
|
5442
5528
|
watcher.close();
|
|
5443
|
-
|
|
5529
|
+
process.stderr.write(chalk2.dim("\n[watch] stopped\n"));
|
|
5444
5530
|
process.exit(0);
|
|
5445
5531
|
});
|
|
5446
5532
|
await new Promise(() => {
|
package/dist/mcp/server.js
CHANGED
|
@@ -2,10 +2,10 @@ import {
|
|
|
2
2
|
SPEC_BASE,
|
|
3
3
|
TEST_DEFINITIONS,
|
|
4
4
|
runComplianceSuite
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-X5CVUDPW.js";
|
|
6
6
|
|
|
7
7
|
// src/mcp/server.ts
|
|
8
|
-
import { existsSync, readFileSync } from "fs";
|
|
8
|
+
import { existsSync, readFileSync, realpathSync } from "fs";
|
|
9
9
|
import { dirname, join, resolve } from "path";
|
|
10
10
|
import { fileURLToPath } from "url";
|
|
11
11
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
@@ -211,8 +211,16 @@ async function startServer() {
|
|
|
211
211
|
const transport = new StdioServerTransport();
|
|
212
212
|
await server.connect(transport);
|
|
213
213
|
}
|
|
214
|
-
|
|
215
|
-
|
|
214
|
+
function isInvokedDirectly() {
|
|
215
|
+
const argv1 = process.argv[1];
|
|
216
|
+
if (!argv1) return false;
|
|
217
|
+
try {
|
|
218
|
+
return realpathSync(argv1) === realpathSync(fileURLToPath(import.meta.url));
|
|
219
|
+
} catch {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
if (isInvokedDirectly()) {
|
|
216
224
|
startServer().catch((err) => {
|
|
217
225
|
console.error("MCP server error:", err);
|
|
218
226
|
process.exit(1);
|
package/dist/runner.d.ts
CHANGED
|
@@ -142,6 +142,14 @@ declare function parseSSEResponse(text: string): any;
|
|
|
142
142
|
|
|
143
143
|
declare const SPEC_VERSION = "2025-11-25";
|
|
144
144
|
declare const SPEC_BASE = "https://modelcontextprotocol.io/specification/2025-11-25";
|
|
145
|
+
/**
|
|
146
|
+
* Dedupe and cap a list of warnings, preserving insertion order and
|
|
147
|
+
* appending a truncation sentinel when capped. Extracted so the cap
|
|
148
|
+
* semantics can be unit-tested without spinning up a suite run.
|
|
149
|
+
*
|
|
150
|
+
* @internal Exported for testing.
|
|
151
|
+
*/
|
|
152
|
+
declare function dedupAndCapWarnings(warnings: readonly string[], max: number): string[];
|
|
145
153
|
|
|
146
154
|
interface PreviewOptions {
|
|
147
155
|
/** Transport to filter against. Defaults to "http". */
|
|
@@ -206,4 +214,4 @@ interface RunOptions {
|
|
|
206
214
|
*/
|
|
207
215
|
declare function runComplianceSuite(target: string | TransportTarget, options?: RunOptions): Promise<ComplianceReport>;
|
|
208
216
|
|
|
209
|
-
export { type ComplianceReport, type PreviewOptions, type RunOptions, SPEC_BASE, SPEC_VERSION, TEST_DEFINITIONS, type TestResult, computeGrade, computeScore, generateBadge, parseSSEResponse, previewTests, runComplianceSuite, urlHash };
|
|
217
|
+
export { type ComplianceReport, type PreviewOptions, type RunOptions, SPEC_BASE, SPEC_VERSION, TEST_DEFINITIONS, type TestResult, computeGrade, computeScore, dedupAndCapWarnings, generateBadge, parseSSEResponse, previewTests, runComplianceSuite, urlHash };
|
package/dist/runner.js
CHANGED
|
@@ -4,18 +4,20 @@ import {
|
|
|
4
4
|
TEST_DEFINITIONS,
|
|
5
5
|
computeGrade,
|
|
6
6
|
computeScore,
|
|
7
|
+
dedupAndCapWarnings,
|
|
7
8
|
generateBadge,
|
|
8
9
|
parseSSEResponse,
|
|
9
10
|
previewTests,
|
|
10
11
|
runComplianceSuite,
|
|
11
12
|
urlHash
|
|
12
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-X5CVUDPW.js";
|
|
13
14
|
export {
|
|
14
15
|
SPEC_BASE,
|
|
15
16
|
SPEC_VERSION,
|
|
16
17
|
TEST_DEFINITIONS,
|
|
17
18
|
computeGrade,
|
|
18
19
|
computeScore,
|
|
20
|
+
dedupAndCapWarnings,
|
|
19
21
|
generateBadge,
|
|
20
22
|
parseSSEResponse,
|
|
21
23
|
previewTests,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yawlabs/mcp-compliance",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.2",
|
|
4
4
|
"description": "CLI tool and MCP server that tests MCP servers for spec compliance",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Yaw Labs <contact@yaw.sh> (https://yaw.sh)",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"vitest": "^3.1.1"
|
|
58
58
|
},
|
|
59
59
|
"engines": {
|
|
60
|
-
"node": ">=
|
|
60
|
+
"node": ">=20"
|
|
61
61
|
},
|
|
62
62
|
"keywords": [
|
|
63
63
|
"mcp",
|