meritmcp 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of meritmcp might be problematic. Click here for more details.
- package/LICENSE +21 -0
- package/README.md +149 -0
- package/action.yml +120 -0
- package/config/scoring.config.json +9 -0
- package/dist/src/adapter/mcpClient.js +72 -0
- package/dist/src/adapter/mcpClient.js.map +1 -0
- package/dist/src/cli.js +121 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/conformance/stdio-proxy.js +102 -0
- package/dist/src/conformance/stdio-proxy.js.map +1 -0
- package/dist/src/conformance/wrapper.js +139 -0
- package/dist/src/conformance/wrapper.js.map +1 -0
- package/dist/src/functional/asserts.js +42 -0
- package/dist/src/functional/asserts.js.map +1 -0
- package/dist/src/functional/parsers.js +44 -0
- package/dist/src/functional/parsers.js.map +1 -0
- package/dist/src/functional/runner.js +27 -0
- package/dist/src/functional/runner.js.map +1 -0
- package/dist/src/orchestrator.js +96 -0
- package/dist/src/orchestrator.js.map +1 -0
- package/dist/src/report/badge.js +10 -0
- package/dist/src/report/badge.js.map +1 -0
- package/dist/src/report/console.js +100 -0
- package/dist/src/report/console.js.map +1 -0
- package/dist/src/report/prcomment.js +52 -0
- package/dist/src/report/prcomment.js.map +1 -0
- package/dist/src/report/sarif.js +118 -0
- package/dist/src/report/sarif.js.map +1 -0
- package/dist/src/report/score.js +59 -0
- package/dist/src/report/score.js.map +1 -0
- package/dist/src/security/engine.js +27 -0
- package/dist/src/security/engine.js.map +1 -0
- package/dist/src/security/owasp-map.js +56 -0
- package/dist/src/security/owasp-map.js.map +1 -0
- package/dist/src/security/probe/authz.js +17 -0
- package/dist/src/security/probe/authz.js.map +1 -0
- package/dist/src/security/probe/injection.js +79 -0
- package/dist/src/security/probe/injection.js.map +1 -0
- package/dist/src/security/static/deps-osv.js +129 -0
- package/dist/src/security/static/deps-osv.js.map +1 -0
- package/dist/src/security/static/secrets.js +57 -0
- package/dist/src/security/static/secrets.js.map +1 -0
- package/dist/src/security/static/unicode.js +37 -0
- package/dist/src/security/static/unicode.js.map +1 -0
- package/dist/src/snapshot/canonicalize.js +11 -0
- package/dist/src/snapshot/canonicalize.js.map +1 -0
- package/dist/src/snapshot/capture.js +26 -0
- package/dist/src/snapshot/capture.js.map +1 -0
- package/dist/src/snapshot/diff.js +26 -0
- package/dist/src/snapshot/diff.js.map +1 -0
- package/dist/src/types.js +3 -0
- package/dist/src/types.js.map +1 -0
- package/package.json +70 -0
- package/schemas/tests.schema.json +53 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// Wraps the official MCP conformance suite (@modelcontextprotocol/conformance@0.1.16).
|
|
2
|
+
// We never fork it — we spawn it (via `node <bin>`, no shell, so args are safe) against an
|
|
3
|
+
// HTTP URL, then read the per-scenario checks.json files it writes to -o <dir>.
|
|
4
|
+
//
|
|
5
|
+
// Scoring is capability- and transport-aware: a tools-only server shouldn't be punished for
|
|
6
|
+
// not implementing resources/prompts/logging, and HTTP-transport scenarios (dns-rebinding,
|
|
7
|
+
// sse) reflect our stdio→HTTP proxy, not a stdio target — both are excluded from the score.
|
|
8
|
+
import { spawn } from "node:child_process";
|
|
9
|
+
import { mkdtempSync, readdirSync, readFileSync, existsSync, rmSync } from "node:fs";
|
|
10
|
+
import { tmpdir } from "node:os";
|
|
11
|
+
import { join, dirname } from "node:path";
|
|
12
|
+
import { createRequire } from "node:module";
|
|
13
|
+
const require = createRequire(import.meta.url);
|
|
14
|
+
export async function runConformance(url, opts) {
|
|
15
|
+
let binJs;
|
|
16
|
+
try {
|
|
17
|
+
binJs = join(dirname(require.resolve("@modelcontextprotocol/conformance/package.json")), "dist", "index.js");
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return notRun("conformance package not installed");
|
|
21
|
+
}
|
|
22
|
+
const outDir = mkdtempSync(join(tmpdir(), "merit-conf-"));
|
|
23
|
+
const args = [binJs, "server", "--url", url, "--suite", opts.suite ?? "active", "-o", outDir, "--verbose"];
|
|
24
|
+
if (opts.baseline && existsSync(opts.baseline))
|
|
25
|
+
args.push("--expected-failures", opts.baseline);
|
|
26
|
+
try {
|
|
27
|
+
await exec(process.execPath, args, opts.timeoutMs ?? 180000);
|
|
28
|
+
}
|
|
29
|
+
catch (e) {
|
|
30
|
+
cleanup(outDir);
|
|
31
|
+
return notRun(`could not run conformance: ${e.message}`);
|
|
32
|
+
}
|
|
33
|
+
const scenarios = parseScenarios(outDir, opts);
|
|
34
|
+
cleanup(outDir);
|
|
35
|
+
if (scenarios.length === 0)
|
|
36
|
+
return notRun("conformance produced no results (is the server reachable?)");
|
|
37
|
+
const applicable = scenarios.filter((s) => s.applicable);
|
|
38
|
+
const passed = applicable.filter((s) => s.passed).length;
|
|
39
|
+
const total = applicable.length;
|
|
40
|
+
return {
|
|
41
|
+
ran: true,
|
|
42
|
+
score: total === 0 ? 100 : Math.round((100 * passed) / total),
|
|
43
|
+
passed,
|
|
44
|
+
total,
|
|
45
|
+
rawPassed: scenarios.filter((s) => s.passed).length,
|
|
46
|
+
rawTotal: scenarios.length,
|
|
47
|
+
scenarios,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function parseScenarios(outDir, opts) {
|
|
51
|
+
const out = [];
|
|
52
|
+
for (const entry of readdirSync(outDir, { withFileTypes: true })) {
|
|
53
|
+
if (!entry.isDirectory())
|
|
54
|
+
continue;
|
|
55
|
+
const scenario = entry.name.replace(/^server-/, "").replace(/-\d{4}-\d{2}-\d{2}T.*Z$/, "");
|
|
56
|
+
const checksPath = join(outDir, entry.name, "checks.json");
|
|
57
|
+
if (!existsSync(checksPath))
|
|
58
|
+
continue;
|
|
59
|
+
let checks;
|
|
60
|
+
try {
|
|
61
|
+
checks = JSON.parse(readFileSync(checksPath, "utf8"));
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const passed = checks.length > 0 && checks.every((c) => c.status === "SUCCESS");
|
|
67
|
+
const error = checks.find((c) => c.status !== "SUCCESS")?.errorMessage;
|
|
68
|
+
out.push({ scenario, passed, error, applicable: isApplicable(scenario, opts) });
|
|
69
|
+
}
|
|
70
|
+
return out.sort((a, b) => a.scenario.localeCompare(b.scenario));
|
|
71
|
+
}
|
|
72
|
+
// Allowlist of conformance scenarios that are *universally* checkable against an arbitrary
|
|
73
|
+
// server for a declared capability — pinned to conformance 0.1.16. We deliberately EXCLUDE:
|
|
74
|
+
// • named-fixture scenarios (prompts-get-*, resources-read-*, completion-complete) — these
|
|
75
|
+
// dereference items only the SDK's internal reference server defines (test://static-text,
|
|
76
|
+
// test_simple_prompt…), so every real server "fails" them for the wrong reason;
|
|
77
|
+
// • content/feature-specific tool scenarios (image/audio/progress/sampling/elicitation);
|
|
78
|
+
// • HTTP-transport scenarios (dns-rebinding, sse) when proxying a stdio target — they test
|
|
79
|
+
// our proxy, not the server.
|
|
80
|
+
// What remains is the honest core: "does your server correctly implement the protocol surface
|
|
81
|
+
// it declares?" (initialize, ping, the *-list scenarios, a basic tool call + error, log level).
|
|
82
|
+
function isApplicable(scenario, opts) {
|
|
83
|
+
const caps = opts.capabilities ?? {};
|
|
84
|
+
const has = (k) => caps[k] !== undefined && caps[k] !== null;
|
|
85
|
+
switch (scenario) {
|
|
86
|
+
case "server-initialize":
|
|
87
|
+
case "ping":
|
|
88
|
+
return true;
|
|
89
|
+
case "tools-list":
|
|
90
|
+
case "tools-call-simple-text":
|
|
91
|
+
case "tools-call-error":
|
|
92
|
+
return has("tools");
|
|
93
|
+
case "resources-list":
|
|
94
|
+
return has("resources");
|
|
95
|
+
case "prompts-list":
|
|
96
|
+
return has("prompts");
|
|
97
|
+
case "logging-set-level":
|
|
98
|
+
return has("logging");
|
|
99
|
+
case "dns-rebinding-protection":
|
|
100
|
+
return opts.transport === "http";
|
|
101
|
+
default:
|
|
102
|
+
if (scenario.startsWith("server-sse"))
|
|
103
|
+
return opts.transport === "http";
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function exec(cmd, args, timeoutMs) {
|
|
108
|
+
return new Promise((resolve, reject) => {
|
|
109
|
+
const child = spawn(cmd, args, { shell: false });
|
|
110
|
+
const timer = setTimeout(() => {
|
|
111
|
+
child.kill();
|
|
112
|
+
reject(new Error("conformance timed out"));
|
|
113
|
+
}, timeoutMs);
|
|
114
|
+
// Drain pipes so the child never blocks on a full buffer.
|
|
115
|
+
child.stdout?.on("data", () => { });
|
|
116
|
+
child.stderr?.on("data", () => { });
|
|
117
|
+
child.on("error", (e) => {
|
|
118
|
+
clearTimeout(timer);
|
|
119
|
+
reject(e);
|
|
120
|
+
});
|
|
121
|
+
// Conformance exits non-zero when scenarios fail — that's expected, not an error.
|
|
122
|
+
child.on("close", () => {
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
resolve();
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
function cleanup(dir) {
|
|
129
|
+
try {
|
|
130
|
+
rmSync(dir, { recursive: true, force: true });
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
/* best effort */
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function notRun(note) {
|
|
137
|
+
return { ran: false, score: null, passed: 0, total: 0, rawPassed: 0, rawTotal: 0, scenarios: [], note };
|
|
138
|
+
}
|
|
139
|
+
//# sourceMappingURL=wrapper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wrapper.js","sourceRoot":"","sources":["../../../src/conformance/wrapper.ts"],"names":[],"mappings":"AAAA,uFAAuF;AACvF,2FAA2F;AAC3F,gFAAgF;AAChF,EAAE;AACF,4FAA4F;AAC5F,2FAA2F;AAC3F,4FAA4F;AAC5F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAU/C,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAwB;IACxE,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAC/G,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,mCAAmC,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAC3G,IAAI,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEhG,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,CAAC;QAChB,OAAO,MAAM,CAAC,8BAA+B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC/C,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,4DAA4D,CAAC,CAAC;IAExG,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;IAChC,OAAO;QACL,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,KAAK,CAAC;QAC7D,MAAM;QACN,KAAK;QACL,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QACnD,QAAQ,EAAE,SAAS,CAAC,MAAM;QAC1B,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,MAAc,EAAE,IAAwB;IAC9D,MAAM,GAAG,GAA0B,EAAE,CAAC;IACtC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC,CAAC;QAC3F,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC3D,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;YAAE,SAAS;QACtC,IAAI,MAAyD,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE,YAAY,CAAC;QACvE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,2FAA2F;AAC3F,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,oFAAoF;AACpF,2FAA2F;AAC3F,6FAA6F;AAC7F,iCAAiC;AACjC,8FAA8F;AAC9F,gGAAgG;AAChG,SAAS,YAAY,CAAC,QAAgB,EAAE,IAAwB;IAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;IACrC,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;IAErE,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,mBAAmB,CAAC;QACzB,KAAK,MAAM;YACT,OAAO,IAAI,CAAC;QACd,KAAK,YAAY,CAAC;QAClB,KAAK,wBAAwB,CAAC;QAC9B,KAAK,kBAAkB;YACrB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;QACtB,KAAK,gBAAgB;YACnB,OAAO,GAAG,CAAC,WAAW,CAAC,CAAC;QAC1B,KAAK,cAAc;YACjB,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC;QACxB,KAAK,mBAAmB;YACtB,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC;QACxB,KAAK,0BAA0B;YAC7B,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;QACnC;YACE,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC;gBAAE,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;YACxE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,GAAW,EAAE,IAAc,EAAE,SAAiB;IAC1D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;QAC7C,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,0DAA0D;QAC1D,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;YACtB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,kFAAkF;QAClF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC1B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,iBAAiB;IACnB,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,IAAY;IAC1B,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AAC1G,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Assertion primitives for the functional runner. Ajv powers the structured-output
|
|
2
|
+
// schema checks. Everything returns a list of human-readable failures (empty = pass).
|
|
3
|
+
import { Ajv } from "ajv";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
5
|
+
// ajv-formats is CJS with an ESM-style .d.ts, which trips NodeNext's default-import
|
|
6
|
+
// typing. createRequire returns the real callable and types cleanly.
|
|
7
|
+
const require = createRequire(import.meta.url);
|
|
8
|
+
const addFormats = require("ajv-formats");
|
|
9
|
+
const ajv = new Ajv({ allErrors: true, strict: false });
|
|
10
|
+
addFormats(ajv);
|
|
11
|
+
export function assertCall(spec, ctx) {
|
|
12
|
+
const fails = [];
|
|
13
|
+
if (spec.not_error && ctx.isError)
|
|
14
|
+
fails.push("expected success but the tool returned isError=true");
|
|
15
|
+
if (spec.is_error && !ctx.isError)
|
|
16
|
+
fails.push("expected a tool error but the call succeeded");
|
|
17
|
+
if (spec.content_contains !== undefined && !ctx.text.includes(spec.content_contains))
|
|
18
|
+
fails.push(`output missing "${spec.content_contains}" (got "${trunc(ctx.text)}")`);
|
|
19
|
+
if (spec.content_matches !== undefined && !new RegExp(spec.content_matches).test(ctx.text))
|
|
20
|
+
fails.push(`output did not match /${spec.content_matches}/`);
|
|
21
|
+
if (spec.error_contains !== undefined && !ctx.text.includes(spec.error_contains))
|
|
22
|
+
fails.push(`error text missing "${spec.error_contains}"`);
|
|
23
|
+
if (spec.structured?.json_schema) {
|
|
24
|
+
const validate = ajv.compile(spec.structured.json_schema);
|
|
25
|
+
if (!validate(ctx.structured ?? null))
|
|
26
|
+
fails.push(`structured output failed schema: ${ajv.errorsText(validate.errors)}`);
|
|
27
|
+
}
|
|
28
|
+
return fails;
|
|
29
|
+
}
|
|
30
|
+
export function assertCatalog(spec, ctx) {
|
|
31
|
+
const fails = [];
|
|
32
|
+
if (spec.tool_count !== undefined && ctx.toolNames.length !== spec.tool_count)
|
|
33
|
+
fails.push(`expected ${spec.tool_count} tools, found ${ctx.toolNames.length}`);
|
|
34
|
+
for (const t of spec.contains_tools ?? [])
|
|
35
|
+
if (!ctx.toolNames.includes(t))
|
|
36
|
+
fails.push(`missing expected tool "${t}"`);
|
|
37
|
+
return fails;
|
|
38
|
+
}
|
|
39
|
+
function trunc(s) {
|
|
40
|
+
return s.length > 80 ? `${s.slice(0, 77)}...` : s;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=asserts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"asserts.js","sourceRoot":"","sources":["../../../src/functional/asserts.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,sFAAsF;AACtF,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,oFAAoF;AACpF,qEAAqE;AACrE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAuC,CAAC;AAEhF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AACxD,UAAU,CAAC,GAAG,CAAC,CAAC;AAuBhB,MAAM,UAAU,UAAU,CAAC,IAAgB,EAAE,GAAgB;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IACrG,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC9F,IAAI,IAAI,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC,mBAAmB,IAAI,CAAC,gBAAgB,WAAW,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrF,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;QACxF,KAAK,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;IAC/D,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC5D,IAAI,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,oCAAoC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAgB,EAAE,GAAmB;IACjE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU;QAC3E,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,UAAU,iBAAiB,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE;QACvC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAC7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Parse + validate the YAML functional spec with zod, so a malformed test file
|
|
2
|
+
// fails loudly with a clear message instead of throwing deep in the runner.
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { parse as parseYaml } from "yaml";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
const AssertSchema = z.object({
|
|
7
|
+
not_error: z.boolean().optional(),
|
|
8
|
+
is_error: z.boolean().optional(),
|
|
9
|
+
content_contains: z.string().optional(),
|
|
10
|
+
content_matches: z.string().optional(),
|
|
11
|
+
error_contains: z.string().optional(),
|
|
12
|
+
tool_count: z.number().optional(),
|
|
13
|
+
contains_tools: z.array(z.string()).optional(),
|
|
14
|
+
structured: z.object({ json_schema: z.record(z.any()).optional() }).optional(),
|
|
15
|
+
});
|
|
16
|
+
const TestSchema = z.object({
|
|
17
|
+
name: z.string(),
|
|
18
|
+
op: z.enum(["list_tools", "call_tool"]),
|
|
19
|
+
tool: z.string().optional(),
|
|
20
|
+
arguments: z.record(z.any()).optional(),
|
|
21
|
+
assert: AssertSchema,
|
|
22
|
+
});
|
|
23
|
+
const TargetSchema = z
|
|
24
|
+
.object({
|
|
25
|
+
transport: z.enum(["stdio", "http"]).optional(),
|
|
26
|
+
command: z.string().optional(),
|
|
27
|
+
url: z.string().optional(),
|
|
28
|
+
env: z.record(z.string()).optional(),
|
|
29
|
+
})
|
|
30
|
+
.optional();
|
|
31
|
+
const TestFileSchema = z.object({
|
|
32
|
+
version: z.literal(1).optional(),
|
|
33
|
+
target: TargetSchema,
|
|
34
|
+
tests: z.array(TestSchema),
|
|
35
|
+
});
|
|
36
|
+
export function loadTests(path) {
|
|
37
|
+
const raw = parseYaml(readFileSync(path, "utf8"));
|
|
38
|
+
const parsed = TestFileSchema.safeParse(raw);
|
|
39
|
+
if (!parsed.success) {
|
|
40
|
+
throw new Error(`invalid test file ${path}: ${parsed.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}`);
|
|
41
|
+
}
|
|
42
|
+
return parsed.data;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=parsers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../../src/functional/parsers.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9C,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC/E,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,MAAM,EAAE,YAAY;CACrB,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CAAC;KACnB,MAAM,CAAC;IACN,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrC,CAAC;KACD,QAAQ,EAAE,CAAC;AAEd,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;CAC3B,CAAC,CAAC;AAKH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,GAAG,GAAG,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjI,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { loadTests } from "./parsers.js";
|
|
2
|
+
import { assertCall, assertCatalog } from "./asserts.js";
|
|
3
|
+
export async function runFunctional(client, testsPath) {
|
|
4
|
+
const file = loadTests(testsPath);
|
|
5
|
+
const results = [];
|
|
6
|
+
const toolNames = (await client.listTools()).map((t) => t.name);
|
|
7
|
+
for (const tc of file.tests) {
|
|
8
|
+
try {
|
|
9
|
+
if (tc.op === "list_tools") {
|
|
10
|
+
const fails = assertCatalog(tc.assert, { toolNames });
|
|
11
|
+
results.push({ name: tc.name, passed: fails.length === 0, detail: fails.join("; ") || "ok" });
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
if (!tc.tool)
|
|
15
|
+
throw new Error("call_tool requires a 'tool' field");
|
|
16
|
+
const res = await client.callTool(tc.tool, tc.arguments ?? {});
|
|
17
|
+
const fails = assertCall(tc.assert, { text: res.text, isError: res.isError, structured: res.structured });
|
|
18
|
+
results.push({ name: tc.name, passed: fails.length === 0, detail: fails.join("; ") || "ok" });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
catch (e) {
|
|
22
|
+
results.push({ name: tc.name, passed: false, detail: `runtime error: ${e.message}` });
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return results;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/functional/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGzD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAiB,EAAE,SAAiB;IACtE,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IAClC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAEhE,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,EAAE,KAAK,YAAY,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAChG,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,EAAE,CAAC,IAAI;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBACnE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAC/D,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC1G,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAChG,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAmB,CAAW,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACnG,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// Wires the engines together into one Report. MVP runs functional + schema-drift;
|
|
2
|
+
// conformance + security are declared "not run" and the scorer redistributes weight.
|
|
3
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
4
|
+
import { McpClient } from "./adapter/mcpClient.js";
|
|
5
|
+
import { runFunctional } from "./functional/runner.js";
|
|
6
|
+
import { capture } from "./snapshot/capture.js";
|
|
7
|
+
import { diff } from "./snapshot/diff.js";
|
|
8
|
+
import { assemble, DEFAULT_SCORING } from "./report/score.js";
|
|
9
|
+
import { runSecurity } from "./security/engine.js";
|
|
10
|
+
import { runConformance } from "./conformance/wrapper.js";
|
|
11
|
+
import { startStdioHttpProxy } from "./conformance/stdio-proxy.js";
|
|
12
|
+
const NOT_RUN_CONFORMANCE = {
|
|
13
|
+
ran: false,
|
|
14
|
+
score: null,
|
|
15
|
+
passed: 0,
|
|
16
|
+
total: 0,
|
|
17
|
+
rawPassed: 0,
|
|
18
|
+
rawTotal: 0,
|
|
19
|
+
scenarios: [],
|
|
20
|
+
};
|
|
21
|
+
export async function run(cfg) {
|
|
22
|
+
const client = new McpClient();
|
|
23
|
+
await client.connect(cfg.connect);
|
|
24
|
+
try {
|
|
25
|
+
const tools = await client.listTools();
|
|
26
|
+
// --- Functional engine ---
|
|
27
|
+
let results = [];
|
|
28
|
+
let functionalRan = false;
|
|
29
|
+
if (cfg.testsPath && existsSync(cfg.testsPath)) {
|
|
30
|
+
results = await runFunctional(client, cfg.testsPath);
|
|
31
|
+
functionalRan = true;
|
|
32
|
+
}
|
|
33
|
+
// --- Schema-drift (part of the functional engine) ---
|
|
34
|
+
let drift;
|
|
35
|
+
if (cfg.snapshotPath && existsSync(cfg.snapshotPath)) {
|
|
36
|
+
const prev = JSON.parse(readFileSync(cfg.snapshotPath, "utf8"));
|
|
37
|
+
drift = diff(prev, capture(tools));
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
drift = { hasSnapshot: false, changed: false, added: [], removed: [], schemaChanged: [], descriptionOnly: [] };
|
|
41
|
+
}
|
|
42
|
+
const passed = results.filter((r) => r.passed).length;
|
|
43
|
+
const total = results.length;
|
|
44
|
+
const functional = {
|
|
45
|
+
ran: functionalRan,
|
|
46
|
+
score: functionalRan ? (total === 0 ? 100 : Math.round((100 * passed) / total)) : null,
|
|
47
|
+
passed,
|
|
48
|
+
total,
|
|
49
|
+
results,
|
|
50
|
+
drift,
|
|
51
|
+
};
|
|
52
|
+
// --- Security engine (OWASP-MCP-Top-10, 5 high-confidence MVP checks) ---
|
|
53
|
+
let security = { ran: false, score: null, findings: [] };
|
|
54
|
+
if (cfg.security ?? true) {
|
|
55
|
+
const sec = await runSecurity({
|
|
56
|
+
client,
|
|
57
|
+
tools,
|
|
58
|
+
transport: cfg.connect.transport,
|
|
59
|
+
url: cfg.connect.url,
|
|
60
|
+
srcDir: cfg.srcDir,
|
|
61
|
+
probe: cfg.probe ?? false,
|
|
62
|
+
});
|
|
63
|
+
security = { ran: true, score: sec.score, findings: sec.findings };
|
|
64
|
+
}
|
|
65
|
+
// --- Conformance (wrap the official suite; proxy stdio→HTTP since it's HTTP-only) ---
|
|
66
|
+
const capabilities = client.serverCapabilities();
|
|
67
|
+
let conformance = NOT_RUN_CONFORMANCE;
|
|
68
|
+
if (cfg.conformance ?? true) {
|
|
69
|
+
if (cfg.connect.transport === "http") {
|
|
70
|
+
conformance = await runConformance(cfg.connect.url, {
|
|
71
|
+
transport: "http",
|
|
72
|
+
capabilities,
|
|
73
|
+
baseline: cfg.conformanceBaseline,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
else if (cfg.connect.command) {
|
|
77
|
+
const proxy = await startStdioHttpProxy(cfg.connect.command);
|
|
78
|
+
try {
|
|
79
|
+
conformance = await runConformance(proxy.url, {
|
|
80
|
+
transport: "stdio",
|
|
81
|
+
capabilities,
|
|
82
|
+
baseline: cfg.conformanceBaseline,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await proxy.close();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return assemble({ functional, conformance, security }, DEFAULT_SCORING);
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
await client.close();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=orchestrator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/orchestrator.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,qFAAqF;AACrF,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,SAAS,EAAkB,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAY,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAcnE,MAAM,mBAAmB,GAAuB;IAC9C,GAAG,EAAE,KAAK;IACV,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,EAAE;CACd,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,GAAc;IACtC,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;IAC/B,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,OAAO,GAAiB,EAAE,CAAC;QAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,IAAI,GAAG,CAAC,SAAS,IAAI,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/C,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;YACrD,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;QAED,uDAAuD;QACvD,IAAI,KAAmB,CAAC;QACxB,IAAI,GAAG,CAAC,YAAY,IAAI,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,CAAa,CAAC;YAC5E,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;QACjH,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,MAAM,UAAU,GAAsB;YACpC,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YACtF,MAAM;YACN,KAAK;YACL,OAAO;YACP,KAAK;SACN,CAAC;QAEF,2EAA2E;QAC3E,IAAI,QAAQ,GAAoB,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAC1E,IAAI,GAAG,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC;gBAC5B,MAAM;gBACN,KAAK;gBACL,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS;gBAChC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,KAAK;aAC1B,CAAC,CAAC;YACH,QAAQ,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;QACrE,CAAC;QAED,uFAAuF;QACvF,MAAM,YAAY,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC;QACjD,IAAI,WAAW,GAAuB,mBAAmB,CAAC;QAC1D,IAAI,GAAG,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;YAC5B,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;gBACrC,WAAW,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,GAAI,EAAE;oBACnD,SAAS,EAAE,MAAM;oBACjB,YAAY;oBACZ,QAAQ,EAAE,GAAG,CAAC,mBAAmB;iBAClC,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7D,IAAI,CAAC;oBACH,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,GAAG,EAAE;wBAC5C,SAAS,EAAE,OAAO;wBAClB,YAAY;wBACZ,QAAQ,EAAE,GAAG,CAAC,mBAAmB;qBAClC,CAAC,CAAC;gBACL,CAAC;wBAAS,CAAC;oBACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,eAAe,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { bandColor } from "./score.js";
|
|
2
|
+
export function toBadge(report) {
|
|
3
|
+
return {
|
|
4
|
+
schemaVersion: 1,
|
|
5
|
+
label: "Merit",
|
|
6
|
+
message: `${report.score}/100 · ${report.verdict === "PASS" ? "passing" : "failing"}`,
|
|
7
|
+
color: bandColor(report.score, report.verdict === "FAIL"),
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=badge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"badge.js","sourceRoot":"","sources":["../../../src/report/badge.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AASvC,MAAM,UAAU,OAAO,CAAC,MAAc;IACpC,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,KAAK,EAAE,OAAO;QACd,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE;QACrF,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC;KAC1D,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
const GREEN = "\x1b[32m";
|
|
2
|
+
const RED = "\x1b[31m";
|
|
3
|
+
const YELLOW = "\x1b[33m";
|
|
4
|
+
const DIM = "\x1b[2m";
|
|
5
|
+
const BOLD = "\x1b[1m";
|
|
6
|
+
const RESET = "\x1b[0m";
|
|
7
|
+
export function printReport(r) {
|
|
8
|
+
const verdict = r.verdict === "PASS" ? `${GREEN}${BOLD}PASS${RESET}` : `${RED}${BOLD}FAIL${RESET}`;
|
|
9
|
+
const capNote = r.capped ? ` ${RED}(capped — high-severity security finding)${RESET}` : "";
|
|
10
|
+
console.log(`\nMerit — ${verdict} · Safety score ${scoreColor(r)}${r.score}/100${RESET}${capNote}\n`);
|
|
11
|
+
if (r.functional.ran) {
|
|
12
|
+
console.log(`Functional: ${r.functional.passed}/${r.functional.total} passed`);
|
|
13
|
+
for (const t of r.functional.results) {
|
|
14
|
+
const mark = t.passed ? `${GREEN}✔${RESET}` : `${RED}✖${RESET}`;
|
|
15
|
+
console.log(` ${mark} ${t.name}${t.passed ? "" : `\n ${DIM}${t.detail}${RESET}`}`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
console.log(`${DIM}Functional: skipped (no test file found)${RESET}`);
|
|
20
|
+
}
|
|
21
|
+
const d = r.functional.drift;
|
|
22
|
+
if (d?.hasSnapshot) {
|
|
23
|
+
if (!d.changed) {
|
|
24
|
+
console.log(`${DIM}Schema: no drift${RESET}`);
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
const parts = [];
|
|
28
|
+
if (d.removed.length)
|
|
29
|
+
parts.push(`${RED}removed: ${d.removed.join(", ")}${RESET}`);
|
|
30
|
+
if (d.schemaChanged.length)
|
|
31
|
+
parts.push(`${RED}schema-changed: ${d.schemaChanged.join(", ")}${RESET}`);
|
|
32
|
+
if (d.added.length)
|
|
33
|
+
parts.push(`${GREEN}added: ${d.added.join(", ")}${RESET}`);
|
|
34
|
+
if (d.descriptionOnly.length)
|
|
35
|
+
parts.push(`${YELLOW}rug-pull: ${d.descriptionOnly.join(", ")}${RESET}`);
|
|
36
|
+
console.log(`Schema drift — ${parts.join(" · ")}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
console.log(`${DIM}Schema: no snapshot yet (run \`merit snapshot\` to enable drift detection)${RESET}`);
|
|
41
|
+
}
|
|
42
|
+
const conf = r.conformance;
|
|
43
|
+
if (conf.ran) {
|
|
44
|
+
const skipped = conf.rawTotal - conf.total;
|
|
45
|
+
console.log(`Conformance: ${conf.passed}/${conf.total} applicable scenarios passed (score ${conf.score}/100)` +
|
|
46
|
+
`${skipped > 0 ? ` ${DIM}· ${skipped} N/A skipped${RESET}` : ""}`);
|
|
47
|
+
for (const s of conf.scenarios.filter((x) => x.applicable && !x.passed)) {
|
|
48
|
+
console.log(` ${RED}✖${RESET} ${s.scenario}${s.error ? `\n ${DIM}${s.error}${RESET}` : ""}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
console.log(`${DIM}Conformance: skipped${conf.note ? ` (${conf.note})` : ""}${RESET}`);
|
|
53
|
+
}
|
|
54
|
+
if (r.security.ran) {
|
|
55
|
+
const f = r.security.findings;
|
|
56
|
+
const c = countBySeverity(f);
|
|
57
|
+
const summary = f.length === 0 ? `${GREEN}no findings${RESET}` : severitySummary(c);
|
|
58
|
+
console.log(`Security: ${summary} (score ${r.security.score}/100)`);
|
|
59
|
+
for (const finding of sortBySeverity(f).slice(0, 8)) {
|
|
60
|
+
console.log(` ${sevMark(finding.severity)} [${finding.owasp ?? finding.ruleId}] ${finding.message}` +
|
|
61
|
+
`${finding.location ? `\n ${DIM}${finding.location}${RESET}` : ""}` +
|
|
62
|
+
` ${DIM}(${finding.confidence} confidence)${RESET}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
console.log(`${DIM}Security: skipped${RESET}`);
|
|
67
|
+
}
|
|
68
|
+
console.log("");
|
|
69
|
+
}
|
|
70
|
+
const SEV_ORDER = ["critical", "high", "medium", "low"];
|
|
71
|
+
function countBySeverity(findings) {
|
|
72
|
+
const c = { critical: 0, high: 0, medium: 0, low: 0 };
|
|
73
|
+
for (const f of findings)
|
|
74
|
+
c[f.severity]++;
|
|
75
|
+
return c;
|
|
76
|
+
}
|
|
77
|
+
function severitySummary(c) {
|
|
78
|
+
const parts = SEV_ORDER.filter((s) => c[s] > 0).map((s) => `${c[s]} ${s}`);
|
|
79
|
+
return `${RED}${parts.join(" · ")}${RESET}`;
|
|
80
|
+
}
|
|
81
|
+
function sortBySeverity(findings) {
|
|
82
|
+
return [...findings].sort((a, b) => SEV_ORDER.indexOf(a.severity) - SEV_ORDER.indexOf(b.severity));
|
|
83
|
+
}
|
|
84
|
+
function sevMark(sev) {
|
|
85
|
+
if (sev === "critical" || sev === "high")
|
|
86
|
+
return `${RED}✖${RESET}`;
|
|
87
|
+
if (sev === "medium")
|
|
88
|
+
return `${YELLOW}▲${RESET}`;
|
|
89
|
+
return `${DIM}•${RESET}`;
|
|
90
|
+
}
|
|
91
|
+
function scoreColor(r) {
|
|
92
|
+
if (r.verdict === "FAIL" || r.score < 30)
|
|
93
|
+
return RED;
|
|
94
|
+
if (r.score >= 85)
|
|
95
|
+
return GREEN;
|
|
96
|
+
if (r.score >= 50)
|
|
97
|
+
return YELLOW;
|
|
98
|
+
return RED;
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=console.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"console.js","sourceRoot":"","sources":["../../../src/report/console.ts"],"names":[],"mappings":"AAGA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,MAAM,GAAG,UAAU,CAAC;AAC1B,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,MAAM,UAAU,WAAW,CAAC,CAAS;IACnC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,IAAI,OAAO,KAAK,EAAE,CAAC;IACnG,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,4CAA4C,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,mBAAmB,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,OAAO,IAAI,CAAC,CAAC;IAEtG,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,CAAC;QAC/E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,2CAA2C,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;IAC7B,IAAI,CAAC,EAAE,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,mBAAmB,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;YACnF,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,mBAAmB,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;YACtG,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,CAAC,eAAe,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,aAAa,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;YACvG,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,6EAA6E,KAAK,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC;IAC3B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3C,OAAO,CAAC,GAAG,CACT,gBAAgB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,uCAAuC,IAAI,CAAC,KAAK,OAAO;YAC/F,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,OAAO,eAAe,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACpE,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACxE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,uBAAuB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC9B,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,cAAc,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACpF,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,CAAC;QACpE,KAAK,MAAM,OAAO,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACpD,OAAO,CAAC,GAAG,CACT,KAAK,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE;gBACtF,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,OAAO,CAAC,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxE,IAAI,GAAG,IAAI,OAAO,CAAC,UAAU,eAAe,KAAK,EAAE,CACtD,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,SAAS,GAAe,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AAEpE,SAAS,eAAe,CAAC,QAAmB;IAC1C,MAAM,CAAC,GAA6B,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IAChF,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC1C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,eAAe,CAAC,CAA2B;IAClD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3E,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,cAAc,CAAC,QAAmB;IACzC,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrG,CAAC;AAED,SAAS,OAAO,CAAC,GAAa;IAC5B,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;IACnE,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC;IAClD,OAAO,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,IAAI,CAAC,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,KAAK,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC;IACrD,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC;IACjC,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
export const PR_COMMENT_MARKER = "<!-- merit-report -->";
|
|
2
|
+
const SEV_ORDER = ["critical", "high", "medium", "low"];
|
|
3
|
+
export function toMarkdown(report) {
|
|
4
|
+
const verdict = report.verdict === "PASS" ? "✅ **PASS**" : "❌ **FAIL**";
|
|
5
|
+
const cap = report.capped ? " _(capped — high-severity security finding)_" : "";
|
|
6
|
+
const lines = [];
|
|
7
|
+
lines.push(PR_COMMENT_MARKER);
|
|
8
|
+
lines.push(`## Merit — ${verdict} · ${report.score}/100${cap}`);
|
|
9
|
+
lines.push("");
|
|
10
|
+
lines.push("| Check | Result | Weight |");
|
|
11
|
+
lines.push("|---|---|---|");
|
|
12
|
+
lines.push(`| Functional | ${cell(report.functional.ran, `${report.functional.passed}/${report.functional.total} passed`)} | 30% |`);
|
|
13
|
+
lines.push(`| Conformance | ${cell(report.conformance.ran, `${report.conformance.passed}/${report.conformance.total} applicable passed`)} | 30% |`);
|
|
14
|
+
lines.push(`| Security | ${cell(report.security.ran, securityCell(report.security.findings))} | 40% |`);
|
|
15
|
+
const d = report.functional.drift;
|
|
16
|
+
lines.push(`| Schema | ${d?.hasSnapshot ? (d.changed ? "⚠️ drift detected" : "no drift") : "_no snapshot_"} | — |`);
|
|
17
|
+
lines.push("");
|
|
18
|
+
const findings = report.security.findings;
|
|
19
|
+
if (findings.length) {
|
|
20
|
+
lines.push("### Top security findings");
|
|
21
|
+
for (const f of sortBySeverity(findings).slice(0, 8)) {
|
|
22
|
+
lines.push(`- **${f.severity.toUpperCase()}** \`${f.owasp ?? f.ruleId}\` — ${f.message} _(${f.confidence} confidence)_`);
|
|
23
|
+
}
|
|
24
|
+
lines.push("");
|
|
25
|
+
}
|
|
26
|
+
const funcFails = report.functional.results.filter((r) => !r.passed);
|
|
27
|
+
if (funcFails.length) {
|
|
28
|
+
lines.push("### Failing functional tests");
|
|
29
|
+
for (const t of funcFails.slice(0, 8))
|
|
30
|
+
lines.push(`- ❌ ${t.name} — ${t.detail}`);
|
|
31
|
+
lines.push("");
|
|
32
|
+
}
|
|
33
|
+
lines.push("<sub>Full results in the **Security** tab (SARIF). Scoring is open + reproducible — see `merit.json`.</sub>");
|
|
34
|
+
return lines.join("\n");
|
|
35
|
+
}
|
|
36
|
+
function cell(ran, text) {
|
|
37
|
+
return ran ? text : "_skipped_";
|
|
38
|
+
}
|
|
39
|
+
function securityCell(findings) {
|
|
40
|
+
if (findings.length === 0)
|
|
41
|
+
return "✅ no findings";
|
|
42
|
+
const counts = { critical: 0, high: 0, medium: 0, low: 0 };
|
|
43
|
+
for (const f of findings)
|
|
44
|
+
counts[f.severity]++;
|
|
45
|
+
return SEV_ORDER.filter((s) => counts[s] > 0)
|
|
46
|
+
.map((s) => `${counts[s]} ${s}`)
|
|
47
|
+
.join(" · ");
|
|
48
|
+
}
|
|
49
|
+
function sortBySeverity(findings) {
|
|
50
|
+
return [...findings].sort((a, b) => SEV_ORDER.indexOf(a.severity) - SEV_ORDER.indexOf(b.severity));
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=prcomment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prcomment.js","sourceRoot":"","sources":["../../../src/report/prcomment.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAEzD,MAAM,SAAS,GAAe,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AAEpE,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC;IACxE,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,MAAM,MAAM,CAAC,KAAK,OAAO,GAAG,EAAE,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1C,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,UAAU,CAAC,CAAC;IACrI,KAAK,CAAC,IAAI,CACR,mBAAmB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW,CAAC,KAAK,oBAAoB,CAAC,UAAU,CACxI,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC;IACxG,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,eAAe,QAAQ,CAAC,CAAC;IACpH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC1C,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACxC,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,UAAU,eAAe,CAAC,CAAC;QAC3H,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACrE,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACjF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,6GAA6G,CAAC,CAAC;IAC1H,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,IAAI,CAAC,GAAY,EAAE,IAAY;IACtC,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC;AAClC,CAAC;AAED,SAAS,YAAY,CAAC,QAAmB;IACvC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,eAAe,CAAC;IAClD,MAAM,MAAM,GAA6B,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACrF,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC/C,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;SAC/B,IAAI,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,QAAmB;IACzC,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrG,CAAC"}
|