@yawlabs/mcp-compliance 0.9.2 → 0.11.0

package/dist/mcp/server.js

@@ -2,7 +2,7 @@ import {
   SPEC_BASE,
   TEST_DEFINITIONS,
   runComplianceSuite
-} from "../chunk-FKTEFLK5.js";
+} from "../chunk-DGGPE3ZM.js";
 
 // src/mcp/server.ts
 import { existsSync, readFileSync } from "fs";

package/dist/runner.d.ts

@@ -12,6 +12,8 @@ interface TestResult {
 type Grade = "A" | "B" | "C" | "D" | "F";
 type Overall = "pass" | "partial" | "fail";
 interface ComplianceReport {
+    /** Stable report-format version. See REPORT_SCHEMA_VERSION. */
+    schemaVersion: string;
     specVersion: string;
     toolVersion: string;
     url: string;
@@ -96,6 +98,16 @@ declare function computeScore(tests: TestResult[]): {
     }>;
 };
 
+/**
+ * Generate a short, deterministic hash of a URL for badge paths.
+ * SHA-256 truncated to 24 hex chars (96 bits of entropy) — matches the
+ * server-side hash width used by mcp.hosting for `/compliance/ext/<hash>`.
+ *
+ * Exported so mcp.hosting (and other consumers) can compute matching
+ * hashes when looking up reports/badges by URL. The hash is the canonical
+ * key for `/compliance/ext/<hash>` and `/api/compliance/ext/<hash>/badge`.
+ */
+declare function urlHash(url: string): string;
 /**
  * Generate badge URLs and markdown for a compliance report.
  * Badge images are served by mcp.hosting.
@@ -135,8 +147,15 @@ interface PreviewOptions {
  */
 declare function previewTests(opts?: PreviewOptions): TestDefinition[];
 interface RunOptions {
-    /** Optional callback for progress updates */
+    /** Optional callback for progress updates (legacy minimal signature). */
     onProgress?: (testId: string, passed: boolean, details: string) => void;
+    /**
+     * Optional callback fired after each test completes with the full
+     * TestResult (category, required, durationMs, specRef). Prefer this
+     * over onProgress for live dashboards and streaming UIs that need
+     * structured data per test.
+     */
+    onTestComplete?: (result: TestResult) => void;
     /** Extra headers to include on all requests */
     headers?: Record<string, string>;
     /** Request timeout in milliseconds (default: 15000) */
@@ -156,4 +175,4 @@ interface RunOptions {
  */
 declare function runComplianceSuite(target: string | TransportTarget, options?: RunOptions): Promise<ComplianceReport>;
 
-export { type ComplianceReport, type PreviewOptions, type RunOptions, SPEC_BASE, SPEC_VERSION, TEST_DEFINITIONS, type TestResult, computeGrade, computeScore, generateBadge, parseSSEResponse, previewTests, runComplianceSuite };
+export { type ComplianceReport, type PreviewOptions, type RunOptions, SPEC_BASE, SPEC_VERSION, TEST_DEFINITIONS, type TestResult, computeGrade, computeScore, generateBadge, parseSSEResponse, previewTests, runComplianceSuite, urlHash };

package/dist/runner.js

@@ -7,8 +7,9 @@ import {
   generateBadge,
   parseSSEResponse,
   previewTests,
-  runComplianceSuite
-} from "./chunk-FKTEFLK5.js";
+  runComplianceSuite,
+  urlHash
+} from "./chunk-DGGPE3ZM.js";
 export {
   SPEC_BASE,
   SPEC_VERSION,
@@ -18,5 +19,6 @@ export {
   generateBadge,
   parseSSEResponse,
   previewTests,
-  runComplianceSuite
+  runComplianceSuite,
+  urlHash
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yawlabs/mcp-compliance",
-  "version": "0.9.2",
+  "version": "0.11.0",
   "description": "CLI tool and MCP server that tests MCP servers for spec compliance",
   "license": "MIT",
   "author": "Yaw Labs <contact@yaw.sh> (https://yaw.sh)",
@@ -23,6 +23,7 @@
   "files": [
     "dist",
     "!dist/**/*.test.*",
+    "schemas",
     "README.md",
     "LICENSE"
   ],
@@ -47,6 +48,8 @@
   "devDependencies": {
     "@biomejs/biome": "^1.9.4",
     "@types/node": "^25.5.2",
+    "ajv": "^8.18.0",
+    "ajv-formats": "^3.0.1",
     "tsup": "^8.4.0",
     "typescript": "^5.8.3",
     "vitest": "^3.1.1"

package/schemas/report.v1.json

@@ -0,0 +1,165 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://mcp.hosting/schemas/compliance/report.v1.json",
+  "title": "MCP Compliance Report",
+  "description": "Structured output of the mcp-compliance test suite. Stable, versioned contract — increment schemaVersion on breaking changes. Consumed by mcp.hosting and third-party dashboards.",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "specVersion",
+    "toolVersion",
+    "url",
+    "timestamp",
+    "score",
+    "grade",
+    "overall",
+    "summary",
+    "categories",
+    "tests",
+    "warnings",
+    "serverInfo",
+    "toolCount",
+    "toolNames",
+    "resourceCount",
+    "resourceNames",
+    "promptCount",
+    "promptNames",
+    "badge"
+  ],
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "1",
+      "description": "Version of this report schema. Breaking changes bump the major version."
+    },
+    "specVersion": {
+      "type": "string",
+      "description": "MCP spec version tested against, e.g. \"2025-11-25\"."
+    },
+    "toolVersion": {
+      "type": "string",
+      "description": "Version of the mcp-compliance tool that produced this report."
+    },
+    "url": {
+      "type": "string",
+      "description": "HTTP URL or stdio display string for the server under test."
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp of when the report was generated."
+    },
+    "score": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 100,
+      "description": "Weighted compliance score: required tests 70%, optional 30%."
+    },
+    "grade": {
+      "type": "string",
+      "enum": ["A", "B", "C", "D", "F"],
+      "description": "Letter grade derived from score. A>=90, B>=75, C>=60, D>=40, F<40."
+    },
+    "overall": {
+      "type": "string",
+      "enum": ["pass", "partial", "fail"],
+      "description": "High-level verdict: pass = all required tests passed, partial = some failed, fail = many failed."
+    },
+    "summary": {
+      "type": "object",
+      "required": ["total", "passed", "failed", "required", "requiredPassed"],
+      "properties": {
+        "total": { "type": "integer", "minimum": 0 },
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "required": { "type": "integer", "minimum": 0 },
+        "requiredPassed": { "type": "integer", "minimum": 0 }
+      },
+      "additionalProperties": false
+    },
+    "categories": {
+      "type": "object",
+      "description": "Per-category pass/total counts, keyed by TestCategory.",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["passed", "total"],
+        "properties": {
+          "passed": { "type": "integer", "minimum": 0 },
+          "total": { "type": "integer", "minimum": 0 }
+        },
+        "additionalProperties": false
+      }
+    },
+    "tests": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/TestResult" }
+    },
+    "warnings": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Non-fatal diagnostics from the test run. Capped at 100 entries."
+    },
+    "serverInfo": {
+      "type": "object",
+      "required": ["protocolVersion", "name", "version", "capabilities"],
+      "properties": {
+        "protocolVersion": { "type": ["string", "null"] },
+        "name": { "type": ["string", "null"] },
+        "version": { "type": ["string", "null"] },
+        "capabilities": { "type": "object" }
+      },
+      "additionalProperties": false
+    },
+    "toolCount": { "type": "integer", "minimum": 0 },
+    "toolNames": { "type": "array", "items": { "type": "string" } },
+    "resourceCount": { "type": "integer", "minimum": 0 },
+    "resourceNames": { "type": "array", "items": { "type": "string" } },
+    "promptCount": { "type": "integer", "minimum": 0 },
+    "promptNames": { "type": "array", "items": { "type": "string" } },
+    "badge": {
+      "type": "object",
+      "required": ["imageUrl", "reportUrl", "markdown", "html"],
+      "properties": {
+        "imageUrl": { "type": "string", "format": "uri" },
+        "reportUrl": { "type": "string", "format": "uri" },
+        "markdown": { "type": "string" },
+        "html": { "type": "string" }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false,
+  "$defs": {
+    "TestResult": {
+      "type": "object",
+      "required": ["id", "name", "category", "passed", "required", "details", "durationMs"],
+      "properties": {
+        "id": { "type": "string" },
+        "name": { "type": "string" },
+        "category": {
+          "type": "string",
+          "enum": [
+            "transport",
+            "lifecycle",
+            "tools",
+            "resources",
+            "prompts",
+            "errors",
+            "schema",
+            "security"
+          ]
+        },
+        "passed": { "type": "boolean" },
+        "required": { "type": "boolean" },
+        "details": { "type": "string" },
+        "durationMs": { "type": "integer", "minimum": 0 },
+        "specRef": {
+          "type": "string",
+          "format": "uri",
+          "description": "Deep link to the relevant spec section."
+        }
+      },
+      "additionalProperties": false
+    }
+  }
+}