mcp-medic 1.2.2 → 1.2.4

package/dist/quality-score.js

@@ -0,0 +1,242 @@
+import { categoryOf, inferDiagnosticCategory } from './diagnostics.js';
+import { allChecks } from './checks/index.js';
+export const QUALITY_DIMENSIONS = [
+    'protocol',
+    'schema',
+    'usability',
+    'security',
+    'reliability',
+];
+/** Suggested weights from the product spec; documented here since every
+ * point deduction must be explainable in terms of these. Do not change
+ * without a documented reason — see .agent-room/DECISIONS.md. */
+export const QUALITY_DIMENSION_WEIGHTS = {
+    protocol: 0.25,
+    schema: 0.2,
+    usability: 0.2,
+    security: 0.2,
+    reliability: 0.15,
+};
+/**
+ * The score is a measurement, not a certification: it reports what
+ * mcp-medic's passive checks actually found (or, per `coverage`, actually
+ * looked for) — never a formal audit of the server's real security or
+ * behavior. Surfaced in both JSON (`ReportQualityScore.disclaimer`) and the
+ * human report.
+ */
+export const QUALITY_SCORE_DISCLAIMER = "This score reflects issues detectable by mcp-medic's passive inspection (see \"coverage\") — " +
+    'it is not a certification of the server\'s actual security, correctness, or behavior.';
+// Deduction weights: chosen so a handful of real problems meaningfully move
+// the score, while a single noisy checkId can never wipe out a whole
+// dimension on its own (the per-checkId cap) — this directly implements
+// "avoid double-counting" / "sensible caps" from the product spec: e.g. 20
+// tools sharing one description problem cost at most 15 points total, not 60.
+const SEVERITY_POINTS = { error: 8, warning: 3, info: 1 };
+const SEVERITY_CAP_PER_CHECK = { error: 25, warning: 15, info: 5 };
+function categoryToDimension(category) {
+    switch (category) {
+        case 'protocol':
+            return 'protocol';
+        case 'schema':
+            return 'schema';
+        case 'security':
+            return 'security';
+        case 'reliability':
+            return 'reliability';
+        // 'usability', 'quality' (the broad bucket most tool/resource/prompt
+        // quality checks use), and 'configuration' (policy.* checks) all land
+        // under the "Agent usability" dimension from the product spec.
+        case 'usability':
+        case 'quality':
+        case 'configuration':
+        default:
+            return 'usability';
+    }
+}
+function describeDeduction(checkId, severity, count) {
+    const plural = count === 1 ? '' : 's';
+    return `${count} ${severity}${plural} from "${checkId}"`;
+}
+function deductionsFromDiagnostics(dimension, diagnostics) {
+    const grouped = new Map();
+    for (const d of diagnostics) {
+        if (categoryToDimension(categoryOf(d)) !== dimension)
+            continue;
+        const key = `${d.checkId}|${d.severity}`;
+        const entry = grouped.get(key) ?? { checkId: d.checkId, severity: d.severity, count: 0 };
+        entry.count += 1;
+        grouped.set(key, entry);
+    }
+    const deductions = [];
+    for (const { checkId, severity, count } of grouped.values()) {
+        const points = Math.min(count * SEVERITY_POINTS[severity], SEVERITY_CAP_PER_CHECK[severity]);
+        deductions.push({ dimension, checkId, severity, count, points, description: describeDeduction(checkId, severity, count) });
+    }
+    return deductions;
+}
+function dimensionScore(deductions) {
+    const totalPoints = deductions.reduce((sum, d) => sum + d.points, 0);
+    return Math.max(0, Math.min(100, Math.round(100 - totalPoints)));
+}
+/**
+ * Computes a quality score for one connection, purely from `diagnostics`
+ * (filtered to that server) — no connection metadata is consulted
+ * directly. Returns `undefined` for a connection that never connected —
+ * there's nothing meaningful to score (no tools/schema/capabilities were
+ * ever observed); see `computeReportQualityScore` for how the report-level
+ * score surfaces that instead of silently dropping it.
+ */
+export function computeConnectionQualityScore(connection, diagnostics) {
+    if (connection.status !== 'connected')
+        return undefined;
+    const relevant = diagnostics.filter((d) => d.serverName === connection.server.name);
+    const allDeductions = QUALITY_DIMENSIONS.flatMap((dim) => deductionsFromDiagnostics(dim, relevant));
+    const dimensions = Object.fromEntries(QUALITY_DIMENSIONS.map((dim) => [dim, dimensionScore(allDeductions.filter((d) => d.dimension === dim))]));
+    const overall = Math.round(QUALITY_DIMENSIONS.reduce((sum, dim) => sum + dimensions[dim] * QUALITY_DIMENSION_WEIGHTS[dim], 0));
+    return {
+        overall,
+        dimensions,
+        deductions: allDeductions.filter((d) => d.points > 0).sort((a, b) => b.points - a.points),
+    };
+}
+const COVERAGE_WEIGHT = { covered: 1, partial: 0.5, 'not-covered': 0 };
+/**
+ * Coverage answers "was this dimension actually evaluated," independent of
+ * what score it got — a 100 from zero checks run means "nothing was
+ * looked for," not "nothing is wrong." Derived from the *ids* of the
+ * checks that actually ran (`executedChecks`), compared against the
+ * built-in `allChecks` reference set grouped by dimension. Reliability is
+ * special-cased to always 'covered': its signal (did the connection
+ * succeed) isn't produced by any optional `Check` — it's inherent to
+ * attempting a connection at all, so it's never "not covered."
+ */
+export function computeQualityCoverage(executedChecks) {
+    const referenceByDimension = new Map();
+    for (const check of allChecks) {
+        const dim = categoryToDimension(inferDiagnosticCategory(check.id));
+        if (dim === 'reliability')
+            continue; // no built-in check drives reliability; handled unconditionally below
+        if (!referenceByDimension.has(dim))
+            referenceByDimension.set(dim, new Set());
+        referenceByDimension.get(dim).add(check.id);
+    }
+    const coverage = {};
+    for (const dim of QUALITY_DIMENSIONS) {
+        if (dim === 'reliability') {
+            coverage[dim] = 'covered';
+            continue;
+        }
+        const reference = referenceByDimension.get(dim) ?? new Set();
+        // Every executed check that maps to this dimension — including custom/
+        // community checks not in the built-in reference set, so an unmapped
+        // check still counts as (at least partial) coverage for the dimension
+        // it was inferred into, rather than being invisible to this count.
+        const executedForDim = executedChecks.filter((c) => categoryToDimension(inferDiagnosticCategory(c.id)) === dim);
+        if (executedForDim.length === 0) {
+            coverage[dim] = 'not-covered';
+            continue;
+        }
+        if (reference.size === 0) {
+            // No built-in check exists for this dimension at all, but something
+            // (necessarily a custom check) ran for it — can't call that "full"
+            // coverage without a reference to compare against.
+            coverage[dim] = 'partial';
+            continue;
+        }
+        const executedKnownIds = new Set(executedForDim.map((c) => c.id).filter((id) => reference.has(id)));
+        coverage[dim] = executedKnownIds.size === reference.size ? 'covered' : 'partial';
+    }
+    return coverage;
+}
+function coveragePercent(coverage) {
+    const total = QUALITY_DIMENSIONS.reduce((sum, dim) => sum + COVERAGE_WEIGHT[coverage[dim]], 0);
+    return Math.round((total / QUALITY_DIMENSIONS.length) * 100);
+}
+/**
+ * Aggregate score across every connected server in a report (simple mean —
+ * deliberately not weighted by tool count, so one huge server can't drown
+ * out the rest of a fleet). `executedChecks` should be the exact list
+ * passed to `runChecks({ checks })` for this report, so `coverage` reflects
+ * what actually ran (defaults to the full built-in set if omitted, for
+ * callers that haven't been updated to pass it through).
+ * Returns `undefined` if no server connected — nothing to score at all.
+ */
+export function computeReportQualityScore(report, executedChecks = allChecks) {
+    const perServer = {};
+    for (const connection of report.connections) {
+        const breakdown = computeConnectionQualityScore(connection, report.diagnostics);
+        if (breakdown)
+            perServer[connection.server.name] = breakdown;
+    }
+    const scoredServers = Object.keys(perServer);
+    const unscoredServers = report.connections
+        .map((c) => c.server.name)
+        .filter((name) => !perServer[name]);
+    if (scoredServers.length === 0)
+        return undefined;
+    const dimensions = Object.fromEntries(QUALITY_DIMENSIONS.map((dim) => [
+        dim,
+        Math.round(scoredServers.reduce((sum, n) => sum + perServer[n].dimensions[dim], 0) / scoredServers.length),
+    ]));
+    const overall = Math.round(scoredServers.reduce((sum, n) => sum + perServer[n].overall, 0) / scoredServers.length);
+    const deductions = scoredServers
+        .flatMap((n) => perServer[n].deductions)
+        .sort((a, b) => b.points - a.points);
+    const coverage = computeQualityCoverage(executedChecks);
+    return {
+        overall,
+        dimensions,
+        deductions,
+        perServer,
+        coverage,
+        coveragePercent: coveragePercent(coverage),
+        scoredServers,
+        unscoredServers,
+        disclaimer: QUALITY_SCORE_DISCLAIMER,
+    };
+}
+/**
+ * Enforces `.mcp-medic-policy.json`'s `quality.minimumScore` as a CI gate.
+ * Pure function so it's directly unit-testable without needing a real CLI
+ * invocation or a restricted check set — see PHASE 12 ("policy/CI") in
+ * .agent-room/DECISIONS.md.
+ *
+ * Returns diagnostics to append to the report (0, 1, or 2):
+ *  - an 'error' if the score is below `minimumScore` (fails the run).
+ *  - a 'warning' if the score was computed from a partial assessment
+ *    (incomplete coverage, or a server that couldn't be scored) — a
+ *    minimumScore gate must never silently "pass" as if a full check had
+ *    run when it didn't.
+ */
+export function checkMinimumScorePolicy(quality, minimumScore, serverNamesLabel) {
+    const results = [];
+    if (quality.overall < minimumScore) {
+        results.push({
+            checkId: 'policy.minimum-quality-score',
+            severity: 'error',
+            message: `MCP quality score ${quality.overall} is below the policy minimum of ${minimumScore}.`,
+            serverName: serverNamesLabel,
+            category: 'configuration',
+        });
+    }
+    if (quality.coveragePercent < 100 || quality.unscoredServers.length > 0) {
+        const gaps = Object.entries(quality.coverage)
+            .filter(([, status]) => status !== 'covered')
+            .map(([dim, status]) => `${dim}: ${status}`)
+            .join(', ');
+        const parts = [
+            gaps ? `coverage is ${quality.coveragePercent}% (${gaps})` : undefined,
+            quality.unscoredServers.length > 0
+                ? `${quality.unscoredServers.length} server(s) could not be scored: ${quality.unscoredServers.join(', ')}`
+                : undefined,
+        ].filter(Boolean);
+        results.push({
+            checkId: 'policy.partial-coverage-with-minimum-score',
+            severity: 'warning',
+            message: `quality.minimumScore was evaluated against a partial assessment (${parts.join('; ')}) — it may not reflect a full check of this server.`,
+            serverName: serverNamesLabel,
+            category: 'configuration',
+        });
+    }
+    return results;
+}

package/dist/report.d.ts

@@ -1,6 +1,9 @@
 import type { RunReport } from './types.js';
 export interface FormatReportOptions {
     showFixes?: boolean;
+    /** Include the MCP quality score section (Phase 7). Off by default so a
+     * plain `check` report stays short; `--score`/`score` turn it on. */
+    showScore?: boolean;
 }
 /** Human-readable plain text report formatter. */
 export declare function formatReportHuman(report: RunReport, options?: FormatReportOptions): string;

package/dist/report.js

@@ -1,3 +1,44 @@
+const DIMENSION_LABELS = {
+    protocol: 'Protocol',
+    schema: 'Schema',
+    usability: 'Agent usability',
+    security: 'Security',
+    reliability: 'Reliability',
+};
+function formatScoreBlock(label, score) {
+    const lines = [];
+    lines.push(label);
+    for (const [dim, value] of Object.entries(score.dimensions)) {
+        const name = (DIMENSION_LABELS[dim] ?? dim).padEnd(17, ' ');
+        lines.push(`  ${name} ${String(value).padStart(3, ' ')}/100`);
+    }
+    lines.push(`  ${'MCP QUALITY SCORE'.padEnd(17, ' ')} ${String(score.overall).padStart(3, ' ')}/100`);
+    if (score.deductions.length > 0) {
+        lines.push('  Deductions:');
+        for (const d of score.deductions) {
+            lines.push(`    -${d.points} ${d.description}`);
+        }
+    }
+    return lines;
+}
+/** Printed once per report (not per-server) — coverage and the disclaimer
+ * are report-level facts, and repeating them per server would overwhelm
+ * the output for a multi-server fleet. */
+function formatCoverageAndDisclaimer(quality) {
+    const lines = [];
+    const gaps = Object.entries(quality.coverage).filter(([, status]) => status !== 'covered');
+    const coverageLine = gaps.length === 0
+        ? `Coverage: ${quality.coveragePercent}% (all dimensions fully evaluated)`
+        : `Coverage: ${quality.coveragePercent}% (${gaps
+            .map(([dim, status]) => `${DIMENSION_LABELS[dim] ?? dim}: ${status}`)
+            .join(', ')})`;
+    lines.push(coverageLine);
+    if (quality.unscoredServers.length > 0) {
+        lines.push(`Note: ${quality.unscoredServers.length} server(s) could not be scored (connection failed): ${quality.unscoredServers.join(', ')}`);
+    }
+    lines.push(quality.disclaimer);
+    return lines;
+}
 /** Human-readable plain text report formatter. */
 export function formatReportHuman(report, options = {}) {
     const lines = [];
@@ -30,6 +71,19 @@ export function formatReportHuman(report, options = {}) {
         if (conn.error) {
             lines.push(`  ${conn.error.stage}: ${conn.error.message}`);
         }
+        const perServerScore = options.showScore ? report.quality?.perServer[conn.server.name] : undefined;
+        if (perServerScore) {
+            lines.push('');
+            lines.push(...formatScoreBlock('QUALITY', perServerScore));
+        }
+    }
+    if (options.showScore && report.quality) {
+        if (Object.keys(report.quality.perServer).length > 1) {
+            lines.push('');
+            lines.push(...formatScoreBlock('OVERALL QUALITY (all servers)', report.quality));
+        }
+        lines.push('');
+        lines.push(...formatCoverageAndDisclaimer(report.quality));
     }
     if (report.diagnostics.length > 0) {
         lines.push('');

package/dist/types.d.ts

@@ -14,21 +14,48 @@ export interface MCPConfig {
     servers: MCPServerConfig[];
     sourcePath?: string;
 }
+/** Per the MCP spec's `ToolAnnotations`: optional client *hints*, not
+ * authoritative security guarantees — a server can lie about any of these. */
+export interface MCPToolAnnotations {
+    title?: string;
+    readOnlyHint?: boolean;
+    destructiveHint?: boolean;
+    idempotentHint?: boolean;
+    openWorldHint?: boolean;
+}
 export interface MCPToolDefinition {
     name: string;
+    title?: string;
     description?: string;
     inputSchema: unknown;
+    /** Optional per spec; same shape as `inputSchema` when present. */
+    outputSchema?: unknown;
+    annotations?: MCPToolAnnotations;
 }
 export interface MCPResourceDefinition {
     uri: string;
+    /** Required by the MCP spec's `Resource` (extends `BaseMetadata`) — kept
+     * optional here because mcp-medic reports a missing name as a diagnostic
+     * rather than failing the whole `resources/list` response over it. */
     name?: string;
+    title?: string;
     description?: string;
     mimeType?: string;
+    size?: number;
+}
+export interface MCPPromptArgument {
+    /** Required by the MCP spec's `PromptArgument` (extends `BaseMetadata`) —
+     * kept optional here for the same reason as `MCPResourceDefinition.name`. */
+    name?: string;
+    title?: string;
+    description?: string;
+    required?: boolean;
 }
 export interface MCPPromptDefinition {
     name: string;
+    title?: string;
     description?: string;
-    arguments?: unknown[];
+    arguments?: MCPPromptArgument[];
 }
 export interface ProtocolVersionInfo {
     /** The protocolVersion this client sent in `initialize`. */
@@ -68,6 +95,10 @@ export interface MCPConnection {
     latencyMs?: number;
 }
 export type Severity = 'error' | 'warning' | 'info';
+/** Stable diagnostic taxonomy. Optional and additive — existing checks that
+ * don't set this are categorized by `inferDiagnosticCategory()` (src/diagnostics.ts)
+ * from their `checkId` prefix, so nothing that already works needs to change. */
+export type DiagnosticCategory = 'protocol' | 'schema' | 'quality' | 'security' | 'reliability' | 'usability' | 'configuration';
 export interface SuggestedFix {
     description: string;
     patch?: unknown;
@@ -80,6 +111,12 @@ export interface DiagnosticResult {
     toolName?: string;
     details?: unknown;
     suggestedFix?: SuggestedFix;
+    /** Optional; see `DiagnosticCategory`. */
+    category?: DiagnosticCategory;
+    /** Optional: how confident the check is that this is a real problem (vs. a heuristic guess). */
+    confidence?: 'low' | 'medium' | 'high';
+    /** Optional: link to a doc explaining the rule this diagnostic enforces. */
+    documentationUrl?: string;
 }
 export interface Check {
     id: string;
@@ -94,6 +131,46 @@ export interface RunOptions {
     /** "auto" (default) requests the newest protocol version this client supports; an explicit version string requests that version instead. */
     protocolVersion?: string;
 }
+/** The five scored dimensions of MCP quality (see src/quality-score.ts for the scoring model). */
+export type QualityDimension = 'protocol' | 'schema' | 'usability' | 'security' | 'reliability';
+export interface QualityDeduction {
+    dimension: QualityDimension;
+    checkId: string;
+    severity: Severity;
+    /** How many diagnostics from this checkId (at this severity) contributed. */
+    count: number;
+    /** Points actually deducted, after the per-checkId cap. */
+    points: number;
+    description: string;
+}
+export interface QualityScoreBreakdown {
+    overall: number;
+    dimensions: Record<QualityDimension, number>;
+    /** Only deductions with points > 0, most-costly first — every deduction here explains itself. */
+    deductions: QualityDeduction[];
+}
+/** Whether the checks that feed a dimension actually ran this time.
+ * 'covered': every built-in check for this dimension ran.
+ * 'partial': at least one check contributing to this dimension ran, but not all of them.
+ * 'not-covered': no check contributing to this dimension ran — a 100 in
+ * that dimension means "nothing flagged it," not "nothing wrong exists." */
+export type CoverageStatus = 'covered' | 'partial' | 'not-covered';
+export type QualityCoverage = Record<QualityDimension, CoverageStatus>;
+export interface ReportQualityScore extends QualityScoreBreakdown {
+    /** Per-connected-server breakdown; a server that never connected has no entry (nothing to score). */
+    perServer: Record<string, QualityScoreBreakdown>;
+    /** Per-dimension coverage, derived from which checks actually ran (`RunOptions.checks`) —
+     * never assume a dimension was fully evaluated just because it scored 100. */
+    coverage: QualityCoverage;
+    /** 0-100: 'covered' dimensions count as 1, 'partial' as 0.5, 'not-covered' as 0, averaged across all 5. */
+    coveragePercent: number;
+    /** Names of servers actually included in this score (connected + scored). */
+    scoredServers: string[];
+    /** Names of servers that could NOT be scored (failed to connect) — never silently dropped from view. */
+    unscoredServers: string[];
+    /** A short, load-bearing reminder of what this number does and doesn't mean — see src/quality-score.ts. */
+    disclaimer: string;
+}
 export interface RunReport {
     configSource?: string;
     connections: MCPConnection[];
@@ -105,4 +182,7 @@ export interface RunReport {
         errors: number;
         warnings: number;
     };
+    /** Deterministic quality score derived from `diagnostics`/`connections` — see src/quality-score.ts.
+     * `undefined` when no server connected (nothing to score). */
+    quality?: ReportQualityScore;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-medic",
-  "version": "1.2.2",
+  "version": "1.2.4",
   "description": "Diagnose broken MCP (Model Context Protocol) server configs before they break your agent silently.",
   "type": "module",
   "main": "dist/index.js",