mcp-server-diff 2.1.0 → 2.1.6

package/src/probe.ts

@@ -1,327 +0,0 @@
-/**
- * MCP Server Probe
- *
- * Probes an MCP server and collects capability snapshots.
- */
-
-import { Client } from "@modelcontextprotocol/sdk/client/index.js";
-import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
-import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
-import { z } from "zod";
-import type {
-  ProbeResult,
-  InitializeInfo,
-  ToolsResult,
-  PromptsResult,
-  ResourcesResult,
-  ResourceTemplatesResult,
-  CustomMessage,
-} from "./types.js";
-import { log } from "./logger.js";
-
-export interface ProbeOptions {
-  transport: "stdio" | "streamable-http";
-  command?: string;
-  args?: string[];
-  url?: string;
-  headers?: Record<string, string>;
-  workingDir?: string;
-  envVars?: Record<string, string>;
-  customMessages?: CustomMessage[];
-}
-
-/**
- * Probes an MCP server and returns capability snapshots
- */
-export async function probeServer(options: ProbeOptions): Promise<ProbeResult> {
-  const result: ProbeResult = {
-    initialize: null,
-    tools: null,
-    prompts: null,
-    resources: null,
-    resourceTemplates: null,
-    customResponses: new Map(),
-  };
-
-  const client = new Client(
-    {
-      name: "mcp-server-diff-probe",
-      version: "2.0.0",
-    },
-    {
-      capabilities: {},
-    }
-  );
-
-  let transport: StdioClientTransport | StreamableHTTPClientTransport;
-
-  try {
-    if (options.transport === "stdio") {
-      if (!options.command) {
-        throw new Error("Command is required for stdio transport");
-      }
-
-      log.info(`  Connecting via stdio: ${options.command} ${(options.args || []).join(" ")}`);
-
-      // Merge environment variables
-      const env: Record<string, string> = {};
-      for (const [key, value] of Object.entries(process.env)) {
-        if (value !== undefined) {
-          env[key] = value;
-        }
-      }
-      for (const [key, value] of Object.entries(options.envVars || {})) {
-        env[key] = value;
-      }
-
-      transport = new StdioClientTransport({
-        command: options.command,
-        args: options.args || [],
-        env,
-        cwd: options.workingDir,
-      });
-    } else {
-      if (!options.url) {
-        throw new Error("URL is required for streamable-http transport");
-      }
-
-      log.info(`  Connecting via streamable-http: ${options.url}`);
-      const transportOptions: { requestInit?: RequestInit } = {};
-      if (options.headers && Object.keys(options.headers).length > 0) {
-        transportOptions.requestInit = { headers: options.headers };
-        log.info(`  With headers: ${Object.keys(options.headers).join(", ")}`);
-      }
-      transport = new StreamableHTTPClientTransport(new URL(options.url), transportOptions);
-    }
-
-    // Connect to the server
-    await client.connect(transport);
-    log.info("  Connected successfully");
-
-    // Get server info and capabilities
-    const serverCapabilities = client.getServerCapabilities();
-    const serverInfo = client.getServerVersion();
-
-    result.initialize = {
-      serverInfo,
-      capabilities: serverCapabilities,
-    } as InitializeInfo;
-
-    // Probe tools if supported
-    if (serverCapabilities?.tools) {
-      try {
-        const toolsResult = await client.listTools();
-        result.tools = toolsResult as ToolsResult;
-        log.info(`  Listed ${result.tools.tools.length} tools`);
-      } catch (error) {
-        log.warning(`  Failed to list tools: ${error}`);
-      }
-    } else {
-      log.info("  Server does not support tools");
-    }
-
-    // Probe prompts if supported
-    if (serverCapabilities?.prompts) {
-      try {
-        const promptsResult = await client.listPrompts();
-        result.prompts = promptsResult as PromptsResult;
-        log.info(`  Listed ${result.prompts.prompts.length} prompts`);
-      } catch (error) {
-        log.warning(`  Failed to list prompts: ${error}`);
-      }
-    } else {
-      log.info("  Server does not support prompts");
-    }
-
-    // Probe resources if supported
-    if (serverCapabilities?.resources) {
-      try {
-        const resourcesResult = await client.listResources();
-        result.resources = resourcesResult as ResourcesResult;
-        log.info(`  Listed ${result.resources.resources.length} resources`);
-      } catch (error) {
-        log.warning(`  Failed to list resources: ${error}`);
-      }
-
-      // Also get resource templates
-      try {
-        const templatesResult = await client.listResourceTemplates();
-        result.resourceTemplates = templatesResult as ResourceTemplatesResult;
-        log.info(
-          `  Listed ${result.resourceTemplates.resourceTemplates.length} resource templates`
-        );
-      } catch (error) {
-        log.warning(`  Failed to list resource templates: ${error}`);
-      }
-    } else {
-      log.info("  Server does not support resources");
-    }
-
-    // Send custom messages if provided
-    if (options.customMessages && options.customMessages.length > 0) {
-      // Schema that accepts any response for custom messages
-      const anyResponseSchema = z.record(z.unknown());
-
-      for (const customMsg of options.customMessages) {
-        try {
-          // Cast message to the expected request type - custom messages should have a method field
-          const response = await client.request(
-            customMsg.message as { method: string; params?: Record<string, unknown> },
-            anyResponseSchema
-          );
-          result.customResponses.set(customMsg.name, response);
-          log.info(`  Custom message '${customMsg.name}' successful`);
-        } catch (error) {
-          log.warning(`  Custom message '${customMsg.name}' failed: ${error}`);
-        }
-      }
-    }
-
-    log.info("  Probe complete");
-
-    // Close the connection
-    await client.close();
-  } catch (error) {
-    result.error = String(error);
-    log.error(`  Error probing server: ${error}`);
-
-    // Try to close client on error
-    try {
-      await client.close();
-    } catch {
-      // Ignore close errors
-    }
-  }
-
-  return result;
-}
-
-/**
- * Get a sort key for an array element based on common MCP entity patterns.
- * This ensures deterministic sorting for tools, prompts, resources, etc.
- */
-function getSortKey(item: unknown): string {
-  if (item === null || item === undefined) {
-    return "";
-  }
-
-  if (typeof item !== "object") {
-    return String(item);
-  }
-
-  const obj = item as Record<string, unknown>;
-
-  // Primary sort keys for MCP entities (in priority order)
-  // Tools, prompts, arguments: use "name"
-  // Resources, resource templates: use "uri" or "uriTemplate"
-  // Content items: use "uri" or "type"
-  if (typeof obj.name === "string") {
-    return obj.name;
-  }
-  if (typeof obj.uri === "string") {
-    return obj.uri;
-  }
-  if (typeof obj.uriTemplate === "string") {
-    return obj.uriTemplate;
-  }
-  if (typeof obj.type === "string") {
-    return obj.type;
-  }
-  if (typeof obj.method === "string") {
-    return obj.method;
-  }
-
-  // Fallback to JSON string - but normalize first to ensure deterministic output
-  return JSON.stringify(normalizeProbeResult(item));
-}
-
-/**
- * Normalize a probe result for comparison by sorting keys and arrays recursively.
- * Also handles embedded JSON strings in "text" fields (from tool call responses).
- *
- * Sorting strategy:
- * - Object keys: sorted alphabetically
- * - Arrays of objects: sorted by primary key (name, uri, type) for deterministic output
- * - Primitive arrays: sorted by string representation
- * - Embedded JSON in "text" fields: parsed, normalized, and re-serialized
- */
-export function normalizeProbeResult(result: unknown): unknown {
-  if (result === null || result === undefined) {
-    return result;
-  }
-
-  if (Array.isArray(result)) {
-    // First normalize all elements
-    const normalized = result.map(normalizeProbeResult);
-
-    // Then sort by sort key for deterministic output
-    return normalized.sort((a, b) => {
-      const aKey = getSortKey(a);
-      const bKey = getSortKey(b);
-      return aKey.localeCompare(bKey);
-    });
-  }
-
-  if (typeof result === "object") {
-    const obj = result as Record<string, unknown>;
-    const normalized: Record<string, unknown> = {};
-
-    // Sort keys alphabetically
-    const keys = Object.keys(obj).sort();
-
-    for (const key of keys) {
-      let value = obj[key];
-
-      // Handle embedded JSON in "text" fields (tool call responses)
-      if (key === "text" && typeof value === "string") {
-        const trimmed = value.trim();
-        if (trimmed.startsWith("[") || trimmed.startsWith("{")) {
-          try {
-            const parsed = JSON.parse(value);
-            // Re-serialize the normalized JSON to keep it as a string
-            value = JSON.stringify(normalizeProbeResult(parsed));
-          } catch {
-            // Not valid JSON, keep as-is
-          }
-        }
-      }
-
-      normalized[key] = normalizeProbeResult(value);
-    }
-    return normalized;
-  }
-
-  return result;
-}
-
-/**
- * Convert probe result to a map of endpoint -> JSON string
- */
-export function probeResultToFiles(result: ProbeResult): Map<string, string> {
-  const files = new Map<string, string>();
-
-  if (result.initialize) {
-    files.set("initialize", JSON.stringify(normalizeProbeResult(result.initialize), null, 2));
-  }
-  if (result.tools) {
-    files.set("tools", JSON.stringify(normalizeProbeResult(result.tools), null, 2));
-  }
-  if (result.prompts) {
-    files.set("prompts", JSON.stringify(normalizeProbeResult(result.prompts), null, 2));
-  }
-  if (result.resources) {
-    files.set("resources", JSON.stringify(normalizeProbeResult(result.resources), null, 2));
-  }
-  if (result.resourceTemplates) {
-    files.set(
-      "resource_templates",
-      JSON.stringify(normalizeProbeResult(result.resourceTemplates), null, 2)
-    );
-  }
-
-  for (const [name, response] of result.customResponses.entries()) {
-    files.set(`custom_${name}`, JSON.stringify(normalizeProbeResult(response), null, 2));
-  }
-
-  return files;
-}

package/src/reporter.ts

@@ -1,214 +0,0 @@
-/**
- * Report generator for MCP server diff
- */
-
-import * as core from "@actions/core";
-import * as fs from "fs";
-import * as path from "path";
-import type { TestResult, ConformanceReport } from "./types.js";
-
-/**
- * Generate a diff report from test results
- */
-export function generateReport(
-  results: TestResult[],
-  currentBranch: string,
-  compareRef: string
-): ConformanceReport {
-  const totalBranchTime = results.reduce((sum, r) => sum + r.branchTime, 0);
-  const totalBaseTime = results.reduce((sum, r) => sum + r.baseTime, 0);
-  const passedCount = results.filter((r) => !r.hasDifferences).length;
-  const diffCount = results.filter((r) => r.hasDifferences).length;
-
-  return {
-    generatedAt: new Date().toISOString(),
-    currentBranch,
-    compareRef,
-    results,
-    totalBranchTime,
-    totalBaseTime,
-    passedCount,
-    diffCount,
-  };
-}
-
-/**
- * Generate markdown report
- */
-export function generateMarkdownReport(report: ConformanceReport): string {
-  const lines: string[] = [];
-
-  lines.push("# MCP Conformance Test Report");
-  lines.push("");
-  lines.push(`**Generated:** ${report.generatedAt}`);
-  lines.push(`**Current Branch:** ${report.currentBranch}`);
-  lines.push(`**Compared Against:** ${report.compareRef}`);
-  lines.push("");
-
-  // Summary
-  lines.push("## Summary");
-  lines.push("");
-  lines.push(`| Metric | Value |`);
-  lines.push(`|--------|-------|`);
-  lines.push(`| Total Configurations | ${report.results.length} |`);
-  lines.push(`| Passed | ${report.passedCount} |`);
-  lines.push(`| With Differences | ${report.diffCount} |`);
-  lines.push(`| Branch Total Time | ${formatTime(report.totalBranchTime)} |`);
-  lines.push(`| Base Total Time | ${formatTime(report.totalBaseTime)} |`);
-  lines.push("");
-
-  // Overall status
-  if (report.diffCount === 0) {
-    lines.push("## ✅ No API Changes");
-    lines.push("");
-    lines.push("No differences detected between the current branch and the comparison ref.");
-  } else {
-    lines.push("## 📋 API Changes Detected");
-    lines.push("");
-    lines.push(
-      `${report.diffCount} configuration(s) have changes. Review below to ensure they are intentional.`
-    );
-  }
-  lines.push("");
-
-  // Per-configuration results
-  lines.push("## Configuration Results");
-  lines.push("");
-
-  for (const result of report.results) {
-    const statusIcon = result.error ? "❌" : result.hasDifferences ? "⚠️" : "✅";
-    lines.push(`### ${statusIcon} ${result.configName}`);
-    lines.push("");
-    lines.push(`- **Transport:** ${result.transport}`);
-
-    // Show primitive counts if available
-    if (result.branchCounts) {
-      const counts = result.branchCounts;
-      const countParts: string[] = [];
-      if (counts.tools > 0) countParts.push(`${counts.tools} tools`);
-      if (counts.prompts > 0) countParts.push(`${counts.prompts} prompts`);
-      if (counts.resources > 0) countParts.push(`${counts.resources} resources`);
-      if (counts.resourceTemplates > 0)
-        countParts.push(`${counts.resourceTemplates} resource templates`);
-      if (countParts.length > 0) {
-        lines.push(`- **Primitives:** ${countParts.join(", ")}`);
-      }
-    }
-
-    lines.push(`- **Branch Time:** ${formatTime(result.branchTime)}`);
-    lines.push(`- **Base Time:** ${formatTime(result.baseTime)}`);
-    lines.push("");
-
-    if (result.hasDifferences) {
-      lines.push("#### Changes");
-      lines.push("");
-
-      for (const [endpoint, diff] of result.diffs) {
-        lines.push(`**${endpoint}**`);
-        lines.push("");
-        lines.push("```diff");
-        lines.push(diff);
-        lines.push("```");
-        lines.push("");
-      }
-    } else {
-      lines.push("No differences detected.");
-      lines.push("");
-    }
-  }
-
-  return lines.join("\n");
-}
-
-/**
- * Format milliseconds to human readable time
- */
-function formatTime(ms: number): string {
-  if (ms < 1000) {
-    return `${ms}ms`;
-  }
-  const seconds = (ms / 1000).toFixed(2);
-  return `${seconds}s`;
-}
-
-/**
- * Save report to file and set outputs
- */
-export function saveReport(report: ConformanceReport, markdown: string, outputDir: string): void {
-  // Ensure output directory exists
-  const reportDir = path.join(outputDir, "mcp-diff-report");
-  fs.mkdirSync(reportDir, { recursive: true });
-
-  // Save JSON report
-  const jsonPath = path.join(reportDir, "mcp-diff-report.json");
-  fs.writeFileSync(
-    jsonPath,
-    JSON.stringify(
-      {
-        ...report,
-        results: report.results.map((r) => ({
-          ...r,
-          diffs: Object.fromEntries(r.diffs),
-        })),
-      },
-      null,
-      2
-    )
-  );
-  core.info(`📄 JSON report saved to: ${jsonPath}`);
-
-  // Save markdown report
-  const mdPath = path.join(reportDir, "MCP_DIFF_REPORT.md");
-  fs.writeFileSync(mdPath, markdown);
-  core.info(`📄 Markdown report saved to: ${mdPath}`);
-
-  // Set outputs using GITHUB_OUTPUT file (for composite actions)
-  const githubOutput = process.env.GITHUB_OUTPUT;
-  if (githubOutput) {
-    const status = report.diffCount > 0 ? "differences" : "passed";
-    fs.appendFileSync(githubOutput, `status=${status}\n`);
-    fs.appendFileSync(githubOutput, `report_path=${mdPath}\n`);
-    fs.appendFileSync(githubOutput, `json_report_path=${jsonPath}\n`);
-    fs.appendFileSync(githubOutput, `has_differences=${report.diffCount > 0}\n`);
-    fs.appendFileSync(githubOutput, `passed_count=${report.passedCount}\n`);
-    fs.appendFileSync(githubOutput, `diff_count=${report.diffCount}\n`);
-  }
-
-  // Also set via core for compatibility
-  core.setOutput("report_path", mdPath);
-  core.setOutput("json_report_path", jsonPath);
-  core.setOutput("has_differences", report.diffCount > 0);
-  core.setOutput("passed_count", report.passedCount);
-  core.setOutput("diff_count", report.diffCount);
-}
-
-/**
- * Write a simple summary for PR comments
- */
-export function generatePRSummary(report: ConformanceReport): string {
-  const lines: string[] = [];
-
-  if (report.diffCount === 0) {
-    lines.push("## ✅ MCP Conformance: No Changes");
-    lines.push("");
-    lines.push(`Tested ${report.results.length} configuration(s) - no API changes detected.`);
-  } else {
-    lines.push("## 📋 MCP Conformance: API Changes Detected");
-    lines.push("");
-    lines.push(
-      `**${report.diffCount}** of ${report.results.length} configuration(s) have changes.`
-    );
-    lines.push("");
-    lines.push("### Changed Endpoints");
-    lines.push("");
-
-    for (const result of report.results.filter((r) => r.hasDifferences)) {
-      lines.push(`- **${result.configName}:** ${Array.from(result.diffs.keys()).join(", ")}`);
-    }
-
-    lines.push("");
-    lines.push("See the full report in the job summary for details.");
-  }
-
-  return lines.join("\n");
-}