@bryan-thompson/inspector-assessment-cli 1.30.1 → 1.31.0

package/build/lib/assessment-runner/config-builder.js

@@ -68,6 +68,15 @@ export function buildConfig(options) {
     if (options.temporalInvocations) {
         config.temporalInvocations = options.temporalInvocations;
     }
+    // Official MCP conformance testing (opt-in via --conformance flag)
+    // Requires HTTP/SSE transport with serverUrl - STDIO transport will skip gracefully
+    if (options.conformanceEnabled) {
+        config.assessmentCategories = {
+            ...config.assessmentCategories,
+            conformance: true,
+        };
+        console.log("🔍 Official MCP conformance testing enabled");
+    }
     if (options.claudeEnabled) {
         // Check for HTTP transport via --claude-http flag or environment variables
         const useHttpTransport = options.claudeHttp || process.env.INSPECTOR_CLAUDE === "true";

package/build/lib/assessment-runner/server-configSchemas.js

@@ -0,0 +1,122 @@
+/**
+ * Zod Schemas for Server Configuration Files
+ *
+ * Runtime validation for MCP server configuration loaded from JSON files.
+ * Supports both Claude Desktop config format and standalone config format.
+ *
+ * @module cli/lib/assessment-runner/server-configSchemas
+ *
+ * @remarks
+ * This module provides type-discriminated schemas for config FILE parsing,
+ * with separate schemas for each transport type (HttpSseServerConfigSchema,
+ * StdioServerConfigSchema). This enables type-safe handling of transport-specific
+ * fields.
+ *
+ * For flexible CLI argument parsing where transport may not be specified,
+ * see cli-parserSchemas.ts (ServerConfigSchema) which uses a single schema
+ * with refinement-based validation.
+ *
+ * **Design Decision (Issue #114):**
+ * Both schema approaches are kept because they serve different purposes:
+ * - server-configSchemas.ts: Type-safe discriminated unions for file parsing
+ * - cli-parserSchemas.ts: Flexible validation for CLI argument parsing
+ *
+ * @see cli-parserSchemas.ts for CLI argument validation
+ * @see sharedSchemas.ts for common schemas (TransportTypeSchema, etc.)
+ */
+import { z } from "zod";
+// Import TransportTypeSchema from shared schemas for consistency
+import { TransportTypeSchema } from "../../../../client/lib/lib/assessment/sharedSchemas.js";
+// Re-export for backwards compatibility
+export { TransportTypeSchema };
+/**
+ * Schema for HTTP/SSE transport server configuration.
+ */
+export const HttpSseServerConfigSchema = z.object({
+    transport: z.enum(["http", "sse"]).optional(),
+    url: z.string().url("url must be a valid URL"),
+});
+/**
+ * Schema for stdio transport server configuration.
+ */
+export const StdioServerConfigSchema = z.object({
+    transport: z.literal("stdio").optional(),
+    command: z.string().min(1, "command is required for stdio transport"),
+    args: z.array(z.string()).optional().default([]),
+    env: z.record(z.string()).optional().default({}),
+    cwd: z.string().optional(),
+});
+/**
+ * Schema for a single server entry (either transport type).
+ * Used within mcpServers object or as standalone config.
+ */
+export const ServerEntrySchema = z.union([
+    HttpSseServerConfigSchema,
+    StdioServerConfigSchema,
+]);
+/**
+ * Schema for Claude Desktop config format.
+ * Contains nested mcpServers object with server configurations.
+ */
+export const ClaudeDesktopConfigSchema = z.object({
+    mcpServers: z.record(ServerEntrySchema).optional(),
+});
+/**
+ * Schema for standalone config file format.
+ * Direct server configuration without nesting.
+ */
+export const StandaloneConfigSchema = ServerEntrySchema;
+/**
+ * Combined schema for any valid config file format.
+ */
+export const ConfigFileSchema = z.union([
+    ClaudeDesktopConfigSchema,
+    StandaloneConfigSchema,
+]);
+/**
+ * Parse a config file's JSON content and validate its structure.
+ *
+ * @param jsonContent - Raw JSON object from file
+ * @returns Validated config file content
+ * @throws ZodError if validation fails
+ */
+export function parseConfigFile(jsonContent) {
+    return ConfigFileSchema.parse(jsonContent);
+}
+/**
+ * Safely parse a config file without throwing.
+ *
+ * @param jsonContent - Raw JSON object from file
+ * @returns SafeParseResult with success status and data/error
+ */
+export function safeParseConfigFile(jsonContent) {
+    return ConfigFileSchema.safeParse(jsonContent);
+}
+/**
+ * Validate a server entry matches expected structure.
+ *
+ * @param entry - Server configuration entry
+ * @returns Array of validation error messages (empty if valid)
+ */
+export function validateServerEntry(entry) {
+    const result = ServerEntrySchema.safeParse(entry);
+    if (result.success) {
+        return [];
+    }
+    return result.error.errors.map((e) => {
+        const path = e.path.length > 0 ? `${e.path.join(".")}: ` : "";
+        return `${path}${e.message}`;
+    });
+}
+/**
+ * Check if a server entry is HTTP/SSE transport.
+ */
+export function isHttpSseConfig(entry) {
+    return ("url" in entry || entry.transport === "http" || entry.transport === "sse");
+}
+/**
+ * Check if a server entry is stdio transport.
+ */
+export function isStdioConfig(entry) {
+    return "command" in entry && !("url" in entry);
+}

package/build/lib/cli-parser.js

@@ -9,36 +9,33 @@
  * @module cli/lib/cli-parser
  */
 import { ASSESSMENT_CATEGORY_METADATA, } from "../../../client/lib/lib/assessmentTypes.js";
-import { ASSESSMENT_PROFILES, isValidProfileName, getProfileHelpText, TIER_1_CORE_SECURITY, TIER_2_COMPLIANCE, TIER_3_CAPABILITY, TIER_4_EXTENDED, } from "../profiles.js";
+import { ASSESSMENT_PROFILES, getProfileHelpText, TIER_1_CORE_SECURITY, TIER_2_COMPLIANCE, TIER_3_CAPABILITY, TIER_4_EXTENDED, } from "../profiles.js";
 import packageJson from "../../package.json" with { type: "json" };
+import { safeParseModuleNames, LogLevelSchema, ReportFormatSchema, AssessmentProfileNameSchema, } from "./cli-parserSchemas.js";
 // ============================================================================
 // Constants
 // ============================================================================
-// Valid module names derived from ASSESSMENT_CATEGORY_METADATA
+// Valid module names derived from ASSESSMENT_CATEGORY_METADATA (used for help text)
 const VALID_MODULE_NAMES = Object.keys(ASSESSMENT_CATEGORY_METADATA);
 // ============================================================================
 // Validation Functions
 // ============================================================================
 /**
- * Validate module names from CLI input
+ * Validate module names from CLI input using Zod schema.
  *
  * @param input - Comma-separated module names
  * @param flagName - Flag name for error messages (e.g., "--skip-modules")
  * @returns Array of validated module names, or empty array if invalid
  */
 export function validateModuleNames(input, flagName) {
-    const names = input
-        .split(",")
-        .map((n) => n.trim())
-        .filter(Boolean);
-    const invalid = names.filter((n) => !VALID_MODULE_NAMES.includes(n));
-    if (invalid.length > 0) {
-        console.error(`Error: Invalid module name(s) for ${flagName}: ${invalid.join(", ")}`);
+    const result = safeParseModuleNames(input);
+    if (result.invalid.length > 0) {
+        console.error(`Error: Invalid module name(s) for ${flagName}: ${result.invalid.join(", ")}`);
         console.error(`Valid modules: ${VALID_MODULE_NAMES.join(", ")}`);
         setTimeout(() => process.exit(1), 10);
         return [];
     }
-    return names;
+    return result.valid;
 }
 /**
  * Validate parsed options for consistency and requirements
@@ -146,20 +143,14 @@ export function parseArgs(argv) {
                 break;
             case "--log-level": {
                 const levelValue = args[++i];
-                const validLevels = [
-                    "silent",
-                    "error",
-                    "warn",
-                    "info",
-                    "debug",
-                ];
-                if (!validLevels.includes(levelValue)) {
-                    console.error(`Invalid log level: ${levelValue}. Valid options: ${validLevels.join(", ")}`);
+                const parseResult = LogLevelSchema.safeParse(levelValue);
+                if (!parseResult.success) {
+                    console.error(`Invalid log level: ${levelValue}. Valid options: silent, error, warn, info, debug`);
                     setTimeout(() => process.exit(1), 10);
                     options.helpRequested = true;
                     return options;
                 }
-                options.logLevel = levelValue;
+                options.logLevel = parseResult.data;
                 break;
             }
             case "--json":
@@ -168,13 +159,14 @@ export function parseArgs(argv) {
             case "--format":
             case "-f": {
                 const formatValue = args[++i];
-                if (formatValue !== "json" && formatValue !== "markdown") {
+                const parseResult = ReportFormatSchema.safeParse(formatValue);
+                if (!parseResult.success) {
                     console.error(`Invalid format: ${formatValue}. Valid options: json, markdown`);
                     setTimeout(() => process.exit(1), 10);
                     options.helpRequested = true;
                     return options;
                 }
-                options.format = formatValue;
+                options.format = parseResult.data;
                 break;
             }
             case "--include-policy":
@@ -201,6 +193,10 @@ export function parseArgs(argv) {
             case "--skip-temporal":
                 options.skipTemporal = true;
                 break;
+            case "--conformance":
+                // Enable official MCP conformance tests (requires HTTP/SSE transport with serverUrl)
+                options.conformanceEnabled = true;
+                break;
             case "--profile": {
                 const profileValue = args[++i];
                 if (!profileValue) {
@@ -210,14 +206,15 @@ export function parseArgs(argv) {
                     options.helpRequested = true;
                     return options;
                 }
-                if (!isValidProfileName(profileValue)) {
+                const parseResult = AssessmentProfileNameSchema.safeParse(profileValue);
+                if (!parseResult.success) {
                     console.error(`Error: Invalid profile name: ${profileValue}`);
                     console.error(`Valid profiles: ${Object.keys(ASSESSMENT_PROFILES).join(", ")}`);
                     setTimeout(() => process.exit(1), 10);
                     options.helpRequested = true;
                     return options;
                 }
-                options.profile = profileValue;
+                options.profile = parseResult.data;
                 break;
             }
             case "--skip-modules": {
@@ -364,6 +361,7 @@ Options:
   --profile <name>       Use predefined module profile (quick, security, compliance, full)
   --temporal-invocations <n>  Number of invocations per tool for rug pull detection (default: 25)
   --skip-temporal        Skip temporal/rug pull testing (faster assessment)
+  --conformance          Enable official MCP conformance tests (experimental, requires HTTP/SSE transport)
   --skip-modules <list>  Skip specific modules (comma-separated)
   --only-modules <list>  Run only specific modules (comma-separated)
   --json                 Output only JSON path (no console summary)

package/build/lib/cli-parserSchemas.js

@@ -0,0 +1,231 @@
+/**
+ * Zod Schemas for CLI Argument Parsing
+ *
+ * Runtime validation schemas for CLI arguments and server configuration.
+ * Replaces manual validation in parseArgs() and validateArgs() functions.
+ *
+ * @module cli/lib/cli-parserSchemas
+ */
+import { z } from "zod";
+// Import shared schemas from single source of truth
+import { LogLevelSchema, ReportFormatSchema, TransportTypeSchema, ZOD_SCHEMA_VERSION, } from "../../../client/lib/lib/assessment/sharedSchemas.js";
+// Re-export shared schemas for backwards compatibility
+export { LogLevelSchema, ReportFormatSchema, TransportTypeSchema };
+// Export schema version for consumers
+export { ZOD_SCHEMA_VERSION };
+/**
+ * Valid assessment profile names.
+ */
+export const AssessmentProfileNameSchema = z.enum([
+    "quick",
+    "security",
+    "compliance",
+    "full",
+]);
+/**
+ * Valid assessment module names.
+ * Derived from ASSESSMENT_CATEGORY_METADATA in coreTypes.ts.
+ */
+export const AssessmentModuleNameSchema = z.enum([
+    "functionality",
+    "security",
+    "documentation",
+    "errorHandling",
+    "usability",
+    "mcpSpecCompliance",
+    "aupCompliance",
+    "toolAnnotations",
+    "prohibitedLibraries",
+    "manifestValidation",
+    "portability",
+    "externalAPIScanner",
+    "authentication",
+    "temporal",
+    "resources",
+    "prompts",
+    "crossCapability",
+    "protocolConformance",
+]);
+// ============================================================================
+// Server Configuration Schema
+// ============================================================================
+/**
+ * Schema for server connection configuration.
+ * Validates transport-specific required fields.
+ *
+ * @remarks
+ * This schema provides flexible validation for CLI argument parsing.
+ * For type-safe config file parsing with discriminated unions, see
+ * server-configSchemas.ts (HttpSseServerConfigSchema, StdioServerConfigSchema).
+ */
+export const ServerConfigSchema = z
+    .object({
+    transport: TransportTypeSchema.optional(),
+    command: z.string().optional(),
+    args: z.array(z.string()).optional(),
+    env: z.record(z.string()).optional(),
+    cwd: z.string().optional(),
+    url: z.string().optional(),
+})
+    .refine((data) => {
+    // For http/sse transport, url is required
+    if (data.transport === "http" || data.transport === "sse") {
+        return !!data.url;
+    }
+    // For stdio transport, command is required
+    if (data.transport === "stdio") {
+        return !!data.command;
+    }
+    // If no transport specified, either url or command must be present
+    return !!data.url || !!data.command;
+}, {
+    message: "For http/sse transport, 'url' is required. For stdio transport, 'command' is required.",
+});
+// ============================================================================
+// Assessment Options Schema
+// ============================================================================
+/**
+ * Schema for assessment CLI options.
+ * Validates all command-line arguments and their constraints.
+ */
+export const AssessmentOptionsSchema = z
+    .object({
+    serverName: z.string().min(1, "--server is required"),
+    serverConfigPath: z.string().optional(),
+    outputPath: z.string().optional(),
+    sourceCodePath: z.string().optional(),
+    patternConfigPath: z.string().optional(),
+    performanceConfigPath: z.string().optional(),
+    claudeEnabled: z.boolean().optional(),
+    claudeHttp: z.boolean().optional(),
+    mcpAuditorUrl: z
+        .string()
+        .url("--mcp-auditor-url must be a valid URL")
+        .optional(),
+    fullAssessment: z.boolean().optional(),
+    verbose: z.boolean().optional(),
+    jsonOnly: z.boolean().optional(),
+    helpRequested: z.boolean().optional(),
+    versionRequested: z.boolean().optional(),
+    format: ReportFormatSchema.optional(),
+    includePolicy: z.boolean().optional(),
+    preflightOnly: z.boolean().optional(),
+    comparePath: z.string().optional(),
+    diffOnly: z.boolean().optional(),
+    resume: z.boolean().optional(),
+    noResume: z.boolean().optional(),
+    temporalInvocations: z.number().int().positive().optional(),
+    skipTemporal: z.boolean().optional(),
+    skipModules: z.array(AssessmentModuleNameSchema).optional(),
+    onlyModules: z.array(AssessmentModuleNameSchema).optional(),
+    profile: AssessmentProfileNameSchema.optional(),
+    logLevel: LogLevelSchema.optional(),
+    listModules: z.boolean().optional(),
+})
+    .refine((data) => !(data.profile && (data.skipModules?.length || data.onlyModules?.length)), {
+    message: "--profile cannot be used with --skip-modules or --only-modules",
+    path: ["profile"],
+})
+    .refine((data) => !(data.skipModules?.length && data.onlyModules?.length), {
+    message: "--skip-modules and --only-modules are mutually exclusive",
+    path: ["skipModules"],
+});
+/**
+ * Schema for validation result.
+ */
+export const ValidationResultSchema = z.object({
+    valid: z.boolean(),
+    errors: z.array(z.string()),
+});
+// ============================================================================
+// Validation Helpers
+// ============================================================================
+/**
+ * Validate assessment options and return error messages.
+ *
+ * @param options - Options to validate
+ * @returns Array of validation error messages (empty if valid)
+ */
+export function validateAssessmentOptions(options) {
+    const result = AssessmentOptionsSchema.safeParse(options);
+    if (result.success) {
+        return [];
+    }
+    return result.error.errors.map((e) => {
+        const path = e.path.length > 0 ? `${e.path.join(".")}: ` : "";
+        return `${path}${e.message}`;
+    });
+}
+/**
+ * Validate a server configuration.
+ *
+ * @param config - Server config to validate
+ * @returns Array of validation error messages (empty if valid)
+ */
+export function validateServerConfig(config) {
+    const result = ServerConfigSchema.safeParse(config);
+    if (result.success) {
+        return [];
+    }
+    return result.error.errors.map((e) => {
+        const path = e.path.length > 0 ? `${e.path.join(".")}: ` : "";
+        return `${path}${e.message}`;
+    });
+}
+/**
+ * Parse assessment options with validation.
+ *
+ * @param options - Raw options object
+ * @returns Validated options
+ * @throws ZodError if validation fails
+ */
+export function parseAssessmentOptions(options) {
+    return AssessmentOptionsSchema.parse(options);
+}
+/**
+ * Safely parse assessment options without throwing.
+ *
+ * @param options - Raw options object
+ * @returns SafeParseResult with success status and data/error
+ */
+export function safeParseAssessmentOptions(options) {
+    return AssessmentOptionsSchema.safeParse(options);
+}
+/**
+ * Validate module names from a comma-separated string.
+ *
+ * @param input - Comma-separated module names
+ * @returns Array of validated module names
+ * @throws ZodError if any module name is invalid
+ */
+export function parseModuleNames(input) {
+    const names = input
+        .split(",")
+        .map((n) => n.trim())
+        .filter(Boolean);
+    return z.array(AssessmentModuleNameSchema).parse(names);
+}
+/**
+ * Safely validate module names without throwing.
+ *
+ * @param input - Comma-separated module names
+ * @returns Object with valid module names and any invalid names
+ */
+export function safeParseModuleNames(input) {
+    const names = input
+        .split(",")
+        .map((n) => n.trim())
+        .filter(Boolean);
+    const valid = [];
+    const invalid = [];
+    for (const name of names) {
+        const result = AssessmentModuleNameSchema.safeParse(name);
+        if (result.success) {
+            valid.push(result.data);
+        }
+        else {
+            invalid.push(name);
+        }
+    }
+    return { valid, invalid };
+}

package/build/lib/zodErrorFormatter.js

@@ -0,0 +1,91 @@
+/**
+ * Zod Error Formatting Utilities
+ *
+ * CLI-friendly error message formatting for Zod validation errors.
+ *
+ * @module cli/lib/zodErrorFormatter
+ */
+/**
+ * Format a single Zod issue into a readable string.
+ *
+ * @param issue - Zod validation issue
+ * @returns Formatted error message
+ */
+export function formatZodIssue(issue) {
+    const path = issue.path.length > 0 ? `${issue.path.join(".")}: ` : "";
+    return `${path}${issue.message}`;
+}
+/**
+ * Format a ZodError into a readable string.
+ *
+ * @param error - Zod validation error
+ * @returns Formatted error messages joined by newlines
+ */
+export function formatZodError(error) {
+    return error.errors.map(formatZodIssue).join("\n");
+}
+/**
+ * Format a ZodError with indentation for CLI output.
+ *
+ * @param error - Zod validation error
+ * @param indent - Indentation string (default: "  ")
+ * @returns Formatted error messages with indentation
+ */
+export function formatZodErrorIndented(error, indent = "  ") {
+    return error.errors.map((e) => `${indent}${formatZodIssue(e)}`).join("\n");
+}
+/**
+ * Print a formatted Zod error to stderr with context.
+ *
+ * @param error - Zod validation error
+ * @param context - Context description (e.g., "config file", "CLI arguments")
+ */
+export function printZodErrorForCli(error, context) {
+    const prefix = context ? `Error in ${context}:\n` : "Validation error:\n";
+    console.error(prefix + formatZodErrorIndented(error));
+}
+/**
+ * Convert ZodError to an array of error strings.
+ * Useful for accumulating errors in a validation result.
+ *
+ * @param error - Zod validation error
+ * @returns Array of formatted error strings
+ */
+export function zodErrorToArray(error) {
+    return error.errors.map(formatZodIssue);
+}
+/**
+ * Format validation errors for JSON output.
+ *
+ * @param error - Zod validation error
+ * @returns Object with structured error information
+ */
+export function formatZodErrorForJson(error) {
+    return {
+        message: "Validation failed",
+        errors: error.errors.map((e) => ({
+            path: e.path,
+            message: e.message,
+            code: e.code,
+        })),
+    };
+}
+/**
+ * Create a user-friendly error message for common validation issues.
+ *
+ * @param error - Zod validation error
+ * @param fieldLabels - Optional map of field names to user-friendly labels
+ * @returns User-friendly error message
+ */
+export function formatUserFriendlyError(error, fieldLabels) {
+    const messages = error.errors.map((issue) => {
+        const fieldPath = issue.path.join(".");
+        const label = fieldLabels?.[fieldPath] || fieldPath;
+        const prefix = label ? `${label}: ` : "";
+        return `${prefix}${issue.message}`;
+    });
+    if (messages.length === 1) {
+        return messages[0];
+    }
+    return `Multiple validation errors:\n${messages.map((m) => `  - ${m}`).join("\n")}`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bryan-thompson/inspector-assessment-cli",
-  "version": "1.30.1",
+  "version": "1.31.0",
   "description": "CLI for the Enhanced MCP Inspector with assessment capabilities",
   "license": "MIT",
   "author": "Bryan Thompson <bryan@triepod.ai>",