@bryan-thompson/inspector-assessment-cli 1.24.2 → 1.25.0

package/build/assess-full.js

@@ -32,6 +32,7 @@ import { compareAssessments } from "../../client/lib/lib/assessmentDiffer.js";
 import { formatDiffAsMarkdown } from "../../client/lib/lib/reportFormatters/DiffReportFormatter.js";
 import { AssessmentStateManager } from "./assessmentState.js";
 import { emitServerConnected, emitToolDiscovered, emitToolsDiscoveryComplete, emitAssessmentComplete, emitTestBatch, emitVulnerabilityFound, emitAnnotationMissing, emitAnnotationMisaligned, emitAnnotationReviewRecommended, emitAnnotationAligned, emitModulesConfigured, } from "./lib/jsonl-events.js";
+import { ASSESSMENT_PROFILES, isValidProfileName, getProfileModules, resolveModuleNames, modulesToLegacyConfig, getProfileHelpText, } from "./profiles.js";
 // Valid module names derived from ASSESSMENT_CATEGORY_METADATA
 const VALID_MODULE_NAMES = Object.keys(ASSESSMENT_CATEGORY_METADATA);
 /**
@@ -342,28 +343,45 @@ function buildConfig(options) {
         enableSourceCodeAnalysis: Boolean(options.sourceCodePath),
     };
     if (options.fullAssessment !== false) {
-        // Derive module config from ASSESSMENT_CATEGORY_METADATA (single source of truth)
-        const allModules = getAllModulesConfig({
-            sourceCodePath: Boolean(options.sourceCodePath),
-            skipTemporal: options.skipTemporal,
-        });
-        // Apply --only-modules filter (whitelist mode)
-        if (options.onlyModules?.length) {
-            for (const key of Object.keys(allModules)) {
-                // Disable all modules except those in the whitelist
-                allModules[key] = options.onlyModules.includes(key);
-            }
+        // Priority: --profile > --only-modules > --skip-modules > default (all)
+        if (options.profile) {
+            // Use profile-based module selection
+            const profileModules = getProfileModules(options.profile, {
+                hasSourceCode: Boolean(options.sourceCodePath),
+                skipTemporal: options.skipTemporal,
+            });
+            // Convert new-style module list to legacy config format
+            // (until orchestrator is updated to use new naming)
+            config.assessmentCategories = modulesToLegacyConfig(profileModules);
         }
-        // Apply --skip-modules filter (blacklist mode)
-        if (options.skipModules?.length) {
-            for (const module of options.skipModules) {
-                if (module in allModules) {
-                    allModules[module] = false;
+        else {
+            // Derive module config from ASSESSMENT_CATEGORY_METADATA (single source of truth)
+            const allModules = getAllModulesConfig({
+                sourceCodePath: Boolean(options.sourceCodePath),
+                skipTemporal: options.skipTemporal,
+            });
+            // Apply --only-modules filter (whitelist mode)
+            if (options.onlyModules?.length) {
+                // Resolve any deprecated module names
+                const resolved = resolveModuleNames(options.onlyModules);
+                for (const key of Object.keys(allModules)) {
+                    // Disable all modules except those in the whitelist
+                    allModules[key] = resolved.includes(key);
                 }
             }
+            // Apply --skip-modules filter (blacklist mode)
+            if (options.skipModules?.length) {
+                // Resolve any deprecated module names
+                const resolved = resolveModuleNames(options.skipModules);
+                for (const module of resolved) {
+                    if (module in allModules) {
+                        allModules[module] = false;
+                    }
+                }
+            }
+            config.assessmentCategories =
+                allModules;
         }
-        config.assessmentCategories =
-            allModules;
     }
     // Temporal/rug pull detection configuration
     if (options.temporalInvocations) {
@@ -922,6 +940,25 @@ function parseArgs() {
             case "--skip-temporal":
                 options.skipTemporal = true;
                 break;
+            case "--profile": {
+                const profileValue = args[++i];
+                if (!profileValue) {
+                    console.error("Error: --profile requires a profile name");
+                    console.error(`Valid profiles: ${Object.keys(ASSESSMENT_PROFILES).join(", ")}`);
+                    setTimeout(() => process.exit(1), 10);
+                    options.helpRequested = true;
+                    return options;
+                }
+                if (!isValidProfileName(profileValue)) {
+                    console.error(`Error: Invalid profile name: ${profileValue}`);
+                    console.error(`Valid profiles: ${Object.keys(ASSESSMENT_PROFILES).join(", ")}`);
+                    setTimeout(() => process.exit(1), 10);
+                    options.helpRequested = true;
+                    return options;
+                }
+                options.profile = profileValue;
+                break;
+            }
             case "--skip-modules": {
                 const skipValue = args[++i];
                 if (!skipValue) {
@@ -972,7 +1009,13 @@ function parseArgs() {
                 }
         }
     }
-    // Validate mutual exclusivity of --skip-modules and --only-modules
+    // Validate mutual exclusivity of --profile, --skip-modules, and --only-modules
+    if (options.profile && (options.skipModules?.length || options.onlyModules?.length)) {
+        console.error("Error: --profile cannot be used with --skip-modules or --only-modules");
+        setTimeout(() => process.exit(1), 10);
+        options.helpRequested = true;
+        return options;
+    }
     if (options.skipModules?.length && options.onlyModules?.length) {
         console.error("Error: --skip-modules and --only-modules are mutually exclusive");
         setTimeout(() => process.exit(1), 10);
@@ -995,7 +1038,7 @@ function printHelp() {
     console.log(`
 Usage: mcp-assess-full [options] [server-name]
 
-Run comprehensive MCP server assessment with all 17 assessor modules.
+Run comprehensive MCP server assessment with 16 assessor modules organized in 4 tiers.
 
 Options:
   --server, -s <name>    Server name (required, or pass as first positional arg)
@@ -1012,6 +1055,7 @@ Options:
   --no-resume            Force fresh start, clear any existing state
   --claude-enabled       Enable Claude Code integration for intelligent analysis
   --full                 Enable all assessment modules (default)
+  --profile <name>       Use predefined module profile (quick, security, compliance, full)
   --temporal-invocations <n>  Number of invocations per tool for rug pull detection (default: 25)
   --skip-temporal        Skip temporal/rug pull testing (faster assessment)
   --skip-modules <list>  Skip specific modules (comma-separated)
@@ -1023,47 +1067,65 @@ Options:
                          Also supports LOG_LEVEL environment variable
   --help, -h             Show this help message
 
+${getProfileHelpText()}
 Module Selection:
-  --skip-modules and --only-modules are mutually exclusive.
-  Use --skip-modules for faster runs by disabling expensive modules.
+  --profile, --skip-modules, and --only-modules are mutually exclusive.
+  Use --profile for common assessment scenarios.
+  Use --skip-modules for custom runs by disabling expensive modules.
   Use --only-modules to focus on specific areas (e.g., tool annotation PRs).
 
-  Valid module names:
-    functionality, security, documentation, errorHandling, usability,
-    mcpSpecCompliance, aupCompliance, toolAnnotations, prohibitedLibraries,
-    externalAPIScanner, authentication, temporal, resources, prompts,
-    crossCapability, manifestValidation, portability
+  Valid module names (new naming):
+    functionality, security, errorHandling, protocolCompliance, aupCompliance,
+    toolAnnotations, prohibitedLibraries, manifestValidation, authentication,
+    temporal, resources, prompts, crossCapability, developerExperience,
+    portability, externalAPIScanner
+
+  Legacy module names (deprecated, will map to new names):
+    documentation -> developerExperience
+    usability -> developerExperience
+    mcpSpecCompliance -> protocolCompliance
+    protocolConformance -> protocolCompliance
+
+Module Tiers (16 total):
+  Tier 1 - Core Security (Always Run):
+    • Functionality      - Tests all tools work correctly
+    • Security           - Prompt injection & vulnerability testing
+    • Error Handling     - Validates error responses
+    • Protocol Compliance - MCP protocol + JSON-RPC validation
+    • AUP Compliance     - Acceptable Use Policy checks
+    • Temporal           - Rug pull/temporal behavior change detection
 
-Assessment Modules (17 total):
-  • Functionality      - Tests all tools work correctly
-  • Security           - Prompt injection & vulnerability testing
-  • Documentation      - README completeness checks
-  • Error Handling     - Validates error responses
-  • Usability          - Input validation & UX
-  • MCP Spec           - Protocol compliance
-  • AUP Compliance     - Acceptable Use Policy checks
-  • Tool Annotations   - readOnlyHint/destructiveHint validation
-  • Prohibited Libs    - Dependency security checks
-  • External API       - External service detection
-  • Authentication     - OAuth/auth evaluation
-  • Temporal           - Rug pull/temporal behavior change detection
-  • Resources          - Resource capability assessment
-  • Prompts            - Prompt capability assessment
-  • Cross-Capability   - Chained vulnerability detection
-  • Manifest           - MCPB manifest.json validation (optional)
-  • Portability        - Cross-platform compatibility (optional)
+  Tier 2 - Compliance (MCP Directory):
+    • Tool Annotations   - readOnlyHint/destructiveHint validation
+    • Prohibited Libs    - Dependency security checks
+    • Manifest           - MCPB manifest.json validation
+    • Authentication     - OAuth/auth evaluation
+
+  Tier 3 - Capability-Based (Conditional):
+    • Resources          - Resource capability assessment
+    • Prompts            - Prompt capability assessment
+    • Cross-Capability   - Chained vulnerability detection
+
+  Tier 4 - Extended (Optional):
+    • Developer Experience - Documentation + usability assessment
+    • Portability        - Cross-platform compatibility
+    • External API       - External service detection
 
 Examples:
-  mcp-assess-full my-server
-  mcp-assess-full --server broken-mcp --claude-enabled
-  mcp-assess-full --server my-server --source ./my-server --output ./results.json
-  mcp-assess-full --server my-server --format markdown --include-policy
-  mcp-assess-full --server my-server --compare ./baseline.json
-  mcp-assess-full --server my-server --compare ./baseline.json --diff-only --format markdown
+  # Profile-based (recommended):
+  mcp-assess-full my-server --profile quick         # CI/CD fast check (~30s)
+  mcp-assess-full my-server --profile security      # Security audit (~2-3min)
+  mcp-assess-full my-server --profile compliance    # Directory submission (~5min)
+  mcp-assess-full my-server --profile full          # Comprehensive audit (~10-15min)
 
-  # Module selection examples:
-  mcp-assess-full my-server --skip-modules security,aupCompliance    # Fast CI run
+  # Custom module selection:
+  mcp-assess-full my-server --skip-modules temporal,resources  # Skip expensive modules
   mcp-assess-full my-server --only-modules functionality,toolAnnotations  # Annotation PR review
+
+  # Advanced options:
+  mcp-assess-full --server my-server --source ./my-server --output ./results.json
+  mcp-assess-full --server my-server --format markdown --include-policy
+  mcp-assess-full --server my-server --compare ./baseline.json --diff-only
   `);
 }
 /**

package/build/profiles.js

@@ -0,0 +1,286 @@
+/**
+ * Assessment Profiles
+ *
+ * Pre-configured module sets for common assessment scenarios.
+ * Profiles map to the 4-tier module organization:
+ *
+ * Tier 1: Core Security (Always Run)
+ *   - functionality, security, temporal, errorHandling, protocolCompliance, aupCompliance
+ *
+ * Tier 2: Compliance (MCP Directory)
+ *   - toolAnnotations, prohibitedLibraries, manifestValidation, authentication
+ *
+ * Tier 3: Capability-Based (Conditional)
+ *   - resources, prompts, crossCapability
+ *
+ * Tier 4: Extended (Optional)
+ *   - developerExperience, portability, externalAPIScanner
+ *
+ * @module cli/profiles
+ */
+/**
+ * Module alias mappings for backward compatibility.
+ * Maps deprecated module names to their replacements.
+ */
+export const MODULE_ALIASES = {
+    // Protocol compliance merge (MCPSpec + ProtocolConformance → ProtocolCompliance)
+    mcpSpecCompliance: "protocolCompliance",
+    protocolConformance: "protocolCompliance",
+    // Developer experience merge (Documentation + Usability → DeveloperExperience)
+    documentation: "developerExperience",
+    usability: "developerExperience",
+};
+/**
+ * Deprecated module names that trigger a warning when used.
+ */
+export const DEPRECATED_MODULES = new Set(Object.keys(MODULE_ALIASES));
+/**
+ * Tier 1: Core Security modules
+ * Essential for any MCP server assessment - security-focused
+ */
+export const TIER_1_CORE_SECURITY = [
+    "functionality",
+    "security",
+    "temporal",
+    "errorHandling",
+    "protocolCompliance",
+    "aupCompliance",
+];
+/**
+ * Tier 2: Compliance modules
+ * Required for MCP Directory submission compliance
+ */
+export const TIER_2_COMPLIANCE = [
+    "toolAnnotations",
+    "prohibitedLibraries",
+    "manifestValidation",
+    "authentication",
+];
+/**
+ * Tier 3: Capability-Based modules
+ * Only run when server has corresponding capabilities
+ */
+export const TIER_3_CAPABILITY = [
+    "resources",
+    "prompts",
+    "crossCapability",
+];
+/**
+ * Tier 4: Extended modules
+ * Optional assessments for comprehensive audits
+ */
+export const TIER_4_EXTENDED = [
+    "developerExperience",
+    "portability",
+    "externalAPIScanner",
+];
+/**
+ * All available modules (new naming)
+ */
+export const ALL_MODULES = [
+    ...TIER_1_CORE_SECURITY,
+    ...TIER_2_COMPLIANCE,
+    ...TIER_3_CAPABILITY,
+    ...TIER_4_EXTENDED,
+];
+/**
+ * Assessment profile definitions
+ * Each profile includes a specific set of modules optimized for the use case.
+ */
+export const ASSESSMENT_PROFILES = {
+    /**
+     * Quick profile: Minimal testing for fast CI/CD checks
+     * Use when: Pre-commit hooks, quick validation
+     * Time: ~30 seconds
+     */
+    quick: ["functionality", "security"],
+    /**
+     * Security profile: Core security modules (Tier 1)
+     * Use when: Security-focused audits, vulnerability scanning
+     * Time: ~2-3 minutes
+     */
+    security: [...TIER_1_CORE_SECURITY],
+    /**
+     * Compliance profile: Security + Directory compliance (Tier 1 + 2)
+     * Use when: Pre-submission validation for MCP Directory
+     * Time: ~5 minutes
+     */
+    compliance: [...TIER_1_CORE_SECURITY, ...TIER_2_COMPLIANCE],
+    /**
+     * Full profile: All modules (Tier 1 + 2 + 3 + 4)
+     * Use when: Comprehensive audits, initial server review
+     * Time: ~10-15 minutes
+     */
+    full: [
+        ...TIER_1_CORE_SECURITY,
+        ...TIER_2_COMPLIANCE,
+        ...TIER_3_CAPABILITY,
+        ...TIER_4_EXTENDED,
+    ],
+};
+export const PROFILE_METADATA = {
+    quick: {
+        description: "Fast validation (functionality + security only)",
+        estimatedTime: "~30 seconds",
+        moduleCount: ASSESSMENT_PROFILES.quick.length,
+        tiers: ["Tier 1 (partial)"],
+    },
+    security: {
+        description: "Core security modules for vulnerability scanning",
+        estimatedTime: "~2-3 minutes",
+        moduleCount: ASSESSMENT_PROFILES.security.length,
+        tiers: ["Tier 1 (Core Security)"],
+    },
+    compliance: {
+        description: "Security + MCP Directory compliance validation",
+        estimatedTime: "~5 minutes",
+        moduleCount: ASSESSMENT_PROFILES.compliance.length,
+        tiers: ["Tier 1 (Core Security)", "Tier 2 (Compliance)"],
+    },
+    full: {
+        description: "Comprehensive audit with all assessment modules",
+        estimatedTime: "~10-15 minutes",
+        moduleCount: ASSESSMENT_PROFILES.full.length,
+        tiers: [
+            "Tier 1 (Core Security)",
+            "Tier 2 (Compliance)",
+            "Tier 3 (Capability)",
+            "Tier 4 (Extended)",
+        ],
+    },
+};
+/**
+ * Resolve module names, applying aliases for deprecated names.
+ * Emits warnings for deprecated module usage.
+ *
+ * @param modules - Array of module names (may include deprecated names)
+ * @param warn - If true, emit console warnings for deprecated names
+ * @returns Array of resolved module names (with aliases applied)
+ */
+export function resolveModuleNames(modules, warn = true) {
+    const resolved = new Set();
+    for (const module of modules) {
+        if (DEPRECATED_MODULES.has(module)) {
+            const replacement = MODULE_ALIASES[module];
+            if (warn) {
+                console.warn(`Warning: Module '${module}' is deprecated, using '${replacement}' instead`);
+            }
+            resolved.add(replacement);
+        }
+        else {
+            resolved.add(module);
+        }
+    }
+    return Array.from(resolved);
+}
+/**
+ * Get modules for a profile, with optional source code awareness.
+ *
+ * @param profileName - Profile to get modules for
+ * @param options.hasSourceCode - If true, includes source-dependent modules
+ * @param options.skipTemporal - If true, excludes temporal module
+ * @returns Array of module names to run
+ */
+export function getProfileModules(profileName, options = {}) {
+    let modules = [...ASSESSMENT_PROFILES[profileName]];
+    // Filter out temporal if requested
+    if (options.skipTemporal) {
+        modules = modules.filter((m) => m !== "temporal");
+    }
+    // externalAPIScanner requires source code
+    if (!options.hasSourceCode) {
+        modules = modules.filter((m) => m !== "externalAPIScanner");
+    }
+    // prohibitedLibraries is most effective with source code
+    // (still runs but with limited capability)
+    return modules;
+}
+/**
+ * Validate a profile name.
+ *
+ * @param name - Profile name to validate
+ * @returns True if valid profile name
+ */
+export function isValidProfileName(name) {
+    return name in ASSESSMENT_PROFILES;
+}
+/**
+ * Get help text for profiles.
+ */
+export function getProfileHelpText() {
+    const lines = ["Assessment Profiles:", ""];
+    for (const [name, metadata] of Object.entries(PROFILE_METADATA)) {
+        lines.push(`  ${name.padEnd(12)} ${metadata.description}`);
+        lines.push(`               Modules: ${metadata.moduleCount}, Time: ${metadata.estimatedTime}`);
+        lines.push(`               Tiers: ${metadata.tiers.join(", ")}`);
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Map old module config keys to new profile-based module list.
+ * Used for backward compatibility with existing configs.
+ *
+ * @param oldConfig - Old-style assessmentCategories object
+ * @returns Array of enabled module names (new naming)
+ */
+export function mapLegacyConfigToModules(oldConfig) {
+    const enabled = [];
+    for (const [key, value] of Object.entries(oldConfig)) {
+        if (value === true) {
+            // Apply alias resolution for deprecated names
+            const resolved = MODULE_ALIASES[key] || key;
+            if (!enabled.includes(resolved)) {
+                enabled.push(resolved);
+            }
+        }
+    }
+    return enabled;
+}
+/**
+ * Convert new module list to old-style config object.
+ * Used for backward compatibility with existing orchestrator.
+ *
+ * @param modules - Array of module names (new naming)
+ * @returns Old-style assessmentCategories object
+ */
+export function modulesToLegacyConfig(modules) {
+    // Start with all modules disabled
+    const config = {
+        functionality: false,
+        security: false,
+        documentation: false,
+        errorHandling: false,
+        usability: false,
+        mcpSpecCompliance: false,
+        aupCompliance: false,
+        toolAnnotations: false,
+        prohibitedLibraries: false,
+        manifestValidation: false,
+        portability: false,
+        externalAPIScanner: false,
+        authentication: false,
+        temporal: false,
+        resources: false,
+        prompts: false,
+        crossCapability: false,
+        protocolConformance: false,
+    };
+    // Enable requested modules, mapping new names to old where needed
+    for (const module of modules) {
+        if (module === "protocolCompliance") {
+            // Map to both old modules for backward compat until orchestrator updated
+            config.mcpSpecCompliance = true;
+            config.protocolConformance = true;
+        }
+        else if (module === "developerExperience") {
+            // Map to both old modules for backward compat until orchestrator updated
+            config.documentation = true;
+            config.usability = true;
+        }
+        else if (module in config) {
+            config[module] = true;
+        }
+    }
+    return config;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bryan-thompson/inspector-assessment-cli",
-  "version": "1.24.2",
+  "version": "1.25.0",
   "description": "CLI for the Enhanced MCP Inspector with assessment capabilities",
   "license": "MIT",
   "author": "Bryan Thompson <bryan@triepod.ai>",