@bryan-thompson/inspector-assessment-client 1.6.0 → 1.7.1

package/lib/services/assessment/AssessmentOrchestrator.js

@@ -0,0 +1,418 @@
+/**
+ * Assessment Orchestrator
+ * Coordinates all assessment modules and manages the assessment workflow
+ */
+import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
+// Core assessment modules
+import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
+import { SecurityAssessor } from "./modules/SecurityAssessor.js";
+import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
+import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
+import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
+// Extended assessment modules
+import { MCPSpecComplianceAssessor } from "./modules/MCPSpecComplianceAssessor.js";
+// New MCP Directory Compliance Gap assessors
+import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
+import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
+import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
+import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
+import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
+// Claude Code integration for intelligent analysis
+import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
+import { TestDataGenerator } from "./TestDataGenerator.js";
+export class AssessmentOrchestrator {
+    config;
+    startTime = 0;
+    totalTestsRun = 0;
+    // Claude Code Bridge for intelligent analysis
+    claudeBridge;
+    claudeEnabled = false;
+    // Core assessors
+    functionalityAssessor;
+    securityAssessor;
+    documentationAssessor;
+    errorHandlingAssessor;
+    usabilityAssessor;
+    // Extended assessors
+    mcpSpecAssessor;
+    // New MCP Directory Compliance Gap assessors
+    aupComplianceAssessor;
+    toolAnnotationAssessor;
+    prohibitedLibrariesAssessor;
+    manifestValidationAssessor;
+    portabilityAssessor;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
+        // Initialize Claude Code Bridge if enabled in config
+        if (this.config.claudeCode?.enabled) {
+            this.initializeClaudeBridge(this.config.claudeCode);
+        }
+        // Initialize core assessors
+        this.functionalityAssessor = new FunctionalityAssessor(this.config);
+        this.securityAssessor = new SecurityAssessor(this.config);
+        this.documentationAssessor = new DocumentationAssessor(this.config);
+        this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
+        this.usabilityAssessor = new UsabilityAssessor(this.config);
+        // Initialize extended assessors if enabled
+        if (this.config.enableExtendedAssessment) {
+            if (this.config.assessmentCategories?.mcpSpecCompliance) {
+                this.mcpSpecAssessor = new MCPSpecComplianceAssessor(this.config);
+            }
+            // Initialize new MCP Directory Compliance Gap assessors
+            if (this.config.assessmentCategories?.aupCompliance) {
+                this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
+                // Wire up Claude bridge for semantic analysis
+                if (this.claudeBridge) {
+                    this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
+                }
+            }
+            if (this.config.assessmentCategories?.toolAnnotations) {
+                this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
+                // Wire up Claude bridge for behavior inference
+                if (this.claudeBridge) {
+                    this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
+                }
+            }
+            if (this.config.assessmentCategories?.prohibitedLibraries) {
+                this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
+            }
+            if (this.config.assessmentCategories?.manifestValidation) {
+                this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
+            }
+            if (this.config.assessmentCategories?.portability) {
+                this.portabilityAssessor = new PortabilityAssessor(this.config);
+            }
+        }
+        // Wire up Claude bridge to TestDataGenerator for intelligent test generation
+        if (this.claudeBridge) {
+            TestDataGenerator.setClaudeBridge(this.claudeBridge);
+        }
+    }
+    /**
+     * Initialize Claude Code Bridge for intelligent analysis
+     * This enables semantic AUP violation analysis, behavior inference, and intelligent test generation
+     */
+    initializeClaudeBridge(bridgeConfig) {
+        try {
+            this.claudeBridge = new ClaudeCodeBridge(bridgeConfig);
+            this.claudeEnabled = true;
+            console.log("[AssessmentOrchestrator] Claude Code Bridge initialized with features:", bridgeConfig.features);
+        }
+        catch (error) {
+            console.warn("[AssessmentOrchestrator] Failed to initialize Claude Code Bridge:", error);
+            this.claudeEnabled = false;
+        }
+    }
+    /**
+     * Enable Claude Code integration programmatically
+     * Call this method to enable Claude features after construction
+     */
+    enableClaudeCode(config) {
+        const bridgeConfig = {
+            ...FULL_CLAUDE_CODE_CONFIG,
+            ...config,
+            enabled: true,
+        };
+        this.initializeClaudeBridge(bridgeConfig);
+        // Wire up to existing assessors
+        if (this.claudeBridge) {
+            if (this.aupComplianceAssessor) {
+                this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
+            }
+            if (this.toolAnnotationAssessor) {
+                this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
+            }
+            TestDataGenerator.setClaudeBridge(this.claudeBridge);
+        }
+    }
+    /**
+     * Check if Claude Code integration is enabled and available
+     */
+    isClaudeEnabled() {
+        return this.claudeEnabled && this.claudeBridge !== undefined;
+    }
+    /**
+     * Get Claude Code Bridge for external access
+     */
+    getClaudeBridge() {
+        return this.claudeBridge;
+    }
+    /**
+     * Reset test counts for all assessors
+     */
+    resetAllTestCounts() {
+        this.functionalityAssessor.resetTestCount();
+        this.securityAssessor.resetTestCount();
+        this.documentationAssessor.resetTestCount();
+        this.errorHandlingAssessor.resetTestCount();
+        this.usabilityAssessor.resetTestCount();
+        if (this.mcpSpecAssessor) {
+            this.mcpSpecAssessor.resetTestCount();
+        }
+        // Reset new assessors
+        if (this.aupComplianceAssessor) {
+            this.aupComplianceAssessor.resetTestCount();
+        }
+        if (this.toolAnnotationAssessor) {
+            this.toolAnnotationAssessor.resetTestCount();
+        }
+        if (this.prohibitedLibrariesAssessor) {
+            this.prohibitedLibrariesAssessor.resetTestCount();
+        }
+        if (this.manifestValidationAssessor) {
+            this.manifestValidationAssessor.resetTestCount();
+        }
+        if (this.portabilityAssessor) {
+            this.portabilityAssessor.resetTestCount();
+        }
+    }
+    /**
+     * Run a complete assessment on an MCP server
+     */
+    async runFullAssessment(context) {
+        this.startTime = Date.now();
+        this.totalTestsRun = 0;
+        this.resetAllTestCounts();
+        // Run assessments in parallel if enabled
+        const assessmentPromises = [];
+        const assessmentResults = {};
+        if (this.config.parallelTesting) {
+            // Core assessments
+            assessmentPromises.push(this.functionalityAssessor
+                .assess(context)
+                .then((r) => (assessmentResults.functionality = r)), this.securityAssessor
+                .assess(context)
+                .then((r) => (assessmentResults.security = r)), this.documentationAssessor
+                .assess(context)
+                .then((r) => (assessmentResults.documentation = r)), this.errorHandlingAssessor
+                .assess(context)
+                .then((r) => (assessmentResults.errorHandling = r)), this.usabilityAssessor
+                .assess(context)
+                .then((r) => (assessmentResults.usability = r)));
+            // Extended assessments
+            if (this.mcpSpecAssessor) {
+                assessmentPromises.push(this.mcpSpecAssessor
+                    .assess(context)
+                    .then((r) => (assessmentResults.mcpSpecCompliance = r)));
+            }
+            // New MCP Directory Compliance Gap assessments
+            if (this.aupComplianceAssessor) {
+                assessmentPromises.push(this.aupComplianceAssessor
+                    .assess(context)
+                    .then((r) => (assessmentResults.aupCompliance = r)));
+            }
+            if (this.toolAnnotationAssessor) {
+                assessmentPromises.push(this.toolAnnotationAssessor
+                    .assess(context)
+                    .then((r) => (assessmentResults.toolAnnotations = r)));
+            }
+            if (this.prohibitedLibrariesAssessor) {
+                assessmentPromises.push(this.prohibitedLibrariesAssessor
+                    .assess(context)
+                    .then((r) => (assessmentResults.prohibitedLibraries = r)));
+            }
+            if (this.manifestValidationAssessor) {
+                assessmentPromises.push(this.manifestValidationAssessor
+                    .assess(context)
+                    .then((r) => (assessmentResults.manifestValidation = r)));
+            }
+            if (this.portabilityAssessor) {
+                assessmentPromises.push(this.portabilityAssessor
+                    .assess(context)
+                    .then((r) => (assessmentResults.portability = r)));
+            }
+            await Promise.all(assessmentPromises);
+        }
+        else {
+            // Sequential execution
+            assessmentResults.functionality =
+                await this.functionalityAssessor.assess(context);
+            assessmentResults.security = await this.securityAssessor.assess(context);
+            assessmentResults.documentation =
+                await this.documentationAssessor.assess(context);
+            assessmentResults.errorHandling =
+                await this.errorHandlingAssessor.assess(context);
+            assessmentResults.usability =
+                await this.usabilityAssessor.assess(context);
+            if (this.mcpSpecAssessor) {
+                assessmentResults.mcpSpecCompliance =
+                    await this.mcpSpecAssessor.assess(context);
+            }
+            // New MCP Directory Compliance Gap assessments (sequential)
+            if (this.aupComplianceAssessor) {
+                assessmentResults.aupCompliance =
+                    await this.aupComplianceAssessor.assess(context);
+            }
+            if (this.toolAnnotationAssessor) {
+                assessmentResults.toolAnnotations =
+                    await this.toolAnnotationAssessor.assess(context);
+            }
+            if (this.prohibitedLibrariesAssessor) {
+                assessmentResults.prohibitedLibraries =
+                    await this.prohibitedLibrariesAssessor.assess(context);
+            }
+            if (this.manifestValidationAssessor) {
+                assessmentResults.manifestValidation =
+                    await this.manifestValidationAssessor.assess(context);
+            }
+            if (this.portabilityAssessor) {
+                assessmentResults.portability =
+                    await this.portabilityAssessor.assess(context);
+            }
+        }
+        // Collect test counts from all assessors
+        this.totalTestsRun = this.collectTotalTestCount();
+        // Determine overall status
+        const overallStatus = this.determineOverallStatus(assessmentResults);
+        // Generate summary and recommendations
+        const summary = this.generateSummary(assessmentResults);
+        const recommendations = this.generateRecommendations(assessmentResults);
+        const executionTime = Date.now() - this.startTime;
+        return {
+            serverName: context.serverName,
+            assessmentDate: new Date().toISOString(),
+            assessorVersion: "2.0.0",
+            ...assessmentResults,
+            overallStatus,
+            summary,
+            recommendations,
+            executionTime,
+            totalTestsRun: this.totalTestsRun,
+            mcpProtocolVersion: this.config.mcpProtocolVersion,
+        };
+    }
+    /**
+     * Legacy assess method for backward compatibility
+     */
+    async assess(serverName, tools, callTool, serverInfo, readmeContent, packageJson) {
+        const context = {
+            serverName,
+            tools,
+            callTool,
+            readmeContent,
+            packageJson,
+            serverInfo,
+            config: this.config,
+        };
+        return this.runFullAssessment(context);
+    }
+    collectTotalTestCount() {
+        let total = 0;
+        // Get actual test counts from assessors
+        const functionalityCount = this.functionalityAssessor.getTestCount();
+        const securityCount = this.securityAssessor.getTestCount();
+        const documentationCount = this.documentationAssessor.getTestCount();
+        const errorHandlingCount = this.errorHandlingAssessor.getTestCount();
+        const usabilityCount = this.usabilityAssessor.getTestCount();
+        const mcpSpecCount = this.mcpSpecAssessor?.getTestCount() || 0;
+        // New assessor counts
+        const aupCount = this.aupComplianceAssessor?.getTestCount() || 0;
+        const annotationCount = this.toolAnnotationAssessor?.getTestCount() || 0;
+        const librariesCount = this.prohibitedLibrariesAssessor?.getTestCount() || 0;
+        const manifestCount = this.manifestValidationAssessor?.getTestCount() || 0;
+        const portabilityCount = this.portabilityAssessor?.getTestCount() || 0;
+        console.log("[AssessmentOrchestrator] Test counts by assessor:", {
+            functionality: functionalityCount,
+            security: securityCount,
+            documentation: documentationCount,
+            errorHandling: errorHandlingCount,
+            usability: usabilityCount,
+            mcpSpec: mcpSpecCount,
+            aupCompliance: aupCount,
+            toolAnnotations: annotationCount,
+            prohibitedLibraries: librariesCount,
+            manifestValidation: manifestCount,
+            portability: portabilityCount,
+        });
+        total =
+            functionalityCount +
+                securityCount +
+                documentationCount +
+                errorHandlingCount +
+                usabilityCount +
+                mcpSpecCount +
+                aupCount +
+                annotationCount +
+                librariesCount +
+                manifestCount +
+                portabilityCount;
+        console.log("[AssessmentOrchestrator] Total test count:", total);
+        return total;
+    }
+    determineOverallStatus(results) {
+        const statuses = [];
+        // Collect all statuses
+        Object.values(results).forEach((assessment) => {
+            if (assessment?.status) {
+                statuses.push(assessment.status);
+            }
+        });
+        // If any critical category fails, overall fails
+        if (statuses.includes("FAIL"))
+            return "FAIL";
+        // If any category needs more info, overall needs more info
+        if (statuses.includes("NEED_MORE_INFO"))
+            return "NEED_MORE_INFO";
+        // All must pass for overall pass
+        return "PASS";
+    }
+    generateSummary(results) {
+        const parts = [];
+        const totalCategories = Object.keys(results).length;
+        const passedCategories = Object.values(results).filter((r) => r?.status === "PASS").length;
+        parts.push(`Assessment complete: ${passedCategories}/${totalCategories} categories passed.`);
+        // Add key findings
+        if (results.security?.vulnerabilities?.length > 0) {
+            parts.push(`Found ${results.security.vulnerabilities.length} security vulnerabilities.`);
+        }
+        if (results.functionality?.brokenTools?.length > 0) {
+            parts.push(`${results.functionality.brokenTools.length} tools are not functioning correctly.`);
+        }
+        // New assessor findings
+        if (results.aupCompliance?.violations?.length > 0) {
+            const criticalCount = results.aupCompliance.violations.filter((v) => v.severity === "CRITICAL").length;
+            if (criticalCount > 0) {
+                parts.push(`CRITICAL: ${criticalCount} AUP violation(s) detected.`);
+            }
+            else {
+                parts.push(`${results.aupCompliance.violations.length} AUP item(s) flagged for review.`);
+            }
+        }
+        if (results.toolAnnotations?.missingAnnotationsCount > 0) {
+            parts.push(`${results.toolAnnotations.missingAnnotationsCount} tools missing annotations.`);
+        }
+        if (results.prohibitedLibraries?.matches?.length > 0) {
+            const blockingCount = results.prohibitedLibraries.matches.filter((m) => m.severity === "BLOCKING").length;
+            if (blockingCount > 0) {
+                parts.push(`BLOCKING: ${blockingCount} prohibited library/libraries detected.`);
+            }
+        }
+        if (results.portability?.usesBundleRoot) {
+            parts.push("Uses ${BUNDLE_ROOT} anti-pattern.");
+        }
+        return parts.join(" ");
+    }
+    generateRecommendations(results) {
+        const recommendations = [];
+        // Aggregate recommendations from all assessments
+        Object.values(results).forEach((assessment) => {
+            if (assessment?.recommendations) {
+                recommendations.push(...assessment.recommendations);
+            }
+        });
+        // De-duplicate and prioritize
+        return [...new Set(recommendations)].slice(0, 10);
+    }
+    /**
+     * Get assessment configuration
+     */
+    getConfig() {
+        return this.config;
+    }
+    /**
+     * Update assessment configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+    }
+}

package/lib/services/assessment/ResponseValidator.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Response Validator for MCP Tool Testing
+ * Validates that tool responses are actually functional, not just present
+ */
+import { CompatibilityCallToolResult, Tool } from "@modelcontextprotocol/sdk/types.js";
+export interface ValidationResult {
+    isValid: boolean;
+    isError: boolean;
+    confidence: number;
+    issues: string[];
+    evidence: string[];
+    classification: "fully_working" | "partially_working" | "connectivity_only" | "broken" | "error";
+}
+export interface ValidationContext {
+    tool: Tool;
+    input: Record<string, unknown>;
+    response: CompatibilityCallToolResult;
+    scenarioCategory?: "happy_path" | "edge_case" | "boundary" | "error_case";
+}
+export declare class ResponseValidator {
+    /**
+     * Validate a tool response comprehensively
+     */
+    static validateResponse(context: ValidationContext): ValidationResult;
+    /**
+     * Check if error is a business logic error (not a tool failure)
+     * These errors indicate the tool is working correctly but rejecting invalid business data
+     */
+    static isBusinessLogicError(context: ValidationContext): boolean;
+    /**
+     * Validate error responses are proper and informative
+     * NOTE: Currently unused - kept for potential future use
+     */
+    private static validateErrorResponse;
+    /**
+     * Validate response structure matches expectations
+     * NOTE: Currently unused - kept for potential future use
+     */
+    private static validateResponseStructure;
+    /**
+     * Validate response content is meaningful
+     * NOTE: Currently unused - kept for potential future use
+     */
+    private static validateResponseContent;
+    /**
+     * Validate semantic correctness based on input/output relationship
+     * NOTE: Currently unused - kept for potential future use
+     */
+    private static validateSemanticCorrectness;
+    /**
+     * Validate tool-specific logic and patterns
+     * NOTE: Currently unused - kept for potential future use
+     */
+    private static validateToolSpecificLogic;
+    /**
+     * Find query-like parameter in input
+     */
+    private static findQueryParameter;
+    /**
+     * Validate structured output against outputSchema (MCP 2025-06-18 feature)
+     * NOTE: Currently unused - kept for potential future use
+     */
+    private static validateStructuredOutput;
+    /**
+     * Calculate confidence score for a set of validation results
+     */
+    static calculateOverallConfidence(results: ValidationResult[]): number;
+}
+//# sourceMappingURL=ResponseValidator.d.ts.map

package/lib/services/assessment/ResponseValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ResponseValidator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/ResponseValidator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAE5C,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,EACV,eAAe,GACf,mBAAmB,GACnB,mBAAmB,GACnB,QAAQ,GACR,OAAO,CAAC;CACb;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,QAAQ,EAAE,2BAA2B,CAAC;IACtC,gBAAgB,CAAC,EAAE,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,CAAC;CAC3E;AAED,qBAAa,iBAAiB;IAC5B;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IA2ErE;;;OAGG;IACH,MAAM,CAAC,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO;IAmShE;;;OAGG;IAEH,OAAO,CAAC,MAAM,CAAC,qBAAqB;IA0CpC;;;OAGG;IAEH,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAoCxC;;;OAGG;IAEH,OAAO,CAAC,MAAM,CAAC,uBAAuB;IAsMtC;;;OAGG;IAEH,OAAO,CAAC,MAAM,CAAC,2BAA2B;IA6S1C;;;OAGG;IAEH,OAAO,CAAC,MAAM,CAAC,yBAAyB;IAwSxC;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IA4BjC;;;OAGG;IAEH,OAAO,CAAC,MAAM,CAAC,wBAAwB;IAkDvC;;OAEG;IACH,MAAM,CAAC,0BAA0B,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM;CAsBvE"}