@bryan-thompson/inspector-assessment-client 1.31.0 → 1.32.0

package/lib/services/assessment/registry/AssessorDefinitions.js

@@ -0,0 +1,370 @@
+/**
+ * Assessor Definitions
+ *
+ * Declarative configuration for all 19 assessors.
+ * Single source of truth for assessor registration, phase ordering,
+ * config flags, and Claude bridge support.
+ *
+ * @module assessment/registry/AssessorDefinitions
+ * @see GitHub Issue #91
+ */
+import { AssessmentPhase } from "./types.js";
+// Core assessor imports
+import { FunctionalityAssessor } from "../modules/FunctionalityAssessor.js";
+import { SecurityAssessor } from "../modules/SecurityAssessor.js";
+import { DocumentationAssessor } from "../modules/DocumentationAssessor.js";
+import { ErrorHandlingAssessor } from "../modules/ErrorHandlingAssessor.js";
+import { UsabilityAssessor } from "../modules/UsabilityAssessor.js";
+// Protocol compliance
+import { ProtocolComplianceAssessor } from "../modules/ProtocolComplianceAssessor.js";
+// MCP Directory compliance gap assessors
+import { AUPComplianceAssessor } from "../modules/AUPComplianceAssessor.js";
+import { ToolAnnotationAssessor } from "../modules/ToolAnnotationAssessor.js";
+import { ProhibitedLibrariesAssessor } from "../modules/ProhibitedLibrariesAssessor.js";
+import { ManifestValidationAssessor } from "../modules/ManifestValidationAssessor.js";
+import { PortabilityAssessor } from "../modules/PortabilityAssessor.js";
+import { ExternalAPIScannerAssessor } from "../modules/ExternalAPIScannerAssessor.js";
+import { TemporalAssessor } from "../modules/TemporalAssessor.js";
+import { AuthenticationAssessor } from "../modules/AuthenticationAssessor.js";
+// Capability assessors
+import { ResourceAssessor } from "../modules/ResourceAssessor.js";
+import { PromptAssessor } from "../modules/PromptAssessor.js";
+import { CrossCapabilitySecurityAssessor } from "../modules/CrossCapabilitySecurityAssessor.js";
+// Code quality assessors
+import { FileModularizationAssessor } from "../modules/FileModularizationAssessor.js";
+import { ConformanceAssessor } from "../modules/ConformanceAssessor.js";
+// Pattern configuration for ToolAnnotationAssessor
+import { loadPatternConfig, compilePatterns, } from "../config/annotationPatterns.js";
+// Test estimators
+import { estimateTemporalTests, estimateFunctionalityTests, estimateSecurityTests, estimateDocumentationTests, estimateErrorHandlingTests, estimateUsabilityTests, estimateProtocolComplianceTests, estimateAUPComplianceTests, estimateToolAnnotationTests, estimateProhibitedLibrariesTests, estimateManifestValidationTests, estimatePortabilityTests, estimateExternalAPIScannerTests, estimateAuthenticationTests, estimateResourceTests, estimatePromptTests, estimateCrossCapabilityTests, estimateFileModularizationTests, estimateConformanceTests, } from "./estimators.js";
+/**
+ * All assessor definitions in phase order.
+ * This is the single source of truth for assessor registration.
+ *
+ * Order within phases:
+ * - Phase 0 (PRE): Temporal only
+ * - Phase 1 (CORE): Functionality, Security, Documentation, ErrorHandling, Usability
+ * - Phase 2 (PROTOCOL): ProtocolCompliance
+ * - Phase 3 (COMPLIANCE): AUP, Annotations, Libraries, Manifest, Portability, APIs, Auth
+ * - Phase 4 (CAPABILITY): Resources, Prompts, CrossCapability
+ * - Phase 5 (QUALITY): FileModularization, Conformance
+ */
+export const ASSESSOR_DEFINITIONS = [
+    // ============================================================================
+    // Phase 0: PRE - Temporal (must run first for clean baseline)
+    // ============================================================================
+    {
+        id: "temporal",
+        displayName: "Temporal",
+        assessorClass: TemporalAssessor,
+        resultField: "temporal",
+        phase: AssessmentPhase.PRE,
+        configFlags: {
+            primary: "temporal",
+            defaultEnabled: false, // Opt-in
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateTemporalTests,
+    },
+    // ============================================================================
+    // Phase 1: CORE - The original 5 assessors
+    // ============================================================================
+    {
+        id: "functionality",
+        displayName: "Functionality",
+        assessorClass: FunctionalityAssessor,
+        resultField: "functionality",
+        phase: AssessmentPhase.CORE,
+        configFlags: {
+            primary: "functionality",
+            defaultEnabled: true, // Enabled unless explicitly disabled
+        },
+        requiresExtended: false,
+        supportsClaudeBridge: false,
+        estimateTests: estimateFunctionalityTests,
+    },
+    {
+        id: "security",
+        displayName: "Security",
+        assessorClass: SecurityAssessor,
+        resultField: "security",
+        phase: AssessmentPhase.CORE,
+        configFlags: {
+            primary: "security",
+            defaultEnabled: true,
+        },
+        requiresExtended: false,
+        supportsClaudeBridge: true, // Supports Claude semantic analysis
+        estimateTests: estimateSecurityTests,
+    },
+    {
+        id: "documentation",
+        displayName: "Documentation",
+        assessorClass: DocumentationAssessor,
+        resultField: "documentation",
+        phase: AssessmentPhase.CORE,
+        configFlags: {
+            primary: "documentation",
+            defaultEnabled: true,
+        },
+        requiresExtended: false,
+        supportsClaudeBridge: false,
+        estimateTests: estimateDocumentationTests,
+    },
+    {
+        id: "errorHandling",
+        displayName: "Error Handling",
+        assessorClass: ErrorHandlingAssessor,
+        resultField: "errorHandling",
+        phase: AssessmentPhase.CORE,
+        configFlags: {
+            primary: "errorHandling",
+            defaultEnabled: true,
+        },
+        requiresExtended: false,
+        supportsClaudeBridge: false,
+        estimateTests: estimateErrorHandlingTests,
+    },
+    {
+        id: "usability",
+        displayName: "Usability",
+        assessorClass: UsabilityAssessor,
+        resultField: "usability",
+        phase: AssessmentPhase.CORE,
+        configFlags: {
+            primary: "usability",
+            defaultEnabled: true,
+        },
+        requiresExtended: false,
+        supportsClaudeBridge: false,
+        estimateTests: estimateUsabilityTests,
+    },
+    // ============================================================================
+    // Phase 2: PROTOCOL - Unified protocol compliance
+    // ============================================================================
+    {
+        id: "protocolCompliance",
+        displayName: "Protocol Compliance",
+        assessorClass: ProtocolComplianceAssessor,
+        resultField: "mcpSpecCompliance", // Legacy field name for BC
+        phase: AssessmentPhase.PROTOCOL,
+        configFlags: {
+            primary: "protocolCompliance",
+            // BC: Enable if ANY of these deprecated flags is true
+            deprecated: ["mcpSpecCompliance", "protocolConformance"],
+            defaultEnabled: false, // Opt-in
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateProtocolComplianceTests,
+    },
+    // ============================================================================
+    // Phase 3: COMPLIANCE - MCP Directory compliance gap assessors
+    // ============================================================================
+    {
+        id: "aupCompliance",
+        displayName: "AUP",
+        assessorClass: AUPComplianceAssessor,
+        resultField: "aupCompliance",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "aupCompliance",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: true, // Supports Claude semantic analysis
+        estimateTests: estimateAUPComplianceTests,
+    },
+    {
+        id: "toolAnnotations",
+        displayName: "Annotations",
+        assessorClass: ToolAnnotationAssessor,
+        resultField: "toolAnnotations",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "toolAnnotations",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: true, // Supports Claude behavior inference
+        estimateTests: estimateToolAnnotationTests,
+        // Custom setup for pattern configuration
+        customSetup: (assessor, config, logger) => {
+            if (config.patternConfigPath) {
+                const patternConfig = loadPatternConfig(config.patternConfigPath, logger);
+                const compiledPatterns = compilePatterns(patternConfig);
+                assessor.setPatterns(compiledPatterns);
+            }
+        },
+    },
+    {
+        id: "prohibitedLibraries",
+        displayName: "Libraries",
+        assessorClass: ProhibitedLibrariesAssessor,
+        resultField: "prohibitedLibraries",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "prohibitedLibraries",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateProhibitedLibrariesTests,
+    },
+    {
+        id: "manifestValidation",
+        displayName: "Manifest",
+        assessorClass: ManifestValidationAssessor,
+        resultField: "manifestValidation",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "manifestValidation",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateManifestValidationTests,
+    },
+    {
+        id: "portability",
+        displayName: "Portability",
+        assessorClass: PortabilityAssessor,
+        resultField: "portability",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "portability",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimatePortabilityTests,
+    },
+    {
+        id: "externalAPIScanner",
+        displayName: "External APIs",
+        assessorClass: ExternalAPIScannerAssessor,
+        resultField: "externalAPIScanner",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "externalAPIScanner",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateExternalAPIScannerTests,
+    },
+    {
+        id: "authentication",
+        displayName: "Authentication",
+        assessorClass: AuthenticationAssessor,
+        resultField: "authentication",
+        phase: AssessmentPhase.COMPLIANCE,
+        configFlags: {
+            primary: "authentication",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateAuthenticationTests,
+    },
+    // ============================================================================
+    // Phase 4: CAPABILITY - Resources, Prompts, Cross-capability
+    // ============================================================================
+    {
+        id: "resources",
+        displayName: "Resources",
+        assessorClass: ResourceAssessor,
+        resultField: "resources",
+        phase: AssessmentPhase.CAPABILITY,
+        configFlags: {
+            primary: "resources",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateResourceTests,
+    },
+    {
+        id: "prompts",
+        displayName: "Prompts",
+        assessorClass: PromptAssessor,
+        resultField: "prompts",
+        phase: AssessmentPhase.CAPABILITY,
+        configFlags: {
+            primary: "prompts",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimatePromptTests,
+    },
+    {
+        id: "crossCapability",
+        displayName: "Cross-Capability",
+        assessorClass: CrossCapabilitySecurityAssessor,
+        resultField: "crossCapability",
+        phase: AssessmentPhase.CAPABILITY,
+        configFlags: {
+            primary: "crossCapability",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateCrossCapabilityTests,
+    },
+    // ============================================================================
+    // Phase 5: QUALITY - Code quality and conformance
+    // ============================================================================
+    {
+        id: "fileModularization",
+        displayName: "File Modularization",
+        assessorClass: FileModularizationAssessor,
+        resultField: "fileModularization",
+        phase: AssessmentPhase.QUALITY,
+        configFlags: {
+            primary: "fileModularization",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateFileModularizationTests,
+    },
+    {
+        id: "conformance",
+        displayName: "Conformance",
+        assessorClass: ConformanceAssessor,
+        resultField: "conformance",
+        phase: AssessmentPhase.QUALITY,
+        configFlags: {
+            primary: "conformance",
+            defaultEnabled: false,
+        },
+        requiresExtended: true,
+        supportsClaudeBridge: false,
+        estimateTests: estimateConformanceTests,
+    },
+];
+/**
+ * Map of assessor ID to definition for fast lookup.
+ */
+export const ASSESSOR_DEFINITION_MAP = new Map(ASSESSOR_DEFINITIONS.map((def) => [def.id, def]));
+/**
+ * Get assessor definitions by phase.
+ */
+export function getDefinitionsByPhase(phase) {
+    return ASSESSOR_DEFINITIONS.filter((def) => def.phase === phase);
+}
+/**
+ * Get all phases in execution order.
+ */
+export function getOrderedPhases() {
+    return [
+        AssessmentPhase.PRE,
+        AssessmentPhase.CORE,
+        AssessmentPhase.PROTOCOL,
+        AssessmentPhase.COMPLIANCE,
+        AssessmentPhase.CAPABILITY,
+        AssessmentPhase.QUALITY,
+    ];
+}

package/lib/services/assessment/registry/AssessorRegistry.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Assessor Registry
+ *
+ * Central registry for managing assessor instances with metadata.
+ * Provides phase-ordered execution, Claude bridge wiring, and test count aggregation.
+ *
+ * @module assessment/registry/AssessorRegistry
+ * @see GitHub Issue #91
+ */
+import type { AssessmentConfiguration } from "../../../lib/assessment/configTypes.js";
+import type { MCPDirectoryAssessment } from "../../../lib/assessment/resultTypes.js";
+import type { BaseAssessor } from "../modules/BaseAssessor.js";
+import type { AssessmentContext } from "../AssessmentOrchestrator.js";
+import type { ClaudeCodeBridge } from "../lib/claudeCodeBridge.js";
+import { type AssessorDefinition, type RegisteredAssessor, AssessmentPhase } from "./types.js";
+/**
+ * AssessorRegistry manages assessor instances and their execution.
+ *
+ * Key responsibilities:
+ * 1. Lazy instantiation based on configuration flags
+ * 2. Phase-ordered execution with Phase 0 always first and sequential
+ * 3. Claude bridge wiring to supporting assessors
+ * 4. Test count aggregation from all assessors
+ * 5. Backward-compatible property access via getAssessor()
+ */
+export declare class AssessorRegistry {
+    private config;
+    private logger;
+    private assessors;
+    private claudeBridge?;
+    constructor(config: AssessmentConfiguration);
+    /**
+     * Register all enabled assessors based on configuration.
+     * Called during orchestrator initialization.
+     */
+    registerAll(definitions?: AssessorDefinition[]): void;
+    /**
+     * Register a single assessor.
+     */
+    private register;
+    /**
+     * Check if an assessor should be enabled based on configuration.
+     * Implements OR logic for backward-compatible deprecated flags.
+     */
+    isEnabled(definition: AssessorDefinition): boolean;
+    /**
+     * Check a single config flag value.
+     * If defaultEnabled is true, returns true unless flag is explicitly false.
+     * If defaultEnabled is false, returns true only if flag is explicitly true.
+     */
+    private checkFlag;
+    /**
+     * Get a registered assessor by ID.
+     * Returns undefined if not registered or disabled.
+     */
+    getAssessor<T extends BaseAssessor = BaseAssessor>(id: string): T | undefined;
+    /**
+     * Get all registered assessors for a specific phase.
+     */
+    getByPhase(phase: AssessmentPhase): RegisteredAssessor[];
+    /**
+     * Set Claude bridge for all supporting assessors.
+     * Called when Claude Code is enabled.
+     */
+    setClaudeBridge(bridge: ClaudeCodeBridge): void;
+    /**
+     * Wire Claude bridge to a single assessor.
+     */
+    private wireClaudeBridge;
+    /**
+     * Execute all assessors in phase order.
+     * Phase 0 (PRE) always runs first and sequentially.
+     * Other phases run based on parallelTesting config.
+     *
+     * @returns Partial MCPDirectoryAssessment with results from all assessors
+     */
+    executeAll(context: AssessmentContext): Promise<Partial<MCPDirectoryAssessment>>;
+    /**
+     * Execute all assessors in a single phase.
+     */
+    private executePhase;
+    /**
+     * Execute assessors in parallel.
+     */
+    private executeParallel;
+    /**
+     * Execute assessors sequentially.
+     */
+    private executeSequential;
+    /**
+     * Extract status string from assessment result.
+     * Most results have a 'status' property.
+     */
+    private extractStatus;
+    /**
+     * Get tool count from context (respects selectedToolsForTesting config).
+     */
+    private getToolCountForContext;
+    /**
+     * Get total test count from all registered assessors.
+     */
+    getTotalTestCount(): number;
+    /**
+     * Get test count for a specific assessor.
+     */
+    getTestCount(id: string): number;
+    /**
+     * Get all registered assessor IDs.
+     */
+    getRegisteredIds(): string[];
+    /**
+     * Check if a specific assessor is registered.
+     */
+    isRegistered(id: string): boolean;
+    /**
+     * Get the count of registered assessors.
+     */
+    get size(): number;
+    /**
+     * Update configuration (e.g., for dynamic config changes).
+     */
+    updateConfig(config: AssessmentConfiguration): void;
+}
+//# sourceMappingURL=AssessorRegistry.d.ts.map

package/lib/services/assessment/registry/AssessorRegistry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AssessorRegistry.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/AssessorRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EAEvB,eAAe,EAEhB,MAAM,SAAS,CAAC;AAOjB;;;;;;;;;GASG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,YAAY,CAAC,CAAmB;gBAE5B,MAAM,EAAE,uBAAuB;IAQ3C;;;OAGG;IACH,WAAW,CAAC,WAAW,GAAE,kBAAkB,EAAyB,GAAG,IAAI;IAc3E;;OAEG;IACH,OAAO,CAAC,QAAQ;IA8BhB;;;OAGG;IACH,SAAS,CAAC,UAAU,EAAE,kBAAkB,GAAG,OAAO;IAmClD;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAajB;;;OAGG;IACH,WAAW,CAAC,CAAC,SAAS,YAAY,GAAG,YAAY,EAC/C,EAAE,EAAE,MAAM,GACT,CAAC,GAAG,SAAS;IAKhB;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,kBAAkB,EAAE;IAMxD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAU/C;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;;;;OAMG;IACG,UAAU,CACd,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IA8B3C;;OAEG;YACW,YAAY;IAoB1B;;OAEG;YACW,eAAe;IA+C7B;;OAEG;YACW,iBAAiB;IAiD/B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAOrB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAS9B;;OAEG;IACH,iBAAiB,IAAI,MAAM;IAQ3B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAKhC;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;IAI5B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIjC;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;OAEG;IACH,YAAY,CAAC,MAAM,EAAE,uBAAuB,GAAG,IAAI;CAGpD"}