@bryan-thompson/inspector-assessment-client 1.11.1 → 1.13.0

package/lib/services/assessment/config/annotationPatterns.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Tool Annotation Pattern Configuration
+ *
+ * Configurable pattern system for inferring expected tool behavior from names.
+ * Supports JSON configuration files for customization.
+ */
+/**
+ * Pattern configuration for tool behavior inference.
+ * Each category contains string patterns that are converted to RegExp at runtime.
+ * Patterns should end with underscore or hyphen (e.g., "get_", "delete-")
+ */
+export interface AnnotationPatternConfig {
+    /** Patterns indicating read-only operations (e.g., "get_", "list_", "fetch_") */
+    readOnly: string[];
+    /** Patterns indicating destructive operations (e.g., "delete_", "remove_", "destroy_") */
+    destructive: string[];
+    /** Patterns indicating write operations that are not destructive (e.g., "create_", "add_") */
+    write: string[];
+    /** Patterns that are semantically ambiguous - behavior varies by implementation */
+    ambiguous: string[];
+}
+/**
+ * Compiled patterns ready for matching.
+ * String patterns are converted to RegExp objects.
+ */
+export interface CompiledPatterns {
+    readOnly: RegExp[];
+    destructive: RegExp[];
+    write: RegExp[];
+    ambiguous: RegExp[];
+}
+/**
+ * Result of pattern matching with confidence scoring.
+ */
+export interface PatternMatchResult {
+    category: "readOnly" | "destructive" | "write" | "ambiguous" | "unknown";
+    pattern: string | null;
+    confidence: "high" | "medium" | "low";
+    isAmbiguous: boolean;
+}
+/**
+ * Default annotation patterns.
+ * These patterns have been validated against real-world MCP servers.
+ */
+export declare const DEFAULT_ANNOTATION_PATTERNS: AnnotationPatternConfig;
+/**
+ * Compile string patterns into RegExp objects for efficient matching.
+ */
+export declare function compilePatterns(config: AnnotationPatternConfig): CompiledPatterns;
+/**
+ * Load pattern configuration from a JSON file.
+ * Partial configs are merged with defaults.
+ *
+ * @param configPath - Path to JSON configuration file
+ * @returns Merged configuration with defaults
+ */
+export declare function loadPatternConfig(configPath?: string): AnnotationPatternConfig;
+/**
+ * Match a tool name against compiled patterns and return the result.
+ *
+ * @param toolName - The tool name to match
+ * @param patterns - Compiled pattern sets
+ * @returns Match result with category, confidence, and ambiguity flag
+ */
+export declare function matchToolPattern(toolName: string, patterns: CompiledPatterns): PatternMatchResult;
+/**
+ * Get compiled default patterns (cached for performance).
+ */
+export declare function getDefaultCompiledPatterns(): CompiledPatterns;
+//# sourceMappingURL=annotationPatterns.d.ts.map

package/lib/services/assessment/config/annotationPatterns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"annotationPatterns.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/config/annotationPatterns.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,iFAAiF;IACjF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,0FAA0F;IAC1F,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,8FAA8F;IAC9F,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,mFAAmF;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,WAAW,GAAG,SAAS,CAAC;IACzE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;;GAGG;AACH,eAAO,MAAM,2BAA2B,EAAE,uBAqKzC,CAAC;AAoBF;;GAEG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,uBAAuB,GAC9B,gBAAgB,CAOlB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,CAAC,EAAE,MAAM,GAClB,uBAAuB,CAyBzB;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,gBAAgB,GACzB,kBAAkB,CA0DpB;AAOD;;GAEG;AACH,wBAAgB,0BAA0B,IAAI,gBAAgB,CAK7D"}

package/lib/services/assessment/config/annotationPatterns.js

@@ -0,0 +1,305 @@
+/**
+ * Tool Annotation Pattern Configuration
+ *
+ * Configurable pattern system for inferring expected tool behavior from names.
+ * Supports JSON configuration files for customization.
+ */
+import * as fs from "fs";
+/**
+ * Default annotation patterns.
+ * These patterns have been validated against real-world MCP servers.
+ */
+export const DEFAULT_ANNOTATION_PATTERNS = {
+    readOnly: [
+        "get_",
+        "get-",
+        "list_",
+        "list-",
+        "fetch_",
+        "fetch-",
+        "read_",
+        "read-",
+        "query_",
+        "query-",
+        "search_",
+        "search-",
+        "find_",
+        "find-",
+        "show_",
+        "show-",
+        "view_",
+        "view-",
+        "describe_",
+        "describe-",
+        "check_",
+        "check-",
+        "verify_",
+        "verify-",
+        "validate_",
+        "validate-",
+        "count_",
+        "count-",
+        "status_",
+        "status-",
+        "info_",
+        "info-",
+        "lookup_",
+        "lookup-",
+        "browse_",
+        "browse-",
+        "preview_",
+        "preview-",
+        "download_",
+        "download-",
+    ],
+    destructive: [
+        "delete_",
+        "delete-",
+        "remove_",
+        "remove-",
+        "destroy_",
+        "destroy-",
+        "drop_",
+        "drop-",
+        "purge_",
+        "purge-",
+        "clear_",
+        "clear-",
+        "wipe_",
+        "wipe-",
+        "erase_",
+        "erase-",
+        "reset_",
+        "reset-",
+        "truncate_",
+        "truncate-",
+        "revoke_",
+        "revoke-",
+        "terminate_",
+        "terminate-",
+        "cancel_",
+        "cancel-",
+        "kill_",
+        "kill-",
+        "force_",
+        "force-",
+    ],
+    write: [
+        "create_",
+        "create-",
+        "add_",
+        "add-",
+        "insert_",
+        "insert-",
+        "update_",
+        "update-",
+        "modify_",
+        "modify-",
+        "edit_",
+        "edit-",
+        "change_",
+        "change-",
+        "set_",
+        "set-",
+        "put_",
+        "put-",
+        "patch_",
+        "patch-",
+        "post_",
+        "post-",
+        "write_",
+        "write-",
+        "upload_",
+        "upload-",
+        "send_",
+        "send-",
+        "submit_",
+        "submit-",
+        "publish_",
+        "publish-",
+        "enable_",
+        "enable-",
+        "disable_",
+        "disable-",
+        "start_",
+        "start-",
+        "stop_",
+        "stop-",
+        "run_",
+        "run-",
+        "execute_",
+        "execute-",
+    ],
+    ambiguous: [
+        "store_",
+        "store-",
+        "queue_",
+        "queue-",
+        "cache_",
+        "cache-",
+        "process_",
+        "process-",
+        "handle_",
+        "handle-",
+        "manage_",
+        "manage-",
+        "sync_",
+        "sync-",
+        "transfer_",
+        "transfer-",
+        "push_",
+        "push-",
+        "pop_",
+        "pop-",
+        "apply_",
+        "apply-",
+        "compute_",
+        "compute-",
+        "calculate_",
+        "calculate-",
+        "transform_",
+        "transform-",
+        "convert_",
+        "convert-",
+        "evaluate_",
+        "evaluate-",
+        "log_",
+        "log-",
+        "record_",
+        "record-",
+        "track_",
+        "track-",
+        "register_",
+        "register-",
+        "save_",
+        "save-",
+    ],
+};
+/**
+ * Convert a string pattern to a RegExp.
+ * Handles patterns like "get_" -> /^get[_-]?/i
+ */
+function patternToRegex(pattern) {
+    // Remove trailing underscore/hyphen for the base pattern
+    const base = pattern.replace(/[_-]$/, "");
+    // Create regex that matches pattern at start of string, with optional underscore/hyphen
+    return new RegExp(`^${escapeRegex(base)}[_-]?`, "i");
+}
+/**
+ * Escape special regex characters in a string.
+ */
+function escapeRegex(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Compile string patterns into RegExp objects for efficient matching.
+ */
+export function compilePatterns(config) {
+    return {
+        readOnly: config.readOnly.map(patternToRegex),
+        destructive: config.destructive.map(patternToRegex),
+        write: config.write.map(patternToRegex),
+        ambiguous: config.ambiguous.map(patternToRegex),
+    };
+}
+/**
+ * Load pattern configuration from a JSON file.
+ * Partial configs are merged with defaults.
+ *
+ * @param configPath - Path to JSON configuration file
+ * @returns Merged configuration with defaults
+ */
+export function loadPatternConfig(configPath) {
+    if (!configPath) {
+        return DEFAULT_ANNOTATION_PATTERNS;
+    }
+    try {
+        const configContent = fs.readFileSync(configPath, "utf-8");
+        const userConfig = JSON.parse(configContent);
+        // Merge with defaults - user config overrides defaults for specified categories
+        return {
+            readOnly: userConfig.readOnly ?? DEFAULT_ANNOTATION_PATTERNS.readOnly,
+            destructive: userConfig.destructive ?? DEFAULT_ANNOTATION_PATTERNS.destructive,
+            write: userConfig.write ?? DEFAULT_ANNOTATION_PATTERNS.write,
+            ambiguous: userConfig.ambiguous ?? DEFAULT_ANNOTATION_PATTERNS.ambiguous,
+        };
+    }
+    catch (error) {
+        console.warn(`Warning: Could not load pattern config from ${configPath}, using defaults`);
+        return DEFAULT_ANNOTATION_PATTERNS;
+    }
+}
+/**
+ * Match a tool name against compiled patterns and return the result.
+ *
+ * @param toolName - The tool name to match
+ * @param patterns - Compiled pattern sets
+ * @returns Match result with category, confidence, and ambiguity flag
+ */
+export function matchToolPattern(toolName, patterns) {
+    const lowerName = toolName.toLowerCase();
+    // Check ambiguous patterns FIRST (highest priority for this feature)
+    for (const pattern of patterns.ambiguous) {
+        if (pattern.test(lowerName)) {
+            return {
+                category: "ambiguous",
+                pattern: pattern.source,
+                confidence: "low",
+                isAmbiguous: true,
+            };
+        }
+    }
+    // Check destructive patterns (high confidence)
+    for (const pattern of patterns.destructive) {
+        if (pattern.test(lowerName)) {
+            return {
+                category: "destructive",
+                pattern: pattern.source,
+                confidence: "high",
+                isAmbiguous: false,
+            };
+        }
+    }
+    // Check read-only patterns (high confidence)
+    for (const pattern of patterns.readOnly) {
+        if (pattern.test(lowerName)) {
+            return {
+                category: "readOnly",
+                pattern: pattern.source,
+                confidence: "high",
+                isAmbiguous: false,
+            };
+        }
+    }
+    // Check write patterns (medium confidence)
+    for (const pattern of patterns.write) {
+        if (pattern.test(lowerName)) {
+            return {
+                category: "write",
+                pattern: pattern.source,
+                confidence: "medium",
+                isAmbiguous: false,
+            };
+        }
+    }
+    // No pattern match
+    return {
+        category: "unknown",
+        pattern: null,
+        confidence: "low",
+        isAmbiguous: true,
+    };
+}
+/**
+ * Singleton instance of compiled default patterns for performance.
+ */
+let defaultCompiledPatterns = null;
+/**
+ * Get compiled default patterns (cached for performance).
+ */
+export function getDefaultCompiledPatterns() {
+    if (!defaultCompiledPatterns) {
+        defaultCompiledPatterns = compilePatterns(DEFAULT_ANNOTATION_PATTERNS);
+    }
+    return defaultCompiledPatterns;
+}

package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts

@@ -12,8 +12,9 @@
  */
 import { BaseAssessor } from "./BaseAssessor.js";
 import { AssessmentContext } from "../AssessmentOrchestrator.js";
-import type { ToolAnnotationAssessment, ToolAnnotationResult } from "../../../lib/assessmentTypes.js";
+import type { ToolAnnotationAssessment, ToolAnnotationResult, AssessmentConfiguration } from "../../../lib/assessmentTypes.js";
 import type { ClaudeCodeBridge } from "../lib/claudeCodeBridge.js";
+import { type CompiledPatterns } from "../config/annotationPatterns.js";
 /**
  * Enhanced tool annotation result with Claude inference
  */
@@ -43,6 +44,12 @@ export interface EnhancedToolAnnotationAssessment extends ToolAnnotationAssessme
 }
 export declare class ToolAnnotationAssessor extends BaseAssessor {
     private claudeBridge?;
+    private compiledPatterns;
+    constructor(config: AssessmentConfiguration);
+    /**
+     * Set custom compiled patterns for behavior inference
+     */
+    setPatterns(patterns: CompiledPatterns): void;
     /**
      * Set Claude Code Bridge for enhanced behavior inference
      */
@@ -69,11 +76,18 @@ export declare class ToolAnnotationAssessor extends BaseAssessor {
     private generateEnhancedRecommendations;
     /**
      * Assess a single tool's annotations
+     * Now includes alignment status with confidence-aware logic
      */
     private assessTool;
     /**
      * Extract annotations from a tool
      * MCP SDK may have annotations in different locations
+     *
+     * Priority order:
+     * 1. tool.annotations (MCP 2024-11 spec) - "mcp" source
+     * 2. Direct properties on tool - "mcp" source
+     * 3. tool.metadata - "mcp" source
+     * 4. No annotations found - "none" source
      */
     private extractAnnotations;
     /**
@@ -82,12 +96,18 @@ export declare class ToolAnnotationAssessor extends BaseAssessor {
     private extractToolParams;
     /**
      * Infer expected behavior from tool name and description
+     * Now returns confidence level and ambiguity flag for better handling
      */
     private inferBehavior;
     /**
-     * Determine overall status
+     * Determine overall status using alignment status.
+     * Only MISALIGNED counts as failure; REVIEW_RECOMMENDED does not fail.
      */
     private determineAnnotationStatus;
+    /**
+     * Calculate metrics and alignment breakdown for the assessment
+     */
+    private calculateMetrics;
     /**
      * Generate explanation
      */

package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAGrB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,oBAAoB;IACxE,eAAe,CAAC,EAAE;QAChB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,oBAAoB,EAAE;YACpB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,eAAe,CAAC,EAAE,OAAO,CAAC;YAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;SAC1B,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;KAC7C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gCAAiC,SAAQ,wBAAwB;IAChF,WAAW,EAAE,4BAA4B,EAAE,CAAC;IAC5C,cAAc,EAAE,OAAO,CAAC;IACxB,2BAA2B,EAAE,4BAA4B,EAAE,CAAC;CAC7D;AAwED,qBAAa,sBAAuB,SAAQ,YAAY;IACtD,OAAO,CAAC,YAAY,CAAC,CAAmB;IAExC;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAK/C;;OAEG;IACH,eAAe,IAAI,OAAO;IAO1B;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,wBAAwB,GAAG,gCAAgC,CAAC;IAsMvE;;OAEG;YACW,0BAA0B;IA+IxC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAiCnC;;OAEG;IACH,OAAO,CAAC,+BAA+B;IAoFvC;;OAEG;IACH,OAAO,CAAC,UAAU;IAqElB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAyC1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;OAEG;IACH,OAAO,CAAC,aAAa;IA0ErB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAsCjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA2ChC"}
+{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAKpB,uBAAuB,EAExB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EACL,KAAK,gBAAgB,EAGtB,MAAM,8BAA8B,CAAC;AAEtC;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,oBAAoB;IACxE,eAAe,CAAC,EAAE;QAChB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,oBAAoB,EAAE;YACpB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,eAAe,CAAC,EAAE,OAAO,CAAC;YAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;SAC1B,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;KAC7C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gCAAiC,SAAQ,wBAAwB;IAChF,WAAW,EAAE,4BAA4B,EAAE,CAAC;IAC5C,cAAc,EAAE,OAAO,CAAC;IACxB,2BAA2B,EAAE,4BAA4B,EAAE,CAAC;CAC7D;AAKD,qBAAa,sBAAuB,SAAQ,YAAY;IACtD,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,gBAAgB,CAAmB;gBAE/B,MAAM,EAAE,uBAAuB;IAM3C;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAK7C;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAK/C;;OAEG;IACH,eAAe,IAAI,OAAO;IAO1B;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,wBAAwB,GAAG,gCAAgC,CAAC;IA8QvE;;OAEG;YACW,0BAA0B;IA+IxC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAiCnC;;OAEG;IACH,OAAO,CAAC,+BAA+B;IAoFvC;;;OAGG;IACH,OAAO,CAAC,UAAU;IA+GlB;;;;;;;;;OASG;IACH,OAAO,CAAC,kBAAkB;IAyE1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAgGrB;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAkDjC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiDxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA2ChC"}