@bryan-thompson/inspector-assessment 1.37.0 → 1.38.0

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js

@@ -365,11 +365,15 @@ export class ErrorHandlingAssessor extends BaseAssessor {
     }
     async testInvalidValues(tool, callTool, isExternalAPI = false) {
         const schema = this.getToolSchema(tool);
-        const testInput = this.generateInvalidValueParams(schema);
+        // Issue #173: Destructure metadata from new return type
+        const { params: testInput, testedParameter, parameterIsRequired, } = this.generateInvalidValueParams(schema);
         try {
             const response = await this.executeWithTimeout(callTool(tool.name, testInput), 5000);
             const isError = this.isErrorResponse(response);
             const errorInfo = this.extractErrorInfo(response);
+            const responseText = this.extractResponseTextSafe(response);
+            // Issue #173: Detect suggestions in response
+            const { hasSuggestions, suggestions } = this.detectSuggestionPatterns(responseText);
             // Issue #168: For external API tools, check if error is an external service error
             if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
                 return {
@@ -385,6 +389,11 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     },
                     passed: true,
                     reason: "External API service error (validation cannot be tested when service unavailable)",
+                    // Issue #173 metadata
+                    testedParameter,
+                    parameterIsRequired,
+                    hasSuggestions,
+                    suggestions: suggestions.length > 0 ? suggestions : undefined,
                 };
             }
             // For invalid values, any error response is good
@@ -402,6 +411,11 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 },
                 passed: isError,
                 reason: isError ? undefined : "Tool accepted invalid values",
+                // Issue #173 metadata
+                testedParameter,
+                parameterIsRequired,
+                hasSuggestions,
+                suggestions: suggestions.length > 0 ? suggestions : undefined,
             };
         }
         catch (error) {
@@ -422,6 +436,9 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     passed: false,
                     reason: "Connection error - unable to test",
                     isConnectionError: true,
+                    // Issue #173 metadata
+                    testedParameter,
+                    parameterIsRequired,
                 };
             }
             // Check if the error message is meaningful (not just a generic crash)
@@ -434,6 +451,8 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 messageLower.includes("validation") ||
                 messageLower.includes("error");
             // Removed: (errorInfo.message?.length ?? 0) > 15 - this was causing false positives
+            // Issue #173: Detect suggestions in error message
+            const { hasSuggestions, suggestions } = this.detectSuggestionPatterns(messageLower);
             return {
                 toolName: tool.name,
                 testType: "invalid_values",
@@ -447,6 +466,11 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 },
                 passed: isMeaningfulError,
                 reason: isMeaningfulError ? undefined : "Generic unhandled exception",
+                // Issue #173 metadata
+                testedParameter,
+                parameterIsRequired,
+                hasSuggestions,
+                suggestions: suggestions.length > 0 ? suggestions : undefined,
             };
         }
     }
@@ -569,11 +593,26 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         }
         return params;
     }
+    /**
+     * Issue #173: Return type for generateInvalidValueParams with metadata
+     * Tracks which parameter is being tested and whether it's required
+     */
     generateInvalidValueParams(schema) {
         const params = {};
-        if (!schema?.properties)
-            return { value: null };
+        let testedParameter = "value";
+        let parameterIsRequired = false;
+        if (!schema?.properties) {
+            return { params: { value: null }, testedParameter, parameterIsRequired };
+        }
+        const requiredSet = new Set(schema.required ?? []);
+        let firstParamSet = false;
         for (const [key, prop] of Object.entries(schema.properties)) {
+            // Track the first parameter being tested (for contextual scoring)
+            if (!firstParamSet) {
+                testedParameter = key;
+                parameterIsRequired = requiredSet.has(key);
+                firstParamSet = true;
+            }
             if (prop.type === "string") {
                 if (prop.enum) {
                     params[key] = "not_in_enum"; // Value not in enum
@@ -600,7 +639,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 }
             }
         }
-        return params;
+        return { params, testedParameter, parameterIsRequired };
     }
     generateParamsWithValue(tool, value) {
         const schema = this.getToolSchema(tool);
@@ -623,11 +662,13 @@ export class ErrorHandlingAssessor extends BaseAssessor {
     /**
      * Analyze invalid_values response to determine scoring impact
      * Issue #99: Contextual empty string validation scoring
+     * Issue #173: Bonus points for suggestions and graceful degradation
      *
      * Classifications:
      * - safe_rejection: Tool rejected with error (no penalty)
      * - safe_reflection: Tool stored/echoed without executing (no penalty)
      * - defensive_programming: Tool handled gracefully (no penalty)
+     * - graceful_degradation: Optional param handled with neutral response (no penalty + bonus)
      * - execution_detected: Tool executed input (penalty)
      * - unknown: Cannot determine (partial penalty)
      */
@@ -635,14 +676,30 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         const responseText = this.extractResponseTextSafe(test.actualResponse.rawResponse);
         // Case 1: Tool rejected with error - best case (no penalty)
         if (test.actualResponse.isError) {
+            // Issue #173: Check for suggestions bonus
+            const suggestionBonus = test.hasSuggestions ? 10 : 0;
             return {
                 shouldPenalize: false,
                 penaltyAmount: 0,
                 classification: "safe_rejection",
                 reason: "Tool properly rejected invalid input",
+                bonusPoints: suggestionBonus,
+            };
+        }
+        // Issue #173 Case 2: Graceful degradation for OPTIONAL parameters
+        // If the parameter is optional and the response is neutral (empty results),
+        // this is valid graceful degradation behavior, not a failure
+        if (test.parameterIsRequired === false &&
+            this.isNeutralGracefulResponse(responseText)) {
+            return {
+                shouldPenalize: false,
+                penaltyAmount: 0,
+                classification: "graceful_degradation",
+                reason: "Tool handled optional empty parameter gracefully (valid behavior)",
+                bonusPoints: 15, // Graceful degradation bonus
             };
         }
-        // Case 2: Defensive programming patterns (no penalty)
+        // Case 3: Defensive programming patterns (no penalty)
         // Check BEFORE execution detection because patterns like "query returned 0"
         // might match execution indicators but are actually safe
         if (this.isDefensiveProgrammingResponse(responseText)) {
@@ -651,18 +708,20 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 penaltyAmount: 0,
                 classification: "defensive_programming",
                 reason: "Tool handled empty input defensively",
+                bonusPoints: 0,
             };
         }
-        // Case 3: Safe reflection patterns (no penalty)
+        // Case 4: Safe reflection patterns (no penalty)
         if (this.safeResponseDetector.isReflectionResponse(responseText)) {
             return {
                 shouldPenalize: false,
                 penaltyAmount: 0,
                 classification: "safe_reflection",
                 reason: "Tool safely reflected input without execution",
+                bonusPoints: 0,
             };
         }
-        // Case 4: Check for execution evidence - VULNERABLE (full penalty)
+        // Case 5: Check for execution evidence - VULNERABLE (full penalty)
         if (this.executionDetector.hasExecutionEvidence(responseText) ||
             this.executionDetector.detectExecutionArtifacts(responseText)) {
             return {
@@ -670,14 +729,16 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 penaltyAmount: 100,
                 classification: "execution_detected",
                 reason: "Tool executed input without validation",
+                bonusPoints: 0,
             };
         }
-        // Case 5: Unknown - partial penalty for manual review
+        // Case 6: Unknown - partial penalty for manual review
         return {
             shouldPenalize: true,
             penaltyAmount: 25,
             classification: "unknown",
             reason: "Unable to determine safety - manual review recommended",
+            bonusPoints: 0,
         };
     }
     /**
@@ -715,10 +776,76 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         ];
         return patterns.some((p) => p.test(responseText));
     }
+    /**
+     * Issue #173: Detect helpful suggestion patterns in error responses
+     * Patterns like: "Did you mean: Button, Checkbox?"
+     * Returns extracted suggestions for bonus scoring
+     */
+    detectSuggestionPatterns(responseText) {
+        // Issue #173: ReDoS protection - limit input length before regex matching
+        const truncatedText = responseText.slice(0, 2000);
+        // Issue #173: Bonus points - see docs/ASSESSMENT_CATALOG.md for scoring table
+        // Suggestions: +10 points for helpful error messages like "Did you mean: X?"
+        const suggestionPatterns = [
+            /did\s+you\s+mean[:\s]+([^?.]+)/i,
+            /perhaps\s+you\s+meant[:\s]+([^?.]+)/i,
+            /similar\s+to[:\s]+([^?.]+)/i,
+            /suggestions?[:\s]+([^?.]+)/i,
+            /valid\s+(options?|values?)[:\s]+([^?.]+)/i,
+            /available[:\s]+([^?.]+)/i,
+            /\btry[:\s]+([^?.]+)/i,
+            /expected\s+one\s+of[:\s]+([^?.]+)/i,
+        ];
+        for (const pattern of suggestionPatterns) {
+            const match = truncatedText.match(pattern);
+            if (match) {
+                // Get the captured group (last non-undefined group)
+                const suggestionText = match[match.length - 1] || match[1] || "";
+                const suggestions = suggestionText
+                    .split(/[,;]/)
+                    .map((s) => s.trim())
+                    .filter((s) => s.length > 0 && s.length < 50);
+                if (suggestions.length > 0) {
+                    return { hasSuggestions: true, suggestions };
+                }
+            }
+        }
+        return { hasSuggestions: false, suggestions: [] };
+    }
+    /**
+     * Issue #173: Check for neutral/graceful responses on optional parameters
+     * These indicate the tool handled empty/missing optional input appropriately
+     */
+    isNeutralGracefulResponse(responseText) {
+        // Issue #173: ReDoS protection - limit input length before regex matching
+        const truncatedText = responseText.slice(0, 2000);
+        const gracefulPatterns = [
+            /^\s*\[\s*\]\s*$/, // Empty JSON array (standalone)
+            /^\s*\{\s*\}\s*$/, // Empty JSON object (standalone)
+            /^\s*$/, // Empty/whitespace only response
+            /no\s+results?\s*(found)?/i, // "No results" / "No results found"
+            /^results?:\s*\[\s*\]/i, // "results: []"
+            /returned\s+0\s+/i, // "returned 0 items"
+            /found\s+0\s+/i, // "found 0 matches"
+            /empty\s+list/i, // "empty list"
+            /no\s+matching/i, // "no matching items"
+            /default\s+value/i, // "using default value"
+            /^null$/i, // Explicit null
+            /no\s+data/i, // "no data"
+            /"results"\s*:\s*\[\s*\]/, // JSON with empty results array
+            /"items"\s*:\s*\[\s*\]/, // JSON with empty items array
+            /"data"\s*:\s*\[\s*\]/, // JSON with empty data array
+        ];
+        return gracefulPatterns.some((pattern) => pattern.test(truncatedText));
+    }
     calculateMetrics(tests, _passed) {
         // Calculate enhanced score with bonus points for quality
         let enhancedScore = 0;
         let maxPossibleScore = 0;
+        // Issue #173: Track graceful degradation and suggestion metrics
+        let gracefulDegradationCount = 0;
+        let suggestionCount = 0;
+        let suggestionBonusPoints = 0;
         tests.forEach((test) => {
             // Issue #99: Contextual scoring for invalid_values tests
             // Instead of blanket exclusion, analyze response patterns to determine if
@@ -726,9 +853,23 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             // or if it executed without validation (security concern).
             if (test.testType === "invalid_values") {
                 const analysis = this.analyzeInvalidValuesResponse(test);
+                // Issue #173: Track graceful degradation
+                if (analysis.classification === "graceful_degradation") {
+                    gracefulDegradationCount++;
+                }
+                // Issue #173: Track suggestions
+                if (test.hasSuggestions) {
+                    suggestionCount++;
+                }
+                // Issue #173: Apply bonus points for graceful handling and suggestions
+                if (analysis.bonusPoints > 0) {
+                    enhancedScore += analysis.bonusPoints;
+                    maxPossibleScore += analysis.bonusPoints;
+                    suggestionBonusPoints += analysis.bonusPoints;
+                }
                 if (!analysis.shouldPenalize) {
-                    // Safe response (rejection, reflection, or defensive programming)
-                    // Skip scoring to preserve backward compatibility for well-behaved tools
+                    // Safe response (rejection, reflection, defensive programming, graceful degradation)
+                    // Skip base scoring to preserve backward compatibility for well-behaved tools
                     return;
                 }
                 // Execution detected or unknown - include in scoring with penalty
@@ -756,6 +897,13 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     enhancedScore += 5;
                     maxPossibleScore += 5;
                 }
+                // Issue #173: Extra points for suggestions in other test types
+                if (test.hasSuggestions) {
+                    suggestionCount++;
+                    enhancedScore += 10;
+                    maxPossibleScore += 10;
+                    suggestionBonusPoints += 10;
+                }
             }
         });
         const score = maxPossibleScore > 0 ? (enhancedScore / maxPossibleScore) * 100 : 0;
@@ -796,6 +944,10 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             hasDescriptiveMessages,
             validatesInputs,
             testDetails: tests,
+            // Issue #173: Graceful degradation and suggestion metrics
+            gracefulDegradationCount,
+            suggestionCount,
+            suggestionBonusPoints,
         };
     }
     determineErrorHandlingStatus(metrics, testCount) {

package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAMxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAyIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwC7B;;OAEG;YACW,mBAAmB;IAiCjC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAiDnC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;YACW,wBAAwB;IA4GtC;;OAEG;YACW,uBAAuB;IA2FrC;;OAEG;YACW,4BAA4B;IAoD1C,OAAO,CAAC,yBAAyB;IAkEjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,qBAAqB;IAgC7B,OAAO,CAAC,oBAAoB;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAqEhC"}
+{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAMxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAyIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwC7B;;OAEG;YACW,mBAAmB;IAiCjC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAiDnC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;YACW,wBAAwB;IA4GtC;;OAEG;YACW,uBAAuB;IA2FrC;;OAEG;YACW,4BAA4B;IAoD1C,OAAO,CAAC,yBAAyB;IAwGjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,qBAAqB;IAgC7B,OAAO,CAAC,oBAAoB;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAqEhC"}

package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.js

@@ -587,6 +587,36 @@ export class ProtocolComplianceAssessor extends BaseAssessor {
     // Legacy compatibility methods (from MCPSpecComplianceAssessor)
     // ============================================================================
     assessTransportCompliance(context) {
+        // Issue #172: Check source-based transport detection first
+        // This fixes incorrect FAIL for valid stdio servers without serverInfo metadata
+        if (context.transportDetection?.supportsStdio) {
+            return {
+                supportsStreamableHTTP: context.transportDetection.supportsHTTP,
+                deprecatedSSE: context.transportDetection.supportsSSE,
+                transportValidation: "passed",
+                supportsStdio: true,
+                supportsSSE: context.transportDetection.supportsSSE,
+                confidence: context.transportDetection.confidence,
+                detectionMethod: "source-code-analysis",
+                requiresManualCheck: false,
+                transportEvidence: context.transportDetection.evidence.map((e) => `${e.source}: ${e.detail}`),
+            };
+        }
+        // Also check HTTP-only detection (no serverInfo but HTTP transport detected)
+        if (context.transportDetection?.supportsHTTP &&
+            !context.transportDetection?.supportsStdio) {
+            return {
+                supportsStreamableHTTP: true,
+                deprecatedSSE: context.transportDetection.supportsSSE,
+                transportValidation: "passed",
+                supportsStdio: false,
+                supportsSSE: context.transportDetection.supportsSSE,
+                confidence: context.transportDetection.confidence,
+                detectionMethod: "source-code-analysis",
+                requiresManualCheck: false,
+                transportEvidence: context.transportDetection.evidence.map((e) => `${e.source}: ${e.detail}`),
+            };
+        }
         if (!context.serverInfo) {
             return {
                 supportsStreamableHTTP: false,

package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiB9D,OAAO,EACL,gBAAgB,EAGjB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAiC;IAErD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAStD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAOjC;;;OAGG;YACW,0BAA0B;gBAwBtC,MAAM,EAAE,OAAO,8BAA8B,EAAE,uBAAuB;IAwClE,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8PrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;OAEG;YACW,+BAA+B;IAiC7C;;;OAGG;YACW,yBAAyB;IA0CvC;;;;;;;OAOG;YACW,yBAAyB;IAmFvC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAgDnC"}
+{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiB9D,OAAO,EACL,gBAAgB,EAGjB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAiC;IAErD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAStD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAOjC;;;OAGG;YACW,0BAA0B;gBAwBtC,MAAM,EAAE,OAAO,8BAA8B,EAAE,uBAAuB;IAwClE,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAyQrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;OAEG;YACW,+BAA+B;IAiC7C;;;OAGG;YACW,yBAAyB;IA0CvC;;;;;;;OAOG;YACW,yBAAyB;IAmFvC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAgDnC"}

package/client/lib/services/assessment/modules/SecurityAssessor.js

@@ -101,6 +101,12 @@ export class SecurityAssessor extends BaseAssessor {
     async assess(context) {
         // Select tools for testing first
         const toolsToTest = this.selectToolsForTesting(context.tools);
+        // Issue #170: Set tool annotations context for severity adjustment
+        // This enables annotation-aware false positive reduction for read-only servers
+        if (!context.toolAnnotationsContext) {
+            this.logger.warn("No tool annotations context provided - severity adjustment disabled");
+        }
+        this.payloadTester.setToolAnnotationsContext(context.toolAnnotationsContext);
         // Run universal security testing via extracted payload tester
         const allTests = await this.payloadTester.runUniversalSecurityTests(toolsToTest, context.callTool, context.onProgress);
         // Separate connection errors from valid tests

package/client/lib/services/assessment/modules/securityTests/AnnotationAwareSeverity.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Annotation-Aware Severity Adjustment
+ *
+ * Reduces false positives by considering tool annotations when scoring
+ * vulnerability severity.
+ *
+ * Issue #170: Security module should consider tool annotations to reduce
+ * false positives for read-only servers.
+ *
+ * @module securityTests/AnnotationAwareSeverity
+ */
+import type { SecurityAnnotations, SecurityRiskLevel } from "../../../../lib/assessment/coreTypes.js";
+/**
+ * Result of annotation-aware severity adjustment.
+ */
+export interface SeverityAdjustment {
+    /** Adjusted risk level after considering annotations */
+    adjustedRiskLevel: SecurityRiskLevel;
+    /** Whether an adjustment was made */
+    wasAdjusted: boolean;
+    /** Reason for adjustment (human-readable) */
+    adjustmentReason?: string;
+    /** Original risk level before adjustment */
+    originalRiskLevel: SecurityRiskLevel;
+}
+/**
+ * Adjust vulnerability severity based on tool annotations.
+ *
+ * This function implements the false positive reduction logic from Issue #170.
+ * Read-only tools (readOnlyHint=true) have execution-type vulnerabilities
+ * downgraded to LOW, and closed-world tools (openWorldHint=false) have
+ * exfiltration-type vulnerabilities downgraded to LOW.
+ *
+ * @param attackName - Name of the attack pattern (e.g., "Command Injection")
+ * @param originalRiskLevel - Original risk level from payload definition
+ * @param toolAnnotations - Extracted annotations for this specific tool
+ * @param serverIsReadOnly - Whether ALL server tools are read-only
+ * @param serverIsClosed - Whether ALL server tools are closed-world
+ * @returns SeverityAdjustment with potentially adjusted risk level
+ *
+ * @example
+ * ```typescript
+ * const adjustment = adjustSeverityForAnnotations(
+ *   "Command Injection",
+ *   "HIGH",
+ *   { readOnlyHint: true, source: "mcp" },
+ *   true,
+ *   false
+ * );
+ * // adjustment.wasAdjusted === true
+ * // adjustment.adjustedRiskLevel === "LOW"
+ * ```
+ */
+export declare function adjustSeverityForAnnotations(attackName: string, originalRiskLevel: SecurityRiskLevel, toolAnnotations: SecurityAnnotations | undefined, serverIsReadOnly: boolean, serverIsClosed: boolean): SeverityAdjustment;
+//# sourceMappingURL=AnnotationAwareSeverity.d.ts.map

package/client/lib/services/assessment/modules/securityTests/AnnotationAwareSeverity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AnnotationAwareSeverity.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/AnnotationAwareSeverity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EACV,mBAAmB,EACnB,iBAAiB,EAClB,MAAM,4BAA4B,CAAC;AA+BpC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,qCAAqC;IACrC,WAAW,EAAE,OAAO,CAAC;IACrB,6CAA6C;IAC7C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4CAA4C;IAC5C,iBAAiB,EAAE,iBAAiB,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,iBAAiB,EACpC,eAAe,EAAE,mBAAmB,GAAG,SAAS,EAChD,gBAAgB,EAAE,OAAO,EACzB,cAAc,EAAE,OAAO,GACtB,kBAAkB,CA0DpB"}

package/client/lib/services/assessment/modules/securityTests/AnnotationAwareSeverity.js

@@ -0,0 +1,135 @@
+/**
+ * Annotation-Aware Severity Adjustment
+ *
+ * Reduces false positives by considering tool annotations when scoring
+ * vulnerability severity.
+ *
+ * Issue #170: Security module should consider tool annotations to reduce
+ * false positives for read-only servers.
+ *
+ * @module securityTests/AnnotationAwareSeverity
+ */
+/**
+ * Attack patterns that should be downgraded for read-only tools.
+ * These involve code/command execution which read-only tools cannot perform.
+ */
+const EXECUTION_TYPE_ATTACKS = [
+    "Command Injection", // RCE via shell commands
+    "Calculator Injection", // Code evaluation via calculator
+    "Code Execution", // Direct code execution
+    "Path Traversal", // File system modification
+    "Cross-Tool State Bypass", // State manipulation attacks
+    "Chained Exploitation", // Multi-tool execution chains
+    "Tool Output Injection", // Output tampering
+    "Nested Injection", // Recursive injection attacks
+    "Auth Bypass", // Authentication manipulation
+    "Session Management", // Session state modification
+];
+/**
+ * Attack patterns that should be downgraded for closed-world tools.
+ * These involve external network access which closed-world tools don't have.
+ */
+const EXFILTRATION_TYPE_ATTACKS = [
+    "Indirect Prompt Injection", // External content injection
+    "Data Exfiltration", // Data leakage to external services
+    "Token Theft", // Credential exfiltration
+    "Secret Leakage", // Sensitive data exposure
+    "SSRF", // Server-side request forgery
+];
+/**
+ * Adjust vulnerability severity based on tool annotations.
+ *
+ * This function implements the false positive reduction logic from Issue #170.
+ * Read-only tools (readOnlyHint=true) have execution-type vulnerabilities
+ * downgraded to LOW, and closed-world tools (openWorldHint=false) have
+ * exfiltration-type vulnerabilities downgraded to LOW.
+ *
+ * @param attackName - Name of the attack pattern (e.g., "Command Injection")
+ * @param originalRiskLevel - Original risk level from payload definition
+ * @param toolAnnotations - Extracted annotations for this specific tool
+ * @param serverIsReadOnly - Whether ALL server tools are read-only
+ * @param serverIsClosed - Whether ALL server tools are closed-world
+ * @returns SeverityAdjustment with potentially adjusted risk level
+ *
+ * @example
+ * ```typescript
+ * const adjustment = adjustSeverityForAnnotations(
+ *   "Command Injection",
+ *   "HIGH",
+ *   { readOnlyHint: true, source: "mcp" },
+ *   true,
+ *   false
+ * );
+ * // adjustment.wasAdjusted === true
+ * // adjustment.adjustedRiskLevel === "LOW"
+ * ```
+ */
+export function adjustSeverityForAnnotations(attackName, originalRiskLevel, toolAnnotations, serverIsReadOnly, serverIsClosed) {
+    // Check if we have valid per-tool annotations
+    const hasValidAnnotations = toolAnnotations && toolAnnotations.source !== "none";
+    // Check 1: Per-tool read-only for execution-type attacks
+    // If tool declares readOnlyHint=true, it cannot execute commands
+    if (hasValidAnnotations && toolAnnotations.readOnlyHint === true) {
+        if (isExecutionTypeAttack(attackName)) {
+            return {
+                adjustedRiskLevel: "LOW",
+                wasAdjusted: true,
+                adjustmentReason: `Tool has readOnlyHint=true; ${attackName} downgraded from ${originalRiskLevel} to LOW (cannot execute)`,
+                originalRiskLevel,
+            };
+        }
+    }
+    // Check 2: Per-tool closed-world for exfiltration-type attacks
+    // If tool declares openWorldHint=false, it cannot access external resources
+    if (hasValidAnnotations && toolAnnotations.openWorldHint === false) {
+        if (isExfiltrationType(attackName)) {
+            return {
+                adjustedRiskLevel: "LOW",
+                wasAdjusted: true,
+                adjustmentReason: `Tool has openWorldHint=false; ${attackName} downgraded from ${originalRiskLevel} to LOW (no external access)`,
+                originalRiskLevel,
+            };
+        }
+    }
+    // Check 3: Server-level read-only flag provides additional context
+    // Even if specific tool annotation is missing, server-level flag applies
+    if (serverIsReadOnly && isExecutionTypeAttack(attackName)) {
+        return {
+            adjustedRiskLevel: "LOW",
+            wasAdjusted: true,
+            adjustmentReason: `Server is 100% read-only; ${attackName} downgraded from ${originalRiskLevel} to LOW`,
+            originalRiskLevel,
+        };
+    }
+    // Check 4: Server-level closed flag
+    if (serverIsClosed && isExfiltrationType(attackName)) {
+        return {
+            adjustedRiskLevel: "LOW",
+            wasAdjusted: true,
+            adjustmentReason: `Server is 100% closed-world; ${attackName} downgraded from ${originalRiskLevel} to LOW`,
+            originalRiskLevel,
+        };
+    }
+    // No adjustment needed
+    return {
+        adjustedRiskLevel: originalRiskLevel,
+        wasAdjusted: false,
+        originalRiskLevel,
+    };
+}
+/**
+ * Check if attack name matches execution-type patterns.
+ * Only checks if attackName contains the pattern (not bidirectional)
+ * to prevent security bypass (e.g., "command" matching "Command Injection").
+ */
+function isExecutionTypeAttack(attackName) {
+    return EXECUTION_TYPE_ATTACKS.some((pattern) => attackName.toLowerCase().includes(pattern.toLowerCase()));
+}
+/**
+ * Check if attack name matches exfiltration-type patterns.
+ * Only checks if attackName contains the pattern (not bidirectional)
+ * to prevent security bypass.
+ */
+function isExfiltrationType(attackName) {
+    return EXFILTRATION_TYPE_ATTACKS.some((pattern) => attackName.toLowerCase().includes(pattern.toLowerCase()));
+}

package/client/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts

@@ -34,6 +34,12 @@ export declare class SafeResponseDetector {
      * Check if response is an HTTP error (Issue #26)
      */
     isHttpErrorResponse(responseText: string): boolean;
+    /**
+     * Check if response is an AppleScript syntax error (Issue #175)
+     * These errors should not be flagged as XXE vulnerabilities even when
+     * the XXE payload is echoed back in the error message.
+     */
+    isAppleScriptSyntaxError(responseText: string): boolean;
     /**
      * Check if response is just reflection (safe)
      * Two-layer defense: Match reflection patterns, verify NO execution evidence

package/client/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SafeResponseDetector.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/SafeResponseDetector.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,2BAA2B,EAAE,MAAM,oCAAoC,CAAC;AAgBjF;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,iBAAiB,CAA4B;;IAMrD;;OAEG;IACH,oBAAoB,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO;IAQzE;;OAEG;IACH,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIlD;;;;;;OAMG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IA8EnD;;OAEG;IACH,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIrD;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIjD;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,OAAO;IA0CrE;;OAEG;IACH,sBAAsB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,MAAM;CAUtE"}
+{"version":3,"file":"SafeResponseDetector.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/SafeResponseDetector.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,2BAA2B,EAAE,MAAM,oCAAoC,CAAC;AAiBjF;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,iBAAiB,CAA4B;;IAMrD;;OAEG;IACH,oBAAoB,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO;IAQzE;;OAEG;IACH,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIlD;;;;OAIG;IACH,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIvD;;;;;;OAMG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IA8EnD;;OAEG;IACH,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIrD;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIjD;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,OAAO;IA0CrE;;OAEG;IACH,sBAAsB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,MAAM;CAUtE"}

package/client/lib/services/assessment/modules/securityTests/SafeResponseDetector.js

@@ -5,7 +5,7 @@
  * Extracted from SecurityResponseAnalyzer.ts (Issue #53)
  * Handles: MCP validation, HTTP errors, reflection detection, validation rejection
  */
-import { VALIDATION_ERROR_PATTERNS, STATUS_PATTERNS, REFLECTION_PATTERNS, SEARCH_RESULT_PATTERNS, CREATION_PATTERNS, TEXT_REJECTION_PATTERNS, RESULT_REJECTION_PATTERNS, isHttpError, matchesAny, hasLLMInjectionMarkers, hasOutputInjectionVulnerability, } from "./SecurityPatternLibrary.js";
+import { VALIDATION_ERROR_PATTERNS, STATUS_PATTERNS, REFLECTION_PATTERNS, SEARCH_RESULT_PATTERNS, CREATION_PATTERNS, TEXT_REJECTION_PATTERNS, RESULT_REJECTION_PATTERNS, isHttpError, matchesAny, hasLLMInjectionMarkers, hasOutputInjectionVulnerability, isAppleScriptSyntaxError as isAppleScriptSyntaxErrorPattern, } from "./SecurityPatternLibrary.js";
 import { ExecutionArtifactDetector } from "./ExecutionArtifactDetector.js";
 /**
  * Detects safe response patterns indicating proper tool behavior
@@ -30,6 +30,14 @@ export class SafeResponseDetector {
     isHttpErrorResponse(responseText) {
         return isHttpError(responseText);
     }
+    /**
+     * Check if response is an AppleScript syntax error (Issue #175)
+     * These errors should not be flagged as XXE vulnerabilities even when
+     * the XXE payload is echoed back in the error message.
+     */
+    isAppleScriptSyntaxError(responseText) {
+        return isAppleScriptSyntaxErrorPattern(responseText);
+    }
     /**
      * Check if response is just reflection (safe)
      * Two-layer defense: Match reflection patterns, verify NO execution evidence

package/client/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts

@@ -133,6 +133,26 @@ export declare const PERMANENT_ERROR_PATTERNS: readonly [RegExp, RegExp, RegExp,
  * @returns true if error is transient and should be retried
  */
 export declare function isTransientErrorPattern(text: string): boolean;
+/**
+ * Issue #175: AppleScript syntax error patterns to exclude from XXE detection
+ *
+ * AppleScript errors can trigger false positives when:
+ * 1. The tool returns an AppleScript syntax error (e.g., -2750 duplicate parameter)
+ * 2. The XXE payload is echoed back in the error message
+ * 3. XXE evidence patterns match "parameter" + "entity" combination
+ *
+ * These patterns detect AppleScript-specific errors by:
+ * - Error code ranges (-27xx, -25xx are AppleScript domain)
+ * - AppleScript-specific syntax error messages
+ * - Common AppleScript error patterns
+ */
+export declare const APPLESCRIPT_SYNTAX_ERROR_PATTERNS: RegExp[];
+/**
+ * Check if error text indicates an AppleScript syntax error (Issue #175)
+ * @param text Error message or response text
+ * @returns true if error is an AppleScript syntax error
+ */
+export declare function isAppleScriptSyntaxError(text: string): boolean;
 /**
  * Status patterns indicating safe response handling
  * Used by: isReflectionResponse()