@bryan-thompson/inspector-assessment 1.36.5 → 1.38.0

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js

@@ -28,7 +28,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         const limit = createConcurrencyLimit(concurrency, this.logger);
         this.logger.info(`Testing ${toolsToTest.length} tools for error handling with concurrency limit of ${concurrency}`);
         const allToolTests = await Promise.all(toolsToTest.map((tool) => limit(async () => {
-            const toolTests = await this.testToolErrorHandling(tool, context.callTool);
+            const toolTests = await this.testToolErrorHandling(tool, context.callTool, context);
             // Emit per-tool validation summary for auditor UI (Phase 7)
             if (context.onProgress) {
                 // Count failures by test type (failed = tool didn't reject invalid input)
@@ -125,21 +125,23 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         this.logger.info(`Testing ${maxTools} out of ${tools.length} tools for error handling`);
         return tools.slice(0, maxTools);
     }
-    async testToolErrorHandling(tool, callTool) {
+    async testToolErrorHandling(tool, callTool, context) {
         const tests = [];
+        // Issue #168: Check if tool depends on external API
+        const isExternalAPI = context.externalAPIDependencies?.toolsWithExternalAPIDependency.has(tool.name) ?? false;
         // Scored tests first (affect compliance score)
         // Test 1: Missing required parameters
-        tests.push(await this.testMissingParameters(tool, callTool));
+        tests.push(await this.testMissingParameters(tool, callTool, isExternalAPI));
         // Test 2: Wrong parameter types
-        tests.push(await this.testWrongTypes(tool, callTool));
+        tests.push(await this.testWrongTypes(tool, callTool, isExternalAPI));
         // Test 3: Excessive input size
-        tests.push(await this.testExcessiveInput(tool, callTool));
+        tests.push(await this.testExcessiveInput(tool, callTool, isExternalAPI));
         // Informational tests last (do not affect compliance score)
         // Test 4: Invalid parameter values (edge case handling)
-        tests.push(await this.testInvalidValues(tool, callTool));
+        tests.push(await this.testInvalidValues(tool, callTool, isExternalAPI));
         return tests;
     }
-    async testMissingParameters(tool, callTool) {
+    async testMissingParameters(tool, callTool, isExternalAPI = false) {
         const testInput = {}; // Empty params
         // Check if tool has any required parameters
         const schema = this.getToolSchema(tool);
@@ -178,6 +180,24 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     messageLower.includes("must specify") ||
                     // Also accept field-specific errors (even better!)
                     /\b(query|field|parameter|argument|value|input)\b/i.test(errorInfo.message ?? ""));
+            // Issue #168: For external API tools, check if error is an external service error
+            // External service errors should be treated as passed (validation can't be tested)
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "missing_required",
+                    testInput,
+                    expectedError: "Missing required parameters",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             return {
                 toolName: tool.name,
                 testType: "missing_required",
@@ -239,7 +259,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             };
         }
     }
-    async testWrongTypes(tool, callTool) {
+    async testWrongTypes(tool, callTool, isExternalAPI = false) {
         const schema = this.getToolSchema(tool);
         const testInput = this.generateWrongTypeParams(schema);
         try {
@@ -264,6 +284,23 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     messageLower.includes("object") ||
                     // Also accept validation framework messages
                     /\b(validation|validate|schema|format)\b/i.test(errorInfo.message ?? ""));
+            // Issue #168: For external API tools, check if error is an external service error
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "wrong_type",
+                    testInput,
+                    expectedError: "Type validation error",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             return {
                 toolName: tool.name,
                 testType: "wrong_type",
@@ -326,13 +363,39 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             };
         }
     }
-    async testInvalidValues(tool, callTool) {
+    async testInvalidValues(tool, callTool, isExternalAPI = false) {
         const schema = this.getToolSchema(tool);
-        const testInput = this.generateInvalidValueParams(schema);
+        // Issue #173: Destructure metadata from new return type
+        const { params: testInput, testedParameter, parameterIsRequired, } = this.generateInvalidValueParams(schema);
         try {
             const response = await this.executeWithTimeout(callTool(tool.name, testInput), 5000);
             const isError = this.isErrorResponse(response);
             const errorInfo = this.extractErrorInfo(response);
+            const responseText = this.extractResponseTextSafe(response);
+            // Issue #173: Detect suggestions in response
+            const { hasSuggestions, suggestions } = this.detectSuggestionPatterns(responseText);
+            // Issue #168: For external API tools, check if error is an external service error
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "invalid_values",
+                    testInput,
+                    expectedError: "Invalid parameter values",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                    // Issue #173 metadata
+                    testedParameter,
+                    parameterIsRequired,
+                    hasSuggestions,
+                    suggestions: suggestions.length > 0 ? suggestions : undefined,
+                };
+            }
             // For invalid values, any error response is good
             // The server is validating inputs properly
             return {
@@ -348,6 +411,11 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 },
                 passed: isError,
                 reason: isError ? undefined : "Tool accepted invalid values",
+                // Issue #173 metadata
+                testedParameter,
+                parameterIsRequired,
+                hasSuggestions,
+                suggestions: suggestions.length > 0 ? suggestions : undefined,
             };
         }
         catch (error) {
@@ -368,6 +436,9 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     passed: false,
                     reason: "Connection error - unable to test",
                     isConnectionError: true,
+                    // Issue #173 metadata
+                    testedParameter,
+                    parameterIsRequired,
                 };
             }
             // Check if the error message is meaningful (not just a generic crash)
@@ -380,6 +451,8 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 messageLower.includes("validation") ||
                 messageLower.includes("error");
             // Removed: (errorInfo.message?.length ?? 0) > 15 - this was causing false positives
+            // Issue #173: Detect suggestions in error message
+            const { hasSuggestions, suggestions } = this.detectSuggestionPatterns(messageLower);
             return {
                 toolName: tool.name,
                 testType: "invalid_values",
@@ -393,16 +466,38 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 },
                 passed: isMeaningfulError,
                 reason: isMeaningfulError ? undefined : "Generic unhandled exception",
+                // Issue #173 metadata
+                testedParameter,
+                parameterIsRequired,
+                hasSuggestions,
+                suggestions: suggestions.length > 0 ? suggestions : undefined,
             };
         }
     }
-    async testExcessiveInput(tool, callTool) {
+    async testExcessiveInput(tool, callTool, isExternalAPI = false) {
         const largeString = "x".repeat(100000); // 100KB string
         const testInput = this.generateParamsWithValue(tool, largeString);
         try {
             const response = await this.executeWithTimeout(callTool(tool.name, testInput), 5000);
             const isError = this.isErrorResponse(response);
             const errorInfo = this.extractErrorInfo(response);
+            // Issue #168: For external API tools, check if error is an external service error
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "excessive_input",
+                    testInput: { ...testInput, value: "[100KB string]" },
+                    expectedError: "Input size limit exceeded",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response ? "[response omitted]" : undefined,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             return {
                 toolName: tool.name,
                 testType: "excessive_input",
@@ -498,11 +593,26 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         }
         return params;
     }
+    /**
+     * Issue #173: Return type for generateInvalidValueParams with metadata
+     * Tracks which parameter is being tested and whether it's required
+     */
     generateInvalidValueParams(schema) {
         const params = {};
-        if (!schema?.properties)
-            return { value: null };
+        let testedParameter = "value";
+        let parameterIsRequired = false;
+        if (!schema?.properties) {
+            return { params: { value: null }, testedParameter, parameterIsRequired };
+        }
+        const requiredSet = new Set(schema.required ?? []);
+        let firstParamSet = false;
         for (const [key, prop] of Object.entries(schema.properties)) {
+            // Track the first parameter being tested (for contextual scoring)
+            if (!firstParamSet) {
+                testedParameter = key;
+                parameterIsRequired = requiredSet.has(key);
+                firstParamSet = true;
+            }
             if (prop.type === "string") {
                 if (prop.enum) {
                     params[key] = "not_in_enum"; // Value not in enum
@@ -529,7 +639,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 }
             }
         }
-        return params;
+        return { params, testedParameter, parameterIsRequired };
     }
     generateParamsWithValue(tool, value) {
         const schema = this.getToolSchema(tool);
@@ -552,11 +662,13 @@ export class ErrorHandlingAssessor extends BaseAssessor {
     /**
      * Analyze invalid_values response to determine scoring impact
      * Issue #99: Contextual empty string validation scoring
+     * Issue #173: Bonus points for suggestions and graceful degradation
      *
      * Classifications:
      * - safe_rejection: Tool rejected with error (no penalty)
      * - safe_reflection: Tool stored/echoed without executing (no penalty)
      * - defensive_programming: Tool handled gracefully (no penalty)
+     * - graceful_degradation: Optional param handled with neutral response (no penalty + bonus)
      * - execution_detected: Tool executed input (penalty)
      * - unknown: Cannot determine (partial penalty)
      */
@@ -564,14 +676,30 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         const responseText = this.extractResponseTextSafe(test.actualResponse.rawResponse);
         // Case 1: Tool rejected with error - best case (no penalty)
         if (test.actualResponse.isError) {
+            // Issue #173: Check for suggestions bonus
+            const suggestionBonus = test.hasSuggestions ? 10 : 0;
             return {
                 shouldPenalize: false,
                 penaltyAmount: 0,
                 classification: "safe_rejection",
                 reason: "Tool properly rejected invalid input",
+                bonusPoints: suggestionBonus,
             };
         }
-        // Case 2: Defensive programming patterns (no penalty)
+        // Issue #173 Case 2: Graceful degradation for OPTIONAL parameters
+        // If the parameter is optional and the response is neutral (empty results),
+        // this is valid graceful degradation behavior, not a failure
+        if (test.parameterIsRequired === false &&
+            this.isNeutralGracefulResponse(responseText)) {
+            return {
+                shouldPenalize: false,
+                penaltyAmount: 0,
+                classification: "graceful_degradation",
+                reason: "Tool handled optional empty parameter gracefully (valid behavior)",
+                bonusPoints: 15, // Graceful degradation bonus
+            };
+        }
+        // Case 3: Defensive programming patterns (no penalty)
         // Check BEFORE execution detection because patterns like "query returned 0"
         // might match execution indicators but are actually safe
         if (this.isDefensiveProgrammingResponse(responseText)) {
@@ -580,18 +708,20 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 penaltyAmount: 0,
                 classification: "defensive_programming",
                 reason: "Tool handled empty input defensively",
+                bonusPoints: 0,
             };
         }
-        // Case 3: Safe reflection patterns (no penalty)
+        // Case 4: Safe reflection patterns (no penalty)
         if (this.safeResponseDetector.isReflectionResponse(responseText)) {
             return {
                 shouldPenalize: false,
                 penaltyAmount: 0,
                 classification: "safe_reflection",
                 reason: "Tool safely reflected input without execution",
+                bonusPoints: 0,
             };
         }
-        // Case 4: Check for execution evidence - VULNERABLE (full penalty)
+        // Case 5: Check for execution evidence - VULNERABLE (full penalty)
         if (this.executionDetector.hasExecutionEvidence(responseText) ||
             this.executionDetector.detectExecutionArtifacts(responseText)) {
             return {
@@ -599,14 +729,16 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                 penaltyAmount: 100,
                 classification: "execution_detected",
                 reason: "Tool executed input without validation",
+                bonusPoints: 0,
             };
         }
-        // Case 5: Unknown - partial penalty for manual review
+        // Case 6: Unknown - partial penalty for manual review
         return {
             shouldPenalize: true,
             penaltyAmount: 25,
             classification: "unknown",
             reason: "Unable to determine safety - manual review recommended",
+            bonusPoints: 0,
         };
     }
     /**
@@ -644,10 +776,76 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         ];
         return patterns.some((p) => p.test(responseText));
     }
+    /**
+     * Issue #173: Detect helpful suggestion patterns in error responses
+     * Patterns like: "Did you mean: Button, Checkbox?"
+     * Returns extracted suggestions for bonus scoring
+     */
+    detectSuggestionPatterns(responseText) {
+        // Issue #173: ReDoS protection - limit input length before regex matching
+        const truncatedText = responseText.slice(0, 2000);
+        // Issue #173: Bonus points - see docs/ASSESSMENT_CATALOG.md for scoring table
+        // Suggestions: +10 points for helpful error messages like "Did you mean: X?"
+        const suggestionPatterns = [
+            /did\s+you\s+mean[:\s]+([^?.]+)/i,
+            /perhaps\s+you\s+meant[:\s]+([^?.]+)/i,
+            /similar\s+to[:\s]+([^?.]+)/i,
+            /suggestions?[:\s]+([^?.]+)/i,
+            /valid\s+(options?|values?)[:\s]+([^?.]+)/i,
+            /available[:\s]+([^?.]+)/i,
+            /\btry[:\s]+([^?.]+)/i,
+            /expected\s+one\s+of[:\s]+([^?.]+)/i,
+        ];
+        for (const pattern of suggestionPatterns) {
+            const match = truncatedText.match(pattern);
+            if (match) {
+                // Get the captured group (last non-undefined group)
+                const suggestionText = match[match.length - 1] || match[1] || "";
+                const suggestions = suggestionText
+                    .split(/[,;]/)
+                    .map((s) => s.trim())
+                    .filter((s) => s.length > 0 && s.length < 50);
+                if (suggestions.length > 0) {
+                    return { hasSuggestions: true, suggestions };
+                }
+            }
+        }
+        return { hasSuggestions: false, suggestions: [] };
+    }
+    /**
+     * Issue #173: Check for neutral/graceful responses on optional parameters
+     * These indicate the tool handled empty/missing optional input appropriately
+     */
+    isNeutralGracefulResponse(responseText) {
+        // Issue #173: ReDoS protection - limit input length before regex matching
+        const truncatedText = responseText.slice(0, 2000);
+        const gracefulPatterns = [
+            /^\s*\[\s*\]\s*$/, // Empty JSON array (standalone)
+            /^\s*\{\s*\}\s*$/, // Empty JSON object (standalone)
+            /^\s*$/, // Empty/whitespace only response
+            /no\s+results?\s*(found)?/i, // "No results" / "No results found"
+            /^results?:\s*\[\s*\]/i, // "results: []"
+            /returned\s+0\s+/i, // "returned 0 items"
+            /found\s+0\s+/i, // "found 0 matches"
+            /empty\s+list/i, // "empty list"
+            /no\s+matching/i, // "no matching items"
+            /default\s+value/i, // "using default value"
+            /^null$/i, // Explicit null
+            /no\s+data/i, // "no data"
+            /"results"\s*:\s*\[\s*\]/, // JSON with empty results array
+            /"items"\s*:\s*\[\s*\]/, // JSON with empty items array
+            /"data"\s*:\s*\[\s*\]/, // JSON with empty data array
+        ];
+        return gracefulPatterns.some((pattern) => pattern.test(truncatedText));
+    }
     calculateMetrics(tests, _passed) {
         // Calculate enhanced score with bonus points for quality
         let enhancedScore = 0;
         let maxPossibleScore = 0;
+        // Issue #173: Track graceful degradation and suggestion metrics
+        let gracefulDegradationCount = 0;
+        let suggestionCount = 0;
+        let suggestionBonusPoints = 0;
         tests.forEach((test) => {
             // Issue #99: Contextual scoring for invalid_values tests
             // Instead of blanket exclusion, analyze response patterns to determine if
@@ -655,9 +853,23 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             // or if it executed without validation (security concern).
             if (test.testType === "invalid_values") {
                 const analysis = this.analyzeInvalidValuesResponse(test);
+                // Issue #173: Track graceful degradation
+                if (analysis.classification === "graceful_degradation") {
+                    gracefulDegradationCount++;
+                }
+                // Issue #173: Track suggestions
+                if (test.hasSuggestions) {
+                    suggestionCount++;
+                }
+                // Issue #173: Apply bonus points for graceful handling and suggestions
+                if (analysis.bonusPoints > 0) {
+                    enhancedScore += analysis.bonusPoints;
+                    maxPossibleScore += analysis.bonusPoints;
+                    suggestionBonusPoints += analysis.bonusPoints;
+                }
                 if (!analysis.shouldPenalize) {
-                    // Safe response (rejection, reflection, or defensive programming)
-                    // Skip scoring to preserve backward compatibility for well-behaved tools
+                    // Safe response (rejection, reflection, defensive programming, graceful degradation)
+                    // Skip base scoring to preserve backward compatibility for well-behaved tools
                     return;
                 }
                 // Execution detected or unknown - include in scoring with penalty
@@ -685,6 +897,13 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     enhancedScore += 5;
                     maxPossibleScore += 5;
                 }
+                // Issue #173: Extra points for suggestions in other test types
+                if (test.hasSuggestions) {
+                    suggestionCount++;
+                    enhancedScore += 10;
+                    maxPossibleScore += 10;
+                    suggestionBonusPoints += 10;
+                }
             }
         });
         const score = maxPossibleScore > 0 ? (enhancedScore / maxPossibleScore) * 100 : 0;
@@ -725,6 +944,10 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             hasDescriptiveMessages,
             validatesInputs,
             testDetails: tests,
+            // Issue #173: Graceful degradation and suggestion metrics
+            gracefulDegradationCount,
+            suggestionCount,
+            suggestionBonusPoints,
         };
     }
     determineErrorHandlingStatus(metrics, testCount) {
@@ -780,6 +1003,18 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         parts.push(`Tested ${toolsTested} tools with ${totalScoredTests} scored scenarios (${totalTests} total including informational).`);
         return parts.join(" ");
     }
+    /**
+     * Check if an error indicates an external service failure
+     * Issue #168: External API tools may fail due to service unavailability,
+     * which should not count as validation failure
+     */
+    isExternalServiceError(errorInfo) {
+        const message = errorInfo.message?.toLowerCase() ?? "";
+        const code = String(errorInfo.code ?? "").toLowerCase();
+        // Common external service error patterns
+        const externalErrorPatterns = /rate\s*limit|429|503|502|504|service\s*unavailable|temporarily|timeout|connection\s*refused|network\s*error|api\s*error|external\s*service|upstream|gateway|unreachable|econnrefused|enotfound|etimedout|socket\s*hang\s*up/i;
+        return (externalErrorPatterns.test(message) || externalErrorPatterns.test(code));
+    }
     generateRecommendations(metrics, tests) {
         const recommendations = [];
         if (!metrics.hasProperErrorCodes) {

package/client/lib/services/assessment/modules/FunctionalityAssessor.d.ts

@@ -31,5 +31,15 @@ export declare class FunctionalityAssessor extends BaseAssessor {
     private determineStrategy;
     generateTestInput(schema: JSONSchema7): unknown;
     private generateExplanation;
+    /**
+     * Issue #168: Check if an error response indicates an expected external API error.
+     * External APIs may return rate limit (429), service unavailable (503), timeout,
+     * or similar errors that are expected behavior, not broken functionality.
+     */
+    private isExpectedAPIError;
+    /**
+     * Extract text content from a response for pattern matching.
+     */
+    private extractResponseText;
 }
 //# sourceMappingURL=FunctionalityAssessor.d.ts.map

package/client/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FunctionalityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/FunctionalityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAGvB,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAc9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,cAAc,CAAwB;IAE9C;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoCvB,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;YAoI5D,QAAQ;IAoGtB,OAAO,CAAC,qBAAqB;IAoE7B,OAAO,CAAC,kBAAkB;IAoH1B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAe7C;IAEF;;;OAGG;IACH,OAAO,CAAC,mCAAmC;IAsF3C;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWlB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO;IAItD,OAAO,CAAC,mBAAmB;CA+B5B"}
+{"version":3,"file":"FunctionalityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/FunctionalityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAGvB,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAc9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,cAAc,CAAwB;IAE9C;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoCvB,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;YAoI5D,QAAQ;IA6HtB,OAAO,CAAC,qBAAqB;IAoE7B,OAAO,CAAC,kBAAkB;IAoH1B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAe7C;IAEF;;;OAGG;IACH,OAAO,CAAC,mCAAmC;IAsF3C;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWlB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO;IAItD,OAAO,CAAC,mBAAmB;IAgC3B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAyB5B"}

package/client/lib/services/assessment/modules/FunctionalityAssessor.js

@@ -76,7 +76,7 @@ export class FunctionalityAssessor extends BaseAssessor {
             this.testCount++;
             completedTests++;
             batchCount++;
-            const result = await this.testTool(tool, context.callTool);
+            const result = await this.testTool(tool, context.callTool, context);
             // Emit progress batch if threshold reached
             const timeSinceLastBatch = Date.now() - lastBatchTime;
             if (batchCount >= BATCH_SIZE ||
@@ -131,7 +131,7 @@ export class FunctionalityAssessor extends BaseAssessor {
             tools,
         };
     }
-    async testTool(tool, callTool) {
+    async testTool(tool, callTool, context) {
         const startTime = Date.now();
         // Generate minimal valid parameters with metadata
         const { params: testParams, metadata } = this.generateMinimalParams(tool);
@@ -173,7 +173,25 @@ export class FunctionalityAssessor extends BaseAssessor {
                         responseMetadata,
                     };
                 }
-                // Real tool failure (not just validation)
+                // Issue #168: Check for expected external API errors
+                // External API tools may fail due to rate limits, service unavailability, etc.
+                // These are expected behaviors, not broken functionality
+                const isExternalAPI = context.externalAPIDependencies?.toolsWithExternalAPIDependency.has(tool.name);
+                if (isExternalAPI && this.isExpectedAPIError(response)) {
+                    this.logger.info(`${tool.name}: External API error (expected behavior for external API tool)`);
+                    return {
+                        toolName: tool.name,
+                        tested: true,
+                        status: "working",
+                        executionTime,
+                        testParameters: cleanedParams,
+                        response,
+                        testInputMetadata: metadata,
+                        responseMetadata,
+                        note: "External API returned error (expected behavior)",
+                    };
+                }
+                // Real tool failure (not just validation or expected API error)
                 return {
                     toolName: tool.name,
                     tested: true,
@@ -472,4 +490,48 @@ export class FunctionalityAssessor extends BaseAssessor {
         }
         return parts.join(" ");
     }
+    /**
+     * Issue #168: Check if an error response indicates an expected external API error.
+     * External APIs may return rate limit (429), service unavailable (503), timeout,
+     * or similar errors that are expected behavior, not broken functionality.
+     */
+    isExpectedAPIError(response) {
+        const content = this.extractResponseText(response);
+        if (!content)
+            return false;
+        // Match common external API error patterns
+        const expectedErrorPatterns = /rate\s*limit|429|503|service\s*unavailable|temporarily|timeout|connection\s*refused|network\s*error|api\s*error|external\s*service|upstream/i;
+        return expectedErrorPatterns.test(content);
+    }
+    /**
+     * Extract text content from a response for pattern matching.
+     */
+    extractResponseText(response) {
+        if (typeof response === "string")
+            return response;
+        if (!response || typeof response !== "object")
+            return "";
+        const obj = response;
+        // Check common response content locations
+        if (typeof obj.content === "string")
+            return obj.content;
+        if (typeof obj.message === "string")
+            return obj.message;
+        if (typeof obj.error === "string")
+            return obj.error;
+        // Handle MCP response format with content array
+        if (Array.isArray(obj.content)) {
+            return obj.content
+                .map((item) => {
+                if (typeof item === "string")
+                    return item;
+                if (typeof item?.text === "string")
+                    return item.text;
+                return "";
+            })
+                .join(" ");
+        }
+        // Fallback to JSON stringify for deep search
+        return JSON.stringify(response);
+    }
 }

package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAMxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAyIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwC7B;;OAEG;YACW,mBAAmB;IAiCjC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAiDnC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;YACW,wBAAwB;IA4GtC;;OAEG;YACW,uBAAuB;IA2FrC;;OAEG;YACW,4BAA4B;IAoD1C,OAAO,CAAC,yBAAyB;IAkEjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,qBAAqB;IAgC7B,OAAO,CAAC,oBAAoB;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAqEhC"}
+{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAMxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAyIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwC7B;;OAEG;YACW,mBAAmB;IAiCjC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAiDnC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;YACW,wBAAwB;IA4GtC;;OAEG;YACW,uBAAuB;IA2FrC;;OAEG;YACW,4BAA4B;IAoD1C,OAAO,CAAC,yBAAyB;IAwGjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,qBAAqB;IAgC7B,OAAO,CAAC,oBAAoB;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAqEhC"}

package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.js

@@ -587,6 +587,36 @@ export class ProtocolComplianceAssessor extends BaseAssessor {
     // Legacy compatibility methods (from MCPSpecComplianceAssessor)
     // ============================================================================
     assessTransportCompliance(context) {
+        // Issue #172: Check source-based transport detection first
+        // This fixes incorrect FAIL for valid stdio servers without serverInfo metadata
+        if (context.transportDetection?.supportsStdio) {
+            return {
+                supportsStreamableHTTP: context.transportDetection.supportsHTTP,
+                deprecatedSSE: context.transportDetection.supportsSSE,
+                transportValidation: "passed",
+                supportsStdio: true,
+                supportsSSE: context.transportDetection.supportsSSE,
+                confidence: context.transportDetection.confidence,
+                detectionMethod: "source-code-analysis",
+                requiresManualCheck: false,
+                transportEvidence: context.transportDetection.evidence.map((e) => `${e.source}: ${e.detail}`),
+            };
+        }
+        // Also check HTTP-only detection (no serverInfo but HTTP transport detected)
+        if (context.transportDetection?.supportsHTTP &&
+            !context.transportDetection?.supportsStdio) {
+            return {
+                supportsStreamableHTTP: true,
+                deprecatedSSE: context.transportDetection.supportsSSE,
+                transportValidation: "passed",
+                supportsStdio: false,
+                supportsSSE: context.transportDetection.supportsSSE,
+                confidence: context.transportDetection.confidence,
+                detectionMethod: "source-code-analysis",
+                requiresManualCheck: false,
+                transportEvidence: context.transportDetection.evidence.map((e) => `${e.source}: ${e.detail}`),
+            };
+        }
         if (!context.serverInfo) {
             return {
                 supportsStreamableHTTP: false,

package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiB9D,OAAO,EACL,gBAAgB,EAGjB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAiC;IAErD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAStD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAOjC;;;OAGG;YACW,0BAA0B;gBAwBtC,MAAM,EAAE,OAAO,8BAA8B,EAAE,uBAAuB;IAwClE,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8PrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;OAEG;YACW,+BAA+B;IAiC7C;;;OAGG;YACW,yBAAyB;IA0CvC;;;;;;;OAOG;YACW,yBAAyB;IAmFvC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAgDnC"}
+{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiB9D,OAAO,EACL,gBAAgB,EAGjB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAiC;IAErD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAStD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAOjC;;;OAGG;YACW,0BAA0B;gBAwBtC,MAAM,EAAE,OAAO,8BAA8B,EAAE,uBAAuB;IAwClE,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAyQrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;OAEG;YACW,+BAA+B;IAiC7C;;;OAGG;YACW,yBAAyB;IA0CvC;;;;;;;OAOG;YACW,yBAAyB;IAmFvC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAgDnC"}

package/client/lib/services/assessment/modules/SecurityAssessor.js

@@ -101,6 +101,12 @@ export class SecurityAssessor extends BaseAssessor {
     async assess(context) {
         // Select tools for testing first
         const toolsToTest = this.selectToolsForTesting(context.tools);
+        // Issue #170: Set tool annotations context for severity adjustment
+        // This enables annotation-aware false positive reduction for read-only servers
+        if (!context.toolAnnotationsContext) {
+            this.logger.warn("No tool annotations context provided - severity adjustment disabled");
+        }
+        this.payloadTester.setToolAnnotationsContext(context.toolAnnotationsContext);
         // Run universal security testing via extracted payload tester
         const allTests = await this.payloadTester.runUniversalSecurityTests(toolsToTest, context.callTool, context.onProgress);
         // Separate connection errors from valid tests