@bryan-thompson/inspector-assessment 1.36.4 → 1.37.0

package/client/lib/services/assessment/config/performanceConfigSchemas.js

@@ -83,6 +83,26 @@ export const PerformanceConfigSchema = z.object({
         .min(PERF_CONFIG_RANGES.eventEmitterMaxListeners.min, `eventEmitterMaxListeners must be >= ${PERF_CONFIG_RANGES.eventEmitterMaxListeners.min}`)
         .max(PERF_CONFIG_RANGES.eventEmitterMaxListeners.max, `eventEmitterMaxListeners must be <= ${PERF_CONFIG_RANGES.eventEmitterMaxListeners.max}`)
         .optional(),
+    /**
+     * Maximum retry attempts for transient errors in security tests.
+     * Issue #157: Connection retry logic for reliability
+     */
+    securityRetryMaxAttempts: z
+        .number()
+        .int("securityRetryMaxAttempts must be an integer")
+        .min(PERF_CONFIG_RANGES.securityRetryMaxAttempts.min, `securityRetryMaxAttempts must be >= ${PERF_CONFIG_RANGES.securityRetryMaxAttempts.min}`)
+        .max(PERF_CONFIG_RANGES.securityRetryMaxAttempts.max, `securityRetryMaxAttempts must be <= ${PERF_CONFIG_RANGES.securityRetryMaxAttempts.max}`)
+        .optional(),
+    /**
+     * Initial backoff delay in milliseconds for security test retries.
+     * Issue #157: Connection retry logic for reliability
+     */
+    securityRetryBackoffMs: z
+        .number()
+        .int("securityRetryBackoffMs must be an integer")
+        .min(PERF_CONFIG_RANGES.securityRetryBackoffMs.min, `securityRetryBackoffMs must be >= ${PERF_CONFIG_RANGES.securityRetryBackoffMs.min}`)
+        .max(PERF_CONFIG_RANGES.securityRetryBackoffMs.max, `securityRetryBackoffMs must be <= ${PERF_CONFIG_RANGES.securityRetryBackoffMs.max}`)
+        .optional(),
 });
 /**
  * Validate a partial performance config using Zod.

package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.d.ts

@@ -0,0 +1,74 @@
+/**
+ * External API Dependency Detector
+ *
+ * Identifies tools that depend on external APIs based on name and description patterns.
+ * This information enables downstream assessors to adjust their behavior:
+ * - TemporalAssessor: Relaxed variance thresholds for external API tools
+ * - FunctionalityAssessor: Accept API errors as valid responses
+ * - ErrorHandlingAssessor: Account for external service failures
+ *
+ * Issue #168: New module for external API dependency detection
+ *
+ * @module helpers/ExternalAPIDependencyDetector
+ */
+import { Tool } from "@modelcontextprotocol/sdk/types.js";
+/**
+ * External API dependency detection results
+ * @public
+ */
+export interface ExternalAPIDependencyInfo {
+    /** Set of tool names that depend on external APIs */
+    toolsWithExternalAPIDependency: Set<string>;
+    /** Number of tools detected with external API dependencies */
+    detectedCount: number;
+    /** Detection confidence based on pattern strength */
+    confidence: "high" | "medium" | "low";
+    /** List of detected tool names (for serialization) */
+    detectedTools: string[];
+}
+/**
+ * Detects external API dependencies in MCP tools based on name and description patterns.
+ * Designed to run during context preparation before assessors execute.
+ *
+ * @public
+ */
+export declare class ExternalAPIDependencyDetector {
+    /**
+     * Tool name patterns that suggest external API dependency.
+     * Uses word-boundary matching to prevent false positives.
+     *
+     * Extracted from VarianceClassifier (Issue #166) for reuse across modules.
+     */
+    private readonly EXTERNAL_API_PATTERNS;
+    /**
+     * Description patterns that suggest external API dependency.
+     * Regex patterns for more flexible matching.
+     */
+    private readonly EXTERNAL_API_DESCRIPTION_PATTERNS;
+    /**
+     * Detect external API dependencies from a list of tools.
+     *
+     * @param tools - List of MCP tools to analyze
+     * @returns Detection results with tool names and confidence
+     */
+    detect(tools: Tool[]): ExternalAPIDependencyInfo;
+    /**
+     * Check if a single tool depends on external APIs.
+     * Uses BOTH name patterns AND description analysis for detection.
+     *
+     * @param tool - MCP tool to check
+     * @returns true if tool appears to depend on external APIs
+     */
+    isExternalAPITool(tool: Tool): boolean;
+    /**
+     * Get the list of name patterns used for detection.
+     * Useful for debugging and documentation.
+     */
+    getNamePatterns(): readonly string[];
+    /**
+     * Get the list of description patterns used for detection.
+     * Useful for debugging and documentation.
+     */
+    getDescriptionPatterns(): readonly RegExp[];
+}
+//# sourceMappingURL=ExternalAPIDependencyDetector.d.ts.map

package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExternalAPIDependencyDetector.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/helpers/ExternalAPIDependencyDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAE1D;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,qDAAqD;IACrD,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC5C,8DAA8D;IAC9D,aAAa,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,sDAAsD;IACtD,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,6BAA6B;IACxC;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAuBpC;IAEF;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAQhD;IAEF;;;;;OAKG;IACH,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,yBAAyB;IA8BhD;;;;;;OAMG;IACH,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO;IAmBtC;;;OAGG;IACH,eAAe,IAAI,SAAS,MAAM,EAAE;IAIpC;;;OAGG;IACH,sBAAsB,IAAI,SAAS,MAAM,EAAE;CAG5C"}

package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.js

@@ -0,0 +1,131 @@
+/**
+ * External API Dependency Detector
+ *
+ * Identifies tools that depend on external APIs based on name and description patterns.
+ * This information enables downstream assessors to adjust their behavior:
+ * - TemporalAssessor: Relaxed variance thresholds for external API tools
+ * - FunctionalityAssessor: Accept API errors as valid responses
+ * - ErrorHandlingAssessor: Account for external service failures
+ *
+ * Issue #168: New module for external API dependency detection
+ *
+ * @module helpers/ExternalAPIDependencyDetector
+ */
+/**
+ * Detects external API dependencies in MCP tools based on name and description patterns.
+ * Designed to run during context preparation before assessors execute.
+ *
+ * @public
+ */
+export class ExternalAPIDependencyDetector {
+    /**
+     * Tool name patterns that suggest external API dependency.
+     * Uses word-boundary matching to prevent false positives.
+     *
+     * Extracted from VarianceClassifier (Issue #166) for reuse across modules.
+     */
+    EXTERNAL_API_PATTERNS = [
+        // API-related prefixes
+        "api",
+        "external",
+        "remote",
+        "live",
+        // Data type patterns (typically from external sources)
+        "weather",
+        "stock",
+        "price",
+        "market",
+        "currency",
+        "exchange",
+        "rate",
+        "forex",
+        // Service-specific prefixes
+        "wb", // World Bank
+        "worldbank",
+        // Action patterns suggesting external fetch
+        "fetch_from",
+        "poll",
+        "realtime",
+        "current",
+    ];
+    /**
+     * Description patterns that suggest external API dependency.
+     * Regex patterns for more flexible matching.
+     */
+    EXTERNAL_API_DESCRIPTION_PATTERNS = [
+        /external\s*(api|service)/i,
+        /fetche?s?\s*(from|data\s+from)/i,
+        /calls?\s*(external|remote)/i,
+        /live\s*(data|feed|stream)/i,
+        /real[- ]?time/i,
+        /world\s*bank/i,
+        /third[- ]?party\s*(api|service)/i,
+    ];
+    /**
+     * Detect external API dependencies from a list of tools.
+     *
+     * @param tools - List of MCP tools to analyze
+     * @returns Detection results with tool names and confidence
+     */
+    detect(tools) {
+        const toolsWithExternalAPI = new Set();
+        for (const tool of tools) {
+            if (this.isExternalAPITool(tool)) {
+                toolsWithExternalAPI.add(tool.name);
+            }
+        }
+        const detectedCount = toolsWithExternalAPI.size;
+        // Determine confidence based on detection count
+        // More detections = higher confidence in pattern accuracy
+        let confidence;
+        if (detectedCount === 0) {
+            confidence = "low";
+        }
+        else if (detectedCount >= 3) {
+            confidence = "high";
+        }
+        else {
+            confidence = "medium";
+        }
+        return {
+            toolsWithExternalAPIDependency: toolsWithExternalAPI,
+            detectedCount,
+            confidence,
+            detectedTools: Array.from(toolsWithExternalAPI),
+        };
+    }
+    /**
+     * Check if a single tool depends on external APIs.
+     * Uses BOTH name patterns AND description analysis for detection.
+     *
+     * @param tool - MCP tool to check
+     * @returns true if tool appears to depend on external APIs
+     */
+    isExternalAPITool(tool) {
+        const toolName = tool.name.toLowerCase();
+        const description = (tool.description || "").toLowerCase();
+        // Check name patterns with word-boundary matching
+        // "weather_api" matches "api" but "capital_gains" doesn't match "api"
+        const nameMatch = this.EXTERNAL_API_PATTERNS.some((pattern) => {
+            const wordBoundaryRegex = new RegExp(`(^|_|-)${pattern}($|_|-|s)`);
+            return wordBoundaryRegex.test(toolName);
+        });
+        // Check description for external API indicators
+        const descriptionMatch = this.EXTERNAL_API_DESCRIPTION_PATTERNS.some((regex) => regex.test(description));
+        return nameMatch || descriptionMatch;
+    }
+    /**
+     * Get the list of name patterns used for detection.
+     * Useful for debugging and documentation.
+     */
+    getNamePatterns() {
+        return this.EXTERNAL_API_PATTERNS;
+    }
+    /**
+     * Get the list of description patterns used for detection.
+     * Useful for debugging and documentation.
+     */
+    getDescriptionPatterns() {
+        return this.EXTERNAL_API_DESCRIPTION_PATTERNS;
+    }
+}

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts

@@ -46,6 +46,12 @@ export declare class ErrorHandlingAssessor extends BaseAssessor {
     private calculateMetrics;
     private determineErrorHandlingStatus;
     private generateExplanation;
+    /**
+     * Check if an error indicates an external service failure
+     * Issue #168: External API tools may fail due to service unavailability,
+     * which should not count as validation failure
+     */
+    private isExternalServiceError;
     private generateRecommendations;
 }
 //# sourceMappingURL=ErrorHandlingAssessor.d.ts.map

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAKxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,iBAAiB,CAA4B;IACrD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,eAAe,CAAkB;gBAE7B,MAAM,EAAE,uBAAuB;IAOrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IA4G1E,OAAO,CAAC,qBAAqB;YAuDf,qBAAqB;YA0BrB,qBAAqB;YA0HrB,cAAc;YA0Gd,iBAAiB;YAqFjB,kBAAkB;IAoFhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAkC/B,OAAO,CAAC,0BAA0B;IAkClC,OAAO,CAAC,uBAAuB;IA4B/B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,4BAA4B;IAgEpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;;OAGG;IACH,OAAO,CAAC,8BAA8B;IAetC,OAAO,CAAC,gBAAgB;IA8GxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,uBAAuB;CA4ChC"}
+{"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAKxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,iBAAiB,CAA4B;IACrD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,eAAe,CAAkB;gBAE7B,MAAM,EAAE,uBAAuB;IAOrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IA6G1E,OAAO,CAAC,qBAAqB;YAuDf,qBAAqB;YAiCrB,qBAAqB;YA+IrB,cAAc;YA8Hd,iBAAiB;YAyGjB,kBAAkB;IAwGhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAkC/B,OAAO,CAAC,0BAA0B;IAkClC,OAAO,CAAC,uBAAuB;IA4B/B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,4BAA4B;IAgEpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;;OAGG;IACH,OAAO,CAAC,8BAA8B;IAetC,OAAO,CAAC,gBAAgB;IA8GxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAgB9B,OAAO,CAAC,uBAAuB;CA4ChC"}

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js

@@ -28,7 +28,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         const limit = createConcurrencyLimit(concurrency, this.logger);
         this.logger.info(`Testing ${toolsToTest.length} tools for error handling with concurrency limit of ${concurrency}`);
         const allToolTests = await Promise.all(toolsToTest.map((tool) => limit(async () => {
-            const toolTests = await this.testToolErrorHandling(tool, context.callTool);
+            const toolTests = await this.testToolErrorHandling(tool, context.callTool, context);
             // Emit per-tool validation summary for auditor UI (Phase 7)
             if (context.onProgress) {
                 // Count failures by test type (failed = tool didn't reject invalid input)
@@ -125,21 +125,23 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         this.logger.info(`Testing ${maxTools} out of ${tools.length} tools for error handling`);
         return tools.slice(0, maxTools);
     }
-    async testToolErrorHandling(tool, callTool) {
+    async testToolErrorHandling(tool, callTool, context) {
         const tests = [];
+        // Issue #168: Check if tool depends on external API
+        const isExternalAPI = context.externalAPIDependencies?.toolsWithExternalAPIDependency.has(tool.name) ?? false;
         // Scored tests first (affect compliance score)
         // Test 1: Missing required parameters
-        tests.push(await this.testMissingParameters(tool, callTool));
+        tests.push(await this.testMissingParameters(tool, callTool, isExternalAPI));
         // Test 2: Wrong parameter types
-        tests.push(await this.testWrongTypes(tool, callTool));
+        tests.push(await this.testWrongTypes(tool, callTool, isExternalAPI));
         // Test 3: Excessive input size
-        tests.push(await this.testExcessiveInput(tool, callTool));
+        tests.push(await this.testExcessiveInput(tool, callTool, isExternalAPI));
         // Informational tests last (do not affect compliance score)
         // Test 4: Invalid parameter values (edge case handling)
-        tests.push(await this.testInvalidValues(tool, callTool));
+        tests.push(await this.testInvalidValues(tool, callTool, isExternalAPI));
         return tests;
     }
-    async testMissingParameters(tool, callTool) {
+    async testMissingParameters(tool, callTool, isExternalAPI = false) {
         const testInput = {}; // Empty params
         // Check if tool has any required parameters
         const schema = this.getToolSchema(tool);
@@ -178,6 +180,24 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     messageLower.includes("must specify") ||
                     // Also accept field-specific errors (even better!)
                     /\b(query|field|parameter|argument|value|input)\b/i.test(errorInfo.message ?? ""));
+            // Issue #168: For external API tools, check if error is an external service error
+            // External service errors should be treated as passed (validation can't be tested)
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "missing_required",
+                    testInput,
+                    expectedError: "Missing required parameters",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             return {
                 toolName: tool.name,
                 testType: "missing_required",
@@ -239,7 +259,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             };
         }
     }
-    async testWrongTypes(tool, callTool) {
+    async testWrongTypes(tool, callTool, isExternalAPI = false) {
         const schema = this.getToolSchema(tool);
         const testInput = this.generateWrongTypeParams(schema);
         try {
@@ -264,6 +284,23 @@ export class ErrorHandlingAssessor extends BaseAssessor {
                     messageLower.includes("object") ||
                     // Also accept validation framework messages
                     /\b(validation|validate|schema|format)\b/i.test(errorInfo.message ?? ""));
+            // Issue #168: For external API tools, check if error is an external service error
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "wrong_type",
+                    testInput,
+                    expectedError: "Type validation error",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             return {
                 toolName: tool.name,
                 testType: "wrong_type",
@@ -326,13 +363,30 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             };
         }
     }
-    async testInvalidValues(tool, callTool) {
+    async testInvalidValues(tool, callTool, isExternalAPI = false) {
         const schema = this.getToolSchema(tool);
         const testInput = this.generateInvalidValueParams(schema);
         try {
             const response = await this.executeWithTimeout(callTool(tool.name, testInput), 5000);
             const isError = this.isErrorResponse(response);
             const errorInfo = this.extractErrorInfo(response);
+            // Issue #168: For external API tools, check if error is an external service error
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "invalid_values",
+                    testInput,
+                    expectedError: "Invalid parameter values",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             // For invalid values, any error response is good
             // The server is validating inputs properly
             return {
@@ -396,13 +450,30 @@ export class ErrorHandlingAssessor extends BaseAssessor {
             };
         }
     }
-    async testExcessiveInput(tool, callTool) {
+    async testExcessiveInput(tool, callTool, isExternalAPI = false) {
         const largeString = "x".repeat(100000); // 100KB string
         const testInput = this.generateParamsWithValue(tool, largeString);
         try {
             const response = await this.executeWithTimeout(callTool(tool.name, testInput), 5000);
             const isError = this.isErrorResponse(response);
             const errorInfo = this.extractErrorInfo(response);
+            // Issue #168: For external API tools, check if error is an external service error
+            if (isExternalAPI && isError && this.isExternalServiceError(errorInfo)) {
+                return {
+                    toolName: tool.name,
+                    testType: "excessive_input",
+                    testInput: { ...testInput, value: "[100KB string]" },
+                    expectedError: "Input size limit exceeded",
+                    actualResponse: {
+                        isError,
+                        errorCode: errorInfo.code,
+                        errorMessage: errorInfo.message,
+                        rawResponse: response ? "[response omitted]" : undefined,
+                    },
+                    passed: true,
+                    reason: "External API service error (validation cannot be tested when service unavailable)",
+                };
+            }
             return {
                 toolName: tool.name,
                 testType: "excessive_input",
@@ -780,6 +851,18 @@ export class ErrorHandlingAssessor extends BaseAssessor {
         parts.push(`Tested ${toolsTested} tools with ${totalScoredTests} scored scenarios (${totalTests} total including informational).`);
         return parts.join(" ");
     }
+    /**
+     * Check if an error indicates an external service failure
+     * Issue #168: External API tools may fail due to service unavailability,
+     * which should not count as validation failure
+     */
+    isExternalServiceError(errorInfo) {
+        const message = errorInfo.message?.toLowerCase() ?? "";
+        const code = String(errorInfo.code ?? "").toLowerCase();
+        // Common external service error patterns
+        const externalErrorPatterns = /rate\s*limit|429|503|502|504|service\s*unavailable|temporarily|timeout|connection\s*refused|network\s*error|api\s*error|external\s*service|upstream|gateway|unreachable|econnrefused|enotfound|etimedout|socket\s*hang\s*up/i;
+        return (externalErrorPatterns.test(message) || externalErrorPatterns.test(code));
+    }
     generateRecommendations(metrics, tests) {
         const recommendations = [];
         if (!metrics.hasProperErrorCodes) {

package/client/lib/services/assessment/modules/FunctionalityAssessor.d.ts

@@ -31,5 +31,15 @@ export declare class FunctionalityAssessor extends BaseAssessor {
     private determineStrategy;
     generateTestInput(schema: JSONSchema7): unknown;
     private generateExplanation;
+    /**
+     * Issue #168: Check if an error response indicates an expected external API error.
+     * External APIs may return rate limit (429), service unavailable (503), timeout,
+     * or similar errors that are expected behavior, not broken functionality.
+     */
+    private isExpectedAPIError;
+    /**
+     * Extract text content from a response for pattern matching.
+     */
+    private extractResponseText;
 }
 //# sourceMappingURL=FunctionalityAssessor.d.ts.map

package/client/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FunctionalityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/FunctionalityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAGvB,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAc9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,cAAc,CAAwB;IAE9C;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoCvB,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;YAoI5D,QAAQ;IAoGtB,OAAO,CAAC,qBAAqB;IAoE7B,OAAO,CAAC,kBAAkB;IAoH1B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAe7C;IAEF;;;OAGG;IACH,OAAO,CAAC,mCAAmC;IAsF3C;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWlB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO;IAItD,OAAO,CAAC,mBAAmB;CA+B5B"}
+{"version":3,"file":"FunctionalityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/FunctionalityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAGvB,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAc9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,cAAc,CAAwB;IAE9C;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoCvB,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;YAoI5D,QAAQ;IA6HtB,OAAO,CAAC,qBAAqB;IAoE7B,OAAO,CAAC,kBAAkB;IAoH1B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAe7C;IAEF;;;OAGG;IACH,OAAO,CAAC,mCAAmC;IAsF3C;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWlB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO;IAItD,OAAO,CAAC,mBAAmB;IAgC3B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAyB5B"}

package/client/lib/services/assessment/modules/FunctionalityAssessor.js

@@ -76,7 +76,7 @@ export class FunctionalityAssessor extends BaseAssessor {
             this.testCount++;
             completedTests++;
             batchCount++;
-            const result = await this.testTool(tool, context.callTool);
+            const result = await this.testTool(tool, context.callTool, context);
             // Emit progress batch if threshold reached
             const timeSinceLastBatch = Date.now() - lastBatchTime;
             if (batchCount >= BATCH_SIZE ||
@@ -131,7 +131,7 @@ export class FunctionalityAssessor extends BaseAssessor {
             tools,
         };
     }
-    async testTool(tool, callTool) {
+    async testTool(tool, callTool, context) {
         const startTime = Date.now();
         // Generate minimal valid parameters with metadata
         const { params: testParams, metadata } = this.generateMinimalParams(tool);
@@ -173,7 +173,25 @@ export class FunctionalityAssessor extends BaseAssessor {
                         responseMetadata,
                     };
                 }
-                // Real tool failure (not just validation)
+                // Issue #168: Check for expected external API errors
+                // External API tools may fail due to rate limits, service unavailability, etc.
+                // These are expected behaviors, not broken functionality
+                const isExternalAPI = context.externalAPIDependencies?.toolsWithExternalAPIDependency.has(tool.name);
+                if (isExternalAPI && this.isExpectedAPIError(response)) {
+                    this.logger.info(`${tool.name}: External API error (expected behavior for external API tool)`);
+                    return {
+                        toolName: tool.name,
+                        tested: true,
+                        status: "working",
+                        executionTime,
+                        testParameters: cleanedParams,
+                        response,
+                        testInputMetadata: metadata,
+                        responseMetadata,
+                        note: "External API returned error (expected behavior)",
+                    };
+                }
+                // Real tool failure (not just validation or expected API error)
                 return {
                     toolName: tool.name,
                     tested: true,
@@ -472,4 +490,48 @@ export class FunctionalityAssessor extends BaseAssessor {
         }
         return parts.join(" ");
     }
+    /**
+     * Issue #168: Check if an error response indicates an expected external API error.
+     * External APIs may return rate limit (429), service unavailable (503), timeout,
+     * or similar errors that are expected behavior, not broken functionality.
+     */
+    isExpectedAPIError(response) {
+        const content = this.extractResponseText(response);
+        if (!content)
+            return false;
+        // Match common external API error patterns
+        const expectedErrorPatterns = /rate\s*limit|429|503|service\s*unavailable|temporarily|timeout|connection\s*refused|network\s*error|api\s*error|external\s*service|upstream/i;
+        return expectedErrorPatterns.test(content);
+    }
+    /**
+     * Extract text content from a response for pattern matching.
+     */
+    extractResponseText(response) {
+        if (typeof response === "string")
+            return response;
+        if (!response || typeof response !== "object")
+            return "";
+        const obj = response;
+        // Check common response content locations
+        if (typeof obj.content === "string")
+            return obj.content;
+        if (typeof obj.message === "string")
+            return obj.message;
+        if (typeof obj.error === "string")
+            return obj.error;
+        // Handle MCP response format with content array
+        if (Array.isArray(obj.content)) {
+            return obj.content
+                .map((item) => {
+                if (typeof item === "string")
+                    return item;
+                if (typeof item?.text === "string")
+                    return item.text;
+                return "";
+            })
+                .join(" ");
+        }
+        // Fallback to JSON stringify for deep search
+        return JSON.stringify(response);
+    }
 }

package/client/lib/services/assessment/modules/TemporalAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"TemporalAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/TemporalAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,uBAAuB,EAEvB,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAiB9C,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IAGjD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAK;gBAE5B,MAAM,EAAE,uBAAuB;IAQrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;YAqEvD,UAAU;IAuHxB,OAAO,CAAC,gBAAgB;IAkKxB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,uBAAuB;IAa/B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsC3B,OAAO,CAAC,uBAAuB;IAa/B,OAAO,CAAC,mBAAmB;IA+C3B,OAAO,CAAC,uBAAuB;CA+DhC"}
+{"version":3,"file":"TemporalAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/TemporalAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,uBAAuB,EAEvB,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAiB9C,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IAGjD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAK;gBAE5B,MAAM,EAAE,uBAAuB;IAQrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;YAqEvD,UAAU;IAwHxB,OAAO,CAAC,gBAAgB;IA2LxB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,uBAAuB;IAa/B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsC3B,OAAO,CAAC,uBAAuB;IAa/B,OAAO,CAAC,mBAAmB;IA+C3B,OAAO,CAAC,uBAAuB;CA+DhC"}

package/client/lib/services/assessment/modules/TemporalAssessor.js

@@ -144,7 +144,8 @@ export class TemporalAssessor extends BaseAssessor {
             }
         }
         // Analyze responses for temporal behavior changes
-        const result = this.analyzeResponses(tool, responses);
+        // Issue #168: Pass context for external API dependency awareness
+        const result = this.analyzeResponses(tool, responses, context);
         // Analyze definitions for mutation (rug pull via description change)
         const definitionMutation = this.mutationDetector.detectDefinitionMutation(definitionSnapshots);
         return {
@@ -167,7 +168,7 @@ export class TemporalAssessor extends BaseAssessor {
             severity: definitionMutation !== null || result.vulnerable ? "HIGH" : "NONE",
         };
     }
-    analyzeResponses(tool, responses) {
+    analyzeResponses(tool, responses, context) {
         if (responses.length === 0) {
             return {
                 tool: tool.name,
@@ -205,6 +206,17 @@ export class TemporalAssessor extends BaseAssessor {
             else if (isStateful) {
                 // Original stateful tool logic: schema comparison + behavioral content check
                 // Content variance is allowed as long as schema is consistent
+                // Issue #166: Check for isError variance first (external API behavior)
+                // For stateful tools, error vs success responses are expected from external APIs
+                const baselineIsError = responses[0].response?.isError === true;
+                const currentIsError = responses[i].response?.isError === true;
+                // Issue #168: Check context-based detection first, fall back to VarianceClassifier
+                const isExternalAPI = context.externalAPIDependencies?.toolsWithExternalAPIDependency.has(tool.name) ?? this.varianceClassifier.isExternalAPITool(tool);
+                if (baselineIsError !== currentIsError && isExternalAPI) {
+                    // External API tool with error vs success variance - LEGITIMATE, not a deviation
+                    this.logger.info(`${tool.name}: API error vs success variance at invocation ${i + 1} (expected for external API)`);
+                    continue; // Skip to next invocation, don't count as deviation
+                }
                 let isDifferent = !this.varianceClassifier.compareSchemas(responses[0].response, responses[i].response);
                 // Secondary detection: Check for content semantic changes (rug pull patterns)
                 // This catches cases where schema is same but content shifts from helpful to harmful
@@ -222,7 +234,8 @@ export class TemporalAssessor extends BaseAssessor {
             else if (isResourceCreating) {
                 // Issue #69: Use variance classification for resource-creating tools
                 // These need intelligent classification to distinguish ID variance from rug pulls
-                const classification = this.varianceClassifier.classifyVariance(responses[0].response, responses[i].response);
+                // Issue #166: Pass tool for external API error variance handling
+                const classification = this.varianceClassifier.classifyVariance(responses[0].response, responses[i].response, tool);
                 varianceDetails.push({
                     invocation: i + 1,
                     classification,

package/client/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"AlignmentChecker.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/annotations/AlignmentChecker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAC/D,OAAO,KAAK,EACV,oBAAoB,EACpB,gBAAgB,EAEhB,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACzB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,gCAAgC,CAAC;AAuFxC;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,kBAAkB,EAAE;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AA0CD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAE7D;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,OAAO,CAElD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,IAAI,GAAG,oBAAoB,CAiNnE;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,IAAI,GACT,oBAAoB,CAAC,kBAAkB,CAAC,CA6D1C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,GAAG,iBAAiB,EAAE,CAqBtE;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,IAAI,GAAG,mBAAmB,CAmD3E;AAqCD;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,IAAI,EACV,gBAAgB,EAAE,gBAAgB,EAClC,kBAAkB,CAAC,EAAE,wBAAwB,GAC5C,oBAAoB,CA0JtB;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,oBAAoB,EAAE,EAC/B,UAAU,EAAE,MAAM,GACjB,gBAAgB,CA8BlB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,oBAAoB,EAAE,EAC/B,UAAU,EAAE,MAAM,GACjB,sBAAsB,CA2BxB"}
+{"version":3,"file":"AlignmentChecker.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/annotations/AlignmentChecker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAC/D,OAAO,KAAK,EACV,oBAAoB,EACpB,gBAAgB,EAEhB,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACzB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,gCAAgC,CAAC;AAuFxC;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,kBAAkB,EAAE;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AA0CD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAE7D;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,OAAO,CAElD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,IAAI,GAAG,oBAAoB,CAiNnE;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,IAAI,GACT,oBAAoB,CAAC,kBAAkB,CAAC,CA6D1C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,GAAG,iBAAiB,EAAE,CAqBtE;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,IAAI,GAAG,mBAAmB,CAmD3E;AAqCD;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,IAAI,EACV,gBAAgB,EAAE,gBAAgB,EAClC,kBAAkB,CAAC,EAAE,wBAAwB,GAC5C,oBAAoB,CA0JtB;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,oBAAoB,EAAE,EAC/B,UAAU,EAAE,MAAM,GACjB,gBAAgB,CAoClB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,oBAAoB,EAAE,EAC/B,UAAU,EAAE,MAAM,GACjB,sBAAsB,CA2BxB"}

package/client/lib/services/assessment/modules/annotations/AlignmentChecker.js

@@ -433,8 +433,12 @@ export function determineAnnotationStatus(results, totalTools) {
     if (totalTools === 0)
         return "PASS";
     const annotatedCount = results.filter((r) => r.hasAnnotations).length;
-    const poisonedCount = results.filter((r) => r.descriptionPoisoning?.detected === true).length;
-    if (poisonedCount > 0)
+    // Issue #167: Only fail for actionable poisoning (MEDIUM or HIGH risk)
+    // LOW risk (e.g., length-only) is informational and should not cause FAIL
+    const actionablePoisonedCount = results.filter((r) => r.descriptionPoisoning?.detected === true &&
+        (r.descriptionPoisoning.riskLevel === "MEDIUM" ||
+            r.descriptionPoisoning.riskLevel === "HIGH")).length;
+    if (actionablePoisonedCount > 0)
         return "FAIL";
     const misalignedCount = results.filter((r) => r.alignmentStatus === "MISALIGNED").length;
     const destructiveWithoutHint = results.filter((r) => r.inferredBehavior?.expectedDestructive === true &&