@bryan-thompson/inspector-assessment 1.36.5 → 1.38.0

package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.js

@@ -0,0 +1,317 @@
+/**
+ * External API Dependency Detector
+ *
+ * Identifies tools that depend on external APIs based on:
+ * 1. Tool name and description patterns (fast, always available)
+ * 2. Source code scanning for API calls (more accurate, when source available)
+ *
+ * This information enables downstream assessors to adjust their behavior:
+ * - TemporalAssessor: Relaxed variance thresholds for external API tools
+ * - FunctionalityAssessor: Accept API errors as valid responses
+ * - ErrorHandlingAssessor: Account for external service failures
+ *
+ * Issue #168: Enhanced with source code scanning support
+ *
+ * @module helpers/ExternalAPIDependencyDetector
+ */
+/**
+ * Detects external API dependencies in MCP tools based on name and description patterns.
+ * Designed to run during context preparation before assessors execute.
+ *
+ * @public
+ */
+export class ExternalAPIDependencyDetector {
+    /**
+     * Tool name patterns that suggest external API dependency.
+     * Uses word-boundary matching to prevent false positives.
+     *
+     * Extracted from VarianceClassifier (Issue #166) for reuse across modules.
+     */
+    EXTERNAL_API_PATTERNS = [
+        // API-related prefixes
+        "api",
+        "external",
+        "remote",
+        "live",
+        // Data type patterns (typically from external sources)
+        "weather",
+        "stock",
+        "price",
+        "market",
+        "currency",
+        "exchange",
+        "rate",
+        "forex",
+        // Service-specific prefixes
+        "wb", // World Bank
+        "worldbank",
+        // Action patterns suggesting external fetch
+        "fetch_from",
+        "poll",
+        "realtime",
+        "current",
+    ];
+    /**
+     * Description patterns that suggest external API dependency.
+     * Regex patterns for more flexible matching.
+     */
+    EXTERNAL_API_DESCRIPTION_PATTERNS = [
+        /external\s*(api|service)/i,
+        /fetche?s?\s*(from|data\s+from)/i,
+        /calls?\s*(external|remote)/i,
+        /live\s*(data|feed|stream)/i,
+        /real[- ]?time/i,
+        /world\s*bank/i,
+        /third[- ]?party\s*(api|service)/i,
+    ];
+    /**
+     * Source code patterns that indicate external API calls.
+     * Each pattern captures the URL in group 1.
+     *
+     * Issue #168: Patterns from proposal for source code scanning
+     */
+    SOURCE_CODE_API_PATTERNS = [
+        // fetch() calls - JavaScript/TypeScript
+        /fetch\s*\(\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+        // axios HTTP client calls
+        /axios\s*\.\s*(?:get|post|put|patch|delete|request)\s*\(\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+        // URL construction
+        /new\s+URL\s*\(\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+        // Common API base URL constants
+        /(?:API_BASE_URL|BASE_URL|API_URL|ENDPOINT)\s*=\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+        // Generic HTTP client .get/.post calls
+        /\.\s*(?:get|post)\s*\(\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+        // Python requests library
+        /requests\s*\.\s*(?:get|post|put|patch|delete)\s*\(\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+        // Python httpx library
+        /httpx\s*\.\s*(?:get|post|put|patch|delete)\s*\(\s*['"`](https?:\/\/[^'"`\s]+)/gi,
+    ];
+    /**
+     * URL patterns to skip (localhost, local networks, documentation)
+     */
+    LOCALHOST_PATTERNS = [
+        /localhost/i,
+        /127\.0\.0\.1/,
+        /0\.0\.0\.0/,
+        /192\.168\./,
+        /10\.\d+\./,
+        /172\.(?:1[6-9]|2[0-9]|3[01])\./,
+        /\.local\b/i,
+        /example\.com/i,
+        /test\.com/i,
+    ];
+    /**
+     * File patterns to skip during source code scanning
+     */
+    SKIP_FILE_PATTERNS = [
+        /node_modules/i,
+        /\.test\.(ts|js|tsx|jsx)$/i,
+        /\.spec\.(ts|js|tsx|jsx)$/i,
+        /\.d\.ts$/i,
+        /package-lock\.json$/i,
+        /yarn\.lock$/i,
+        /\.map$/i,
+        /\.git\//i,
+        /dist\//i,
+        /build\//i,
+        /__tests__\//i,
+        /__mocks__\//i,
+    ];
+    /**
+     * Detect external API dependencies from tools and optionally source code.
+     *
+     * Detection strategy:
+     * 1. Always analyze tool names and descriptions (fast, no source needed)
+     * 2. If sourceCodeFiles provided, scan for actual API calls (more accurate)
+     * 3. Combine results and compute confidence
+     *
+     * @param tools - List of MCP tools to analyze
+     * @param sourceCodeFiles - Optional map of file paths to content for source scanning
+     * @returns Detection results with tool names, domains, and implications
+     */
+    detect(tools, sourceCodeFiles) {
+        // Phase 1: Name/description pattern matching (always runs)
+        const toolsWithExternalAPI = new Set();
+        for (const tool of tools) {
+            if (this.isExternalAPITool(tool)) {
+                toolsWithExternalAPI.add(tool.name);
+            }
+        }
+        const detectedCount = toolsWithExternalAPI.size;
+        // Phase 2: Source code scanning (when available)
+        let domains;
+        let sourceCodeScanned = false;
+        if (sourceCodeFiles && sourceCodeFiles.size > 0) {
+            sourceCodeScanned = true;
+            domains = this.scanSourceCode(sourceCodeFiles);
+        }
+        // Compute confidence based on both detection methods
+        const confidence = this.computeConfidence(detectedCount, domains);
+        // Generate implications if any external APIs were detected
+        const hasExternalDependencies = detectedCount > 0 || (domains && domains.length > 0);
+        const implications = hasExternalDependencies
+            ? this.generateImplications(domains)
+            : undefined;
+        return {
+            toolsWithExternalAPIDependency: toolsWithExternalAPI,
+            detectedCount,
+            confidence,
+            detectedTools: Array.from(toolsWithExternalAPI),
+            domains,
+            sourceCodeScanned,
+            implications,
+        };
+    }
+    /** Maximum content length per file (500KB) - prevents ReDoS attacks */
+    MAX_CONTENT_LENGTH = 500_000;
+    /** Maximum matches per file - prevents runaway matching */
+    MAX_MATCHES_PER_FILE = 100;
+    /**
+     * Scan source code files for external API URLs.
+     * Returns unique external domains found in the code.
+     *
+     * @param sourceCodeFiles - Map of file paths to content
+     * @returns Array of unique external domain names
+     */
+    scanSourceCode(sourceCodeFiles) {
+        const domains = new Set();
+        sourceCodeFiles.forEach((content, filePath) => {
+            // Skip test files, node_modules, etc.
+            if (this.shouldSkipFile(filePath))
+                return;
+            // Skip oversized files to prevent ReDoS
+            if (content.length > this.MAX_CONTENT_LENGTH)
+                return;
+            // Try each API call pattern using matchAll (thread-safe, no lastIndex issues)
+            for (const pattern of this.SOURCE_CODE_API_PATTERNS) {
+                // Use Array.from for compatibility with older TS targets
+                const matches = Array.from(content.matchAll(pattern));
+                let matchCount = 0;
+                for (const match of matches) {
+                    if (matchCount >= this.MAX_MATCHES_PER_FILE)
+                        break;
+                    matchCount++;
+                    const url = match[1];
+                    // Skip localhost and local network URLs
+                    if (this.isLocalhost(url))
+                        continue;
+                    // Extract domain from URL
+                    const domain = this.extractDomain(url);
+                    if (domain) {
+                        domains.add(domain);
+                    }
+                }
+            }
+        });
+        return Array.from(domains);
+    }
+    /**
+     * Extract the hostname from a URL string.
+     *
+     * @param url - URL string (may be partial)
+     * @returns Hostname or null if extraction fails
+     */
+    extractDomain(url) {
+        try {
+            // Handle URLs that may not have protocol
+            const fullUrl = url.startsWith("http") ? url : `https://${url}`;
+            return new URL(fullUrl).hostname;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Check if a URL points to localhost or local network.
+     *
+     * @param url - URL string to check
+     * @returns true if URL is local
+     */
+    isLocalhost(url) {
+        return this.LOCALHOST_PATTERNS.some((pattern) => pattern.test(url));
+    }
+    /**
+     * Check if a file should be skipped during source scanning.
+     *
+     * @param filePath - Path to check
+     * @returns true if file should be skipped
+     */
+    shouldSkipFile(filePath) {
+        return this.SKIP_FILE_PATTERNS.some((pattern) => pattern.test(filePath));
+    }
+    /**
+     * Compute detection confidence based on both methods.
+     * Source code confirmation boosts confidence.
+     *
+     * @param toolCount - Number of tools detected via name/description
+     * @param domains - Domains found in source code
+     * @returns Confidence level
+     */
+    computeConfidence(toolCount, domains) {
+        const domainCount = domains?.length ?? 0;
+        // Both methods agree = high confidence
+        if (toolCount > 0 && domainCount > 0) {
+            return "high";
+        }
+        // Either method found multiple = high confidence
+        if (toolCount >= 3 || domainCount >= 3) {
+            return "high";
+        }
+        // Either method found something = medium confidence
+        if (toolCount > 0 || domainCount > 0) {
+            return "medium";
+        }
+        // Nothing found = low confidence (no external APIs)
+        return "low";
+    }
+    /**
+     * Generate implications for downstream assessors.
+     *
+     * @param domains - External domains found
+     * @returns Implications object
+     */
+    generateImplications(domains) {
+        const domainList = domains && domains.length > 0 ? domains.join(", ") : "external services";
+        return {
+            temporalVariance: "Expected - external data changes between invocations",
+            availabilityDependency: `Server depends on ${domainList} uptime`,
+            rateLimitingRisk: domains && domains.length > 0
+                ? `May encounter rate limits from ${domainList}`
+                : undefined,
+        };
+    }
+    /**
+     * Check if a single tool depends on external APIs.
+     * Uses BOTH name patterns AND description analysis for detection.
+     *
+     * @param tool - MCP tool to check
+     * @returns true if tool appears to depend on external APIs
+     */
+    isExternalAPITool(tool) {
+        const toolName = tool.name.toLowerCase();
+        const description = (tool.description || "").toLowerCase();
+        // Check name patterns with word-boundary matching
+        // "weather_api" matches "api" but "capital_gains" doesn't match "api"
+        const nameMatch = this.EXTERNAL_API_PATTERNS.some((pattern) => {
+            const wordBoundaryRegex = new RegExp(`(^|_|-)${pattern}($|_|-|s)`);
+            return wordBoundaryRegex.test(toolName);
+        });
+        // Check description for external API indicators
+        const descriptionMatch = this.EXTERNAL_API_DESCRIPTION_PATTERNS.some((regex) => regex.test(description));
+        return nameMatch || descriptionMatch;
+    }
+    /**
+     * Get the list of name patterns used for detection.
+     * Useful for debugging and documentation.
+     */
+    getNamePatterns() {
+        return this.EXTERNAL_API_PATTERNS;
+    }
+    /**
+     * Get the list of description patterns used for detection.
+     * Useful for debugging and documentation.
+     */
+    getDescriptionPatterns() {
+        return this.EXTERNAL_API_DESCRIPTION_PATTERNS;
+    }
+}

package/client/lib/services/assessment/helpers/StdioTransportDetector.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Stdio Transport Detector
+ *
+ * Identifies stdio transport support from multiple sources:
+ * 1. server.json manifest (packages[0].transport.type)
+ * 2. package.json bin entries (indicates CLI/stdio)
+ * 3. Source code scanning for transport patterns
+ * 4. Runtime transport configuration
+ *
+ * This fixes Issue #172: C6/F6 incorrectly fails for valid stdio servers
+ * because transport detection previously relied solely on serverInfo metadata.
+ *
+ * @module helpers/StdioTransportDetector
+ */
+import type { TransportMode } from "../config/architecturePatterns.js";
+/**
+ * Evidence source for transport detection
+ */
+export type TransportEvidenceSource = "server.json" | "package.json" | "source-code" | "runtime-config";
+/**
+ * Individual piece of transport detection evidence
+ */
+export interface TransportEvidence {
+    /** Source of the evidence */
+    source: TransportEvidenceSource;
+    /** Transport type detected */
+    transport: TransportMode;
+    /** Confidence level for this evidence */
+    confidence: "high" | "medium" | "low";
+    /** Human-readable detail about the detection */
+    detail: string;
+}
+/**
+ * Transport detection results
+ */
+export interface TransportDetectionResult {
+    /** Set of detected transport modes */
+    detectedTransports: Set<TransportMode>;
+    /** Overall detection confidence */
+    confidence: "high" | "medium" | "low";
+    /** All evidence collected during detection */
+    evidence: TransportEvidence[];
+    /** Whether stdio transport is supported */
+    supportsStdio: boolean;
+    /** Whether HTTP transport is supported */
+    supportsHTTP: boolean;
+    /** Whether SSE transport is supported */
+    supportsSSE: boolean;
+    /** Whether source code was scanned */
+    sourceCodeScanned: boolean;
+}
+/**
+ * server.json structure (partial - transport fields only)
+ */
+export interface ServerJsonTransport {
+    packages?: Array<{
+        transport?: {
+            type?: string;
+        };
+    }>;
+}
+/**
+ * package.json structure (partial - bin field only)
+ */
+export interface PackageJsonBin {
+    bin?: Record<string, string> | string;
+}
+/**
+ * Detects transport capabilities from multiple sources.
+ *
+ * Detection priority (highest confidence first):
+ * 1. Runtime transport configuration (actual runtime proof)
+ * 2. server.json transport declaration (explicit manifest)
+ * 3. package.json bin entries (strong CLI/stdio indicator)
+ * 4. Source code patterns (StdioServerTransport, mcp.run, etc.)
+ *
+ * @public
+ */
+export declare class StdioTransportDetector {
+    /**
+     * TypeScript/JavaScript patterns for stdio transport
+     */
+    private readonly STDIO_CODE_PATTERNS;
+    /**
+     * Python/FastMCP patterns for stdio transport
+     */
+    private readonly PYTHON_STDIO_PATTERNS;
+    /**
+     * HTTP/SSE transport patterns
+     */
+    private readonly HTTP_CODE_PATTERNS;
+    /**
+     * File patterns to skip during source code scanning
+     */
+    private readonly SKIP_FILE_PATTERNS;
+    /** Maximum file size for source scanning (500KB) */
+    private readonly MAX_FILE_SIZE;
+    /**
+     * Detect transport capabilities from all available sources.
+     *
+     * @param sourceCodeFiles - Map of file paths to content
+     * @param packageJson - Parsed package.json content
+     * @param serverJson - Parsed server.json content
+     * @param runtimeTransport - Transport type from runtime config
+     * @returns Transport detection results
+     */
+    detect(sourceCodeFiles?: Map<string, string>, packageJson?: PackageJsonBin, serverJson?: ServerJsonTransport, runtimeTransport?: TransportMode): TransportDetectionResult;
+    /**
+     * Scan source code files for transport patterns.
+     *
+     * @param sourceCodeFiles - Map of file paths to content
+     * @returns Array of evidence from source code analysis
+     */
+    private scanSourceCode;
+    /**
+     * Check if a transport type is valid.
+     */
+    private isValidTransport;
+    /**
+     * Check if a file should be skipped during scanning.
+     */
+    private shouldSkipFile;
+    /**
+     * Shorten file path for display.
+     */
+    private shortenPath;
+    /**
+     * Compute overall confidence from collected evidence.
+     *
+     * Confidence rules:
+     * - High: Any high-confidence evidence present
+     * - Medium: Only medium-confidence evidence OR multiple sources agree
+     * - Low: No evidence or only weak patterns
+     */
+    private computeConfidence;
+}
+//# sourceMappingURL=StdioTransportDetector.d.ts.map

package/client/lib/services/assessment/helpers/StdioTransportDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StdioTransportDetector.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/helpers/StdioTransportDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAC/B,aAAa,GACb,cAAc,GACd,aAAa,GACb,gBAAgB,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,6BAA6B;IAC7B,MAAM,EAAE,uBAAuB,CAAC;IAChC,8BAA8B;IAC9B,SAAS,EAAE,aAAa,CAAC;IACzB,yCAAyC;IACzC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,sCAAsC;IACtC,kBAAkB,EAAE,GAAG,CAAC,aAAa,CAAC,CAAC;IACvC,mCAAmC;IACnC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,8CAA8C;IAC9C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,2CAA2C;IAC3C,aAAa,EAAE,OAAO,CAAC;IACvB,0CAA0C;IAC1C,YAAY,EAAE,OAAO,CAAC;IACtB,yCAAyC;IACzC,WAAW,EAAE,OAAO,CAAC;IACrB,sCAAsC;IACtC,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,SAAS,CAAC,EAAE;YACV,IAAI,CAAC,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC;CACvC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,sBAAsB;IACjC;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAwBlC;IAEF;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAoBpC;IAEF;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAwCjC;IAEF;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAejC;IAEF,oDAAoD;IACpD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAW;IAEzC;;;;;;;;OAQG;IACH,MAAM,CACJ,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACrC,WAAW,CAAC,EAAE,cAAc,EAC5B,UAAU,CAAC,EAAE,mBAAmB,EAChC,gBAAgB,CAAC,EAAE,aAAa,GAC/B,wBAAwB;IAwE3B;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IA4DtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAIxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,WAAW;IAQnB;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;CA0B1B"}