@bryan-thompson/inspector-assessment 1.37.0 → 1.38.0

package/client/lib/services/assessment/helpers/StdioTransportDetector.js

@@ -0,0 +1,315 @@
+/**
+ * Stdio Transport Detector
+ *
+ * Identifies stdio transport support from multiple sources:
+ * 1. server.json manifest (packages[0].transport.type)
+ * 2. package.json bin entries (indicates CLI/stdio)
+ * 3. Source code scanning for transport patterns
+ * 4. Runtime transport configuration
+ *
+ * This fixes Issue #172: C6/F6 incorrectly fails for valid stdio servers
+ * because transport detection previously relied solely on serverInfo metadata.
+ *
+ * @module helpers/StdioTransportDetector
+ */
+/**
+ * Detects transport capabilities from multiple sources.
+ *
+ * Detection priority (highest confidence first):
+ * 1. Runtime transport configuration (actual runtime proof)
+ * 2. server.json transport declaration (explicit manifest)
+ * 3. package.json bin entries (strong CLI/stdio indicator)
+ * 4. Source code patterns (StdioServerTransport, mcp.run, etc.)
+ *
+ * @public
+ */
+export class StdioTransportDetector {
+    /**
+     * TypeScript/JavaScript patterns for stdio transport
+     */
+    STDIO_CODE_PATTERNS = [
+        {
+            pattern: /StdioServerTransport/,
+            description: "Uses StdioServerTransport class",
+        },
+        {
+            pattern: /from\s*['"]@modelcontextprotocol\/sdk\/server\/stdio/,
+            description: "Imports MCP SDK stdio module",
+        },
+        {
+            pattern: /from\s*['"].*\/stdio/,
+            description: "Imports stdio transport module",
+        },
+        {
+            pattern: /transport\s*[:=]\s*['"]stdio['"]/i,
+            description: "Declares transport as stdio",
+        },
+        {
+            pattern: /createStdioTransport/,
+            description: "Creates stdio transport",
+        },
+    ];
+    /**
+     * Python/FastMCP patterns for stdio transport
+     */
+    PYTHON_STDIO_PATTERNS = [
+        {
+            pattern: /mcp\.run\s*\(\s*transport\s*=\s*['"]stdio['"]/,
+            description: "FastMCP run with stdio transport",
+        },
+        {
+            pattern: /StdioTransport/,
+            description: "Uses StdioTransport class",
+        },
+        {
+            pattern: /transport\s*=\s*['"]stdio['"]/i,
+            description: "Python stdio transport declaration",
+        },
+        {
+            pattern: /from\s+mcp\.server\.stdio\s+import/,
+            description: "Imports MCP stdio module (Python)",
+        },
+    ];
+    /**
+     * HTTP/SSE transport patterns
+     */
+    HTTP_CODE_PATTERNS = [
+        {
+            pattern: /StreamableHTTPServerTransport/,
+            transport: "http",
+            description: "Uses StreamableHTTPServerTransport",
+        },
+        {
+            pattern: /SSEServerTransport/,
+            transport: "sse",
+            description: "Uses SSEServerTransport",
+        },
+        {
+            pattern: /transport\s*[:=]\s*['"]http['"]/i,
+            transport: "http",
+            description: "Declares transport as http",
+        },
+        {
+            pattern: /transport\s*[:=]\s*['"]sse['"]/i,
+            transport: "sse",
+            description: "Declares transport as sse",
+        },
+        {
+            pattern: /transport\s*[:=]\s*['"]streamable-http['"]/i,
+            transport: "http",
+            description: "Declares transport as streamable-http",
+        },
+        {
+            pattern: /express|fastify|koa|hono/i,
+            transport: "http",
+            description: "Uses HTTP framework",
+        },
+        {
+            pattern: /app\.listen\s*\(/,
+            transport: "http",
+            description: "HTTP server listen call",
+        },
+    ];
+    /**
+     * File patterns to skip during source code scanning
+     */
+    SKIP_FILE_PATTERNS = [
+        /node_modules/i,
+        /\.test\.(ts|js|tsx|jsx|py)$/i,
+        /\.spec\.(ts|js|tsx|jsx|py)$/i,
+        /\.d\.ts$/i,
+        /package-lock\.json$/i,
+        /yarn\.lock$/i,
+        /\.map$/i,
+        /\.git\//i,
+        /dist\//i,
+        /build\//i,
+        /__tests__\//i,
+        /__mocks__\//i,
+        /__pycache__\//i,
+        /\.pytest_cache\//i,
+    ];
+    /** Maximum file size for source scanning (500KB) */
+    MAX_FILE_SIZE = 500_000;
+    /**
+     * Detect transport capabilities from all available sources.
+     *
+     * @param sourceCodeFiles - Map of file paths to content
+     * @param packageJson - Parsed package.json content
+     * @param serverJson - Parsed server.json content
+     * @param runtimeTransport - Transport type from runtime config
+     * @returns Transport detection results
+     */
+    detect(sourceCodeFiles, packageJson, serverJson, runtimeTransport) {
+        const evidence = [];
+        const detectedTransports = new Set();
+        // 1. Runtime transport config (highest confidence)
+        if (runtimeTransport) {
+            detectedTransports.add(runtimeTransport);
+            evidence.push({
+                source: "runtime-config",
+                transport: runtimeTransport,
+                confidence: "high",
+                detail: `Runtime transport configured as ${runtimeTransport}`,
+            });
+        }
+        // 2. server.json transport declaration
+        if (serverJson?.packages?.[0]?.transport?.type) {
+            const transportType = serverJson.packages[0].transport
+                .type;
+            if (this.isValidTransport(transportType)) {
+                detectedTransports.add(transportType);
+                evidence.push({
+                    source: "server.json",
+                    transport: transportType,
+                    confidence: "high",
+                    detail: `server.json declares transport.type="${transportType}"`,
+                });
+            }
+        }
+        // 3. package.json bin entries (strong stdio indicator)
+        if (packageJson?.bin) {
+            const binEntry = typeof packageJson.bin === "string"
+                ? packageJson.bin
+                : Object.keys(packageJson.bin)[0];
+            if (binEntry) {
+                detectedTransports.add("stdio");
+                evidence.push({
+                    source: "package.json",
+                    transport: "stdio",
+                    confidence: "high",
+                    detail: `package.json has bin entry (CLI tools use stdio)`,
+                });
+            }
+        }
+        // 4. Source code scanning
+        const sourceCodeScanned = sourceCodeFiles !== undefined && sourceCodeFiles.size > 0;
+        if (sourceCodeFiles !== undefined && sourceCodeFiles.size > 0) {
+            const sourceEvidence = this.scanSourceCode(sourceCodeFiles);
+            for (const ev of sourceEvidence) {
+                evidence.push(ev);
+                detectedTransports.add(ev.transport);
+            }
+        }
+        // Compute overall confidence
+        const confidence = this.computeConfidence(evidence);
+        return {
+            detectedTransports,
+            confidence,
+            evidence,
+            supportsStdio: detectedTransports.has("stdio"),
+            supportsHTTP: detectedTransports.has("http"),
+            supportsSSE: detectedTransports.has("sse"),
+            sourceCodeScanned,
+        };
+    }
+    /**
+     * Scan source code files for transport patterns.
+     *
+     * @param sourceCodeFiles - Map of file paths to content
+     * @returns Array of evidence from source code analysis
+     */
+    scanSourceCode(sourceCodeFiles) {
+        const evidence = [];
+        const foundPatterns = new Set(); // Deduplicate
+        sourceCodeFiles.forEach((content, filePath) => {
+            // Skip test files, node_modules, etc.
+            if (this.shouldSkipFile(filePath))
+                return;
+            // Skip oversized files
+            if (content.length > this.MAX_FILE_SIZE)
+                return;
+            // Check stdio patterns (TypeScript/JavaScript)
+            for (const { pattern, description } of this.STDIO_CODE_PATTERNS) {
+                const key = `stdio:${description}`;
+                if (!foundPatterns.has(key) && pattern.test(content)) {
+                    foundPatterns.add(key);
+                    evidence.push({
+                        source: "source-code",
+                        transport: "stdio",
+                        confidence: "medium",
+                        detail: `${description} in ${this.shortenPath(filePath)}`,
+                    });
+                }
+            }
+            // Check stdio patterns (Python)
+            for (const { pattern, description } of this.PYTHON_STDIO_PATTERNS) {
+                const key = `stdio-py:${description}`;
+                if (!foundPatterns.has(key) && pattern.test(content)) {
+                    foundPatterns.add(key);
+                    evidence.push({
+                        source: "source-code",
+                        transport: "stdio",
+                        confidence: "medium",
+                        detail: `${description} in ${this.shortenPath(filePath)}`,
+                    });
+                }
+            }
+            // Check HTTP/SSE patterns
+            for (const { pattern, transport, description } of this
+                .HTTP_CODE_PATTERNS) {
+                const key = `${transport}:${description}`;
+                if (!foundPatterns.has(key) && pattern.test(content)) {
+                    foundPatterns.add(key);
+                    evidence.push({
+                        source: "source-code",
+                        transport,
+                        confidence: "medium",
+                        detail: `${description} in ${this.shortenPath(filePath)}`,
+                    });
+                }
+            }
+        });
+        return evidence;
+    }
+    /**
+     * Check if a transport type is valid.
+     */
+    isValidTransport(transport) {
+        return ["stdio", "http", "sse"].includes(transport);
+    }
+    /**
+     * Check if a file should be skipped during scanning.
+     */
+    shouldSkipFile(filePath) {
+        return this.SKIP_FILE_PATTERNS.some((pattern) => pattern.test(filePath));
+    }
+    /**
+     * Shorten file path for display.
+     */
+    shortenPath(filePath) {
+        const parts = filePath.split("/");
+        if (parts.length > 2) {
+            return `.../${parts.slice(-2).join("/")}`;
+        }
+        return filePath;
+    }
+    /**
+     * Compute overall confidence from collected evidence.
+     *
+     * Confidence rules:
+     * - High: Any high-confidence evidence present
+     * - Medium: Only medium-confidence evidence OR multiple sources agree
+     * - Low: No evidence or only weak patterns
+     */
+    computeConfidence(evidence) {
+        if (evidence.length === 0) {
+            return "low";
+        }
+        // Any high-confidence evidence = overall high
+        if (evidence.some((e) => e.confidence === "high")) {
+            return "high";
+        }
+        // Multiple sources agreeing boosts confidence
+        const uniqueSources = new Set(evidence.map((e) => e.source));
+        if (uniqueSources.size >= 2) {
+            return "high";
+        }
+        // Multiple medium-confidence findings = overall medium
+        if (evidence.length >= 2) {
+            return "medium";
+        }
+        // Single medium-confidence finding
+        return "medium";
+    }
+}

package/client/lib/services/assessment/helpers/ToolAnnotationExtractor.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Tool Annotation Extractor
+ *
+ * Pre-extracts annotations from all tools for security assessment context.
+ * Issue #170: Enables annotation-aware security severity adjustment.
+ *
+ * This helper is used during context preparation to extract annotations
+ * BEFORE security testing begins, allowing SecurityAssessor to adjust
+ * vulnerability severity based on tool capabilities.
+ *
+ * @module helpers/ToolAnnotationExtractor
+ */
+import type { Tool } from "@modelcontextprotocol/sdk/types.js";
+import type { ToolAnnotationsContext } from "../../../lib/assessment/coreTypes.js";
+/**
+ * Extract tool annotations context from all tools.
+ *
+ * Iterates through all tools, extracts annotations using the existing
+ * 5-tier priority system, and computes server-level flags for read-only
+ * and closed-world servers.
+ *
+ * @param tools - Array of MCP tools to extract annotations from
+ * @returns ToolAnnotationsContext with per-tool annotations and server flags
+ *
+ * @example
+ * ```typescript
+ * const context = extractToolAnnotationsContext(tools);
+ * if (context.serverIsReadOnly) {
+ *   console.log("Server is 100% read-only");
+ * }
+ * ```
+ */
+export declare function extractToolAnnotationsContext(tools: Tool[]): ToolAnnotationsContext;
+//# sourceMappingURL=ToolAnnotationExtractor.d.ts.map

package/client/lib/services/assessment/helpers/ToolAnnotationExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ToolAnnotationExtractor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/helpers/ToolAnnotationExtractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAE/D,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,4BAA4B,CAAC;AAEpC;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,IAAI,EAAE,GACZ,sBAAsB,CAkExB"}

package/client/lib/services/assessment/helpers/ToolAnnotationExtractor.js

@@ -0,0 +1,85 @@
+/**
+ * Tool Annotation Extractor
+ *
+ * Pre-extracts annotations from all tools for security assessment context.
+ * Issue #170: Enables annotation-aware security severity adjustment.
+ *
+ * This helper is used during context preparation to extract annotations
+ * BEFORE security testing begins, allowing SecurityAssessor to adjust
+ * vulnerability severity based on tool capabilities.
+ *
+ * @module helpers/ToolAnnotationExtractor
+ */
+import { extractAnnotations } from "../modules/annotations/AlignmentChecker.js";
+/**
+ * Extract tool annotations context from all tools.
+ *
+ * Iterates through all tools, extracts annotations using the existing
+ * 5-tier priority system, and computes server-level flags for read-only
+ * and closed-world servers.
+ *
+ * @param tools - Array of MCP tools to extract annotations from
+ * @returns ToolAnnotationsContext with per-tool annotations and server flags
+ *
+ * @example
+ * ```typescript
+ * const context = extractToolAnnotationsContext(tools);
+ * if (context.serverIsReadOnly) {
+ *   console.log("Server is 100% read-only");
+ * }
+ * ```
+ */
+export function extractToolAnnotationsContext(tools) {
+    const toolAnnotations = new Map();
+    let readOnlyCount = 0;
+    let closedWorldCount = 0;
+    let annotatedCount = 0;
+    for (const tool of tools) {
+        // Use existing extraction logic with 5-tier priority
+        const extracted = extractAnnotations(tool);
+        // Validate extracted data has valid source
+        if (!extracted || !extracted.source) {
+            continue; // Skip tools with invalid extraction
+        }
+        // Convert to SecurityAnnotations (subset of ExtractedAnnotations)
+        // Ensure boolean fields are properly validated
+        const securityAnnotations = {
+            readOnlyHint: typeof extracted.readOnlyHint === "boolean"
+                ? extracted.readOnlyHint
+                : undefined,
+            destructiveHint: typeof extracted.destructiveHint === "boolean"
+                ? extracted.destructiveHint
+                : undefined,
+            idempotentHint: typeof extracted.idempotentHint === "boolean"
+                ? extracted.idempotentHint
+                : undefined,
+            openWorldHint: typeof extracted.openWorldHint === "boolean"
+                ? extracted.openWorldHint
+                : undefined,
+            source: extracted.source,
+        };
+        toolAnnotations.set(tool.name, securityAnnotations);
+        // Count tools with annotations
+        if (extracted.source !== "none") {
+            annotatedCount++;
+            // Count read-only tools
+            if (extracted.readOnlyHint === true) {
+                readOnlyCount++;
+            }
+            // Count closed-world tools
+            if (extracted.openWorldHint === false) {
+                closedWorldCount++;
+            }
+        }
+    }
+    const totalCount = tools.length;
+    return {
+        toolAnnotations,
+        // Server is read-only only if ALL annotated tools are read-only
+        serverIsReadOnly: annotatedCount > 0 && readOnlyCount === annotatedCount,
+        // Server is closed only if ALL annotated tools are closed
+        serverIsClosed: annotatedCount > 0 && closedWorldCount === annotatedCount,
+        annotatedToolCount: annotatedCount,
+        totalToolCount: totalCount,
+    };
+}

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts

@@ -20,16 +20,22 @@ export declare class ErrorHandlingAssessor extends BaseAssessor {
     private testExcessiveInput;
     private getToolSchema;
     private generateWrongTypeParams;
+    /**
+     * Issue #173: Return type for generateInvalidValueParams with metadata
+     * Tracks which parameter is being tested and whether it's required
+     */
     private generateInvalidValueParams;
     private generateParamsWithValue;
     /**
      * Analyze invalid_values response to determine scoring impact
      * Issue #99: Contextual empty string validation scoring
+     * Issue #173: Bonus points for suggestions and graceful degradation
      *
      * Classifications:
      * - safe_rejection: Tool rejected with error (no penalty)
      * - safe_reflection: Tool stored/echoed without executing (no penalty)
      * - defensive_programming: Tool handled gracefully (no penalty)
+     * - graceful_degradation: Optional param handled with neutral response (no penalty + bonus)
      * - execution_detected: Tool executed input (penalty)
      * - unknown: Cannot determine (partial penalty)
      */
@@ -43,6 +49,17 @@ export declare class ErrorHandlingAssessor extends BaseAssessor {
      * Examples: "Deleted 0 keys", "No results found", "Query returned 0"
      */
     private isDefensiveProgrammingResponse;
+    /**
+     * Issue #173: Detect helpful suggestion patterns in error responses
+     * Patterns like: "Did you mean: Button, Checkbox?"
+     * Returns extracted suggestions for bonus scoring
+     */
+    private detectSuggestionPatterns;
+    /**
+     * Issue #173: Check for neutral/graceful responses on optional parameters
+     * These indicate the tool handled empty/missing optional input appropriately
+     */
+    private isNeutralGracefulResponse;
     private calculateMetrics;
     private determineErrorHandlingStatus;
     private generateExplanation;

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAKxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,iBAAiB,CAA4B;IACrD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,eAAe,CAAkB;gBAE7B,MAAM,EAAE,uBAAuB;IAOrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IA6G1E,OAAO,CAAC,qBAAqB;YAuDf,qBAAqB;YAiCrB,qBAAqB;YA+IrB,cAAc;YA8Hd,iBAAiB;YAyGjB,kBAAkB;IAwGhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAkC/B,OAAO,CAAC,0BAA0B;IAkClC,OAAO,CAAC,uBAAuB;IA4B/B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,4BAA4B;IAgEpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;;OAGG;IACH,OAAO,CAAC,8BAA8B;IAetC,OAAO,CAAC,gBAAgB;IA8GxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAgB9B,OAAO,CAAC,uBAAuB;CA4ChC"}
+{"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAKxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,iBAAiB,CAA4B;IACrD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,eAAe,CAAkB;gBAE7B,MAAM,EAAE,uBAAuB;IAOrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IA6G1E,OAAO,CAAC,qBAAqB;YAuDf,qBAAqB;YAiCrB,qBAAqB;YA+IrB,cAAc;YA8Hd,iBAAiB;YAyIjB,kBAAkB;IAwGhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAkC/B;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IAkDlC,OAAO,CAAC,uBAAuB;IA4B/B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,4BAA4B;IAyFpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;;OAGG;IACH,OAAO,CAAC,8BAA8B;IAetC;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAuChC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAyBjC,OAAO,CAAC,gBAAgB;IAiJxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAgB9B,OAAO,CAAC,uBAAuB;CA4ChC"}