@bryan-thompson/inspector-assessment 1.5.0 → 1.7.0

package/client/lib/services/assessment/modules/BaseAssessor.js

@@ -0,0 +1,171 @@
+/**
+ * Base Assessor Class
+ * Provides common functionality for all assessment modules
+ */
+export class BaseAssessor {
+    config;
+    testCount = 0;
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Common method to determine status based on pass rate
+     */
+    determineStatus(passed, total, threshold = 0.8) {
+        if (total === 0)
+            return "NEED_MORE_INFO";
+        const passRate = passed / total;
+        if (passRate >= threshold)
+            return "PASS";
+        if (passRate >= threshold * 0.5)
+            return "NEED_MORE_INFO";
+        return "FAIL";
+    }
+    /**
+     * Log assessment progress
+     */
+    log(message) {
+        console.log(`[${this.constructor.name}] ${message}`);
+    }
+    /**
+     * Log error
+     */
+    logError(message, error) {
+        console.error(`[${this.constructor.name}] ${message}`, error);
+    }
+    /**
+     * Get test count for this assessor
+     */
+    getTestCount() {
+        return this.testCount;
+    }
+    /**
+     * Reset test count
+     */
+    resetTestCount() {
+        this.testCount = 0;
+    }
+    /**
+     * Check if a feature is enabled in configuration
+     */
+    isFeatureEnabled(feature) {
+        return this.config.assessmentCategories?.[feature] ?? false;
+    }
+    /**
+     * Sleep for specified milliseconds (useful for rate limiting)
+     */
+    async sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Execute with timeout
+     */
+    async executeWithTimeout(promise, timeoutMs = this.config.testTimeout) {
+        const timeoutPromise = new Promise((_, reject) => {
+            setTimeout(() => reject(new Error(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
+        });
+        return Promise.race([promise, timeoutPromise]);
+    }
+    /**
+     * Safe JSON parse with error handling
+     */
+    safeJsonParse(text) {
+        try {
+            return JSON.parse(text);
+        }
+        catch (error) {
+            this.logError(`Failed to parse JSON: ${text}`, error);
+            return null;
+        }
+    }
+    /**
+     * Extract error message from various error types
+     */
+    extractErrorMessage(error) {
+        if (typeof error === "string")
+            return error;
+        if (error?.message)
+            return error.message;
+        if (error?.error)
+            return this.extractErrorMessage(error.error);
+        if (error?.content) {
+            if (Array.isArray(error.content)) {
+                return error.content
+                    .map((c) => c.text || c.content || "")
+                    .join(" ");
+            }
+            return error.content;
+        }
+        return JSON.stringify(error);
+    }
+    /**
+     * Check if a response indicates an error
+     * Handles various MCP response formats
+     *
+     * @param response - The response to check
+     * @param strictMode - If true, only check explicit error indicators (default: false)
+     */
+    isErrorResponse(response, strictMode = false) {
+        if (!response)
+            return false;
+        // Check explicit error flag first (always check these)
+        if (response.isError === true || response.error !== undefined) {
+            return true;
+        }
+        // In strict mode, only rely on explicit error indicators
+        // Used by FunctionalityAssessor where we expect valid responses
+        if (strictMode) {
+            return false;
+        }
+        // In non-strict mode, also check content for error patterns
+        // Used by ErrorHandlingAssessor where we're deliberately triggering errors
+        if (response.content) {
+            if (typeof response.content === "string") {
+                const lower = response.content.toLowerCase();
+                // Only flag if error appears at start or with strong indicators
+                return (lower.startsWith("error:") ||
+                    lower.startsWith("error ") ||
+                    lower.includes("error occurred") ||
+                    lower.includes("failed to") ||
+                    lower.includes("exception:"));
+            }
+            else if (Array.isArray(response.content)) {
+                // Check if any text content starts with error indicators
+                return response.content.some((c) => {
+                    if (c.type !== "text" || !c.text)
+                        return false;
+                    const lower = c.text.toLowerCase();
+                    return (lower.startsWith("error:") ||
+                        lower.startsWith("error ") ||
+                        lower.includes("error occurred") ||
+                        lower.includes("failed to") ||
+                        lower.includes("exception:"));
+                });
+            }
+        }
+        return false;
+    }
+    /**
+     * Extract error information from a response
+     */
+    extractErrorInfo(response) {
+        if (!response)
+            return {};
+        // Extract text from content array if present
+        let contentText;
+        if (Array.isArray(response.content)) {
+            const textContent = response.content.find((c) => c.type === "text");
+            contentText = textContent?.text;
+        }
+        else if (typeof response.content === "string") {
+            contentText = response.content;
+        }
+        return {
+            code: response.errorCode || response.code || response.error?.code,
+            message: response.errorMessage ||
+                response.message ||
+                response.error?.message ||
+                contentText,
+        };
+    }
+}

package/client/lib/services/assessment/modules/DocumentationAssessor.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Documentation Assessor Module
+ * Evaluates documentation quality and completeness
+ */
+import { DocumentationAssessment } from "../../../lib/assessmentTypes.js";
+import { BaseAssessor } from "./BaseAssessor.js";
+import { AssessmentContext } from "../AssessmentOrchestrator.js";
+export declare class DocumentationAssessor extends BaseAssessor {
+    assess(context: AssessmentContext): Promise<DocumentationAssessment>;
+    private analyzeDocumentation;
+    /**
+     * Extract functional example prompts from documentation.
+     * Only counts user-facing examples, not configuration or installation code.
+     */
+    private extractFunctionalExamples;
+    /**
+     * Check if a code block is non-functional (config, install, implementation).
+     */
+    private isNonFunctionalCodeBlock;
+    /**
+     * Score functional example quality (returns true if meets minimum threshold).
+     */
+    private scoreFunctionalExample;
+    /**
+     * Get line number for a position in content.
+     */
+    private getLineNumber;
+    /**
+     * Remove duplicate examples based on similarity.
+     */
+    private deduplicateExamples;
+    /**
+     * @deprecated Use extractFunctionalExamples() instead.
+     * This method counts ALL code blocks including configs and install commands.
+     */
+    private extractCodeExamples;
+    private checkInstallInstructions;
+    private checkUsageGuide;
+    private checkAPIReference;
+    private extractSection;
+    private determineDocumentationStatus;
+    private generateExplanation;
+    private generateRecommendations;
+}
+//# sourceMappingURL=DocumentationAssessor.d.ts.map

package/client/lib/services/assessment/modules/DocumentationAssessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/DocumentationAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAIxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,qBAAsB,SAAQ,YAAY;IAC/C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAkB1E,OAAO,CAAC,oBAAoB;IAsF5B;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAuEjC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAiBhC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAmC9B;;OAEG;IACH,OAAO,CAAC,aAAa;IAKrB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqB3B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAyC3B,OAAO,CAAC,wBAAwB;IAchC,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,4BAA4B;IAmBpC,OAAO,CAAC,mBAAmB;IAyB3B,OAAO,CAAC,uBAAuB;CA+BhC"}

package/client/lib/services/assessment/modules/DocumentationAssessor.js

@@ -0,0 +1,355 @@
+/**
+ * Documentation Assessor Module
+ * Evaluates documentation quality and completeness
+ */
+import { BaseAssessor } from "./BaseAssessor.js";
+export class DocumentationAssessor extends BaseAssessor {
+    async assess(context) {
+        this.log("Starting documentation assessment");
+        const readmeContent = context.readmeContent || "";
+        const metrics = this.analyzeDocumentation(readmeContent, context.tools);
+        const status = this.determineDocumentationStatus(metrics);
+        const explanation = this.generateExplanation(metrics);
+        const recommendations = this.generateRecommendations(metrics);
+        return {
+            metrics,
+            status,
+            explanation,
+            recommendations,
+        };
+    }
+    analyzeDocumentation(content, tools) {
+        const hasReadme = content.length > 0;
+        // Use new functional examples method that filters out configs/installs
+        const functionalExamples = this.extractFunctionalExamples(content);
+        // Keep old method for backward compatibility (extractedExamples field)
+        const allCodeExamples = this.extractCodeExamples(content);
+        const hasInstallInstructions = this.checkInstallInstructions(content);
+        const hasUsageGuide = this.checkUsageGuide(content);
+        const hasAPIReference = this.checkAPIReference(content);
+        // Check which tools are documented
+        const missingExamples = [];
+        let documentedToolsCount = 0;
+        if (tools && tools.length > 0) {
+            // Check each tool for documentation
+            for (const tool of tools) {
+                const toolName = tool.name;
+                const hasDescription = tool.description && tool.description.trim().length > 0;
+                // Check if tool is mentioned in headings (any level) or code examples
+                const headingRegex = new RegExp(`^#{1,6}\\s+${toolName}`, "mi");
+                const mentionRegex = new RegExp(`\\b${toolName}\\b`, "i");
+                const hasHeading = headingRegex.test(content);
+                const hasMention = mentionRegex.test(content);
+                // Count as documented if mentioned in README
+                if (hasHeading || hasMention) {
+                    documentedToolsCount++;
+                }
+                // Tool is missing if it has no description AND not documented in README
+                if (!hasDescription && !hasHeading && !hasMention) {
+                    missingExamples.push(toolName);
+                }
+            }
+        }
+        else {
+            // Fallback to generic example checking if no tools provided
+            if (functionalExamples.length < 1)
+                missingExamples.push("Basic usage example");
+            if (!functionalExamples.some((e) => e.description?.includes("error")) &&
+                !allCodeExamples.some((e) => e.description?.includes("error"))) {
+                missingExamples.push("Error handling example");
+            }
+            if (!functionalExamples.some((e) => e.description?.includes("config")) &&
+                !allCodeExamples.some((e) => e.description?.includes("config"))) {
+                missingExamples.push("Configuration example");
+            }
+        }
+        // Required examples: 3 minimum (Anthropic's standard)
+        const requiredExamples = 3;
+        // Count functional examples only (not configs/installs)
+        // For servers with tools, also count documented tools
+        const functionalExampleCount = functionalExamples.length +
+            (tools && tools.length > 0 ? documentedToolsCount : 0);
+        return {
+            hasReadme,
+            exampleCount: functionalExampleCount, // Use functional examples instead of all code blocks
+            requiredExamples,
+            missingExamples,
+            hasInstallInstructions,
+            hasUsageGuide,
+            hasAPIReference,
+            extractedExamples: allCodeExamples, // Keep for backward compatibility
+            installInstructions: hasInstallInstructions
+                ? this.extractSection(content, "install")
+                : undefined,
+            usageInstructions: hasUsageGuide
+                ? this.extractSection(content, "usage")
+                : undefined,
+        };
+    }
+    /**
+     * Extract functional example prompts from documentation.
+     * Only counts user-facing examples, not configuration or installation code.
+     */
+    extractFunctionalExamples(content) {
+        const functionalExamples = [];
+        // Pattern 1: Look for standalone lines with tool triggers (use context7, use library, with X)
+        const standalonePromptRegex = /^[ \t]*([A-Z][^\n]{10,300}?(?:use (?:context7|library|@?[\w-]+\/[\w-]+)|with \S+)[^\n]*?)[ \t]*$/gim;
+        let standaloneMatch;
+        while ((standaloneMatch = standalonePromptRegex.exec(content)) !== null) {
+            const prompt = standaloneMatch[1].trim();
+            // Filter out non-functional examples
+            if (!this.isNonFunctionalCodeBlock(prompt) &&
+                this.scoreFunctionalExample(prompt)) {
+                functionalExamples.push({
+                    code: prompt,
+                    language: "prompt",
+                    description: "Functional example prompt",
+                    lineNumber: this.getLineNumber(content, standaloneMatch.index),
+                });
+            }
+        }
+        // Pattern 2: Look for bulleted examples
+        const bulletedExampleRegex = /^[ \t]*[-*][ \t]+([A-Z][^\n]{10,300}?)[ \t]*$/gim;
+        let bulletMatch;
+        while ((bulletMatch = bulletedExampleRegex.exec(content)) !== null) {
+            const prompt = bulletMatch[1].trim();
+            // Must have tool trigger or action verb + technical term
+            if (!this.isNonFunctionalCodeBlock(prompt) &&
+                this.scoreFunctionalExample(prompt)) {
+                functionalExamples.push({
+                    code: prompt,
+                    language: "prompt",
+                    description: "Bulleted example",
+                    lineNumber: this.getLineNumber(content, bulletMatch.index),
+                });
+            }
+        }
+        // Pattern 3: Look for inline examples with labels
+        const inlineExampleRegex = /(?:Try|Example|Usage):\s*["`']([^"`']{10,300}?)["`']/gi;
+        let inlineMatch;
+        while ((inlineMatch = inlineExampleRegex.exec(content)) !== null) {
+            const prompt = inlineMatch[1].trim();
+            if (!this.isNonFunctionalCodeBlock(prompt) &&
+                this.scoreFunctionalExample(prompt)) {
+                functionalExamples.push({
+                    code: prompt,
+                    language: "prompt",
+                    description: "Inline example",
+                    lineNumber: this.getLineNumber(content, inlineMatch.index),
+                });
+            }
+        }
+        // Remove duplicates based on similar content
+        return this.deduplicateExamples(functionalExamples);
+    }
+    /**
+     * Check if a code block is non-functional (config, install, implementation).
+     */
+    isNonFunctionalCodeBlock(text) {
+        const excludePatterns = [
+            /^\s*{\s*["']mcpServers["']/i, // JSON config
+            /^\s*{\s*["']command["']/i, // MCP config
+            /^\s*(npx|npm|yarn|pnpm|docker|git)\s+/i, // Install commands
+            /^\s*FROM\s+\w+:/i, // Dockerfile
+            /^\s*import\s+.*\s+from/i, // Import statements
+            /^\s*\[.*\]\s*=\s*["']/i, // TOML/config assignments
+            /^\s*<\w+>/i, // HTML/XML tags
+            /^\s*(export|const|let|var|function)\s+/i, // Code declarations
+            /^\s*\/\//i, // Code comments
+            /^\s*#\s*\w+/i, // Shell comments or headers
+        ];
+        return excludePatterns.some((pattern) => pattern.test(text));
+    }
+    /**
+     * Score functional example quality (returns true if meets minimum threshold).
+     */
+    scoreFunctionalExample(text) {
+        let score = 0;
+        // Has clear action verb (create, configure, implement, show, etc.)
+        if (/\b(create|configure|implement|show|generate|build|write|add|get|set|use|run|start)\b/i.test(text)) {
+            score += 2;
+        }
+        // Includes tool trigger ("use context7", "with library", "use library")
+        if (/\b(?:use|with)\s+(?:context7|library|@?\w+\/\w+)\b/i.test(text)) {
+            score += 2;
+        }
+        // Is a complete sentence (starts with capital, ends with punctuation or is substantial)
+        if (/^[A-Z].{10,}/.test(text)) {
+            score += 1;
+        }
+        // Has technical context (mentions framework/library/technology)
+        if (/\b(Next\.js|React|Vue|Angular|PostgreSQL|MySQL|MongoDB|Redis|AWS|Cloudflare|API|HTTP|REST|GraphQL|TypeScript|JavaScript|Python|Docker|Kubernetes|Supabase|Firebase|Auth|JWT|OAuth)\b/i.test(text)) {
+            score += 1;
+        }
+        // Minimum score: 4 out of 6 points
+        return score >= 4;
+    }
+    /**
+     * Get line number for a position in content.
+     */
+    getLineNumber(content, position) {
+        const beforeMatch = content.substring(0, position);
+        return beforeMatch.split("\n").length;
+    }
+    /**
+     * Remove duplicate examples based on similarity.
+     */
+    deduplicateExamples(examples) {
+        const seen = new Set();
+        const unique = [];
+        for (const example of examples) {
+            // Create a normalized version for comparison
+            const normalized = example.code
+                .toLowerCase()
+                .replace(/[^\w\s]/g, "") // Remove punctuation
+                .replace(/\s+/g, " ") // Normalize multiple spaces to single space
+                .trim();
+            if (!seen.has(normalized)) {
+                seen.add(normalized);
+                unique.push(example);
+            }
+        }
+        return unique;
+    }
+    /**
+     * @deprecated Use extractFunctionalExamples() instead.
+     * This method counts ALL code blocks including configs and install commands.
+     */
+    extractCodeExamples(content) {
+        const examples = [];
+        const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+        let match;
+        let lineNumber = 0;
+        const lines = content.split("\n");
+        while ((match = codeBlockRegex.exec(content)) !== null) {
+            const language = match[1] || "plaintext";
+            const code = match[2].trim();
+            // Find line number
+            const position = match.index;
+            const beforeMatch = content.substring(0, position);
+            lineNumber = beforeMatch.split("\n").length;
+            // Try to find description from preceding lines
+            let description = "";
+            const lineIndex = lines.findIndex((_, i) => lines.slice(0, i + 1).join("\n").length >= position);
+            if (lineIndex > 0) {
+                const prevLine = lines[lineIndex - 1].trim();
+                if (prevLine && !prevLine.startsWith("#")) {
+                    description = prevLine;
+                }
+            }
+            examples.push({
+                code,
+                language,
+                description,
+                lineNumber,
+            });
+        }
+        return examples;
+    }
+    checkInstallInstructions(content) {
+        const installKeywords = [
+            "install",
+            "npm install",
+            "yarn add",
+            "pip install",
+            "setup",
+            "getting started",
+        ];
+        const contentLower = content.toLowerCase();
+        return installKeywords.some((keyword) => contentLower.includes(keyword));
+    }
+    checkUsageGuide(content) {
+        const usageKeywords = [
+            "usage",
+            "how to use",
+            "example",
+            "quick start",
+            "tutorial",
+        ];
+        const contentLower = content.toLowerCase();
+        return usageKeywords.some((keyword) => contentLower.includes(keyword));
+    }
+    checkAPIReference(content) {
+        const apiKeywords = [
+            "api",
+            "reference",
+            "methods",
+            "functions",
+            "parameters",
+            "returns",
+            "endpoints",
+        ];
+        const contentLower = content.toLowerCase();
+        return apiKeywords.some((keyword) => contentLower.includes(keyword));
+    }
+    extractSection(content, section) {
+        const sectionRegex = new RegExp(`#+\\s*${section}[\\s\\S]*?(?=\\n#|$)`, "gi");
+        const match = content.match(sectionRegex);
+        return match ? match[0].trim() : "";
+    }
+    determineDocumentationStatus(metrics) {
+        let score = 0;
+        const maxScore = 5;
+        if (metrics.hasReadme)
+            score++;
+        if (metrics.hasInstallInstructions)
+            score++;
+        if (metrics.hasUsageGuide)
+            score++;
+        if (metrics.hasAPIReference)
+            score++;
+        if (metrics.exampleCount >= metrics.requiredExamples)
+            score++;
+        const percentage = (score / maxScore) * 100;
+        if (percentage >= 80)
+            return "PASS";
+        if (percentage >= 50)
+            return "NEED_MORE_INFO";
+        return "FAIL";
+    }
+    generateExplanation(metrics) {
+        const parts = [];
+        if (!metrics.hasReadme) {
+            parts.push("No README documentation found.");
+        }
+        else {
+            parts.push(`README contains ${metrics.exampleCount} code examples.`);
+            const features = [];
+            if (metrics.hasInstallInstructions)
+                features.push("installation");
+            if (metrics.hasUsageGuide)
+                features.push("usage guide");
+            if (metrics.hasAPIReference)
+                features.push("API reference");
+            if (features.length > 0) {
+                parts.push(`Documentation includes: ${features.join(", ")}.`);
+            }
+            if (metrics.missingExamples.length > 0) {
+                parts.push(`Missing examples: ${metrics.missingExamples.join(", ")}.`);
+            }
+        }
+        return parts.join(" ");
+    }
+    generateRecommendations(metrics) {
+        const recommendations = [];
+        if (!metrics.hasReadme) {
+            recommendations.push("Create a comprehensive README.md file");
+        }
+        if (!metrics.hasInstallInstructions) {
+            recommendations.push("Add clear installation instructions");
+        }
+        if (!metrics.hasUsageGuide) {
+            recommendations.push("Include a usage guide with examples");
+        }
+        if (!metrics.hasAPIReference) {
+            recommendations.push("Document all available tools and parameters");
+        }
+        if (metrics.exampleCount < metrics.requiredExamples) {
+            recommendations.push(`Add ${metrics.requiredExamples - metrics.exampleCount} more code examples`);
+        }
+        metrics.missingExamples.forEach((missing) => {
+            recommendations.push(`Add ${missing}`);
+        });
+        return recommendations;
+    }
+}

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Error Handling Assessor Module
+ * Tests error handling and input validation
+ */
+import { ErrorHandlingAssessment } from "../../../lib/assessmentTypes.js";
+import { BaseAssessor } from "./BaseAssessor.js";
+import { AssessmentContext } from "../AssessmentOrchestrator.js";
+export declare class ErrorHandlingAssessor extends BaseAssessor {
+    assess(context: AssessmentContext): Promise<ErrorHandlingAssessment>;
+    private selectToolsForTesting;
+    private testToolErrorHandling;
+    private testMissingParameters;
+    private testWrongTypes;
+    private testInvalidValues;
+    private testExcessiveInput;
+    private getToolSchema;
+    private generateWrongTypeParams;
+    private generateInvalidValueParams;
+    private generateParamsWithValue;
+    private calculateMetrics;
+    private determineErrorHandlingStatus;
+    private generateExplanation;
+    private generateRecommendations;
+}
+//# sourceMappingURL=ErrorHandlingAssessor.d.ts.map

package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAIxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,qBAAsB,SAAQ,YAAY;IAC/C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAyC1E,OAAO,CAAC,qBAAqB;YA8Cf,qBAAqB;YAuBrB,qBAAqB;YAmGrB,cAAc;YAmFd,iBAAiB;YA8DjB,kBAAkB;IA6DhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAgC/B,OAAO,CAAC,0BAA0B;IAgClC,OAAO,CAAC,uBAAuB;IA4B/B,OAAO,CAAC,gBAAgB;IAoGxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,uBAAuB;CA4ChC"}