@bryan-thompson/inspector-assessment 1.5.0 → 1.7.0

package/client/lib/services/assessment/modules/UsabilityAssessor.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Usability Assessor Module
+ * Evaluates tool naming, parameter clarity, and best practices
+ */
+import { UsabilityAssessment } from "../../../lib/assessmentTypes.js";
+import { BaseAssessor } from "./BaseAssessor.js";
+import { AssessmentContext } from "../AssessmentOrchestrator.js";
+export declare class UsabilityAssessor extends BaseAssessor {
+    assess(context: AssessmentContext): Promise<UsabilityAssessment>;
+    private analyzeUsability;
+    private analyzeNamingConvention;
+    private analyzeParameterClarity;
+    private checkDescriptions;
+    private checkBestPractices;
+    private isDescriptiveName;
+    private getToolSchema;
+    private determineUsabilityStatus;
+    private generateExplanation;
+    private generateRecommendations;
+}
+//# sourceMappingURL=UsabilityAssessor.d.ts.map

package/client/lib/services/assessment/modules/UsabilityAssessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UsabilityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/UsabilityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EAGpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,iBAAkB,SAAQ,YAAY;IAC3C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAgBtE,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,uBAAuB;IAqC/B,OAAO,CAAC,uBAAuB;IAwC/B,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,kBAAkB;IA6C1B,OAAO,CAAC,iBAAiB;IAoCzB,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,wBAAwB;IAkBhC,OAAO,CAAC,mBAAmB;IAqB3B,OAAO,CAAC,uBAAuB;CAyBhC"}

package/client/lib/services/assessment/modules/UsabilityAssessor.js

@@ -0,0 +1,241 @@
+/**
+ * Usability Assessor Module
+ * Evaluates tool naming, parameter clarity, and best practices
+ */
+import { BaseAssessor } from "./BaseAssessor.js";
+export class UsabilityAssessor extends BaseAssessor {
+    async assess(context) {
+        this.log("Starting usability assessment");
+        const metrics = this.analyzeUsability(context.tools);
+        const status = this.determineUsabilityStatus(metrics);
+        const explanation = this.generateExplanation(metrics, context.tools);
+        const recommendations = this.generateRecommendations(metrics);
+        return {
+            metrics,
+            status,
+            explanation,
+            recommendations,
+        };
+    }
+    analyzeUsability(tools) {
+        const toolNamingConvention = this.analyzeNamingConvention(tools);
+        const parameterClarity = this.analyzeParameterClarity(tools);
+        const hasHelpfulDescriptions = this.checkDescriptions(tools);
+        const followsBestPractices = this.checkBestPractices(tools);
+        return {
+            toolNamingConvention,
+            parameterClarity,
+            hasHelpfulDescriptions,
+            followsBestPractices,
+        };
+    }
+    analyzeNamingConvention(tools) {
+        if (tools.length === 0)
+            return "consistent";
+        const namingPatterns = {
+            camelCase: 0,
+            snake_case: 0,
+            kebab_case: 0,
+            PascalCase: 0,
+        };
+        for (const tool of tools) {
+            const name = tool.name;
+            if (/^[a-z][a-zA-Z0-9]*$/.test(name)) {
+                namingPatterns.camelCase++;
+            }
+            else if (/^[a-z]+(_[a-z]+)*$/.test(name)) {
+                namingPatterns.snake_case++;
+            }
+            else if (/^[a-z]+(-[a-z]+)*$/.test(name)) {
+                namingPatterns.kebab_case++;
+            }
+            else if (/^[A-Z][a-zA-Z0-9]*$/.test(name)) {
+                namingPatterns.PascalCase++;
+            }
+        }
+        // Check if one pattern dominates (>70%)
+        const total = tools.length;
+        const threshold = total * 0.7;
+        for (const count of Object.values(namingPatterns)) {
+            if (count >= threshold) {
+                return "consistent";
+            }
+        }
+        return "inconsistent";
+    }
+    analyzeParameterClarity(tools) {
+        if (tools.length === 0)
+            return "clear";
+        let clearCount = 0;
+        let unclearCount = 0;
+        for (const tool of tools) {
+            const schema = this.getToolSchema(tool);
+            if (!schema?.properties)
+                continue;
+            for (const [paramName, paramDef] of Object.entries(schema.properties)) {
+                // Check if parameter name is self-descriptive
+                if (this.isDescriptiveName(paramName)) {
+                    clearCount++;
+                }
+                else {
+                    unclearCount++;
+                }
+                // Check if parameter has description
+                if (paramDef.description) {
+                    clearCount++;
+                }
+                else {
+                    unclearCount++;
+                }
+            }
+        }
+        const total = clearCount + unclearCount;
+        if (total === 0)
+            return "clear";
+        const clarityRatio = clearCount / total;
+        if (clarityRatio >= 0.8)
+            return "clear";
+        if (clarityRatio <= 0.3)
+            return "unclear";
+        return "mixed";
+    }
+    checkDescriptions(tools) {
+        if (tools.length === 0)
+            return false;
+        let toolsWithDescriptions = 0;
+        for (const tool of tools) {
+            if (tool.description && tool.description.length > 10) {
+                toolsWithDescriptions++;
+            }
+        }
+        // Consider helpful if >70% of tools have descriptions
+        return toolsWithDescriptions / tools.length >= 0.7;
+    }
+    checkBestPractices(tools) {
+        const practices = {
+            hasVersioning: false,
+            hasErrorHandling: false,
+            hasValidation: false,
+            hasDocumentation: false,
+        };
+        // Check various best practices
+        for (const tool of tools) {
+            const schema = this.getToolSchema(tool);
+            // Check for validation (required fields, enums, etc.)
+            if (schema?.required && schema.required.length > 0) {
+                practices.hasValidation = true;
+            }
+            // Check for proper parameter constraints
+            if (schema?.properties) {
+                for (const prop of Object.values(schema.properties)) {
+                    if (prop.enum ||
+                        prop.minimum !== undefined ||
+                        prop.maximum !== undefined) {
+                        practices.hasValidation = true;
+                    }
+                }
+            }
+            // Check for documentation
+            if (tool.description) {
+                practices.hasDocumentation = true;
+            }
+        }
+        // Count how many practices are followed
+        const followedPractices = Object.values(practices).filter((v) => v).length;
+        // Consider following best practices if at least 2 are met
+        return followedPractices >= 2;
+    }
+    isDescriptiveName(name) {
+        // Check if name is self-descriptive
+        const goodNames = [
+            "query",
+            "search",
+            "input",
+            "output",
+            "data",
+            "content",
+            "message",
+            "text",
+            "file",
+            "path",
+            "url",
+            "name",
+            "id",
+            "value",
+            "result",
+            "response",
+            "request",
+            "params",
+        ];
+        const nameLower = name.toLowerCase();
+        // Check if name contains any good keywords
+        for (const goodName of goodNames) {
+            if (nameLower.includes(goodName)) {
+                return true;
+            }
+        }
+        // Check if name is not too short or cryptic
+        return name.length > 3 && !/^[a-z]$/.test(name);
+    }
+    getToolSchema(tool) {
+        if (!tool.inputSchema)
+            return null;
+        return typeof tool.inputSchema === "string"
+            ? this.safeJsonParse(tool.inputSchema)
+            : tool.inputSchema;
+    }
+    determineUsabilityStatus(metrics) {
+        let score = 0;
+        const maxScore = 4;
+        if (metrics.toolNamingConvention === "consistent")
+            score++;
+        if (metrics.parameterClarity === "clear")
+            score++;
+        if (metrics.hasHelpfulDescriptions)
+            score++;
+        if (metrics.followsBestPractices)
+            score++;
+        const percentage = (score / maxScore) * 100;
+        if (percentage >= 75)
+            return "PASS";
+        if (percentage >= 50)
+            return "NEED_MORE_INFO";
+        return "FAIL";
+    }
+    generateExplanation(metrics, tools) {
+        const parts = [];
+        parts.push(`Analyzed ${tools.length} tools for usability.`);
+        parts.push(`Naming convention: ${metrics.toolNamingConvention}.`);
+        parts.push(`Parameter clarity: ${metrics.parameterClarity}.`);
+        const features = [];
+        if (metrics.hasHelpfulDescriptions)
+            features.push("helpful descriptions");
+        if (metrics.followsBestPractices)
+            features.push("follows best practices");
+        if (features.length > 0) {
+            parts.push(`Features: ${features.join(", ")}.`);
+        }
+        else {
+            parts.push("Missing key usability features.");
+        }
+        return parts.join(" ");
+    }
+    generateRecommendations(metrics) {
+        const recommendations = [];
+        if (metrics.toolNamingConvention === "inconsistent") {
+            recommendations.push("Adopt a consistent naming convention for all tools");
+        }
+        if (metrics.parameterClarity !== "clear") {
+            recommendations.push("Use descriptive parameter names");
+            recommendations.push("Add descriptions for all parameters");
+        }
+        if (!metrics.hasHelpfulDescriptions) {
+            recommendations.push("Provide detailed descriptions for each tool");
+        }
+        if (!metrics.followsBestPractices) {
+            recommendations.push("Implement input validation with constraints");
+            recommendations.push("Follow MCP best practices for tool design");
+        }
+        return recommendations;
+    }
+}

package/client/lib/services/assessment/modules/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * MCP Server Assessment Modules
+ *
+ * This module exports all assessors for comprehensive MCP server evaluation.
+ *
+ * Original Assessors (from MCP Inspector):
+ * - FunctionalityAssessor - Tests tool execution and response handling
+ * - DocumentationAssessor - Evaluates README and tool documentation
+ * - SecurityAssessor - Checks for security vulnerabilities
+ * - ErrorHandlingAssessor - Tests error handling patterns
+ * - UsabilityAssessor - Evaluates tool naming and schemas
+ * - MCPSpecComplianceAssessor - Verifies MCP specification compliance
+ *
+ * MCP Directory Compliance Assessors (new):
+ * - AUPComplianceAssessor - Checks for Acceptable Use Policy violations (14 categories)
+ * - ToolAnnotationAssessor - Verifies tool annotations per Policy #17
+ * - ProhibitedLibrariesAssessor - Detects prohibited libraries per Policy #28-30
+ * - ManifestValidationAssessor - Validates MCPB manifest.json
+ * - PortabilityAssessor - Checks for portability issues
+ */
+export { BaseAssessor } from "./BaseAssessor.js";
+export { FunctionalityAssessor } from "./FunctionalityAssessor.js";
+export { DocumentationAssessor } from "./DocumentationAssessor.js";
+export { SecurityAssessor } from "./SecurityAssessor.js";
+export { ErrorHandlingAssessor } from "./ErrorHandlingAssessor.js";
+export { UsabilityAssessor } from "./UsabilityAssessor.js";
+export { MCPSpecComplianceAssessor } from "./MCPSpecComplianceAssessor.js";
+export { AUPComplianceAssessor } from "./AUPComplianceAssessor.js";
+export { ToolAnnotationAssessor } from "./ToolAnnotationAssessor.js";
+export { ProhibitedLibrariesAssessor } from "./ProhibitedLibrariesAssessor.js";
+export { ManifestValidationAssessor } from "./ManifestValidationAssessor.js";
+export { PortabilityAssessor } from "./PortabilityAssessor.js";
+//# sourceMappingURL=index.d.ts.map

package/client/lib/services/assessment/modules/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC"}

package/client/lib/services/assessment/modules/index.js

@@ -0,0 +1,35 @@
+/**
+ * MCP Server Assessment Modules
+ *
+ * This module exports all assessors for comprehensive MCP server evaluation.
+ *
+ * Original Assessors (from MCP Inspector):
+ * - FunctionalityAssessor - Tests tool execution and response handling
+ * - DocumentationAssessor - Evaluates README and tool documentation
+ * - SecurityAssessor - Checks for security vulnerabilities
+ * - ErrorHandlingAssessor - Tests error handling patterns
+ * - UsabilityAssessor - Evaluates tool naming and schemas
+ * - MCPSpecComplianceAssessor - Verifies MCP specification compliance
+ *
+ * MCP Directory Compliance Assessors (new):
+ * - AUPComplianceAssessor - Checks for Acceptable Use Policy violations (14 categories)
+ * - ToolAnnotationAssessor - Verifies tool annotations per Policy #17
+ * - ProhibitedLibrariesAssessor - Detects prohibited libraries per Policy #28-30
+ * - ManifestValidationAssessor - Validates MCPB manifest.json
+ * - PortabilityAssessor - Checks for portability issues
+ */
+// Base class
+export { BaseAssessor } from "./BaseAssessor.js";
+// Original MCP Inspector Assessors
+export { FunctionalityAssessor } from "./FunctionalityAssessor.js";
+export { DocumentationAssessor } from "./DocumentationAssessor.js";
+export { SecurityAssessor } from "./SecurityAssessor.js";
+export { ErrorHandlingAssessor } from "./ErrorHandlingAssessor.js";
+export { UsabilityAssessor } from "./UsabilityAssessor.js";
+export { MCPSpecComplianceAssessor } from "./MCPSpecComplianceAssessor.js";
+// MCP Directory Compliance Assessors
+export { AUPComplianceAssessor } from "./AUPComplianceAssessor.js";
+export { ToolAnnotationAssessor } from "./ToolAnnotationAssessor.js";
+export { ProhibitedLibrariesAssessor } from "./ProhibitedLibrariesAssessor.js";
+export { ManifestValidationAssessor } from "./ManifestValidationAssessor.js";
+export { PortabilityAssessor } from "./PortabilityAssessor.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bryan-thompson/inspector-assessment",
-  "version": "1.5.0",
+  "version": "1.7.0",
   "description": "Enhanced MCP Inspector with comprehensive assessment capabilities for server validation",
   "license": "MIT",
   "author": "Bryan Thompson <bryan@triepod.ai>",
@@ -24,11 +24,14 @@
   ],
   "type": "module",
   "bin": {
-    "mcp-inspector-assess": "cli/build/cli.js"
+    "mcp-inspector-assess": "cli/build/cli.js",
+    "mcp-assess-full": "cli/build/assess-full.js",
+    "mcp-assess-security": "cli/build/assess-security.js"
   },
   "files": [
     "client/bin",
     "client/dist",
+    "client/lib",
     "server/build",
     "cli/build"
   ],
@@ -48,6 +51,7 @@
     "kill": "node scripts/kill-server.js",
     "restart": "npm run kill && npm run dev",
     "assess": "tsx --tsconfig client/tsconfig.app.json scripts/run-security-assessment.ts",
+    "assess:full": "tsx --tsconfig client/tsconfig.app.json scripts/run-full-assessment.ts",
     "dev:sdk": "npm run link:sdk && concurrently \"npm run dev\" \"cd sdk && npm run build:esm:w\"",
     "link:sdk": "(test -d sdk || ln -sf ${MCP_SDK:-$PWD/../typescript-sdk} sdk) && (cd sdk && npm link && (test -d node_modules || npm i)) && npm link @modelcontextprotocol/sdk",
     "unlink:sdk": "(cd sdk && npm unlink -g) && rm sdk && npm unlink @modelcontextprotocol/sdk",
@@ -73,6 +77,7 @@
     "@bryan-thompson/inspector-assessment-cli": "^1.0.0",
     "@bryan-thompson/inspector-assessment-client": "^1.0.0",
     "@bryan-thompson/inspector-assessment-server": "^1.0.0",
+    "@esbuild/darwin-arm64": "^0.27.1",
     "@modelcontextprotocol/sdk": "^1.23.0",
     "concurrently": "^9.2.0",
     "node-fetch": "^3.3.2",