@bryan-thompson/inspector-assessment-client 1.6.0 → 1.7.1

package/lib/services/assessment/lib/claudeCodeBridge.js

@@ -0,0 +1,357 @@
+/**
+ * Claude Code Bridge
+ *
+ * Provides integration with Claude Code CLI for intelligent analysis tasks.
+ * Uses shell execution with `claude --print` for stateless reasoning.
+ *
+ * This bridge enables:
+ * - Intelligent test parameter generation
+ * - Semantic AUP violation analysis
+ * - Tool behavior inference for annotation validation
+ * - Documentation quality assessment
+ */
+import { execSync } from "child_process";
+/**
+ * Default configuration with minimal features
+ */
+export const DEFAULT_CLAUDE_CODE_CONFIG = {
+    enabled: false,
+    timeout: 30000,
+    maxRetries: 1,
+    features: {
+        intelligentTestGeneration: false,
+        aupSemanticAnalysis: false,
+        behaviorInference: false,
+        annotationInference: false,
+        documentationAssessment: false,
+        documentationQuality: false,
+    },
+};
+/**
+ * Full configuration with all features enabled
+ */
+export const FULL_CLAUDE_CODE_CONFIG = {
+    enabled: true,
+    timeout: 60000,
+    maxRetries: 2,
+    features: {
+        intelligentTestGeneration: true,
+        aupSemanticAnalysis: true,
+        behaviorInference: true,
+        annotationInference: true,
+        documentationAssessment: true,
+        documentationQuality: true,
+    },
+};
+/**
+ * Claude Code Bridge
+ * Executes Claude CLI for intelligent analysis during MCP assessments
+ */
+export class ClaudeCodeBridge {
+    config;
+    isAvailable = false;
+    constructor(config) {
+        this.config = config;
+        this.isAvailable = this.checkClaudeAvailability();
+        if (!this.isAvailable) {
+            console.warn("[ClaudeCodeBridge] Claude CLI not available - features will be disabled");
+        }
+    }
+    /**
+     * Check if a specific feature is enabled
+     * Note: annotationInference is an alias for behaviorInference
+     */
+    isFeatureEnabled(feature) {
+        if (!this.isAvailable || !this.config.enabled) {
+            return false;
+        }
+        // annotationInference is an alias for behaviorInference
+        if (feature === "annotationInference") {
+            return (this.config.features.annotationInference === true ||
+                this.config.features.behaviorInference === true);
+        }
+        return this.config.features[feature] === true;
+    }
+    /**
+     * Check if Claude CLI is available on the system
+     */
+    checkClaudeAvailability() {
+        try {
+            execSync("which claude", { stdio: "pipe", timeout: 5000 });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Execute Claude CLI with a prompt
+     */
+    executeClaudeCommand(prompt) {
+        const startTime = Date.now();
+        try {
+            const timeout = this.config.timeout || 30000;
+            // Escape the prompt for shell execution
+            const escapedPrompt = prompt.replace(/'/g, "'\\''");
+            const output = execSync(`claude --print '${escapedPrompt}'`, {
+                encoding: "utf-8",
+                timeout,
+                stdio: ["pipe", "pipe", "pipe"],
+                maxBuffer: 10 * 1024 * 1024, // 10MB buffer
+            });
+            return {
+                success: true,
+                output: output.trim(),
+                executionTimeMs: Date.now() - startTime,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                output: "",
+                error: error instanceof Error ? error.message : String(error),
+                executionTimeMs: Date.now() - startTime,
+            };
+        }
+    }
+    /**
+     * Execute with retries
+     */
+    async executeWithRetry(prompt) {
+        const maxRetries = this.config.maxRetries || 1;
+        let lastError = null;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            const response = this.executeClaudeCommand(prompt);
+            if (response.success) {
+                return response;
+            }
+            lastError = response;
+            // Wait before retry (exponential backoff)
+            if (attempt < maxRetries) {
+                await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, attempt)));
+            }
+        }
+        return lastError;
+    }
+    /**
+     * Parse JSON from Claude response, handling markdown code blocks
+     */
+    parseJsonResponse(response) {
+        if (!response.success) {
+            return null;
+        }
+        try {
+            let jsonStr = response.output;
+            // Handle markdown code blocks
+            const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+            if (jsonMatch) {
+                jsonStr = jsonMatch[1].trim();
+            }
+            return JSON.parse(jsonStr);
+        }
+        catch {
+            console.warn("[ClaudeCodeBridge] Failed to parse JSON response");
+            return null;
+        }
+    }
+    /**
+     * Analyze an AUP violation for false positives using semantic analysis
+     * @param matchedText - The text that matched the AUP pattern
+     * @param context - Context about the violation
+     */
+    async analyzeAUPViolation(matchedText, context) {
+        if (!this.isFeatureEnabled("aupSemanticAnalysis")) {
+            return null;
+        }
+        const prompt = `You are analyzing a potential AUP (Acceptable Use Policy) violation detected in an MCP server tool.
+
+CONTEXT:
+- Flagged text: "${matchedText}"
+- Tool name: ${context.toolName}
+- Tool description: ${context.toolDescription || "No description"}
+- Category: ${context.category} (${context.categoryName})
+- Location: ${context.location}
+
+TASK:
+Determine if this is a TRUE violation or a FALSE POSITIVE. Consider:
+1. Is this a legitimate security/defensive tool?
+2. Is there appropriate context (disclaimers, educational purpose)?
+3. Does the tool's purpose justify the flagged content?
+
+Respond with JSON only:
+{
+  "isViolation": boolean,
+  "confidence": number (0-100),
+  "reasoning": "explanation",
+  "category": "${context.category}",
+  "suggestedAction": "block" | "flag_for_review" | "allow",
+  "contextFactors": ["factor1", "factor2"]
+}`;
+        const response = await this.executeWithRetry(prompt);
+        const result = this.parseJsonResponse(response);
+        // Normalize: ensure both isViolation and isConfirmedViolation are present
+        if (result) {
+            result.isViolation = result.isViolation ?? result.isConfirmedViolation;
+            result.isConfirmedViolation =
+                result.isConfirmedViolation ?? result.isViolation;
+        }
+        return result;
+    }
+    /**
+     * Infer expected tool behavior for annotation validation
+     */
+    async inferToolBehavior(tool, currentAnnotations) {
+        // Check both behaviorInference and annotationInference feature flags
+        if (!this.isFeatureEnabled("behaviorInference") &&
+            !this.isFeatureEnabled("annotationInference")) {
+            return null;
+        }
+        const annotationsStr = currentAnnotations
+            ? JSON.stringify(currentAnnotations, null, 2)
+            : "No annotations provided";
+        const prompt = `You are analyzing an MCP tool to infer its expected behavior and validate annotations.
+
+TOOL:
+- Name: ${tool.name}
+- Description: ${tool.description || "No description provided"}
+- Input Schema: ${JSON.stringify(tool.inputSchema, null, 2)}
+
+CURRENT ANNOTATIONS:
+${annotationsStr}
+
+TASK:
+Analyze the tool and determine:
+1. Is this tool read-only (doesn't modify state)?
+2. Is this tool destructive (can delete/destroy data)?
+3. Do the current annotations match expected behavior?
+
+Respond with JSON only:
+{
+  "expectedReadOnly": boolean,
+  "expectedDestructive": boolean,
+  "confidence": number (0-100),
+  "reasoning": "explanation",
+  "suggestedAnnotations": {
+    "readOnlyHint": boolean,
+    "destructiveHint": boolean,
+    "idempotentHint": boolean
+  },
+  "misalignmentDetected": boolean,
+  "misalignmentDetails": "details if misaligned, null otherwise"
+}`;
+        const response = await this.executeWithRetry(prompt);
+        return this.parseJsonResponse(response);
+    }
+    /**
+     * Generate intelligent test scenarios for a tool
+     */
+    async generateTestScenarios(tool, existingScenarios) {
+        if (!this.isFeatureEnabled("intelligentTestGeneration")) {
+            return null;
+        }
+        const prompt = `You are generating test scenarios for an MCP tool.
+
+TOOL:
+- Name: ${tool.name}
+- Description: ${tool.description || "No description provided"}
+- Input Schema: ${JSON.stringify(tool.inputSchema, null, 2)}
+
+EXISTING SCENARIOS: ${existingScenarios} already generated via schema analysis
+
+TASK:
+Generate 3-5 additional test scenarios that would catch edge cases the schema-based generator might miss. Focus on:
+1. Real-world usage patterns
+2. Boundary conditions
+3. Error conditions
+4. Security-relevant inputs
+
+Respond with JSON only:
+{
+  "scenarios": [
+    {
+      "name": "scenario_name",
+      "description": "what this tests",
+      "params": { "param1": "value1" },
+      "expectedBehavior": "what should happen",
+      "category": "happy_path" | "edge_case" | "boundary" | "error_case"
+    }
+  ],
+  "reasoning": "why these scenarios are valuable"
+}`;
+        const response = await this.executeWithRetry(prompt);
+        return this.parseJsonResponse(response);
+    }
+    /**
+     * Generate test parameters for a tool
+     * This returns just the parameter sets, used by TestDataGenerator
+     */
+    async generateTestParameters(tool) {
+        if (!this.isFeatureEnabled("intelligentTestGeneration")) {
+            return null;
+        }
+        const prompt = `You are generating test parameters for an MCP tool.
+
+TOOL:
+- Name: ${tool.name}
+- Description: ${tool.description || "No description provided"}
+- Input Schema: ${JSON.stringify(tool.inputSchema, null, 2)}
+
+TASK:
+Generate 3-5 sets of valid test parameters that exercise different scenarios:
+1. Happy path / typical usage
+2. Edge cases (empty strings, zeros, minimum values)
+3. Boundary values (max length strings, large numbers)
+4. Alternative valid inputs
+
+Return ONLY valid parameter sets (no intentionally invalid inputs).
+
+Respond with JSON only:
+{
+  "parameters": [
+    { "param1": "value1", "param2": 123 },
+    { "param1": "", "param2": 0 },
+    { "param1": "very long string...", "param2": 999999 }
+  ]
+}`;
+        const response = await this.executeWithRetry(prompt);
+        const result = this.parseJsonResponse(response);
+        return result?.parameters || null;
+    }
+    /**
+     * Assess documentation quality
+     */
+    async assessDocumentation(readmeContent, toolCount) {
+        if (!this.isFeatureEnabled("documentationAssessment")) {
+            return null;
+        }
+        // Truncate very long READMEs
+        const truncatedReadme = readmeContent.length > 10000
+            ? readmeContent.substring(0, 10000) + "\n...[truncated]..."
+            : readmeContent;
+        const prompt = `You are assessing the documentation quality of an MCP server.
+
+README CONTENT:
+${truncatedReadme}
+
+SERVER INFO:
+- Number of tools: ${toolCount}
+
+TASK:
+Assess the documentation quality. Check for:
+1. Clear description of what the server does
+2. Installation instructions
+3. Configuration requirements
+4. Tool documentation
+5. Examples of usage
+6. Security considerations
+
+Respond with JSON only:
+{
+  "score": number (0-100),
+  "issues": ["issue1", "issue2"],
+  "suggestions": ["suggestion1", "suggestion2"]
+}`;
+        const response = await this.executeWithRetry(prompt);
+        return this.parseJsonResponse(response);
+    }
+}

package/lib/services/assessment/modules/AUPComplianceAssessor.d.ts

@@ -0,0 +1,100 @@
+/**
+ * AUP Compliance Assessor
+ * Scans MCP server for Acceptable Use Policy violations
+ *
+ * Checks:
+ * - Tool names and descriptions
+ * - README content
+ * - Source code (if sourceCodePath provided)
+ *
+ * Based on Anthropic's 14 AUP categories (A-N)
+ *
+ * Supports optional Claude Code integration for semantic analysis
+ * to reduce false positives (e.g., security tools, disclaimers).
+ */
+import { BaseAssessor } from "./BaseAssessor.js";
+import { AssessmentContext } from "../AssessmentOrchestrator.js";
+import type { AUPComplianceAssessment, AUPViolation } from "../../../lib/assessmentTypes.js";
+import type { ClaudeCodeBridge } from "../lib/claudeCodeBridge.js";
+/**
+ * Extended AUP violation with semantic analysis results
+ */
+export interface EnhancedAUPViolation extends AUPViolation {
+    semanticAnalysis?: {
+        isConfirmedViolation: boolean;
+        confidence: number;
+        reasoning: string;
+        source: "claude-verified" | "pattern-only";
+    };
+}
+/**
+ * Extended AUP compliance assessment with semantic analysis
+ */
+export interface EnhancedAUPComplianceAssessment extends AUPComplianceAssessment {
+    confirmedViolations: EnhancedAUPViolation[];
+    flaggedForReview: EnhancedAUPViolation[];
+    semanticAnalysisEnabled: boolean;
+    falsePositivesFiltered: number;
+}
+export declare class AUPComplianceAssessor extends BaseAssessor {
+    private claudeBridge;
+    /**
+     * Set the Claude Code bridge for semantic violation analysis
+     */
+    setClaudeBridge(bridge: ClaudeCodeBridge | null): void;
+    /**
+     * Check if Claude semantic analysis is enabled
+     */
+    private isSemanticAnalysisEnabled;
+    /**
+     * Run AUP compliance assessment
+     * If Claude semantic analysis is enabled, violations are verified to reduce false positives.
+     */
+    assess(context: AssessmentContext): Promise<AUPComplianceAssessment | EnhancedAUPComplianceAssessment>;
+    /**
+     * Run Claude semantic analysis on flagged violations
+     * Separates confirmed violations from likely false positives
+     */
+    private runSemanticAnalysis;
+    /**
+     * Generate explanation for semantic analysis results
+     */
+    private generateSemanticExplanation;
+    /**
+     * Generate recommendations for semantic analysis results
+     */
+    private generateSemanticRecommendations;
+    /**
+     * Scan a tool name for AUP violations
+     */
+    private scanToolName;
+    /**
+     * Scan a tool description for AUP violations
+     */
+    private scanToolDescription;
+    /**
+     * Scan README content for AUP violations
+     */
+    private scanReadme;
+    /**
+     * Scan a source file for AUP violations
+     */
+    private scanSourceFile;
+    /**
+     * Check if a file should be skipped for AUP scanning
+     */
+    private shouldSkipFile;
+    /**
+     * Determine overall status based on violations
+     */
+    private determineAUPStatus;
+    /**
+     * Generate explanation text
+     */
+    private generateExplanation;
+    /**
+     * Generate recommendations
+     */
+    private generateRecommendations;
+}
+//# sourceMappingURL=AUPComplianceAssessor.d.ts.map

package/lib/services/assessment/modules/AUPComplianceAssessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AUPComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/AUPComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,uBAAuB,EACvB,YAAY,EAEb,MAAM,uBAAuB,CAAC;AAK/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,gBAAgB,CAAC,EAAE;QACjB,oBAAoB,EAAE,OAAO,CAAC;QAC9B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,iBAAiB,GAAG,cAAc,CAAC;KAC5C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,+BAAgC,SAAQ,uBAAuB;IAC9E,mBAAmB,EAAE,oBAAoB,EAAE,CAAC;IAC5C,gBAAgB,EAAE,oBAAoB,EAAE,CAAC;IACzC,uBAAuB,EAAE,OAAO,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,qBAAa,qBAAsB,SAAQ,YAAY;IAErD,OAAO,CAAC,YAAY,CAAiC;IAErD;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAItD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAMjC;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,uBAAuB,GAAG,+BAA+B,CAAC;IA+HrE;;;OAGG;YACW,mBAAmB;IAyHjC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAqDnC;;OAEG;IACH,OAAO,CAAC,+BAA+B;IAiEvC;;OAEG;IACH,OAAO,CAAC,YAAY;IAgBpB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmB3B;;OAEG;IACH,OAAO,CAAC,UAAU;IAgBlB;;OAEG;IACH,OAAO,CAAC,cAAc;IA4BtB;;OAEG;IACH,OAAO,CAAC,cAAc;IAetB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA0B1B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAkD3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAkDhC"}