@bryan-thompson/inspector-assessment-client 1.43.1 → 1.43.3

package/lib/lib/assessmentTypes.d.ts

@@ -1,25 +1,681 @@
 /**
  * MCP Directory Review Assessment Types
- *
- * @deprecated This file has been split into focused modules for better maintainability.
- * All exports are re-exported from the new `assessment/` directory for backward compatibility.
- *
- * For new code, prefer importing from specific modules:
- * - `@/lib/assessment/coreTypes` - AssessmentStatus, SecurityRiskLevel, AlignmentStatus
- * - `@/lib/assessment/configTypes` - AssessmentConfiguration, config presets
- * - `@/lib/assessment/resultTypes` - MCPDirectoryAssessment, assessment result types
- * - `@/lib/assessment/extendedTypes` - AUP, Annotation, Temporal assessment types
- * - `@/lib/assessment/progressTypes` - Progress event types for JSONL streaming
- * - `@/lib/assessment/constants` - PROMPT_INJECTION_TESTS constant
- *
- * Or import everything from `@/lib/assessment`:
- * ```typescript
- * import { MCPDirectoryAssessment, AssessmentConfiguration } from "../lib/assessment/index.js";
- * ```
- *
- * See GitHub Issue #21 for details on this refactoring.
- *
- * @module assessmentTypes
- */
-export * from "./assessment/index.js";
+ * Based on Anthropic's 5 core requirements for MCP directory submission
+ */
+export type AssessmentStatus = "PASS" | "FAIL" | "NEED_MORE_INFO";
+export type SecurityRiskLevel = "LOW" | "MEDIUM" | "HIGH";
+export interface ToolTestResult {
+    toolName: string;
+    tested: boolean;
+    status: "working" | "broken" | "untested";
+    error?: string;
+    executionTime?: number;
+    testParameters?: Record<string, unknown>;
+    response?: unknown;
+}
+export interface EnhancedToolTestResult {
+    toolName: string;
+    tested: boolean;
+    status: "fully_working" | "partially_working" | "connectivity_only" | "broken" | "untested";
+    confidence: number;
+    scenariosExecuted: number;
+    scenariosPassed: number;
+    scenariosFailed: number;
+    executionTime: number;
+    validationSummary: {
+        happyPathSuccess: boolean;
+        edgeCasesHandled: number;
+        edgeCasesTotal: number;
+        boundariesRespected: number;
+        boundariesTotal: number;
+        errorHandlingWorks: boolean;
+    };
+    recommendations: string[];
+    detailedResults?: Array<{
+        scenarioName: string;
+        category: "happy_path" | "edge_case" | "boundary" | "error_case";
+        passed: boolean;
+        confidence: number;
+        issues: string[];
+        evidence: string[];
+    }>;
+}
+export interface SecurityTestResult {
+    testName: string;
+    description: string;
+    payload: string;
+    vulnerable: boolean;
+    evidence?: string;
+    riskLevel: SecurityRiskLevel;
+    toolName?: string;
+    response?: string;
+    confidence?: "high" | "medium" | "low";
+    requiresManualReview?: boolean;
+    manualReviewReason?: string;
+    reviewGuidance?: string;
+    connectionError?: boolean;
+    errorType?: "connection" | "server" | "protocol";
+    testReliability?: "completed" | "failed" | "retried";
+    vulnerableHighConfidence?: boolean;
+    toolCategory?: string;
+}
+export interface CodeExample {
+    code: string;
+    language?: string;
+    description?: string;
+    lineNumber?: number;
+}
+export interface DocumentationMetrics {
+    hasReadme: boolean;
+    exampleCount: number;
+    requiredExamples: number;
+    missingExamples: string[];
+    hasInstallInstructions: boolean;
+    hasUsageGuide: boolean;
+    hasAPIReference: boolean;
+    extractedExamples?: CodeExample[];
+    installInstructions?: string;
+    usageInstructions?: string;
+}
+export interface ErrorTestDetail {
+    toolName: string;
+    testType: string;
+    testInput: Record<string, unknown>;
+    testDescription?: string;
+    expectedError: string;
+    actualResponse: {
+        isError: boolean;
+        errorCode?: string | number;
+        errorMessage?: string;
+        rawResponse: unknown;
+    };
+    passed: boolean;
+    reason?: string;
+}
+export interface ErrorHandlingMetrics {
+    mcpComplianceScore: number;
+    errorResponseQuality: "excellent" | "good" | "fair" | "poor";
+    hasProperErrorCodes: boolean;
+    hasDescriptiveMessages: boolean;
+    validatesInputs: boolean;
+    validationCoverage?: {
+        wrongType: number;
+        wrongTypeCount?: {
+            passed: number;
+            total: number;
+        };
+        extraParams: number;
+        extraParamsCount?: {
+            passed: number;
+            total: number;
+        };
+        missingRequired: number;
+        missingRequiredCount?: {
+            passed: number;
+            total: number;
+        };
+        nullValues: number;
+        nullValuesCount?: {
+            passed: number;
+            total: number;
+        };
+        totalTests: number;
+        overallPassRate?: number;
+    };
+    testDetails?: ErrorTestDetail[];
+}
+export interface UsabilityMetrics {
+    toolNamingConvention: "consistent" | "inconsistent";
+    parameterClarity: "clear" | "unclear" | "mixed";
+    hasHelpfulDescriptions: boolean;
+    followsBestPractices: boolean;
+    detailedAnalysis?: {
+        tools: Array<{
+            toolName: string;
+            namingPattern: string;
+            description?: string;
+            descriptionLength: number;
+            hasDescription: boolean;
+            parameterCount: number;
+            hasRequiredParams: boolean;
+            hasSchema: boolean;
+            schemaQuality: string;
+            parameters?: Array<{
+                name: string;
+                type?: string;
+                required: boolean;
+                description?: string;
+                hasDescription: boolean;
+            }>;
+        }>;
+        naming: {
+            patterns: string[];
+            breakdown: Record<string, number>;
+            dominant: string;
+        };
+        descriptions: {
+            withDescriptions: number;
+            withoutDescriptions: number;
+            averageLength: number;
+            tooShort: Array<{
+                toolName: string;
+                namingPattern: string;
+                description?: string;
+                descriptionLength: number;
+                hasDescription: boolean;
+                parameterCount: number;
+                hasRequiredParams: boolean;
+                hasSchema: boolean;
+                schemaQuality: string;
+                parameters?: Array<{
+                    name: string;
+                    type?: string;
+                    required: boolean;
+                    description?: string;
+                    hasDescription: boolean;
+                }>;
+            }>;
+            adequate: Array<{
+                toolName: string;
+                namingPattern: string;
+                description?: string;
+                descriptionLength: number;
+                hasDescription: boolean;
+                parameterCount: number;
+                hasRequiredParams: boolean;
+                hasSchema: boolean;
+                schemaQuality: string;
+                parameters?: Array<{
+                    name: string;
+                    type?: string;
+                    required: boolean;
+                    description?: string;
+                    hasDescription: boolean;
+                }>;
+            }>;
+            detailed: Array<{
+                toolName: string;
+                namingPattern: string;
+                description?: string;
+                descriptionLength: number;
+                hasDescription: boolean;
+                parameterCount: number;
+                hasRequiredParams: boolean;
+                hasSchema: boolean;
+                schemaQuality: string;
+                parameters?: Array<{
+                    name: string;
+                    type?: string;
+                    required: boolean;
+                    description?: string;
+                    hasDescription: boolean;
+                }>;
+            }>;
+        };
+        parameterIssues: string[];
+        bestPracticeScore: {
+            naming: number;
+            descriptions: number;
+            schemas: number;
+            clarity: number;
+            total: number;
+        };
+        overallScore: number;
+    };
+}
+export interface FunctionalityAssessment {
+    totalTools: number;
+    testedTools: number;
+    workingTools: number;
+    brokenTools: string[];
+    coveragePercentage: number;
+    status: AssessmentStatus;
+    explanation: string;
+    toolResults: ToolTestResult[];
+}
+export interface SecurityAssessment {
+    promptInjectionTests: SecurityTestResult[];
+    vulnerabilities: string[];
+    overallRiskLevel: SecurityRiskLevel;
+    status: AssessmentStatus;
+    explanation: string;
+    auditAnalysis?: {
+        highConfidenceVulnerabilities: string[];
+        needsReview: string[];
+        falsePositiveLikelihood: Record<string, "HIGH" | "MEDIUM" | "LOW">;
+        responseUniformity: Record<string, {
+            uniqueResponses: number;
+            totalTests: number;
+        }>;
+    };
+}
+export interface DocumentationAssessment {
+    metrics: DocumentationMetrics;
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+export interface ErrorHandlingAssessment {
+    metrics: ErrorHandlingMetrics;
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+export interface UsabilityAssessment {
+    metrics: UsabilityMetrics;
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+export interface StructuredRecommendation {
+    id: string;
+    title: string;
+    severity: "critical" | "warning" | "enhancement";
+    confidence: "high" | "medium" | "low";
+    detectionMethod: "automated" | "manual-required";
+    category: string;
+    description: string;
+    requiresManualVerification: boolean;
+    manualVerificationSteps?: string[];
+    contextNote?: string;
+    actionItems: string[];
+}
+/**
+ * Individual protocol check result with evidence
+ */
+export interface ProtocolCheckResult {
+    passed: boolean;
+    confidence: "high" | "medium" | "low";
+    evidence?: string;
+    warnings?: string[];
+    rawResponse?: unknown;
+}
+/**
+ * Protocol checks that are actually tested via MCP calls
+ * HIGH CONFIDENCE - these are verified through actual protocol interaction
+ */
+export interface ProtocolChecks {
+    jsonRpcCompliance: ProtocolCheckResult;
+    serverInfoValidity: ProtocolCheckResult;
+    schemaCompliance: ProtocolCheckResult;
+    errorResponseCompliance: ProtocolCheckResult;
+    structuredOutputSupport: ProtocolCheckResult;
+}
+/**
+ * Metadata-based hints parsed from serverInfo
+ * LOW CONFIDENCE - these are NOT tested, just parsed from metadata
+ */
+export interface MetadataHints {
+    confidence: "low";
+    requiresManualVerification: true;
+    transportHints?: {
+        detectedTransport?: string;
+        supportsStdio: boolean;
+        supportsHTTP: boolean;
+        supportsSSE: boolean;
+        detectionMethod: "metadata" | "assumed";
+    };
+    oauthHints?: {
+        hasOAuthConfig: boolean;
+        supportsOAuth: boolean;
+        supportsPKCE: boolean;
+        resourceIndicators?: string[];
+    };
+    annotationHints?: {
+        supportsReadOnlyHint: boolean;
+        supportsDestructiveHint: boolean;
+        supportsTitleAnnotation: boolean;
+        customAnnotations?: string[];
+    };
+    streamingHints?: {
+        supportsStreaming: boolean;
+        streamingProtocol?: "http-streaming" | "sse" | "websocket";
+    };
+    manualVerificationSteps: string[];
+}
+/**
+ * MCP Spec Compliance Assessment - Hybrid Structure
+ * Clearly separates verified protocol checks from unverified metadata hints
+ */
+export interface MCPSpecComplianceAssessment {
+    protocolVersion: string;
+    protocolChecks: ProtocolChecks;
+    metadataHints?: MetadataHints;
+    status: AssessmentStatus;
+    complianceScore: number;
+    explanation: string;
+    recommendations: string[];
+    /** @deprecated Use protocolChecks and metadataHints instead */
+    transportCompliance?: TransportComplianceMetrics;
+    /** @deprecated Use metadataHints.oauthHints instead */
+    oauthImplementation?: OAuthComplianceMetrics;
+    /** @deprecated Use metadataHints.annotationHints instead */
+    annotationSupport?: AnnotationSupportMetrics;
+    /** @deprecated Use metadataHints.streamingHints instead */
+    streamingSupport?: StreamingSupportMetrics;
+}
+export interface TransportComplianceMetrics {
+    supportsStreamableHTTP: boolean;
+    deprecatedSSE: boolean;
+    transportValidation: "passed" | "failed" | "partial";
+    errors?: string[];
+    supportsStdio?: boolean;
+    supportsSSE?: boolean;
+    confidence?: "high" | "medium" | "low";
+    detectionMethod?: "automated" | "manual-required";
+    requiresManualCheck?: boolean;
+    manualVerificationSteps?: string[];
+}
+export interface OAuthComplianceMetrics {
+    implementsResourceServer: boolean;
+    supportsRFC8707: boolean;
+    resourceIndicators: string[];
+    tokenValidation: boolean;
+    scopeEnforcement: boolean;
+    errors?: string[];
+    supportsOAuth?: boolean;
+    supportsPKCE?: boolean;
+}
+export interface AnnotationSupportMetrics {
+    supportsReadOnlyHint: boolean;
+    supportsDestructiveHint: boolean;
+    supportsTitleAnnotation: boolean;
+    customAnnotations?: string[];
+}
+export interface StreamingSupportMetrics {
+    supportsStreaming: boolean;
+    streamingProtocol?: "http-streaming" | "sse" | "websocket";
+    performanceMetrics?: {
+        latency: number;
+        throughput: number;
+    };
+}
+export interface DependencyAnalysis {
+    totalDependencies: number;
+    directDependencies: number;
+    transitiveDependencies: number;
+    outdatedPackages: number;
+    abandonedPackages: number;
+    riskyLicenses: string[];
+    licenseCompliance?: boolean;
+}
+export interface VulnerabilityReport {
+    packageName: string;
+    version: string;
+    vulnerability: string;
+    severity: "CRITICAL" | "HIGH" | "MEDIUM" | "LOW";
+    cve?: string;
+    fixAvailable: boolean;
+    fixVersion?: string;
+    package?: string;
+}
+export interface SoftwareBillOfMaterials {
+    format: "SPDX" | "CycloneDX";
+    components: number;
+    licenses: string[];
+    attestations?: string[];
+}
+export interface PackageIntegrityMetrics {
+    signedPackages: number;
+    verifiedPublishers: number;
+    integrityChecksPassed: boolean;
+    integrityScore: number;
+    squattingRisk: "HIGH" | "MEDIUM" | "LOW";
+}
+export interface RuntimeTestResult {
+    testName: string;
+    category: "memory" | "filesystem" | "network" | "process";
+    passed: boolean;
+    findings?: string[];
+    severity?: SecurityRiskLevel;
+}
+export interface FuzzingReport {
+    totalInputsTested: number;
+    crashesFound: number;
+    hangsDetected: number;
+    memoryLeaks: number;
+    unexpectedBehaviors: string[];
+    coveragePercentage: number;
+    passed: number;
+    failed: number;
+}
+export interface SandboxTestResult {
+    escapeTechnique: string;
+    successful: boolean;
+    containmentLevel: "full" | "partial" | "none";
+    details?: string;
+}
+export interface BehaviorAnalysisReport {
+    suspiciousBehaviors: string[];
+    networkConnections: string[];
+    fileSystemAccess: string[];
+    processSpawning: boolean;
+    anomalyScore: number;
+}
+export interface MCPDirectoryAssessment {
+    serverName: string;
+    assessmentDate: string;
+    assessorVersion: string;
+    functionality: FunctionalityAssessment;
+    security: SecurityAssessment;
+    documentation: DocumentationAssessment;
+    errorHandling: ErrorHandlingAssessment;
+    usability: UsabilityAssessment;
+    mcpSpecCompliance?: MCPSpecComplianceAssessment;
+    aupCompliance?: AUPComplianceAssessment;
+    toolAnnotations?: ToolAnnotationAssessment;
+    prohibitedLibraries?: ProhibitedLibrariesAssessment;
+    manifestValidation?: ManifestValidationAssessment;
+    portability?: PortabilityAssessment;
+    overallStatus: AssessmentStatus;
+    summary: string;
+    recommendations: string[];
+    executionTime: number;
+    totalTestsRun: number;
+    evidenceFiles?: string[];
+    mcpProtocolVersion?: string;
+}
+/**
+ * AUP (Acceptable Use Policy) Compliance Types
+ * Based on Anthropic's 14 AUP categories (A-N)
+ */
+export type AUPCategory = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" | "K" | "L" | "M" | "N";
+export type AUPSeverity = "CRITICAL" | "HIGH" | "MEDIUM" | "FLAG";
+export interface AUPViolation {
+    category: AUPCategory;
+    categoryName: string;
+    severity: AUPSeverity;
+    pattern: string;
+    matchedText: string;
+    location: "tool_name" | "tool_description" | "readme" | "source_code";
+    filePath?: string;
+    lineNumber?: number;
+    confidence: "high" | "medium" | "low";
+    requiresHumanReview: boolean;
+    reviewGuidance?: string;
+}
+export interface AUPComplianceAssessment {
+    violations: AUPViolation[];
+    highRiskDomains: string[];
+    scannedLocations: {
+        toolNames: boolean;
+        toolDescriptions: boolean;
+        readme: boolean;
+        sourceCode: boolean;
+    };
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+/**
+ * Tool Annotation Types (Policy #17)
+ * Verifies readOnlyHint, destructiveHint presence
+ */
+export interface ToolAnnotationResult {
+    toolName: string;
+    hasAnnotations: boolean;
+    annotations?: {
+        readOnlyHint?: boolean;
+        destructiveHint?: boolean;
+        title?: string;
+        description?: string;
+        idempotentHint?: boolean;
+        openWorldHint?: boolean;
+    };
+    inferredBehavior?: {
+        expectedReadOnly: boolean;
+        expectedDestructive: boolean;
+        reason: string;
+    };
+    issues: string[];
+    recommendations: string[];
+}
+export interface ToolAnnotationAssessment {
+    toolResults: ToolAnnotationResult[];
+    annotatedCount: number;
+    missingAnnotationsCount: number;
+    misalignedAnnotationsCount: number;
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+/**
+ * Prohibited Libraries Types (Policy #28-30)
+ * Detects financial and media processing libraries
+ */
+export type ProhibitedLibraryCategory = "financial" | "media" | "payments" | "banking";
+export interface ProhibitedLibraryMatch {
+    name: string;
+    category: ProhibitedLibraryCategory;
+    location: "package.json" | "source_import" | "requirements.txt" | "cargo.toml";
+    filePath?: string;
+    lineNumber?: number;
+    severity: "BLOCKING" | "HIGH" | "MEDIUM";
+    reason: string;
+    policyReference: string;
+}
+export interface ProhibitedLibrariesAssessment {
+    matches: ProhibitedLibraryMatch[];
+    scannedFiles: string[];
+    hasFinancialLibraries: boolean;
+    hasMediaLibraries: boolean;
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+/**
+ * MCPB Manifest Validation Types
+ * Based on manifest_version 0.3 spec
+ */
+export interface ManifestJsonSchema {
+    manifest_version: string;
+    name: string;
+    version: string;
+    description?: string;
+    author?: string;
+    repository?: string;
+    license?: string;
+    mcp_config: {
+        command: string;
+        args?: string[];
+        env?: Record<string, string>;
+    };
+    icon?: string;
+    homepage?: string;
+    keywords?: string[];
+}
+export interface ManifestValidationResult {
+    field: string;
+    valid: boolean;
+    value?: unknown;
+    expectedType?: string;
+    issue?: string;
+    severity: "ERROR" | "WARNING" | "INFO";
+}
+export interface ManifestValidationAssessment {
+    hasManifest: boolean;
+    manifestVersion?: string;
+    validationResults: ManifestValidationResult[];
+    hasIcon: boolean;
+    hasRequiredFields: boolean;
+    missingFields: string[];
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+/**
+ * Portability Assessment Types
+ * Detects hardcoded paths, platform-specific code
+ */
+export interface PortabilityIssue {
+    type: "hardcoded_path" | "platform_specific" | "bundle_root_antipattern" | "absolute_path" | "user_home_path";
+    filePath: string;
+    lineNumber?: number;
+    matchedText: string;
+    severity: "HIGH" | "MEDIUM" | "LOW";
+    recommendation: string;
+}
+export interface PortabilityAssessment {
+    issues: PortabilityIssue[];
+    scannedFiles: number;
+    platformSpecificCount: number;
+    hardcodedPathCount: number;
+    usesDirname: boolean;
+    usesBundleRoot: boolean;
+    status: AssessmentStatus;
+    explanation: string;
+    recommendations: string[];
+}
+export declare const PROMPT_INJECTION_TESTS: Omit<SecurityTestResult, "vulnerable" | "evidence">[];
+/**
+ * Claude Code Bridge Configuration
+ * Enables integration with Claude Code CLI for intelligent analysis
+ */
+export interface ClaudeCodeConfig {
+    enabled: boolean;
+    features: {
+        intelligentTestGeneration: boolean;
+        aupSemanticAnalysis: boolean;
+        annotationInference: boolean;
+        documentationQuality: boolean;
+    };
+    timeout: number;
+    workingDir?: string;
+    maxRetries?: number;
+}
+export interface AssessmentConfiguration {
+    testTimeout: number;
+    delayBetweenTests?: number;
+    skipBrokenTools: boolean;
+    reviewerMode?: boolean;
+    enableExtendedAssessment?: boolean;
+    parallelTesting?: boolean;
+    maxParallelTests?: number;
+    scenariosPerTool?: number;
+    maxToolsToTestForErrors?: number;
+    selectedToolsForTesting?: string[];
+    securityPatternsToTest?: number;
+    enableDomainTesting?: boolean;
+    mcpProtocolVersion?: string;
+    enableSourceCodeAnalysis?: boolean;
+    claudeCode?: ClaudeCodeConfig;
+    assessmentCategories?: {
+        functionality: boolean;
+        security: boolean;
+        documentation: boolean;
+        errorHandling: boolean;
+        usability: boolean;
+        mcpSpecCompliance?: boolean;
+        aupCompliance?: boolean;
+        toolAnnotations?: boolean;
+        prohibitedLibraries?: boolean;
+        manifestValidation?: boolean;
+        portability?: boolean;
+    };
+}
+export declare const DEFAULT_ASSESSMENT_CONFIG: AssessmentConfiguration;
+export declare const REVIEWER_MODE_CONFIG: AssessmentConfiguration;
+export declare const DEVELOPER_MODE_CONFIG: AssessmentConfiguration;
+export declare const AUDIT_MODE_CONFIG: AssessmentConfiguration;
+export declare const CLAUDE_ENHANCED_AUDIT_CONFIG: AssessmentConfiguration;
 //# sourceMappingURL=assessmentTypes.d.ts.map