@bryan-thompson/inspector-assessment-client 1.34.2 → 1.35.2

package/lib/lib/assessment/summarizer/AssessmentSummarizer.js

@@ -0,0 +1,452 @@
+/**
+ * Assessment Summarizer
+ *
+ * Generates tiered output for large assessment results to fit within
+ * LLM context windows. Creates executive summaries and per-tool digests.
+ *
+ * Issue #136: Tiered output strategy for large assessments
+ *
+ * @module assessment/summarizer/AssessmentSummarizer
+ */
+import { calculateModuleScore } from "../../moduleScoring.js";
+import { estimateTokens } from "./tokenEstimator.js";
+import { DEFAULT_SUMMARIZER_CONFIG, } from "./types.js";
+import { buildToolSummaryStageBEnrichment, buildToolDetailStageBEnrichment, } from "./stageBEnrichmentBuilder.js";
+// ============================================================================
+// Assessment Summarizer Class
+// ============================================================================
+/**
+ * Generates tiered summaries from assessment results.
+ *
+ * @example
+ * ```typescript
+ * const summarizer = new AssessmentSummarizer();
+ * const executive = summarizer.generateExecutiveSummary(results);
+ * const toolSummaries = summarizer.generateToolSummaries(results);
+ * ```
+ */
+export class AssessmentSummarizer {
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_SUMMARIZER_CONFIG, ...config };
+    }
+    // ==========================================================================
+    // Tier 1: Executive Summary
+    // ==========================================================================
+    /**
+     * Generate executive summary (Tier 1) from assessment results.
+     * Targets ~5K tokens for guaranteed LLM context fit.
+     *
+     * @param results - Full assessment results
+     * @returns Executive summary
+     */
+    generateExecutiveSummary(results) {
+        const modulesSummary = this.extractModulesSummary(results);
+        const criticalFindings = this.extractCriticalFindings(results);
+        const toolRiskDistribution = this.calculateToolRiskDistribution(results);
+        const recommendations = this.aggregateRecommendations(results);
+        const summary = {
+            serverName: results.serverName,
+            overallStatus: results.overallStatus,
+            overallScore: this.calculateOverallScore(results),
+            toolCount: results.functionality?.totalTools ?? 0,
+            testCount: results.totalTestsRun ?? 0,
+            executionTime: results.executionTime ?? 0,
+            modulesSummary,
+            criticalFindings,
+            toolRiskDistribution,
+            recommendations: recommendations.slice(0, this.config.maxRecommendations),
+            estimatedTokens: 0, // Will be calculated after construction
+            generatedAt: new Date().toISOString(),
+        };
+        // Calculate actual token estimate
+        summary.estimatedTokens = estimateTokens(summary);
+        return summary;
+    }
+    /**
+     * Extract per-module status and scores.
+     */
+    extractModulesSummary(results) {
+        const summary = {};
+        const moduleKeys = [
+            "functionality",
+            "security",
+            "errorHandling",
+            "aupCompliance",
+            "toolAnnotations",
+            "temporal",
+            "resources",
+            "prompts",
+            "crossCapability",
+            "protocolCompliance",
+            "developerExperience",
+            "prohibitedLibraries",
+            "manifestValidation",
+            "authentication",
+            "portability",
+            "externalAPIScanner",
+            // Legacy keys for backwards compatibility
+            "mcpSpecCompliance",
+            "documentation",
+            "usability",
+        ];
+        for (const key of moduleKeys) {
+            const module = results[key];
+            if (module && module.status) {
+                const score = calculateModuleScore(module) ?? 50;
+                summary[key] = {
+                    status: module.status,
+                    score,
+                };
+            }
+        }
+        return summary;
+    }
+    /**
+     * Extract critical findings counts from all modules.
+     */
+    extractCriticalFindings(results) {
+        return {
+            securityVulnerabilities: results.security?.vulnerabilities?.length ?? 0,
+            aupViolations: results.aupCompliance?.violations?.length ?? 0,
+            brokenTools: results.functionality?.brokenTools?.length ?? 0,
+            missingAnnotations: results.toolAnnotations?.missingAnnotationsCount ?? 0,
+        };
+    }
+    /**
+     * Calculate tool risk distribution from security test results.
+     */
+    calculateToolRiskDistribution(results) {
+        const distribution = { high: 0, medium: 0, low: 0, safe: 0 };
+        const tests = results.security?.promptInjectionTests ?? [];
+        const toolVulnCounts = new Map();
+        // Count vulnerabilities per tool
+        for (const test of tests) {
+            if (test.vulnerable && test.toolName) {
+                const current = toolVulnCounts.get(test.toolName) ?? 0;
+                toolVulnCounts.set(test.toolName, current + 1);
+            }
+        }
+        // Get all tool names
+        const allTools = new Set();
+        for (const test of tests) {
+            if (test.toolName) {
+                allTools.add(test.toolName);
+            }
+        }
+        // Categorize each tool
+        for (const toolName of allTools) {
+            const vulnCount = toolVulnCounts.get(toolName) ?? 0;
+            const riskLevel = this.calculateToolRiskLevel(vulnCount);
+            switch (riskLevel) {
+                case "HIGH":
+                    distribution.high++;
+                    break;
+                case "MEDIUM":
+                    distribution.medium++;
+                    break;
+                case "LOW":
+                    distribution.low++;
+                    break;
+                case "SAFE":
+                    distribution.safe++;
+                    break;
+            }
+        }
+        return distribution;
+    }
+    /**
+     * Calculate risk level based on vulnerability count.
+     */
+    calculateToolRiskLevel(vulnCount) {
+        if (vulnCount >= 5)
+            return "HIGH";
+        if (vulnCount >= 2)
+            return "MEDIUM";
+        if (vulnCount >= 1)
+            return "LOW";
+        return "SAFE";
+    }
+    /**
+     * Aggregate recommendations from all modules.
+     */
+    aggregateRecommendations(results) {
+        const recommendations = [];
+        // Top-level recommendations
+        if (results.recommendations) {
+            recommendations.push(...results.recommendations);
+        }
+        // Module-specific recommendations
+        const modulesWithRecs = [
+            results.errorHandling,
+            results.aupCompliance,
+            results.toolAnnotations,
+            results.developerExperience,
+            results.documentation,
+            results.usability,
+            results.prohibitedLibraries,
+            results.portability,
+        ];
+        for (const module of modulesWithRecs) {
+            if (module?.recommendations) {
+                recommendations.push(...module.recommendations);
+            }
+        }
+        // Deduplicate
+        return [...new Set(recommendations)];
+    }
+    /**
+     * Calculate overall score from module scores.
+     */
+    calculateOverallScore(results) {
+        const scores = [];
+        const coreModules = [
+            results.functionality,
+            results.security,
+            results.errorHandling,
+        ];
+        for (const module of coreModules) {
+            const score = calculateModuleScore(module);
+            if (score !== null) {
+                scores.push(score);
+            }
+        }
+        if (scores.length === 0) {
+            return results.overallStatus === "PASS" ? 100 : 0;
+        }
+        return Math.round(scores.reduce((a, b) => a + b, 0) / scores.length);
+    }
+    // ==========================================================================
+    // Tier 2: Tool Summaries
+    // ==========================================================================
+    /**
+     * Generate tool summaries (Tier 2) from assessment results.
+     * Targets ~500 tokens per tool for efficient LLM processing.
+     *
+     * @param results - Full assessment results
+     * @returns Collection of tool summaries
+     */
+    generateToolSummaries(results) {
+        const tools = [];
+        // Get all unique tool names from security tests
+        const toolNames = this.extractToolNames(results);
+        for (const toolName of toolNames) {
+            const summary = this.generateSingleToolSummary(toolName, results);
+            tools.push(summary);
+        }
+        // Sort by risk level (highest first)
+        tools.sort((a, b) => {
+            const riskOrder = {
+                HIGH: 0,
+                MEDIUM: 1,
+                LOW: 2,
+                SAFE: 3,
+            };
+            return riskOrder[a.riskLevel] - riskOrder[b.riskLevel];
+        });
+        const aggregate = this.calculateAggregate(tools);
+        const collection = {
+            tools,
+            totalTools: tools.length,
+            aggregate,
+            estimatedTokens: 0,
+            generatedAt: new Date().toISOString(),
+        };
+        collection.estimatedTokens = estimateTokens(collection);
+        return collection;
+    }
+    /**
+     * Extract all unique tool names from assessment results.
+     */
+    extractToolNames(results) {
+        const toolNames = new Set();
+        // From security tests
+        const tests = results.security?.promptInjectionTests ?? [];
+        for (const test of tests) {
+            if (test.toolName) {
+                toolNames.add(test.toolName);
+            }
+        }
+        // From functionality results
+        const funcResults = results.functionality?.toolResults ?? [];
+        for (const result of funcResults) {
+            if (result.toolName) {
+                toolNames.add(result.toolName);
+            }
+        }
+        // From annotation results
+        const annotationResults = results.toolAnnotations?.toolResults ?? [];
+        for (const result of annotationResults) {
+            if (result.toolName) {
+                toolNames.add(result.toolName);
+            }
+        }
+        return [...toolNames].sort();
+    }
+    /**
+     * Generate summary for a single tool.
+     */
+    generateSingleToolSummary(toolName, results) {
+        const tests = this.getToolTests(toolName, results);
+        const vulnCount = tests.filter((t) => t.vulnerable).length;
+        const patterns = this.extractTopPatterns(tests);
+        const passRate = this.calculatePassRate(tests);
+        const annotationInfo = this.getToolAnnotationInfo(toolName, results);
+        const summary = {
+            toolName,
+            riskLevel: this.calculateToolRiskLevel(vulnCount),
+            vulnerabilityCount: vulnCount,
+            topPatterns: patterns.slice(0, this.config.maxPatternsPerTool),
+            testCount: tests.length,
+            passRate,
+            recommendations: this.generateToolRecommendations(toolName, vulnCount, annotationInfo),
+            estimatedTokens: 0,
+            hasAnnotations: annotationInfo.hasAnnotations,
+            annotationStatus: annotationInfo.status,
+        };
+        // Issue #137: Add Stage B enrichment if enabled
+        if (this.config.stageBVerbose) {
+            const allTests = results.security?.promptInjectionTests ?? [];
+            summary.stageBEnrichment = buildToolSummaryStageBEnrichment(toolName, allTests, 3);
+        }
+        summary.estimatedTokens = estimateTokens(summary);
+        return summary;
+    }
+    /**
+     * Get all security tests for a specific tool.
+     */
+    getToolTests(toolName, results) {
+        const tests = results.security?.promptInjectionTests ?? [];
+        return tests.filter((t) => t.toolName === toolName);
+    }
+    /**
+     * Extract top vulnerability patterns from tests.
+     */
+    extractTopPatterns(tests) {
+        const patternCounts = new Map();
+        for (const test of tests) {
+            if (test.vulnerable && test.testName) {
+                const current = patternCounts.get(test.testName) ?? 0;
+                patternCounts.set(test.testName, current + 1);
+            }
+        }
+        // Sort by count and return names
+        return [...patternCounts.entries()]
+            .sort((a, b) => b[1] - a[1])
+            .map(([name]) => name);
+    }
+    /**
+     * Calculate pass rate for tests.
+     */
+    calculatePassRate(tests) {
+        if (tests.length === 0)
+            return 100;
+        const passed = tests.filter((t) => !t.vulnerable).length;
+        return Math.round((passed / tests.length) * 100);
+    }
+    /**
+     * Get annotation info for a tool.
+     */
+    getToolAnnotationInfo(toolName, results) {
+        const annotationResults = results.toolAnnotations?.toolResults ?? [];
+        const toolResult = annotationResults.find((r) => r.toolName === toolName);
+        if (!toolResult) {
+            return { hasAnnotations: false, status: "MISSING" };
+        }
+        const hasAnnotations = toolResult.annotations?.readOnlyHint !== undefined ||
+            toolResult.annotations?.destructiveHint !== undefined;
+        let status;
+        if (!hasAnnotations) {
+            status = "MISSING";
+        }
+        else if (toolResult.alignmentStatus === "ALIGNED") {
+            status = "ALIGNED";
+        }
+        else if (toolResult.alignmentStatus === "MISALIGNED") {
+            status = "MISALIGNED";
+        }
+        return { hasAnnotations, status };
+    }
+    /**
+     * Generate recommendations for a specific tool.
+     */
+    generateToolRecommendations(toolName, vulnCount, annotationInfo) {
+        const recommendations = [];
+        if (vulnCount >= 5) {
+            recommendations.push(`Critical: ${toolName} has ${vulnCount} vulnerabilities - requires immediate security review`);
+        }
+        else if (vulnCount >= 2) {
+            recommendations.push(`${toolName} has ${vulnCount} vulnerabilities - review input validation`);
+        }
+        else if (vulnCount >= 1) {
+            recommendations.push(`${toolName} has a vulnerability - investigate and patch`);
+        }
+        if (!annotationInfo.hasAnnotations) {
+            recommendations.push(`Add readOnlyHint/destructiveHint annotations to ${toolName}`);
+        }
+        else if (annotationInfo.status === "MISALIGNED") {
+            recommendations.push(`Review annotation alignment for ${toolName}`);
+        }
+        return recommendations;
+    }
+    /**
+     * Calculate aggregate statistics across all tool summaries.
+     */
+    calculateAggregate(tools) {
+        const totalVulns = tools.reduce((sum, t) => sum + t.vulnerabilityCount, 0);
+        const avgPassRate = tools.length > 0
+            ? Math.round(tools.reduce((sum, t) => sum + t.passRate, 0) / tools.length)
+            : 100;
+        const misaligned = tools.filter((t) => t.annotationStatus === "MISALIGNED").length;
+        return {
+            totalVulnerabilities: totalVulns,
+            averagePassRate: avgPassRate,
+            misalignedAnnotations: misaligned,
+        };
+    }
+    // ==========================================================================
+    // Tier 3: Per-Tool Details (extraction helpers)
+    // ==========================================================================
+    /**
+     * Extract full detail data for a specific tool.
+     * Used when generating Tier 3 per-tool detail files.
+     *
+     * @param toolName - Tool name to extract
+     * @param results - Full assessment results
+     * @returns Tool-specific detail data
+     */
+    extractToolDetail(toolName, results) {
+        const securityTests = this.getToolTests(toolName, results);
+        const functionalityResult = results.functionality?.toolResults?.find((r) => r.toolName === toolName);
+        const annotationResult = results.toolAnnotations?.toolResults?.find((r) => r.toolName === toolName);
+        const detail = {
+            toolName,
+            extractedAt: new Date().toISOString(),
+            security: {
+                tests: securityTests,
+                vulnerableCount: securityTests.filter((t) => t.vulnerable).length,
+                totalTests: securityTests.length,
+            },
+            functionality: functionalityResult ?? null,
+            annotations: annotationResult ?? null,
+            estimatedTokens: estimateTokens({
+                security: { tests: securityTests },
+                functionality: functionalityResult,
+                annotations: annotationResult,
+            }),
+        };
+        // Issue #137: Add Stage B enrichment for Tier 3 if enabled
+        if (this.config.stageBVerbose) {
+            const allTests = results.security?.promptInjectionTests ?? [];
+            const aupViolations = results.aupCompliance?.violations;
+            detail.stageBEnrichment = buildToolDetailStageBEnrichment(toolName, allTests, annotationResult, aupViolations, 50);
+        }
+        return detail;
+    }
+    /**
+     * Get all tool names for Tier 3 file generation.
+     */
+    getAllToolNames(results) {
+        return this.extractToolNames(results);
+    }
+}

package/lib/lib/assessment/summarizer/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Assessment Summarizer Module
+ *
+ * Generates tiered output for large assessment results to fit within
+ * LLM context windows.
+ *
+ * Issue #136: Tiered output strategy for large assessments
+ * Issue #137: Stage B enrichment for Claude semantic analysis
+ *
+ * @module assessment/summarizer
+ */
+export type { OutputFormat, ToolRiskLevel, ExecutiveSummary, ToolSummary, ToolSummariesCollection, ToolDetailReference, TieredOutput, SummarizerConfig, } from "./types.js";
+export { DEFAULT_SUMMARIZER_CONFIG } from "./types.js";
+export type { FindingEvidence, PayloadCorrelation, ToolSummaryStageBEnrichment, ToolDetailStageBEnrichment, StageBEnrichment, } from "./stageBTypes.js";
+export { STAGE_B_ENRICHMENT_VERSION, DEFAULT_TIER2_MAX_SAMPLES, DEFAULT_TIER3_MAX_CORRELATIONS, MAX_RESPONSE_LENGTH, MAX_CONTEXT_WINDOW, } from "./stageBTypes.js";
+export { buildToolSummaryStageBEnrichment, buildToolDetailStageBEnrichment, } from "./stageBEnrichmentBuilder.js";
+export { estimateTokens, estimateJsonFileTokens, shouldAutoTier, formatTokenEstimate, estimateSectionTokens, getTopSections, } from "./tokenEstimator.js";
+export { AssessmentSummarizer } from "./AssessmentSummarizer.js";
+//# sourceMappingURL=index.d.ts.map

package/lib/lib/assessment/summarizer/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/assessment/summarizer/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,YAAY,EACV,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,uBAAuB,EACvB,mBAAmB,EACnB,YAAY,EACZ,gBAAgB,GACjB,MAAM,SAAS,CAAC;AAEjB,OAAO,EAAE,yBAAyB,EAAE,MAAM,SAAS,CAAC;AAGpD,YAAY,EACV,eAAe,EACf,kBAAkB,EAClB,2BAA2B,EAC3B,0BAA0B,EAC1B,gBAAgB,GACjB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,EACzB,8BAA8B,EAC9B,mBAAmB,EACnB,kBAAkB,GACnB,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EACL,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,GACf,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC"}

package/lib/lib/assessment/summarizer/index.js

@@ -0,0 +1,19 @@
+/**
+ * Assessment Summarizer Module
+ *
+ * Generates tiered output for large assessment results to fit within
+ * LLM context windows.
+ *
+ * Issue #136: Tiered output strategy for large assessments
+ * Issue #137: Stage B enrichment for Claude semantic analysis
+ *
+ * @module assessment/summarizer
+ */
+export { DEFAULT_SUMMARIZER_CONFIG } from "./types.js";
+export { STAGE_B_ENRICHMENT_VERSION, DEFAULT_TIER2_MAX_SAMPLES, DEFAULT_TIER3_MAX_CORRELATIONS, MAX_RESPONSE_LENGTH, MAX_CONTEXT_WINDOW, } from "./stageBTypes.js";
+// Stage B Enrichment Builders (Issue #137)
+export { buildToolSummaryStageBEnrichment, buildToolDetailStageBEnrichment, } from "./stageBEnrichmentBuilder.js";
+// Token estimation utilities
+export { estimateTokens, estimateJsonFileTokens, shouldAutoTier, formatTokenEstimate, estimateSectionTokens, getTopSections, } from "./tokenEstimator.js";
+// Main summarizer class
+export { AssessmentSummarizer } from "./AssessmentSummarizer.js";

package/lib/lib/assessment/summarizer/stageBEnrichmentBuilder.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Stage B Enrichment Builder
+ *
+ * Functions to build Stage B enrichment data from assessment results.
+ * Extracts evidence, correlations, and confidence details for Claude
+ * semantic analysis.
+ *
+ * Issue #137: Stage A data enrichment for Stage B Claude analysis
+ *
+ * @module assessment/summarizer/stageBEnrichmentBuilder
+ */
+import type { SecurityTestResult } from "../resultTypes.js";
+import type { AUPViolation } from "../extendedTypes.js";
+import type { EnhancedToolAnnotationResult } from "../../../services/assessment/modules/annotations/types.js";
+import { type ToolSummaryStageBEnrichment, type ToolDetailStageBEnrichment } from "./stageBTypes.js";
+/**
+ * Build Stage B enrichment for Tier 2 tool summaries.
+ *
+ * @param toolName - Name of the tool
+ * @param tests - Security test results for this tool
+ * @param maxSamples - Maximum evidence samples to include
+ * @returns Tool summary Stage B enrichment
+ */
+export declare function buildToolSummaryStageBEnrichment(toolName: string, tests: SecurityTestResult[], maxSamples?: number): ToolSummaryStageBEnrichment;
+/**
+ * Build Stage B enrichment for Tier 3 per-tool detail files.
+ *
+ * @param toolName - Name of the tool
+ * @param tests - Security test results for this tool
+ * @param annotationResult - Tool annotation result (if available)
+ * @param aupViolations - AUP violations for this tool (if any)
+ * @param maxCorrelations - Maximum correlations to include
+ * @returns Tool detail Stage B enrichment
+ */
+export declare function buildToolDetailStageBEnrichment(toolName: string, tests: SecurityTestResult[], annotationResult?: EnhancedToolAnnotationResult, aupViolations?: AUPViolation[], maxCorrelations?: number): ToolDetailStageBEnrichment;
+//# sourceMappingURL=stageBEnrichmentBuilder.d.ts.map

package/lib/lib/assessment/summarizer/stageBEnrichmentBuilder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stageBEnrichmentBuilder.d.ts","sourceRoot":"","sources":["../../../../src/lib/assessment/summarizer/stageBEnrichmentBuilder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,wDAAwD,CAAC;AAC3G,OAAO,EAGL,KAAK,2BAA2B,EAChC,KAAK,0BAA0B,EAKhC,MAAM,eAAe,CAAC;AA6HvB;;;;;;;GAOG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,kBAAkB,EAAE,EAC3B,UAAU,GAAE,MAAkC,GAC7C,2BAA2B,CA6C7B;AAMD;;;;;;;;;GASG;AACH,wBAAgB,+BAA+B,CAC7C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,kBAAkB,EAAE,EAC3B,gBAAgB,CAAC,EAAE,4BAA4B,EAC/C,aAAa,CAAC,EAAE,YAAY,EAAE,EAC9B,eAAe,GAAE,MAAuC,GACvD,0BAA0B,CA4I5B"}