@bryan-thompson/inspector-assessment-client 1.34.1 → 1.35.1

package/lib/lib/assessment/summarizer/tokenEstimator.js

@@ -0,0 +1,225 @@
+/**
+ * Token Estimation Utilities
+ *
+ * Provides token counting and threshold detection for tiered output strategy.
+ * Uses industry-standard approximation of ~4 characters per token.
+ *
+ * Issue #136: Tiered output strategy for large assessments
+ *
+ * @module assessment/summarizer/tokenEstimator
+ */
+import { DEFAULT_SUMMARIZER_CONFIG } from "./types.js";
+// ============================================================================
+// Constants
+// ============================================================================
+/**
+ * Average characters per token for modern LLMs (GPT, Claude).
+ * This is an approximation; actual tokenization varies by model and content.
+ */
+const CHARS_PER_TOKEN = 4;
+/**
+ * Buffer factor to account for JSON formatting overhead.
+ * Pretty-printed JSON adds whitespace that increases character count.
+ */
+const JSON_FORMAT_BUFFER = 1.1;
+// ============================================================================
+// Token Estimation Functions
+// ============================================================================
+/**
+ * Estimate the number of tokens for any content.
+ *
+ * Uses the industry-standard approximation of ~4 characters per token.
+ * For JSON content, applies a buffer for formatting overhead.
+ *
+ * @param content - Content to estimate (string, object, or array)
+ * @returns Estimated token count
+ *
+ * @example
+ * ```typescript
+ * // String content
+ * estimateTokens("Hello world"); // ~3 tokens
+ *
+ * // Object content (will be JSON stringified)
+ * estimateTokens({ name: "test", value: 123 }); // ~10 tokens
+ *
+ * // Large assessment results
+ * estimateTokens(assessmentResults); // ~50,000+ tokens
+ * ```
+ */
+export function estimateTokens(content) {
+    let charCount;
+    if (typeof content === "string") {
+        charCount = content.length;
+    }
+    else if (content === null || content === undefined) {
+        return 0;
+    }
+    else {
+        // JSON stringify for objects/arrays
+        try {
+            const json = JSON.stringify(content, null, 2);
+            charCount = Math.ceil(json.length * JSON_FORMAT_BUFFER);
+        }
+        catch {
+            // Fallback for circular references or other stringify issues
+            return 0;
+        }
+    }
+    return Math.ceil(charCount / CHARS_PER_TOKEN);
+}
+/**
+ * Estimate tokens for a JSON file that would be written.
+ * Accounts for pretty-printing with indent=2.
+ *
+ * @param content - Content that would be JSON.stringify'd
+ * @returns Estimated token count
+ */
+export function estimateJsonFileTokens(content) {
+    if (content === null || content === undefined) {
+        return 0;
+    }
+    try {
+        const json = JSON.stringify(content, null, 2);
+        return Math.ceil(json.length / CHARS_PER_TOKEN);
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Determine if assessment results should automatically use tiered output.
+ *
+ * Returns true when estimated token count exceeds the threshold,
+ * indicating the full output would not fit in typical LLM context windows.
+ *
+ * @param results - Full assessment results
+ * @param threshold - Token threshold (default: 100,000)
+ * @returns true if results should be tiered
+ *
+ * @example
+ * ```typescript
+ * const results = await runAssessment(server);
+ *
+ * if (shouldAutoTier(results)) {
+ *   // Use tiered output
+ *   saveTieredResults(serverName, results, options);
+ * } else {
+ *   // Use standard full output
+ *   saveResults(serverName, results, options);
+ * }
+ * ```
+ */
+export function shouldAutoTier(results, threshold = DEFAULT_SUMMARIZER_CONFIG.autoTierThreshold) {
+    const estimated = estimateTokens(results);
+    return estimated > threshold;
+}
+/**
+ * Get a human-readable token estimate with size category.
+ *
+ * @param tokenCount - Number of tokens
+ * @returns Object with formatted token count and size category
+ *
+ * @example
+ * ```typescript
+ * formatTokenEstimate(5000);
+ * // { tokens: "5,000", category: "small", fitsContext: true }
+ *
+ * formatTokenEstimate(500000);
+ * // { tokens: "500,000", category: "very-large", fitsContext: false }
+ * ```
+ */
+export function formatTokenEstimate(tokenCount) {
+    const formatted = tokenCount.toLocaleString();
+    let category;
+    let fitsContext;
+    let recommendation;
+    if (tokenCount <= 10_000) {
+        category = "small";
+        fitsContext = true;
+        recommendation = "Full output recommended";
+    }
+    else if (tokenCount <= 50_000) {
+        category = "medium";
+        fitsContext = true;
+        recommendation = "Full output should fit most contexts";
+    }
+    else if (tokenCount <= 100_000) {
+        category = "large";
+        fitsContext = true;
+        recommendation = "Consider tiered output for smaller context windows";
+    }
+    else if (tokenCount <= 200_000) {
+        category = "very-large";
+        fitsContext = false;
+        recommendation = "Tiered output recommended";
+    }
+    else {
+        category = "oversized";
+        fitsContext = false;
+        recommendation = "Tiered output required";
+    }
+    return { tokens: formatted, category, fitsContext, recommendation };
+}
+/**
+ * Estimate tokens for each major section of assessment results.
+ * Useful for understanding which modules contribute most to output size.
+ *
+ * @param results - Assessment results to analyze
+ * @returns Map of section name to estimated token count
+ */
+export function estimateSectionTokens(results) {
+    const sections = {};
+    // Core assessment sections
+    const sectionKeys = [
+        "functionality",
+        "security",
+        "errorHandling",
+        "aupCompliance",
+        "toolAnnotations",
+        "temporal",
+        "resources",
+        "prompts",
+        "crossCapability",
+        "protocolCompliance",
+        "developerExperience",
+        "prohibitedLibraries",
+        "manifestValidation",
+        "authentication",
+        "portability",
+        "externalAPIScanner",
+    ];
+    for (const key of sectionKeys) {
+        const section = results[key];
+        if (section !== undefined) {
+            sections[key] = estimateTokens(section);
+        }
+    }
+    // Metadata and summary
+    sections["metadata"] = estimateTokens({
+        serverName: results.serverName,
+        overallStatus: results.overallStatus,
+        summary: results.summary,
+        recommendations: results.recommendations,
+        totalTestsRun: results.totalTestsRun,
+        executionTime: results.executionTime,
+    });
+    // Calculate total
+    sections["_total"] = Object.entries(sections)
+        .filter(([key]) => !key.startsWith("_"))
+        .reduce((sum, [, tokens]) => sum + tokens, 0);
+    return sections;
+}
+/**
+ * Get the top N largest sections by token count.
+ *
+ * @param results - Assessment results
+ * @param topN - Number of sections to return (default: 5)
+ * @returns Array of [sectionName, tokenCount] sorted by size descending
+ */
+export function getTopSections(results, topN = 5) {
+    const sections = estimateSectionTokens(results);
+    return Object.entries(sections)
+        .filter(([key]) => !key.startsWith("_"))
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, topN);
+}

package/lib/lib/assessment/summarizer/types.d.ts

@@ -0,0 +1,187 @@
+/**
+ * Tiered Output Types
+ *
+ * Type definitions for the tiered output strategy that generates
+ * LLM-consumable summaries from large assessment results.
+ *
+ * Issue #136: Tiered output strategy for large assessments
+ *
+ * @module assessment/summarizer/types
+ */
+import type { AssessmentStatus } from "../coreTypes.js";
+import type { ToolSummaryStageBEnrichment } from "./stageBTypes.js";
+/**
+ * Output format for assessment results.
+ * - "full": Complete JSON output (default, existing behavior)
+ * - "tiered": Directory structure with executive summary, tool summaries, and per-tool details
+ * - "summary-only": Only executive summary and tool summaries (no per-tool detail files)
+ */
+export type OutputFormat = "full" | "tiered" | "summary-only";
+/**
+ * Risk level categorization for tools based on security assessment results.
+ */
+export type ToolRiskLevel = "HIGH" | "MEDIUM" | "LOW" | "SAFE";
+/**
+ * Executive Summary - Tier 1 output.
+ * Always generated, always fits in LLM context window.
+ * Provides high-level overview for quick assessment understanding.
+ */
+export interface ExecutiveSummary {
+    /** Server name from assessment */
+    serverName: string;
+    /** Overall assessment status (PASS/FAIL/NEED_MORE_INFO) */
+    overallStatus: AssessmentStatus;
+    /** Calculated overall score (0-100) */
+    overallScore: number;
+    /** Total number of tools discovered */
+    toolCount: number;
+    /** Total number of tests executed */
+    testCount: number;
+    /** Total execution time in milliseconds */
+    executionTime: number;
+    /**
+     * Per-module status and score summary.
+     * Key is module name (e.g., "security", "functionality")
+     */
+    modulesSummary: Record<string, {
+        status: AssessmentStatus;
+        score: number;
+    }>;
+    /** Critical findings aggregated from all modules */
+    criticalFindings: {
+        /** Number of security vulnerabilities detected */
+        securityVulnerabilities: number;
+        /** Number of AUP violations detected */
+        aupViolations: number;
+        /** Number of broken/non-functional tools */
+        brokenTools: number;
+        /** Number of tools missing required annotations */
+        missingAnnotations: number;
+    };
+    /**
+     * Distribution of tools by risk level.
+     * Helps quickly understand overall risk profile.
+     */
+    toolRiskDistribution: {
+        high: number;
+        medium: number;
+        low: number;
+        safe: number;
+    };
+    /** Top recommendations aggregated from all modules */
+    recommendations: string[];
+    /** Estimated token count for this summary */
+    estimatedTokens: number;
+    /** ISO timestamp when summary was generated */
+    generatedAt: string;
+}
+/**
+ * Tool Summary - Tier 2 output.
+ * Per-tool digest without individual test results.
+ * Enables focused analysis on specific tools without full detail.
+ */
+export interface ToolSummary {
+    /** Tool name from MCP server */
+    toolName: string;
+    /** Calculated risk level based on security findings */
+    riskLevel: ToolRiskLevel;
+    /** Number of vulnerabilities found for this tool */
+    vulnerabilityCount: number;
+    /**
+     * Top vulnerability patterns detected.
+     * Limited to top 5 for token efficiency.
+     */
+    topPatterns: string[];
+    /** Total number of tests run on this tool */
+    testCount: number;
+    /** Percentage of tests that passed (0-100) */
+    passRate: number;
+    /** Tool-specific recommendations */
+    recommendations: string[];
+    /** Estimated token count for this summary */
+    estimatedTokens: number;
+    /** Whether the tool has proper annotations */
+    hasAnnotations: boolean;
+    /** Annotation alignment status if available */
+    annotationStatus?: "ALIGNED" | "MISALIGNED" | "MISSING";
+    /** Stage B enrichment for Claude semantic analysis (Issue #137) */
+    stageBEnrichment?: ToolSummaryStageBEnrichment;
+}
+/**
+ * Collection of tool summaries with aggregate metadata.
+ */
+export interface ToolSummariesCollection {
+    /** Individual tool summaries */
+    tools: ToolSummary[];
+    /** Total number of tools */
+    totalTools: number;
+    /** Aggregate statistics */
+    aggregate: {
+        /** Total vulnerabilities across all tools */
+        totalVulnerabilities: number;
+        /** Average pass rate across all tools */
+        averagePassRate: number;
+        /** Tools with misaligned annotations */
+        misalignedAnnotations: number;
+    };
+    /** Estimated total tokens for all summaries */
+    estimatedTokens: number;
+    /** ISO timestamp */
+    generatedAt: string;
+}
+/**
+ * Reference to a per-tool detail file (Tier 3).
+ * Full test results stored in separate files for deep-dive analysis.
+ */
+export interface ToolDetailReference {
+    /** Tool name */
+    toolName: string;
+    /** Relative path to detail file (e.g., "tools/my_tool.json") */
+    relativePath: string;
+    /** Absolute path to detail file */
+    absolutePath: string;
+    /** File size in bytes */
+    fileSizeBytes: number;
+    /** Estimated token count for full detail */
+    estimatedTokens: number;
+}
+/**
+ * Complete tiered output structure.
+ * Contains all tiers with paths to generated files.
+ */
+export interface TieredOutput {
+    /** Tier 1: Executive summary */
+    executiveSummary: ExecutiveSummary;
+    /** Tier 2: Tool summaries */
+    toolSummaries: ToolSummariesCollection;
+    /** Tier 3: References to per-tool detail files */
+    toolDetailRefs: ToolDetailReference[];
+    /** Output directory path */
+    outputDir: string;
+    /** File paths for each tier */
+    paths: {
+        executiveSummary: string;
+        toolSummaries: string;
+        toolDetailsDir: string;
+    };
+}
+/**
+ * Configuration options for the summarizer.
+ */
+export interface SummarizerConfig {
+    /** Maximum number of recommendations to include in executive summary */
+    maxRecommendations?: number;
+    /** Maximum number of top patterns per tool in tool summaries */
+    maxPatternsPerTool?: number;
+    /** Token threshold for auto-tiering (default: 100,000) */
+    autoTierThreshold?: number;
+    /** Whether to include tool detail files (Tier 3) */
+    includeToolDetails?: boolean;
+    /** Enable Stage B enrichment for Claude semantic analysis (Issue #137) */
+    stageBVerbose?: boolean;
+}
+/**
+ * Default summarizer configuration values.
+ */
+export declare const DEFAULT_SUMMARIZER_CONFIG: Required<SummarizerConfig>;
+//# sourceMappingURL=types.d.ts.map

package/lib/lib/assessment/summarizer/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/lib/assessment/summarizer/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAMjE;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,QAAQ,GAAG,cAAc,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAM/D;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IAEnB,2DAA2D;IAC3D,aAAa,EAAE,gBAAgB,CAAC;IAEhC,uCAAuC;IACvC,YAAY,EAAE,MAAM,CAAC;IAErB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAElB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAElB,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,cAAc,EAAE,MAAM,CACpB,MAAM,EACN;QACE,MAAM,EAAE,gBAAgB,CAAC;QACzB,KAAK,EAAE,MAAM,CAAC;KACf,CACF,CAAC;IAEF,oDAAoD;IACpD,gBAAgB,EAAE;QAChB,kDAAkD;QAClD,uBAAuB,EAAE,MAAM,CAAC;QAChC,wCAAwC;QACxC,aAAa,EAAE,MAAM,CAAC;QACtB,4CAA4C;QAC5C,WAAW,EAAE,MAAM,CAAC;QACpB,mDAAmD;QACnD,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IAEF;;;OAGG;IACH,oBAAoB,EAAE;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IAEF,sDAAsD;IACtD,eAAe,EAAE,MAAM,EAAE,CAAC;IAE1B,6CAA6C;IAC7C,eAAe,EAAE,MAAM,CAAC;IAExB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IAEjB,uDAAuD;IACvD,SAAS,EAAE,aAAa,CAAC;IAEzB,oDAAoD;IACpD,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IAEtB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAElB,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IAEjB,oCAAoC;IACpC,eAAe,EAAE,MAAM,EAAE,CAAC;IAE1B,6CAA6C;IAC7C,eAAe,EAAE,MAAM,CAAC;IAExB,8CAA8C;IAC9C,cAAc,EAAE,OAAO,CAAC;IAExB,+CAA+C;IAC/C,gBAAgB,CAAC,EAAE,SAAS,GAAG,YAAY,GAAG,SAAS,CAAC;IAExD,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,2BAA2B,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,gCAAgC;IAChC,KAAK,EAAE,WAAW,EAAE,CAAC;IAErB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IAEnB,2BAA2B;IAC3B,SAAS,EAAE;QACT,6CAA6C;QAC7C,oBAAoB,EAAE,MAAM,CAAC;QAC7B,yCAAyC;QACzC,eAAe,EAAE,MAAM,CAAC;QACxB,wCAAwC;QACxC,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;IAEF,+CAA+C;IAC/C,eAAe,EAAE,MAAM,CAAC;IAExB,oBAAoB;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAC;IAEjB,gEAAgE;IAChE,YAAY,EAAE,MAAM,CAAC;IAErB,mCAAmC;IACnC,YAAY,EAAE,MAAM,CAAC;IAErB,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IAEtB,4CAA4C;IAC5C,eAAe,EAAE,MAAM,CAAC;CACzB;AAMD;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,gCAAgC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IAEnC,6BAA6B;IAC7B,aAAa,EAAE,uBAAuB,CAAC;IAEvC,kDAAkD;IAClD,cAAc,EAAE,mBAAmB,EAAE,CAAC;IAEtC,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;IAElB,+BAA+B;IAC/B,KAAK,EAAE;QACL,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,gEAAgE;IAChE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,0DAA0D;IAC1D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B,0EAA0E;IAC1E,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,EAAE,QAAQ,CAAC,gBAAgB,CAMhE,CAAC"}

package/lib/lib/assessment/summarizer/types.js

@@ -0,0 +1,20 @@
+/**
+ * Tiered Output Types
+ *
+ * Type definitions for the tiered output strategy that generates
+ * LLM-consumable summaries from large assessment results.
+ *
+ * Issue #136: Tiered output strategy for large assessments
+ *
+ * @module assessment/summarizer/types
+ */
+/**
+ * Default summarizer configuration values.
+ */
+export const DEFAULT_SUMMARIZER_CONFIG = {
+    maxRecommendations: 10,
+    maxPatternsPerTool: 5,
+    autoTierThreshold: 100_000,
+    includeToolDetails: true,
+    stageBVerbose: false,
+};

package/lib/lib/moduleScoring.d.ts

@@ -32,6 +32,11 @@ export declare const INSPECTOR_VERSION: string;
  * - scripts/lib/jsonl-events.ts
  * - cli/src/lib/jsonl-events.ts
  * - client/src/services/assessment/orchestratorHelpers.ts
+ *
+ * Version History:
+ * - v1: Initial schema
+ * - v2: Added TestValidityWarningEvent (Issue #134)
+ * - v3: Added Stage B enrichment for Claude semantic analysis (Issue #137)
  */
-export declare const SCHEMA_VERSION = 1;
+export declare const SCHEMA_VERSION = 3;
 //# sourceMappingURL=moduleScoring.d.ts.map

package/lib/lib/moduleScoring.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"moduleScoring.d.ts","sourceRoot":"","sources":["../../src/lib/moduleScoring.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAkCnE;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,QAAsB,CAAC;AAErD;;;;;;;;;GASG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC"}
+{"version":3,"file":"moduleScoring.d.ts","sourceRoot":"","sources":["../../src/lib/moduleScoring.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAkCnE;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,QAAsB,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC"}

package/lib/lib/moduleScoring.js

@@ -69,5 +69,10 @@ export const INSPECTOR_VERSION = packageJson.version;
  * - scripts/lib/jsonl-events.ts
  * - cli/src/lib/jsonl-events.ts
  * - client/src/services/assessment/orchestratorHelpers.ts
+ *
+ * Version History:
+ * - v1: Initial schema
+ * - v2: Added TestValidityWarningEvent (Issue #134)
+ * - v3: Added Stage B enrichment for Claude semantic analysis (Issue #137)
  */
-export const SCHEMA_VERSION = 1;
+export const SCHEMA_VERSION = 3;

package/lib/services/assessment/modules/SecurityAssessor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAa9D,OAAO,EACL,gBAAgB,EAGjB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAiC;IAErD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAStD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAOjC;;;OAGG;YACW,0BAA0B;gBAwBtC,MAAM,EAAE,OAAO,8BAA8B,EAAE,uBAAuB;IAwClE,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAsMrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;OAEG;YACW,+BAA+B;IAiC7C;;;OAGG;YACW,yBAAyB;IA0CvC;;;;;;;OAOG;YACW,yBAAyB;IAmFvC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAgDnC"}
+{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiB9D,OAAO,EACL,gBAAgB,EAGjB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,aAAa,CAAwB;IAC7C,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAiC;IAErD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI;IAStD;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAOjC;;;OAGG;YACW,0BAA0B;gBAwBtC,MAAM,EAAE,OAAO,8BAA8B,EAAE,uBAAuB;IAwClE,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA+OrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;OAEG;YACW,+BAA+B;IAiC7C;;;OAGG;YACW,yBAAyB;IA0CvC;;;;;;;OAOG;YACW,yBAAyB;IAmFvC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;CAgDnC"}

package/lib/services/assessment/modules/SecurityAssessor.js

@@ -23,7 +23,7 @@
  *   Tests for multi-tool chain exploitation attacks
  */
 import { BaseAssessor } from "./BaseAssessor.js";
-import { SecurityPayloadTester, SecurityPayloadGenerator, CrossToolStateTester, ChainExecutionTester, } from "./securityTests/index.js";
+import { SecurityPayloadTester, SecurityPayloadGenerator, CrossToolStateTester, ChainExecutionTester, TestValidityAnalyzer, } from "./securityTests/index.js";
 import { ToolClassifier, ToolCategory } from "../ToolClassifier.js";
 export class SecurityAssessor extends BaseAssessor {
     payloadTester;
@@ -217,9 +217,40 @@ export class SecurityAssessor extends BaseAssessor {
         // Determine overall risk level
         const overallRiskLevel = this.determineOverallRiskLevel(highRiskCount, mediumRiskCount, vulnerabilities.length);
         // Determine status (pass validTests array to check confidence levels, not allTests)
-        const status = this.determineSecurityStatus(validTests, vulnerabilities.length, validTests.length, connectionErrors.length);
+        let status = this.determineSecurityStatus(validTests, vulnerabilities.length, validTests.length, connectionErrors.length);
+        // Issue #134: Analyze test validity (detect uniform responses)
+        const validityAnalyzer = new TestValidityAnalyzer();
+        const validityResult = validityAnalyzer.analyze(validTests);
+        // Adjust status if test validity is compromised
+        let overallConfidence;
+        if (validityResult.isCompromised) {
+            overallConfidence = validityResult.recommendedConfidence;
+            // If tests are compromised and reporting PASS, change to NEED_MORE_INFO
+            if (status === "PASS" && validityResult.warningLevel === "critical") {
+                status = "NEED_MORE_INFO";
+                this.logger.info(`⚠️ Security status changed to NEED_MORE_INFO due to ${validityResult.warning?.percentageIdentical}% identical responses`);
+            }
+            // Emit test validity warning event
+            if (context.onProgress && validityResult.warning) {
+                const validityWarningEvent = {
+                    type: "test_validity_warning",
+                    module: "security",
+                    identicalResponseCount: validityResult.warning.identicalResponseCount,
+                    totalResponses: validityResult.warning.totalResponses,
+                    percentageIdentical: validityResult.warning.percentageIdentical,
+                    detectedPattern: validityResult.warning.detectedPattern,
+                    warningLevel: validityResult.warningLevel,
+                    recommendedConfidence: validityResult.recommendedConfidence,
+                };
+                context.onProgress(validityWarningEvent);
+            }
+        }
         // Generate explanation (pass both validTests and connectionErrors)
-        const explanation = this.generateSecurityExplanation(validTests, connectionErrors, vulnerabilities, overallRiskLevel);
+        let explanation = this.generateSecurityExplanation(validTests, connectionErrors, vulnerabilities, overallRiskLevel);
+        // Prepend validity warning to explanation if compromised
+        if (validityResult.isCompromised && validityResult.warning) {
+            explanation = `⚠️ TEST VALIDITY WARNING: ${validityResult.warning.explanation}\n\n${explanation}`;
+        }
         // Issue #75: Aggregate auth bypass detection results
         const authBypassSummary = this.aggregateAuthBypassResults(allTests);
         return {
@@ -229,6 +260,9 @@ export class SecurityAssessor extends BaseAssessor {
             status,
             explanation,
             authBypassSummary,
+            // Issue #134: Test validity warning for response uniformity detection
+            testValidityWarning: validityResult.warning,
+            overallConfidence,
         };
     }
     /**

package/lib/services/assessment/modules/securityTests/TestValidityAnalyzer.d.ts

@@ -0,0 +1,118 @@
+/**
+ * Test Validity Analyzer
+ *
+ * Detects when security test responses are suspiciously uniform,
+ * indicating tests may not have reached security-relevant code paths.
+ *
+ * @see Issue #134: Detect identical security test responses (test validity masking)
+ */
+import type { SecurityTestResult } from "../../../../lib/assessment/resultTypes.js";
+import type { TestValidityWarning } from "../../../../lib/assessment/resultTypes.js";
+/**
+ * Configuration for test validity analysis
+ */
+export interface TestValidityConfig {
+    /** Percentage threshold to trigger warning (default: 80) */
+    warningThresholdPercent: number;
+    /** Percentage threshold to reduce confidence (default: 90) */
+    confidenceReduceThresholdPercent: number;
+    /** Minimum tests required for analysis (default: 10) */
+    minimumTestsForAnalysis: number;
+    /** Maximum response length to compare (default: 1000) */
+    maxResponseCompareLength: number;
+    /** Maximum sample payload-response pairs (default: 10) */
+    maxSamplePairs: number;
+    /** Maximum response distribution entries (default: 5) */
+    maxDistributionEntries: number;
+}
+/**
+ * Result of test validity analysis
+ */
+export interface TestValidityResult {
+    /** Whether test validity is compromised */
+    isCompromised: boolean;
+    /** Warning level: none, warning, critical */
+    warningLevel: "none" | "warning" | "critical";
+    /** Recommended confidence adjustment */
+    recommendedConfidence: "high" | "medium" | "low";
+    /** Detailed warning information */
+    warning?: TestValidityWarning;
+    /** Per-tool uniformity analysis */
+    toolUniformity?: Map<string, {
+        identicalCount: number;
+        totalCount: number;
+        percentageIdentical: number;
+    }>;
+}
+/**
+ * Analyzes security test results for response uniformity.
+ *
+ * When a high percentage of test responses are identical, it indicates
+ * that tests may be hitting a configuration error, connection issue,
+ * or other problem that prevents them from reaching security-relevant code.
+ */
+export declare class TestValidityAnalyzer {
+    private config;
+    constructor(config?: Partial<TestValidityConfig>);
+    /**
+     * Analyze test results for response uniformity
+     *
+     * @param testResults - Array of security test results with responses
+     * @returns Analysis result with warning details if uniformity detected
+     */
+    analyze(testResults: SecurityTestResult[]): TestValidityResult;
+    /**
+     * Normalize response for comparison.
+     * Removes timestamps, UUIDs, request IDs, and other variable content.
+     */
+    private normalizeResponse;
+    /**
+     * Count occurrences of normalized responses
+     */
+    private countNormalizedResponses;
+    /**
+     * Find the most common response
+     */
+    private findMostCommon;
+    /**
+     * Find original (non-normalized) sample that matches the normalized pattern
+     */
+    private findOriginalSample;
+    /**
+     * Detect the category of the response pattern
+     */
+    private detectPatternCategory;
+    /**
+     * Analyze uniformity per tool
+     */
+    private analyzePerTool;
+    /**
+     * Generate human-readable explanation
+     */
+    private generateExplanation;
+    /**
+     * Calculate Shannon entropy for response diversity (0=uniform, 1=max diversity)
+     */
+    private calculateEntropy;
+    /**
+     * Build response distribution sorted by frequency
+     */
+    private buildResponseDistribution;
+    /**
+     * Extract attack category from test name
+     */
+    private extractAttackCategory;
+    /**
+     * Analyze attack pattern correlation by category
+     */
+    private analyzeAttackPatterns;
+    /**
+     * Collect sample payload-response pairs with category diversity
+     */
+    private collectSamplePairs;
+    /**
+     * Collect response metadata statistics
+     */
+    private collectResponseMetadata;
+}
+//# sourceMappingURL=TestValidityAnalyzer.d.ts.map

package/lib/services/assessment/modules/securityTests/TestValidityAnalyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestValidityAnalyzer.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/TestValidityAnalyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,4DAA4D;IAC5D,uBAAuB,EAAE,MAAM,CAAC;IAChC,8DAA8D;IAC9D,gCAAgC,EAAE,MAAM,CAAC;IACzC,wDAAwD;IACxD,uBAAuB,EAAE,MAAM,CAAC;IAChC,yDAAyD;IACzD,wBAAwB,EAAE,MAAM,CAAC;IAEjC,0DAA0D;IAC1D,cAAc,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,aAAa,EAAE,OAAO,CAAC;IACvB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;IAC9C,wCAAwC;IACxC,qBAAqB,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACjD,mCAAmC;IACnC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,mCAAmC;IACnC,cAAc,CAAC,EAAE,GAAG,CAClB,MAAM,EACN;QACE,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,CACF,CAAC;CACH;AAYD;;;;;;GAMG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAqB;gBAEvB,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IAIhD;;;;;OAKG;IACH,OAAO,CAAC,WAAW,EAAE,kBAAkB,EAAE,GAAG,kBAAkB;IAyF9D;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgCzB;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAehC;;OAEG;IACH,OAAO,CAAC,cAAc;IActB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAe1B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA2C7B;;OAEG;IACH,OAAO,CAAC,cAAc;IA4CtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAcjC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAc7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA2C7B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAkD1B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAwDhC"}