@dotsetlabs/bellwether 0.10.0

package/dist/baseline/calibration.js

@@ -0,0 +1,136 @@
+/**
+ * Confidence Calibration for Drift Detection
+ *
+ * Calibrates raw confidence scores to match actual accuracy.
+ * A calibrated confidence of 80% means the algorithm is correct ~80% of the time
+ * when it reports that confidence level.
+ *
+ * Calibration is based on evaluation against the golden dataset.
+ */
+/**
+ * Default calibration model based on golden dataset evaluation.
+ *
+ * These values should be updated as the algorithm improves.
+ * Current baseline: v1.0.1 (50 test cases)
+ */
+export const DEFAULT_CALIBRATION_MODEL = [
+    // High confidence predictions
+    { min: 90, max: 101, calibratedAccuracy: 85, sampleCount: 12 },
+    { min: 80, max: 90, calibratedAccuracy: 75, sampleCount: 15 },
+    // Medium confidence predictions
+    { min: 70, max: 80, calibratedAccuracy: 65, sampleCount: 10 },
+    { min: 60, max: 70, calibratedAccuracy: 55, sampleCount: 8 },
+    // Low confidence predictions
+    { min: 50, max: 60, calibratedAccuracy: 45, sampleCount: 5 },
+    { min: 0, max: 50, calibratedAccuracy: 35, sampleCount: 10 },
+];
+/**
+ * Calibrate a raw confidence score to reflect actual accuracy.
+ *
+ * @param rawScore - Raw confidence score (0-100)
+ * @param model - Calibration model to use (defaults to DEFAULT_CALIBRATION_MODEL)
+ * @returns Calibrated confidence score
+ */
+export function calibrateConfidence(rawScore, model = DEFAULT_CALIBRATION_MODEL) {
+    // Find the bucket for this score
+    const bucket = model.find(b => rawScore >= b.min && rawScore < b.max);
+    if (!bucket) {
+        // Score outside all buckets, return as-is
+        return rawScore;
+    }
+    return bucket.calibratedAccuracy;
+}
+/**
+ * Format confidence score with calibration information.
+ *
+ * @param rawScore - Raw confidence score
+ * @param showRaw - Whether to show raw score alongside calibrated
+ * @returns Formatted string
+ */
+export function formatCalibratedConfidence(rawScore, showRaw = false) {
+    const calibrated = calibrateConfidence(rawScore);
+    if (showRaw && calibrated !== rawScore) {
+        return `${calibrated}% (raw: ${rawScore}%)`;
+    }
+    return `${calibrated}%`;
+}
+/**
+ * Get confidence label based on calibrated score.
+ */
+export function getCalibratedConfidenceLabel(rawScore) {
+    const calibrated = calibrateConfidence(rawScore);
+    if (calibrated >= 75)
+        return 'high';
+    if (calibrated >= 55)
+        return 'medium';
+    if (calibrated >= 40)
+        return 'low';
+    return 'very-low';
+}
+/**
+ * Check if a calibrated confidence meets a threshold.
+ *
+ * @param rawScore - Raw confidence score
+ * @param threshold - Minimum required calibrated confidence
+ * @returns True if calibrated confidence meets threshold
+ */
+export function meetsCalibratedThreshold(rawScore, threshold) {
+    return calibrateConfidence(rawScore) >= threshold;
+}
+/**
+ * Update calibration model based on evaluation results.
+ * This recalculates accuracy for each bucket from test results.
+ *
+ * @param results - Array of {predictedConfidence, wasCorrect} pairs
+ * @returns Updated calibration model
+ */
+export function updateCalibrationModel(results) {
+    const bucketRanges = [
+        { min: 90, max: 101 },
+        { min: 80, max: 90 },
+        { min: 70, max: 80 },
+        { min: 60, max: 70 },
+        { min: 50, max: 60 },
+        { min: 0, max: 50 },
+    ];
+    return bucketRanges.map(range => {
+        const bucketResults = results.filter(r => r.predictedConfidence >= range.min && r.predictedConfidence < range.max);
+        if (bucketResults.length === 0) {
+            return {
+                ...range,
+                calibratedAccuracy: (range.min + range.max) / 2,
+                sampleCount: 0,
+            };
+        }
+        const correctCount = bucketResults.filter(r => r.wasCorrect).length;
+        const accuracy = Math.round((correctCount / bucketResults.length) * 100);
+        return {
+            ...range,
+            calibratedAccuracy: accuracy,
+            sampleCount: bucketResults.length,
+        };
+    });
+}
+/**
+ * Calculate calibration error (ECE - Expected Calibration Error).
+ * Lower is better. 0 = perfectly calibrated.
+ *
+ * @param model - Calibration model
+ * @returns ECE as a percentage (0-100)
+ */
+export function calculateCalibrationError(model) {
+    let totalError = 0;
+    let totalSamples = 0;
+    for (const bucket of model) {
+        if (bucket.sampleCount > 0) {
+            const midpoint = (bucket.min + bucket.max) / 2;
+            const error = Math.abs(midpoint - bucket.calibratedAccuracy);
+            totalError += error * bucket.sampleCount;
+            totalSamples += bucket.sampleCount;
+        }
+    }
+    if (totalSamples === 0)
+        return 0;
+    return Math.round(totalError / totalSamples);
+}
+//# sourceMappingURL=calibration.js.map

package/dist/baseline/category-matching.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Category Matching and Relationship Scoring
+ *
+ * Provides multi-category detection and relationship scoring for improved
+ * recall in semantic matching. Instead of first-match-wins, this module
+ * extracts ALL matching categories and scores relationships between them.
+ *
+ * Key improvements over single-category matching:
+ * 1. Multi-category detection - finds all matching categories
+ * 2. Relationship scoring - related categories get partial credit
+ * 3. Best-match selection - finds highest-confidence category pair
+ */
+/**
+ * Result of category extraction with confidence.
+ */
+export interface CategoryMatch {
+    category: string;
+    confidence: number;
+    matchedKeywords: string[];
+}
+/**
+ * Category relationship groups.
+ * Categories in the same group are related and should get partial credit.
+ *
+ * IMPORTANT (v1.3.0): Groups are now more conservative to prevent false positives.
+ * - Authentication and Authorization are NOT grouped (they're different concerns)
+ * - Injection types are more narrowly grouped
+ * - Only truly similar vulnerabilities are grouped
+ */
+export declare const SECURITY_CATEGORY_GROUPS: Record<string, string[]>;
+/**
+ * Limitation category relationship groups.
+ */
+export declare const LIMITATION_CATEGORY_GROUPS: Record<string, string[]>;
+/**
+ * Extract ALL matching security categories from text with confidence scores.
+ * Unlike single-category extraction, this returns all matches ranked by confidence.
+ */
+export declare function extractSecurityCategories(text: string): CategoryMatch[];
+/**
+ * Extract ALL matching limitation categories from text with confidence scores.
+ */
+export declare function extractLimitationCategories(text: string): CategoryMatch[];
+/**
+ * Calculate relationship score between two security categories.
+ * Returns 0-100 where:
+ * - 100: Same category
+ * - 70-90: Categories in same group
+ * - 40-60: Related categories
+ * - 0-30: Unrelated categories
+ */
+export declare function calculateSecurityCategoryRelationship(cat1: string, cat2: string): number;
+/**
+ * Calculate relationship score between two limitation categories.
+ */
+export declare function calculateLimitationCategoryRelationship(cat1: string, cat2: string): number;
+/**
+ * Find the best category match between two texts.
+ * Returns the highest-scoring category pair and their relationship score.
+ */
+export declare function findBestSecurityCategoryMatch(categories1: CategoryMatch[], categories2: CategoryMatch[]): {
+    cat1: string;
+    cat2: string;
+    relationshipScore: number;
+    combinedConfidence: number;
+} | null;
+/**
+ * Find the best limitation category match between two texts.
+ */
+export declare function findBestLimitationCategoryMatch(categories1: CategoryMatch[], categories2: CategoryMatch[]): {
+    cat1: string;
+    cat2: string;
+    relationshipScore: number;
+    combinedConfidence: number;
+} | null;
+/**
+ * Check if two security categories are considered matching.
+ * Uses relationship scoring for partial credit.
+ */
+export declare function securityCategoriesMatch(cat1: string, cat2: string): boolean;
+/**
+ * Check if two limitation categories are considered matching.
+ */
+export declare function limitationCategoriesMatch(cat1: string, cat2: string): boolean;
+//# sourceMappingURL=category-matching.d.ts.map

package/dist/baseline/category-matching.js

@@ -0,0 +1,289 @@
+/**
+ * Category Matching and Relationship Scoring
+ *
+ * Provides multi-category detection and relationship scoring for improved
+ * recall in semantic matching. Instead of first-match-wins, this module
+ * extracts ALL matching categories and scores relationships between them.
+ *
+ * Key improvements over single-category matching:
+ * 1. Multi-category detection - finds all matching categories
+ * 2. Relationship scoring - related categories get partial credit
+ * 3. Best-match selection - finds highest-confidence category pair
+ */
+import { EXTENDED_SECURITY_KEYWORDS } from '../utils/semantic.js';
+/**
+ * Category relationship groups.
+ * Categories in the same group are related and should get partial credit.
+ *
+ * IMPORTANT (v1.3.0): Groups are now more conservative to prevent false positives.
+ * - Authentication and Authorization are NOT grouped (they're different concerns)
+ * - Injection types are more narrowly grouped
+ * - Only truly similar vulnerabilities are grouped
+ */
+export const SECURITY_CATEGORY_GROUPS = {
+    // File/path related vulnerabilities (narrowed)
+    file_access: ['path_traversal', 'file_upload'],
+    // SQL-specific injection (narrowed - SQL and command injection are different)
+    sql_issues: ['sql_injection'],
+    // Command execution issues
+    command_issues: ['command_injection'],
+    // XSS-specific
+    xss_issues: ['xss', 'output_encoding'],
+    // Access control (without auth/authz - they're different)
+    access: ['access_control'],
+    // Data handling issues (narrowed)
+    data: ['information_disclosure', 'cryptography'],
+    // Input handling
+    input_handling: ['input_validation'],
+    // Server-side issues (narrowed)
+    server_side: ['ssrf'],
+    // Deserialization standalone
+    deserialization_issues: ['deserialization'],
+    // XXE standalone (different from deserialization despite both being data handling)
+    xxe_issues: ['xxe'],
+};
+/**
+ * Limitation category relationship groups.
+ */
+export const LIMITATION_CATEGORY_GROUPS = {
+    // Resource constraints
+    resource: ['size_limit', 'memory', 'rate_limit'],
+    // Time-related
+    temporal: ['timeout', 'rate_limit'],
+    // Access-related
+    access: ['permission', 'platform'],
+    // Format-related
+    format: ['encoding', 'format'],
+};
+/**
+ * Direct category similarity scores (0-100).
+ * Used for categories that are similar but not in the same group.
+ *
+ * IMPORTANT (v1.3.0): Reduced similarity scores to prevent false positives.
+ * Authentication vs Authorization are now considered DIFFERENT (not related).
+ * Only truly similar categories get partial credit.
+ */
+const CATEGORY_SIMILARITY = {
+    path_traversal: {
+        file_upload: 40, // Reduced - only somewhat related
+        information_disclosure: 30, // Reduced
+    },
+    sql_injection: {
+        input_validation: 40, // Reduced - input validation is generic
+    },
+    xss: {
+        output_encoding: 50, // Reduced
+        input_validation: 30, // Reduced
+    },
+    // IMPORTANT: authentication and authorization are now DIFFERENT
+    // They are distinct security concerns and should not match
+    authentication: {
+        session_management: 40, // Only session mgmt is somewhat related
+    },
+    authorization: {
+        access_control: 50, // Access control is related but distinct
+    },
+    input_validation: {
+        output_encoding: 40,
+    },
+};
+/**
+ * Extract ALL matching security categories from text with confidence scores.
+ * Unlike single-category extraction, this returns all matches ranked by confidence.
+ */
+export function extractSecurityCategories(text) {
+    const matches = [];
+    const lowerText = text.toLowerCase();
+    for (const [category, keywords] of Object.entries(EXTENDED_SECURITY_KEYWORDS)) {
+        const matchedKeywords = keywords.filter(keyword => lowerText.includes(keyword));
+        if (matchedKeywords.length > 0) {
+            // Calculate confidence based on keyword match quality
+            const confidence = calculateCategoryConfidence(text, matchedKeywords, keywords);
+            matches.push({
+                category,
+                confidence,
+                matchedKeywords,
+            });
+        }
+    }
+    // Sort by confidence (highest first)
+    return matches.sort((a, b) => b.confidence - a.confidence);
+}
+/**
+ * Limitation category keywords for multi-category extraction.
+ */
+const LIMITATION_KEYWORDS = {
+    size_limit: ['size limit', 'max size', 'file size', 'mb', 'gb', 'kb', 'bytes', 'too large', 'megabytes', 'gigabytes', 'kilobytes', 'maximum'],
+    rate_limit: ['rate limit', 'throttle', 'requests per', 'quota', 'too many requests', 'rate limiting'],
+    timeout: ['timeout', 'time out', 'time limit', 'seconds', 'timed out', 'deadline', 'expires'],
+    encoding: ['encoding', 'utf-8', 'ascii', 'binary', 'charset', 'unicode'],
+    format: ['format', 'json', 'xml', 'csv', 'type', 'mime', 'content-type'],
+    permission: ['permission', 'access', 'denied', 'forbidden', 'read-only', 'write', 'privileges'],
+    platform: ['platform', 'windows', 'linux', 'macos', 'os-specific', 'operating system'],
+    dependency: ['dependency', 'requires', 'prerequisite', 'library', 'package', 'module'],
+    concurrency: ['concurrent', 'parallel', 'thread', 'lock', 'race condition', 'simultaneous'],
+    memory: ['memory', 'ram', 'heap', 'out of memory', 'memory limit'],
+    network: ['network', 'connection', 'offline', 'unreachable', 'connectivity'],
+};
+/**
+ * Extract ALL matching limitation categories from text with confidence scores.
+ */
+export function extractLimitationCategories(text) {
+    const matches = [];
+    const lowerText = text.toLowerCase();
+    for (const [category, keywords] of Object.entries(LIMITATION_KEYWORDS)) {
+        const matchedKeywords = keywords.filter(keyword => lowerText.includes(keyword));
+        if (matchedKeywords.length > 0) {
+            const confidence = calculateCategoryConfidence(text, matchedKeywords, keywords);
+            matches.push({
+                category,
+                confidence,
+                matchedKeywords,
+            });
+        }
+    }
+    return matches.sort((a, b) => b.confidence - a.confidence);
+}
+/**
+ * Calculate confidence for a category match based on keyword quality.
+ */
+function calculateCategoryConfidence(text, matchedKeywords, allKeywords) {
+    if (matchedKeywords.length === 0)
+        return 0;
+    // Base confidence from keyword count
+    const keywordRatio = matchedKeywords.length / Math.min(allKeywords.length, 5);
+    let confidence = Math.min(keywordRatio * 60, 60); // Max 60 from keyword count
+    // Bonus for longer/more specific keywords
+    const avgKeywordLength = matchedKeywords.reduce((sum, k) => sum + k.length, 0) / matchedKeywords.length;
+    if (avgKeywordLength > 10)
+        confidence += 20;
+    else if (avgKeywordLength > 5)
+        confidence += 10;
+    // Bonus for multiple distinct keywords
+    if (matchedKeywords.length >= 3)
+        confidence += 15;
+    else if (matchedKeywords.length >= 2)
+        confidence += 10;
+    // Penalty if text is very long but few keywords matched
+    const textLength = text.length;
+    if (textLength > 200 && matchedKeywords.length === 1) {
+        confidence -= 10;
+    }
+    return Math.min(Math.max(confidence, 10), 100);
+}
+/**
+ * Calculate relationship score between two security categories.
+ * Returns 0-100 where:
+ * - 100: Same category
+ * - 70-90: Categories in same group
+ * - 40-60: Related categories
+ * - 0-30: Unrelated categories
+ */
+export function calculateSecurityCategoryRelationship(cat1, cat2) {
+    if (cat1 === cat2)
+        return 100;
+    // Check direct similarity scores
+    const directScore = CATEGORY_SIMILARITY[cat1]?.[cat2] ?? CATEGORY_SIMILARITY[cat2]?.[cat1];
+    if (directScore !== undefined)
+        return directScore;
+    // Check if in same group
+    for (const groupCategories of Object.values(SECURITY_CATEGORY_GROUPS)) {
+        if (groupCategories.includes(cat1) && groupCategories.includes(cat2)) {
+            return 70; // Same group gets 70%
+        }
+    }
+    // Unrelated
+    return 0;
+}
+/**
+ * Calculate relationship score between two limitation categories.
+ */
+export function calculateLimitationCategoryRelationship(cat1, cat2) {
+    if (cat1 === cat2)
+        return 100;
+    // Check if in same group
+    for (const groupCategories of Object.values(LIMITATION_CATEGORY_GROUPS)) {
+        if (groupCategories.includes(cat1) && groupCategories.includes(cat2)) {
+            return 70;
+        }
+    }
+    return 0;
+}
+/**
+ * Find the best category match between two texts.
+ * Returns the highest-scoring category pair and their relationship score.
+ */
+export function findBestSecurityCategoryMatch(categories1, categories2) {
+    if (categories1.length === 0 || categories2.length === 0) {
+        return null;
+    }
+    let bestMatch = null;
+    let bestScore = 0;
+    for (const c1 of categories1) {
+        for (const c2 of categories2) {
+            const relationshipScore = calculateSecurityCategoryRelationship(c1.category, c2.category);
+            if (relationshipScore > 0) {
+                // Combined score considers both category confidence and relationship
+                const combinedConfidence = Math.round((c1.confidence * 0.4 + c2.confidence * 0.4 + relationshipScore * 0.2));
+                // Prefer higher relationship scores, then higher combined confidence
+                const totalScore = relationshipScore * 100 + combinedConfidence;
+                if (totalScore > bestScore) {
+                    bestScore = totalScore;
+                    bestMatch = {
+                        cat1: c1.category,
+                        cat2: c2.category,
+                        relationshipScore,
+                        combinedConfidence,
+                    };
+                }
+            }
+        }
+    }
+    return bestMatch;
+}
+/**
+ * Find the best limitation category match between two texts.
+ */
+export function findBestLimitationCategoryMatch(categories1, categories2) {
+    if (categories1.length === 0 || categories2.length === 0) {
+        return null;
+    }
+    let bestMatch = null;
+    let bestScore = 0;
+    for (const c1 of categories1) {
+        for (const c2 of categories2) {
+            const relationshipScore = calculateLimitationCategoryRelationship(c1.category, c2.category);
+            if (relationshipScore > 0) {
+                const combinedConfidence = Math.round((c1.confidence * 0.4 + c2.confidence * 0.4 + relationshipScore * 0.2));
+                const totalScore = relationshipScore * 100 + combinedConfidence;
+                if (totalScore > bestScore) {
+                    bestScore = totalScore;
+                    bestMatch = {
+                        cat1: c1.category,
+                        cat2: c2.category,
+                        relationshipScore,
+                        combinedConfidence,
+                    };
+                }
+            }
+        }
+    }
+    return bestMatch;
+}
+/**
+ * Check if two security categories are considered matching.
+ * Uses relationship scoring for partial credit.
+ */
+export function securityCategoriesMatch(cat1, cat2) {
+    const relationshipScore = calculateSecurityCategoryRelationship(cat1, cat2);
+    // Consider matching if relationship score is 50 or higher
+    return relationshipScore >= 50;
+}
+/**
+ * Check if two limitation categories are considered matching.
+ */
+export function limitationCategoriesMatch(cat1, cat2) {
+    const relationshipScore = calculateLimitationCategoryRelationship(cat1, cat2);
+    return relationshipScore >= 50;
+}
+//# sourceMappingURL=category-matching.js.map

package/dist/baseline/change-impact-analyzer.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Change Impact Analyzer
+ *
+ * Provides semantic understanding of schema changes and their impact.
+ * Goes beyond simple hash comparison to understand what actually breaks.
+ */
+import type { BehaviorChange, ChangeSeverity, ToolFingerprint, BehavioralBaseline, BehavioralDiff, WorkflowSignature } from './types.js';
+/**
+ * Type of schema change detected.
+ */
+export type SchemaChangeType = 'parameter_removed' | 'parameter_added' | 'parameter_type_changed' | 'parameter_required_added' | 'parameter_required_removed' | 'enum_value_removed' | 'enum_value_added' | 'constraint_added' | 'constraint_removed' | 'constraint_tightened' | 'constraint_relaxed' | 'description_changed' | 'default_changed' | 'format_changed';
+/**
+ * Detailed information about a single schema change.
+ */
+export interface SchemaChangeDetail {
+    type: SchemaChangeType;
+    parameterPath: string;
+    breaking: boolean;
+    before: unknown;
+    after: unknown;
+    description: string;
+}
+/**
+ * Migration complexity levels.
+ */
+export type MigrationComplexity = 'trivial' | 'simple' | 'moderate' | 'complex';
+/**
+ * Comprehensive impact analysis for a change.
+ */
+export interface ChangeImpact {
+    /** Overall severity of the change */
+    severity: ChangeSeverity;
+    /** List of affected workflow IDs */
+    affectedWorkflows: string[];
+    /** List of affected parameter paths */
+    affectedParameters: string[];
+    /** Estimated complexity to migrate */
+    migrationComplexity: MigrationComplexity;
+    /** Suggested migration approach */
+    suggestedMigration: string;
+    /** Detailed breakdown of schema changes */
+    schemaChanges: SchemaChangeDetail[];
+    /** Whether this change is backwards compatible */
+    backwardsCompatible: boolean;
+    /** Risk score (0-100) */
+    riskScore: number;
+}
+/**
+ * Impact analysis results for the entire diff.
+ */
+export interface DiffImpactAnalysis {
+    /** Overall severity of all changes */
+    overallSeverity: ChangeSeverity;
+    /** Total number of breaking changes */
+    breakingChangesCount: number;
+    /** Per-tool impact analysis */
+    toolImpacts: Map<string, ChangeImpact>;
+    /** Workflows that will fail due to changes */
+    brokenWorkflows: string[];
+    /** Overall migration complexity */
+    overallMigrationComplexity: MigrationComplexity;
+    /** Summary of all changes */
+    summary: string;
+    /** Action items for addressing the changes */
+    actionItems: ActionItem[];
+}
+/**
+ * Action item for addressing a change.
+ */
+export interface ActionItem {
+    priority: 'critical' | 'high' | 'medium' | 'low';
+    tool: string;
+    description: string;
+    suggestedAction: string;
+}
+export { CHANGE_IMPACT } from '../constants.js';
+/**
+ * Analyze the impact of changes between two tool fingerprints.
+ */
+export declare function analyzeToolChangeImpact(oldTool: ToolFingerprint, newTool: ToolFingerprint, workflows?: WorkflowSignature[]): ChangeImpact;
+/**
+ * Analyze a complete diff and provide comprehensive impact analysis.
+ */
+export declare function analyzeDiffImpact(diff: BehavioralDiff, oldBaseline: BehavioralBaseline, newBaseline: BehavioralBaseline): DiffImpactAnalysis;
+/**
+ * Analyze changes between two schemas and return detailed change information.
+ */
+export declare function analyzeSchemaChanges(oldSchema: Record<string, unknown> | undefined, newSchema: Record<string, unknown> | undefined): SchemaChangeDetail[];
+/**
+ * Check if a behavior change is actually breaking based on semantic analysis.
+ * This enhances the simple hash-based comparison with semantic understanding.
+ */
+export declare function isBreakingChange(change: BehaviorChange): boolean;
+/**
+ * Get a quick summary of breaking changes for CI output.
+ */
+export declare function getBreakingChangeSummary(analysis: DiffImpactAnalysis): string;
+//# sourceMappingURL=change-impact-analyzer.d.ts.map