@dotsetlabs/bellwether 0.10.0

package/dist/baseline/migrations.js

@@ -0,0 +1,197 @@
+/**
+ * Baseline format migrations.
+ *
+ * This module handles upgrading baselines from older CLI versions to the current version.
+ * Migrations are triggered when the CLI major version changes.
+ *
+ * Migration Strategy:
+ * - Migrations are keyed by their TARGET major version
+ * - Each migration upgrades from the previous major version
+ * - Migrations are applied sequentially in version order
+ * - Downgrading is not supported
+ */
+import { VERSION } from '../version.js';
+import { parseVersion, compareVersions } from './version.js';
+/**
+ * Registry of migrations keyed by target major version.
+ *
+ * When adding a new migration for CLI v2.0.0:
+ * 1. Add an entry with '2' as the key (major version)
+ * 2. Implement the migration function to transform from v1.x to v2.x format
+ *
+ * Example for future migration from v1.x to v2.0.0:
+ *
+ * '2': (baseline) => {
+ *   return {
+ *     ...baseline,
+ *     version: '2.0.0',
+ *     // Add new required fields with defaults
+ *     newField: baseline.newField ?? 'default',
+ *     // Transform renamed fields
+ *     renamedField: baseline.oldFieldName,
+ *     // Remove deprecated fields (just don't include them)
+ *   };
+ * },
+ */
+const MIGRATIONS = {
+    // Migration from legacy format version "1.0.0" to CLI version format
+    // This handles baselines created before the versioning simplification
+    '0': (baseline) => {
+        const version = baseline.version;
+        // If version looks like old format version (1.0.0), convert to CLI version
+        if (version === '1.0.0' || version === '1.0' || version === 1) {
+            return {
+                ...baseline,
+                version: VERSION, // Use current CLI version
+            };
+        }
+        return baseline;
+    },
+    // Future migrations would be added here when CLI major version changes:
+    // '1': (baseline) => { ... }, // Migrates v0.x baselines to v1.x format
+    // '2': (baseline) => { ... }, // Migrates v1.x baselines to v2.x format
+};
+/**
+ * Get the current CLI major version.
+ */
+function getCurrentMajorVersion() {
+    return parseVersion(VERSION).major;
+}
+/**
+ * Check if a version is the legacy format version (before CLI version was used).
+ * Legacy format versions were "1.0.0", "1.0", or numeric 1.
+ * These need special handling because they're not CLI versions.
+ */
+function isLegacyFormatVersion(version) {
+    return version === '1.0.0' || version === '1.0' || version === 1;
+}
+/**
+ * Check if a baseline can be migrated to the current CLI version.
+ *
+ * Migration is possible if:
+ * - Source version has a different major version than current
+ * - A migration path exists
+ *
+ * @param fromVersion - Source version (string, number, or undefined)
+ * @returns true if migration is possible
+ */
+export function canMigrate(fromVersion) {
+    const from = parseVersion(fromVersion);
+    const currentMajor = getCurrentMajorVersion();
+    // Same major version - no migration needed
+    if (from.major === currentMajor) {
+        return true;
+    }
+    // Check if we have a migration for the current major version
+    return MIGRATIONS[String(currentMajor)] !== undefined;
+}
+/**
+ * Get the list of migrations that would be applied.
+ *
+ * @param fromVersion - Source version
+ * @returns Array of major version strings for migrations that would be applied
+ */
+export function getMigrationsToApply(fromVersion) {
+    const currentMajor = getCurrentMajorVersion();
+    // Legacy format version needs the '0' migration
+    if (isLegacyFormatVersion(fromVersion)) {
+        return MIGRATIONS['0'] ? ['0'] : [];
+    }
+    const from = parseVersion(fromVersion);
+    // Same major version - no migrations needed
+    if (from.major === currentMajor) {
+        return [];
+    }
+    // Return migrations from source major to current major
+    const toApply = [];
+    for (let major = from.major; major <= currentMajor; major++) {
+        if (MIGRATIONS[String(major)]) {
+            toApply.push(String(major));
+        }
+    }
+    return toApply;
+}
+/**
+ * Migrate a baseline to the current CLI version format.
+ *
+ * @param baseline - The baseline object to migrate (can be any version)
+ * @returns Migrated baseline conforming to BehavioralBaseline interface
+ * @throws Error if migration is not possible (e.g., downgrade attempt)
+ */
+export function migrateBaseline(baseline) {
+    const sourceVersion = baseline.version;
+    const current = parseVersion(VERSION);
+    // Handle legacy format version specially (not a real CLI version)
+    if (isLegacyFormatVersion(sourceVersion)) {
+        let migrated = { ...baseline };
+        const migration = MIGRATIONS['0'];
+        if (migration) {
+            migrated = migration(migrated);
+        }
+        migrated.version = VERSION;
+        return migrated;
+    }
+    const from = parseVersion(sourceVersion);
+    // Already at current version
+    if (compareVersions(from, current) === 0) {
+        return baseline;
+    }
+    // Same major version - just update the version string
+    if (from.major === current.major) {
+        return {
+            ...baseline,
+            version: VERSION,
+        };
+    }
+    // Cannot downgrade (only applies to actual CLI versions, not legacy format)
+    if (from.major > current.major) {
+        throw new Error(`Cannot downgrade baseline from v${from.raw} to v${current.raw}. ` +
+            `Downgrading baselines is not supported.`);
+    }
+    // Apply migrations
+    let migrated = { ...baseline };
+    const migrationsToApply = getMigrationsToApply(sourceVersion);
+    for (const majorVersion of migrationsToApply) {
+        const migration = MIGRATIONS[majorVersion];
+        if (migration) {
+            migrated = migration(migrated);
+        }
+    }
+    // Ensure version is set to current CLI version
+    migrated.version = VERSION;
+    return migrated;
+}
+/**
+ * Check if a baseline needs migration.
+ *
+ * @param baseline - The baseline to check
+ * @returns true if the baseline major version differs from current CLI major version
+ */
+export function needsMigration(baseline) {
+    const version = baseline.version;
+    const from = parseVersion(version);
+    const currentMajor = getCurrentMajorVersion();
+    // Check for legacy format version (1.0.0) which needs migration to CLI version
+    if (version === '1.0.0' || version === '1.0' || version === 1) {
+        return true;
+    }
+    return from.major !== currentMajor;
+}
+/**
+ * Get information about what migrations would be applied.
+ *
+ * @param baseline - The baseline to analyze
+ * @returns Object with migration details
+ */
+export function getMigrationInfo(baseline) {
+    const version = baseline.version;
+    const from = parseVersion(version);
+    return {
+        currentVersion: from.raw,
+        targetVersion: VERSION,
+        needsMigration: needsMigration(baseline),
+        migrationsToApply: getMigrationsToApply(version),
+        canMigrate: canMigrate(version),
+    };
+}
+//# sourceMappingURL=migrations.js.map

package/dist/baseline/performance-tracker.d.ts

@@ -0,0 +1,214 @@
+/**
+ * Performance Regression Detection
+ *
+ * Tracks response times across tool executions and detects performance regressions.
+ * Provides percentile-based metrics (p50, p95, p99) for comprehensive latency analysis.
+ */
+import type { BehavioralBaseline, ChangeSeverity, PerformanceConfidence } from './types.js';
+/**
+ * Latency trend direction.
+ */
+export type LatencyTrend = 'improving' | 'stable' | 'degrading';
+/**
+ * Performance metrics for a single tool.
+ */
+export interface ToolPerformanceMetrics {
+    /** Tool name */
+    toolName: string;
+    /** 50th percentile latency in milliseconds */
+    p50Ms: number;
+    /** 95th percentile latency in milliseconds */
+    p95Ms: number;
+    /** 99th percentile latency in milliseconds */
+    p99Ms: number;
+    /** Success rate (0-1) */
+    successRate: number;
+    /** Total number of executions */
+    sampleCount: number;
+    /** Average latency in milliseconds */
+    avgMs: number;
+    /** Minimum latency in milliseconds */
+    minMs: number;
+    /** Maximum latency in milliseconds */
+    maxMs: number;
+    /** Standard deviation of latency */
+    stdDevMs: number;
+    /** Timestamp of when metrics were collected */
+    collectedAt: Date;
+    /** Statistical confidence metrics */
+    confidence?: PerformanceConfidence;
+}
+/**
+ * Performance baseline for a tool (stored in baseline file).
+ */
+export interface PerformanceBaseline {
+    /** Tool name */
+    toolName: string;
+    /** Baseline 50th percentile latency */
+    baselineP50: number;
+    /** Baseline 95th percentile latency */
+    baselineP95: number;
+    /** Baseline 99th percentile latency */
+    baselineP99: number;
+    /** Baseline success rate */
+    baselineSuccessRate: number;
+    /** Maximum allowed regression percentage (default from config) */
+    maxAllowedRegression: number;
+    /** When the baseline was established */
+    establishedAt: Date;
+}
+/**
+ * Performance comparison result for a single tool.
+ */
+export interface PerformanceComparison {
+    /** Tool name */
+    toolName: string;
+    /** Current metrics */
+    current: ToolPerformanceMetrics;
+    /** Baseline metrics (if available) */
+    baseline?: PerformanceBaseline;
+    /** Latency trend */
+    trend: LatencyTrend;
+    /** Regression percentage for p50 (positive = slower, negative = faster) */
+    p50RegressionPercent: number | null;
+    /** Regression percentage for p95 */
+    p95RegressionPercent: number | null;
+    /** Regression percentage for p99 */
+    p99RegressionPercent: number | null;
+    /** Whether this tool has regressed beyond threshold */
+    hasRegression: boolean;
+    /** Severity of the regression */
+    severity: ChangeSeverity;
+    /** Human-readable summary */
+    summary: string;
+    /** Current confidence level */
+    confidence?: PerformanceConfidence;
+    /** Whether the regression is statistically reliable (based on confidence) */
+    isReliable: boolean;
+}
+/**
+ * Overall performance report for a baseline comparison.
+ */
+export interface PerformanceReport {
+    /** Individual tool comparisons */
+    toolComparisons: PerformanceComparison[];
+    /** Number of tools with performance regressions */
+    regressionCount: number;
+    /** Number of tools with improved performance */
+    improvementCount: number;
+    /** Number of tools with stable performance */
+    stableCount: number;
+    /** Overall performance trend */
+    overallTrend: LatencyTrend;
+    /** Overall severity */
+    overallSeverity: ChangeSeverity;
+    /** Human-readable summary */
+    summary: string;
+    /** Number of tools with low confidence */
+    lowConfidenceCount: number;
+    /** Tools with low confidence (names) */
+    lowConfidenceTools: string[];
+    /** Number of reliable regressions (regressions with good confidence) */
+    reliableRegressionCount: number;
+}
+/**
+ * Raw latency sample for calculating metrics.
+ */
+export interface LatencySample {
+    toolName: string;
+    durationMs: number;
+    success: boolean;
+    timestamp: Date;
+    /**
+     * Expected outcome of this test.
+     * - 'success': Happy path test, expects tool to succeed
+     * - 'error': Validation test, expects tool to reject/fail
+     * - 'either': Edge case, either outcome is acceptable
+     */
+    expectedOutcome?: 'success' | 'error' | 'either';
+    /**
+     * Whether the outcome was correct based on expectations.
+     * True if: (expected success && got success) OR (expected error && got error)
+     */
+    outcomeCorrect?: boolean;
+}
+export { PERFORMANCE_TRACKING as PERFORMANCE } from '../constants.js';
+/**
+ * Calculate statistical confidence for performance metrics.
+ *
+ * Confidence is determined by:
+ * 1. Sample count - more samples = higher confidence
+ * 2. Coefficient of variation (CV) - lower variability = higher confidence
+ *
+ * Key insight: For confidence calculation, we only count happy_path tests that
+ * expect success. Validation tests (expectedOutcome: 'error') are tracked
+ * separately because their failure doesn't indicate tool problems.
+ *
+ * Note: The first sample is excluded from variance calculation because it includes
+ * cold-start overhead (JIT compilation, connection establishment, cache warming).
+ * This gives more accurate confidence scores for steady-state performance.
+ *
+ * @param samples - The latency samples to analyze
+ * @param options - Optional configuration
+ * @returns Performance confidence metrics
+ */
+export declare function calculatePerformanceConfidence(samples: LatencySample[], options?: {
+    excludeWarmup?: boolean;
+}): PerformanceConfidence;
+/**
+ * Calculate performance confidence from ToolPerformanceMetrics.
+ * Use this when you already have calculated metrics but need confidence.
+ *
+ * Note: This function assumes the metrics are from happy path tests only.
+ * For full validation/success separation, use calculatePerformanceConfidence with raw samples.
+ */
+export declare function calculateConfidenceFromMetrics(metrics: ToolPerformanceMetrics, options?: {
+    validationSamples?: number;
+    totalTests?: number;
+}): PerformanceConfidence;
+/**
+ * Format confidence level for display.
+ */
+export declare function formatConfidenceLevel(confidence: PerformanceConfidence, includeIndicator?: boolean): string;
+/**
+ * Check if performance data has sufficient confidence for reliable comparisons.
+ */
+export declare function hasReliableConfidence(confidence: PerformanceConfidence): boolean;
+/**
+ * Calculate performance metrics from raw latency samples.
+ */
+export declare function calculateMetrics(samples: LatencySample[]): ToolPerformanceMetrics | null;
+/**
+ * Create a performance baseline from metrics.
+ */
+export declare function createPerformanceBaseline(metrics: ToolPerformanceMetrics, maxAllowedRegression?: number): PerformanceBaseline;
+/**
+ * Extract performance baselines from a behavioral baseline.
+ * Uses the performance metrics stored in tool fingerprints.
+ */
+export declare function extractPerformanceBaselines(baseline: BehavioralBaseline, regressionThreshold?: number): Map<string, PerformanceBaseline>;
+/**
+ * Compare current metrics against baseline.
+ */
+export declare function comparePerformance(current: ToolPerformanceMetrics, baseline: PerformanceBaseline | undefined, regressionThreshold?: number): PerformanceComparison;
+/**
+ * Generate a complete performance report comparing current and baseline.
+ */
+export declare function generatePerformanceReport(currentMetrics: Map<string, ToolPerformanceMetrics>, baselines: Map<string, PerformanceBaseline>, regressionThreshold?: number): PerformanceReport;
+/**
+ * Format performance metrics for display.
+ */
+export declare function formatMetrics(metrics: ToolPerformanceMetrics): string;
+/**
+ * Format performance comparison for display.
+ */
+export declare function formatComparison(comparison: PerformanceComparison): string;
+/**
+ * Check if metrics indicate acceptable performance.
+ */
+export declare function isPerformanceAcceptable(comparison: PerformanceComparison, failOnRegression?: boolean): boolean;
+/**
+ * Aggregate multiple samples into metrics grouped by tool.
+ */
+export declare function aggregateSamplesByTool(samples: LatencySample[]): Map<string, ToolPerformanceMetrics>;
+//# sourceMappingURL=performance-tracker.d.ts.map