@dotsetlabs/bellwether 0.10.0

package/dist/baseline/telemetry.js

@@ -0,0 +1,294 @@
+/**
+ * Telemetry and Decision Logging for Drift Detection
+ *
+ * Records comparison decisions for post-hoc analysis and algorithm improvement.
+ * Decisions are logged locally and can be exported for feedback or A/B testing.
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+import { randomUUID } from 'crypto';
+/**
+ * Get the telemetry directory path.
+ */
+function getTelemetryDir() {
+    return join(homedir(), '.bellwether', 'telemetry');
+}
+/**
+ * Ensure telemetry directory exists.
+ */
+function ensureTelemetryDir() {
+    const dir = getTelemetryDir();
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+}
+/**
+ * Decision logger for recording and analyzing comparison decisions.
+ */
+export class DecisionLogger {
+    decisions = [];
+    logPath;
+    enabled;
+    constructor(options = {}) {
+        this.enabled = options.enabled ?? true;
+        ensureTelemetryDir();
+        this.logPath = options.logPath ?? join(getTelemetryDir(), 'comparison-log.jsonl');
+    }
+    /**
+     * Log a comparison decision.
+     */
+    log(decision) {
+        const fullDecision = {
+            ...decision,
+            id: randomUUID(),
+            timestamp: new Date(),
+            bellwetherVersion: getVersion(),
+        };
+        this.decisions.push(fullDecision);
+        if (this.enabled) {
+            try {
+                appendFileSync(this.logPath, JSON.stringify(fullDecision) + '\n');
+            }
+            catch {
+                // Silently fail if we can't write to log file
+            }
+        }
+        return fullDecision.id;
+    }
+    /**
+     * Get all logged decisions from this session.
+     */
+    getSessionDecisions() {
+        return [...this.decisions];
+    }
+    /**
+     * Load all decisions from the log file.
+     */
+    loadAllDecisions() {
+        if (!existsSync(this.logPath)) {
+            return [];
+        }
+        try {
+            const content = readFileSync(this.logPath, 'utf-8');
+            const lines = content.trim().split('\n').filter(Boolean);
+            return lines.map(line => {
+                const decision = JSON.parse(line);
+                decision.timestamp = new Date(decision.timestamp);
+                return decision;
+            });
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Get a specific decision by ID.
+     */
+    getDecision(id) {
+        // First check session decisions
+        const sessionDecision = this.decisions.find(d => d.id === id);
+        if (sessionDecision)
+            return sessionDecision;
+        // Then check log file
+        const allDecisions = this.loadAllDecisions();
+        return allDecisions.find(d => d.id === id);
+    }
+    /**
+     * Export decisions to a JSON file.
+     */
+    exportToFile(filePath) {
+        const decisions = this.loadAllDecisions();
+        writeFileSync(filePath, JSON.stringify(decisions, null, 2));
+    }
+    /**
+     * Get statistics about logged decisions.
+     */
+    getStatistics() {
+        const decisions = this.loadAllDecisions();
+        if (decisions.length === 0) {
+            return {
+                totalDecisions: 0,
+                byType: {},
+                averageConfidence: 0,
+                matchRate: 0,
+            };
+        }
+        const byType = {};
+        let totalConfidence = 0;
+        let matchCount = 0;
+        for (const decision of decisions) {
+            byType[decision.type] = (byType[decision.type] || 0) + 1;
+            totalConfidence += decision.rawConfidence;
+            if (decision.matchDecision)
+                matchCount++;
+        }
+        return {
+            totalDecisions: decisions.length,
+            byType,
+            averageConfidence: Math.round(totalConfidence / decisions.length),
+            matchRate: Math.round((matchCount / decisions.length) * 100),
+        };
+    }
+    /**
+     * Clear all logged decisions.
+     */
+    clear() {
+        this.decisions = [];
+        if (existsSync(this.logPath)) {
+            writeFileSync(this.logPath, '');
+        }
+    }
+}
+/**
+ * Feedback manager for recording and analyzing user feedback.
+ */
+export class FeedbackManager {
+    feedbackPath;
+    constructor(options = {}) {
+        ensureTelemetryDir();
+        this.feedbackPath = options.feedbackPath ?? join(getTelemetryDir(), 'feedback.jsonl');
+    }
+    /**
+     * Submit feedback on a comparison decision.
+     */
+    submit(feedback) {
+        const fullFeedback = {
+            ...feedback,
+            timestamp: new Date(),
+        };
+        try {
+            appendFileSync(this.feedbackPath, JSON.stringify(fullFeedback) + '\n');
+        }
+        catch {
+            // Silently fail if we can't write
+        }
+    }
+    /**
+     * Load all feedback reports.
+     */
+    loadAll() {
+        if (!existsSync(this.feedbackPath)) {
+            return [];
+        }
+        try {
+            const content = readFileSync(this.feedbackPath, 'utf-8');
+            const lines = content.trim().split('\n').filter(Boolean);
+            return lines.map(line => {
+                const report = JSON.parse(line);
+                report.timestamp = new Date(report.timestamp);
+                return report;
+            });
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Analyze all feedback to identify patterns.
+     */
+    analyze() {
+        const reports = this.loadAll();
+        if (reports.length === 0) {
+            return {
+                totalReports: 0,
+                falsePositiveRate: 0,
+                falseNegativeRate: 0,
+                confidenceIssueRate: 0,
+                commonPatterns: [],
+            };
+        }
+        const fpCount = reports.filter(r => r.feedbackType === 'false_positive').length;
+        const fnCount = reports.filter(r => r.feedbackType === 'false_negative').length;
+        const confCount = reports.filter(r => r.feedbackType === 'confidence_wrong').length;
+        // Extract common patterns from user comments
+        const patternCounts = new Map();
+        for (const report of reports) {
+            if (report.userComment) {
+                // Simple keyword extraction
+                const keywords = report.userComment.toLowerCase()
+                    .split(/\s+/)
+                    .filter(word => word.length > 4);
+                for (const keyword of keywords) {
+                    const existing = patternCounts.get(keyword);
+                    if (existing) {
+                        existing.count++;
+                    }
+                    else {
+                        patternCounts.set(keyword, { count: 1, feedbackType: report.feedbackType });
+                    }
+                }
+            }
+        }
+        const commonPatterns = Array.from(patternCounts.entries())
+            .filter(([, data]) => data.count >= 2)
+            .sort((a, b) => b[1].count - a[1].count)
+            .slice(0, 10)
+            .map(([pattern, data]) => ({
+            pattern,
+            count: data.count,
+            feedbackType: data.feedbackType,
+        }));
+        return {
+            totalReports: reports.length,
+            falsePositiveRate: Math.round((fpCount / reports.length) * 100),
+            falseNegativeRate: Math.round((fnCount / reports.length) * 100),
+            confidenceIssueRate: Math.round((confCount / reports.length) * 100),
+            commonPatterns,
+        };
+    }
+    /**
+     * Clear all feedback.
+     */
+    clear() {
+        if (existsSync(this.feedbackPath)) {
+            writeFileSync(this.feedbackPath, '');
+        }
+    }
+}
+/**
+ * Get the package version.
+ */
+function getVersion() {
+    try {
+        // Try to read from package.json
+        const packageJsonPath = join(__dirname, '..', '..', 'package.json');
+        if (existsSync(packageJsonPath)) {
+            const pkg = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
+            return pkg.version || '0.0.0';
+        }
+    }
+    catch {
+        // Ignore errors
+    }
+    return '0.0.0';
+}
+// Singleton instances for global access
+let globalDecisionLogger = null;
+let globalFeedbackManager = null;
+/**
+ * Get the global decision logger instance.
+ */
+export function getDecisionLogger(options) {
+    if (!globalDecisionLogger) {
+        globalDecisionLogger = new DecisionLogger(options);
+    }
+    return globalDecisionLogger;
+}
+/**
+ * Get the global feedback manager instance.
+ */
+export function getFeedbackManager() {
+    if (!globalFeedbackManager) {
+        globalFeedbackManager = new FeedbackManager();
+    }
+    return globalFeedbackManager;
+}
+/**
+ * Reset global instances (for testing).
+ */
+export function resetTelemetry() {
+    globalDecisionLogger = null;
+    globalFeedbackManager = null;
+}
+//# sourceMappingURL=telemetry.js.map

package/dist/baseline/test-pruner.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Intelligent Test Pruning.
+ *
+ * Determines which test categories to run or skip based on tool characteristics
+ * and testing history. Reduces test time while maintaining coverage where it matters.
+ */
+import type { MCPTool } from '../transport/types.js';
+import type { ToolFingerprint, ErrorPattern } from './types.js';
+/**
+ * Test category type.
+ */
+export type TestCategory = 'happy_path' | 'boundary' | 'enum' | 'optional_combinations' | 'error_handling' | 'security' | 'semantic';
+/**
+ * Decision about a test category.
+ */
+export interface TestCategoryDecision {
+    /** The test category */
+    category: TestCategory;
+    /** Whether to run this category */
+    shouldRun: boolean;
+    /** Reason for the decision */
+    reason: string;
+    /** Priority if running (higher = more important) */
+    priority: number;
+}
+/**
+ * Complete pruning decision for a tool.
+ */
+export interface ToolPruningDecision {
+    /** Tool name */
+    toolName: string;
+    /** Overall tool priority (0-100) */
+    priority: number;
+    /** Decisions for each category */
+    categories: TestCategoryDecision[];
+    /** Categories that will run */
+    categoriesToRun: TestCategory[];
+    /** Categories that will skip */
+    categoriesToSkip: TestCategory[];
+    /** Estimated test reduction percentage */
+    reductionPercent: number;
+}
+/**
+ * Tool characteristics for pruning decisions.
+ */
+export interface ToolCharacteristics {
+    /** Number of parameters */
+    parameterCount: number;
+    /** Number of required parameters */
+    requiredParamCount: number;
+    /** Whether tool has numeric parameters */
+    hasNumericParams: boolean;
+    /** Whether tool has enum parameters */
+    hasEnumParams: boolean;
+    /** Whether tool has optional parameters */
+    hasOptionalParams: boolean;
+    /** Whether tool has string parameters (for security testing) */
+    hasStringParams: boolean;
+    /** Maximum nesting depth in schema */
+    maxNestingDepth: number;
+    /** Whether tool has external dependencies */
+    hasExternalDependency: boolean;
+    /** Error rate from previous testing (0-1) */
+    errorRate: number;
+    /** Hours since last test (null if never tested) */
+    hoursSinceLastTest: number | null;
+    /** Consecutive successful runs */
+    consecutiveSuccesses: number;
+}
+/**
+ * Input for making pruning decisions.
+ */
+export interface PruningInput {
+    /** Tool definition */
+    tool: MCPTool;
+    /** Previous baseline fingerprint (if available) */
+    fingerprint?: ToolFingerprint;
+    /** Error patterns from baseline */
+    errorPatterns?: ErrorPattern[];
+    /** All test categories that could run */
+    availableCategories: TestCategory[];
+}
+/**
+ * Calculate pruning decisions for a set of tools.
+ */
+export declare function calculatePruningDecisions(inputs: PruningInput[]): ToolPruningDecision[];
+/**
+ * Calculate pruning decision for a single tool.
+ */
+export declare function calculateToolPruning(input: PruningInput): ToolPruningDecision;
+/**
+ * Prioritize tools for testing order.
+ */
+export declare function prioritizeTools(decisions: ToolPruningDecision[]): ToolPruningDecision[];
+/**
+ * Generate summary of pruning decisions.
+ */
+export interface PruningSummary {
+    /** Total tools analyzed */
+    totalTools: number;
+    /** Total categories that would run without pruning */
+    totalCategoriesWithoutPruning: number;
+    /** Total categories that will run with pruning */
+    totalCategoriesWithPruning: number;
+    /** Overall reduction percentage */
+    overallReduction: number;
+    /** Tools with highest priority */
+    highPriorityTools: string[];
+    /** Tools with most categories skipped */
+    mostPrunedTools: string[];
+}
+/**
+ * Generate pruning summary.
+ */
+export declare function generatePruningSummary(decisions: ToolPruningDecision[]): PruningSummary;
+/**
+ * Generate markdown report for pruning decisions.
+ */
+export declare function generatePruningMarkdown(decisions: ToolPruningDecision[], summary: PruningSummary): string;
+//# sourceMappingURL=test-pruner.d.ts.map