@dotsetlabs/bellwether 0.10.0

package/dist/baseline/evaluation/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Drift Detection Evaluation Framework
+ *
+ * Provides tools for measuring the accuracy of semantic comparison
+ * algorithms used in behavioral drift detection.
+ *
+ * Usage:
+ *   import { evaluate, formatEvaluationReport } from './evaluation';
+ *   const result = evaluate();
+ *   console.log(formatEvaluationReport(result));
+ */
+export * from './types.js';
+export * from './golden-dataset.js';
+export * from './evaluator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/baseline/evaluation/index.js

@@ -0,0 +1,15 @@
+/**
+ * Drift Detection Evaluation Framework
+ *
+ * Provides tools for measuring the accuracy of semantic comparison
+ * algorithms used in behavioral drift detection.
+ *
+ * Usage:
+ *   import { evaluate, formatEvaluationReport } from './evaluation';
+ *   const result = evaluate();
+ *   console.log(formatEvaluationReport(result));
+ */
+export * from './types.js';
+export * from './golden-dataset.js';
+export * from './evaluator.js';
+//# sourceMappingURL=index.js.map

package/dist/baseline/evaluation/types.d.ts

@@ -0,0 +1,186 @@
+/**
+ * Types for the drift detection evaluation framework.
+ *
+ * This framework enables systematic measurement of semantic comparison
+ * accuracy, including precision, recall, and confidence calibration.
+ */
+/**
+ * A labeled test case for evaluating semantic comparison accuracy.
+ */
+export interface GoldenTestCase {
+    /** Unique identifier for this test case */
+    id: string;
+    /** Category of comparison being tested */
+    category: 'security' | 'limitation' | 'assertion';
+    /** First text to compare */
+    text1: string;
+    /** Second text to compare */
+    text2: string;
+    /** Tool name for context (affects fingerprinting) */
+    toolName: string;
+    /** Whether these should be considered semantically equivalent */
+    expectedMatch: boolean;
+    /** Expected confidence range (optional) */
+    expectedConfidence?: {
+        min: number;
+        max: number;
+    };
+    /** Human reasoning for why this is the expected outcome */
+    reasoning: string;
+    /** Source of this test case */
+    source: 'manual' | 'llm-generated' | 'production' | 'user-feedback';
+    /** Tags for filtering and analysis */
+    tags?: string[];
+}
+/**
+ * Result of evaluating a single test case.
+ */
+export interface TestCaseResult {
+    /** The test case that was evaluated */
+    testCase: GoldenTestCase;
+    /** Whether the comparison returned match */
+    actualMatch: boolean;
+    /** The confidence score returned */
+    actualConfidence: number;
+    /** Whether this test passed */
+    passed: boolean;
+    /** Type of failure if not passed */
+    failureType?: 'false_positive' | 'false_negative' | 'confidence_out_of_range';
+    /** Time taken for this comparison (ms) */
+    durationMs: number;
+    /** Detailed confidence factors */
+    confidenceFactors?: Array<{
+        name: string;
+        weight: number;
+        value: number;
+        description: string;
+    }>;
+}
+/**
+ * Metrics for a specific category of comparisons.
+ */
+export interface CategoryMetrics {
+    /** Category name */
+    category: string;
+    /** Number of test cases */
+    totalCases: number;
+    /** Accuracy for this category */
+    accuracy: number;
+    /** Precision for this category */
+    precision: number;
+    /** Recall for this category */
+    recall: number;
+    /** F1 score for this category */
+    f1Score: number;
+    /** Confusion matrix counts */
+    truePositives: number;
+    trueNegatives: number;
+    falsePositives: number;
+    falseNegatives: number;
+}
+/**
+ * Calibration bucket for analyzing confidence score accuracy.
+ */
+export interface CalibrationBucket {
+    /** Range of predicted confidence scores */
+    predictedRange: {
+        min: number;
+        max: number;
+    };
+    /** Actual accuracy for predictions in this range */
+    actualAccuracy: number;
+    /** Number of samples in this bucket */
+    sampleCount: number;
+    /** Calibration error (|predicted - actual|) */
+    calibrationError: number;
+}
+/**
+ * Complete evaluation result for a drift detection algorithm.
+ */
+export interface EvaluationResult {
+    /** Timestamp of evaluation */
+    timestamp: Date;
+    /** Algorithm version being evaluated */
+    algorithmVersion: string;
+    /** Dataset version used */
+    datasetVersion: string;
+    /** Total number of test cases */
+    totalCases: number;
+    /** Overall accuracy: (TP + TN) / Total */
+    accuracy: number;
+    /** Precision: TP / (TP + FP) - low false positive rate */
+    precision: number;
+    /** Recall: TP / (TP + FN) - catches real drift */
+    recall: number;
+    /** F1 Score: harmonic mean of precision and recall */
+    f1Score: number;
+    /** Correctly identified as matching */
+    truePositives: number;
+    /** Correctly identified as different */
+    trueNegatives: number;
+    /** Incorrectly flagged as different (noise/false alarm) */
+    falsePositives: number;
+    /** Missed real differences (dangerous) */
+    falseNegatives: number;
+    /** Average |predicted_confidence - actual_accuracy| */
+    calibrationError: number;
+    /** Mean squared error of probabilistic predictions */
+    brierScore: number;
+    /** Calibration buckets for detailed analysis */
+    calibrationBuckets: CalibrationBucket[];
+    /** Metrics per category */
+    categoryMetrics: CategoryMetrics[];
+    /** All individual test results */
+    testResults: TestCaseResult[];
+    /** Failed test cases for analysis */
+    failures: TestCaseResult[];
+    /** Total evaluation time (ms) */
+    totalDurationMs: number;
+    /** Average comparison time (ms) */
+    averageComparisonMs: number;
+}
+/**
+ * Summary for display/reporting.
+ */
+export interface EvaluationSummary {
+    accuracy: string;
+    precision: string;
+    recall: string;
+    f1Score: string;
+    falsePositiveRate: string;
+    falseNegativeRate: string;
+    calibrationError: string;
+    totalCases: number;
+    passedCases: number;
+    failedCases: number;
+}
+/**
+ * Options for running evaluation.
+ */
+export interface EvaluationOptions {
+    /** Filter to specific categories */
+    categories?: Array<'security' | 'limitation' | 'assertion'>;
+    /** Filter to specific tags */
+    tags?: string[];
+    /** Verbose output */
+    verbose?: boolean;
+    /** Include detailed confidence factors in results */
+    includeFactors?: boolean;
+}
+/**
+ * Semantic comparator interface for pluggable algorithms.
+ */
+export interface SemanticComparator {
+    /** Compare two texts and return match result with confidence */
+    compare(text1: string, text2: string, toolName: string, category: 'security' | 'limitation' | 'assertion'): {
+        matches: boolean;
+        confidence: number;
+        factors?: Array<{
+            name: string;
+            weight: number;
+            value: number;
+            description: string;
+        }>;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/baseline/evaluation/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for the drift detection evaluation framework.
+ *
+ * This framework enables systematic measurement of semantic comparison
+ * accuracy, including precision, recall, and confidence calibration.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/baseline/external-dependency-detector.d.ts

@@ -0,0 +1,181 @@
+/**
+ * External Dependency Detection
+ *
+ * Detects and categorizes errors from known external services (Plaid, Stripe, AWS, etc.)
+ * to distinguish between:
+ * - Environment misconfiguration (missing credentials)
+ * - External API failures (service down, rate limited)
+ * - Actual code bugs
+ *
+ * This helps users understand whether test failures are due to their MCP server code
+ * or external factors beyond their control.
+ */
+import { EXTERNAL_DEPENDENCIES } from '../constants.js';
+import type { ErrorPattern } from './response-fingerprint.js';
+import type { ExternalServicesConfig } from '../interview/types.js';
+/** Known external service names */
+export type ExternalServiceName = keyof typeof EXTERNAL_DEPENDENCIES.SERVICES;
+/** Error source classification */
+export type ErrorSource = keyof typeof EXTERNAL_DEPENDENCIES.ERROR_SOURCES;
+/**
+ * Confidence level for dependency detection.
+ * - 'confirmed': Actual error messages from the service were observed
+ * - 'likely': Strong evidence from tool name/description patterns
+ * - 'possible': Weak evidence, only partial matches
+ */
+export type DependencyConfidenceLevel = 'confirmed' | 'likely' | 'possible';
+/** Information about a detected external dependency */
+export interface ExternalDependencyInfo {
+    /** Name of the external service (e.g., 'plaid', 'stripe') */
+    serviceName: ExternalServiceName;
+    /** Display name of the service (e.g., 'Plaid', 'Stripe') */
+    displayName: string;
+    /** Confidence level of the detection (0-1) */
+    confidence: number;
+    /**
+     * Whether this dependency was confirmed by actual errors.
+     * - 'confirmed': Error message matched service-specific patterns
+     * - 'likely': Tool name/description strongly suggests this service
+     * - 'possible': Weak evidence, might be a false positive
+     */
+    confidenceLevel: DependencyConfidenceLevel;
+    /** Whether this appears to be a transient/temporary error */
+    isTransient: boolean;
+    /** Suggested remediation for this error */
+    remediation: string;
+    /** Matched patterns that led to detection */
+    matchedPatterns: string[];
+    /** Evidence breakdown for transparency */
+    evidence: {
+        /** True if error message patterns matched */
+        fromErrorMessage: boolean;
+        /** True if tool name patterns matched */
+        fromToolName: boolean;
+        /** True if tool description patterns matched */
+        fromDescription: boolean;
+        /** Number of actual errors attributed to this dependency */
+        actualErrorCount: number;
+    };
+}
+/** Configuration status for a known external service */
+export interface ServiceStatus {
+    /** External service name */
+    service: ExternalServiceName;
+    /** Whether the service is fully configured */
+    configured: boolean;
+    /** Missing credential keys or env vars */
+    missingCredentials: string[];
+    /** Whether a sandbox is available */
+    sandboxAvailable: boolean;
+    /** Whether a mock response is available */
+    mockAvailable: boolean;
+}
+/** Result of analyzing an error for external dependencies */
+export interface ExternalDependencyAnalysis {
+    /** The error source classification */
+    source: ErrorSource;
+    /** Detected external dependency info (if source is 'external_dependency') */
+    dependency?: ExternalDependencyInfo;
+    /** Whether the error appears transient */
+    isTransient: boolean;
+    /** Human-readable explanation of the classification */
+    explanation: string;
+    /** Remediation suggestion */
+    remediation?: string;
+}
+/** Summary of external dependencies across all tools */
+export interface ExternalDependencySummary {
+    /** Services detected across all tools */
+    services: Map<ExternalServiceName, ExternalServiceSummary>;
+    /** Total number of external dependency errors */
+    totalExternalErrors: number;
+    /** Total number of environment configuration errors */
+    totalEnvironmentErrors: number;
+    /** Total number of likely code bugs */
+    totalCodeBugErrors: number;
+    /** Total number of unclassified errors */
+    totalUnknownErrors: number;
+    /** Tools affected by external dependencies */
+    affectedTools: Map<string, ExternalServiceName[]>;
+}
+/** Summary for a single external service */
+export interface ExternalServiceSummary {
+    /** Display name of the service */
+    displayName: string;
+    /** Number of errors from this service */
+    errorCount: number;
+    /** Number of confirmed errors (from error message patterns) */
+    confirmedErrorCount: number;
+    /** Tools that use this service (confirmed from errors) */
+    confirmedTools: string[];
+    /** Tools that likely use this service (from name/description only) */
+    detectedTools: string[];
+    /** All tools associated with this service */
+    tools: string[];
+    /** Whether errors appear to be transient */
+    hasTransientErrors: boolean;
+    /** Primary remediation suggestion */
+    remediation: string;
+    /** Highest confidence level for this service */
+    highestConfidenceLevel: DependencyConfidenceLevel;
+}
+/**
+ * Detect if an error message indicates an external dependency.
+ *
+ * @param errorMessage - The error message to analyze
+ * @param toolName - Optional tool name for context
+ * @param toolDescription - Optional tool description for context
+ * @returns External dependency info if detected, null otherwise
+ */
+export declare function detectExternalDependency(errorMessage: string, toolName?: string, toolDescription?: string): ExternalDependencyInfo | null;
+/**
+ * Detect external service dependencies based on tool name/description alone.
+ */
+export declare function detectExternalServiceFromTool(toolName: string, toolDescription?: string): ExternalDependencyInfo | null;
+/**
+ * Determine whether an external service is configured.
+ */
+export declare function getExternalServiceStatus(serviceName: ExternalServiceName, config?: ExternalServicesConfig): ServiceStatus;
+/**
+ * Categorize the source of an error.
+ *
+ * @param errorMessage - The error message to analyze
+ * @param toolName - Optional tool name for context
+ * @param toolDescription - Optional tool description for context
+ * @returns Analysis of the error source
+ */
+export declare function categorizeErrorSource(errorMessage: string, toolName?: string, toolDescription?: string): ExternalDependencyAnalysis;
+/**
+ * Check if an error appears to be transient (temporary).
+ *
+ * @param errorMessage - The error message to check
+ * @returns True if the error appears transient
+ */
+export declare function isTransientError(errorMessage: string): boolean;
+/**
+ * Analyze multiple error patterns and generate a summary.
+ *
+ * @param errors - Array of tool names and their error patterns
+ * @returns Summary of external dependencies
+ */
+export declare function analyzeExternalDependencies(errors: Array<{
+    toolName: string;
+    toolDescription?: string;
+    patterns: ErrorPattern[];
+}>): ExternalDependencySummary;
+/**
+ * Format external dependency summary for display.
+ *
+ * @param summary - The summary to format
+ * @param useColors - Whether to use ANSI colors
+ * @returns Formatted string
+ */
+export declare function formatExternalDependencySummary(summary: ExternalDependencySummary, useColors?: boolean): string;
+/**
+ * Generate markdown table for external dependencies.
+ *
+ * @param summary - The summary to format
+ * @returns Markdown string
+ */
+export declare function formatExternalDependenciesMarkdown(summary: ExternalDependencySummary): string;
+//# sourceMappingURL=external-dependency-detector.d.ts.map