@dotsetlabs/bellwether 0.10.0

package/dist/baseline/embeddings.d.ts

@@ -0,0 +1,118 @@
+/**
+ * Semantic Embeddings for Drift Detection
+ *
+ * Provides optional embedding-based similarity for improved paraphrase detection.
+ * Uses Ollama for local embeddings (free, no API cost) or can be extended
+ * for other providers.
+ */
+import type { ConfidenceFactor } from './types.js';
+/**
+ * Interface for embedding providers.
+ */
+export interface EmbeddingProvider {
+    /** Provider name */
+    name: string;
+    /** Generate embedding vector for text */
+    embed(text: string): Promise<number[]>;
+    /** Calculate similarity between two embedding vectors */
+    similarity(a: number[], b: number[]): number;
+    /** Check if the provider is available */
+    isAvailable(): Promise<boolean>;
+}
+/**
+ * Ollama-based local embeddings using nomic-embed-text model.
+ * Free and runs locally without API costs.
+ */
+export declare class OllamaEmbeddings implements EmbeddingProvider {
+    name: string;
+    private baseUrl;
+    private model;
+    private cache;
+    constructor(options?: {
+        baseUrl?: string;
+        model?: string;
+    });
+    isAvailable(): Promise<boolean>;
+    embed(text: string): Promise<number[]>;
+    similarity(a: number[], b: number[]): number;
+    clearCache(): void;
+}
+/**
+ * Calculate cosine similarity between two vectors.
+ * Returns a value between -1 and 1, where 1 is identical.
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/**
+ * Configuration for embedding-enhanced comparison.
+ */
+export interface EmbeddingConfig {
+    /** Whether to use embeddings (default: false) */
+    enabled: boolean;
+    /** Embedding provider to use */
+    provider: EmbeddingProvider;
+    /** Weight for embedding similarity in overall confidence (0-1) */
+    weight: number;
+    /** Minimum similarity to consider a match (0-1) */
+    matchThreshold: number;
+    /** Timeout for embedding requests in ms */
+    timeout: number;
+}
+/**
+ * Default embedding configuration.
+ */
+export declare const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig;
+/**
+ * Result of embedding-based comparison.
+ */
+export interface EmbeddingComparisonResult {
+    /** Whether embeddings were successfully used */
+    used: boolean;
+    /** Cosine similarity score (0-1) */
+    similarity: number;
+    /** Similarity as percentage (0-100) */
+    similarityPercent: number;
+    /** Whether similarity meets match threshold */
+    meetsThreshold: boolean;
+    /** Error message if embeddings failed */
+    error?: string;
+}
+/**
+ * Compare two texts using embeddings.
+ */
+export declare function compareWithEmbeddings(text1: string, text2: string, config?: Partial<EmbeddingConfig>): Promise<EmbeddingComparisonResult>;
+/**
+ * Create a confidence factor from embedding comparison.
+ */
+export declare function createEmbeddingFactor(result: EmbeddingComparisonResult): ConfidenceFactor | null;
+/**
+ * Enhanced semantic comparator that optionally uses embeddings.
+ */
+export declare class EmbeddingEnhancedComparator {
+    private config;
+    private providerChecked;
+    private providerAvailable;
+    constructor(config?: Partial<EmbeddingConfig>);
+    /**
+     * Check provider availability (cached).
+     */
+    checkAvailability(): Promise<boolean>;
+    /**
+     * Get embedding similarity as an additional confidence factor.
+     * Falls back gracefully if embeddings are unavailable.
+     */
+    getEmbeddingFactor(text1: string, text2: string): Promise<ConfidenceFactor | null>;
+    /**
+     * Determine if texts match based on embedding similarity alone.
+     */
+    matchesByEmbedding(text1: string, text2: string): Promise<boolean>;
+}
+/**
+ * Check if Ollama is available and has the embedding model.
+ */
+export declare function checkOllamaEmbeddings(): Promise<{
+    available: boolean;
+    hasModel: boolean;
+    modelName: string;
+    error?: string;
+}>;
+//# sourceMappingURL=embeddings.d.ts.map

package/dist/baseline/embeddings.js

@@ -0,0 +1,251 @@
+/**
+ * Semantic Embeddings for Drift Detection
+ *
+ * Provides optional embedding-based similarity for improved paraphrase detection.
+ * Uses Ollama for local embeddings (free, no API cost) or can be extended
+ * for other providers.
+ */
+import { MATH_FACTORS, TIME_CONSTANTS } from '../constants.js';
+/**
+ * Ollama-based local embeddings using nomic-embed-text model.
+ * Free and runs locally without API costs.
+ */
+export class OllamaEmbeddings {
+    name = 'ollama';
+    baseUrl;
+    model;
+    cache = new Map();
+    constructor(options = {}) {
+        this.baseUrl = options.baseUrl ?? process.env.OLLAMA_BASE_URL ?? 'http://localhost:11434';
+        this.model = options.model ?? 'nomic-embed-text';
+    }
+    async isAvailable() {
+        try {
+            const response = await fetch(`${this.baseUrl}/api/tags`, {
+                method: 'GET',
+                signal: AbortSignal.timeout(2000),
+            });
+            return response.ok;
+        }
+        catch {
+            return false;
+        }
+    }
+    async embed(text) {
+        // Check cache first
+        const cacheKey = `${this.model}:${text}`;
+        const cached = this.cache.get(cacheKey);
+        if (cached)
+            return cached;
+        const response = await fetch(`${this.baseUrl}/api/embeddings`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: this.model,
+                prompt: text,
+            }),
+        });
+        if (!response.ok) {
+            throw new Error(`Ollama embedding failed: ${response.statusText}`);
+        }
+        const result = await response.json();
+        const embedding = result.embedding;
+        // Cache the result
+        this.cache.set(cacheKey, embedding);
+        return embedding;
+    }
+    similarity(a, b) {
+        return cosineSimilarity(a, b);
+    }
+    clearCache() {
+        this.cache.clear();
+    }
+}
+/**
+ * Calculate cosine similarity between two vectors.
+ * Returns a value between -1 and 1, where 1 is identical.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error('Vectors must have the same length');
+    }
+    let dotProduct = 0;
+    let magnitudeA = 0;
+    let magnitudeB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        magnitudeA += a[i] * a[i];
+        magnitudeB += b[i] * b[i];
+    }
+    magnitudeA = Math.sqrt(magnitudeA);
+    magnitudeB = Math.sqrt(magnitudeB);
+    if (magnitudeA === 0 || magnitudeB === 0) {
+        return 0;
+    }
+    return dotProduct / (magnitudeA * magnitudeB);
+}
+/**
+ * Default embedding configuration.
+ */
+export const DEFAULT_EMBEDDING_CONFIG = {
+    enabled: false,
+    provider: new OllamaEmbeddings(),
+    weight: MATH_FACTORS.EMBEDDING_WEIGHT,
+    matchThreshold: MATH_FACTORS.EMBEDDING_MATCH_THRESHOLD,
+    timeout: TIME_CONSTANTS.EMBEDDING_TIMEOUT,
+};
+/**
+ * Compare two texts using embeddings.
+ */
+export async function compareWithEmbeddings(text1, text2, config = {}) {
+    const fullConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
+    if (!fullConfig.enabled) {
+        return {
+            used: false,
+            similarity: 0,
+            similarityPercent: 0,
+            meetsThreshold: false,
+        };
+    }
+    try {
+        // Check if provider is available
+        const available = await Promise.race([
+            fullConfig.provider.isAvailable(),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Availability check timeout')), 2000)),
+        ]);
+        if (!available) {
+            return {
+                used: false,
+                similarity: 0,
+                similarityPercent: 0,
+                meetsThreshold: false,
+                error: 'Embedding provider not available',
+            };
+        }
+        // Generate embeddings with timeout
+        const embedPromise = Promise.all([
+            fullConfig.provider.embed(text1),
+            fullConfig.provider.embed(text2),
+        ]);
+        const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Embedding timeout')), fullConfig.timeout));
+        const [emb1, emb2] = await Promise.race([embedPromise, timeoutPromise]);
+        // Calculate similarity
+        const similarity = fullConfig.provider.similarity(emb1, emb2);
+        const similarityPercent = Math.round(similarity * 100);
+        return {
+            used: true,
+            similarity,
+            similarityPercent,
+            meetsThreshold: similarity >= fullConfig.matchThreshold,
+        };
+    }
+    catch (error) {
+        return {
+            used: false,
+            similarity: 0,
+            similarityPercent: 0,
+            meetsThreshold: false,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}
+/**
+ * Create a confidence factor from embedding comparison.
+ */
+export function createEmbeddingFactor(result) {
+    if (!result.used) {
+        return null;
+    }
+    return {
+        name: 'embedding_similarity',
+        weight: DEFAULT_EMBEDDING_CONFIG.weight,
+        value: result.similarityPercent,
+        description: `${result.similarityPercent}% semantic embedding similarity`,
+    };
+}
+/**
+ * Enhanced semantic comparator that optionally uses embeddings.
+ */
+export class EmbeddingEnhancedComparator {
+    config;
+    providerChecked = false;
+    providerAvailable = false;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
+    }
+    /**
+     * Check provider availability (cached).
+     */
+    async checkAvailability() {
+        if (this.providerChecked) {
+            return this.providerAvailable;
+        }
+        this.providerAvailable = await this.config.provider.isAvailable();
+        this.providerChecked = true;
+        return this.providerAvailable;
+    }
+    /**
+     * Get embedding similarity as an additional confidence factor.
+     * Falls back gracefully if embeddings are unavailable.
+     */
+    async getEmbeddingFactor(text1, text2) {
+        if (!this.config.enabled) {
+            return null;
+        }
+        const result = await compareWithEmbeddings(text1, text2, this.config);
+        return createEmbeddingFactor(result);
+    }
+    /**
+     * Determine if texts match based on embedding similarity alone.
+     */
+    async matchesByEmbedding(text1, text2) {
+        if (!this.config.enabled) {
+            return false;
+        }
+        const result = await compareWithEmbeddings(text1, text2, this.config);
+        return result.meetsThreshold;
+    }
+}
+/**
+ * Check if Ollama is available and has the embedding model.
+ */
+export async function checkOllamaEmbeddings() {
+    const provider = new OllamaEmbeddings();
+    try {
+        const available = await provider.isAvailable();
+        if (!available) {
+            return {
+                available: false,
+                hasModel: false,
+                modelName: 'nomic-embed-text',
+                error: 'Ollama is not running or not accessible',
+            };
+        }
+        // Try to generate a test embedding
+        try {
+            await provider.embed('test');
+            return {
+                available: true,
+                hasModel: true,
+                modelName: 'nomic-embed-text',
+            };
+        }
+        catch (embedError) {
+            return {
+                available: true,
+                hasModel: false,
+                modelName: 'nomic-embed-text',
+                error: `Model not found. Run: ollama pull nomic-embed-text`,
+            };
+        }
+    }
+    catch (error) {
+        return {
+            available: false,
+            hasModel: false,
+            modelName: 'nomic-embed-text',
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}
+//# sourceMappingURL=embeddings.js.map

package/dist/baseline/error-analyzer.d.ts

@@ -0,0 +1,198 @@
+/**
+ * Rich error analysis with remediation suggestions.
+ *
+ * This module provides enhanced error analysis capabilities:
+ * - HTTP status code parsing and categorization
+ * - Root cause inference from error messages
+ * - Remediation suggestion generation
+ * - Related parameter extraction
+ * - Transient error detection
+ * - Error trend analysis across baselines
+ */
+import type { ErrorPattern } from './response-fingerprint.js';
+import type { ErrorTrend, ErrorTrendReport } from './types.js';
+export type { ErrorTrend, ErrorTrendReport };
+/** HTTP status code categories for error classification */
+export type HttpStatusCategory = 'client_error_validation' | 'client_error_auth' | 'client_error_not_found' | 'client_error_conflict' | 'client_error_rate_limit' | 'server_error' | 'validation_expected' | 'unknown';
+/** Severity level for error analysis */
+export type ErrorSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info';
+/**
+ * Context about the test that generated the error.
+ * Used to distinguish expected validation errors from actual bugs.
+ */
+export interface ErrorContext {
+    /** Whether the error was expected (from a validation test) */
+    wasExpected?: boolean;
+    /** The expected outcome of the test */
+    expectedOutcome?: 'success' | 'error' | 'either';
+    /** The test category */
+    testCategory?: string;
+}
+/** Enhanced error analysis result */
+export interface EnhancedErrorAnalysis {
+    /** Original error pattern */
+    pattern: ErrorPattern;
+    /** HTTP status code if detected */
+    httpStatus?: number;
+    /** Status category */
+    statusCategory: HttpStatusCategory;
+    /** Root cause analysis */
+    rootCause: string;
+    /** Remediation suggestion */
+    remediation: string;
+    /** Related parameters (if identifiable) */
+    relatedParameters: string[];
+    /** Whether this error is likely transient */
+    transient: boolean;
+    /** Severity assessment */
+    severity: ErrorSeverity;
+    /**
+     * Whether this error was expected (from a validation test).
+     * Expected errors are intentional test outcomes, not actual bugs.
+     */
+    wasExpected?: boolean;
+}
+/** Error analysis summary for a tool */
+export interface ErrorAnalysisSummary {
+    /** Tool name */
+    tool: string;
+    /** Total errors analyzed */
+    totalErrors: number;
+    /** Enhanced analyses */
+    analyses: EnhancedErrorAnalysis[];
+    /** Most common error category */
+    dominantCategory: HttpStatusCategory;
+    /** Count of transient errors */
+    transientErrors: number;
+    /** Count of actionable errors (with clear remediation) */
+    actionableCount: number;
+    /** Unique remediations suggested */
+    remediations: string[];
+    /** Counts by error category */
+    categoryCounts: Map<string, number>;
+    /** Top root causes (most common) */
+    topRootCauses: string[];
+    /** Top remediations (most actionable) */
+    topRemediations: string[];
+    /** Related parameters across all errors */
+    relatedParameters: string[];
+}
+/**
+ * Analyze an error message for enhanced information.
+ *
+ * @param errorMessage - The error message to analyze
+ * @param context - Optional context about the test that generated the error
+ * @returns Enhanced error analysis with root cause and remediation
+ */
+export declare function analyzeError(errorMessage: string, context?: ErrorContext): EnhancedErrorAnalysis;
+/**
+ * Error pattern with context for analysis.
+ */
+export interface ErrorPatternWithContext {
+    /** The error pattern */
+    pattern: ErrorPattern;
+    /** Context about the test that generated the error */
+    context?: ErrorContext;
+}
+/**
+ * Analyze multiple error patterns and return enhanced analyses.
+ *
+ * @param patterns - Array of error patterns to analyze
+ * @param defaultContext - Default context to apply to all patterns without individual context
+ * @returns Array of enhanced error analyses
+ */
+export declare function analyzeErrorPatterns(patterns: ErrorPattern[] | ErrorPatternWithContext[], defaultContext?: ErrorContext): EnhancedErrorAnalysis[];
+/**
+ * Generate an error analysis summary for a tool.
+ *
+ * @param toolName - Name of the tool
+ * @param patterns - Error patterns for the tool
+ * @returns Error analysis summary
+ */
+export declare function generateErrorSummary(toolName: string, patterns: ErrorPattern[]): ErrorAnalysisSummary;
+/**
+ * Compare error patterns and identify trends.
+ *
+ * @param previous - Error patterns from previous baseline
+ * @param current - Error patterns from current baseline
+ * @returns Error trend report
+ */
+export declare function analyzeErrorTrends(previous: ErrorPattern[], current: ErrorPattern[]): ErrorTrendReport;
+/**
+ * Extract HTTP status code from error message.
+ *
+ * @param message - Error message to parse
+ * @returns HTTP status code if found, undefined otherwise
+ */
+export declare function extractHttpStatus(message: string): number | undefined;
+/**
+ * Categorize HTTP status code into a category.
+ *
+ * @param status - HTTP status code
+ * @returns HTTP status category
+ */
+export declare function categorizeHttpStatus(status: number | undefined): HttpStatusCategory;
+/**
+ * Infer root cause from error message and category.
+ *
+ * @param message - Error message
+ * @param category - HTTP status category
+ * @returns Root cause description
+ */
+export declare function inferRootCause(message: string, category: HttpStatusCategory): string;
+/**
+ * Generate remediation suggestion based on category and message.
+ *
+ * @param category - HTTP status category
+ * @param message - Error message
+ * @returns Remediation suggestion
+ */
+export declare function generateRemediation(category: HttpStatusCategory, message: string): string;
+/**
+ * Extract parameter names mentioned in error message.
+ *
+ * @param message - Error message
+ * @returns Array of parameter names
+ */
+export declare function extractRelatedParameters(message: string): string[];
+/**
+ * Determine if error is likely transient (temporary).
+ *
+ * @param category - HTTP status category
+ * @param message - Error message
+ * @returns Whether the error is likely transient
+ */
+export declare function isTransientError(category: HttpStatusCategory, message: string): boolean;
+/**
+ * Assess error severity based on category and message.
+ *
+ * @param category - HTTP status category
+ * @param message - Error message
+ * @returns Error severity level
+ */
+export declare function assessErrorSeverity(category: HttpStatusCategory, message: string): ErrorSeverity;
+/**
+ * Map HTTP status category to ErrorPattern category.
+ */
+export declare function mapStatusToErrorCategory(category: HttpStatusCategory): 'validation' | 'not_found' | 'permission' | 'timeout' | 'internal' | 'unknown';
+/**
+ * Format enhanced error analysis for display.
+ *
+ * @param analysis - Enhanced error analysis
+ * @param useColors - Whether to use ANSI colors
+ * @returns Formatted string
+ */
+export declare function formatEnhancedError(analysis: EnhancedErrorAnalysis, useColors?: boolean): string;
+/**
+ * Format error trend report for display.
+ *
+ * @param report - Error trend report
+ * @param useColors - Whether to use ANSI colors
+ * @returns Formatted string
+ */
+export declare function formatErrorTrendReport(report: ErrorTrendReport, useColors?: boolean): string;
+/**
+ * Format category name for display.
+ */
+export declare function formatCategoryName(category: HttpStatusCategory): string;
+//# sourceMappingURL=error-analyzer.d.ts.map