observability-toolkit 1.8.4 → 2.0.0

package/dist/lib/llm-as-judge.js

@@ -0,0 +1,1397 @@
+/**
+ * LLM-as-Judge Implementation
+ *
+ * Provides patterns and utilities for using LLMs to evaluate LLM outputs.
+ * Implements G-Eval, QAG, and production-ready evaluation patterns per
+ * industry best practices and OTel GenAI semantic conventions.
+ *
+ * @security
+ * - All user inputs are sanitized for prompt injection protection
+ * - LLM calls have timeout protection (default 30s)
+ * - Input sizes are validated to prevent resource exhaustion
+ * - JSON parsing has depth limits to prevent DoS
+ *
+ * @security Known Limitations
+ * - Script homoglyphs (Cyrillic, Greek characters visually similar to Latin)
+ *   are NOT currently filtered in all cases. The confusables library provides
+ *   partial coverage but may miss some Unicode TR39 edge cases.
+ *   Example: Cyrillic "а" (U+0430) looks identical to Latin "a".
+ *   This is a known gap tracked for future enhancement.
+ *
+ * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
+ */
+import { InputValidationError } from './input-validator.js';
+import { HttpStatus } from './constants.js';
+import { remove as removeConfusables } from 'confusables';
+import sbd from 'sbd';
+// ============================================================================
+// Typed Error Classes
+// ============================================================================
+/**
+ * Error for prompt injection detection.
+ *
+ * NOTE: This class is exported for type-checking and external use but is
+ * intentionally NOT thrown by sanitizeForPrompt(). The design decision is
+ * to silently replace injection patterns with '[filtered]' markers rather
+ * than fail-fast, allowing evaluation to proceed with sanitized input.
+ *
+ * @example
+ * // External code can throw this for stricter handling:
+ * if (detectsInjection(input)) {
+ *   throw new PromptInjectionError('Injection detected in user input');
+ * }
+ */
+export class PromptInjectionError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'PromptInjectionError';
+    }
+}
+/**
+ * Error thrown when an LLM call exceeds the configured timeout.
+ *
+ * Thrown by withTimeout() when the wrapped function does not complete
+ * within the specified duration. Callers should catch this to implement
+ * fallback behavior or retry logic.
+ *
+ * @example
+ * try {
+ *   await withTimeout(() => llmCall(), 5000);
+ * } catch (e) {
+ *   if (e instanceof LLMTimeoutError) {
+ *     // Handle timeout - use cached result or return default
+ *   }
+ * }
+ *
+ * @see withTimeout
+ */
+export class LLMTimeoutError extends Error {
+    constructor(timeoutMs) {
+        super(`LLM call timed out after ${timeoutMs}ms`);
+        this.name = 'LLMTimeoutError';
+    }
+}
+/**
+ * Error thrown when score extraction or normalization fails.
+ *
+ * Thrown by normalizeWithLogprobs() when the LLM response cannot be
+ * parsed into a valid score, or when probability weighting fails.
+ *
+ * @example
+ * try {
+ *   const score = normalizeWithLogprobs(response, [1,2,3,4,5]);
+ * } catch (e) {
+ *   if (e instanceof ScoreNormalizationError) {
+ *     // Fallback to regex-based extraction
+ *   }
+ * }
+ *
+ * @see normalizeWithLogprobs
+ * @see extractScoreFromText
+ */
+export class ScoreNormalizationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ScoreNormalizationError';
+    }
+}
+// ============================================================================
+// Security Constants
+// ============================================================================
+/** Maximum input size in bytes (64KB) */
+export const MAX_INPUT_SIZE_BYTES = 65536;
+/** Maximum text length per field (10KB) */
+export const MAX_TEXT_LENGTH = 10000;
+/** Maximum context array length */
+export const MAX_CONTEXT_ITEMS = 20;
+/** Maximum statements to process in QAG pattern */
+export const MAX_STATEMENTS = 20;
+/** Default timeout for LLM calls (30 seconds) */
+export const DEFAULT_LLM_TIMEOUT_MS = 30000;
+/** Maximum JSON nesting depth */
+export const MAX_JSON_DEPTH = 5;
+/**
+ * Current log level for the module.
+ * Controls verbosity of console output for production flexibility.
+ * - 'debug': All logs including verbose debugging info
+ * - 'info': Informational messages and above
+ * - 'warn': Warnings and errors only (default)
+ * - 'error': Only error messages
+ * - 'silent': No logging output
+ */
+export const LOG_LEVEL = process.env.LLM_JUDGE_LOG_LEVEL || 'warn';
+/**
+ * Check if a log level should be output based on current LOG_LEVEL.
+ * @param level - Level to check
+ * @returns True if the level should be logged
+ */
+function shouldLog(level) {
+    const levels = ['debug', 'info', 'warn', 'error', 'silent'];
+    const currentIndex = levels.indexOf(LOG_LEVEL);
+    const levelIndex = levels.indexOf(level);
+    return levelIndex >= currentIndex && LOG_LEVEL !== 'silent';
+}
+// ============================================================================
+// G-Eval Score Range Constants
+// ============================================================================
+/** G-Eval minimum score (inclusive) */
+export const G_EVAL_MIN_SCORE = 1;
+/** G-Eval maximum score (inclusive) */
+export const G_EVAL_MAX_SCORE = 5;
+/** G-Eval valid score values array */
+export const G_EVAL_VALID_SCORES = [1, 2, 3, 4, 5];
+/** G-Eval score range for normalization (max - min) */
+export const G_EVAL_SCORE_RANGE = G_EVAL_MAX_SCORE - G_EVAL_MIN_SCORE;
+/**
+ * G-Eval default/middle score for fallback scenarios.
+ * @note This constant is exported for library consumers who need a sensible
+ * default when implementing custom fallback logic. The core gEval() function
+ * throws errors rather than using fallback values to ensure explicit handling.
+ * @example
+ * ```typescript
+ * // Custom fallback in consumer code
+ * const score = parseResult(response) ?? G_EVAL_DEFAULT_SCORE;
+ * ```
+ */
+export const G_EVAL_DEFAULT_SCORE = 3;
+// ============================================================================
+// LLM Configuration Constants
+// ============================================================================
+/** Default temperature for deterministic LLM calls (e.g., extraction, answering) */
+export const LLM_TEMPERATURE_DETERMINISTIC = 0;
+/** Default temperature for evaluation LLM calls (slight variation allowed) */
+export const LLM_TEMPERATURE_EVALUATION = 0.1;
+/** Minimum evaluation steps to generate in G-Eval */
+export const G_EVAL_MIN_STEPS = 3;
+/** Maximum evaluation steps to generate in G-Eval */
+export const G_EVAL_MAX_STEPS = 5;
+// ============================================================================
+// QAG Pattern Constants
+// ============================================================================
+/** Minimum statement length to be considered valid for QAG extraction */
+export const MIN_STATEMENT_LENGTH = 10;
+// ============================================================================
+// Retry and Circuit Breaker Constants
+// ============================================================================
+/** Default maximum retry attempts for evaluateWithRetry */
+export const DEFAULT_MAX_RETRIES = 3;
+/** Default circuit breaker failure threshold */
+export const DEFAULT_CIRCUIT_BREAKER_THRESHOLD = 5;
+/** Default circuit breaker reset timeout in milliseconds */
+export const DEFAULT_CIRCUIT_BREAKER_RESET_MS = 30000;
+/** Base delay multiplier for exponential backoff (1 second) */
+export const BACKOFF_BASE_MS = 1000;
+// ============================================================================
+// Score Validation Constants
+// ============================================================================
+/** Minimum valid normalized score */
+export const NORMALIZED_SCORE_MIN = 0;
+/** Maximum valid normalized score */
+export const NORMALIZED_SCORE_MAX = 1;
+/**
+ * OTel attribute mapping for evaluation events
+ */
+export const EVALUATION_OTEL_ATTRIBUTES = {
+    evaluationName: 'gen_ai.evaluation.name',
+    scoreValue: 'gen_ai.evaluation.score.value',
+    scoreLabel: 'gen_ai.evaluation.score.label',
+    explanation: 'gen_ai.evaluation.explanation',
+    errorType: 'error.type',
+    durationMs: 'gen_ai.evaluation.duration',
+    inputTokens: 'gen_ai.usage.input_tokens',
+    outputTokens: 'gen_ai.usage.output_tokens',
+};
+// ============================================================================
+// Security Utilities
+// ============================================================================
+/**
+ * Prompt injection detection patterns (case-insensitive, Unicode-normalized).
+ *
+ * @security These patterns use non-capturing groups (?:...) and avoid nested
+ * quantifiers that could cause catastrophic backtracking on adversarial input.
+ * For example, `\s+(all\s+)?` is rewritten as `\s+(?:all\s+)?` with the outer
+ * `\s+` matching minimally before the optional group.
+ */
+const PROMPT_INJECTION_PATTERNS = [
+    /ignore\s+(?:all\s+)?previous\s+instructions/gi,
+    /system\s+prompt/gi,
+    /you\s+are\s+now/gi,
+    /forget\s+everything/gi,
+    /disregard\s+(?:all\s+)?(?:previous|prior)/gi,
+    /new\s+instructions?:/gi,
+    /override\s+(?:system|instructions)/gi,
+    /act\s+as\s+(?:if\s+)?(?:you\s+are|an?)\s/gi,
+    /pretend\s+(?:you\s+are|to\s+be)/gi,
+    /jailbreak/gi,
+    /\bDAN\b/gi, // "Do Anything Now" prompt - case-insensitive to catch "dan", "Dan" variants
+    /developer\s+mode/gi,
+    /ignore\s+safety/gi,
+    /bypass\s+(?:filter|restriction|rule)/gi,
+];
+/**
+ * Compiles an array of RegExp patterns into a single combined regex.
+ * Optimizes O(n*m) pattern matching to O(n) by using alternation.
+ *
+ * @param patterns - Array of regex patterns to compile (must be non-empty)
+ * @returns Combined regex with all patterns as alternations (global + case-insensitive)
+ * @performance Reduces 14 pattern checks to single regex evaluation
+ * @security All patterns are normalized to 'gi' flags for consistent case-insensitive matching
+ * @throws {Error} If patterns array is empty
+ */
+function compilePatterns(patterns) {
+    if (patterns.length === 0) {
+        throw new Error('Cannot compile empty patterns array');
+    }
+    const sources = patterns.map(p => `(?:${p.source})`);
+    return new RegExp(sources.join('|'), 'gi');
+}
+/**
+ * Pre-compiled injection detection regex for O(n) performance.
+ * Combines all 14 injection patterns into single regex via alternation.
+ *
+ * @performance Single regex.test() instead of 14 individual pattern checks.
+ * For 10KB text with 14 patterns, reduces from O(10KB*14) to O(10KB).
+ */
+const COMPILED_INJECTION_PATTERN = compilePatterns(PROMPT_INJECTION_PATTERNS);
+/**
+ * Normalize text for prompt injection detection.
+ * Handles Unicode homoglyphs and common obfuscation tricks.
+ *
+ * @security CRITICAL ORDERING:
+ * 1. Homoglyph mapping FIRST using confusables library (Unicode TR39 coverage)
+ * 2. NFKC normalization to decompose composed characters
+ * 3. Zero-width removal prevents attacks like "ign\u2060ore" bypassing detection
+ * 4. Quote normalization prevents "ignore" vs 'ignore' bypasses
+ *
+ * @see https://unicode.org/reports/tr39/#Confusable_Detection
+ */
+function normalizeForDetection(text) {
+    // Step 1: Map homoglyphs from other scripts to Latin equivalents
+    // Uses confusables library with full Unicode TR39 coverage
+    const normalized = removeConfusables(text);
+    return normalized
+        .normalize('NFKC') // Step 2: Decompose composed characters
+        .replace(/[\u200B-\u200D\u2060\u180E\uFEFF\u034F\uFE00-\uFE0F]/g, '') // Step 3: Remove zero-width chars
+        .replace(/['']/g, "'") // Step 4: Normalize quotes
+        .replace(/[""]/g, '"')
+        .toLowerCase();
+}
+/** Zero-width characters regex for removal from output */
+const ZERO_WIDTH_CHARS_REGEX = /[\u200B-\u200D\u2060\u180E\uFEFF\u034F\uFE00-\uFE0F]/g;
+/** Smart single quotes normalization regex */
+const SMART_SINGLE_QUOTES_REGEX = /['']/g;
+/** Smart double quotes normalization regex */
+const SMART_DOUBLE_QUOTES_REGEX = /[""]/g;
+/** Double newline delimiter regex for section injection prevention */
+const DOUBLE_NEWLINE_REGEX = /\n\n/g;
+/** Section keyword regex for prompt delimiter escaping */
+const SECTION_KEYWORD_REGEX = /\n(Output|Input|Context|Expected Output|Criteria|Score):/gi;
+/**
+ * Sanitizes an array of context strings for safe prompt inclusion.
+ * Truncates to MAX_CONTEXT_ITEMS and sanitizes each item.
+ *
+ * @param context - Array of context strings to sanitize
+ * @returns Sanitized and truncated context array (max MAX_CONTEXT_ITEMS items)
+ * @security Applies prompt injection protection to each context item
+ */
+export function sanitizeContextArray(context) {
+    return context.slice(0, MAX_CONTEXT_ITEMS).map(c => sanitizeForPrompt(c));
+}
+/**
+ * Sanitize text for safe inclusion in prompts.
+ * Detects and removes potential prompt injection attempts.
+ *
+ * SECURITY DESIGN:
+ * - Uses normalized text (homoglyphs → Latin) for injection DETECTION
+ * - Preserves original text when no injection patterns are found
+ * - Only returns normalized text when malicious patterns are detected
+ * - This preserves legitimate Cyrillic/Greek text while catching attacks
+ *
+ * @param text - Text to sanitize
+ * @param maxLength - Maximum allowed length (default: MAX_TEXT_LENGTH)
+ * @returns Sanitized text with injection patterns replaced by '[filtered]'
+ * @security Removes zero-width characters and neutralizes prompt injection patterns
+ */
+export function sanitizeForPrompt(text, maxLength = MAX_TEXT_LENGTH) {
+    // Truncate to prevent context overflow
+    let sanitized = text.slice(0, maxLength);
+    // Remove zero-width characters that could be used to bypass detection
+    // This must happen before pattern matching to prevent word-breaking attacks
+    sanitized = sanitized.replace(ZERO_WIDTH_CHARS_REGEX, '');
+    // Create normalized text for injection DETECTION only
+    // Uses confusables library for full Unicode TR39 homoglyph coverage
+    let detectionText = removeConfusables(sanitized);
+    // Apply NFKC normalization for full-width characters and other Unicode tricks
+    detectionText = detectionText
+        .normalize('NFKC')
+        .replace(ZERO_WIDTH_CHARS_REGEX, '')
+        .replace(SMART_SINGLE_QUOTES_REGEX, "'")
+        .replace(SMART_DOUBLE_QUOTES_REGEX, '"');
+    // Check for injection patterns in normalized detection text
+    // Uses pre-compiled regex for O(n) performance vs O(n*m) with individual patterns
+    const hasInjection = COMPILED_INJECTION_PATTERN.test(detectionText);
+    // Reset lastIndex after test() to ensure consistent behavior
+    COMPILED_INJECTION_PATTERN.lastIndex = 0;
+    // If injection detected, use normalized text with patterns filtered
+    // Data loss is acceptable when filtering malicious input
+    if (hasInjection) {
+        // Use compiled pattern for single-pass replacement
+        const normalized = detectionText.replace(COMPILED_INJECTION_PATTERN, '[filtered]');
+        // Escape prompt delimiters
+        return normalized
+            .replace(DOUBLE_NEWLINE_REGEX, '\n \n')
+            .replace(SECTION_KEYWORD_REGEX, '\n $1:');
+    }
+    // No injection - preserve original text, only escape delimiters
+    // This preserves legitimate Cyrillic/Greek text
+    return sanitized
+        .replace(DOUBLE_NEWLINE_REGEX, '\n \n')
+        .replace(SECTION_KEYWORD_REGEX, '\n $1:');
+}
+/**
+ * Create a customizable sanitizer with additional injection patterns.
+ *
+ * Use this factory when you need to extend the default prompt injection
+ * patterns with domain-specific patterns for your use case.
+ *
+ * @param additionalPatterns - Additional regex patterns to detect as prompt injection
+ * @returns Sanitizer function with signature (text: string, maxLength?: number) => string
+ *   that applies all default patterns plus additionalPatterns, truncates to maxLength,
+ *   and returns sanitized text with '[filtered]' markers for detected injections.
+ * @throws {InputValidationError} If additionalPatterns contains non-RegExp items
+ *
+ * @example
+ * ```typescript
+ * const customSanitizer = createSanitizer([
+ *   /my\s+custom\s+attack\s+pattern/gi,
+ *   /another\s+pattern/gi,
+ * ]);
+ * const sanitized = customSanitizer(userInput);
+ * const truncated = customSanitizer(longInput, 1000); // Override maxLength per-call
+ * ```
+ *
+ * @security Custom patterns MUST avoid ReDoS vulnerabilities:
+ * - Use non-capturing groups (?:...) when grouping is not needed
+ * - Avoid nested quantifiers like (a+)+ or (a*)*
+ * - Test patterns with tools like safe-regex before deployment
+ * - Example vulnerable: /^(a+)+$/ with input "aaaaaaaaaaaaaaaaX"
+ * - Example safe: /^a+$/ or /^(?:a+)$/
+ */
+export function createSanitizer(additionalPatterns = []) {
+    // Validate patterns at factory time
+    for (let i = 0; i < additionalPatterns.length; i++) {
+        if (!(additionalPatterns[i] instanceof RegExp)) {
+            throw new InputValidationError(`additionalPatterns[${i}] must be a RegExp, got ${typeof additionalPatterns[i]}`, 'additionalPatterns', 'type');
+        }
+    }
+    // Compile all patterns at factory time for O(n) performance
+    const allPatterns = [...PROMPT_INJECTION_PATTERNS, ...additionalPatterns];
+    const compiledPattern = compilePatterns(allPatterns);
+    return (text, maxLength = MAX_TEXT_LENGTH) => {
+        // Truncate to prevent context overflow
+        let sanitized = text.slice(0, maxLength);
+        // Remove zero-width characters
+        sanitized = sanitized.replace(ZERO_WIDTH_CHARS_REGEX, '');
+        // Create normalized text for injection DETECTION only
+        let detectionText = removeConfusables(sanitized);
+        detectionText = detectionText
+            .normalize('NFKC')
+            .replace(ZERO_WIDTH_CHARS_REGEX, '')
+            .replace(SMART_SINGLE_QUOTES_REGEX, "'")
+            .replace(SMART_DOUBLE_QUOTES_REGEX, '"');
+        // Check for injection patterns using compiled regex
+        const hasInjection = compiledPattern.test(detectionText);
+        compiledPattern.lastIndex = 0; // Reset for consistent behavior
+        // If injection detected, use normalized text with patterns filtered
+        if (hasInjection) {
+            const normalized = detectionText.replace(compiledPattern, '[filtered]');
+            return normalized
+                .replace(DOUBLE_NEWLINE_REGEX, '\n \n')
+                .replace(SECTION_KEYWORD_REGEX, '\n $1:');
+        }
+        // No injection - preserve original text, only escape delimiters
+        return sanitized
+            .replace(DOUBLE_NEWLINE_REGEX, '\n \n')
+            .replace(SECTION_KEYWORD_REGEX, '\n $1:');
+    };
+}
+/**
+ * Validate test case input sizes against security limits.
+ * Checks individual field lengths and total byte size.
+ *
+ * @param testCase - Test case to validate
+ * @returns void
+ * @throws {InputValidationError} If any field exceeds MAX_TEXT_LENGTH
+ * @throws {InputValidationError} If context exceeds MAX_CONTEXT_ITEMS
+ * @throws {InputValidationError} If total size exceeds MAX_INPUT_SIZE_BYTES
+ * @security Prevents resource exhaustion by enforcing size limits
+ */
+export function validateTestCase(testCase) {
+    if (testCase.input.length > MAX_TEXT_LENGTH) {
+        throw new InputValidationError(`Input exceeds ${MAX_TEXT_LENGTH} character limit`, 'input', 'maxLength');
+    }
+    if (testCase.output.length > MAX_TEXT_LENGTH) {
+        throw new InputValidationError(`Output exceeds ${MAX_TEXT_LENGTH} character limit`, 'output', 'maxLength');
+    }
+    if (testCase.context && testCase.context.length > MAX_CONTEXT_ITEMS) {
+        throw new InputValidationError(`Context exceeds ${MAX_CONTEXT_ITEMS} items limit`, 'context', 'maxLength');
+    }
+    // Validate individual context item types and sizes
+    if (testCase.context) {
+        for (let i = 0; i < testCase.context.length; i++) {
+            const item = testCase.context[i];
+            // Validate type - context items must be strings
+            if (typeof item !== 'string') {
+                throw new InputValidationError(`Context item ${i} must be a string, got ${typeof item}`, 'context', 'type');
+            }
+            if (item.length > MAX_TEXT_LENGTH) {
+                throw new InputValidationError(`Context item ${i} exceeds ${MAX_TEXT_LENGTH} character limit`, 'context', 'maxLength');
+            }
+        }
+    }
+    if (testCase.expectedOutput && testCase.expectedOutput.length > MAX_TEXT_LENGTH) {
+        throw new InputValidationError(`Expected output exceeds ${MAX_TEXT_LENGTH} character limit`, 'expectedOutput', 'maxLength');
+    }
+    // Validate total size to prevent memory exhaustion
+    // Individual fields may pass but combined could exceed MAX_INPUT_SIZE_BYTES
+    let totalBytes = testCase.input.length + testCase.output.length;
+    if (testCase.context) {
+        totalBytes += testCase.context.reduce((sum, c) => sum + c.length, 0);
+    }
+    if (testCase.expectedOutput) {
+        totalBytes += testCase.expectedOutput.length;
+    }
+    if (totalBytes > MAX_INPUT_SIZE_BYTES) {
+        throw new InputValidationError(`Total test case size ${totalBytes} exceeds ${MAX_INPUT_SIZE_BYTES} bytes`, 'testCase', 'maxSize');
+    }
+}
+/**
+ * Safe JSON parsing with depth limit to prevent DoS attacks.
+ *
+ * @param text - JSON text to parse
+ * @param maxDepth - Maximum nesting depth (default: MAX_JSON_DEPTH)
+ * @returns Parsed value
+ * @throws {Error} If JSON is invalid, too large, or too deeply nested
+ */
+export function safeJSONParse(text, maxDepth = MAX_JSON_DEPTH) {
+    // Limit size
+    if (text.length > MAX_INPUT_SIZE_BYTES) {
+        throw new Error('JSON response too large');
+    }
+    const parsed = JSON.parse(text);
+    // Check depth recursively - iterates directly without array allocation for performance
+    const checkDepth = (obj, depth = 0) => {
+        if (depth > maxDepth) {
+            throw new Error('JSON nesting too deep');
+        }
+        if (typeof obj === 'object' && obj !== null) {
+            if (Array.isArray(obj)) {
+                for (const value of obj) {
+                    checkDepth(value, depth + 1);
+                }
+            }
+            else {
+                for (const key in obj) {
+                    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                        checkDepth(obj[key], depth + 1);
+                    }
+                }
+            }
+        }
+    };
+    checkDepth(parsed);
+    return parsed;
+}
+/**
+ * Execute an async function with timeout protection.
+ *
+ * Uses AbortController for atomic cancellation to prevent race conditions
+ * between the function completing and the timeout firing. The abort signal
+ * provides atomic state that both code paths can check safely.
+ *
+ * @param fn - Async function to execute, receives AbortSignal for cancellation
+ * @param timeoutMs - Timeout in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
+ * @returns Result of the function
+ * @throws {LLMTimeoutError} If function times out
+ */
+export async function withTimeout(fn, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    let timeoutId;
+    const abortController = new AbortController();
+    const clearTimer = () => {
+        if (timeoutId !== undefined) {
+            clearTimeout(timeoutId);
+            timeoutId = undefined;
+        }
+    };
+    const timeoutPromise = new Promise((_, reject) => {
+        timeoutId = setTimeout(() => {
+            // Check and set abort atomically - abort() MUST be first action after check
+            // to prevent race between timeout and fn() completion
+            if (!abortController.signal.aborted) {
+                abortController.abort(); // Signal timeout to function
+                clearTimer();
+                reject(new LLMTimeoutError(timeoutMs));
+            }
+        }, timeoutMs);
+    });
+    try {
+        const result = await Promise.race([fn(abortController.signal), timeoutPromise]);
+        clearTimer(); // Success: just clear timer, don't abort (signal stays not-aborted)
+        return result;
+    }
+    catch (error) {
+        clearTimer();
+        // Only abort if not already aborted (e.g., from timeout)
+        if (!abortController.signal.aborted) {
+            abortController.abort();
+        }
+        throw error;
+    }
+}
+// ============================================================================
+// G-Eval Pattern Helpers
+// ============================================================================
+/**
+ * Build evaluation prompt from config and test case.
+ * Used by G-Eval pattern for structured evaluation prompts.
+ *
+ * @param config - G-Eval configuration with criteria and parameters
+ * @param testCase - Test case containing input, output, and optional context
+ * @param steps - Evaluation steps generated by chain-of-thought
+ * @returns Formatted prompt string for the judge model
+ * @security All user-provided content is sanitized for prompt injection
+ */
+export function buildEvalPrompt(config, testCase, steps) {
+    const parts = [
+        `You are evaluating: ${config.name}`,
+        `\nCriteria: ${config.criteria}`,
+        `\nEvaluation Steps:\n${steps}`,
+    ];
+    if (config.evaluationParams.includes('input')) {
+        parts.push(`\nInput: ${sanitizeForPrompt(testCase.input)}`);
+    }
+    if (config.evaluationParams.includes('output')) {
+        parts.push(`\nOutput: ${sanitizeForPrompt(testCase.output)}`);
+    }
+    if (config.evaluationParams.includes('context') && testCase.context) {
+        const sanitizedContext = sanitizeContextArray(testCase.context);
+        parts.push(`\nContext: ${sanitizedContext.join('\n')}`);
+    }
+    if (config.evaluationParams.includes('expectedOutput') && testCase.expectedOutput) {
+        parts.push(`\nExpected Output: ${sanitizeForPrompt(testCase.expectedOutput)}`);
+    }
+    parts.push(`\nProvide a score from ${G_EVAL_MIN_SCORE}-${G_EVAL_MAX_SCORE} and explain your reasoning.`);
+    return parts.join('');
+}
+/**
+ * Normalize score using token log probabilities.
+ * Calculates weighted average score based on probability distribution.
+ *
+ * @param logprobs - Token log probabilities from LLM response
+ * @param validScores - Valid score values (e.g., [1, 2, 3, 4, 5])
+ * @returns Normalized score as weighted average
+ * @throws {ScoreNormalizationError} When no valid score tokens found in logprobs
+ */
+export function normalizeWithLogprobs(logprobs, validScores) {
+    // Validate validScores contains only finite numbers
+    if (!validScores.every(s => typeof s === 'number' && Number.isFinite(s))) {
+        throw new ScoreNormalizationError('validScores must contain only finite numbers');
+    }
+    const scoreProbs = new Map();
+    for (const score of validScores) {
+        scoreProbs.set(score, 0);
+    }
+    for (const { token, logprob } of logprobs) {
+        const scoreValue = parseInt(token.trim(), 10);
+        if (validScores.includes(scoreValue)) {
+            const prob = Math.exp(logprob);
+            scoreProbs.set(scoreValue, (scoreProbs.get(scoreValue) || 0) + prob);
+        }
+    }
+    let weightedSum = 0;
+    let totalProb = 0;
+    for (const [score, prob] of scoreProbs) {
+        weightedSum += score * prob;
+        totalProb += prob;
+    }
+    if (totalProb === 0) {
+        throw new ScoreNormalizationError('No valid score tokens found in logprobs - cannot normalize');
+    }
+    return weightedSum / totalProb;
+}
+// ============================================================================
+// Score Extraction Pattern Constants
+// ============================================================================
+/**
+ * Matches explicit score declarations like "Score: 4" or "score 3".
+ * Case-insensitive. Captures the digit (1-5) in group 1.
+ * @example "Score: 4" → captures "4"
+ * @example "score 3" → captures "3"
+ */
+const EXPLICIT_SCORE_PATTERN = /\bscore[:\s]+([1-5])\b/i;
+/**
+ * Matches rating format like "Rating: 4" or "rating: 2".
+ * Case-insensitive. Captures the digit (1-5) in group 1.
+ * @example "Rating: 4" → captures "4"
+ */
+const RATING_PATTERN = /\brating[:\s]+([1-5])\b/i;
+/**
+ * Matches fractional format like "4 out of 5" or "4/5".
+ * Case-insensitive. Captures the numerator digit (1-5) in group 1.
+ * @example "4 out of 5" → captures "4"
+ * @example "3/5" → captures "3"
+ */
+const FRACTION_PATTERN = /\b([1-5])\s*(?:out of|\/)\s*5\b/i;
+/**
+ * Matches standalone digit on its own line.
+ * Useful for responses that just return the score number.
+ * Captures the digit (1-5) in group 1.
+ * @example "  3  " (with newlines) → captures "3"
+ */
+const STANDALONE_DIGIT_PATTERN = /^\s*([1-5])\s*$/m;
+/**
+ * Score extraction patterns in order of specificity.
+ * More specific patterns are tried first to avoid false positives.
+ *
+ * These patterns match G-Eval scores in the range [G_EVAL_MIN_SCORE, G_EVAL_MAX_SCORE] (1-5).
+ */
+const SCORE_PATTERNS = [
+    EXPLICIT_SCORE_PATTERN,
+    RATING_PATTERN,
+    FRACTION_PATTERN,
+    STANDALONE_DIGIT_PATTERN,
+];
+/** Maximum characters to search for fallback score extraction */
+const SCORE_FALLBACK_WINDOW = 100;
+/**
+ * Extract score from LLM response text.
+ *
+ * Uses specific patterns to avoid false positives from incidental digits.
+ * Falls back to the last digit in the valid range within the last 100 characters,
+ * since LLMs typically provide their final answer at the end.
+ *
+ * @param text - LLM response text
+ * @returns Extracted score in G-Eval range
+ * @throws {ScoreNormalizationError} If no valid score found
+ */
+export function extractScoreFromText(text) {
+    // Try specific patterns first (more reliable) - search entire text
+    for (const pattern of SCORE_PATTERNS) {
+        const match = text.match(pattern);
+        if (match) {
+            return parseInt(match[1], 10);
+        }
+    }
+    // Fallback: last digit in G-Eval range within last 100 chars only
+    // This reduces false positives from incidental numbers in prose
+    // e.g., "This response has 3 main points" won't extract 3 if score is at end
+    const tailText = text.slice(-SCORE_FALLBACK_WINDOW);
+    const allDigits = tailText.match(/\b([1-5])\b/g);
+    if (allDigits && allDigits.length > 0) {
+        return parseInt(allDigits[allDigits.length - 1], 10);
+    }
+    // No valid score found - throw rather than mask failure with default
+    throw new ScoreNormalizationError('No valid score found in LLM response');
+}
+/**
+ * G-Eval implementation using chain-of-thought prompting with token probability normalization.
+ *
+ * @param llm - LLM provider for judge calls
+ * @param config - G-Eval configuration with name, criteria, and evaluation parameters
+ * @param testCase - Test case to evaluate
+ * @param timeoutMs - Timeout for LLM calls in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
+ * @returns Evaluation result with normalized score (0-1), reason, and raw response
+ * @throws {InputValidationError} If test case exceeds size limits
+ * @throws {Error} If LLM call times out or returns invalid score
+ * @security Input validation, prompt injection protection, and timeout protection
+ */
+export async function gEval(llm, config, testCase, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    // Validate input sizes
+    validateTestCase(testCase);
+    // Step 1: Generate evaluation steps via CoT (with timeout)
+    const stepsPrompt = `
+Given the criteria: ${config.criteria}
+Generate detailed evaluation steps to assess this criterion.
+List ${G_EVAL_MIN_STEPS}-${G_EVAL_MAX_STEPS} specific steps the evaluator should follow.
+`;
+    const stepsResponse = await withTimeout((_signal) => llm.generate(stepsPrompt, { temperature: config.temperature ?? LLM_TEMPERATURE_EVALUATION }), timeoutMs);
+    // Step 2: Evaluate with generated steps (with timeout)
+    const evalPrompt = buildEvalPrompt(config, testCase, stepsResponse.text);
+    const response = await withTimeout((_signal) => llm.generate(evalPrompt, { temperature: config.temperature ?? LLM_TEMPERATURE_EVALUATION, logprobs: true }), timeoutMs);
+    // Step 3: Normalize score using token probabilities if available
+    let rawScore;
+    if (response.logprobs && response.logprobs.length > 0) {
+        try {
+            rawScore = normalizeWithLogprobs(response.logprobs, [...G_EVAL_VALID_SCORES]);
+        }
+        catch {
+            // Logprobs normalization failed - fallback to text extraction
+            // This is expected when LLM returns unexpected token distribution
+            rawScore = extractScoreFromText(response.text);
+        }
+    }
+    else {
+        // Fallback: extract score from response text using specific patterns
+        rawScore = extractScoreFromText(response.text);
+    }
+    // Convert G-Eval scale (1-5) to normalized score (0-1)
+    // Formula: (score - min) / (max - min) = (score - 1) / 4
+    const normalizedScore = (rawScore - G_EVAL_MIN_SCORE) / G_EVAL_SCORE_RANGE;
+    // Validate score is in valid range
+    if (!isValidScore(normalizedScore)) {
+        throw new Error(`Invalid normalized score: ${normalizedScore} (raw: ${rawScore})`);
+    }
+    return {
+        score: normalizedScore,
+        reason: response.text,
+        rawResponse: response.text,
+    };
+}
+// ============================================================================
+// QAG Pattern Helpers
+// ============================================================================
+/**
+ * Extract atomic statements from LLM output.
+ * Each statement should be independently verifiable.
+ *
+ * @param llm - LLM provider for extraction
+ * @param output - Text to extract statements from
+ * @param timeoutMs - Timeout for LLM call in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
+ * @returns Array of atomic statements (max MAX_STATEMENTS)
+ * @throws {Error} If LLM call times out
+ * @security Output sanitized, safe JSON parsing with depth limits, result capped at MAX_STATEMENTS
+ */
+export async function extractStatements(llm, output, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const sanitizedOutput = sanitizeForPrompt(output);
+    const prompt = `
+Extract all factual claims from the following text as a JSON array of strings.
+Each claim should be a single, atomic statement that can be verified independently.
+
+Text: ${sanitizedOutput}
+
+Return ONLY a JSON array, e.g.: ["claim 1", "claim 2", "claim 3"]
+`;
+    const response = await withTimeout((_signal) => llm.generate(prompt, { temperature: LLM_TEMPERATURE_DETERMINISTIC }), timeoutMs);
+    try {
+        const parsed = safeJSONParse(response.text);
+        // Validate parsed result is an array before processing
+        if (!Array.isArray(parsed)) {
+            throw new Error(`Expected array from statement extraction, got ${typeof parsed}`);
+        }
+        // Type guard filters non-strings and empty strings, caps at MAX_STATEMENTS
+        return parsed
+            .filter((item) => typeof item === 'string' && item.trim().length > 0)
+            .slice(0, MAX_STATEMENTS);
+    }
+    catch (error) {
+        // JSON parse failed - fallback to sentence splitting
+        // Log with context for production debugging
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        const responsePreview = response.text.length > 200
+            ? response.text.slice(0, 200) + '...'
+            : response.text;
+        // Structured logging with context for distributed tracing
+        // Callers can pass traceId/spanId via AsyncLocalStorage or context propagation
+        // Respects LOG_LEVEL environment variable for production flexibility
+        if (shouldLog('warn')) {
+            console.warn('[llm-as-judge] Statement extraction JSON parse failed, using sentence fallback.', {
+                error: errorMessage,
+                responsePreview,
+                outputLength: output.length,
+                // Note: traceId/spanId can be added by wrapping this module with OTel instrumentation
+                // e.g., using @opentelemetry/api context.active().getValue(SPAN_KEY)
+            });
+        }
+    }
+    // Use sbd (sentence boundary detection) for proper sentence splitting
+    // Handles abbreviations like "Dr. Smith" without splitting incorrectly
+    return sbd.sentences(output)
+        .map(s => s.trim())
+        .filter(s => s.length > MIN_STATEMENT_LENGTH)
+        .slice(0, MAX_STATEMENTS);
+}
+/**
+ * Generate verification question for a statement.
+ *
+ * @param llm - LLM provider for question generation
+ * @param statement - Statement to verify
+ * @param timeoutMs - Timeout for LLM call in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
+ * @returns Yes/no verification question string
+ * @throws {Error} If LLM call times out
+ * @security Statement is sanitized for prompt injection before use
+ */
+export async function generateVerificationQuestion(llm, statement, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const sanitizedStatement = sanitizeForPrompt(statement);
+    const prompt = `
+Convert this statement into a yes/no question that can verify its accuracy:
+
+Statement: ${sanitizedStatement}
+
+Return ONLY the question, nothing else.
+`;
+    const response = await withTimeout((_signal) => llm.generate(prompt, { temperature: LLM_TEMPERATURE_DETERMINISTIC }), timeoutMs);
+    return response.text.trim();
+}
+/**
+ * Answer verification question using provided context.
+ *
+ * @param llm - LLM provider for answering
+ * @param question - Yes/no question to answer
+ * @param context - Context documents to use for answering
+ * @param timeoutMs - Timeout for LLM call in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
+ * @returns 'yes', 'no', or 'unknown' based on context
+ * @throws {Error} If LLM call times out
+ * @security Question and context are sanitized for prompt injection
+ */
+export async function answerQuestion(llm, question, context, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const sanitizedQuestion = sanitizeForPrompt(question);
+    const sanitizedContext = sanitizeContextArray(context);
+    const prompt = `
+Based ONLY on the following context, answer the question with "yes", "no", or "unknown".
+
+Context:
+${sanitizedContext.join('\n\n')}
+
+Question: ${sanitizedQuestion}
+
+Answer (yes/no/unknown):
+`;
+    const response = await withTimeout((_signal) => llm.generate(prompt, { temperature: LLM_TEMPERATURE_DETERMINISTIC }), timeoutMs);
+    const normalized = response.text.trim().toLowerCase();
+    // Use word boundary matching to avoid false positives like "yesterday" or "notwithstanding"
+    const yesMatch = /\b(yes|yeah|correct|true|affirmative)\b/i.test(normalized);
+    const noMatch = /\b(no|nope|incorrect|false|negative)\b/i.test(normalized);
+    // If both or neither, check what comes first for ambiguous cases
+    if (yesMatch && noMatch) {
+        const yesPos = normalized.search(/\b(yes|yeah|correct|true|affirmative)\b/i);
+        const noPos = normalized.search(/\b(no|nope|incorrect|false|negative)\b/i);
+        return yesPos < noPos ? 'yes' : 'no';
+    }
+    if (yesMatch)
+        return 'yes';
+    if (noMatch)
+        return 'no';
+    return 'unknown';
+}
+/**
+ * QAG (Question-Answer Generation) evaluation.
+ * Decomposes evaluation into atomic yes/no questions.
+ *
+ * Uses Promise.allSettled for graceful degradation - partial failures
+ * don't abort the entire evaluation. Score is calculated from successful
+ * verifications only.
+ *
+ * @param llm - LLM provider for all operations
+ * @param input - Original user input (unused but included for API consistency)
+ * @param output - LLM output to evaluate for faithfulness
+ * @param context - Context documents for verification
+ * @param options - Optional configuration object
+ * @param options.timeoutMs - Timeout for each LLM call (default: DEFAULT_LLM_TIMEOUT_MS)
+ * @returns Faithfulness score (0-1) as proportion of verified statements
+ * @security Timeout protection on all calls; graceful degradation on partial failures
+ * @performance Makes 2N+1 LLM calls for N statements (capped at MAX_STATEMENTS=20):
+ *   - 1 call to extract atomic statements from output
+ *   - N parallel calls to generate verification questions (one per statement)
+ *   - N parallel calls to answer questions from context (one per statement)
+ *   For MAX_STATEMENTS (20), this is 41 LLM calls total.
+ *   Typical latency: 10-30s (parallel execution) depending on LLM provider.
+ */
+export async function qagEvaluate(llm, input, output, context, options) {
+    const timeoutMs = options?.timeoutMs ?? DEFAULT_LLM_TIMEOUT_MS;
+    // Step 1: Extract statements from output
+    const statements = await extractStatements(llm, output, timeoutMs);
+    if (statements.length === 0) {
+        return 1; // No claims to verify = fully faithful
+    }
+    // Step 2: Generate verification questions with graceful degradation
+    const questionResults = await Promise.allSettled(statements.map(s => generateVerificationQuestion(llm, s, timeoutMs)));
+    // Collect successful questions with their indices
+    const successfulQuestions = [];
+    for (let i = 0; i < questionResults.length; i++) {
+        const result = questionResults[i];
+        if (result.status === 'fulfilled') {
+            successfulQuestions.push({ question: result.value, index: i });
+        }
+    }
+    // If all question generation failed, throw error (0 would mean "unfaithful" not "failed")
+    if (successfulQuestions.length === 0) {
+        throw new Error('QAG evaluation failed: no verification questions generated');
+    }
+    // Step 3: Answer questions with graceful degradation
+    const answerResults = await Promise.allSettled(successfulQuestions.map(({ question }) => answerQuestion(llm, question, context, timeoutMs)));
+    // Step 4: Calculate score from successful answers only
+    let yesCount = 0;
+    let successfulAnswers = 0;
+    for (const result of answerResults) {
+        if (result.status === 'fulfilled') {
+            successfulAnswers++;
+            if (result.value === 'yes') {
+                yesCount++;
+            }
+        }
+    }
+    // If all answer calls failed, throw error (0 would mean "unfaithful" not "failed")
+    if (successfulAnswers === 0) {
+        throw new Error('QAG evaluation failed: no verification answers obtained');
+    }
+    return yesCount / successfulAnswers;
+}
+// ============================================================================
+// Bias Mitigation
+// ============================================================================
+/** Valid winner values for pairwise evaluation */
+const VALID_PAIRWISE_WINNERS = ['A', 'B', 'tie'];
+/**
+ * Validate that an evaluate function returned a valid pairwise result.
+ * Runtime type guard to prevent type safety bypass from external functions.
+ *
+ * @param result - Result from evaluate function to validate
+ * @param ordering - Ordering label for error message ('AB' or 'BA')
+ * @throws {InputValidationError} If result is not a valid pairwise result object
+ */
+function validatePairwiseResult(result, ordering) {
+    if (!result ||
+        typeof result !== 'object' ||
+        typeof result.winner !== 'string' ||
+        !VALID_PAIRWISE_WINNERS.includes(result.winner)) {
+        throw new InputValidationError(`Invalid evaluate result for ${ordering} ordering: expected { winner: 'A' | 'B' | 'tie' }, got ${JSON.stringify(result)}`, 'evaluate', 'type');
+    }
+}
+/**
+ * Mitigated pairwise evaluation with position bias correction.
+ * Evaluates both orderings and only counts consistent wins.
+ *
+ * @param evaluate - Evaluation function that compares two outputs
+ * @param input - User input
+ * @param outputA - First output to compare
+ * @param outputB - Second output to compare
+ * @returns Winner ('A', 'B', or 'tie')
+ * @throws {Error} If evaluate function is not provided
+ * @throws {InputValidationError} If input, outputA, or outputB is empty
+ * @throws {InputValidationError} If any string exceeds MAX_TEXT_LENGTH
+ */
+export async function mitigatedPairwiseEval(evaluate, input, outputA, outputB) {
+    // Validate evaluate function
+    if (typeof evaluate !== 'function') {
+        throw new Error('mitigatedPairwiseEval requires an evaluate function');
+    }
+    // Validate input is non-empty
+    if (!input || input.trim().length === 0) {
+        throw new InputValidationError('Input cannot be empty', 'input', 'required');
+    }
+    // Validate outputA is non-empty
+    if (!outputA || outputA.trim().length === 0) {
+        throw new InputValidationError('Output A cannot be empty', 'outputA', 'required');
+    }
+    // Validate outputB is non-empty
+    if (!outputB || outputB.trim().length === 0) {
+        throw new InputValidationError('Output B cannot be empty', 'outputB', 'required');
+    }
+    // Validate input sizes to prevent resource exhaustion
+    if (input.length > MAX_TEXT_LENGTH) {
+        throw new InputValidationError(`Input exceeds ${MAX_TEXT_LENGTH} character limit`, 'input', 'maxLength');
+    }
+    if (outputA.length > MAX_TEXT_LENGTH) {
+        throw new InputValidationError(`Output A exceeds ${MAX_TEXT_LENGTH} character limit`, 'outputA', 'maxLength');
+    }
+    if (outputB.length > MAX_TEXT_LENGTH) {
+        throw new InputValidationError(`Output B exceeds ${MAX_TEXT_LENGTH} character limit`, 'outputB', 'maxLength');
+    }
+    // Evaluate both orderings to detect position bias
+    const [resultAB, resultBA] = await Promise.all([
+        evaluate(input, outputA, outputB),
+        evaluate(input, outputB, outputA),
+    ]);
+    // Validate evaluate function returned valid results at runtime
+    validatePairwiseResult(resultAB, 'AB');
+    validatePairwiseResult(resultBA, 'BA');
+    // Map BA result back to AB perspective:
+    // - 'A' winner in BA (reversed order) means B won in original ordering
+    // - 'B' winner in BA (reversed order) means A won in original ordering
+    // - 'tie' remains 'tie'
+    const baMapped = resultBA.winner === 'A' ? 'B' : resultBA.winner === 'B' ? 'A' : 'tie';
+    // Only count consistent wins
+    if (resultAB.winner === 'A' && baMapped === 'A') {
+        return 'A';
+    }
+    else if (resultAB.winner === 'B' && baMapped === 'B') {
+        return 'B';
+    }
+    else {
+        return 'tie';
+    }
+}
+/**
+ * Multi-judge panel evaluation.
+ * Uses multiple judge models and returns median score.
+ *
+ * @param evaluators - Array of evaluation functions for different models
+ * @param testCase - Test case to evaluate
+ * @returns Median score from all judges
+ * @throws {Error} If evaluators array is empty
+ */
+export async function panelEvaluation(evaluators, testCase) {
+    if (evaluators.length === 0) {
+        throw new Error('panelEvaluation requires at least one evaluator');
+    }
+    const scores = await Promise.all(evaluators.map(evaluate => evaluate(testCase)));
+    // Return median
+    const sorted = [...scores].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0) {
+        return (sorted[mid - 1] + sorted[mid]) / 2;
+    }
+    return sorted[mid];
+}
+// ============================================================================
+// Production Utilities
+// ============================================================================
+/**
+ * Validate that a score is within expected range [0, 1].
+ *
+ * @param score - Score value to validate
+ * @returns True if score is a number between 0 and 1 (inclusive), false otherwise
+ */
+export function isValidScore(score) {
+    return typeof score === 'number' && !isNaN(score) && score >= NORMALIZED_SCORE_MIN && score <= NORMALIZED_SCORE_MAX;
+}
+/**
+ * Maximum exponent for backoff calculation.
+ * Derived from: Math.floor(Math.log2(MAX_BACKOFF_MS / BACKOFF_BASE_MS))
+ * = Math.floor(Math.log2(60000 / 1000)) = Math.floor(5.9) = 5
+ * This caps backoff at 2^5 * 1000ms = 32 seconds before MAX_BACKOFF_MS takes over.
+ */
+const MAX_BACKOFF_EXPONENT = 5;
+/** Maximum backoff delay in milliseconds (60 seconds) */
+const MAX_BACKOFF_MS = 60000;
+/**
+ * Delay utility for retry backoff.
+ */
+function delay(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/**
+ * Evaluate with retry logic and exponential backoff.
+ *
+ * Retries a single evaluation function on transient failures. Use this for
+ * simple retry scenarios where you want automatic backoff.
+ *
+ * Compare to {@link JudgeCircuitBreaker.evaluate}:
+ * - `evaluateWithRetry`: Retries same operation with backoff (1s → 2s → 4s)
+ * - `JudgeCircuitBreaker.evaluate`: Fails fast when service is degraded, optional fallback
+ *
+ * These can be combined: wrap evaluateWithRetry inside circuit breaker for
+ * retries on transient errors while circuit-breaking on sustained failures.
+ *
+ * @param evaluate - Evaluation function to retry on failure
+ * @param testCase - Test case to evaluate
+ * @param maxRetries - Maximum number of retry attempts (default: DEFAULT_MAX_RETRIES)
+ * @returns Evaluation result with retryCount indicating number of failed attempts
+ * @throws {Error} If all retry attempts fail. The thrown error is the last error
+ *   encountered. For non-Error thrown values, the error wraps the original value
+ *   in `error.cause` for debugging context (ECMAScript 2022 Error.cause).
+ *
+ * @example
+ * const result = await evaluateWithRetry(
+ *   (tc) => gEval(tc, criteria, llmFn),
+ *   testCase,
+ *   3
+ * );
+ *
+ * @example
+ * // Accessing error cause for debugging
+ * try {
+ *   await evaluateWithRetry(evaluate, testCase, 3);
+ * } catch (error) {
+ *   console.error('Final error:', error.message);
+ *   if (error.cause) {
+ *     console.error('Original cause:', error.cause);
+ *   }
+ * }
+ */
+export async function evaluateWithRetry(evaluate, testCase, maxRetries = DEFAULT_MAX_RETRIES) {
+    let lastError = new Error('No attempts made');
+    let retryCount = 0;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const result = await evaluate(testCase);
+            // Validate result has valid score
+            if (isValidScore(result.score)) {
+                return { ...result, retryCount };
+            }
+            // Invalid score - treat as error and retry
+            throw new Error(`Invalid score: ${result.score}`);
+        }
+        catch (error) {
+            // Preserve original error as cause for debugging context
+            // Use JSON.stringify for objects to get meaningful message instead of "[object Object]"
+            const errorMessage = error instanceof Error
+                ? error.message
+                : (typeof error === 'object' && error !== null ? JSON.stringify(error) : String(error));
+            lastError = error instanceof Error ? error : new Error(errorMessage, { cause: error });
+            retryCount++;
+            // Don't wait after the last attempt
+            if (attempt < maxRetries) {
+                // Exponential backoff: 2^(attempt-1) seconds, capped at MAX_BACKOFF_EXPONENT
+                // First retry (attempt=1) waits 1s, second (attempt=2) waits 2s, etc.
+                const cappedExponent = Math.min(attempt - 1, MAX_BACKOFF_EXPONENT);
+                const backoffMs = Math.min(BACKOFF_BASE_MS * 2 ** cappedExponent, MAX_BACKOFF_MS);
+                await delay(backoffMs);
+            }
+        }
+    }
+    throw lastError;
+}
+/**
+ * Circuit breaker for judge model failures.
+ * Prevents cascading failures when judge model is unavailable.
+ * Rate limit errors (429) are not counted toward the threshold.
+ *
+ * Compare to {@link evaluateWithRetry}:
+ * - `evaluateWithRetry`: Retries same operation with backoff (1s → 2s → 4s)
+ * - `JudgeCircuitBreaker.evaluate`: Fails fast when service is degraded, optional fallback
+ */
+export class JudgeCircuitBreaker {
+    threshold;
+    resetTimeout;
+    failures = 0;
+    lastFailure = null;
+    isOpen = false;
+    /**
+     * Flag to prevent multiple concurrent resets (race condition protection).
+     * @note This implementation assumes single-threaded Node.js execution.
+     * For worker threads or multi-process deployments, use external synchronization
+     * (e.g., Redis-based distributed locks) instead of this in-memory flag.
+     */
+    resetting = false;
+    /**
+     * Count of times circuit has opened (for flapping detection).
+     * @note This counter is unbounded for simplicity. In practice, overflow would
+     * take ~584 million years at 1 state change per second. For long-running
+     * services requiring bounded counters, use external observability tools
+     * (e.g., Prometheus counters) or implement a sliding window approach.
+     */
+    openCount = 0;
+    /**
+     * Count of times circuit has been reset (for flapping detection).
+     * @note Unbounded counter - see openCount for rationale.
+     */
+    resetCount = 0;
+    /**
+     * Create a new circuit breaker instance.
+     *
+     * @param threshold - Number of failures before circuit opens (default: DEFAULT_CIRCUIT_BREAKER_THRESHOLD)
+     * @param resetTimeout - Time in ms before circuit resets (default: DEFAULT_CIRCUIT_BREAKER_RESET_MS)
+     */
+    constructor(threshold = DEFAULT_CIRCUIT_BREAKER_THRESHOLD, resetTimeout = DEFAULT_CIRCUIT_BREAKER_RESET_MS) {
+        this.threshold = threshold;
+        this.resetTimeout = resetTimeout;
+    }
+    /**
+     * Check if circuit is open (failing).
+     *
+     * @returns True if circuit is open and blocking requests
+     */
+    get open() {
+        return this.isOpen;
+    }
+    /**
+     * Get current failure count.
+     *
+     * @returns Number of consecutive failures (excluding rate limits)
+     */
+    get failureCount() {
+        return this.failures;
+    }
+    /**
+     * Get circuit breaker statistics for observability and flapping detection.
+     *
+     * @returns Object with openCount and resetCount for monitoring circuit health
+     */
+    get stats() {
+        return { openCount: this.openCount, resetCount: this.resetCount };
+    }
+    /**
+     * Reset the circuit breaker to closed state.
+     * Thread-safe: uses resetting flag to prevent concurrent resets.
+     *
+     * @returns void
+     */
+    reset() {
+        this.isOpen = false;
+        this.failures = 0;
+        this.lastFailure = null;
+        this.resetting = false;
+        this.resetCount++;
+    }
+    /**
+     * Check if an error should count toward circuit breaker threshold.
+     *
+     * @param error - Error to check
+     * @returns False for rate limit errors (transient), true for other errors
+     */
+    shouldCountAsFailure(error) {
+        if (!(error instanceof Error)) {
+            return true;
+        }
+        // Type-based checks (robust for typed providers)
+        const rateLimitErrorNames = [
+            'RateLimitError', // OpenAI SDK
+            'ThrottlingException', // AWS Bedrock
+            'TooManyRequestsError', // Generic
+            'RateLimitExceeded', // Anthropic
+        ];
+        if (rateLimitErrorNames.includes(error.name)) {
+            return false;
+        }
+        // HTTP status code check (if available on error object)
+        const errorWithStatus = error;
+        if (errorWithStatus.statusCode === HttpStatus.TOO_MANY_REQUESTS || errorWithStatus.status === HttpStatus.TOO_MANY_REQUESTS) {
+            return false;
+        }
+        // Error code check (AWS-style errors)
+        if (errorWithStatus.code === 'ThrottlingException' ||
+            errorWithStatus.code === 'TooManyRequestsException' ||
+            errorWithStatus.code === 'ProvisionedThroughputExceededException') {
+            return false;
+        }
+        // Fallback to message pattern matching (last resort)
+        // Use word boundary regex to avoid false positives
+        if (error.message && typeof error.message === 'string') {
+            const message = error.message.toLowerCase();
+            // Note: \s+ intentionally matches any amount of whitespace to handle
+            // variations like "too   many    requests" from different providers
+            if (/\brate[_\s-]?limit/i.test(message) ||
+                /\b429\b/.test(message) ||
+                /\bthrottl/i.test(message) ||
+                /\btoo\s+many\s+requests\b/i.test(message)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Execute evaluation with circuit breaker protection.
+     *
+     * @param evaluate - Primary evaluation function to execute
+     * @param fallbackEvaluate - Optional fallback function when circuit is open
+     * @returns Result from evaluate or fallbackEvaluate
+     * @throws {Error} If circuit is open and no fallback provided
+     * @throws {Error} If evaluation fails (error is re-thrown after recording)
+     */
+    async evaluate(evaluate, fallbackEvaluate) {
+        // Check if circuit should be reset - triple-check pattern with resetting flag
+        // prevents race condition where multiple concurrent calls could all reset
+        if (this.isOpen && this.lastFailure && !this.resetting) {
+            const elapsed = Date.now() - this.lastFailure.getTime();
+            if (elapsed > this.resetTimeout && this.isOpen && !this.resetting) {
+                // Set flag BEFORE reset to prevent concurrent resets
+                this.resetting = true;
+                this.reset();
+            }
+        }
+        // If circuit is open, use fallback or throw
+        if (this.isOpen) {
+            if (fallbackEvaluate) {
+                return fallbackEvaluate();
+            }
+            throw new Error('Circuit breaker open - evaluation temporarily unavailable');
+        }
+        try {
+            const result = await evaluate();
+            this.failures = 0;
+            return result;
+        }
+        catch (error) {
+            // Only count non-transient errors
+            if (this.shouldCountAsFailure(error)) {
+                this.failures++;
+                this.lastFailure = new Date();
+                if (this.failures >= this.threshold) {
+                    this.isOpen = true;
+                    this.openCount++;
+                }
+            }
+            throw error;
+        }
+    }
+}
+/**
+ * Default canary test cases for judge pipeline health monitoring.
+ */
+export const DEFAULT_CANARY_CASES = [
+    {
+        name: 'perfect_answer',
+        input: 'What is 2+2?',
+        output: '2+2 equals 4.',
+        metric: 'relevance',
+        expectedScore: { min: 0.9 },
+        description: 'Simple factual answer should score high',
+    },
+    {
+        name: 'hallucination_detection',
+        input: 'What is the capital of France?',
+        output: 'The capital of France is Tokyo, a beautiful city in Asia.',
+        metric: 'faithfulness',
+        expectedScore: { max: 0.3 },
+        description: 'Obvious hallucination should score low',
+    },
+    {
+        name: 'off_topic_detection',
+        input: 'Explain quantum computing',
+        output: 'I love pizza! It is delicious with pepperoni.',
+        metric: 'relevance',
+        expectedScore: { max: 0.2 },
+        description: 'Completely off-topic should score very low',
+    },
+];
+/**
+ * Run canary evaluations to monitor judge pipeline health.
+ *
+ * @param evaluate - Evaluation function to test (takes test case and metric)
+ * @param canaries - Canary test cases to run (defaults to DEFAULT_CANARY_CASES)
+ * @returns Canary report with overall pass/fail and individual results
+ * @throws {Error} If any canary lacks expectedScore.min or expectedScore.max
+ */
+export async function runCanaryEvaluations(evaluate, canaries = DEFAULT_CANARY_CASES) {
+    // Validate evaluate is a function
+    if (typeof evaluate !== 'function') {
+        throw new Error('runCanaryEvaluations requires an evaluate function');
+    }
+    const results = [];
+    for (const canary of canaries) {
+        // Validate canary has at least one threshold defined
+        if (canary.expectedScore.min === undefined && canary.expectedScore.max === undefined) {
+            throw new Error(`Canary '${canary.name}' must define expectedScore.min or expectedScore.max`);
+        }
+        const score = await evaluate({ input: canary.input, output: canary.output }, canary.metric);
+        // Validate score is in valid range
+        if (!isValidScore(score)) {
+            results.push({
+                name: canary.name,
+                score: NaN,
+                expected: canary.expectedScore,
+                passed: false,
+                timestamp: new Date().toISOString(),
+            });
+            continue;
+        }
+        // Determine if score passes threshold - check both min AND max when both defined
+        // We validated above that at least one of min/max is defined
+        const passed = (canary.expectedScore.min === undefined || score >= canary.expectedScore.min) &&
+            (canary.expectedScore.max === undefined || score <= canary.expectedScore.max);
+        results.push({
+            name: canary.name,
+            score,
+            expected: canary.expectedScore,
+            passed,
+            timestamp: new Date().toISOString(),
+        });
+    }
+    return {
+        timestamp: new Date().toISOString(),
+        passed: results.every(r => r.passed),
+        results,
+    };
+}
+//# sourceMappingURL=llm-as-judge.js.map