observability-toolkit 2.0.0 → 2.1.1

package/dist/lib/llm-as-judge.d.ts

@@ -1,657 +0,0 @@
-/**
- * LLM-as-Judge Implementation
- *
- * Provides patterns and utilities for using LLMs to evaluate LLM outputs.
- * Implements G-Eval, QAG, and production-ready evaluation patterns per
- * industry best practices and OTel GenAI semantic conventions.
- *
- * @security
- * - All user inputs are sanitized for prompt injection protection
- * - LLM calls have timeout protection (default 30s)
- * - Input sizes are validated to prevent resource exhaustion
- * - JSON parsing has depth limits to prevent DoS
- *
- * @security Known Limitations
- * - Script homoglyphs (Cyrillic, Greek characters visually similar to Latin)
- *   are NOT currently filtered in all cases. The confusables library provides
- *   partial coverage but may miss some Unicode TR39 edge cases.
- *   Example: Cyrillic "а" (U+0430) looks identical to Latin "a".
- *   This is a known gap tracked for future enhancement.
- *
- * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
- */
-import type { EvaluatorType } from '../backends/index.js';
-/**
- * Error for prompt injection detection.
- *
- * NOTE: This class is exported for type-checking and external use but is
- * intentionally NOT thrown by sanitizeForPrompt(). The design decision is
- * to silently replace injection patterns with '[filtered]' markers rather
- * than fail-fast, allowing evaluation to proceed with sanitized input.
- *
- * @example
- * // External code can throw this for stricter handling:
- * if (detectsInjection(input)) {
- *   throw new PromptInjectionError('Injection detected in user input');
- * }
- */
-export declare class PromptInjectionError extends Error {
-    constructor(message: string);
-}
-/**
- * Error thrown when an LLM call exceeds the configured timeout.
- *
- * Thrown by withTimeout() when the wrapped function does not complete
- * within the specified duration. Callers should catch this to implement
- * fallback behavior or retry logic.
- *
- * @example
- * try {
- *   await withTimeout(() => llmCall(), 5000);
- * } catch (e) {
- *   if (e instanceof LLMTimeoutError) {
- *     // Handle timeout - use cached result or return default
- *   }
- * }
- *
- * @see withTimeout
- */
-export declare class LLMTimeoutError extends Error {
-    constructor(timeoutMs: number);
-}
-/**
- * Error thrown when score extraction or normalization fails.
- *
- * Thrown by normalizeWithLogprobs() when the LLM response cannot be
- * parsed into a valid score, or when probability weighting fails.
- *
- * @example
- * try {
- *   const score = normalizeWithLogprobs(response, [1,2,3,4,5]);
- * } catch (e) {
- *   if (e instanceof ScoreNormalizationError) {
- *     // Fallback to regex-based extraction
- *   }
- * }
- *
- * @see normalizeWithLogprobs
- * @see extractScoreFromText
- */
-export declare class ScoreNormalizationError extends Error {
-    constructor(message: string);
-}
-/** Maximum input size in bytes (64KB) */
-export declare const MAX_INPUT_SIZE_BYTES = 65536;
-/** Maximum text length per field (10KB) */
-export declare const MAX_TEXT_LENGTH = 10000;
-/** Maximum context array length */
-export declare const MAX_CONTEXT_ITEMS = 20;
-/** Maximum statements to process in QAG pattern */
-export declare const MAX_STATEMENTS = 20;
-/** Default timeout for LLM calls (30 seconds) */
-export declare const DEFAULT_LLM_TIMEOUT_MS = 30000;
-/** Maximum JSON nesting depth */
-export declare const MAX_JSON_DEPTH = 5;
-/**
- * Log levels for llm-as-judge module.
- * Set via LLM_JUDGE_LOG_LEVEL environment variable.
- * @default 'warn'
- */
-export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent';
-/**
- * Current log level for the module.
- * Controls verbosity of console output for production flexibility.
- * - 'debug': All logs including verbose debugging info
- * - 'info': Informational messages and above
- * - 'warn': Warnings and errors only (default)
- * - 'error': Only error messages
- * - 'silent': No logging output
- */
-export declare const LOG_LEVEL: LogLevel;
-/** G-Eval minimum score (inclusive) */
-export declare const G_EVAL_MIN_SCORE = 1;
-/** G-Eval maximum score (inclusive) */
-export declare const G_EVAL_MAX_SCORE = 5;
-/** G-Eval valid score values array */
-export declare const G_EVAL_VALID_SCORES: readonly [1, 2, 3, 4, 5];
-/** G-Eval score range for normalization (max - min) */
-export declare const G_EVAL_SCORE_RANGE: number;
-/**
- * G-Eval default/middle score for fallback scenarios.
- * @note This constant is exported for library consumers who need a sensible
- * default when implementing custom fallback logic. The core gEval() function
- * throws errors rather than using fallback values to ensure explicit handling.
- * @example
- * ```typescript
- * // Custom fallback in consumer code
- * const score = parseResult(response) ?? G_EVAL_DEFAULT_SCORE;
- * ```
- */
-export declare const G_EVAL_DEFAULT_SCORE = 3;
-/** Default temperature for deterministic LLM calls (e.g., extraction, answering) */
-export declare const LLM_TEMPERATURE_DETERMINISTIC = 0;
-/** Default temperature for evaluation LLM calls (slight variation allowed) */
-export declare const LLM_TEMPERATURE_EVALUATION = 0.1;
-/** Minimum evaluation steps to generate in G-Eval */
-export declare const G_EVAL_MIN_STEPS = 3;
-/** Maximum evaluation steps to generate in G-Eval */
-export declare const G_EVAL_MAX_STEPS = 5;
-/** Minimum statement length to be considered valid for QAG extraction */
-export declare const MIN_STATEMENT_LENGTH = 10;
-/** Default maximum retry attempts for evaluateWithRetry */
-export declare const DEFAULT_MAX_RETRIES = 3;
-/** Default circuit breaker failure threshold */
-export declare const DEFAULT_CIRCUIT_BREAKER_THRESHOLD = 5;
-/** Default circuit breaker reset timeout in milliseconds */
-export declare const DEFAULT_CIRCUIT_BREAKER_RESET_MS = 30000;
-/** Base delay multiplier for exponential backoff (1 second) */
-export declare const BACKOFF_BASE_MS = 1000;
-/** Minimum valid normalized score */
-export declare const NORMALIZED_SCORE_MIN = 0;
-/** Maximum valid normalized score */
-export declare const NORMALIZED_SCORE_MAX = 1;
-/**
- * Extended evaluation event with error handling, performance metrics, and
- * operational metadata per OTel GenAI semantic conventions.
- */
-export interface EvaluationEvent {
-    timestamp: string;
-    evaluationName: string;
-    scoreValue?: number;
-    scoreLabel?: string;
-    explanation?: string;
-    evaluator?: string;
-    evaluatorType?: EvaluatorType;
-    responseId?: string;
-    traceId?: string;
-    spanId?: string;
-    sessionId?: string;
-    errorType?: 'timeout' | 'rate_limit' | 'invalid_response' | 'model_error';
-    errorMessage?: string;
-    durationMs?: number;
-    inputTokens?: number;
-    outputTokens?: number;
-    totalTokens?: number;
-    retryCount?: number;
-    judgeModel?: string;
-    judgeTemperature?: number;
-    samplingReason?: 'random' | 'error' | 'priority';
-}
-/**
- * OTel attribute mapping for evaluation events
- */
-export declare const EVALUATION_OTEL_ATTRIBUTES: {
-    readonly evaluationName: "gen_ai.evaluation.name";
-    readonly scoreValue: "gen_ai.evaluation.score.value";
-    readonly scoreLabel: "gen_ai.evaluation.score.label";
-    readonly explanation: "gen_ai.evaluation.explanation";
-    readonly errorType: "error.type";
-    readonly durationMs: "gen_ai.evaluation.duration";
-    readonly inputTokens: "gen_ai.usage.input_tokens";
-    readonly outputTokens: "gen_ai.usage.output_tokens";
-};
-/**
- * Test case for evaluation
- */
-export interface TestCase {
-    input: string;
-    output: string;
-    context?: string[];
-    expectedOutput?: string;
-}
-/**
- * Evaluation result from judge model
- */
-export interface EvalResult {
-    score: number;
-    reason: string;
-    rawResponse?: string;
-}
-/**
- * G-Eval configuration
- */
-export interface GEvalConfig {
-    name: string;
-    criteria: string;
-    evaluationParams: ('input' | 'output' | 'context' | 'expectedOutput')[];
-    temperature?: number;
-}
-/**
- * Pairwise comparison result
- */
-export interface PairwiseResult {
-    winner: 'A' | 'B' | 'tie';
-    explanation: string;
-    confidence: number;
-}
-/**
- * LLM provider interface for judge calls.
- *
- * @throws {Error} On timeout (>30s), rate limit, or model errors
- */
-export interface LLMProvider {
-    generate(prompt: string, options?: {
-        temperature?: number;
-        logprobs?: boolean;
-    }): Promise<{
-        text: string;
-        logprobs?: Array<{
-            token: string;
-            logprob: number;
-        }>;
-    }>;
-}
-/**
- * Sanitizes an array of context strings for safe prompt inclusion.
- * Truncates to MAX_CONTEXT_ITEMS and sanitizes each item.
- *
- * @param context - Array of context strings to sanitize
- * @returns Sanitized and truncated context array (max MAX_CONTEXT_ITEMS items)
- * @security Applies prompt injection protection to each context item
- */
-export declare function sanitizeContextArray(context: string[]): string[];
-/**
- * Sanitize text for safe inclusion in prompts.
- * Detects and removes potential prompt injection attempts.
- *
- * SECURITY DESIGN:
- * - Uses normalized text (homoglyphs → Latin) for injection DETECTION
- * - Preserves original text when no injection patterns are found
- * - Only returns normalized text when malicious patterns are detected
- * - This preserves legitimate Cyrillic/Greek text while catching attacks
- *
- * @param text - Text to sanitize
- * @param maxLength - Maximum allowed length (default: MAX_TEXT_LENGTH)
- * @returns Sanitized text with injection patterns replaced by '[filtered]'
- * @security Removes zero-width characters and neutralizes prompt injection patterns
- */
-export declare function sanitizeForPrompt(text: string, maxLength?: number): string;
-/**
- * Create a customizable sanitizer with additional injection patterns.
- *
- * Use this factory when you need to extend the default prompt injection
- * patterns with domain-specific patterns for your use case.
- *
- * @param additionalPatterns - Additional regex patterns to detect as prompt injection
- * @returns Sanitizer function with signature (text: string, maxLength?: number) => string
- *   that applies all default patterns plus additionalPatterns, truncates to maxLength,
- *   and returns sanitized text with '[filtered]' markers for detected injections.
- * @throws {InputValidationError} If additionalPatterns contains non-RegExp items
- *
- * @example
- * ```typescript
- * const customSanitizer = createSanitizer([
- *   /my\s+custom\s+attack\s+pattern/gi,
- *   /another\s+pattern/gi,
- * ]);
- * const sanitized = customSanitizer(userInput);
- * const truncated = customSanitizer(longInput, 1000); // Override maxLength per-call
- * ```
- *
- * @security Custom patterns MUST avoid ReDoS vulnerabilities:
- * - Use non-capturing groups (?:...) when grouping is not needed
- * - Avoid nested quantifiers like (a+)+ or (a*)*
- * - Test patterns with tools like safe-regex before deployment
- * - Example vulnerable: /^(a+)+$/ with input "aaaaaaaaaaaaaaaaX"
- * - Example safe: /^a+$/ or /^(?:a+)$/
- */
-export declare function createSanitizer(additionalPatterns?: RegExp[]): (text: string, maxLength?: number) => string;
-/**
- * Validate test case input sizes against security limits.
- * Checks individual field lengths and total byte size.
- *
- * @param testCase - Test case to validate
- * @returns void
- * @throws {InputValidationError} If any field exceeds MAX_TEXT_LENGTH
- * @throws {InputValidationError} If context exceeds MAX_CONTEXT_ITEMS
- * @throws {InputValidationError} If total size exceeds MAX_INPUT_SIZE_BYTES
- * @security Prevents resource exhaustion by enforcing size limits
- */
-export declare function validateTestCase(testCase: TestCase): void;
-/**
- * Safe JSON parsing with depth limit to prevent DoS attacks.
- *
- * @param text - JSON text to parse
- * @param maxDepth - Maximum nesting depth (default: MAX_JSON_DEPTH)
- * @returns Parsed value
- * @throws {Error} If JSON is invalid, too large, or too deeply nested
- */
-export declare function safeJSONParse(text: string, maxDepth?: number): unknown;
-/**
- * Execute an async function with timeout protection.
- *
- * Uses AbortController for atomic cancellation to prevent race conditions
- * between the function completing and the timeout firing. The abort signal
- * provides atomic state that both code paths can check safely.
- *
- * @param fn - Async function to execute, receives AbortSignal for cancellation
- * @param timeoutMs - Timeout in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
- * @returns Result of the function
- * @throws {LLMTimeoutError} If function times out
- */
-export declare function withTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs?: number): Promise<T>;
-/**
- * Build evaluation prompt from config and test case.
- * Used by G-Eval pattern for structured evaluation prompts.
- *
- * @param config - G-Eval configuration with criteria and parameters
- * @param testCase - Test case containing input, output, and optional context
- * @param steps - Evaluation steps generated by chain-of-thought
- * @returns Formatted prompt string for the judge model
- * @security All user-provided content is sanitized for prompt injection
- */
-export declare function buildEvalPrompt(config: GEvalConfig, testCase: TestCase, steps: string): string;
-/**
- * Normalize score using token log probabilities.
- * Calculates weighted average score based on probability distribution.
- *
- * @param logprobs - Token log probabilities from LLM response
- * @param validScores - Valid score values (e.g., [1, 2, 3, 4, 5])
- * @returns Normalized score as weighted average
- * @throws {ScoreNormalizationError} When no valid score tokens found in logprobs
- */
-export declare function normalizeWithLogprobs(logprobs: Array<{
-    token: string;
-    logprob: number;
-}>, validScores: number[]): number;
-/**
- * Extract score from LLM response text.
- *
- * Uses specific patterns to avoid false positives from incidental digits.
- * Falls back to the last digit in the valid range within the last 100 characters,
- * since LLMs typically provide their final answer at the end.
- *
- * @param text - LLM response text
- * @returns Extracted score in G-Eval range
- * @throws {ScoreNormalizationError} If no valid score found
- */
-export declare function extractScoreFromText(text: string): number;
-/**
- * G-Eval implementation using chain-of-thought prompting with token probability normalization.
- *
- * @param llm - LLM provider for judge calls
- * @param config - G-Eval configuration with name, criteria, and evaluation parameters
- * @param testCase - Test case to evaluate
- * @param timeoutMs - Timeout for LLM calls in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
- * @returns Evaluation result with normalized score (0-1), reason, and raw response
- * @throws {InputValidationError} If test case exceeds size limits
- * @throws {Error} If LLM call times out or returns invalid score
- * @security Input validation, prompt injection protection, and timeout protection
- */
-export declare function gEval(llm: LLMProvider, config: GEvalConfig, testCase: TestCase, timeoutMs?: number): Promise<EvalResult>;
-/**
- * Extract atomic statements from LLM output.
- * Each statement should be independently verifiable.
- *
- * @param llm - LLM provider for extraction
- * @param output - Text to extract statements from
- * @param timeoutMs - Timeout for LLM call in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
- * @returns Array of atomic statements (max MAX_STATEMENTS)
- * @throws {Error} If LLM call times out
- * @security Output sanitized, safe JSON parsing with depth limits, result capped at MAX_STATEMENTS
- */
-export declare function extractStatements(llm: LLMProvider, output: string, timeoutMs?: number): Promise<string[]>;
-/**
- * Generate verification question for a statement.
- *
- * @param llm - LLM provider for question generation
- * @param statement - Statement to verify
- * @param timeoutMs - Timeout for LLM call in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
- * @returns Yes/no verification question string
- * @throws {Error} If LLM call times out
- * @security Statement is sanitized for prompt injection before use
- */
-export declare function generateVerificationQuestion(llm: LLMProvider, statement: string, timeoutMs?: number): Promise<string>;
-/**
- * Answer verification question using provided context.
- *
- * @param llm - LLM provider for answering
- * @param question - Yes/no question to answer
- * @param context - Context documents to use for answering
- * @param timeoutMs - Timeout for LLM call in milliseconds (default: DEFAULT_LLM_TIMEOUT_MS)
- * @returns 'yes', 'no', or 'unknown' based on context
- * @throws {Error} If LLM call times out
- * @security Question and context are sanitized for prompt injection
- */
-export declare function answerQuestion(llm: LLMProvider, question: string, context: string[], timeoutMs?: number): Promise<'yes' | 'no' | 'unknown'>;
-/**
- * QAG (Question-Answer Generation) evaluation.
- * Decomposes evaluation into atomic yes/no questions.
- *
- * Uses Promise.allSettled for graceful degradation - partial failures
- * don't abort the entire evaluation. Score is calculated from successful
- * verifications only.
- *
- * @param llm - LLM provider for all operations
- * @param input - Original user input (unused but included for API consistency)
- * @param output - LLM output to evaluate for faithfulness
- * @param context - Context documents for verification
- * @param options - Optional configuration object
- * @param options.timeoutMs - Timeout for each LLM call (default: DEFAULT_LLM_TIMEOUT_MS)
- * @returns Faithfulness score (0-1) as proportion of verified statements
- * @security Timeout protection on all calls; graceful degradation on partial failures
- * @performance Makes 2N+1 LLM calls for N statements (capped at MAX_STATEMENTS=20):
- *   - 1 call to extract atomic statements from output
- *   - N parallel calls to generate verification questions (one per statement)
- *   - N parallel calls to answer questions from context (one per statement)
- *   For MAX_STATEMENTS (20), this is 41 LLM calls total.
- *   Typical latency: 10-30s (parallel execution) depending on LLM provider.
- */
-export declare function qagEvaluate(llm: LLMProvider, input: string, output: string, context: string[], options?: {
-    timeoutMs?: number;
-}): Promise<number>;
-/**
- * Mitigated pairwise evaluation with position bias correction.
- * Evaluates both orderings and only counts consistent wins.
- *
- * @param evaluate - Evaluation function that compares two outputs
- * @param input - User input
- * @param outputA - First output to compare
- * @param outputB - Second output to compare
- * @returns Winner ('A', 'B', or 'tie')
- * @throws {Error} If evaluate function is not provided
- * @throws {InputValidationError} If input, outputA, or outputB is empty
- * @throws {InputValidationError} If any string exceeds MAX_TEXT_LENGTH
- */
-export declare function mitigatedPairwiseEval(evaluate: (input: string, first: string, second: string) => Promise<{
-    winner: 'A' | 'B' | 'tie';
-}>, input: string, outputA: string, outputB: string): Promise<'A' | 'B' | 'tie'>;
-/**
- * Multi-judge panel evaluation.
- * Uses multiple judge models and returns median score.
- *
- * @param evaluators - Array of evaluation functions for different models
- * @param testCase - Test case to evaluate
- * @returns Median score from all judges
- * @throws {Error} If evaluators array is empty
- */
-export declare function panelEvaluation(evaluators: Array<(testCase: TestCase) => Promise<number>>, testCase: TestCase): Promise<number>;
-/**
- * Validate that a score is within expected range [0, 1].
- *
- * @param score - Score value to validate
- * @returns True if score is a number between 0 and 1 (inclusive), false otherwise
- */
-export declare function isValidScore(score: number): boolean;
-/**
- * Evaluate with retry logic and exponential backoff.
- *
- * Retries a single evaluation function on transient failures. Use this for
- * simple retry scenarios where you want automatic backoff.
- *
- * Compare to {@link JudgeCircuitBreaker.evaluate}:
- * - `evaluateWithRetry`: Retries same operation with backoff (1s → 2s → 4s)
- * - `JudgeCircuitBreaker.evaluate`: Fails fast when service is degraded, optional fallback
- *
- * These can be combined: wrap evaluateWithRetry inside circuit breaker for
- * retries on transient errors while circuit-breaking on sustained failures.
- *
- * @param evaluate - Evaluation function to retry on failure
- * @param testCase - Test case to evaluate
- * @param maxRetries - Maximum number of retry attempts (default: DEFAULT_MAX_RETRIES)
- * @returns Evaluation result with retryCount indicating number of failed attempts
- * @throws {Error} If all retry attempts fail. The thrown error is the last error
- *   encountered. For non-Error thrown values, the error wraps the original value
- *   in `error.cause` for debugging context (ECMAScript 2022 Error.cause).
- *
- * @example
- * const result = await evaluateWithRetry(
- *   (tc) => gEval(tc, criteria, llmFn),
- *   testCase,
- *   3
- * );
- *
- * @example
- * // Accessing error cause for debugging
- * try {
- *   await evaluateWithRetry(evaluate, testCase, 3);
- * } catch (error) {
- *   console.error('Final error:', error.message);
- *   if (error.cause) {
- *     console.error('Original cause:', error.cause);
- *   }
- * }
- */
-export declare function evaluateWithRetry<T extends EvalResult>(evaluate: (testCase: TestCase) => Promise<T>, testCase: TestCase, maxRetries?: number): Promise<T & {
-    retryCount: number;
-}>;
-/**
- * Circuit breaker for judge model failures.
- * Prevents cascading failures when judge model is unavailable.
- * Rate limit errors (429) are not counted toward the threshold.
- *
- * Compare to {@link evaluateWithRetry}:
- * - `evaluateWithRetry`: Retries same operation with backoff (1s → 2s → 4s)
- * - `JudgeCircuitBreaker.evaluate`: Fails fast when service is degraded, optional fallback
- */
-export declare class JudgeCircuitBreaker {
-    private threshold;
-    private resetTimeout;
-    private failures;
-    private lastFailure;
-    private isOpen;
-    /**
-     * Flag to prevent multiple concurrent resets (race condition protection).
-     * @note This implementation assumes single-threaded Node.js execution.
-     * For worker threads or multi-process deployments, use external synchronization
-     * (e.g., Redis-based distributed locks) instead of this in-memory flag.
-     */
-    private resetting;
-    /**
-     * Count of times circuit has opened (for flapping detection).
-     * @note This counter is unbounded for simplicity. In practice, overflow would
-     * take ~584 million years at 1 state change per second. For long-running
-     * services requiring bounded counters, use external observability tools
-     * (e.g., Prometheus counters) or implement a sliding window approach.
-     */
-    private openCount;
-    /**
-     * Count of times circuit has been reset (for flapping detection).
-     * @note Unbounded counter - see openCount for rationale.
-     */
-    private resetCount;
-    /**
-     * Create a new circuit breaker instance.
-     *
-     * @param threshold - Number of failures before circuit opens (default: DEFAULT_CIRCUIT_BREAKER_THRESHOLD)
-     * @param resetTimeout - Time in ms before circuit resets (default: DEFAULT_CIRCUIT_BREAKER_RESET_MS)
-     */
-    constructor(threshold?: number, resetTimeout?: number);
-    /**
-     * Check if circuit is open (failing).
-     *
-     * @returns True if circuit is open and blocking requests
-     */
-    get open(): boolean;
-    /**
-     * Get current failure count.
-     *
-     * @returns Number of consecutive failures (excluding rate limits)
-     */
-    get failureCount(): number;
-    /**
-     * Get circuit breaker statistics for observability and flapping detection.
-     *
-     * @returns Object with openCount and resetCount for monitoring circuit health
-     */
-    get stats(): {
-        openCount: number;
-        resetCount: number;
-    };
-    /**
-     * Reset the circuit breaker to closed state.
-     * Thread-safe: uses resetting flag to prevent concurrent resets.
-     *
-     * @returns void
-     */
-    reset(): void;
-    /**
-     * Check if an error should count toward circuit breaker threshold.
-     *
-     * @param error - Error to check
-     * @returns False for rate limit errors (transient), true for other errors
-     */
-    private shouldCountAsFailure;
-    /**
-     * Execute evaluation with circuit breaker protection.
-     *
-     * @param evaluate - Primary evaluation function to execute
-     * @param fallbackEvaluate - Optional fallback function when circuit is open
-     * @returns Result from evaluate or fallbackEvaluate
-     * @throws {Error} If circuit is open and no fallback provided
-     * @throws {Error} If evaluation fails (error is re-thrown after recording)
-     */
-    evaluate<T>(evaluate: () => Promise<T>, fallbackEvaluate?: () => Promise<T>): Promise<T>;
-}
-/**
- * Canary test case with expected score bounds.
- */
-export interface CanaryTestCase {
-    name: string;
-    input: string;
-    output: string;
-    metric: string;
-    expectedScore: {
-        min?: number;
-        max?: number;
-    };
-    description: string;
-}
-/**
- * Result of a canary evaluation.
- */
-export interface CanaryResult {
-    name: string;
-    score: number;
-    expected: {
-        min?: number;
-        max?: number;
-    };
-    passed: boolean;
-    timestamp: string;
-}
-/**
- * Report from canary evaluation run.
- */
-export interface CanaryReport {
-    timestamp: string;
-    passed: boolean;
-    results: CanaryResult[];
-}
-/**
- * Default canary test cases for judge pipeline health monitoring.
- */
-export declare const DEFAULT_CANARY_CASES: CanaryTestCase[];
-/**
- * Run canary evaluations to monitor judge pipeline health.
- *
- * @param evaluate - Evaluation function to test (takes test case and metric)
- * @param canaries - Canary test cases to run (defaults to DEFAULT_CANARY_CASES)
- * @returns Canary report with overall pass/fail and individual results
- * @throws {Error} If any canary lacks expectedScore.min or expectedScore.max
- */
-export declare function runCanaryEvaluations(evaluate: (testCase: {
-    input: string;
-    output: string;
-}, metric: string) => Promise<number>, canaries?: CanaryTestCase[]): Promise<CanaryReport>;
-//# sourceMappingURL=llm-as-judge.d.ts.map

package/dist/lib/llm-as-judge.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"llm-as-judge.d.ts","sourceRoot":"","sources":["../../src/lib/llm-as-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAU1D;;;;;;;;;;;;;GAaG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,eAAgB,SAAQ,KAAK;gBAC5B,SAAS,EAAE,MAAM;CAI9B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;gBACpC,OAAO,EAAE,MAAM;CAI5B;AAMD,yCAAyC;AACzC,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAE1C,2CAA2C;AAC3C,eAAO,MAAM,eAAe,QAAQ,CAAC;AAErC,mCAAmC;AACnC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,mDAAmD;AACnD,eAAO,MAAM,cAAc,KAAK,CAAC;AAEjC,iDAAiD;AACjD,eAAO,MAAM,sBAAsB,QAAQ,CAAC;AAE5C,iCAAiC;AACjC,eAAO,MAAM,cAAc,IAAI,CAAC;AAMhC;;;;GAIG;AACH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtE;;;;;;;;GAQG;AACH,eAAO,MAAM,SAAS,EAAE,QAAkE,CAAC;AAkB3F,uCAAuC;AACvC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,uCAAuC;AACvC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,sCAAsC;AACtC,eAAO,MAAM,mBAAmB,0BAA2B,CAAC;AAE5D,uDAAuD;AACvD,eAAO,MAAM,kBAAkB,QAAsC,CAAC;AAEtE;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAMtC,oFAAoF;AACpF,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,qDAAqD;AACrD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,qDAAqD;AACrD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAMlC,yEAAyE;AACzE,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAMvC,2DAA2D;AAC3D,eAAO,MAAM,mBAAmB,IAAI,CAAC;AAErC,gDAAgD;AAChD,eAAO,MAAM,iCAAiC,IAAI,CAAC;AAEnD,4DAA4D;AAC5D,eAAO,MAAM,gCAAgC,QAAQ,CAAC;AAEtD,+DAA+D;AAC/D,eAAO,MAAM,eAAe,OAAO,CAAC;AAMpC,qCAAqC;AACrC,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,qCAAqC;AACrC,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAMtC;;;GAGG;AACH,MAAM,WAAW,eAAe;IAE9B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IAGvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,aAAa,CAAC;IAG9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,SAAS,CAAC,EAAE,SAAS,GAAG,YAAY,GAAG,kBAAkB,GAAG,aAAa,CAAC;IAC1E,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,QAAQ,GAAG,OAAO,GAAG,UAAU,CAAC;CAClD;AAED;;GAEG;AACH,eAAO,MAAM,0BAA0B;;;;;;;;;CAS7B,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,CAAC,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,gBAAgB,CAAC,EAAE,CAAC;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GACrD,OAAO,CAAC;QACT,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACtD,CAAC,CAAC;CACJ;AAkGD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAEhE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAwB,GAAG,MAAM,CAyC3F;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,eAAe,CAC7B,kBAAkB,GAAE,MAAM,EAAO,GAChC,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,MAAM,CAgD9C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAwEzD;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAuB,GAAG,OAAO,CA8BtF;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,WAAW,CAAC,CAAC,EACjC,EAAE,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EACvC,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,CAAC,CAAC,CAmCZ;AAMD;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,MAAM,GACZ,MAAM,CAuBR;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EACnD,WAAW,EAAE,MAAM,EAAE,GACpB,MAAM,CAgCR;AAqDD;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAoBzD;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,KAAK,CACzB,GAAG,EAAE,WAAW,EAChB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,QAAQ,EAClB,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,UAAU,CAAC,CAmDrB;AAMD;;;;;;;;;;GAUG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,WAAW,EAChB,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,MAAM,EAAE,CAAC,CAqDnB;AAED;;;;;;;;;GASG;AACH,wBAAsB,4BAA4B,CAChD,GAAG,EAAE,WAAW,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EAAE,EACjB,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,SAAS,CAAC,CAkCnC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,WAAW,EAChB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/B,OAAO,CAAC,MAAM,CAAC,CAoDjB;AAmCD;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,KAAK,CAAA;CAAE,CAAC,EAClG,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,CAkF5B;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,UAAU,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,EAC1D,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC,MAAM,CAAC,CAiBjB;AAMD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEnD;AAoBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,SAAS,UAAU,EAC1D,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,EAC5C,QAAQ,EAAE,QAAQ,EAClB,UAAU,GAAE,MAA4B,GACvC,OAAO,CAAC,CAAC,GAAG;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,CAoCrC;AAED;;;;;;;;GAQG;AACH,qBAAa,mBAAmB;IAgC5B,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,YAAY;IAhCtB,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB;;;;;OAKG;IACH,OAAO,CAAC,SAAS,CAAS;IAC1B;;;;;;OAMG;IACH,OAAO,CAAC,SAAS,CAAK;IACtB;;;OAGG;IACH,OAAO,CAAC,UAAU,CAAK;IAEvB;;;;;OAKG;gBAEO,SAAS,GAAE,MAA0C,EACrD,YAAY,GAAE,MAAyC;IAGjE;;;;OAIG;IACH,IAAI,IAAI,IAAI,OAAO,CAElB;IAED;;;;OAIG;IACH,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;;OAIG;IACH,IAAI,KAAK,IAAI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAErD;IAED;;;;;OAKG;IACH,KAAK,IAAI,IAAI;IAQb;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IA8C5B;;;;;;;;OAQG;IACG,QAAQ,CAAC,CAAC,EACd,QAAQ,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC1B,gBAAgB,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAClC,OAAO,CAAC,CAAC,CAAC;CAuCd;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,eAAO,MAAM,oBAAoB,EAAE,cAAc,EAyBhD,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EAC1F,QAAQ,GAAE,cAAc,EAAyB,GAChD,OAAO,CAAC,YAAY,CAAC,CAmDvB"}