observability-toolkit 2.0.0 → 2.1.0

package/dist/lib/judge/llm-judge-config.js

@@ -0,0 +1,390 @@
+/**
+ * LLM Judge Configuration
+ *
+ * Provides a configuration object pattern for LLM-as-Judge evaluations.
+ * Centralizes timeout, retries, and logging configuration for cleaner API.
+ *
+ * @see BACKLOG.md Configuration Object Pattern section
+ */
+import { z } from 'zod';
+import { normalizedScoreSchema } from '../core/shared-schemas.js';
+import { evaluatorTypeSchema } from '../../backends/index.js';
+import { LLMTimeoutError, gEval, qagEvaluate, withTimeout, } from './llm-as-judge.js';
+import { BACKOFF_BASE_MS } from '../core/constants.js';
+import { BACKOFF_MAX_MS } from '../exports/export-utils.js';
+import { DEFAULT_LLM_TIMEOUT_MS, DEFAULT_HALLUCINATION_THRESHOLD, META_EVAL_SAMPLE_RATE, MAX_TIMEOUT_MS, MAX_RETRIES, PRODUCTION_TIMEOUT_MS, PRODUCTION_MAX_RETRIES, LLM_TEMPERATURE_DETERMINISTIC, } from './llm-judge-constants.js';
+// ============================================================================
+// Types (defined after Zod schemas below)
+// ============================================================================
+// ============================================================================
+// Zod Schemas for LLM Judge Configuration
+// ============================================================================
+export const guardrailResultSchema = z.object({
+    score: normalizedScoreSchema,
+    blocked: z.boolean(),
+    reason: z.string().optional(),
+});
+const logFnSchema = z.function({ input: z.tuple([z.string(), z.record(z.string(), z.unknown()).optional()]), output: z.void() });
+export const llmJudgeLoggerSchema = z.looseObject({
+    debug: logFnSchema.optional(),
+    info: logFnSchema.optional(),
+    warn: logFnSchema.optional(),
+    error: logFnSchema.optional(),
+}); // Allow extra properties for custom loggers
+export const hallucinationGuardrailOptionsSchema = z.object({
+    threshold: normalizedScoreSchema.optional(),
+    judgeConfig: z.lazy(() => llmJudgeConfigSchema).optional(),
+});
+export const llmJudgeConfigSchema = z.object({
+    timeoutMs: z.int().min(1).max(MAX_TIMEOUT_MS).optional(),
+    maxRetries: z.int().min(0).max(MAX_RETRIES).optional(),
+    logger: llmJudgeLoggerSchema.optional(),
+    evaluator: z.string().optional(),
+    evaluatorType: evaluatorTypeSchema.optional(),
+});
+export const configuredEvaluationResultSchema = z.object({
+    score: normalizedScoreSchema,
+    reason: z.string().optional(),
+    metadata: z.object({
+        evaluator: z.string(),
+        evaluatorType: z.string(),
+        timeoutMs: z.int().nonnegative(),
+        retries: z.int().nonnegative(),
+    }),
+});
+export const metaEvalGuardSchema = z.object({
+    isMetaEval: z.boolean(),
+});
+// ============================================================================
+// Constants
+// (Defined in llm-judge-constants.ts)
+// ============================================================================
+export { DEFAULT_HALLUCINATION_THRESHOLD, META_EVAL_SAMPLE_RATE, MAX_TIMEOUT_MS, MAX_RETRIES, PRODUCTION_TIMEOUT_MS, PRODUCTION_MAX_RETRIES };
+/** Default configuration values */
+export const DEFAULT_CONFIG = {
+    timeoutMs: DEFAULT_LLM_TIMEOUT_MS,
+    maxRetries: 0,
+    logger: {},
+    evaluator: 'llm-judge',
+    evaluatorType: 'llm',
+};
+// ============================================================================
+// Pre-defined Evaluation Criteria
+// ============================================================================
+/** Relevance evaluation criteria */
+export const RELEVANCE_CRITERIA = {
+    name: 'relevance',
+    criteria: 'Evaluate how relevant the output is to the given context and input',
+    evaluationParams: ['input', 'output', 'context'],
+};
+/** Faithfulness evaluation criteria */
+export const FAITHFULNESS_CRITERIA = {
+    name: 'faithfulness',
+    criteria: 'Evaluate if the output is factually consistent with the context without hallucination',
+    evaluationParams: ['output', 'context'],
+};
+/** Coherence evaluation criteria */
+export const COHERENCE_CRITERIA = {
+    name: 'coherence',
+    criteria: 'Evaluate the logical flow and structural organization of the output',
+    evaluationParams: ['output'],
+};
+/** Explanation quality meta-evaluation criteria */
+export const EXPLANATION_QUALITY_CRITERIA = {
+    name: 'explanation_quality',
+    criteria: `Evaluate the quality of a judge's explanation for an evaluation score.
+Rate on three dimensions:
+1. Specificity: Does the explanation reference concrete details from the evaluated output?
+2. Evidence citation: Does it quote or paraphrase specific passages as evidence?
+3. Actionability: Does it suggest what could be improved?`,
+    evaluationParams: ['input', 'output'],
+    temperature: LLM_TEMPERATURE_DETERMINISTIC,
+};
+// ============================================================================
+// Helpers
+// ============================================================================
+/**
+ * Resolve configuration with defaults and validation
+ */
+function resolveConfig(config = {}) {
+    // Validate config against schema
+    const parseResult = llmJudgeConfigSchema.safeParse(config);
+    if (!parseResult.success) {
+        throw new TypeError(`Invalid LLM judge config: ${parseResult.error.message}`);
+    }
+    const validatedConfig = parseResult.data;
+    const timeoutMs = validatedConfig.timeoutMs ?? DEFAULT_CONFIG.timeoutMs;
+    const maxRetries = validatedConfig.maxRetries ?? DEFAULT_CONFIG.maxRetries;
+    return {
+        timeoutMs,
+        maxRetries,
+        logger: validatedConfig.logger ?? DEFAULT_CONFIG.logger,
+        evaluator: validatedConfig.evaluator ?? DEFAULT_CONFIG.evaluator,
+        evaluatorType: validatedConfig.evaluatorType ?? DEFAULT_CONFIG.evaluatorType,
+    };
+}
+/**
+ * Execute with retries and logging
+ */
+async function executeWithRetry(operation, config, operationName) {
+    const EXPONENTIAL_BACKOFF_BASE = 2;
+    let lastError;
+    for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
+        try {
+            config.logger.debug?.(`${operationName}: attempt ${attempt + 1}`, {
+                timeoutMs: config.timeoutMs,
+                maxRetries: config.maxRetries,
+            });
+            const result = await withTimeout(operation, config.timeoutMs);
+            config.logger.debug?.(`${operationName}: success`, {
+                attempt: attempt + 1,
+                retries: attempt, // Number of retries = current attempt (0 for first try, 1 for second, etc.)
+            });
+            return { result, retries: attempt };
+        }
+        catch (error) {
+            lastError = error;
+            const isRetryable = error instanceof LLMTimeoutError ||
+                (error instanceof Error && error.message.includes('rate limit'));
+            if (isRetryable && attempt < config.maxRetries) {
+                const backoff = Math.min(BACKOFF_BASE_MS * Math.pow(EXPONENTIAL_BACKOFF_BASE, attempt), BACKOFF_MAX_MS);
+                config.logger.warn?.(`${operationName}: retry after ${backoff}ms`, {
+                    attempt: attempt + 1,
+                    error: lastError.message,
+                });
+                await new Promise(resolve => setTimeout(resolve, backoff));
+            }
+            else {
+                config.logger.error?.(`${operationName}: failed`, {
+                    attempt: attempt + 1,
+                    error: lastError.message,
+                });
+                break;
+            }
+        }
+    }
+    throw lastError;
+}
+// ============================================================================
+// Meta-Evaluation
+// ============================================================================
+function buildMetaEvalInput(originalEvaluation, originalInput) {
+    return `Evaluation: ${originalEvaluation.evaluationName} (score: ${originalEvaluation.score})\nOriginal input: ${originalInput}`;
+}
+/**
+ * Determine whether to run meta-evaluation for this evaluation result.
+ * Uses random sampling at META_EVAL_SAMPLE_RATE (10%).
+ * Never meta-evaluates meta-evaluations (recursion guard).
+ */
+export function shouldMetaEvaluate(evaluationName, guard) {
+    if (guard?.isMetaEval)
+        return false;
+    if (evaluationName === EXPLANATION_QUALITY_CRITERIA.name)
+        return false;
+    return Math.random() < META_EVAL_SAMPLE_RATE;
+}
+/**
+ * Evaluate the quality of a judge's explanation.
+ * Uses G-Eval to score explanation specificity, evidence citation, and actionability.
+ *
+ * @remarks Does not apply timeout or retry logic. For production use,
+ * prefer {@link LLMJudge.evaluateExplanationQuality}.
+ *
+ * @param llm - LLM provider for meta-evaluation
+ * @param originalEvaluation - The evaluation whose explanation is being assessed
+ * @param originalInput - The original input that was evaluated
+ * @returns EvalResult with explanation quality score (0-1)
+ */
+export async function evaluateExplanationQuality(llm, originalEvaluation, originalInput) {
+    const testCase = {
+        input: buildMetaEvalInput(originalEvaluation, originalInput),
+        output: originalEvaluation.reason,
+    };
+    return gEval(llm, EXPLANATION_QUALITY_CRITERIA, testCase);
+}
+// ============================================================================
+// LLMJudge Class
+// ============================================================================
+/**
+ * Configured LLM Judge instance.
+ *
+ * Provides a cleaner API by encapsulating configuration once,
+ * rather than passing timeoutMs to every function call.
+ *
+ * @example
+ * const judge = new LLMJudge(llmProvider, {
+ *   timeoutMs: 60_000,
+ *   maxRetries: 2,
+ *   logger: console,
+ * });
+ *
+ * const result = await judge.evaluateRelevance('output', ['context']);
+ */
+export class LLMJudge {
+    llm;
+    config;
+    constructor(llm, config = {}) {
+        this.llm = llm;
+        this.config = resolveConfig(config);
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Run G-Eval with configured settings
+     */
+    async gEval(criteria, testCase) {
+        const { result, retries } = await executeWithRetry(() => gEval(this.llm, criteria, testCase, this.config.timeoutMs), this.config, 'gEval');
+        return {
+            score: result.score,
+            reason: result.reason,
+            metadata: {
+                evaluator: this.config.evaluator,
+                evaluatorType: this.config.evaluatorType,
+                timeoutMs: this.config.timeoutMs,
+                retries,
+            },
+        };
+    }
+    /**
+     * Run QAG evaluation with configured settings
+     * Returns faithfulness score (0-1)
+     */
+    async qagEvaluate(input, output, context) {
+        const { result: score, retries } = await executeWithRetry(() => qagEvaluate(this.llm, input, output, context, { timeoutMs: this.config.timeoutMs }), this.config, 'qagEvaluate');
+        return {
+            score,
+            metadata: {
+                evaluator: this.config.evaluator,
+                evaluatorType: this.config.evaluatorType,
+                timeoutMs: this.config.timeoutMs,
+                retries,
+            },
+        };
+    }
+    /**
+     * Evaluate relevance with configured settings
+     */
+    async evaluateRelevance(input, output, context) {
+        return this.gEval(RELEVANCE_CRITERIA, { input, output, context });
+    }
+    /**
+     * Evaluate faithfulness with configured settings
+     */
+    async evaluateFaithfulness(input, output, context) {
+        return this.gEval(FAITHFULNESS_CRITERIA, { input, output, context });
+    }
+    /**
+     * Evaluate coherence with configured settings
+     */
+    async evaluateCoherence(output) {
+        return this.gEval(COHERENCE_CRITERIA, { input: output, output });
+    }
+    /**
+     * Evaluate explanation quality of an evaluation result with configured settings
+     */
+    async evaluateExplanationQuality(originalEvaluation, originalInput) {
+        return this.gEval(EXPLANATION_QUALITY_CRITERIA, {
+            input: buildMetaEvalInput(originalEvaluation, originalInput),
+            output: originalEvaluation.reason,
+        });
+    }
+}
+// ============================================================================
+// Factory Functions
+// ============================================================================
+/**
+ * Create a configured LLM Judge instance
+ */
+export function createLLMJudge(llm, config) {
+    return new LLMJudge(llm, config);
+}
+/**
+ * Create a quick evaluation function with minimal configuration
+ */
+export function createQuickEvaluator(llm, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const judge = new LLMJudge(llm, { timeoutMs });
+    return {
+        relevance: async (input, output, context) => {
+            const result = await judge.evaluateRelevance(input, output, context);
+            return result.score;
+        },
+        faithfulness: async (input, output, context) => {
+            const result = await judge.evaluateFaithfulness(input, output, context);
+            return result.score;
+        },
+        coherence: async (output) => {
+            const result = await judge.evaluateCoherence(output);
+            return result.score;
+        },
+    };
+}
+/**
+ * Create a production-ready evaluator with logging and retries
+ */
+export function createProductionEvaluator(llm, logger, options = {}) {
+    return new LLMJudge(llm, {
+        timeoutMs: options.timeoutMs ?? PRODUCTION_TIMEOUT_MS,
+        maxRetries: options.maxRetries ?? PRODUCTION_MAX_RETRIES,
+        logger,
+        evaluator: options.evaluator ?? 'production-judge',
+        evaluatorType: 'llm',
+    });
+}
+// ============================================================================
+// Hallucination Guardrail
+// ============================================================================
+/**
+ * Guardrail that blocks translation or generation outputs when the
+ * hallucination score returned by QAG evaluation exceeds a threshold.
+ *
+ * Hallucination is scored 0–1 where higher = more hallucination.
+ * Outputs with score >= `threshold` are blocked.
+ *
+ * @example
+ * const guardrail = new HallucinationGuardrail(llm, { threshold: 0.10 });
+ * const result = await guardrail.check(input, output, context);
+ * if (result.blocked) {
+ *   throw new Error(`Hallucination detected: ${result.score}`);
+ * }
+ */
+export class HallucinationGuardrail {
+    judge;
+    threshold;
+    constructor(llm, options = {}) {
+        this.threshold = options.threshold ?? DEFAULT_HALLUCINATION_THRESHOLD;
+        if (this.threshold <= 0 || this.threshold > 1) {
+            throw new TypeError(`HallucinationGuardrail threshold must be in (0, 1], got ${this.threshold}`);
+        }
+        this.judge = new LLMJudge(llm, options.judgeConfig);
+    }
+    /**
+     * Check an output for hallucination.
+     * Uses QAG (faithfulness) where score near 0 = faithful, near 1 = hallucinates.
+     * Returns `blocked: true` when hallucination score >= threshold.
+     */
+    async check(input, output, context) {
+        const { score, reason } = await this.judge.evaluateFaithfulness(input, output, context);
+        // Faithfulness is inverse of hallucination: 1 = faithful, 0 = hallucination
+        const hallucinationScore = 1 - score;
+        return {
+            score: hallucinationScore,
+            blocked: hallucinationScore >= this.threshold,
+            reason,
+        };
+    }
+    /** Configured blocking threshold */
+    get blocksAbove() {
+        return this.threshold;
+    }
+}
+/**
+ * Create a hallucination guardrail with the given threshold.
+ */
+export function createHallucinationGuardrail(llm, options) {
+    return new HallucinationGuardrail(llm, options);
+}
+//# sourceMappingURL=llm-judge-config.js.map

package/dist/lib/judge/llm-judge-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-config.js","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-config.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAsB,MAAM,yBAAyB,CAAC;AAClF,OAAO,EACL,eAAe,EACf,KAAK,EACL,WAAW,EACX,WAAW,GAKZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,+BAA+B,EAC/B,qBAAqB,EACrB,cAAc,EACd,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAElC,+EAA+E;AAC/E,0CAA0C;AAC1C,+EAA+E;AAE/E,+EAA+E;AAC/E,0CAA0C;AAC1C,+EAA+E;AAE/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,KAAK,EAAE,qBAAqB;IAC5B,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAEjI,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,WAAW,CAAC;IAChD,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE;IAC7B,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE;IAC5B,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE;IAC5B,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC,CAAC,4CAA4C;AAEhD,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1D,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IAC3C,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;CAC3D,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,SAAS,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IACxD,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE;IACtD,MAAM,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,aAAa,EAAE,mBAAmB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CAAC;AAcH,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,CAAC,MAAM,CAAC;IACvD,KAAK,EAAE,qBAAqB;IAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;QACjB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,SAAS,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAChC,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;KAC/B,CAAC;CACH,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE;CACxB,CAAC,CAAC;AAyBH,+EAA+E;AAC/E,YAAY;AACZ,sCAAsC;AACtC,+EAA+E;AAE/E,OAAO,EAAE,+BAA+B,EAAE,qBAAqB,EAAE,cAAc,EAAE,WAAW,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,CAAC;AAE9I,mCAAmC;AACnC,MAAM,CAAC,MAAM,cAAc,GAA0E;IACnG,SAAS,EAAE,sBAAsB;IACjC,UAAU,EAAE,CAAC;IACb,MAAM,EAAE,EAAE;IACV,SAAS,EAAE,WAAW;IACtB,aAAa,EAAE,KAAK;CACrB,CAAC;AAGF,+EAA+E;AAC/E,kCAAkC;AAClC,+EAA+E;AAE/E,oCAAoC;AACpC,MAAM,CAAC,MAAM,kBAAkB,GAAgB;IAC7C,IAAI,EAAE,WAAW;IACjB,QAAQ,EAAE,oEAAoE;IAC9E,gBAAgB,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC;CACjD,CAAC;AAEF,uCAAuC;AACvC,MAAM,CAAC,MAAM,qBAAqB,GAAgB;IAChD,IAAI,EAAE,cAAc;IACpB,QAAQ,EAAE,uFAAuF;IACjG,gBAAgB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;CACxC,CAAC;AAEF,oCAAoC;AACpC,MAAM,CAAC,MAAM,kBAAkB,GAAgB;IAC7C,IAAI,EAAE,WAAW;IACjB,QAAQ,EAAE,qEAAqE;IAC/E,gBAAgB,EAAE,CAAC,QAAQ,CAAC;CAC7B,CAAC;AAEF,mDAAmD;AACnD,MAAM,CAAC,MAAM,4BAA4B,GAAgB;IACvD,IAAI,EAAE,qBAAqB;IAC3B,QAAQ,EAAE;;;;0DAI8C;IACxD,gBAAgB,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;IACrC,WAAW,EAAE,6BAA6B;CAC3C,CAAC;AAEF,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E;;GAEG;AACH,SAAS,aAAa,CAAC,SAAyB,EAAE;IAChD,iCAAiC;IACjC,MAAM,WAAW,GAAG,oBAAoB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACzB,MAAM,IAAI,SAAS,CAAC,6BAA6B,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC;IAEzC,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,IAAI,cAAc,CAAC,SAAS,CAAC;IACxE,MAAM,UAAU,GAAG,eAAe,CAAC,UAAU,IAAI,cAAc,CAAC,UAAU,CAAC;IAE3E,OAAO;QACL,SAAS;QACT,UAAU;QACV,MAAM,EAAE,eAAe,CAAC,MAAM,IAAI,cAAc,CAAC,MAAM;QACvD,SAAS,EAAE,eAAe,CAAC,SAAS,IAAI,cAAc,CAAC,SAAS;QAChE,aAAa,EAAE,eAAe,CAAC,aAAa,IAAI,cAAc,CAAC,aAAa;KAC7E,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,SAA2B,EAC3B,MAAsB,EACtB,aAAqB;IAErB,MAAM,wBAAwB,GAAG,CAAC,CAAC;IACnC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,aAAa,aAAa,OAAO,GAAG,CAAC,EAAE,EAAE;gBAChE,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAE9D,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,aAAa,WAAW,EAAE;gBACjD,OAAO,EAAE,OAAO,GAAG,CAAC;gBACpB,OAAO,EAAE,OAAO,EAAE,4EAA4E;aAC/F,CAAC,CAAC;YAEH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAc,CAAC;YAE3B,MAAM,WAAW,GACf,KAAK,YAAY,eAAe;gBAChC,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;YAEnE,IAAI,WAAW,IAAI,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,wBAAwB,EAAE,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;gBACxG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,GAAG,aAAa,iBAAiB,OAAO,IAAI,EAAE;oBACjE,OAAO,EAAE,OAAO,GAAG,CAAC;oBACpB,KAAK,EAAE,SAAS,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,aAAa,UAAU,EAAE;oBAChD,OAAO,EAAE,OAAO,GAAG,CAAC;oBACpB,KAAK,EAAE,SAAS,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E,SAAS,kBAAkB,CACzB,kBAA6D,EAC7D,aAAqB;IAErB,OAAO,eAAe,kBAAkB,CAAC,cAAc,YAAY,kBAAkB,CAAC,KAAK,sBAAsB,aAAa,EAAE,CAAC;AACnI,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,cAAsB,EAAE,KAAqB;IAC9E,IAAI,KAAK,EAAE,UAAU;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,cAAc,KAAK,4BAA4B,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACvE,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,qBAAqB,CAAC;AAC/C,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,GAAgB,EAChB,kBAA6E,EAC7E,aAAqB;IAErB,MAAM,QAAQ,GAAa;QACzB,KAAK,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,aAAa,CAAC;QAC5D,MAAM,EAAE,kBAAkB,CAAC,MAAM;KAClC,CAAC;IACF,OAAO,KAAK,CAAC,GAAG,EAAE,4BAA4B,EAAE,QAAQ,CAAC,CAAC;AAC5D,CAAC;AAED,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,QAAQ;IAIT;IAHF,MAAM,CAAiB;IAE/B,YACU,GAAgB,EACxB,SAAyB,EAAE;QADnB,QAAG,GAAH,GAAG,CAAa;QAGxB,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACT,QAAqB,EACrB,QAAkB;QAElB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,gBAAgB,CAChD,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAChE,IAAI,CAAC,MAAM,EACX,OAAO,CACR,CAAC;QAEF,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE;gBACR,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;gBACxC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,OAAO;aACR;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,KAAa,EACb,MAAc,EACd,OAAiB;QAEjB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,gBAAgB,CACvD,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EACzF,IAAI,CAAC,MAAM,EACX,aAAa,CACd,CAAC;QAEF,OAAO;YACL,KAAK;YACL,QAAQ,EAAE;gBACR,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;gBACxC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,OAAO;aACR;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CACrB,KAAa,EACb,MAAc,EACd,OAAiB;QAEjB,OAAO,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CACxB,KAAa,EACb,MAAc,EACd,OAAiB;QAEjB,OAAO,IAAI,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CACrB,MAAc;QAEd,OAAO,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,0BAA0B,CAC9B,kBAA6E,EAC7E,aAAqB;QAErB,OAAO,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE;YAC9C,KAAK,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,aAAa,CAAC;YAC5D,MAAM,EAAE,kBAAkB,CAAC,MAAM;SAClC,CAAC,CAAC;IACL,CAAC;CACF;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,GAAgB,EAChB,MAAuB;IAEvB,OAAO,IAAI,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,GAAgB,EAChB,YAAoB,sBAAsB;IAM1C,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAE/C,OAAO;QACL,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;YAC1C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YACrE,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YACxE,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAC1B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;YACrD,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB,CACvC,GAAgB,EAChB,MAAsB,EACtB,UAII,EAAE;IAEN,OAAO,IAAI,QAAQ,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,qBAAqB;QACrD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,sBAAsB;QACxD,MAAM;QACN,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;QAClD,aAAa,EAAE,KAAK;KACrB,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,0BAA0B;AAC1B,+EAA+E;AAE/E;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,sBAAsB;IACzB,KAAK,CAAW;IAChB,SAAS,CAAS;IAE1B,YAAY,GAAgB,EAAE,UAAyC,EAAE;QACvE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,+BAA+B,CAAC;QACtE,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,SAAS,CACjB,2DAA2D,IAAI,CAAC,SAAS,EAAE,CAC5E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK,CACT,KAAa,EACb,MAAc,EACd,OAAiB;QAEjB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAC7D,KAAK,EACL,MAAM,EACN,OAAO,CACR,CAAC;QACF,4EAA4E;QAC5E,MAAM,kBAAkB,GAAG,CAAC,GAAG,KAAK,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,kBAAkB;YACzB,OAAO,EAAE,kBAAkB,IAAI,IAAI,CAAC,SAAS;YAC7C,MAAM;SACP,CAAC;IACJ,CAAC;IAED,oCAAoC;IACpC,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,4BAA4B,CAC1C,GAAgB,EAChB,OAAuC;IAEvC,OAAO,IAAI,sBAAsB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAClD,CAAC"}

package/dist/lib/judge/llm-judge-config.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for LLM Judge Configuration
+ */
+export {};
+//# sourceMappingURL=llm-judge-config.test.d.ts.map

package/dist/lib/judge/llm-judge-config.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-config.test.d.ts","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-config.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}