observability-toolkit 2.0.0 → 2.1.1

package/dist/lib/judge/llm-judge-config.test.js

@@ -0,0 +1,392 @@
+/**
+ * Tests for LLM Judge Configuration
+ */
+import { describe, it } from 'vitest';
+import * as assert from 'node:assert';
+import { LLMJudge, createLLMJudge, createQuickEvaluator, createProductionEvaluator, HallucinationGuardrail, createHallucinationGuardrail, DEFAULT_CONFIG, DEFAULT_HALLUCINATION_THRESHOLD, MAX_TIMEOUT_MS, MAX_RETRIES, PRODUCTION_TIMEOUT_MS, PRODUCTION_MAX_RETRIES, RELEVANCE_CRITERIA, FAITHFULNESS_CRITERIA, COHERENCE_CRITERIA, } from './llm-judge-config.js';
+import { JUDGE_SCORE_FIVE, JUDGE_SCORE_FOUR, PROBABILITY_FIFTY_PERCENT, PROBABILITY_TWENTY_PERCENT, COUNT_THREE, COUNT_TEN, COUNT_TWO, DELTA, TEST_TIMEOUT_MS_LONG, TEST_TIMEOUT_MS_SHORT, } from '../../test-helpers/test-constants.js';
+const MOCK_STEPS_RESPONSE = '1. Analyze content\n2. Check quality\n3. Rate coherence';
+const MOCK_SCORE_RESPONSE = (score) => `Score: ${score}/5. Good overall quality.`;
+// Mock LLM Provider that returns a valid G-Eval response (steps on call 1, score on call 2+)
+function createMockLLM(score = JUDGE_SCORE_FOUR) {
+    let callIndex = 0;
+    return {
+        generate: async () => {
+            const text = callIndex === 0 ? MOCK_STEPS_RESPONSE : MOCK_SCORE_RESPONSE(score);
+            callIndex++;
+            return { text };
+        },
+    };
+}
+// Mock LLM Provider that fails N times (on any call) then succeeds with valid G-Eval responses.
+// gEval makes 2 LLM calls per attempt: first = steps, second = scoring.
+// This mock tracks total calls: odd calls return steps, even calls return score.
+// The first `failCount` calls throw to simulate rate limit errors.
+function createRetryableMockLLM(failCount) {
+    let callIndex = 0;
+    return {
+        generate: async () => {
+            callIndex++;
+            if (callIndex <= failCount) {
+                throw new Error('rate limit exceeded');
+            }
+            // After failures, calls come in pairs per gEval invocation.
+            // Determine position within current gEval attempt (1-based).
+            const posInAttempt = (callIndex - failCount - 1) % 2;
+            return { text: posInAttempt === 0 ? MOCK_STEPS_RESPONSE : 'Score: 4/5. Retry succeeded.' };
+        },
+    };
+}
+describe('llm-judge-config', () => {
+    // ==========================================================================
+    // LLMJudge Class
+    // ==========================================================================
+    describe('LLMJudge', () => {
+        it('creates instance with default config', () => {
+            const llm = createMockLLM();
+            const judge = new LLMJudge(llm);
+            const config = judge.getConfig();
+            assert.strictEqual(config.timeoutMs, DEFAULT_CONFIG.timeoutMs);
+            assert.strictEqual(config.maxRetries, DEFAULT_CONFIG.maxRetries);
+        });
+        it('creates instance with custom config', () => {
+            const llm = createMockLLM();
+            const judge = new LLMJudge(llm, {
+                timeoutMs: PRODUCTION_TIMEOUT_MS,
+                maxRetries: COUNT_THREE,
+                evaluator: 'custom-judge',
+            });
+            const config = judge.getConfig();
+            assert.strictEqual(config.timeoutMs, PRODUCTION_TIMEOUT_MS);
+            assert.strictEqual(config.maxRetries, COUNT_THREE);
+            assert.strictEqual(config.evaluator, 'custom-judge');
+        });
+        it('throws on invalid timeout', () => {
+            const llm = createMockLLM();
+            assert.throws(() => {
+                new LLMJudge(llm, { timeoutMs: 0 });
+            }, /Invalid LLM judge config/);
+            assert.throws(() => {
+                new LLMJudge(llm, { timeoutMs: MAX_TIMEOUT_MS + 1 });
+            }, /Invalid LLM judge config/);
+        });
+        it('throws on invalid maxRetries', () => {
+            const llm = createMockLLM();
+            assert.throws(() => {
+                new LLMJudge(llm, { maxRetries: -1 });
+            }, /Invalid LLM judge config/);
+            assert.throws(() => {
+                new LLMJudge(llm, { maxRetries: MAX_RETRIES + 1 });
+            }, /Invalid LLM judge config/);
+        });
+    });
+    // ==========================================================================
+    // Pre-defined Criteria
+    // ==========================================================================
+    describe('pre-defined criteria', () => {
+        it('RELEVANCE_CRITERIA has required fields', () => {
+            assert.strictEqual(RELEVANCE_CRITERIA.name, 'relevance');
+            assert.ok(RELEVANCE_CRITERIA.criteria.length > 0);
+            assert.ok(RELEVANCE_CRITERIA.evaluationParams.includes('output'));
+        });
+        it('FAITHFULNESS_CRITERIA has required fields', () => {
+            assert.strictEqual(FAITHFULNESS_CRITERIA.name, 'faithfulness');
+            assert.ok(FAITHFULNESS_CRITERIA.criteria.includes('hallucination'));
+            assert.ok(FAITHFULNESS_CRITERIA.evaluationParams.includes('context'));
+        });
+        it('COHERENCE_CRITERIA has required fields', () => {
+            assert.strictEqual(COHERENCE_CRITERIA.name, 'coherence');
+            assert.ok(COHERENCE_CRITERIA.criteria.includes('logical'));
+            assert.ok(COHERENCE_CRITERIA.evaluationParams.includes('output'));
+        });
+    });
+    // ==========================================================================
+    // Evaluation Methods
+    // ==========================================================================
+    describe('evaluation methods', () => {
+        it('gEval returns configured result', async () => {
+            const llm = createMockLLM(JUDGE_SCORE_FOUR);
+            const judge = new LLMJudge(llm, {
+                timeoutMs: TEST_TIMEOUT_MS_LONG,
+                evaluator: 'test-judge',
+            });
+            const result = await judge.gEval(RELEVANCE_CRITERIA, {
+                input: 'What is AI?',
+                output: 'AI is artificial intelligence.',
+                context: ['AI stands for artificial intelligence.'],
+            });
+            assert.ok(result.score >= 0 && result.score <= 1);
+            assert.strictEqual(result.metadata.evaluator, 'test-judge');
+            assert.strictEqual(result.metadata.timeoutMs, TEST_TIMEOUT_MS_LONG);
+            assert.strictEqual(result.metadata.retries, 0);
+        });
+        it('evaluateRelevance uses RELEVANCE_CRITERIA', async () => {
+            const llm = createMockLLM(JUDGE_SCORE_FOUR);
+            const judge = new LLMJudge(llm, { evaluatorType: 'llm' });
+            const result = await judge.evaluateRelevance('What is AI?', 'AI is artificial intelligence.', ['AI stands for artificial intelligence.']);
+            assert.ok(result.score >= 0 && result.score <= 1);
+            assert.strictEqual(result.metadata.evaluatorType, 'llm');
+        });
+        it('evaluateFaithfulness uses FAITHFULNESS_CRITERIA', async () => {
+            const llm = createMockLLM(JUDGE_SCORE_FIVE);
+            const judge = new LLMJudge(llm);
+            const result = await judge.evaluateFaithfulness('What is AI?', 'AI is artificial intelligence.', ['AI stands for artificial intelligence.']);
+            assert.ok(result.score >= 0 && result.score <= 1);
+        });
+        it('evaluateCoherence uses COHERENCE_CRITERIA', async () => {
+            const llm = createMockLLM(JUDGE_SCORE_FOUR);
+            const judge = new LLMJudge(llm);
+            const result = await judge.evaluateCoherence('AI is artificial intelligence. It can perform many tasks.');
+            assert.ok(result.score >= 0 && result.score <= 1);
+        });
+    });
+    // ==========================================================================
+    // Retry Behavior
+    // ==========================================================================
+    describe('retry behavior', () => {
+        it('retries on rate limit error', async () => {
+            const llm = createRetryableMockLLM(1);
+            const judge = new LLMJudge(llm, {
+                maxRetries: 2,
+                timeoutMs: 1000,
+            });
+            const result = await judge.gEval(COHERENCE_CRITERIA, {
+                input: 'Test input',
+                output: 'Test output',
+            });
+            assert.ok(result.score >= 0);
+            assert.strictEqual(result.metadata.retries, 1);
+        });
+        it('fails after max retries exceeded', async () => {
+            const llm = createRetryableMockLLM(JUDGE_SCORE_FIVE);
+            const judge = new LLMJudge(llm, {
+                maxRetries: 2,
+                timeoutMs: 100,
+            });
+            await assert.rejects(() => judge.gEval(COHERENCE_CRITERIA, { input: 'Test input', output: 'Test' }), /rate limit/);
+        });
+        it('does not retry on non-retryable errors', async () => {
+            let attempts = 0;
+            const llm = {
+                generate: async () => {
+                    attempts++;
+                    throw new Error('Invalid input');
+                },
+            };
+            const judge = new LLMJudge(llm, { maxRetries: 2, timeoutMs: 100 });
+            await assert.rejects(() => judge.gEval(COHERENCE_CRITERIA, { input: 'Test input', output: 'Test' }), /Invalid input/);
+            assert.strictEqual(attempts, 1);
+        });
+    });
+    // ==========================================================================
+    // Logger Integration
+    // ==========================================================================
+    describe('logger integration', () => {
+        it('calls logger.debug on operation start', async () => {
+            const debugCalls = [];
+            const logger = {
+                debug: (msg) => { debugCalls.push(msg); },
+            };
+            const llm = createMockLLM(JUDGE_SCORE_FOUR);
+            const judge = new LLMJudge(llm, { logger, timeoutMs: TEST_TIMEOUT_MS_SHORT });
+            await judge.gEval(COHERENCE_CRITERIA, { input: 'Test input', output: 'Test' });
+            assert.ok(debugCalls.some(msg => msg.includes('attempt 1')));
+            assert.ok(debugCalls.some(msg => msg.includes('success')));
+        });
+        it('calls logger.warn on retry', async () => {
+            const warnCalls = [];
+            const logger = {
+                debug: () => { },
+                warn: (msg) => { warnCalls.push(msg); },
+            };
+            const llm = createRetryableMockLLM(1);
+            const judge = new LLMJudge(llm, {
+                logger,
+                maxRetries: 2,
+                timeoutMs: 100,
+            });
+            await judge.gEval(COHERENCE_CRITERIA, { input: 'Test input', output: 'Test' });
+            assert.ok(warnCalls.some(msg => msg.includes('retry')));
+        });
+        it('calls logger.error on failure', async () => {
+            const errorCalls = [];
+            const logger = {
+                debug: () => { },
+                error: (msg) => { errorCalls.push(msg); },
+            };
+            const llm = {
+                generate: async () => { throw new Error('Fatal'); },
+            };
+            const judge = new LLMJudge(llm, { logger, timeoutMs: 100 });
+            try {
+                await judge.gEval(COHERENCE_CRITERIA, { input: 'Test input', output: 'Test' });
+            }
+            catch {
+                // Expected
+            }
+            assert.ok(errorCalls.some(msg => msg.includes('failed')));
+        });
+    });
+    // ==========================================================================
+    // Factory Functions
+    // ==========================================================================
+    describe('createLLMJudge', () => {
+        it('creates LLMJudge instance', () => {
+            const llm = createMockLLM();
+            const judge = createLLMJudge(llm, { timeoutMs: TEST_TIMEOUT_MS_LONG });
+            assert.ok(judge instanceof LLMJudge);
+            assert.strictEqual(judge.getConfig().timeoutMs, TEST_TIMEOUT_MS_LONG);
+        });
+    });
+    describe('createQuickEvaluator', () => {
+        it('creates simple score-returning functions', async () => {
+            const llm = createMockLLM(JUDGE_SCORE_FOUR);
+            const evaluator = createQuickEvaluator(llm, TEST_TIMEOUT_MS_SHORT);
+            assert.ok(typeof evaluator.relevance === 'function');
+            assert.ok(typeof evaluator.faithfulness === 'function');
+            assert.ok(typeof evaluator.coherence === 'function');
+            const score = await evaluator.coherence('Test output');
+            assert.ok(typeof score === 'number');
+            assert.ok(score >= 0 && score <= 1);
+        });
+    });
+    describe('createProductionEvaluator', () => {
+        it('creates judge with production defaults', () => {
+            const llm = createMockLLM();
+            const logger = { info: () => { } };
+            const judge = createProductionEvaluator(llm, logger);
+            const config = judge.getConfig();
+            assert.strictEqual(config.timeoutMs, PRODUCTION_TIMEOUT_MS);
+            assert.strictEqual(config.maxRetries, PRODUCTION_MAX_RETRIES);
+            assert.strictEqual(config.evaluator, 'production-judge');
+        });
+        it('allows custom options', () => {
+            const llm = createMockLLM();
+            const logger = {};
+            const customTimeoutMs = PRODUCTION_TIMEOUT_MS * COUNT_TWO;
+            const judge = createProductionEvaluator(llm, logger, {
+                timeoutMs: customTimeoutMs,
+                maxRetries: COUNT_THREE,
+                evaluator: 'my-evaluator',
+            });
+            const config = judge.getConfig();
+            assert.strictEqual(config.timeoutMs, customTimeoutMs);
+            assert.strictEqual(config.maxRetries, COUNT_THREE);
+            assert.strictEqual(config.evaluator, 'my-evaluator');
+        });
+    });
+    // ==========================================================================
+    // Constants
+    // ==========================================================================
+    describe('constants', () => {
+        it('DEFAULT_CONFIG has reasonable values', () => {
+            assert.ok(DEFAULT_CONFIG.timeoutMs >= TEST_TIMEOUT_MS_SHORT);
+            assert.ok(DEFAULT_CONFIG.timeoutMs <= MAX_TIMEOUT_MS);
+            assert.strictEqual(DEFAULT_CONFIG.maxRetries, 0);
+            assert.ok(DEFAULT_CONFIG.evaluator);
+        });
+        it('MAX_TIMEOUT_MS is reasonable', () => {
+            assert.ok(MAX_TIMEOUT_MS >= PRODUCTION_TIMEOUT_MS); // At least 1 minute
+            assert.ok(MAX_TIMEOUT_MS <= PRODUCTION_TIMEOUT_MS * COUNT_TEN); // At most 10 minutes
+        });
+        it('MAX_RETRIES is reasonable', () => {
+            assert.ok(MAX_RETRIES >= 1);
+            assert.ok(MAX_RETRIES <= COUNT_TEN);
+        });
+        it('DEFAULT_HALLUCINATION_THRESHOLD is in (0, 1]', () => {
+            assert.ok(DEFAULT_HALLUCINATION_THRESHOLD > 0);
+            assert.ok(DEFAULT_HALLUCINATION_THRESHOLD <= 1);
+        });
+    });
+    // ==========================================================================
+    // HallucinationGuardrail
+    // ==========================================================================
+    describe('HallucinationGuardrail', () => {
+        const FAITHFULNESS_STEPS = '1. Check factual accuracy\n2. Verify source alignment\n3. Detect fabrication';
+        // Faithful output: faithfulness score close to 1 → hallucination near 0
+        function createFaithfulLLM() {
+            let callIndex = 0;
+            return {
+                generate: async () => {
+                    const text = callIndex === 0 ? FAITHFULNESS_STEPS : 'Score: 5/5. The output faithfully reflects the context.';
+                    callIndex++;
+                    return { text };
+                },
+            };
+        }
+        // Hallucinating output: faithfulness score close to 0 → hallucination near 1
+        function createHallucinatingLLM() {
+            let callIndex = 0;
+            return {
+                generate: async () => {
+                    const text = callIndex === 0 ? FAITHFULNESS_STEPS : 'Score: 1/5. The output contains fabricated information.';
+                    callIndex++;
+                    return { text };
+                },
+            };
+        }
+        it('does not block faithful output below threshold', async () => {
+            const guardrail = new HallucinationGuardrail(createFaithfulLLM(), {
+                threshold: DEFAULT_HALLUCINATION_THRESHOLD,
+                judgeConfig: { timeoutMs: 5000 },
+            });
+            const result = await guardrail.check('What is TypeScript?', 'TypeScript is a typed superset of JavaScript.', ['TypeScript adds static typing to JavaScript.']);
+            assert.ok(result.score >= 0 && result.score <= 1);
+            assert.strictEqual(result.blocked, false);
+        });
+        it('blocks hallucinating output at or above threshold', async () => {
+            const guardrail = new HallucinationGuardrail(createHallucinatingLLM(), {
+                threshold: DEFAULT_HALLUCINATION_THRESHOLD,
+                judgeConfig: { timeoutMs: 5000 },
+            });
+            const result = await guardrail.check('What is TypeScript?', 'TypeScript was invented by Google in 2005.', ['TypeScript was created by Microsoft and released in 2012.']);
+            assert.ok(result.score >= 0 && result.score <= 1);
+            assert.strictEqual(result.blocked, true);
+        });
+        it('uses DEFAULT_HALLUCINATION_THRESHOLD when not specified', () => {
+            const guardrail = new HallucinationGuardrail(createFaithfulLLM());
+            assert.strictEqual(guardrail.blocksAbove, DEFAULT_HALLUCINATION_THRESHOLD);
+        });
+        it('respects custom threshold', () => {
+            const guardrail = new HallucinationGuardrail(createFaithfulLLM(), {
+                threshold: DELTA,
+            });
+            assert.strictEqual(guardrail.blocksAbove, DELTA);
+        });
+        it('throws on invalid threshold (zero)', () => {
+            assert.throws(() => new HallucinationGuardrail(createFaithfulLLM(), { threshold: 0 }), /threshold must be in \(0, 1\]/);
+        });
+        it('throws on invalid threshold (above 1)', () => {
+            assert.throws(() => new HallucinationGuardrail(createFaithfulLLM(), { threshold: 1.1 }), /threshold must be in \(0, 1\]/);
+        });
+        it('allows threshold of exactly 1 (never blocks)', () => {
+            const guardrail = new HallucinationGuardrail(createFaithfulLLM(), {
+                threshold: 1,
+            });
+            assert.strictEqual(guardrail.blocksAbove, 1);
+        });
+        it('score is inverse of faithfulness (0=faithful, 1=hallucinates)', async () => {
+            // Score 5/5 → faithfulness = 1.0 → hallucination = 0.0
+            const guardrail = new HallucinationGuardrail(createFaithfulLLM(), {
+                threshold: 0.5,
+                judgeConfig: { timeoutMs: 5000 },
+            });
+            const result = await guardrail.check('q', 'a', ['ctx']);
+            assert.ok(result.score < PROBABILITY_FIFTY_PERCENT, `Expected low hallucination score for faithful LLM, got ${result.score}`);
+        });
+    });
+    describe('createHallucinationGuardrail', () => {
+        it('returns a HallucinationGuardrail instance', () => {
+            const llm = { generate: async () => ({ text: 'Score: 4/5.' }) };
+            const guardrail = createHallucinationGuardrail(llm, { threshold: PROBABILITY_TWENTY_PERCENT });
+            assert.ok(guardrail instanceof HallucinationGuardrail);
+            assert.strictEqual(guardrail.blocksAbove, PROBABILITY_TWENTY_PERCENT);
+        });
+        it('uses default threshold when options omitted', () => {
+            const llm = { generate: async () => ({ text: 'Score: 4/5.' }) };
+            const guardrail = createHallucinationGuardrail(llm);
+            assert.strictEqual(guardrail.blocksAbove, DEFAULT_HALLUCINATION_THRESHOLD);
+        });
+    });
+});
+//# sourceMappingURL=llm-judge-config.test.js.map

package/dist/lib/judge/llm-judge-config.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-config.test.js","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-config.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EACL,QAAQ,EACR,cAAc,EACd,oBAAoB,EACpB,yBAAyB,EACzB,sBAAsB,EACtB,4BAA4B,EAC5B,cAAc,EACd,+BAA+B,EAC/B,cAAc,EACd,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,GAEnB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,yBAAyB,EACzB,0BAA0B,EAC1B,WAAW,EACX,SAAS,EACT,SAAS,EACT,KAAK,EACL,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,sCAAsC,CAAC;AAE9C,MAAM,mBAAmB,GAAG,yDAAyD,CAAC;AACtF,MAAM,mBAAmB,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,UAAU,KAAK,2BAA2B,CAAC;AAE1F,6FAA6F;AAC7F,SAAS,aAAa,CAAC,QAAgB,gBAAgB;IACrD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAChF,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,CAAC;QAClB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,gGAAgG;AAChG,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,SAAS,sBAAsB,CAAC,SAAiB;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,SAAS,EAAE,CAAC;YACZ,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACzC,CAAC;YACD,4DAA4D;YAC5D,6DAA6D;YAC7D,MAAM,YAAY,GAAG,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACrD,OAAO,EAAE,IAAI,EAAE,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,8BAA8B,EAAE,CAAC;QAC7F,CAAC;KACF,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE;gBAC9B,SAAS,EAAE,qBAAqB;gBAChC,UAAU,EAAE,WAAW;gBACvB,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAC5D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YACnD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAE5B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YACtC,CAAC,EAAE,0BAA0B,CAAC,CAAC;YAE/B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,GAAG,CAAC,EAAE,CAAC,CAAC;YACvD,CAAC,EAAE,0BAA0B,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAE5B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACxC,CAAC,EAAE,0BAA0B,CAAC,CAAC;YAE/B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,CAAC,EAAE,0BAA0B,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,uBAAuB;IACvB,6EAA6E;IAE7E,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YACzD,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;YAC/D,MAAM,CAAC,EAAE,CAAC,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;YACpE,MAAM,CAAC,EAAE,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YACzD,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,qBAAqB;IACrB,6EAA6E;IAE7E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,GAAG,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE;gBAC9B,SAAS,EAAE,oBAAoB;gBAC/B,SAAS,EAAE,YAAY;aACxB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE;gBACnD,KAAK,EAAE,aAAa;gBACpB,MAAM,EAAE,gCAAgC;gBACxC,OAAO,EAAE,CAAC,wCAAwC,CAAC;aACpD,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAC5D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACpE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,GAAG,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAC1C,aAAa,EACb,gCAAgC,EAChC,CAAC,wCAAwC,CAAC,CAC3C,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,GAAG,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,oBAAoB,CAC7C,aAAa,EACb,gCAAgC,EAChC,CAAC,wCAAwC,CAAC,CAC3C,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,GAAG,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAC1C,2DAA2D,CAC5D,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,iBAAiB;IACjB,6EAA6E;IAE7E,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,GAAG,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE;gBAC9B,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE;gBACnD,KAAK,EAAE,YAAY;gBACnB,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,GAAG,GAAG,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE;gBAC9B,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,GAAG;aACf,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAC9E,YAAY,CACb,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,MAAM,GAAG,GAAgB;gBACvB,QAAQ,EAAE,KAAK,IAAI,EAAE;oBACnB,QAAQ,EAAE,CAAC;oBACX,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;gBACnC,CAAC;aACF,CAAC;YACF,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YAEnE,MAAM,MAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAC9E,eAAe,CAChB,CAAC;YAEF,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,qBAAqB;IACrB,6EAA6E;IAE7E,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,MAAM,GAAmB;gBAC7B,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aAClD,CAAC;YAEF,MAAM,GAAG,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,EAAE,CAAC,CAAC;YAE9E,MAAM,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAE/E,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAmB;gBAC7B,KAAK,EAAE,GAAG,EAAE,GAAE,CAAC;gBACf,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aAChD,CAAC;YAEF,MAAM,GAAG,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE;gBAC9B,MAAM;gBACN,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,GAAG;aACf,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAE/E,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,MAAM,GAAmB;gBAC7B,KAAK,EAAE,GAAG,EAAE,GAAE,CAAC;gBACf,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aAClD,CAAC;YAEF,MAAM,GAAG,GAAgB;gBACvB,QAAQ,EAAE,KAAK,IAAI,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;aACpD,CAAC;YACF,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YAE5D,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACjF,CAAC;YAAC,MAAM,CAAC;gBACP,WAAW;YACb,CAAC;YAED,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,oBAAoB;IACpB,6EAA6E;IAE7E,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE,CAAC,CAAC;YAEvE,MAAM,CAAC,EAAE,CAAC,KAAK,YAAY,QAAQ,CAAC,CAAC;YACrC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,GAAG,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;YAEnE,MAAM,CAAC,EAAE,CAAC,OAAO,SAAS,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC;YACrD,MAAM,CAAC,EAAE,CAAC,OAAO,SAAS,CAAC,YAAY,KAAK,UAAU,CAAC,CAAC;YACxD,MAAM,CAAC,EAAE,CAAC,OAAO,SAAS,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC;YAErD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACvD,MAAM,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;YACrC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAmB,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;YAElD,MAAM,KAAK,GAAG,yBAAyB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YAEjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAC5D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC/B,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAmB,EAAE,CAAC;YAElC,MAAM,eAAe,GAAG,qBAAqB,GAAG,SAAS,CAAC;YAC1D,MAAM,KAAK,GAAG,yBAAyB,CAAC,GAAG,EAAE,MAAM,EAAE;gBACnD,SAAS,EAAE,eAAe;gBAC1B,UAAU,EAAE,WAAW;gBACvB,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YAEjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YACtD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YACnD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,YAAY;IACZ,6EAA6E;IAE7E,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,SAAS,IAAI,qBAAqB,CAAC,CAAC;YAC7D,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,SAAS,IAAI,cAAc,CAAC,CAAC;YACtD,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,EAAE,CAAC,cAAc,IAAI,qBAAqB,CAAC,CAAC,CAAC,oBAAoB;YACxE,MAAM,CAAC,EAAE,CAAC,cAAc,IAAI,qBAAqB,GAAG,SAAS,CAAC,CAAC,CAAC,qBAAqB;QACvF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;YAC5B,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,CAAC,EAAE,CAAC,+BAA+B,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,CAAC,+BAA+B,IAAI,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,yBAAyB;IACzB,6EAA6E;IAE7E,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,MAAM,kBAAkB,GAAG,8EAA8E,CAAC;QAE1G,wEAAwE;QACxE,SAAS,iBAAiB;YACxB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,OAAO;gBACL,QAAQ,EAAE,KAAK,IAAI,EAAE;oBACnB,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,yDAAyD,CAAC;oBAC9G,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,EAAE,CAAC;gBAClB,CAAC;aACF,CAAC;QACJ,CAAC;QAED,6EAA6E;QAC7E,SAAS,sBAAsB;YAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,OAAO;gBACL,QAAQ,EAAE,KAAK,IAAI,EAAE;oBACnB,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,yDAAyD,CAAC;oBAC9G,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,EAAE,CAAC;gBAClB,CAAC;aACF,CAAC;QACJ,CAAC;QAED,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,EAAE;gBAChE,SAAS,EAAE,+BAA+B;gBAC1C,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;aACjC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,CAClC,qBAAqB,EACrB,+CAA+C,EAC/C,CAAC,8CAA8C,CAAC,CACjD,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,sBAAsB,EAAE,EAAE;gBACrE,SAAS,EAAE,+BAA+B;gBAC1C,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;aACjC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,CAClC,qBAAqB,EACrB,4CAA4C,EAC5C,CAAC,2DAA2D,CAAC,CAC9D,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;YACjE,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAClE,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,EAAE;gBAChE,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,MAAM,CACX,GAAG,EAAE,CAAC,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EACvE,+BAA+B,CAChC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,MAAM,CACX,GAAG,EAAE,CAAC,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EACzE,+BAA+B,CAChC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,EAAE;gBAChE,SAAS,EAAE,CAAC;aACb,CAAC,CAAC;YACH,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,uDAAuD;YACvD,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,EAAE;gBAChE,SAAS,EAAE,GAAG;gBACd,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;aACjC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,EAAE,CACP,MAAM,CAAC,KAAK,GAAG,yBAAyB,EACxC,0DAA0D,MAAM,CAAC,KAAK,EAAE,CACzE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,8BAA8B,EAAE,GAAG,EAAE;QAC5C,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,GAAG,GAAgB,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,SAAS,GAAG,4BAA4B,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;YAC/F,MAAM,CAAC,EAAE,CAAC,SAAS,YAAY,sBAAsB,CAAC,CAAC;YACvD,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,WAAW,EAAE,0BAA0B,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,GAAG,GAAgB,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,SAAS,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;YACpD,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/lib/judge/llm-judge-constants.d.ts

@@ -0,0 +1,111 @@
+/**
+ * LLM-as-Judge Module Constants
+ *
+ * Centralized constants for all LLM evaluation utilities.
+ * Import from here rather than defining magic numbers inline.
+ */
+/** Maximum text length per field (10KB) */
+export declare const MAX_TEXT_LENGTH = 10000;
+/** Maximum context array length */
+export declare const MAX_CONTEXT_ITEMS = 20;
+/** Maximum statements to process in QAG pattern */
+export declare const MAX_STATEMENTS = 20;
+/** Maximum logprob entries to process — prevents memory exhaustion from unbounded arrays */
+export declare const MAX_LOGPROBS = 1000;
+/** Minimum total probability for logprobs-based score normalization.
+ * Below this, text extraction is more reliable than probability weighting. */
+export declare const LOGPROBS_CONFIDENCE_THRESHOLD = 0.1;
+/** Default timeout for LLM calls (30 seconds) */
+export declare const DEFAULT_LLM_TIMEOUT_MS = 30000;
+/** Maximum JSON nesting depth */
+export declare const MAX_JSON_DEPTH = 5;
+/** G-Eval minimum score (inclusive) */
+export declare const G_EVAL_MIN_SCORE = 1;
+/** G-Eval maximum score (inclusive) */
+export declare const G_EVAL_MAX_SCORE = 5;
+/** G-Eval default/middle score for fallback scenarios */
+export declare const G_EVAL_DEFAULT_SCORE = 3;
+/** G-Eval valid score values array */
+export declare const G_EVAL_VALID_SCORES: readonly [1, 2, 3, 4, 5];
+/** G-Eval score range for normalization (max - min) */
+export declare const G_EVAL_SCORE_RANGE: number;
+/** Default temperature for evaluation LLM calls (slight variation allowed) */
+export declare const LLM_TEMPERATURE_EVALUATION = 0.1;
+/** Minimum evaluation steps to generate in G-Eval */
+export declare const G_EVAL_MIN_STEPS = 3;
+/** Maximum evaluation steps to generate in G-Eval */
+export declare const G_EVAL_MAX_STEPS = 5;
+/** Characters of tail text to search for contextual score match in extractScoreFromText */
+export declare const SCORE_FALLBACK_WINDOW = 100;
+/** Default temperature for deterministic LLM calls (e.g., extraction, answering) */
+export declare const LLM_TEMPERATURE_DETERMINISTIC = 0;
+/** Minimum statement length to be considered valid for QAG extraction */
+export declare const MIN_STATEMENT_LENGTH = 10;
+/** Maximum characters of LLM response to include in warning preview logs */
+export declare const QAG_RESPONSE_PREVIEW_LENGTH = 200;
+/** Minimum valid normalized score */
+export declare const NORMALIZED_SCORE_MIN = 0;
+/** Maximum valid normalized score */
+export declare const NORMALIZED_SCORE_MAX = 1;
+/** Default circuit breaker failure threshold */
+export declare const DEFAULT_CIRCUIT_BREAKER_THRESHOLD = 5;
+/** Default circuit breaker reset timeout in milliseconds */
+export declare const DEFAULT_CIRCUIT_BREAKER_RESET_MS = 30000;
+/** Maximum exponent for backoff calculation */
+export declare const MAX_BACKOFF_EXPONENT = 5;
+export declare const MAX_BACKOFF_MS: number;
+/** Maximum canary score for poor/hallucination detection cases */
+export declare const CANARY_SCORE_LOW_MAX = 0.3;
+/** Maximum canary score for very low/off-topic detection cases */
+export declare const CANARY_SCORE_VERY_LOW_MAX = 0.2;
+/** Default weight applied to the syntax stage in the aggregate code evaluation score */
+export declare const CODE_EVAL_SYNTAX_WEIGHT = 0.3;
+/** Default weight applied to the semantic stage in the aggregate code evaluation score */
+export declare const CODE_EVAL_SEMANTIC_WEIGHT = 0.7;
+/** Syntax weight for Python — lower weight since ast.parse is comprehensive but semantics dominate */
+export declare const CODE_EVAL_SYNTAX_WEIGHT_PYTHON = 0.2;
+/** Syntax weight for Go — slightly higher weight since gofmt validates both format and parse */
+export declare const CODE_EVAL_SYNTAX_WEIGHT_GO = 0.35;
+/** Syntax score when code passes syntax validation */
+export declare const CODE_EVAL_SYNTAX_SCORE_PASS = 1;
+/** Syntax score when code fails syntax validation */
+export declare const CODE_EVAL_SYNTAX_SCORE_FAIL = 0;
+/** Default programming language for code evaluation */
+export declare const CODE_EVAL_DEFAULT_LANGUAGE = "typescript";
+/** Default evaluation criteria for code evaluation */
+export declare const CODE_EVAL_DEFAULT_CRITERIA: string[];
+/** Score when runtime error patterns are detected in code */
+export declare const CODE_EVAL_RUNTIME_ERROR_SCORE_DETECTED = 0;
+/** Score when no runtime error patterns are detected in code */
+export declare const CODE_EVAL_RUNTIME_ERROR_SCORE_CLEAN = 1;
+/** Default weight applied to runtimeErrorScore in aggregate score (0 = no effect, backward compat) */
+export declare const CODE_EVAL_RUNTIME_ERROR_WEIGHT_DEFAULT = 0;
+/** Maximum possible population stdDev for a bounded [0, 1] score distribution */
+export declare const PANEL_MAX_STD_DEV = 0.5;
+/** Default hallucination guardrail threshold (blocks when score >= this value) */
+export declare const DEFAULT_HALLUCINATION_THRESHOLD = 0.15;
+/** Sample rate for meta-evaluation (10% of evaluations) */
+export declare const META_EVAL_SAMPLE_RATE = 0.1;
+/** Maximum allowed timeout */
+export declare const MAX_TIMEOUT_MS = 300000;
+/** Maximum allowed retries for LLM judge calls */
+export declare const MAX_RETRIES = 5;
+/** Default timeout for production evaluators (1 minute) */
+export declare const PRODUCTION_TIMEOUT_MS: number;
+/** Default retries for production evaluators */
+export declare const PRODUCTION_MAX_RETRIES = 2;
+/** Maximum bytes per batch (1MB) to prevent memory exhaustion */
+export declare const MAX_BATCH_BYTES: number;
+/** Maximum bytes per single evaluation (100KB) */
+export declare const MAX_SINGLE_EVALUATION_BYTES: number;
+/** Maximum dedup cache entries before LRU eviction */
+export declare const MAX_DEDUP_CACHE_SIZE = 10000;
+/** Maximum number of metadata keys allowed per evaluation (CR-M2) */
+export declare const MAX_METADATA_KEYS = 50;
+/** Maximum length of a metadata key (CR-M2) */
+export declare const MAX_METADATA_KEY_LENGTH = 128;
+/** File name for persisted hook stats */
+export declare const STATS_FILE_NAME = "hook-stats.json";
+/** File name for persisted webhook configs */
+export declare const WEBHOOK_CONFIGS_FILE_NAME = "webhook-configs.json";
+//# sourceMappingURL=llm-judge-constants.d.ts.map

package/dist/lib/judge/llm-judge-constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-constants.d.ts","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-constants.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAaH,2CAA2C;AAC3C,eAAO,MAAM,eAAe,QAAQ,CAAC;AAErC,mCAAmC;AACnC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,mDAAmD;AACnD,eAAO,MAAM,cAAc,KAAK,CAAC;AAEjC,4FAA4F;AAC5F,eAAO,MAAM,YAAY,OAAO,CAAC;AAEjC;8EAC8E;AAC9E,eAAO,MAAM,6BAA6B,MAAM,CAAC;AAEjD,iDAAiD;AACjD,eAAO,MAAM,sBAAsB,QAAQ,CAAC;AAE5C,iCAAiC;AACjC,eAAO,MAAM,cAAc,IAAI,CAAC;AAMhC,uCAAuC;AACvC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,uCAAuC;AACvC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,yDAAyD;AACzD,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAKtC,sCAAsC;AACtC,eAAO,MAAM,mBAAmB,0BAMtB,CAAC;AAEX,uDAAuD;AACvD,eAAO,MAAM,kBAAkB,QAAsC,CAAC;AAEtE,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,qDAAqD;AACrD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,qDAAqD;AACrD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,2FAA2F;AAC3F,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAMzC,oFAAoF;AACpF,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,yEAAyE;AACzE,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,4EAA4E;AAC5E,eAAO,MAAM,2BAA2B,MAAM,CAAC;AAM/C,qCAAqC;AACrC,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,qCAAqC;AACrC,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,gDAAgD;AAChD,eAAO,MAAM,iCAAiC,IAAI,CAAC;AAEnD,4DAA4D;AAC5D,eAAO,MAAM,gCAAgC,QAAQ,CAAC;AAEtD,+CAA+C;AAC/C,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAKtC,eAAO,MAAM,cAAc,QAAkC,CAAC;AAM9D,kEAAkE;AAClE,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC,kEAAkE;AAClE,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAM7C,wFAAwF;AACxF,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0FAA0F;AAC1F,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,sGAAsG;AACtG,eAAO,MAAM,8BAA8B,MAAM,CAAC;AAElD,gGAAgG;AAChG,eAAO,MAAM,0BAA0B,OAAO,CAAC;AAE/C,sDAAsD;AACtD,eAAO,MAAM,2BAA2B,IAAM,CAAC;AAE/C,qDAAqD;AACrD,eAAO,MAAM,2BAA2B,IAAM,CAAC;AAE/C,uDAAuD;AACvD,eAAO,MAAM,0BAA0B,eAAe,CAAC;AAEvD,sDAAsD;AACtD,eAAO,MAAM,0BAA0B,UAAoD,CAAC;AAE5F,6DAA6D;AAC7D,eAAO,MAAM,sCAAsC,IAAM,CAAC;AAE1D,gEAAgE;AAChE,eAAO,MAAM,mCAAmC,IAAM,CAAC;AAEvD,sGAAsG;AACtG,eAAO,MAAM,sCAAsC,IAAI,CAAC;AAMxD,iFAAiF;AACjF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAMrC,kFAAkF;AAClF,eAAO,MAAM,+BAA+B,OAAO,CAAC;AAEpD,2DAA2D;AAC3D,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,8BAA8B;AAC9B,eAAO,MAAM,cAAc,SAAU,CAAC;AAEtC,kDAAkD;AAClD,eAAO,MAAM,WAAW,IAAI,CAAC;AAE7B,2DAA2D;AAC3D,eAAO,MAAM,qBAAqB,QAAkC,CAAC;AAErE,gDAAgD;AAChD,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAMxC,iEAAiE;AACjE,eAAO,MAAM,eAAe,QAAsB,CAAC;AAEnD,kDAAkD;AAClD,eAAO,MAAM,2BAA2B,QAAgC,CAAC;AAEzE,sDAAsD;AACtD,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAE1C,qEAAqE;AACrE,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,+CAA+C;AAC/C,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,yCAAyC;AACzC,eAAO,MAAM,eAAe,oBAAoB,CAAC;AAEjD,8CAA8C;AAC9C,eAAO,MAAM,yBAAyB,yBAAyB,CAAC"}