observability-toolkit 1.8.5 → 2.1.0

package/dist/lib/judge/llm-judge-otel.test.js

@@ -0,0 +1,91 @@
+/**
+ * Tests for OTel evaluation event emission from LLM-as-Judge functions.
+ *
+ * These tests verify that gEval, qagEvaluate, and panelEvaluation
+ * emit evaluation events without breaking evaluation flow.
+ * OTel is disabled in test environment, so events are no-ops.
+ */
+import { describe, it } from 'vitest';
+import assert from 'node:assert';
+import { gEval, qagEvaluate, panelEvaluation, } from './llm-as-judge.js';
+import { BINARY_SCALE_EXCELLENT_THRESHOLD, COHERENCE_WARNING, SCORE_BAND_EXCELLENT, SCORE_BAND_GOOD, } from '../quality/quality-constants.js';
+import { COUNT_TWO } from '../../test-helpers/test-constants.js';
+const MOCK_STEPS = '1. Evaluate relevance\n2. Check accuracy\n3. Assess completeness';
+/** Mock LLM that returns evaluation steps on first call, then a fixed score response */
+function createMockLLM(scoreText) {
+    let callIndex = 0;
+    return {
+        generate: async () => {
+            const text = callIndex === 0 ? MOCK_STEPS : scoreText;
+            callIndex++;
+            return { text, logprobs: undefined };
+        },
+    };
+}
+describe('LLM-as-Judge OTel event emission', () => {
+    describe('gEval emits evaluation event', () => {
+        it('completes successfully with OTel emission (disabled = no-op)', async () => {
+            const llm = createMockLLM('Score: 4\nThe response is relevant.');
+            const config = {
+                name: 'relevance',
+                criteria: 'Evaluate relevance of the response',
+                evaluationParams: ['input', 'output'],
+            };
+            const testCase = {
+                input: 'What is 2+2?',
+                output: '4',
+            };
+            const result = await gEval(llm, config, testCase);
+            assert.ok(typeof result.score === 'number');
+            assert.ok(result.score >= 0 && result.score <= 1);
+        });
+    });
+    describe('qagEvaluate emits evaluation event', () => {
+        it('completes successfully with OTel emission (disabled = no-op)', async () => {
+            // Mock LLM: first call extracts statements, second generates question, third answers
+            let callCount = 0;
+            const llm = {
+                generate: async () => {
+                    callCount++;
+                    if (callCount === 1) {
+                        // extractStatements: return a simple statement
+                        return { text: '["The answer is 4"]', logprobs: undefined };
+                    }
+                    else if (callCount === COUNT_TWO) {
+                        // generateVerificationQuestion
+                        return { text: 'Is the answer 4?', logprobs: undefined };
+                    }
+                    else {
+                        // answerQuestion
+                        return { text: 'yes', logprobs: undefined };
+                    }
+                },
+            };
+            const score = await qagEvaluate(llm, 'What is 2+2?', 'The answer is 4', ['Basic arithmetic: 2+2=4']);
+            assert.ok(typeof score === 'number');
+            assert.ok(score >= 0 && score <= 1);
+        });
+    });
+    describe('panelEvaluation emits evaluation event', () => {
+        it('completes successfully with OTel emission (disabled = no-op)', async () => {
+            const evaluators = [
+                async () => SCORE_BAND_GOOD,
+                async () => SCORE_BAND_EXCELLENT,
+                async () => BINARY_SCALE_EXCELLENT_THRESHOLD,
+            ];
+            const testCase = {
+                input: 'What is 2+2?',
+                output: '4',
+            };
+            const result = await panelEvaluation(evaluators, testCase);
+            assert.strictEqual(result.median, SCORE_BAND_GOOD);
+        });
+        it('handles single evaluator', async () => {
+            const evaluators = [async () => COHERENCE_WARNING];
+            const testCase = { input: 'test', output: 'test' };
+            const result = await panelEvaluation(evaluators, testCase);
+            assert.strictEqual(result.median, COHERENCE_WARNING);
+        });
+    });
+});
+//# sourceMappingURL=llm-judge-otel.test.js.map

package/dist/lib/judge/llm-judge-otel.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-otel.test.js","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-otel.test.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EACL,KAAK,EACL,WAAW,EACX,eAAe,GAIhB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,gCAAgC,EAChC,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,GAChB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAEjE,MAAM,UAAU,GAAG,kEAAkE,CAAC;AAEtF,wFAAwF;AACxF,SAAS,aAAa,CAAC,SAAiB;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO;QACL,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;YACtD,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IAChD,QAAQ,CAAC,8BAA8B,EAAE,GAAG,EAAE;QAC5C,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,GAAG,GAAG,aAAa,CAAC,qCAAqC,CAAC,CAAC;YACjE,MAAM,MAAM,GAAgB;gBAC1B,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,oCAAoC;gBAC9C,gBAAgB,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;aACtC,CAAC;YACF,MAAM,QAAQ,GAAa;gBACzB,KAAK,EAAE,cAAc;gBACrB,MAAM,EAAE,GAAG;aACZ,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YAElD,MAAM,CAAC,EAAE,CAAC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC;YAC5C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAClD,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,qFAAqF;YACrF,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,GAAG,GAAgB;gBACvB,QAAQ,EAAE,KAAK,IAAI,EAAE;oBACnB,SAAS,EAAE,CAAC;oBACZ,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;wBACpB,+CAA+C;wBAC/C,OAAO,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;oBAC9D,CAAC;yBAAM,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;wBACnC,+BAA+B;wBAC/B,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;oBAC3D,CAAC;yBAAM,CAAC;wBACN,iBAAiB;wBACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;oBAC9C,CAAC;gBACH,CAAC;aACF,CAAC;YAEF,MAAM,KAAK,GAAG,MAAM,WAAW,CAC7B,GAAG,EACH,cAAc,EACd,iBAAiB,EACjB,CAAC,yBAAyB,CAAC,CAC5B,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;YACrC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wCAAwC,EAAE,GAAG,EAAE;QACtD,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,UAAU,GAAG;gBACjB,KAAK,IAAI,EAAE,CAAC,eAAe;gBAC3B,KAAK,IAAI,EAAE,CAAC,oBAAoB;gBAChC,KAAK,IAAI,EAAE,CAAC,gCAAgC;aAC7C,CAAC;YACF,MAAM,QAAQ,GAAa;gBACzB,KAAK,EAAE,cAAc;gBACrB,MAAM,EAAE,GAAG;aACZ,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAE3D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,iBAAiB,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAa,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YAE7D,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC3D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/lib/judge/llm-judge-qag.d.ts

@@ -0,0 +1,38 @@
+/**
+ * QAG (Question-Answer Generation) Pattern
+ *
+ * Decomposes evaluation into atomic statements, generates verification
+ * questions, and answers them against context for faithfulness scoring.
+ *
+ * @see llm-as-judge.ts for core types
+ */
+import type { LLMProvider } from './llm-as-judge.js';
+import { LLM_TEMPERATURE_DETERMINISTIC, MIN_STATEMENT_LENGTH } from './llm-judge-constants.js';
+export { LLM_TEMPERATURE_DETERMINISTIC, MIN_STATEMENT_LENGTH };
+/**
+ * Extract atomic statements from LLM output.
+ */
+export declare function extractStatements(llm: LLMProvider, output: string, timeoutMs?: number): Promise<string[]>;
+/**
+ * Generate verification question for a statement.
+ */
+export declare function generateVerificationQuestion(llm: LLMProvider, statement: string, timeoutMs?: number): Promise<string>;
+/**
+ * Answer verification question using provided context.
+ */
+export declare function answerQuestion(llm: LLMProvider, question: string, context: string[], timeoutMs?: number): Promise<'yes' | 'no' | 'unknown'>;
+/** Verification modes for QAG evaluation */
+export type QagVerificationMode = 'faithfulness' | 'fabrication';
+/**
+ * QAG (Question-Answer Generation) evaluation.
+ * Decomposes evaluation into atomic yes/no questions.
+ *
+ * @param options.verificationMode - 'faithfulness' (default): score = fraction of statements
+ *   verified against context (higher = more faithful). 'fabrication': score = fraction of
+ *   statements NOT grounded in context (higher = more hallucinated).
+ */
+export declare function qagEvaluate(llm: LLMProvider, input: string, output: string, context: string[], options?: {
+    timeoutMs?: number;
+    verificationMode?: QagVerificationMode;
+}): Promise<number>;
+//# sourceMappingURL=llm-judge-qag.d.ts.map

package/dist/lib/judge/llm-judge-qag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-qag.d.ts","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-qag.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAarD,OAAO,EAIL,6BAA6B,EAC7B,oBAAoB,EAErB,MAAM,0BAA0B,CAAC;AAYlC,OAAO,EAAE,6BAA6B,EAAE,oBAAoB,EAAE,CAAC;AAO/D;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,WAAW,EAChB,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,MAAM,EAAE,CAAC,CA0CnB;AAED;;GAEG;AACH,wBAAsB,4BAA4B,CAChD,GAAG,EAAE,WAAW,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EAAE,EACjB,SAAS,GAAE,MAA+B,GACzC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,SAAS,CAAC,CA8BnC;AAED,4CAA4C;AAC5C,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,aAAa,CAAC;AAEjE;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,WAAW,EAChB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,gBAAgB,CAAC,EAAE,mBAAmB,CAAA;CAAE,GACvE,OAAO,CAAC,MAAM,CAAC,CAqGjB"}

package/dist/lib/judge/llm-judge-qag.js

@@ -0,0 +1,205 @@
+/**
+ * QAG (Question-Answer Generation) Pattern
+ *
+ * Decomposes evaluation into atomic statements, generates verification
+ * questions, and answers them against context for faithfulness scoring.
+ *
+ * @see llm-as-judge.ts for core types
+ */
+import { PERCENT_MULTIPLIER, } from '../core/units.js';
+import { createLogger } from '../core/logger.js';
+import { sanitizeForPrompt, sanitizeContextArray, safeJSONParse, withTimeout, hashInput, shouldLog, } from './llm-judge-security.js';
+import { MAX_TEXT_LENGTH, MAX_STATEMENTS, DEFAULT_LLM_TIMEOUT_MS, LLM_TEMPERATURE_DETERMINISTIC, MIN_STATEMENT_LENGTH, QAG_RESPONSE_PREVIEW_LENGTH, } from './llm-judge-constants.js';
+import { emitEvaluationEvent } from '../observability/instrumentation.js';
+import { STEP_SCALE_ADEQUATE_THRESHOLD } from '../quality/quality-constants.js';
+import sbd from 'sbd';
+const llmJudgeLog = createLogger('llm-judge');
+// ============================================================================
+// QAG Constants
+// (Defined in llm-judge-constants.ts)
+// ============================================================================
+export { LLM_TEMPERATURE_DETERMINISTIC, MIN_STATEMENT_LENGTH };
+const MIN_VERIFICATION_SUCCESS_RATE = STEP_SCALE_ADEQUATE_THRESHOLD;
+// ============================================================================
+// Functions
+// ============================================================================
+/**
+ * Extract atomic statements from LLM output.
+ */
+export async function extractStatements(llm, output, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const sanitizedOutput = sanitizeForPrompt(output);
+    const prompt = `
+Extract all factual claims from the following text as a JSON array of strings.
+Each claim should be a single, atomic statement that can be verified independently.
+
+Text: ${sanitizedOutput}
+
+Return ONLY a JSON array, e.g.: ["claim 1", "claim 2", "claim 3"]
+`;
+    const response = await withTimeout((signal) => llm.generate(prompt, { temperature: LLM_TEMPERATURE_DETERMINISTIC, signal }), timeoutMs);
+    try {
+        const parsed = safeJSONParse(response.text);
+        if (!Array.isArray(parsed)) {
+            throw new Error(`Expected array from statement extraction, got ${typeof parsed}`);
+        }
+        return parsed
+            .filter((item) => typeof item === 'string' && item.trim().length > 0)
+            .slice(0, MAX_STATEMENTS);
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        const responsePreview = response.text.length > QAG_RESPONSE_PREVIEW_LENGTH
+            ? response.text.slice(0, QAG_RESPONSE_PREVIEW_LENGTH) + '...'
+            : response.text;
+        if (shouldLog('warn')) {
+            llmJudgeLog.warn('Statement extraction JSON parse failed, using sentence fallback', {
+                error: errorMessage,
+                responsePreview,
+                outputLength: output.length,
+            });
+        }
+    }
+    return sbd.sentences(output.slice(0, MAX_TEXT_LENGTH))
+        .map(s => s.trim())
+        .filter(s => s.length > MIN_STATEMENT_LENGTH)
+        .slice(0, MAX_STATEMENTS);
+}
+/**
+ * Generate verification question for a statement.
+ */
+export async function generateVerificationQuestion(llm, statement, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const sanitizedStatement = sanitizeForPrompt(statement);
+    const prompt = `
+Convert this statement into a yes/no question that can verify its accuracy:
+
+Statement: ${sanitizedStatement}
+
+Return ONLY the question, nothing else.
+`;
+    const response = await withTimeout((signal) => llm.generate(prompt, { temperature: LLM_TEMPERATURE_DETERMINISTIC, signal }), timeoutMs);
+    return response.text.trim();
+}
+/**
+ * Answer verification question using provided context.
+ */
+export async function answerQuestion(llm, question, context, timeoutMs = DEFAULT_LLM_TIMEOUT_MS) {
+    const sanitizedQuestion = sanitizeForPrompt(question);
+    const sanitizedContext = sanitizeContextArray(context);
+    const prompt = `
+Based ONLY on the following context, answer the question with "yes", "no", or "unknown".
+
+Context:
+${sanitizedContext.join('\n\n')}
+
+Question: ${sanitizedQuestion}
+
+Answer (yes/no/unknown):
+`;
+    const response = await withTimeout((signal) => llm.generate(prompt, { temperature: LLM_TEMPERATURE_DETERMINISTIC, signal }), timeoutMs);
+    const normalized = response.text.trim().toLowerCase();
+    const yesMatch = /\b(yes|yeah|correct|true|affirmative)\b/i.test(normalized);
+    const noMatch = /\b(no|nope|incorrect|false|negative)\b/i.test(normalized);
+    if (yesMatch && noMatch) {
+        return 'unknown';
+    }
+    if (yesMatch)
+        return 'yes';
+    if (noMatch)
+        return 'no';
+    return 'unknown';
+}
+/**
+ * QAG (Question-Answer Generation) evaluation.
+ * Decomposes evaluation into atomic yes/no questions.
+ *
+ * @param options.verificationMode - 'faithfulness' (default): score = fraction of statements
+ *   verified against context (higher = more faithful). 'fabrication': score = fraction of
+ *   statements NOT grounded in context (higher = more hallucinated).
+ */
+export async function qagEvaluate(llm, input, output, context, options) {
+    const evalStartTime = performance.now();
+    const timeoutMs = options?.timeoutMs ?? DEFAULT_LLM_TIMEOUT_MS;
+    const mode = options?.verificationMode ?? 'faithfulness';
+    const evalName = mode === 'fabrication' ? 'hallucination' : 'faithfulness';
+    try {
+        const statements = await extractStatements(llm, output, timeoutMs);
+        if (statements.length === 0) {
+            return mode === 'fabrication' ? 0 : 1;
+        }
+        const questionResults = await Promise.allSettled(statements.map(s => generateVerificationQuestion(llm, s, timeoutMs)));
+        const successfulQuestions = [];
+        for (let i = 0; i < questionResults.length; i++) {
+            const result = questionResults[i];
+            if (result.status === 'fulfilled') {
+                successfulQuestions.push({ question: result.value, index: i });
+            }
+        }
+        if (successfulQuestions.length === 0) {
+            throw new Error('QAG evaluation failed: no verification questions generated');
+        }
+        const answerResults = await Promise.allSettled(successfulQuestions.map(({ question }) => answerQuestion(llm, question, context, timeoutMs)));
+        let yesCount = 0;
+        let noCount = 0;
+        let successfulAnswers = 0;
+        for (const result of answerResults) {
+            if (result.status === 'fulfilled') {
+                successfulAnswers++;
+                if (result.value === 'yes') {
+                    yesCount++;
+                }
+                else if (result.value === 'no') {
+                    noCount++;
+                }
+            }
+        }
+        if (successfulAnswers === 0) {
+            throw new Error('QAG evaluation failed: no verification answers obtained');
+        }
+        const totalAttempts = answerResults.length;
+        const successRate = successfulAnswers / totalAttempts;
+        if (successRate < MIN_VERIFICATION_SUCCESS_RATE) {
+            throw new Error(`QAG evaluation unreliable: only ${successfulAnswers}/${totalAttempts} verifications succeeded (${Math.round(successRate * PERCENT_MULTIPLIER)}%)`);
+        }
+        let score;
+        let explanation;
+        if (mode === 'fabrication') {
+            // Fabrication: fraction of statements NOT grounded in context (higher = more hallucinated).
+            // Pre-condition: successfulAnswers > 0 (guarded by the early throw above).
+            // The successRate multiplier penalizes low-confidence results: when many verifications
+            // fail, we have less evidence that detected fabrications are real, so the score is
+            // discounted toward zero rather than inflated by unreliable verification attempts.
+            const fabricatedCount = noCount;
+            score = (fabricatedCount / successfulAnswers) * successRate;
+            explanation = `${fabricatedCount}/${successfulAnswers} statements fabricated (${totalAttempts} attempted, ${statements.length} extracted)`;
+        }
+        else {
+            // Faithfulness: fraction of statements verified in context (higher = more faithful)
+            const rawScore = yesCount / successfulAnswers;
+            score = rawScore * successRate;
+            explanation = `${yesCount}/${successfulAnswers} statements verified (${totalAttempts} attempted, ${statements.length} extracted)`;
+        }
+        emitEvaluationEvent({
+            evaluationName: evalName,
+            scoreValue: score,
+            explanation,
+            evaluator: 'qag',
+            evaluatorType: 'llm',
+            durationMs: performance.now() - evalStartTime,
+            inputHash: hashInput(input, output),
+        });
+        return score;
+    }
+    catch (error) {
+        emitEvaluationEvent({
+            evaluationName: evalName,
+            evaluator: 'qag',
+            evaluatorType: 'llm',
+            durationMs: performance.now() - evalStartTime,
+            inputHash: hashInput(input, output),
+            errorType: error instanceof Error ? error.name : 'Error',
+            errorMessage: error instanceof Error ? error.message : String(error),
+        });
+        throw error;
+    }
+}
+//# sourceMappingURL=llm-judge-qag.js.map

package/dist/lib/judge/llm-judge-qag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-qag.js","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-qag.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,aAAa,EACb,WAAW,EACX,SAAS,EACT,SAAS,GACV,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,6BAA6B,EAC7B,oBAAoB,EACpB,2BAA2B,GAC5B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAChF,OAAO,GAAG,MAAM,KAAK,CAAC;AAEtB,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AAE9C,+EAA+E;AAC/E,gBAAgB;AAChB,sCAAsC;AACtC,+EAA+E;AAE/E,OAAO,EAAE,6BAA6B,EAAE,oBAAoB,EAAE,CAAC;AAC/D,MAAM,6BAA6B,GAAG,6BAA6B,CAAC;AAEpE,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAgB,EAChB,MAAc,EACd,YAAoB,sBAAsB;IAE1C,MAAM,eAAe,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG;;;;QAIT,eAAe;;;CAGtB,CAAC;IACA,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,6BAA6B,EAAE,MAAM,EAAE,CAAC,EACxF,SAAS,CACV,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,iDAAiD,OAAO,MAAM,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,OAAO,MAAM;aACV,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAC/B,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;aACpD,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5E,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,2BAA2B;YACxE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,GAAG,KAAK;YAC7D,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAClB,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,WAAW,CAAC,IAAI,CAAC,iEAAiE,EAAE;gBAClF,KAAK,EAAE,YAAY;gBACnB,eAAe;gBACf,YAAY,EAAE,MAAM,CAAC,MAAM;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;SACnD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,oBAAoB,CAAC;SAC5C,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,GAAgB,EAChB,SAAiB,EACjB,YAAoB,sBAAsB;IAE1C,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG;;;aAGJ,kBAAkB;;;CAG9B,CAAC;IACA,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,6BAA6B,EAAE,MAAM,EAAE,CAAC,EACxF,SAAS,CACV,CAAC;IACF,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,QAAgB,EAChB,OAAiB,EACjB,YAAoB,sBAAsB;IAE1C,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG;;;;EAIf,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC;;YAEnB,iBAAiB;;;CAG5B,CAAC;IACA,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,6BAA6B,EAAE,MAAM,EAAE,CAAC,EACxF,SAAS,CACV,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEtD,MAAM,QAAQ,GAAG,0CAA0C,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,yCAAyC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE3E,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC3B,IAAI,OAAO;QAAE,OAAO,IAAI,CAAC;IACzB,OAAO,SAAS,CAAC;AACnB,CAAC;AAKD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAgB,EAChB,KAAa,EACb,MAAc,EACd,OAAiB,EACjB,OAAwE;IAExE,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACxC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,sBAAsB,CAAC;IAC/D,MAAM,IAAI,GAAG,OAAO,EAAE,gBAAgB,IAAI,cAAc,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC;IAE3E,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAEnE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,UAAU,CAC9C,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,4BAA4B,CAAC,GAAG,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CACrE,CAAC;QAEF,MAAM,mBAAmB,GAA+C,EAAE,CAAC;QAC3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,mBAAmB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAChF,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,CAC5C,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAC7F,CAAC;QAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,iBAAiB,EAAE,CAAC;gBACpB,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;oBAC3B,QAAQ,EAAE,CAAC;gBACb,CAAC;qBAAM,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;oBACjC,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC;QAC3C,MAAM,WAAW,GAAG,iBAAiB,GAAG,aAAa,CAAC;QACtD,IAAI,WAAW,GAAG,6BAA6B,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CACb,mCAAmC,iBAAiB,IAAI,aAAa,6BAA6B,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,IAAI,CACnJ,CAAC;QACJ,CAAC;QAED,IAAI,KAAa,CAAC;QAClB,IAAI,WAAmB,CAAC;QAExB,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3B,4FAA4F;YAC5F,2EAA2E;YAC3E,uFAAuF;YACvF,mFAAmF;YACnF,mFAAmF;YACnF,MAAM,eAAe,GAAG,OAAO,CAAC;YAChC,KAAK,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,GAAG,WAAW,CAAC;YAC5D,WAAW,GAAG,GAAG,eAAe,IAAI,iBAAiB,2BAA2B,aAAa,eAAe,UAAU,CAAC,MAAM,aAAa,CAAC;QAC7I,CAAC;aAAM,CAAC;YACN,oFAAoF;YACpF,MAAM,QAAQ,GAAG,QAAQ,GAAG,iBAAiB,CAAC;YAC9C,KAAK,GAAG,QAAQ,GAAG,WAAW,CAAC;YAC/B,WAAW,GAAG,GAAG,QAAQ,IAAI,iBAAiB,yBAAyB,aAAa,eAAe,UAAU,CAAC,MAAM,aAAa,CAAC;QACpI,CAAC;QAED,mBAAmB,CAAC;YAClB,cAAc,EAAE,QAAQ;YACxB,UAAU,EAAE,KAAK;YACjB,WAAW;YACX,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,KAAK;YACpB,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,aAAa;YAC7C,SAAS,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;SACpC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mBAAmB,CAAC;YAClB,cAAc,EAAE,QAAQ;YACxB,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,KAAK;YACpB,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,aAAa;YAC7C,SAAS,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;YACnC,SAAS,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO;YACxD,YAAY,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SACrE,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/lib/judge/llm-judge-qag.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=llm-judge-qag.test.d.ts.map

package/dist/lib/judge/llm-judge-qag.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge-qag.test.d.ts","sourceRoot":"","sources":["../../../src/lib/judge/llm-judge-qag.test.ts"],"names":[],"mappings":""}