judgeval 0.2.9 → 0.3.0

package/dist/esm/scorers/metrics/answer-correctness/prompts.js

@@ -1,171 +0,0 @@
-/**
- * Utility prompts for AnswerCorrectnessScorer
- */
-import { z } from 'zod';
-// Schema definitions for LLM responses
-export const StatementsSchema = z.object({
-    statements: z.array(z.string())
-});
-export const ACVerdictSchema = z.object({
-    verdict: z.string(),
-    reason: z.string()
-});
-export const VerdictsSchema = z.object({
-    verdicts: z.array(ACVerdictSchema)
-});
-export const ReasonSchema = z.object({
-    reason: z.string()
-});
-/**
- * Template prompts for the AnswerCorrectnessScorer
- */
-export class AnswerCorrectnessTemplate {
-    /**
-     * Generate a prompt to extract statements from the expected output
-     */
-    static deduceStatements(expectedOutput) {
-        return `You will be presented with a piece of text. Your task is to break down the text and generate a list of statements contained within the text. Single words and ambiguous phrases should be considered statements.
-
-===== START OF EXAMPLES =====
-Example 1:
-Example text: The weather is sunny today. Temperature is 75 degrees. Don't forget your sunscreen!
-
-Output:
-{
-    "statements": ["The weather is sunny today", "Temperature is 75 degrees", "Don't forget your sunscreen!"]
-}
-
-Example 2:
-Example text: I love pizza. It has cheese and tomato sauce and the crust is crispy.
-
-Output:
-{
-    "statements": ["I love pizza", "It has cheese and tomato sauce", "The crust is crispy"]
-}
-===== END OF EXAMPLES =====
-
-        
-**
-IMPORTANT: Please return your answer in valid JSON format, with the "statements" key mapping to a list of strings. No words or explanation is needed.
-**
-
-==== START OF INPUT ====
-Text:
-${expectedOutput}
-==== END OF INPUT ====
-
-==== YOUR ANSWER ====
-JSON:`;
-    }
-    /**
-     * Generate a prompt to evaluate statements against the actual output
-     */
-    static generateVerdicts(statements, actualOutput) {
-        return `You will be provided with:
-
-- a list of statements from a text that we will refer to as expected output
-- a text that we will refer to as actual output
-
-Your task is to determine whether each statement from the expected output is correct/consistent with the actual output text.
-More specifically, you should generate a JSON object with the key "verdicts". "verdicts" will map to a list of nested JSON objects with two keys: \`verdict\` and \`reason\`.
-The "reason" key should provide an explanation for your choice, regardless of which verdict you select. Try providing quotes from the text(s) to justify your answer where possible.
-The "verdict" key be EXACTLY EITHER "yes" or "no". You should select "yes" if the statement is correct/consistent based on the actual output and "no" otherwise.
-
-==== OUTPUT FORMATTING ====
-IMPORTANT: Please make sure to only return in JSON format, with the "verdicts" key mapping to a list of JSON objects. Each JSON object should contain keys "verdict" (one of "yes" or "no") and "reason" (str). 
-
-==== START OF EXAMPLES ====
-Example input 1: What's the capital of France?
-Example expected output statements 1: ["Paris is the capital of France", "It is located in northern France", "The city has a population of over 2 million"]
-Example actual output 1: "Paris is the capital city of France. It is situated in the northern part of the country and has over 2 million residents."
-Example JSON 1:
-{
-    "verdicts": [
-        {
-            "verdict": "yes",
-            "reason": "The actual output directly states 'Paris is the capital city of France', which matches the statement"
-        },
-        {
-            "verdict": "yes", 
-            "reason": "The actual output confirms this by saying it is 'situated in the northern part of the country'"
-        },
-        {
-            "verdict": "yes",
-            "reason": "The actual output mentions the city 'has over 2 million residents', matching the population statement"
-        }
-    ]
-}
-
-Example input 2: What is the largest planet in our solar system?
-Example expected output statements 2: ["Jupiter is the largest planet", "It is a gas giant", "Jupiter has 79 known moons", "The Great Red Spot is a storm on Jupiter"]
-Example actual output 2: "Jupiter is the biggest planet in the solar system. It is made mostly of gas. The planet has many moons orbiting it."
-Example JSON 2:
-{
-    "verdicts": [
-        {
-            "verdict": "yes",
-            "reason": "The actual output confirms 'Jupiter is the biggest planet', which is equivalent to it being the largest"
-        },
-        {
-            "verdict": "yes",
-            "reason": "The actual output states it is 'made mostly of gas', indicating it is a gas giant"
-        },
-        {
-            "verdict": "no",
-            "reason": "While the actual output mentions Jupiter has 'many moons', it does not specify the exact number of 79 known moons"
-        },
-        {
-            "verdict": "no",
-            "reason": "The actual output makes no mention of the Great Red Spot or any storms on Jupiter"
-        }
-    ]
-}
-==== END OF EXAMPLES ====
-
-** LASTLY **
-Since you are tasked to choose a verdict for each statement, the number of "verdicts" SHOULD BE EXACTLY EQUAL to the number of "statements".
-
-
-==== YOUR TURN =====
-
-Statements:
-${statements}
-
-Actual output:
-${actualOutput}
-
-JSON:`;
-    }
-    /**
-     * Generate a prompt to explain the score based on incorrect statements
-     */
-    static generateReason(incorrectStatements, score) {
-        const incorrectStatementsText = incorrectStatements.map(([statement, reason]) => `statement: ${statement}\nreason: ${reason}\n------`).join('\n');
-        return `==== TASK INSTRUCTIONS ====
-You will provided with two inputs: an answer correctness score and a list of inconsistent/incorrect statements made in a model's output (with the reason why it's irrelevant). Your task is to provide a CLEAR and CONCISE reason for the answer correctness score. 
-For context, there were a list of statements generated from an expected output. The model's actual output was then compared to the expected output, and we collected a list of claims made in the expected output that were either incorrect or inconsistent with the actual output. 
-The score represents how well the model's output matches the expected output.
-You should explain why the score is not higher, but also include why its current score is fair.
-The incorrect statements represent parts of the model output that are incorrect or inconsistent with the expected output. The incorrect statement will be paired with the reason why it's incorrect.
-If there are no incorrect statements, instead respond with a positive remark with an upbeat encouraging tone (but don't overblow the kind attitude).
-
-
-==== FORMATTING YOUR ANSWER ====
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{
-    "reason": "The score is <answer_relevancy_score> because <your_reason>."
-}
-
-==== YOUR TURN ====
----- ANSWER CORRECTNESS SCORE ----
-${score}
-
----- INCORRECT STATEMENTS ----
-${incorrectStatementsText}
-
----- YOUR RESPONSE ----
-JSON:`;
-    }
-}
-//# sourceMappingURL=prompts.js.map

package/dist/esm/scorers/metrics/answer-correctness/prompts.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../../../../src/scorers/metrics/answer-correctness/prompts.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,uCAAuC;AACvC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAChC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;CACnC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,OAAO,yBAAyB;IACpC;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,cAAsB;QAC5C,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2BT,cAAc;;;;MAIV,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,UAAoB,EAAE,YAAoB;QAChE,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoET,UAAU;;;EAGV,YAAY;;MAER,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,cAAc,CAAC,mBAAuC,EAAE,KAAa;QAC1E,MAAM,uBAAuB,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,CAC9E,cAAc,SAAS,aAAa,MAAM,UAAU,CACrD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,OAAO;;;;;;;;;;;;;;;;;;EAkBT,KAAK;;;EAGL,uBAAuB;;;MAGnB,CAAC;IACL,CAAC;CACF"}

package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js

@@ -1,510 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { JudgevalScorer } from '../../base-scorer.js';
-import { APIScorer } from '../../../constants.js';
-import { log, info, warn, error } from '../../../common/logger.js';
-import { AnswerRelevancyTemplate, StatementsSchema, VerdictsSchema, ReasonSchema } from './prompts.js';
-// Import the Judge interface and createJudge from the judges module
-import { createJudge } from '../../../judges/index.js';
-/**
- * AnswerRelevancyScorer evaluates how relevant the actual output is to the input
- * by breaking down the actual output into statements and checking if each statement
- * is relevant to the input.
- */
-export class AnswerRelevancyScorer extends JudgevalScorer {
-    /**
-     * Constructor for AnswerRelevancyScorer
-     * @param threshold Minimum score to consider the evaluation successful (default: 0.5)
-     * @param model LLM to use for evaluation (string or Judge instance)
-     * @param include_reason Whether to generate a reason for the score
-     * @param async_mode Whether to use asynchronous evaluation
-     * @param strict_mode If true, sets threshold to 1.0 (requiring perfect match)
-     * @param verbose_mode Enables detailed logging
-     * @param user Optional user identifier for the LLM
-     * @param additional_metadata Additional metadata to include in the result
-     */
-    constructor(threshold = 0.5, model, include_reason = true, async_mode = true, strict_mode = false, verbose_mode = true, user, additional_metadata) {
-        super(APIScorer.ANSWER_RELEVANCY, strict_mode ? 1.0 : threshold, additional_metadata, include_reason, async_mode, strict_mode, verbose_mode);
-        info(`Initializing AnswerRelevancyScorer with threshold=${this.threshold}, model=${model}, strict_mode=${strict_mode}`);
-        const { judge, usingNativeModel } = createJudge(model, user);
-        this.model = judge;
-        this.usingNativeModel = usingNativeModel;
-        this.evaluation_model = this.model.getModelName();
-        log(`Using model: ${this.evaluation_model}`);
-        // Set required fields for this scorer
-        this.requiredFields = ['input', 'actualOutput'];
-    }
-    /**
-     * Get statements from actual output asynchronously
-     */
-    _aGetStatements(actualOutput) {
-        return __awaiter(this, void 0, void 0, function* () {
-            log("Getting statements asynchronously");
-            // Handle string array
-            const actualOutputStr = Array.isArray(actualOutput) ? actualOutput.join('\n') : actualOutput;
-            const prompt = AnswerRelevancyTemplate.deduceStatements(actualOutputStr);
-            try {
-                const response = yield this.model.aGenerate(prompt);
-                // Parse the response
-                try {
-                    const jsonResponse = JSON.parse(response);
-                    const parsed = StatementsSchema.safeParse(jsonResponse);
-                    if (parsed.success) {
-                        return parsed.data.statements;
-                    }
-                    else {
-                        // Fallback to direct access if schema validation fails
-                        warn("Schema validation failed, falling back to raw response parsing");
-                        if (jsonResponse.statements && Array.isArray(jsonResponse.statements)) {
-                            return jsonResponse.statements;
-                        }
-                    }
-                }
-                catch (parseError) {
-                    warn(`Error parsing JSON response: ${parseError}`);
-                    // Try to extract JSON from the response text
-                    const jsonMatch = response.match(/\{[\s\S]*\}/);
-                    if (jsonMatch) {
-                        try {
-                            const extractedJson = JSON.parse(jsonMatch[0]);
-                            if (extractedJson.statements && Array.isArray(extractedJson.statements)) {
-                                return extractedJson.statements;
-                            }
-                        }
-                        catch (e) {
-                            error(`Failed to extract JSON from response: ${e}`);
-                        }
-                    }
-                }
-                // If all parsing attempts fail, return empty array
-                error("Failed to parse statements from model response");
-                return [];
-            }
-            catch (e) {
-                error(`Error getting statements: ${e}`);
-                return [];
-            }
-        });
-    }
-    /**
-     * Get statements from actual output synchronously
-     */
-    _getStatements(actualOutput) {
-        // Handle string array
-        const actualOutputStr = Array.isArray(actualOutput) ? actualOutput.join('\n') : actualOutput;
-        const prompt = AnswerRelevancyTemplate.deduceStatements(actualOutputStr);
-        try {
-            const response = this.model.generate(prompt);
-            // Parse the response
-            try {
-                const jsonResponse = JSON.parse(response);
-                const parsed = StatementsSchema.safeParse(jsonResponse);
-                if (parsed.success) {
-                    return parsed.data.statements;
-                }
-                else {
-                    // Fallback to direct access if schema validation fails
-                    warn("Schema validation failed, falling back to raw response parsing");
-                    if (jsonResponse.statements && Array.isArray(jsonResponse.statements)) {
-                        return jsonResponse.statements;
-                    }
-                }
-            }
-            catch (parseError) {
-                warn(`Error parsing JSON response: ${parseError}`);
-                // Try to extract JSON from the response text
-                const jsonMatch = response.match(/\{[\s\S]*\}/);
-                if (jsonMatch) {
-                    try {
-                        const extractedJson = JSON.parse(jsonMatch[0]);
-                        if (extractedJson.statements && Array.isArray(extractedJson.statements)) {
-                            return extractedJson.statements;
-                        }
-                    }
-                    catch (e) {
-                        error(`Failed to extract JSON from response: ${e}`);
-                    }
-                }
-            }
-            // If all parsing attempts fail, return empty array
-            error("Failed to parse statements from model response");
-            return [];
-        }
-        catch (e) {
-            error(`Error getting statements: ${e}`);
-            return [];
-        }
-    }
-    /**
-     * Get verdicts for statements against input asynchronously
-     */
-    _aGetVerdicts(input) {
-        return __awaiter(this, void 0, void 0, function* () {
-            log("Getting verdicts asynchronously");
-            if (!this.statements || this.statements.length === 0) {
-                warn("No statements to evaluate");
-                return [];
-            }
-            // Handle string array
-            const inputStr = Array.isArray(input) ? input.join('\n') : input;
-            const prompt = AnswerRelevancyTemplate.generateVerdicts(inputStr, this.statements);
-            try {
-                const response = yield this.model.aGenerate(prompt);
-                // Parse the response
-                try {
-                    const jsonResponse = JSON.parse(response);
-                    const parsed = VerdictsSchema.safeParse(jsonResponse);
-                    if (parsed.success) {
-                        return parsed.data.verdicts;
-                    }
-                    else {
-                        // Fallback to direct access if schema validation fails
-                        warn("Schema validation failed, falling back to raw response parsing");
-                        if (jsonResponse.verdicts && Array.isArray(jsonResponse.verdicts)) {
-                            return jsonResponse.verdicts.map((v) => ({
-                                verdict: v.verdict,
-                                reason: v.reason
-                            }));
-                        }
-                    }
-                }
-                catch (parseError) {
-                    warn(`Error parsing JSON response: ${parseError}`);
-                    // Try to extract JSON from the response text
-                    const jsonMatch = response.match(/\{[\s\S]*\}/);
-                    if (jsonMatch) {
-                        try {
-                            const extractedJson = JSON.parse(jsonMatch[0]);
-                            if (extractedJson.verdicts && Array.isArray(extractedJson.verdicts)) {
-                                return extractedJson.verdicts.map((v) => ({
-                                    verdict: v.verdict,
-                                    reason: v.reason
-                                }));
-                            }
-                        }
-                        catch (e) {
-                            error(`Failed to extract JSON from response: ${e}`);
-                        }
-                    }
-                }
-                // If all parsing attempts fail, return empty array
-                error("Failed to parse verdicts from model response");
-                return [];
-            }
-            catch (e) {
-                error(`Error getting verdicts: ${e}`);
-                return [];
-            }
-        });
-    }
-    /**
-     * Get verdicts for statements against input synchronously
-     */
-    _getVerdicts(input) {
-        if (!this.statements || this.statements.length === 0) {
-            warn("No statements to evaluate");
-            return [];
-        }
-        // Handle string array
-        const inputStr = Array.isArray(input) ? input.join('\n') : input;
-        const prompt = AnswerRelevancyTemplate.generateVerdicts(inputStr, this.statements);
-        try {
-            const response = this.model.generate(prompt);
-            // Parse the response
-            try {
-                const jsonResponse = JSON.parse(response);
-                const parsed = VerdictsSchema.safeParse(jsonResponse);
-                if (parsed.success) {
-                    return parsed.data.verdicts;
-                }
-                else {
-                    // Fallback to direct access if schema validation fails
-                    warn("Schema validation failed, falling back to raw response parsing");
-                    if (jsonResponse.verdicts && Array.isArray(jsonResponse.verdicts)) {
-                        return jsonResponse.verdicts.map((v) => ({
-                            verdict: v.verdict,
-                            reason: v.reason
-                        }));
-                    }
-                }
-            }
-            catch (parseError) {
-                warn(`Error parsing JSON response: ${parseError}`);
-                // Try to extract JSON from the response text
-                const jsonMatch = response.match(/\{[\s\S]*\}/);
-                if (jsonMatch) {
-                    try {
-                        const extractedJson = JSON.parse(jsonMatch[0]);
-                        if (extractedJson.verdicts && Array.isArray(extractedJson.verdicts)) {
-                            return extractedJson.verdicts.map((v) => ({
-                                verdict: v.verdict,
-                                reason: v.reason
-                            }));
-                        }
-                    }
-                    catch (e) {
-                        error(`Failed to extract JSON from response: ${e}`);
-                    }
-                }
-            }
-            // If all parsing attempts fail, return empty array
-            error("Failed to parse verdicts from model response");
-            return [];
-        }
-        catch (e) {
-            error(`Error getting verdicts: ${e}`);
-            return [];
-        }
-    }
-    /**
-     * Get reason for the score asynchronously
-     */
-    _aGetReason(input) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            if (!this.include_reason) {
-                return undefined;
-            }
-            if (!this.verdicts || this.verdicts.length === 0) {
-                return undefined;
-            }
-            try {
-                // Handle string array
-                const inputStr = Array.isArray(input) ? input.join('\n') : input;
-                // Get irrelevant statements with their verdicts
-                const irrelevantStatements = [];
-                for (let i = 0; i < this.statements.length; i++) {
-                    if (i < this.verdicts.length && this.verdicts[i].verdict.toLowerCase() === "no") {
-                        irrelevantStatements.push([this.statements[i], this.verdicts[i].reason]);
-                    }
-                }
-                if (irrelevantStatements.length === 0) {
-                    return "All statements in the actual output are relevant to the input.";
-                }
-                // Generate reason
-                const prompt = AnswerRelevancyTemplate.generateReason(irrelevantStatements, inputStr, ((_a = this.score) === null || _a === void 0 ? void 0 : _a.toString()) || "0");
-                const reasonText = yield this.model.aGenerate(prompt);
-                const parsedReason = ReasonSchema.safeParse(JSON.parse(reasonText));
-                if (!parsedReason.success) {
-                    error(`Failed to parse reason: ${parsedReason.error}`);
-                    return undefined;
-                }
-                return parsedReason.data.reason;
-            }
-            catch (err) {
-                error(`Error getting reason: ${err}`);
-                return undefined;
-            }
-        });
-    }
-    /**
-     * Get reason for the score synchronously
-     */
-    _getReason(input) {
-        var _a;
-        if (!this.include_reason) {
-            return undefined;
-        }
-        if (!this.verdicts || this.verdicts.length === 0) {
-            return undefined;
-        }
-        try {
-            // Handle string array
-            const inputStr = Array.isArray(input) ? input.join('\n') : input;
-            // Get irrelevant statements with their verdicts
-            const irrelevantStatements = [];
-            for (let i = 0; i < this.statements.length; i++) {
-                if (i < this.verdicts.length && this.verdicts[i].verdict.toLowerCase() === "no") {
-                    irrelevantStatements.push([this.statements[i], this.verdicts[i].reason]);
-                }
-            }
-            if (irrelevantStatements.length === 0) {
-                return "All statements in the actual output are relevant to the input.";
-            }
-            // Generate reason
-            const prompt = AnswerRelevancyTemplate.generateReason(irrelevantStatements, inputStr, ((_a = this.score) === null || _a === void 0 ? void 0 : _a.toString()) || "0");
-            const reasonText = this.model.generate(prompt);
-            const parsedReason = ReasonSchema.safeParse(JSON.parse(reasonText));
-            if (!parsedReason.success) {
-                error(`Failed to parse reason: ${parsedReason.error}`);
-                return undefined;
-            }
-            return parsedReason.data.reason;
-        }
-        catch (err) {
-            error(`Error getting reason: ${err}`);
-            return undefined;
-        }
-    }
-    /**
-     * Compute score based on verdicts
-     */
-    _computeScore() {
-        log("Computing score");
-        // If we have no statements or verdicts due to API errors, return 0 instead of 1
-        // This ensures that when API calls fail, we don't incorrectly return a perfect score
-        if (!this.statements || this.statements.length === 0) {
-            return 0;
-        }
-        if (!this.verdicts || this.verdicts.length === 0) {
-            return 0;
-        }
-        let relevantCount = 0;
-        for (const verdict of this.verdicts) {
-            if (verdict.verdict.trim().toLowerCase() !== "no") {
-                relevantCount++;
-            }
-        }
-        const score = relevantCount / this.verdicts.length;
-        // Match Python implementation's handling of strict_mode
-        return this.strict_mode && score < this.threshold ? 0 : score;
-    }
-    /**
-     * Create verbose logs for debugging
-     */
-    _createVerboseLogs() {
-        if (!this.verbose_mode) {
-            return '';
-        }
-        const steps = [
-            `Verdict:\n${JSON.stringify(this.verdicts, null, 2)}`,
-            `Score: ${this.score}\nReason: ${this.reason}`
-        ];
-        return steps.join('\n\n');
-    }
-    /**
-     * Score an example synchronously - this is for compatibility with the Python SDK
-     */
-    syncScoreExample(example) {
-        info("Starting example scoring (sync mode)");
-        try {
-            // Check required parameters
-            this._checkExampleParams(example);
-            // Process example
-            if (this.async_mode) {
-                throw new Error("Cannot use synchronous scoreExample with async_mode=true. Use async scoreExample instead.");
-            }
-            this.statements = this._getStatements(example.actualOutput);
-            this.verdicts = this._getVerdicts(example.input);
-            this.score = this._computeScore();
-            this.reason = this._getReason(example.input);
-            this.success = this._successCheck();
-            this.verbose_logs = this._createVerboseLogs();
-            // Calculate evaluation cost
-            // Track tokens used in LLM calls
-            const promptTokens = 500; // Estimate - in a real implementation, track actual tokens
-            const completionTokens = 200; // Estimate - in a real implementation, track actual tokens
-            this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
-            info(`Scoring completed with score: ${this.score}`);
-            // Ensure all fields match the ScorerData interface
-            return {
-                name: this.type,
-                threshold: this.threshold,
-                success: this.success || false,
-                score: this.score || 0,
-                reason: this.reason !== undefined ? this.reason : null,
-                strict_mode: this.strict_mode || false,
-                evaluation_model: this.evaluation_model || null,
-                error: null,
-                evaluation_cost: this.evaluation_cost || null,
-                verbose_logs: this.verbose_logs ? this.verbose_logs : null,
-                additional_metadata: this.additional_metadata || {}
-            };
-        }
-        catch (error) {
-            // Handle errors
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            this.error = errorMessage;
-            this.success = false;
-            return {
-                name: this.type,
-                threshold: this.threshold,
-                success: false,
-                score: 0,
-                reason: `Error during scoring: ${errorMessage}`,
-                strict_mode: this.strict_mode || false,
-                evaluation_model: this.evaluation_model || null,
-                error: errorMessage,
-                evaluation_cost: null,
-                verbose_logs: null,
-                additional_metadata: this.additional_metadata || {}
-            };
-        }
-    }
-    /**
-     * Score an example - this is the main method that should be called
-     * It will use async or sync methods based on the async_mode setting
-     */
-    scoreExample(example) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.async_mode) {
-                return this.syncScoreExample(example);
-            }
-            info("Starting example scoring (async mode)");
-            try {
-                // Check required parameters
-                this._checkExampleParams(example);
-                // Process example
-                this.statements = yield this._aGetStatements(example.actualOutput);
-                this.verdicts = yield this._aGetVerdicts(example.input);
-                this.score = this._computeScore();
-                this.reason = yield this._aGetReason(example.input);
-                this.success = this._successCheck();
-                this.verbose_logs = this._createVerboseLogs();
-                // Calculate evaluation cost
-                // Track tokens used in LLM calls
-                const promptTokens = 500; // Estimate - in a real implementation, track actual tokens
-                const completionTokens = 200; // Estimate - in a real implementation, track actual tokens
-                this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
-                info(`Scoring completed with score: ${this.score}`);
-                // Ensure all fields match the ScorerData interface
-                return {
-                    name: this.type,
-                    threshold: this.threshold,
-                    success: this.success || false,
-                    score: this.score || 0,
-                    reason: this.reason !== undefined ? this.reason : null,
-                    strict_mode: this.strict_mode || false,
-                    evaluation_model: this.evaluation_model || null,
-                    error: null,
-                    evaluation_cost: this.evaluation_cost || null,
-                    verbose_logs: this.verbose_logs ? this.verbose_logs : null,
-                    additional_metadata: this.additional_metadata || {}
-                };
-            }
-            catch (error) {
-                // Handle errors
-                const errorMessage = error instanceof Error ? error.message : String(error);
-                this.error = errorMessage;
-                this.success = false;
-                return {
-                    name: this.type,
-                    threshold: this.threshold,
-                    success: false,
-                    score: 0,
-                    reason: `Error during scoring: ${errorMessage}`,
-                    strict_mode: this.strict_mode || false,
-                    evaluation_model: this.evaluation_model || null,
-                    error: errorMessage,
-                    evaluation_cost: null,
-                    verbose_logs: null,
-                    additional_metadata: this.additional_metadata || {}
-                };
-            }
-        });
-    }
-    /**
-     * Get the name of the scorer
-     */
-    get name() {
-        return "Answer Relevancy";
-    }
-}
-//# sourceMappingURL=answer-relevancy.js.map