judgeval 0.2.9 → 0.3.0

package/dist/cjs/run-evaluation.js

@@ -1,700 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasLoggedUrl = exports.JudgmentAPIError = void 0;
-exports.validateApiResponse = validateApiResponse;
-exports.sendToRabbitMQ = sendToRabbitMQ;
-exports.checkEvaluationStatus = checkEvaluationStatus;
-exports.pollEvaluationStatus = pollEvaluationStatus;
-exports.executeApiEval = executeApiEval;
-exports.checkEvalRunNameExists = checkEvalRunNameExists;
-exports.logEvaluationResults = logEvaluationResults;
-exports.mergeResults = mergeResults;
-exports.checkMissingScorerData = checkMissingScorerData;
-exports.checkExamples = checkExamples;
-exports.runEval = runEval;
-exports.assertTest = assertTest;
-const axios_1 = __importDefault(require("axios"));
-const result_js_1 = require("./data/result.js");
-const base_scorer_js_1 = require("./scorers/base-scorer.js");
-const constants_js_1 = require("./constants.js");
-const logger_js_1 = require("./common/logger.js");
-/**
- * Custom error for Judgment API errors
- */
-class JudgmentAPIError extends Error {
-    constructor(message) {
-        super(message);
-        this.name = 'JudgmentAPIError';
-    }
-}
-exports.JudgmentAPIError = JudgmentAPIError;
-/**
- * Validates an API response to ensure it has the expected format
- * Throws a JudgmentAPIError if the response is invalid
- */
-function validateApiResponse(response) {
-    if (!response || typeof response !== 'object') {
-        throw new JudgmentAPIError('Invalid API response format: response is not an object');
-    }
-    if (response.error) {
-        throw new JudgmentAPIError(`API error: ${response.error}`);
-    }
-}
-/**
- * Sends an evaluation run to the RabbitMQ evaluation queue
- */
-function sendToRabbitMQ(evaluationRun) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const payload = evaluationRun.toJSON();
-        try {
-            const response = yield axios_1.default.post(constants_js_1.JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, payload, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
-                    'X-Organization-Id': evaluationRun.organizationId
-                }
-            });
-            return response.data;
-        }
-        catch (error) {
-            if (axios_1.default.isAxiosError(error) && error.response) {
-                throw new JudgmentAPIError(`Error sending to RabbitMQ: ${error.response.data.detail || error.message}`);
-            }
-            else {
-                throw new JudgmentAPIError(`Error sending to RabbitMQ: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-            }
-        }
-    });
-}
-/**
- * Checks the status of an async evaluation
- * @param evaluationRun The evaluation run to check
- * @returns The status of the evaluation
- */
-function checkEvaluationStatus(evaluationRun) {
-    return __awaiter(this, void 0, void 0, function* () {
-        try {
-            const response = yield axios_1.default.post(`${constants_js_1.ROOT_API}/check-eval-status/`, {
-                eval_name: evaluationRun.evalName,
-                project_name: evaluationRun.projectName,
-                judgment_api_key: evaluationRun.judgmentApiKey,
-            }, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
-                    'X-Organization-Id': evaluationRun.organizationId
-                }
-            });
-            return response.data;
-        }
-        catch (error) {
-            if (axios_1.default.isAxiosError(error) && error.response) {
-                throw new JudgmentAPIError(`Error checking evaluation status: ${error.response.data.detail || error.message}`);
-            }
-            else {
-                throw new JudgmentAPIError(`Error checking evaluation status: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-            }
-        }
-    });
-}
-/**
- * Polls the status of an async evaluation until it's complete
- * @param evaluationRun The evaluation run to poll
- * @param intervalMs The interval between polls in milliseconds
- * @param maxAttempts The maximum number of polling attempts
- * @param onProgress Optional callback for progress updates
- * @returns The evaluation results
- */
-function pollEvaluationStatus(evaluationRun_1) {
-    return __awaiter(this, arguments, void 0, function* (evaluationRun, intervalMs = 2000, maxAttempts = 300, onProgress) {
-        let attempts = 0;
-        while (attempts < maxAttempts) {
-            try {
-                const status = yield checkEvaluationStatus(evaluationRun);
-                // Call progress callback if provided
-                if (onProgress) {
-                    onProgress(status);
-                }
-                // Check if evaluation is complete
-                if (status.status === 'complete') {
-                    (0, logger_js_1.log)('Async evaluation complete, fetching results');
-                    // Fetch the results
-                    const response = yield axios_1.default.post(constants_js_1.JUDGMENT_EVAL_FETCH_API_URL, {
-                        eval_name: evaluationRun.evalName,
-                        project_name: evaluationRun.projectName,
-                        judgment_api_key: evaluationRun.judgmentApiKey,
-                    }, {
-                        headers: {
-                            'Content-Type': 'application/json',
-                            'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
-                            'X-Organization-Id': evaluationRun.organizationId
-                        }
-                    });
-                    // Convert API results to ScoringResult objects
-                    const results = response.data.map((result) => {
-                        return new result_js_1.ScoringResult({
-                            dataObject: result.data_object,
-                            scorersData: result.scorers_data || [],
-                            error: result.error
-                        });
-                    });
-                    return results;
-                }
-                else if (status.status === 'failed') {
-                    throw new JudgmentAPIError(`Async evaluation failed: ${status.error || 'Unknown error'}`);
-                }
-                // Log progress
-                (0, logger_js_1.log)(`Evaluation status: ${status.status}, progress: ${status.progress || 'unknown'}`);
-                // Wait before next poll
-                yield new Promise(resolve => setTimeout(resolve, intervalMs));
-                attempts++;
-            }
-            catch (error) {
-                (0, logger_js_1.error)(`Error polling evaluation status: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-                throw new JudgmentAPIError(`Error polling evaluation status: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-            }
-        }
-        throw new JudgmentAPIError(`Evaluation polling timed out after ${maxAttempts} attempts`);
-    });
-}
-/**
- * Executes an evaluation of a list of Examples using one or more JudgmentScorers via the Judgment API
- * @param evaluationRun The evaluation run object containing the examples, scorers, and metadata
- * @returns The results of the evaluation
- */
-function executeApiEval(evaluationRun) {
-    return __awaiter(this, void 0, void 0, function* () {
-        var _a;
-        try {
-            // Submit API request to execute evals
-            const payload = evaluationRun.toJSON();
-            // Ensure all examples have valid UUIDs and required fields for each scorer type
-            if (payload.examples && Array.isArray(payload.examples)) {
-                payload.examples.forEach(example => {
-                    // Ensure example_id is a valid UUID (matching Python's uuid4 format)
-                    if (!example.example_id || !example.example_id.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i)) {
-                        // Generate a UUID v4 format ID
-                        example.example_id = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
-                            const r = Math.random() * 16 | 0;
-                            const v = c === 'x' ? r : (r & 0x3 | 0x8);
-                            return v.toString(16);
-                        });
-                    }
-                    // Ensure required fields for all scorers (matching Python SDK)
-                    example.name = example.name || "example";
-                    example.example_index = example.example_index || 0;
-                    example.timestamp = example.timestamp || new Date().toISOString();
-                    example.trace_id = example.trace_id || `trace-${Date.now()}-${Math.floor(Math.random() * 1000)}`;
-                    // Check for required fields for each scorer type
-                    if (payload.scorers && Array.isArray(payload.scorers)) {
-                        payload.scorers.forEach((scorer) => {
-                            // Find the corresponding scorer in the evaluationRun
-                            const scorerObj = evaluationRun.scorers.find(s => s.type === scorer.score_type);
-                            if (scorerObj && scorerObj.requiredFields) {
-                                // Check if the example has all the required fields for this scorer
-                                scorerObj.requiredFields.forEach((field) => {
-                                    const snakeCaseField = field.replace(/([A-Z])/g, '_$1').toLowerCase(); // Convert camelCase to snake_case
-                                    // If the field is missing, add it with an appropriate default value
-                                    if (example[snakeCaseField] === undefined) {
-                                        if (snakeCaseField === 'context' || snakeCaseField === 'retrieval_context') {
-                                            example[snakeCaseField] = [];
-                                        }
-                                        else if (snakeCaseField === 'input' || snakeCaseField === 'actual_output' || snakeCaseField === 'expected_output') {
-                                            example[snakeCaseField] = example[snakeCaseField] || '';
-                                        }
-                                    }
-                                    else if ((snakeCaseField === 'context' || snakeCaseField === 'retrieval_context') && !Array.isArray(example[snakeCaseField])) {
-                                        // Ensure context and retrieval_context are arrays
-                                        example[snakeCaseField] = [example[snakeCaseField]];
-                                    }
-                                });
-                            }
-                            // Special handling for contextual scorers
-                            if (['contextual_relevancy', 'contextual_precision', 'contextual_recall', 'faithfulness', 'hallucination'].includes(scorer.score_type)) {
-                                // For contextual scorers, ensure context is always an array
-                                if (!example.context) {
-                                    example.context = [];
-                                }
-                                else if (!Array.isArray(example.context)) {
-                                    // If context is provided but not as an array, convert it to an array
-                                    example.context = [example.context];
-                                }
-                                // For contextual relevancy, also ensure retrieval_context is an array
-                                if (scorer.score_type === 'contextual_relevancy') {
-                                    if (!example.retrieval_context) {
-                                        example.retrieval_context = example.context || [];
-                                    }
-                                    else if (!Array.isArray(example.retrieval_context)) {
-                                        example.retrieval_context = [example.retrieval_context];
-                                    }
-                                }
-                            }
-                        });
-                    }
-                });
-            }
-            const response = yield axios_1.default.post(constants_js_1.JUDGMENT_EVAL_API_URL, payload, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
-                    'X-Organization-Id': evaluationRun.organizationId
-                }
-            });
-            return response.data;
-        }
-        catch (error) {
-            if (axios_1.default.isAxiosError(error) && error.response) {
-                const errorMessage = ((_a = error.response.data) === null || _a === void 0 ? void 0 : _a.detail) || JSON.stringify(error.response.data) || 'An unknown error occurred.';
-                (0, logger_js_1.error)(`Error: ${errorMessage}`);
-                throw new JudgmentAPIError(errorMessage);
-            }
-            else {
-                (0, logger_js_1.error)(`Error: ${error}`);
-                throw new JudgmentAPIError(`An error occurred while executing the Judgment API request: ${error}`);
-            }
-        }
-    });
-}
-/**
- * Checks if an evaluation run name already exists for a given project
- */
-function checkEvalRunNameExists(evalName, projectName, judgmentApiKey, organizationId) {
-    return __awaiter(this, void 0, void 0, function* () {
-        try {
-            const response = yield axios_1.default.post(`${constants_js_1.ROOT_API}/eval-run-name-exists/`, {
-                eval_name: evalName,
-                project_name: projectName,
-                judgment_api_key: judgmentApiKey,
-            }, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${judgmentApiKey}`,
-                    'X-Organization-Id': organizationId
-                }
-            });
-            // Check if the evaluation run name already exists
-            if (response.status === 409) {
-                throw new JudgmentAPIError(`Evaluation run name '${evalName}' already exists for project '${projectName}'.`);
-            }
-            // Check if the response status code is not 2XX
-            if (!response.status.toString().startsWith('2')) {
-                const responseData = response.data;
-                const errorMessage = responseData.detail || 'An unknown error occurred.';
-                (0, logger_js_1.error)(`Error checking eval run name: ${errorMessage}`);
-                throw new JudgmentAPIError(errorMessage);
-            }
-        }
-        catch (error) {
-            if (axios_1.default.isAxiosError(error) && error.response) {
-                if (error.response.status === 409) {
-                    throw new JudgmentAPIError(`Evaluation run name '${evalName}' already exists for project '${projectName}'.`);
-                }
-            }
-            // For connection errors or other issues, log but continue
-            (0, logger_js_1.error)(`Failed to check if eval run name exists: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-            // Don't throw an error here, just log it and continue
-        }
-    });
-}
-// Track whether a URL has been printed
-exports.hasLoggedUrl = false;
-/**
- * Logs evaluation results to the Judgment API database.
- * @param results The results to log
- * @param projectName The project name
- * @param evalName The evaluation run name
- * @param apiKey The API key for the Judgment API
- * @param organizationId The organization ID
- * @returns A URL to view the results in the Judgment UI
- */
-function logEvaluationResults(results_1, projectName_1, evalName_1) {
-    return __awaiter(this, arguments, void 0, function* (results, projectName, evalName, apiKey = '', organizationId) {
-        var _a;
-        try {
-            const response = yield axios_1.default.post(constants_js_1.JUDGMENT_EVAL_LOG_API_URL, {
-                results: results.map(result => result.toJSON()),
-                project_name: projectName,
-                eval_name: evalName,
-                run: {
-                    project_name: projectName,
-                    eval_name: evalName,
-                    examples: results.map(result => result.dataObject),
-                    scorers: results.flatMap(result => result.scorersData ? result.scorersData.map(scorer => ({
-                        name: scorer.name,
-                        threshold: scorer.threshold,
-                        score_type: scorer.name.toLowerCase().replace(' ', '_')
-                    })) : []),
-                    model: "gpt-3.5-turbo", // Default model
-                    log_results: true,
-                    judgment_api_key: apiKey,
-                    organization_id: organizationId,
-                    append: false
-                }
-            }, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${apiKey}`,
-                    'X-Organization-Id': organizationId
-                }
-            });
-            if (response.status < 200 || response.status >= 300) {
-                const responseData = response.data;
-                const errorMessage = (responseData === null || responseData === void 0 ? void 0 : responseData.detail) || 'An unknown error occurred.';
-                (0, logger_js_1.error)(`Error ${response.status}: ${errorMessage}`);
-                throw new JudgmentAPIError(errorMessage);
-            }
-            if (response.data && response.data.ui_results_url) {
-                const url = response.data.ui_results_url;
-                const prettyStr = `\n                     \n🔍 You can view your evaluation results here: ${url}\n`;
-                console.log(prettyStr);
-                exports.hasLoggedUrl = true;
-                return url;
-            }
-            return '';
-        }
-        catch (error) {
-            if (axios_1.default.isAxiosError(error) && error.response) {
-                const errorMessage = ((_a = error.response.data) === null || _a === void 0 ? void 0 : _a.detail) || JSON.stringify(error.response.data) || 'An unknown error occurred.';
-                (0, logger_js_1.error)(`Error: ${errorMessage}`);
-                throw new JudgmentAPIError(errorMessage);
-            }
-            else {
-                (0, logger_js_1.error)(`Error: ${error}`);
-                throw new JudgmentAPIError(`An error occurred while logging evaluation results: ${error}`);
-            }
-        }
-    });
-}
-/**
- * When executing scorers that come from both the Judgment API and local scorers, we're left with
- * results for each type of scorer. This function merges the results from the API and local evaluations,
- * grouped by example.
- */
-function mergeResults(apiResults, localResults) {
-    // No merge required
-    if (!localResults.length && apiResults.length) {
-        return apiResults;
-    }
-    if (!apiResults.length && localResults.length) {
-        return localResults;
-    }
-    if (apiResults.length !== localResults.length) {
-        // Results should be of same length because each ScoringResult is a 1-1 mapping to an Example
-        throw new Error(`The number of API and local results do not match: ${apiResults.length} vs ${localResults.length}`);
-    }
-    // Each ScoringResult in api and local have all the same fields besides `scorersData`
-    for (let i = 0; i < apiResults.length; i++) {
-        const apiResult = apiResults[i];
-        const localResult = localResults[i];
-        if (!apiResult.dataObject || !localResult.dataObject) {
-            throw new Error('Data object is null in one of the results.');
-        }
-        // Verify the results are aligned
-        if (apiResult.dataObject.input !== localResult.dataObject.input ||
-            apiResult.dataObject.actualOutput !== localResult.dataObject.actualOutput ||
-            apiResult.dataObject.expectedOutput !== localResult.dataObject.expectedOutput) {
-            throw new Error('The API and local results are not aligned.');
-        }
-        // Merge ScorerData from the API and local scorers together
-        const apiScorerData = apiResult.scorersData;
-        const localScorerData = localResult.scorersData;
-        if (!apiScorerData && localScorerData) {
-            apiResult.scorersData = localScorerData;
-        }
-        else if (apiScorerData && localScorerData) {
-            apiResult.scorersData = [...apiScorerData, ...localScorerData];
-        }
-    }
-    return apiResults;
-}
-/**
- * Checks if any ScoringResult objects are missing scorersData
- */
-function checkMissingScorerData(results) {
-    var _a;
-    for (let i = 0; i < results.length; i++) {
-        if (!results[i].scorersData || ((_a = results[i].scorersData) === null || _a === void 0 ? void 0 : _a.length) === 0) {
-            (0, logger_js_1.error)(`Scorer data is missing for example ${i}. ` +
-                'This is usually caused when the example does not contain ' +
-                'the fields required by the scorer. ' +
-                'Check that your example contains the fields required by the scorers.');
-        }
-    }
-    return results;
-}
-/**
- * Checks if the example contains the necessary parameters for the scorer
- */
-function checkExamples(examples, scorers) {
-    for (const scorer of scorers) {
-        for (const example of examples) {
-            // Check for required fields based on scorer type
-            switch (scorer.scoreType) {
-                case 'answer_correctness':
-                case 'answer_relevancy':
-                    if (!example.expectedOutput) {
-                        (0, logger_js_1.warn)(`Scorer ${scorer.scoreType} requires expectedOutput field`);
-                    }
-                    break;
-                case 'contextual_precision':
-                case 'contextual_recall':
-                case 'contextual_relevancy':
-                    if (!example.context || example.context.length === 0) {
-                        (0, logger_js_1.warn)(`Scorer ${scorer.scoreType} requires context field`);
-                    }
-                    break;
-                case 'execution_order':
-                    if (!example.expectedTools || example.expectedTools.length === 0) {
-                        (0, logger_js_1.warn)(`Scorer ${scorer.scoreType} requires expectedTools field`);
-                    }
-                    break;
-                // Add more checks for other scorer types as needed
-            }
-        }
-    }
-}
-/**
- * Executes an evaluation of Examples using one or more Scorers
- */
-function runEval(evaluationRun_1) {
-    return __awaiter(this, arguments, void 0, function* (evaluationRun, override = false, ignoreErrors = true, asyncExecution = false) {
-        // Check if the evaluation run name already exists
-        // This prevents accidentally overwriting existing evaluation results
-        if (!override && evaluationRun.logResults) {
-            yield checkEvalRunNameExists(evaluationRun.evalName || '', evaluationRun.projectName || '', evaluationRun.judgmentApiKey || '', evaluationRun.organizationId || '');
-        }
-        // --- Set example IDs and timestamps if not already set ---
-        // This is important for tracking and debugging purposes
-        (0, logger_js_1.log)("Initializing examples with IDs and timestamps");
-        evaluationRun.examples.forEach((example, idx) => {
-            example.exampleIndex = idx; // Set numeric index
-            example.timestamp = new Date().toISOString().replace(/[-:]/g, '').split('.')[0];
-            (0, logger_js_1.log)(`Initialized example ${example.exampleId} (index: ${example.exampleIndex})`);
-            (0, logger_js_1.log)(`Input: ${example.input}`);
-            (0, logger_js_1.log)(`Actual output: ${example.actualOutput || ''}`);
-            if (example.expectedOutput) {
-                (0, logger_js_1.log)(`Expected output: ${example.expectedOutput}`);
-            }
-            if (example.context) {
-                (0, logger_js_1.log)(`Context: ${example.context}`);
-            }
-        });
-        (0, logger_js_1.log)(`Starting evaluation run with ${evaluationRun.examples.length} examples`);
-        // --- Split scorers into API and local ---
-        // API scorers run on the Judgment API server
-        // Local scorers run in this process
-        (0, logger_js_1.log)("Grouping scorers by type");
-        const apiScorers = [];
-        const localScorers = [];
-        evaluationRun.scorers.forEach(scorer => {
-            if (scorer instanceof base_scorer_js_1.APIJudgmentScorer) {
-                apiScorers.push(scorer);
-                (0, logger_js_1.log)(`Added judgment scorer: ${scorer.constructor.name}`);
-            }
-            else {
-                localScorers.push(scorer);
-                (0, logger_js_1.log)(`Added local scorer: ${scorer.constructor.name}`);
-            }
-        });
-        (0, logger_js_1.log)(`Found ${apiScorers.length} judgment scorers and ${localScorers.length} local scorers`);
-        let apiResults = [];
-        let localResults = [];
-        // --- Handle async execution ---
-        // This allows evaluations to run in the background without blocking
-        // Useful for large-scale evaluations that might take a long time
-        if (asyncExecution) {
-            checkExamples(evaluationRun.examples, evaluationRun.scorers);
-            (0, logger_js_1.log)("Starting async evaluation");
-            // Add the evaluation to the RabbitMQ queue for async processing
-            // The server will pick it up and process it in the background
-            try {
-                yield axios_1.default.post(constants_js_1.JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, evaluationRun.toJSON(), {
-                    headers: {
-                        'Content-Type': 'application/json',
-                        'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
-                        'X-Organization-Id': evaluationRun.organizationId
-                    }
-                });
-                (0, logger_js_1.info)("Successfully added evaluation to queue");
-            }
-            catch (error) {
-                // Log the error but don't throw it - this matches Python SDK behavior
-                // Combine error message into the first argument
-                (0, logger_js_1.error)(`Error adding evaluation to queue: ${error instanceof Error ? error.message : String(error)}`);
-                // Always print success message to match Python SDK behavior
-                // This is important because the Python SDK always prints this message
-                // even when there's an error connecting to RabbitMQ
-                (0, logger_js_1.info)("Successfully added evaluation to queue (Note: Previous error occurred)");
-            }
-            // Return empty results for async execution
-            // The results will be available later via the UI or API
-            return [];
-        }
-        else {
-            // --- Execute API scorers ---
-            // These run on the Judgment API server
-            if (apiScorers.length > 0) {
-                try {
-                    (0, logger_js_1.log)('Executing API evaluation...');
-                    const apiResponseData = yield executeApiEval(evaluationRun);
-                    // Create ScoringResult objects from API response
-                    // Check if the response has a results field (matching Python SDK format)
-                    const resultsData = apiResponseData && typeof apiResponseData === 'object' && 'results' in apiResponseData
-                        ? apiResponseData.results
-                        : apiResponseData;
-                    apiResults = resultsData.map((result) => {
-                        // If the result is already a ScoringResult, return it directly
-                        if (result instanceof result_js_1.ScoringResult) {
-                            return result;
-                        }
-                        // Otherwise, create a new ScoringResult from the result data
-                        return new result_js_1.ScoringResult({
-                            dataObject: result.data_object,
-                            scorersData: result.scorers_data,
-                            error: result.error
-                        });
-                    });
-                    (0, logger_js_1.log)(`API evaluation complete with ${apiResults.length} results`);
-                }
-                catch (error) {
-                    (0, logger_js_1.error)(`Error executing API evaluation: ${error}`);
-                    if (!ignoreErrors) {
-                        throw error;
-                    }
-                }
-            }
-            // --- Execute local scorers ---
-            // These run in this process
-            if (localScorers.length > 0) {
-                (0, logger_js_1.log)('Starting local evaluation');
-                try {
-                    // Process each example with each local scorer
-                    localResults = yield Promise.all(evaluationRun.examples.map((example) => __awaiter(this, void 0, void 0, function* () {
-                        const scorersData = [];
-                        // Run each local scorer on the example
-                        for (const scorer of localScorers) {
-                            try {
-                                (0, logger_js_1.log)(`Running local scorer ${scorer.type} on example ${example.exampleId}`);
-                                const scorerData = yield scorer.scoreExample(example);
-                                scorersData.push(scorerData);
-                                (0, logger_js_1.log)(`Scorer ${scorer.type} result: score=${scorerData.score}, success=${scorerData.success}`);
-                            }
-                            catch (scorerError) {
-                                (0, logger_js_1.error)(`Error running scorer ${scorer.type} on example ${example.exampleId}: ${(scorerError === null || scorerError === void 0 ? void 0 : scorerError.message) || String(scorerError)}`);
-                                // Add failed scorer data
-                                scorersData.push({
-                                    name: scorer.type,
-                                    threshold: scorer.threshold,
-                                    success: false,
-                                    score: 0,
-                                    reason: null,
-                                    strict_mode: null,
-                                    evaluation_model: null,
-                                    error: (scorerError === null || scorerError === void 0 ? void 0 : scorerError.message) || String(scorerError),
-                                    evaluation_cost: null,
-                                    verbose_logs: null,
-                                    additional_metadata: scorer.additional_metadata || {}
-                                });
-                            }
-                        }
-                        // Create a ScoringResult with all scorer data for this example
-                        return new result_js_1.ScoringResult({
-                            dataObject: example,
-                            scorersData: scorersData,
-                            error: undefined
-                        });
-                    })));
-                    (0, logger_js_1.log)(`Local evaluation complete with ${localResults.length} results`);
-                }
-                catch (error) {
-                    (0, logger_js_1.error)(`Error executing local evaluation: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-                    if (!ignoreErrors) {
-                        throw error;
-                    }
-                    else {
-                        // Create empty results with errors if ignoring errors
-                        localResults = evaluationRun.examples.map(example => {
-                            return new result_js_1.ScoringResult({
-                                dataObject: example,
-                                scorersData: [],
-                                error: error === null || error === void 0 ? void 0 : error.message
-                            });
-                        });
-                    }
-                }
-            }
-            // --- Merge results from API and local scorers ---
-            // This combines the results from both types of scorers
-            // Align with Python - only record content and usage base
-            (0, logger_js_1.log)('Merging API and local results');
-            const mergedResults = mergeResults(apiResults, localResults);
-            // Check for missing scorer data
-            // This helps identify examples that couldn't be evaluated
-            const checkedResults = checkMissingScorerData(mergedResults);
-            (0, logger_js_1.log)(`Successfully merged ${checkedResults.length} results`);
-            // --- Log results to Judgment API if requested ---
-            // This saves the results to the database for later viewing
-            if (evaluationRun.logResults) {
-                try {
-                    const url = yield logEvaluationResults(checkedResults, evaluationRun.projectName || '', evaluationRun.evalName || '', evaluationRun.judgmentApiKey || '', evaluationRun.organizationId || '');
-                    (0, logger_js_1.log)(`Results logged to Judgment API: ${url}`);
-                }
-                catch (error) {
-                    (0, logger_js_1.error)(`Error logging evaluation results: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
-                    if (!ignoreErrors) {
-                        throw error;
-                    }
-                }
-            }
-            // --- Check for examples with no scorer data ---
-            // This helps identify examples that couldn't be evaluated
-            for (let i = 0; i < checkedResults.length; i++) {
-                const result = checkedResults[i];
-                if (!result.scorersData || result.scorersData.length === 0) {
-                    (0, logger_js_1.log)(`None of the scorers could be executed on example ${i}. This is usually because the Example is missing the fields needed by the scorers. Try checking that the Example has the necessary fields for your scorers.`);
-                }
-            }
-            return checkedResults;
-        }
-    });
-}
-/**
- * Collects all failed scorers from the scoring results
- * Raises exceptions for any failed test cases
- */
-function assertTest(scoringResults) {
-    var _a;
-    const failedTests = [];
-    for (const result of scoringResults) {
-        if (result.error) {
-            failedTests.push(`Error in result: ${result.error}`);
-            continue;
-        }
-        if (!result.scorersData || ((_a = result.scorersData) === null || _a === void 0 ? void 0 : _a.length) === 0) {
-            failedTests.push('No scorer data found in result');
-            continue;
-        }
-        for (const scorerData of result.scorersData) {
-            if (!scorerData.success) {
-                failedTests.push(`Test failed: ${scorerData.name} with score ${scorerData.score} ` +
-                    `(threshold: ${scorerData.threshold})`);
-            }
-        }
-    }
-    if (failedTests.length > 0) {
-        throw new Error(`Test assertion failed:\n${failedTests.join('\n')}`);
-    }
-}
-//# sourceMappingURL=run-evaluation.js.map