judgeval 0.1.33

package/dist/run-evaluation.js

@@ -0,0 +1,618 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertTest = exports.runEval = exports.checkExamples = exports.checkMissingScorerData = exports.mergeResults = exports.logEvaluationResults = exports.hasLoggedUrl = exports.checkEvalRunNameExists = exports.executeApiEval = exports.pollEvaluationStatus = exports.checkEvaluationStatus = exports.sendToRabbitMQ = exports.validateApiResponse = exports.JudgmentAPIError = void 0;
+const axios_1 = __importDefault(require("axios"));
+const result_1 = require("./data/result");
+const base_scorer_1 = require("./scorers/base-scorer");
+const constants_1 = require("./constants");
+const logger_1 = require("./common/logger");
+/**
+ * Custom error for Judgment API errors
+ */
+class JudgmentAPIError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'JudgmentAPIError';
+    }
+}
+exports.JudgmentAPIError = JudgmentAPIError;
+/**
+ * Validates an API response to ensure it has the expected format
+ * Throws a JudgmentAPIError if the response is invalid
+ */
+function validateApiResponse(response) {
+    if (!response || typeof response !== 'object') {
+        throw new JudgmentAPIError('Invalid API response format: response is not an object');
+    }
+    if (response.error) {
+        throw new JudgmentAPIError(`API error: ${response.error}`);
+    }
+}
+exports.validateApiResponse = validateApiResponse;
+/**
+ * Sends an evaluation run to the RabbitMQ evaluation queue
+ */
+function sendToRabbitMQ(evaluationRun) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const payload = evaluationRun.toJSON();
+        try {
+            const response = yield axios_1.default.post(constants_1.JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, payload, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
+                    'X-Organization-Id': evaluationRun.organizationId
+                }
+            });
+            return response.data;
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error) && error.response) {
+                throw new JudgmentAPIError(`Error sending to RabbitMQ: ${error.response.data.detail || error.message}`);
+            }
+            else {
+                throw new JudgmentAPIError(`Error sending to RabbitMQ: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+            }
+        }
+    });
+}
+exports.sendToRabbitMQ = sendToRabbitMQ;
+/**
+ * Checks the status of an async evaluation
+ * @param evaluationRun The evaluation run to check
+ * @returns The status of the evaluation
+ */
+function checkEvaluationStatus(evaluationRun) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const response = yield axios_1.default.post(`${constants_1.ROOT_API}/check-eval-status/`, {
+                eval_name: evaluationRun.evalName,
+                project_name: evaluationRun.projectName,
+                judgment_api_key: evaluationRun.judgmentApiKey,
+            }, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
+                    'X-Organization-Id': evaluationRun.organizationId
+                }
+            });
+            return response.data;
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error) && error.response) {
+                throw new JudgmentAPIError(`Error checking evaluation status: ${error.response.data.detail || error.message}`);
+            }
+            else {
+                throw new JudgmentAPIError(`Error checking evaluation status: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+            }
+        }
+    });
+}
+exports.checkEvaluationStatus = checkEvaluationStatus;
+/**
+ * Polls the status of an async evaluation until it's complete
+ * @param evaluationRun The evaluation run to poll
+ * @param intervalMs The interval between polls in milliseconds
+ * @param maxAttempts The maximum number of polling attempts
+ * @param onProgress Optional callback for progress updates
+ * @returns The evaluation results
+ */
+function pollEvaluationStatus(evaluationRun_1) {
+    return __awaiter(this, arguments, void 0, function* (evaluationRun, intervalMs = 2000, maxAttempts = 300, onProgress) {
+        let attempts = 0;
+        while (attempts < maxAttempts) {
+            try {
+                const status = yield checkEvaluationStatus(evaluationRun);
+                // Call progress callback if provided
+                if (onProgress) {
+                    onProgress(status);
+                }
+                // Check if evaluation is complete
+                if (status.status === 'complete') {
+                    (0, logger_1.log)('Async evaluation complete, fetching results');
+                    // Fetch the results
+                    const response = yield axios_1.default.post(constants_1.JUDGMENT_EVAL_FETCH_API_URL, {
+                        eval_name: evaluationRun.evalName,
+                        project_name: evaluationRun.projectName,
+                        judgment_api_key: evaluationRun.judgmentApiKey,
+                    }, {
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
+                            'X-Organization-Id': evaluationRun.organizationId
+                        }
+                    });
+                    // Convert API results to ScoringResult objects
+                    const results = response.data.map((result) => {
+                        return new result_1.ScoringResult({
+                            dataObject: result.data_object,
+                            scorersData: result.scorers_data || [],
+                            error: result.error
+                        });
+                    });
+                    return results;
+                }
+                else if (status.status === 'failed') {
+                    throw new JudgmentAPIError(`Async evaluation failed: ${status.error || 'Unknown error'}`);
+                }
+                // Log progress
+                (0, logger_1.log)(`Evaluation status: ${status.status}, progress: ${status.progress || 'unknown'}`);
+                // Wait before next poll
+                yield new Promise(resolve => setTimeout(resolve, intervalMs));
+                attempts++;
+            }
+            catch (error) {
+                (0, logger_1.error)(`Error polling evaluation status: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+                throw new JudgmentAPIError(`Error polling evaluation status: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+            }
+        }
+        throw new JudgmentAPIError(`Evaluation polling timed out after ${maxAttempts} attempts`);
+    });
+}
+exports.pollEvaluationStatus = pollEvaluationStatus;
+/**
+ * Executes an evaluation of a list of Examples using one or more JudgmentScorers via the Judgment API
+ * @param evaluationRun The evaluation run object containing the examples, scorers, and metadata
+ * @returns The results of the evaluation
+ */
+function executeApiEval(evaluationRun) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        try {
+            // Submit API request to execute evals
+            const payload = evaluationRun.toJSON();
+            const response = yield axios_1.default.post(constants_1.JUDGMENT_EVAL_API_URL, payload, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
+                    'X-Organization-Id': evaluationRun.organizationId
+                }
+            });
+            return response.data;
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error) && error.response) {
+                const errorMessage = ((_a = error.response.data) === null || _a === void 0 ? void 0 : _a.detail) || JSON.stringify(error.response.data) || 'An unknown error occurred.';
+                (0, logger_1.error)(`Error: ${errorMessage}`);
+                throw new JudgmentAPIError(errorMessage);
+            }
+            else {
+                (0, logger_1.error)(`Error: ${error}`);
+                throw new JudgmentAPIError(`An error occurred while executing the Judgment API request: ${error}`);
+            }
+        }
+    });
+}
+exports.executeApiEval = executeApiEval;
+/**
+ * Checks if an evaluation run name already exists for a given project
+ */
+function checkEvalRunNameExists(evalName, projectName, judgmentApiKey, organizationId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const response = yield axios_1.default.post(`${constants_1.ROOT_API}/eval-run-name-exists/`, {
+                eval_name: evalName,
+                project_name: projectName,
+                judgment_api_key: judgmentApiKey,
+            }, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${judgmentApiKey}`,
+                    'X-Organization-Id': organizationId
+                }
+            });
+            // Check if the evaluation run name already exists
+            if (response.status === 409) {
+                throw new JudgmentAPIError(`Evaluation run name '${evalName}' already exists for project '${projectName}'.`);
+            }
+            // Check if the response status code is not 2XX
+            if (!response.status.toString().startsWith('2')) {
+                const responseData = response.data;
+                const errorMessage = responseData.detail || 'An unknown error occurred.';
+                (0, logger_1.error)(`Error checking eval run name: ${errorMessage}`);
+                throw new JudgmentAPIError(errorMessage);
+            }
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error) && error.response) {
+                if (error.response.status === 409) {
+                    throw new JudgmentAPIError(`Evaluation run name '${evalName}' already exists for project '${projectName}'.`);
+                }
+            }
+            // For connection errors or other issues, log but continue
+            (0, logger_1.error)(`Failed to check if eval run name exists: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+            // Don't throw an error here, just log it and continue
+        }
+    });
+}
+exports.checkEvalRunNameExists = checkEvalRunNameExists;
+// Track whether a URL has been printed
+exports.hasLoggedUrl = false;
+/**
+ * Logs evaluation results to the Judgment API database.
+ * @param results The results to log
+ * @param projectName The project name
+ * @param evalName The evaluation run name
+ * @param apiKey The API key for the Judgment API
+ * @param organizationId The organization ID
+ * @returns A URL to view the results in the Judgment UI
+ */
+function logEvaluationResults(results_1, projectName_1, evalName_1) {
+    return __awaiter(this, arguments, void 0, function* (results, projectName, evalName, apiKey = '', organizationId) {
+        var _a;
+        try {
+            const response = yield axios_1.default.post(constants_1.JUDGMENT_EVAL_LOG_API_URL, {
+                results: results.map(result => result.toJSON()),
+                project_name: projectName,
+                eval_name: evalName,
+            }, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${apiKey}`,
+                    'X-Organization-Id': organizationId
+                }
+            });
+            if (response.status < 200 || response.status >= 300) {
+                const responseData = response.data;
+                const errorMessage = (responseData === null || responseData === void 0 ? void 0 : responseData.detail) || 'An unknown error occurred.';
+                (0, logger_1.error)(`Error ${response.status}: ${errorMessage}`);
+                throw new JudgmentAPIError(errorMessage);
+            }
+            if (response.data && response.data.ui_results_url) {
+                const url = response.data.ui_results_url;
+                const prettyStr = `\n                     \n🔍 You can view your evaluation results here: ${url}\n`;
+                console.log(prettyStr);
+                exports.hasLoggedUrl = true;
+                return url;
+            }
+            return '';
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error) && error.response) {
+                const errorMessage = ((_a = error.response.data) === null || _a === void 0 ? void 0 : _a.detail) || JSON.stringify(error.response.data) || 'An unknown error occurred.';
+                (0, logger_1.error)(`Error: ${errorMessage}`);
+                throw new JudgmentAPIError(errorMessage);
+            }
+            else {
+                (0, logger_1.error)(`Error: ${error}`);
+                throw new JudgmentAPIError(`An error occurred while logging evaluation results: ${error}`);
+            }
+        }
+    });
+}
+exports.logEvaluationResults = logEvaluationResults;
+/**
+ * When executing scorers that come from both the Judgment API and local scorers, we're left with
+ * results for each type of scorer. This function merges the results from the API and local evaluations,
+ * grouped by example.
+ */
+function mergeResults(apiResults, localResults) {
+    // No merge required
+    if (!localResults.length && apiResults.length) {
+        return apiResults;
+    }
+    if (!apiResults.length && localResults.length) {
+        return localResults;
+    }
+    if (apiResults.length !== localResults.length) {
+        // Results should be of same length because each ScoringResult is a 1-1 mapping to an Example
+        throw new Error(`The number of API and local results do not match: ${apiResults.length} vs ${localResults.length}`);
+    }
+    // Each ScoringResult in api and local have all the same fields besides `scorersData`
+    for (let i = 0; i < apiResults.length; i++) {
+        const apiResult = apiResults[i];
+        const localResult = localResults[i];
+        if (!apiResult.dataObject || !localResult.dataObject) {
+            throw new Error('Data object is null in one of the results.');
+        }
+        // Verify the results are aligned
+        if (apiResult.dataObject.input !== localResult.dataObject.input ||
+            apiResult.dataObject.actualOutput !== localResult.dataObject.actualOutput ||
+            apiResult.dataObject.expectedOutput !== localResult.dataObject.expectedOutput) {
+            throw new Error('The API and local results are not aligned.');
+        }
+        // Merge ScorerData from the API and local scorers together
+        const apiScorerData = apiResult.scorersData;
+        const localScorerData = localResult.scorersData;
+        if (!apiScorerData && localScorerData) {
+            apiResult.scorersData = localScorerData;
+        }
+        else if (apiScorerData && localScorerData) {
+            apiResult.scorersData = [...apiScorerData, ...localScorerData];
+        }
+    }
+    return apiResults;
+}
+exports.mergeResults = mergeResults;
+/**
+ * Checks if any ScoringResult objects are missing scorersData
+ */
+function checkMissingScorerData(results) {
+    var _a;
+    for (let i = 0; i < results.length; i++) {
+        if (!results[i].scorersData || ((_a = results[i].scorersData) === null || _a === void 0 ? void 0 : _a.length) === 0) {
+            (0, logger_1.error)(`Scorer data is missing for example ${i}. ` +
+                'This is usually caused when the example does not contain ' +
+                'the fields required by the scorer. ' +
+                'Check that your example contains the fields required by the scorers.');
+        }
+    }
+    return results;
+}
+exports.checkMissingScorerData = checkMissingScorerData;
+/**
+ * Checks if the example contains the necessary parameters for the scorer
+ */
+function checkExamples(examples, scorers) {
+    for (const scorer of scorers) {
+        for (const example of examples) {
+            // Check for required fields based on scorer type
+            switch (scorer.scoreType) {
+                case 'answer_correctness':
+                case 'answer_relevancy':
+                    if (!example.expectedOutput) {
+                        (0, logger_1.warn)(`Scorer ${scorer.scoreType} requires expectedOutput field`);
+                    }
+                    break;
+                case 'contextual_precision':
+                case 'contextual_recall':
+                case 'contextual_relevancy':
+                    if (!example.context || example.context.length === 0) {
+                        (0, logger_1.warn)(`Scorer ${scorer.scoreType} requires context field`);
+                    }
+                    break;
+                case 'execution_order':
+                    if (!example.expectedTools || example.expectedTools.length === 0) {
+                        (0, logger_1.warn)(`Scorer ${scorer.scoreType} requires expectedTools field`);
+                    }
+                    break;
+                // Add more checks for other scorer types as needed
+            }
+        }
+    }
+}
+exports.checkExamples = checkExamples;
+/**
+ * Executes an evaluation of Examples using one or more Scorers
+ */
+function runEval(evaluationRun_1) {
+    return __awaiter(this, arguments, void 0, function* (evaluationRun, override = false, ignoreErrors = true, asyncExecution = false) {
+        // Check if the evaluation run name already exists
+        // This prevents accidentally overwriting existing evaluation results
+        if (!override && evaluationRun.logResults) {
+            yield checkEvalRunNameExists(evaluationRun.evalName || '', evaluationRun.projectName || '', evaluationRun.judgmentApiKey || '', evaluationRun.organizationId || '');
+        }
+        // --- Set example IDs and timestamps if not already set ---
+        // This is important for tracking and debugging purposes
+        (0, logger_1.log)("Initializing examples with IDs and timestamps");
+        evaluationRun.examples.forEach((example, idx) => {
+            example.exampleIndex = idx; // Set numeric index
+            example.timestamp = new Date().toISOString().replace(/[-:]/g, '').split('.')[0];
+            (0, logger_1.log)(`Initialized example ${example.exampleId} (index: ${example.exampleIndex})`);
+            (0, logger_1.log)(`Input: ${example.input}`);
+            (0, logger_1.log)(`Actual output: ${example.actualOutput || ''}`);
+            if (example.expectedOutput) {
+                (0, logger_1.log)(`Expected output: ${example.expectedOutput}`);
+            }
+            if (example.context) {
+                (0, logger_1.log)(`Context: ${example.context}`);
+            }
+        });
+        (0, logger_1.log)(`Starting evaluation run with ${evaluationRun.examples.length} examples`);
+        // --- Split scorers into API and local ---
+        // API scorers run on the Judgment API server
+        // Local scorers run in this process
+        (0, logger_1.log)("Grouping scorers by type");
+        const apiScorers = [];
+        const localScorers = [];
+        evaluationRun.scorers.forEach(scorer => {
+            if (scorer instanceof base_scorer_1.APIJudgmentScorer) {
+                apiScorers.push(scorer);
+                (0, logger_1.log)(`Added judgment scorer: ${scorer.constructor.name}`);
+            }
+            else {
+                localScorers.push(scorer);
+                (0, logger_1.log)(`Added local scorer: ${scorer.constructor.name}`);
+            }
+        });
+        (0, logger_1.log)(`Found ${apiScorers.length} judgment scorers and ${localScorers.length} local scorers`);
+        let apiResults = [];
+        let localResults = [];
+        // --- Handle async execution ---
+        // This allows evaluations to run in the background without blocking
+        // Useful for large-scale evaluations that might take a long time
+        if (asyncExecution) {
+            checkExamples(evaluationRun.examples, evaluationRun.scorers);
+            (0, logger_1.log)("Starting async evaluation");
+            // Add the evaluation to the RabbitMQ queue for async processing
+            // The server will pick it up and process it in the background
+            try {
+                yield axios_1.default.post(constants_1.JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, evaluationRun.toJSON(), {
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${evaluationRun.judgmentApiKey}`,
+                        'X-Organization-Id': evaluationRun.organizationId
+                    }
+                });
+                (0, logger_1.info)("Successfully added evaluation to queue");
+            }
+            catch (error) {
+                // Log the error but don't throw it - this matches Python SDK behavior
+                // Combine error message into the first argument
+                (0, logger_1.error)(`Error adding evaluation to queue: ${error instanceof Error ? error.message : String(error)}`);
+                // Always print success message to match Python SDK behavior
+                // This is important because the Python SDK always prints this message
+                // even when there's an error connecting to RabbitMQ
+                (0, logger_1.info)("Successfully added evaluation to queue (Note: Previous error occurred)");
+            }
+            // Return empty results for async execution
+            // The results will be available later via the UI or API
+            return [];
+        }
+        else {
+            // --- Execute API scorers ---
+            // These run on the Judgment API server
+            if (apiScorers.length > 0) {
+                try {
+                    (0, logger_1.log)('Executing API evaluation...');
+                    const apiResponseData = yield executeApiEval(evaluationRun);
+                    // Create ScoringResult objects from API response
+                    // Check if the response has a results field (matching Python SDK format)
+                    const resultsData = apiResponseData && typeof apiResponseData === 'object' && 'results' in apiResponseData
+                        ? apiResponseData.results
+                        : apiResponseData;
+                    apiResults = resultsData.map((result) => {
+                        // If the result is already a ScoringResult, return it directly
+                        if (result instanceof result_1.ScoringResult) {
+                            return result;
+                        }
+                        // Otherwise, create a new ScoringResult from the result data
+                        return new result_1.ScoringResult({
+                            dataObject: result.data_object,
+                            scorersData: result.scorers_data,
+                            error: result.error
+                        });
+                    });
+                    (0, logger_1.log)(`API evaluation complete with ${apiResults.length} results`);
+                }
+                catch (error) {
+                    (0, logger_1.error)(`Error executing API evaluation: ${error}`);
+                    if (!ignoreErrors) {
+                        throw error;
+                    }
+                }
+            }
+            // --- Execute local scorers ---
+            // These run in this process
+            if (localScorers.length > 0) {
+                (0, logger_1.log)('Starting local evaluation');
+                try {
+                    // Process each example with each local scorer
+                    localResults = yield Promise.all(evaluationRun.examples.map((example) => __awaiter(this, void 0, void 0, function* () {
+                        const scorersData = [];
+                        // Run each local scorer on the example
+                        for (const scorer of localScorers) {
+                            try {
+                                (0, logger_1.log)(`Running local scorer ${scorer.type} on example ${example.exampleId}`);
+                                const scorerData = yield scorer.scoreExample(example);
+                                scorersData.push(scorerData);
+                                (0, logger_1.log)(`Scorer ${scorer.type} result: score=${scorerData.score}, success=${scorerData.success}`);
+                            }
+                            catch (scorerError) {
+                                (0, logger_1.error)(`Error running scorer ${scorer.type} on example ${example.exampleId}: ${(scorerError === null || scorerError === void 0 ? void 0 : scorerError.message) || String(scorerError)}`);
+                                // Add failed scorer data
+                                scorersData.push({
+                                    name: scorer.type,
+                                    threshold: scorer.threshold,
+                                    success: false,
+                                    score: 0,
+                                    reason: null,
+                                    strict_mode: null,
+                                    evaluation_model: null,
+                                    error: (scorerError === null || scorerError === void 0 ? void 0 : scorerError.message) || String(scorerError),
+                                    evaluation_cost: null,
+                                    verbose_logs: null,
+                                    additional_metadata: scorer.additional_metadata || {}
+                                });
+                            }
+                        }
+                        // Create a ScoringResult with all scorer data for this example
+                        return new result_1.ScoringResult({
+                            dataObject: example,
+                            scorersData: scorersData,
+                            error: undefined
+                        });
+                    })));
+                    (0, logger_1.log)(`Local evaluation complete with ${localResults.length} results`);
+                }
+                catch (error) {
+                    (0, logger_1.error)(`Error executing local evaluation: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+                    if (!ignoreErrors) {
+                        throw error;
+                    }
+                    // Create empty results with errors if ignoring errors
+                    localResults = evaluationRun.examples.map(example => {
+                        return new result_1.ScoringResult({
+                            dataObject: example,
+                            scorersData: [],
+                            error: error === null || error === void 0 ? void 0 : error.message
+                        });
+                    });
+                }
+            }
+            // --- Merge results from API and local scorers ---
+            // This combines the results from both types of scorers
+            // Align with Python - only record content and usage base
+            (0, logger_1.log)('Merging API and local results');
+            const mergedResults = mergeResults(apiResults, localResults);
+            // Check for missing scorer data
+            // This helps identify examples that couldn't be evaluated
+            const checkedResults = checkMissingScorerData(mergedResults);
+            (0, logger_1.log)(`Successfully merged ${checkedResults.length} results`);
+            // --- Log results to Judgment API if requested ---
+            // This saves the results to the database for later viewing
+            if (evaluationRun.logResults) {
+                try {
+                    const url = yield logEvaluationResults(checkedResults, evaluationRun.projectName || '', evaluationRun.evalName || '', evaluationRun.judgmentApiKey || '', evaluationRun.organizationId || '');
+                    (0, logger_1.log)(`Results logged to Judgment API: ${url}`);
+                }
+                catch (error) {
+                    (0, logger_1.error)(`Error logging evaluation results: ${(error === null || error === void 0 ? void 0 : error.message) || String(error)}`);
+                    if (!ignoreErrors) {
+                        throw error;
+                    }
+                }
+            }
+            // --- Check for examples with no scorer data ---
+            // This helps identify examples that couldn't be evaluated
+            for (let i = 0; i < checkedResults.length; i++) {
+                const result = checkedResults[i];
+                if (!result.scorersData || result.scorersData.length === 0) {
+                    (0, logger_1.log)(`None of the scorers could be executed on example ${i}. This is usually because the Example is missing the fields needed by the scorers. Try checking that the Example has the necessary fields for your scorers.`);
+                }
+            }
+            return checkedResults;
+        }
+    });
+}
+exports.runEval = runEval;
+/**
+ * Collects all failed scorers from the scoring results
+ * Raises exceptions for any failed test cases
+ */
+function assertTest(scoringResults) {
+    var _a;
+    const failedTests = [];
+    for (const result of scoringResults) {
+        if (result.error) {
+            failedTests.push(`Error in result: ${result.error}`);
+            continue;
+        }
+        if (!result.scorersData || ((_a = result.scorersData) === null || _a === void 0 ? void 0 : _a.length) === 0) {
+            failedTests.push('No scorer data found in result');
+            continue;
+        }
+        for (const scorerData of result.scorersData) {
+            if (!scorerData.success) {
+                failedTests.push(`Test failed: ${scorerData.name} with score ${scorerData.score} ` +
+                    `(threshold: ${scorerData.threshold})`);
+            }
+        }
+    }
+    if (failedTests.length > 0) {
+        throw new Error(`Test assertion failed:\n${failedTests.join('\n')}`);
+    }
+}
+exports.assertTest = assertTest;
+//# sourceMappingURL=run-evaluation.js.map

package/dist/run-evaluation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-evaluation.js","sourceRoot":"","sources":["../src/run-evaluation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,kDAA0B;AAG1B,0CAA0D;AAC1D,uDAA0E;AAC1E,2CAOqB;AACrB,4CAAiH;AAEjH;;GAEG;AACH,MAAa,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AALD,4CAKC;AAED;;;GAGG;AACH,SAAgB,mBAAmB,CAAC,QAAa;IAC/C,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,gBAAgB,CAAC,wDAAwD,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC,cAAc,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AARD,kDAQC;AAED;;GAEG;AACH,SAAsB,cAAc,CAAC,aAA4B;;QAC/D,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,kDAAsC,EACtC,OAAO,EACP;gBACE,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,aAAa,CAAC,cAAc,EAAE;oBACzD,mBAAmB,EAAE,aAAa,CAAC,cAAc;iBAClD;aACF,CACF,CAAC;YACF,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAChD,MAAM,IAAI,gBAAgB,CAAC,8BAA8B,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1G,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,gBAAgB,CAAC,8BAA8B,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;IACH,CAAC;CAAA;AAtBD,wCAsBC;AAED;;;;GAIG;AACH,SAAsB,qBAAqB,CAAC,aAA4B;;QACtE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,GAAG,oBAAQ,qBAAqB,EAChC;gBACE,SAAS,EAAE,aAAa,CAAC,QAAQ;gBACjC,YAAY,EAAE,aAAa,CAAC,WAAW;gBACvC,gBAAgB,EAAE,aAAa,CAAC,cAAc;aAC/C,EACD;gBACE,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,aAAa,CAAC,cAAc,EAAE;oBACzD,mBAAmB,EAAE,aAAa,CAAC,cAAc;iBAClD;aACF,CACF,CAAC;YACF,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAChD,MAAM,IAAI,gBAAgB,CAAC,qCAAqC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACjH,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,gBAAgB,CAAC,qCAAqC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;IACH,CAAC;CAAA;AAzBD,sDAyBC;AAED;;;;;;;GAOG;AACH,SAAsB,oBAAoB;yDACxC,aAA4B,EAC5B,aAAqB,IAAI,EACzB,cAAsB,GAAG,EACzB,UAAkC;QAElC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,QAAQ,GAAG,WAAW,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,aAAa,CAAC,CAAC;gBAE1D,qCAAqC;gBACrC,IAAI,UAAU,EAAE,CAAC;oBACf,UAAU,CAAC,MAAM,CAAC,CAAC;gBACrB,CAAC;gBAED,kCAAkC;gBAClC,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;oBACjC,IAAA,YAAS,EAAC,6CAA6C,CAAC,CAAC;oBAEzD,oBAAoB;oBACpB,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,uCAA2B,EAC3B;wBACE,SAAS,EAAE,aAAa,CAAC,QAAQ;wBACjC,YAAY,EAAE,aAAa,CAAC,WAAW;wBACvC,gBAAgB,EAAE,aAAa,CAAC,cAAc;qBAC/C,EACD;wBACE,OAAO,EAAE;4BACP,cAAc,EAAE,kBAAkB;4BAClC,eAAe,EAAE,UAAU,aAAa,CAAC,cAAc,EAAE;4BACzD,mBAAmB,EAAE,aAAa,CAAC,cAAc;yBAClD;qBACF,CACF,CAAC;oBAEF,+CAA+C;oBAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE;wBAChD,OAAO,IAAI,sBAAa,CAAC;4BACvB,UAAU,EAAE,MAAM,CAAC,WAAW;4BAC9B,WAAW,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;4BACtC,KAAK,EAAE,MAAM,CAAC,KAAK;yBACpB,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,OAAO,OAAO,CAAC;gBACjB,CAAC;qBAAM,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACtC,MAAM,IAAI,gBAAgB,CAAC,4BAA4B,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;gBAC5F,CAAC;gBAED,eAAe;gBACf,IAAA,YAAS,EAAC,sBAAsB,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;gBAE5F,wBAAwB;gBACxB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;gBAC9D,QAAQ,EAAE,CAAC;YACb,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAA,cAAW,EAAC,oCAAoC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACnF,MAAM,IAAI,gBAAgB,CAAC,oCAAoC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACpG,CAAC;QACH,CAAC;QAED,MAAM,IAAI,gBAAgB,CAAC,sCAAsC,WAAW,WAAW,CAAC,CAAC;IAC3F,CAAC;CAAA;AAjED,oDAiEC;AAED;;;;GAIG;AACH,SAAsB,cAAc,CAAC,aAA4B;;;QAC/D,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,iCAAqB,EACrB,OAAO,EACP;gBACE,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,aAAa,CAAC,cAAc,EAAE;oBACzD,mBAAmB,EAAE,aAAa,CAAC,cAAc;iBAClD;aACF,CACF,CAAC;YAEF,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAChD,MAAM,YAAY,GAAG,CAAA,MAAA,KAAK,CAAC,QAAQ,CAAC,IAAI,0CAAE,MAAM,KAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,4BAA4B,CAAC;gBACxH,IAAA,cAAW,EAAC,UAAU,YAAY,EAAE,CAAC,CAAC;gBACtC,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,IAAA,cAAW,EAAC,UAAU,KAAK,EAAE,CAAC,CAAC;gBAC/B,MAAM,IAAI,gBAAgB,CAAC,+DAA+D,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;IACH,CAAC;CAAA;AA3BD,wCA2BC;AAED;;GAEG;AACH,SAAsB,sBAAsB,CAC1C,QAAgB,EAChB,WAAmB,EACnB,cAAsB,EACtB,cAAsB;;QAEtB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,GAAG,oBAAQ,wBAAwB,EACnC;gBACE,SAAS,EAAE,QAAQ;gBACnB,YAAY,EAAE,WAAW;gBACzB,gBAAgB,EAAE,cAAc;aACjC,EACD;gBACE,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,cAAc,EAAE;oBAC3C,mBAAmB,EAAE,cAAc;iBACpC;aACF,CACF,CAAC;YAEF,kDAAkD;YAClD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,IAAI,gBAAgB,CAAC,wBAAwB,QAAQ,iCAAiC,WAAW,IAAI,CAAC,CAAC;YAC/G,CAAC;YAED,+CAA+C;YAC/C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACnC,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,IAAI,4BAA4B,CAAC;gBACzE,IAAA,cAAW,EAAC,iCAAiC,YAAY,EAAE,CAAC,CAAC;gBAC7D,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAChD,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAClC,MAAM,IAAI,gBAAgB,CAAC,wBAAwB,QAAQ,iCAAiC,WAAW,IAAI,CAAC,CAAC;gBAC/G,CAAC;YACH,CAAC;YACD,0DAA0D;YAC1D,IAAA,cAAW,EAAC,4CAA4C,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3F,sDAAsD;QACxD,CAAC;IACH,CAAC;CAAA;AA7CD,wDA6CC;AAED,uCAAuC;AAC5B,QAAA,YAAY,GAAG,KAAK,CAAC;AAEhC;;;;;;;;GAQG;AACH,SAAsB,oBAAoB;yDACxC,OAAwB,EACxB,WAAmB,EACnB,QAAgB,EAChB,SAAiB,EAAE,EACnB,cAAsB;;QAEtB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,qCAAyB,EACzB;gBACE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC/C,YAAY,EAAE,WAAW;gBACzB,SAAS,EAAE,QAAQ;aACpB,EACD;gBACE,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,mBAAmB,EAAE,cAAc;iBACpC;aACF,CACF,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACnC,MAAM,YAAY,GAAG,CAAA,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,MAAM,KAAI,4BAA4B,CAAC;gBAC1E,IAAA,cAAW,EAAC,SAAS,QAAQ,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC,CAAC;gBACzD,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;YAC3C,CAAC;YAED,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC;gBACzC,MAAM,SAAS,GAAG,0EAA0E,GAAG,IAAI,CAAC;gBACpG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACvB,oBAAY,GAAG,IAAI,CAAC;gBACpB,OAAO,GAAG,CAAC;YACb,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAChD,MAAM,YAAY,GAAG,CAAA,MAAA,KAAK,CAAC,QAAQ,CAAC,IAAI,0CAAE,MAAM,KAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,4BAA4B,CAAC;gBACxH,IAAA,cAAW,EAAC,UAAU,YAAY,EAAE,CAAC,CAAC;gBACtC,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,IAAA,cAAW,EAAC,UAAU,KAAK,EAAE,CAAC,CAAC;gBAC/B,MAAM,IAAI,gBAAgB,CAAC,uDAAuD,KAAK,EAAE,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;IACH,CAAC;CAAA;AAlDD,oDAkDC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,UAA2B,EAAE,YAA6B;IACrF,oBAAoB;IACpB,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QAC9C,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;QAC9C,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC9C,6FAA6F;QAC7F,MAAM,IAAI,KAAK,CAAC,qDAAqD,UAAU,CAAC,MAAM,OAAO,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IACtH,CAAC;IAED,qFAAqF;IACrF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAEpC,IAAI,CAAC,SAAS,CAAC,UAAU,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,iCAAiC;QACjC,IAAI,SAAS,CAAC,UAAU,CAAC,KAAK,KAAK,WAAW,CAAC,UAAU,CAAC,KAAK;YAC3D,SAAS,CAAC,UAAU,CAAC,YAAY,KAAK,WAAW,CAAC,UAAU,CAAC,YAAY;YACzE,SAAS,CAAC,UAAU,CAAC,cAAc,KAAK,WAAW,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,2DAA2D;QAC3D,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC;QAC5C,MAAM,eAAe,GAAG,WAAW,CAAC,WAAW,CAAC;QAEhD,IAAI,CAAC,aAAa,IAAI,eAAe,EAAE,CAAC;YACtC,SAAS,CAAC,WAAW,GAAG,eAAe,CAAC;QAC1C,CAAC;aAAM,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;YAC5C,SAAS,CAAC,WAAW,GAAG,CAAC,GAAG,aAAa,EAAE,GAAG,eAAe,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA1CD,oCA0CC;AAED;;GAEG;AACH,SAAgB,sBAAsB,CAAC,OAAwB;;IAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,CAAA,MAAA,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,0CAAE,MAAM,MAAK,CAAC,EAAE,CAAC;YACpE,IAAA,cAAW,EACT,sCAAsC,CAAC,IAAI;gBAC3C,2DAA2D;gBAC3D,qCAAqC;gBACrC,sEAAsE,CACvE,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAZD,wDAYC;AAED;;GAEG;AACH,SAAgB,aAAa,CAAC,QAAmB,EAAE,OAA4B;IAC7E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,iDAAiD;YACjD,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC;gBACzB,KAAK,oBAAoB,CAAC;gBAC1B,KAAK,kBAAkB;oBACrB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;wBAC5B,IAAA,aAAU,EAAC,UAAU,MAAM,CAAC,SAAS,gCAAgC,CAAC,CAAC;oBACzE,CAAC;oBACD,MAAM;gBACR,KAAK,sBAAsB,CAAC;gBAC5B,KAAK,mBAAmB,CAAC;gBACzB,KAAK,sBAAsB;oBACzB,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACrD,IAAA,aAAU,EAAC,UAAU,MAAM,CAAC,SAAS,yBAAyB,CAAC,CAAC;oBAClE,CAAC;oBACD,MAAM;gBACR,KAAK,iBAAiB;oBACpB,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACjE,IAAA,aAAU,EAAC,UAAU,MAAM,CAAC,SAAS,+BAA+B,CAAC,CAAC;oBACxE,CAAC;oBACD,MAAM;gBACR,mDAAmD;YACrD,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AA3BD,sCA2BC;AAED;;GAEG;AACH,SAAsB,OAAO;yDAC3B,aAA4B,EAC5B,WAAoB,KAAK,EACzB,eAAwB,IAAI,EAC5B,iBAA0B,KAAK;QAE/B,kDAAkD;QAClD,qEAAqE;QACrE,IAAI,CAAC,QAAQ,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;YAC1C,MAAM,sBAAsB,CAC1B,aAAa,CAAC,QAAQ,IAAI,EAAE,EAC5B,aAAa,CAAC,WAAW,IAAI,EAAE,EAC/B,aAAa,CAAC,cAAc,IAAI,EAAE,EAClC,aAAa,CAAC,cAAc,IAAI,EAAE,CACnC,CAAC;QACJ,CAAC;QAED,4DAA4D;QAC5D,wDAAwD;QACxD,IAAA,YAAS,EAAC,+CAA+C,CAAC,CAAC;QAC3D,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;YAC9C,OAAO,CAAC,YAAY,GAAG,GAAG,CAAC,CAAE,oBAAoB;YACjD,OAAO,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,IAAA,YAAS,EAAC,uBAAuB,OAAO,CAAC,SAAS,YAAY,OAAO,CAAC,YAAY,GAAG,CAAC,CAAC;YACvF,IAAA,YAAS,EAAC,UAAU,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACrC,IAAA,YAAS,EAAC,kBAAkB,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC3B,IAAA,YAAS,EAAC,oBAAoB,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,IAAA,YAAS,EAAC,YAAY,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAA,YAAS,EAAC,gCAAgC,aAAa,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;QAEpF,2CAA2C;QAC3C,6CAA6C;QAC7C,oCAAoC;QACpC,IAAA,YAAS,EAAC,0BAA0B,CAAC,CAAC;QACtC,MAAM,UAAU,GAAwB,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAqB,EAAE,CAAC;QAE1C,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YACrC,IAAI,MAAM,YAAY,+BAAiB,EAAE,CAAC;gBACxC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxB,IAAA,YAAS,EAAC,0BAA0B,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;YACjE,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,MAAwB,CAAC,CAAC;gBAC5C,IAAA,YAAS,EAAC,uBAAuB,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAA,YAAS,EAAC,SAAS,UAAU,CAAC,MAAM,yBAAyB,YAAY,CAAC,MAAM,gBAAgB,CAAC,CAAC;QAElG,IAAI,UAAU,GAAoB,EAAE,CAAC;QACrC,IAAI,YAAY,GAAoB,EAAE,CAAC;QAEvC,iCAAiC;QACjC,oEAAoE;QACpE,iEAAiE;QACjE,IAAI,cAAc,EAAE,CAAC;YACnB,aAAa,CAAC,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,OAA8B,CAAC,CAAC;YACpF,IAAA,YAAS,EAAC,2BAA2B,CAAC,CAAC;YAEvC,gEAAgE;YAChE,8DAA8D;YAC9D,IAAI,CAAC;gBACH,MAAM,eAAK,CAAC,IAAI,CACd,kDAAsC,EACtC,aAAa,CAAC,MAAM,EAAE,EACtB;oBACE,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,eAAe,EAAE,UAAU,aAAa,CAAC,cAAc,EAAE;wBACzD,mBAAmB,EAAE,aAAa,CAAC,cAAc;qBAClD;iBACF,CACF,CAAC;gBAEF,IAAA,aAAU,EAAC,wCAAwC,CAAC,CAAC;YACvD,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,sEAAsE;gBACtE,gDAAgD;gBAChD,IAAA,cAAW,EAAC,qCAAqC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAE3G,4DAA4D;gBAC5D,sEAAsE;gBACtE,oDAAoD;gBACpD,IAAA,aAAU,EAAC,wEAAwE,CAAC,CAAC;YACvF,CAAC;YAED,2CAA2C;YAC3C,wDAAwD;YACxD,OAAO,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,8BAA8B;YAC9B,uCAAuC;YACvC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACH,IAAA,YAAS,EAAC,6BAA6B,CAAC,CAAC;oBACzC,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,aAAa,CAAC,CAAC;oBAE5D,iDAAiD;oBACjD,yEAAyE;oBACzE,MAAM,WAAW,GAAG,eAAe,IAAI,OAAO,eAAe,KAAK,QAAQ,IAAI,SAAS,IAAI,eAAe;wBACxG,CAAC,CAAC,eAAe,CAAC,OAAgB;wBAClC,CAAC,CAAC,eAAwB,CAAC;oBAE7B,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE;wBAC3C,+DAA+D;wBAC/D,IAAI,MAAM,YAAY,sBAAa,EAAE,CAAC;4BACpC,OAAO,MAAM,CAAC;wBAChB,CAAC;wBAED,6DAA6D;wBAC7D,OAAO,IAAI,sBAAa,CAAC;4BACvB,UAAU,EAAE,MAAM,CAAC,WAAW;4BAC9B,WAAW,EAAE,MAAM,CAAC,YAAY;4BAChC,KAAK,EAAE,MAAM,CAAC,KAAK;yBACpB,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,IAAA,YAAS,EAAC,gCAAgC,UAAU,CAAC,MAAM,UAAU,CAAC,CAAC;gBACzE,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAA,cAAW,EAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;oBACxD,IAAI,CAAC,YAAY,EAAE,CAAC;wBAClB,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gCAAgC;YAChC,4BAA4B;YAC5B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,IAAA,YAAS,EAAC,2BAA2B,CAAC,CAAC;gBACvC,IAAI,CAAC;oBACH,8CAA8C;oBAC9C,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAO,OAAO,EAAE,EAAE;wBAC5E,MAAM,WAAW,GAAG,EAAE,CAAC;wBAEvB,uCAAuC;wBACvC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;4BAClC,IAAI,CAAC;gCACH,IAAA,YAAS,EAAC,wBAAwB,MAAM,CAAC,IAAI,eAAe,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;gCACjF,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gCACtD,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gCAC7B,IAAA,YAAS,EAAC,UAAU,MAAM,CAAC,IAAI,kBAAkB,UAAU,CAAC,KAAK,aAAa,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;4BACtG,CAAC;4BAAC,OAAO,WAAgB,EAAE,CAAC;gCAC1B,IAAA,cAAW,EAAC,wBAAwB,MAAM,CAAC,IAAI,eAAe,OAAO,CAAC,SAAS,KAAK,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gCACnI,yBAAyB;gCACzB,WAAW,CAAC,IAAI,CAAC;oCACf,IAAI,EAAE,MAAM,CAAC,IAAI;oCACjB,SAAS,EAAE,MAAM,CAAC,SAAS;oCAC3B,OAAO,EAAE,KAAK;oCACd,KAAK,EAAE,CAAC;oCACR,MAAM,EAAE,IAAI;oCACZ,WAAW,EAAE,IAAI;oCACjB,gBAAgB,EAAE,IAAI;oCACtB,KAAK,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,MAAM,CAAC,WAAW,CAAC;oCAClD,eAAe,EAAE,IAAI;oCACrB,YAAY,EAAE,IAAI;oCAClB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,IAAI,EAAE;iCACtD,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;wBAED,+DAA+D;wBAC/D,OAAO,IAAI,sBAAa,CAAC;4BACvB,UAAU,EAAE,OAAO;4BACnB,WAAW,EAAE,WAAW;4BACxB,KAAK,EAAE,SAAS;yBACjB,CAAC,CAAC;oBACL,CAAC,CAAA,CAAC,CAAC,CAAC;oBAEJ,IAAA,YAAS,EAAC,kCAAkC,YAAY,CAAC,MAAM,UAAU,CAAC,CAAC;gBAC7E,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,IAAA,cAAW,EAAC,qCAAqC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACpF,IAAI,CAAC,YAAY,EAAE,CAAC;wBAClB,MAAM,KAAK,CAAC;oBACd,CAAC;oBAED,sDAAsD;oBACtD,YAAY,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;wBAClD,OAAO,IAAI,sBAAa,CAAC;4BACvB,UAAU,EAAE,OAAO;4BACnB,WAAW,EAAE,EAAE;4BACf,KAAK,EAAE,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO;yBACtB,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,mDAAmD;YACnD,uDAAuD;YACvD,yDAAyD;YACzD,IAAA,YAAS,EAAC,+BAA+B,CAAC,CAAC;YAC3C,MAAM,aAAa,GAAG,YAAY,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YAE7D,gCAAgC;YAChC,0DAA0D;YAC1D,MAAM,cAAc,GAAG,sBAAsB,CAAC,aAAa,CAAC,CAAC;YAC7D,IAAA,YAAS,EAAC,uBAAuB,cAAc,CAAC,MAAM,UAAU,CAAC,CAAC;YAElE,mDAAmD;YACnD,2DAA2D;YAC3D,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,oBAAoB,CACpC,cAAc,EACd,aAAa,CAAC,WAAW,IAAI,EAAE,EAC/B,aAAa,CAAC,QAAQ,IAAI,EAAE,EAC5B,aAAa,CAAC,cAAc,IAAI,EAAE,EAClC,aAAa,CAAC,cAAc,IAAI,EAAE,CACnC,CAAC;oBACF,IAAA,YAAS,EAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,IAAA,cAAW,EAAC,qCAAqC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,KAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACpF,IAAI,CAAC,YAAY,EAAE,CAAC;wBAClB,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;YACH,CAAC;YAED,iDAAiD;YACjD,0DAA0D;YAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBACjC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3D,IAAA,YAAS,EAAC,oDAAoD,CAAC,6JAA6J,CAAC,CAAC;gBAChO,CAAC;YACH,CAAC;YAED,OAAO,cAAc,CAAC;QACxB,CAAC;IACH,CAAC;CAAA;AA3OD,0BA2OC;AAED;;;GAGG;AACH,SAAgB,UAAU,CAAC,cAA+B;;IACxD,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,WAAW,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAA,MAAA,MAAM,CAAC,WAAW,0CAAE,MAAM,MAAK,CAAC,EAAE,CAAC;YAC5D,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YACnD,SAAS;QACX,CAAC;QAED,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;gBACxB,WAAW,CAAC,IAAI,CACd,gBAAgB,UAAU,CAAC,IAAI,eAAe,UAAU,CAAC,KAAK,GAAG;oBACjE,eAAe,UAAU,CAAC,SAAS,GAAG,CACvC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,2BAA2B,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AA3BD,gCA2BC"}