@salesforce/agents 1.5.1 → 1.6.0

package/lib/agentEvalRunner.d.ts

@@ -0,0 +1,26 @@
+import { Org } from '@salesforce/core';
+import type { EvalPayload } from './evalNormalizer.js';
+import type { EvalApiResponse, EvalResult, EvalOutput, TestResult } from './evalFormatter.js';
+export type AgentEvalRunResult = {
+    tests: Array<{
+        id: string;
+        status: string;
+        evaluations: EvalResult[];
+        outputs: EvalOutput[];
+    }>;
+    summary: {
+        passed: number;
+        failed: number;
+        scored: number;
+        errors: number;
+    };
+};
+export declare function resolveAgent(org: Org, apiName: string): Promise<{
+    agentId: string;
+    versionId: string;
+}>;
+export declare function executeBatches(org: Org, batches: Array<EvalPayload['tests']>, log?: (msg: string) => void): Promise<TestResult[]>;
+export declare function buildResultSummary(mergedResponse: EvalApiResponse): {
+    summary: AgentEvalRunResult['summary'];
+    testSummaries: AgentEvalRunResult['tests'];
+};

package/lib/agentEvalRunner.js

@@ -0,0 +1,102 @@
+"use strict";
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveAgent = resolveAgent;
+exports.executeBatches = executeBatches;
+exports.buildResultSummary = buildResultSummary;
+/* eslint-disable camelcase */
+const core_1 = require("@salesforce/core");
+const utils_1 = require("./utils");
+async function getApiHeaders(org) {
+    const conn = org.getConnection();
+    const userInfo = await conn.request(`${conn.instanceUrl}/services/oauth2/userinfo`);
+    return {
+        orgId: org.getOrgId(),
+        userId: userInfo.user_id,
+        instanceUrl: conn.instanceUrl,
+    };
+}
+async function callEvalApi(org, payload, headers) {
+    const conn = org.getConnection();
+    return (0, utils_1.requestWithEndpointFallback)(conn, {
+        url: 'https://api.salesforce.com/einstein/evaluation/v1/tests',
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-sfdc-core-tenant-id': `core/prod/${headers.orgId}`,
+            'x-org-id': headers.orgId,
+            'x-sfdc-core-instance-url': headers.instanceUrl,
+            'x-sfdc-user-id': headers.userId,
+            'x-client-feature-id': 'AIPlatformEvaluation',
+            'x-sfdc-app-context': 'EinsteinGPT',
+        },
+        body: JSON.stringify(payload),
+    }, { retry: { maxRetries: 3 } });
+}
+async function resolveAgent(org, apiName) {
+    const conn = org.getConnection();
+    const escapedApiName = apiName.replace(/'/g, "''");
+    const botResult = await conn.query(`SELECT Id FROM BotDefinition WHERE DeveloperName = '${escapedApiName}'`);
+    if (!botResult.records.length) {
+        throw new core_1.SfError(`Agent '${apiName}' not found. Verify the DeveloperName exists in BotDefinition in the target org.`);
+    }
+    const agentId = botResult.records[0].Id;
+    const versionResult = await conn.query(`SELECT Id FROM BotVersion WHERE BotDefinitionId = '${agentId}' ORDER BY VersionNumber DESC LIMIT 1`);
+    if (!versionResult.records.length) {
+        throw new core_1.SfError(`No published version found for agent '${apiName}'. Ensure the agent has been saved and versioned in the target org.`);
+    }
+    const versionId = versionResult.records[0].Id;
+    return { agentId, versionId };
+}
+async function executeBatches(org, batches, log) {
+    const headers = await getApiHeaders(org);
+    if (batches.length > 1) {
+        log?.(`Running ${batches.length} batches in parallel`);
+    }
+    const batchPromises = batches.map(async (batch) => {
+        const batchPayload = { tests: batch };
+        const resultObj = await callEvalApi(org, batchPayload, headers);
+        return resultObj.results ?? [];
+    });
+    const batchResults = await Promise.all(batchPromises);
+    return batchResults.flat();
+}
+function buildResultSummary(mergedResponse) {
+    const summary = { passed: 0, failed: 0, scored: 0, errors: 0 };
+    const testSummaries = [];
+    for (const testResult of mergedResponse.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        const evalResults = testResult.evaluation_results ?? [];
+        const testErrors = testResult.errors ?? [];
+        const passed = evalResults.filter((e) => e.is_pass === true).length;
+        const failed = evalResults.filter((e) => e.is_pass === false).length;
+        const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length;
+        summary.passed += passed;
+        summary.failed += failed;
+        summary.scored += scored;
+        summary.errors += testErrors.length;
+        const outputs = testResult.outputs ?? [];
+        testSummaries.push({
+            id: testId,
+            status: failed > 0 || testErrors.length > 0 ? 'failed' : 'passed',
+            evaluations: evalResults,
+            outputs,
+        });
+    }
+    return { summary, testSummaries };
+}
+//# sourceMappingURL=agentEvalRunner.js.map

package/lib/agentEvalRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentEvalRunner.js","sourceRoot":"","sources":["../src/agentEvalRunner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAsDH,oCA0BC;AAED,wCAmBC;AAED,gDA+BC;AApID,8BAA8B;AAE9B,2CAAgD;AAChD,mCAAsD;AAetD,KAAK,UAAU,aAAa,CAAC,GAAQ;IACnC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAsB,GAAG,IAAI,CAAC,WAAW,2BAA2B,CAAC,CAAC;IAEzG,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE;QACrB,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,OAAoB,EAAE,OAAmB;IAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,OAAO,IAAA,mCAA2B,EAChC,IAAI,EACJ;QACE,GAAG,EAAE,yDAAyD;QAC9D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,uBAAuB,EAAE,aAAa,OAAO,CAAC,KAAK,EAAE;YACrD,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,0BAA0B,EAAE,OAAO,CAAC,WAAW;YAC/C,gBAAgB,EAAE,OAAO,CAAC,MAAM;YAChC,qBAAqB,EAAE,sBAAsB;YAC7C,oBAAoB,EAAE,aAAa;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,EACD,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,CAC7B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAAe;IAC1D,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAEnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAChC,uDAAuD,cAAc,GAAG,CACzE,CAAC;IACF,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,IAAI,cAAO,CACf,UAAU,OAAO,kFAAkF,CACpG,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CACpC,sDAAsD,OAAO,uCAAuC,CACrG,CAAC;IACF,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,IAAI,cAAO,CACf,yCAAyC,OAAO,qEAAqE,CACtH,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAEM,KAAK,UAAU,cAAc,CAClC,GAAQ,EACR,OAAoC,EACpC,GAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAEzC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,EAAE,CAAC,WAAW,OAAO,CAAC,MAAM,sBAAsB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,MAAM,YAAY,GAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACtD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAgB,kBAAkB,CAAC,cAA+B;IAIhE,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,aAAa,GAAgC,EAAE,CAAC;IAEtD,KAAK,MAAM,UAAU,IAAI,cAAc,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAC1C,MAAM,WAAW,GAAiB,UAAU,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACtE,MAAM,UAAU,GAAgB,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;QAExD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;QAEtF,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAEpC,MAAM,OAAO,GAAiB,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC;QACvD,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjE,WAAW,EAAE,WAAW;YACxB,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC"}

package/lib/evalFormatter.d.ts

@@ -0,0 +1,44 @@
+export type ResultFormat = 'human' | 'json' | 'junit' | 'tap';
+type SendMessageResponse = string | {
+    messages: Array<{
+        message: string;
+    }>;
+};
+type GetStateResponse = {
+    planner_response?: {
+        lastExecution?: {
+            topic?: string;
+            latency?: number;
+            invokedActions?: string[];
+        };
+    };
+};
+export type EvalOutput = {
+    type?: string;
+    id?: string;
+    session_id?: string;
+    response?: SendMessageResponse | GetStateResponse;
+};
+export type EvalResult = {
+    id?: string;
+    score?: number | null;
+    is_pass?: boolean | null;
+    actual_value?: string;
+    expected_value?: string;
+    error_message?: string;
+};
+export type TestError = {
+    id?: string;
+    error_message?: string;
+};
+export type TestResult = {
+    id?: string;
+    outputs?: EvalOutput[];
+    evaluation_results?: EvalResult[];
+    errors?: TestError[];
+};
+export type EvalApiResponse = {
+    results?: TestResult[];
+};
+export declare function formatResults(results: EvalApiResponse, format: ResultFormat): string;
+export {};

package/lib/evalFormatter.js

@@ -0,0 +1,267 @@
+"use strict";
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatResults = formatResults;
+function formatResults(results, format) {
+    switch (format) {
+        case 'human':
+            return formatHuman(results);
+        case 'json':
+            return JSON.stringify(results, null, 2);
+        case 'junit':
+            return formatJunit(results);
+        case 'tap':
+            return formatTap(results);
+        default:
+            return formatHuman(results);
+    }
+}
+// --- formatHuman helpers ---
+function formatOutputLines(outputs) {
+    const lines = [];
+    for (const output of outputs) {
+        const stepType = output.type ?? '';
+        const stepId = output.id ?? '';
+        if (stepType === 'agent.create_session') {
+            const sessionId = output.session_id ?? 'N/A';
+            lines.push(`- **Create Session**: ${sessionId}`);
+        }
+        else if (stepType === 'agent.send_message') {
+            const resp = output.response;
+            let msgStr;
+            if (typeof resp === 'string') {
+                msgStr = resp;
+            }
+            else if (resp !== null && typeof resp === 'object' && 'messages' in resp) {
+                msgStr = resp.messages?.[0]?.message ?? '';
+            }
+            else {
+                msgStr = String(resp ?? '');
+            }
+            const displayMsg = msgStr.length > 200 ? msgStr.substring(0, 200) + '...' : msgStr;
+            lines.push(`- **Agent Response** (${stepId}): ${displayMsg}`);
+        }
+        else if (stepType === 'agent.get_state') {
+            const resp = output.response;
+            if (resp !== null && typeof resp === 'object' && 'planner_response' in resp) {
+                const { planner_response: plannerResp } = resp;
+                const lastExec = plannerResp?.lastExecution;
+                lines.push(`- **Topic Selected**: ${lastExec?.topic ?? 'N/A'}`);
+                lines.push(`- **Response Latency**: ${lastExec?.latency ?? 'N/A'}ms`);
+            }
+            else {
+                lines.push(`- **State**: ${String(resp).substring(0, 200)}`);
+            }
+        }
+    }
+    return lines;
+}
+function formatEvaluationTable(evalResults) {
+    const lines = [];
+    if (evalResults.length > 0) {
+        lines.push('### Evaluation Results\n');
+        lines.push('| Metric | Score | Pass | Actual | Expected |');
+        lines.push('|--------|-------|------|--------|----------|');
+        for (const evalR of evalResults) {
+            const metricId = evalR.id ?? 'unknown';
+            const score = evalR.score;
+            const scoreStr = score != null ? score.toFixed(3) : 'N/A';
+            const isPass = evalR.is_pass;
+            const passStr = isPass === true ? 'PASS' : isPass === false ? 'FAIL' : 'N/A';
+            const actual = String(evalR.actual_value ?? '').substring(0, 60);
+            const expected = String(evalR.expected_value ?? '').substring(0, 60);
+            const error = evalR.error_message;
+            if (error) {
+                lines.push(`| ${metricId} | ERROR | - | ${error.substring(0, 80)} | - |`);
+            }
+            else {
+                lines.push(`| ${metricId} | ${scoreStr} | ${passStr} | ${actual} | ${expected} |`);
+            }
+        }
+        lines.push('');
+    }
+    return lines;
+}
+function formatErrorLines(errors) {
+    const lines = [];
+    if (errors.length > 0) {
+        lines.push('### Errors\n');
+        for (const error of errors) {
+            const errorId = error.id ?? 'unknown';
+            const errorMsg = error.error_message ?? String(error);
+            lines.push(`- **${errorId}**: ${errorMsg}`);
+        }
+        lines.push('');
+    }
+    return lines;
+}
+function formatTestSummaryLines(evalResults, errors) {
+    const lines = [];
+    const totalEvals = evalResults.length;
+    const passed = evalResults.filter((e) => e.is_pass === true).length;
+    const failed = evalResults.filter((e) => e.is_pass === false).length;
+    const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length;
+    lines.push(`**Summary**: ${totalEvals} evaluations`);
+    if (passed || failed) {
+        lines.push(`  - Passed: ${passed}, Failed: ${failed}`);
+    }
+    if (scored) {
+        lines.push(`  - Scored (no threshold): ${scored}`);
+    }
+    if (errors.length > 0) {
+        lines.push(`  - Errors: ${errors.length}`);
+    }
+    lines.push('');
+    return lines;
+}
+function formatHuman(results) {
+    const lines = ['# Agent Evaluation Results\n'];
+    for (const testResult of results.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        const errors = testResult.errors ?? [];
+        const evalResults = testResult.evaluation_results ?? [];
+        const outputs = testResult.outputs ?? [];
+        lines.push(`## Test: ${testId}\n`);
+        lines.push(...formatOutputLines(outputs));
+        lines.push('');
+        lines.push(...formatEvaluationTable(evalResults));
+        lines.push(...formatErrorLines(errors));
+        lines.push(...formatTestSummaryLines(evalResults, errors));
+    }
+    return lines.join('\n');
+}
+function formatJunit(results) {
+    const allTests = [];
+    for (const testResult of results.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        for (const evalR of testResult.evaluation_results ?? []) {
+            const stepId = evalR.id ?? 'unknown';
+            const name = `${testId}.${stepId}`;
+            const score = evalR.score;
+            const isPass = evalR.is_pass;
+            const error = evalR.error_message;
+            allTests.push({
+                name,
+                classname: 'agent-eval-labs',
+                failed: isPass === false,
+                errored: !!error,
+                message: error
+                    ? error
+                    : isPass === false
+                        ? `Expected ${String(evalR.expected_value ?? '')} but got ${String(evalR.actual_value ?? '')}`
+                        : '',
+                score: score != null ? score.toFixed(3) : 'N/A',
+            });
+        }
+        for (const err of testResult.errors ?? []) {
+            const stepId = err.id ?? 'unknown';
+            allTests.push({
+                name: `${testId}.${stepId}`,
+                classname: 'agent-eval-labs',
+                failed: false,
+                errored: true,
+                message: err.error_message ?? 'Unknown error',
+                score: 'N/A',
+            });
+        }
+    }
+    const totalTests = allTests.length;
+    const failures = allTests.filter((t) => t.failed).length;
+    const errors = allTests.filter((t) => t.errored).length;
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<testsuites>',
+        `  <testsuite name="agent-eval-labs" tests="${totalTests}" failures="${failures}" errors="${errors}">`,
+    ];
+    for (const tc of allTests) {
+        lines.push(`    <testcase name="${escapeXml(tc.name)}" classname="${escapeXml(tc.classname)}">`);
+        if (tc.errored) {
+            lines.push(`      <error message="${escapeXml(tc.message)}">${escapeXml(tc.message)}</error>`);
+        }
+        else if (tc.failed) {
+            lines.push(`      <failure message="${escapeXml(tc.message)}">Score: ${tc.score}</failure>`);
+        }
+        lines.push('    </testcase>');
+    }
+    lines.push('  </testsuite>');
+    lines.push('</testsuites>');
+    return lines.join('\n');
+}
+function buildTapEntries(results) {
+    const entries = [];
+    for (const testResult of results.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        for (const evalR of testResult.evaluation_results ?? []) {
+            const stepId = evalR.id ?? 'unknown';
+            const name = `${testId}.${stepId}`;
+            const score = evalR.score;
+            const isPass = evalR.is_pass;
+            const error = evalR.error_message;
+            entries.push({
+                ok: isPass !== false && !error,
+                name,
+                score: score != null ? score.toFixed(3) : 'N/A',
+                expected: evalR.expected_value != null ? String(evalR.expected_value) : undefined,
+                actual: evalR.actual_value != null ? String(evalR.actual_value) : undefined,
+                error: error ?? undefined,
+            });
+        }
+        for (const err of testResult.errors ?? []) {
+            const stepId = err.id ?? 'unknown';
+            entries.push({
+                ok: false,
+                name: `${testId}.${stepId}`,
+                score: 'N/A',
+                error: err.error_message ?? 'Unknown error',
+            });
+        }
+    }
+    return entries;
+}
+function formatTap(results) {
+    const entries = buildTapEntries(results);
+    const lines = ['TAP version 13', `1..${entries.length}`];
+    for (let i = 0; i < entries.length; i++) {
+        const e = entries[i];
+        const num = i + 1;
+        const prefix = e.ok ? 'ok' : 'not ok';
+        lines.push(`${prefix} ${num} - ${e.name} (score: ${e.score})`);
+        if (!e.ok) {
+            lines.push('  ---');
+            if (e.expected !== undefined) {
+                lines.push(`  expected: "${e.expected}"`);
+            }
+            if (e.actual !== undefined) {
+                lines.push(`  actual: "${e.actual}"`);
+            }
+            if (e.error) {
+                lines.push(`  error: "${e.error}"`);
+            }
+            lines.push('  ...');
+        }
+    }
+    return lines.join('\n');
+}
+function escapeXml(str) {
+    return str
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+//# sourceMappingURL=evalFormatter.js.map

package/lib/evalFormatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalFormatter.js","sourceRoot":"","sources":["../src/evalFormatter.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAgDH,sCAaC;AAbD,SAAgB,aAAa,CAAC,OAAwB,EAAE,MAAoB;IAC1E,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;QAC9B,KAAK,MAAM;YACT,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;QAC9B,KAAK,KAAK;YACR,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B;YACE,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED,8BAA8B;AAE9B,SAAS,iBAAiB,CAAC,OAAqB;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;QAE/B,IAAI,QAAQ,KAAK,sBAAsB,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QACnD,CAAC;aAAM,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC7B,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;iBAAM,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;gBAC3E,MAAM,GAAI,IAAiD,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;YAC3F,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;YACnF,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,MAAM,UAAU,EAAE,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,QAAQ,KAAK,iBAAiB,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC7B,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;gBAC5E,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,IAAmE,CAAC;gBAC9G,MAAM,QAAQ,GAAG,WAAW,EAAE,aAAa,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,yBAAyB,QAAQ,EAAE,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,2BAA2B,QAAQ,EAAE,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,WAAyB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC1D,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7E,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YAElC,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,kBAAkB,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,MAAM,QAAQ,MAAM,OAAO,MAAM,MAAM,MAAM,QAAQ,IAAI,CAAC,CAAC;YACrF,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAmB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,OAAO,QAAQ,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,sBAAsB,CAAC,WAAyB,EAAE,MAAmB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;IAEtF,KAAK,CAAC,IAAI,CAAC,gBAAgB,UAAU,cAAc,CAAC,CAAC;IACrD,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,aAAa,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,8BAA8B,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAwB;IAC3C,MAAM,KAAK,GAAa,CAAC,8BAA8B,CAAC,CAAC;IAEzD,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC;QAEzC,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,IAAI,CAAC,CAAC;QAEnC,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,WAAW,CAAC,OAAwB;IAC3C,MAAM,QAAQ,GAOT,EAAE,CAAC;IAER,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YAElC,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI;gBACJ,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,MAAM,KAAK,KAAK;gBACxB,OAAO,EAAE,CAAC,CAAC,KAAK;gBAChB,OAAO,EAAE,KAAK;oBACZ,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,MAAM,KAAK,KAAK;wBAClB,CAAC,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE;wBAC9F,CAAC,CAAC,EAAE;gBACN,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;aAChD,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,IAAI,SAAS,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE;gBAC3B,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,GAAG,CAAC,aAAa,IAAI,eAAe;gBAC7C,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAExD,MAAM,KAAK,GAAa;QACtB,wCAAwC;QACxC,cAAc;QACd,8CAA8C,UAAU,eAAe,QAAQ,aAAa,MAAM,IAAI;KACvG,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,uBAAuB,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACjG,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACjG,CAAC;aAAM,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,2BAA2B,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,YAAY,CAAC,CAAC;QAC/F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAE5B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAaD,SAAS,eAAe,CAAC,OAAwB;IAC/C,MAAM,OAAO,GAAe,EAAE,CAAC;IAE/B,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YAElC,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC,KAAK;gBAC9B,IAAI;gBACJ,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC/C,QAAQ,EAAE,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;gBACjF,MAAM,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC3E,KAAK,EAAE,KAAK,IAAI,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,IAAI,SAAS,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,KAAK;gBACT,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE;gBAC3B,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,GAAG,CAAC,aAAa,IAAI,eAAe;aAC5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAwB;IACzC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAEzC,MAAM,KAAK,GAAa,CAAC,gBAAgB,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;QAE/D,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACZ,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YACtC,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,GAAG;SACP,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC"}

package/lib/evalNormalizer.d.ts

@@ -0,0 +1,57 @@
+export type EvalPayload = {
+    tests: EvalTest[];
+};
+export type EvalTest = {
+    id: string;
+    steps: EvalStep[];
+};
+export type EvalStep = {
+    [key: string]: unknown;
+    type: string;
+    id: string;
+};
+/**
+ * Apply all normalizations to a test payload.
+ * Passes run in order: mcp-shorthand -> auto-correct -> camelCase -> evaluator fields -> shorthand refs -> defaults -> strip.
+ */
+export declare function normalizePayload(payload: EvalPayload): EvalPayload;
+/**
+ * Convert MCP shorthand format to raw Eval API format.
+ * MCP uses type="evaluator" + evaluator_type, raw API uses type="evaluator.xxx".
+ * Also maps `field` to `actual` with proper JSONPath and auto-generates missing `id` fields.
+ */
+export declare function normalizeMcpShorthand(steps: EvalStep[]): EvalStep[];
+/**
+ * Auto-correct common field name mistakes.
+ * Maps wrong field names to correct ones (agentId->agent_id, text->utterance, etc.)
+ */
+export declare function autoCorrectFields(steps: EvalStep[]): EvalStep[];
+/**
+ * Normalize camelCase agent field names to snake_case.
+ * useAgentApi->use_agent_api, plannerDefinitionId->planner_id, etc.
+ */
+export declare function normalizeCamelCase(steps: EvalStep[]): EvalStep[];
+/**
+ * Normalize evaluator field names based on evaluator category.
+ * Maps actual/expected <-> generated_output/reference_answer.
+ * Also auto-lowercases operator values and auto-injects metric_name.
+ */
+export declare function normalizeEvaluatorFields(steps: EvalStep[]): EvalStep[];
+/**
+ * Convert {step_id.field} shorthand references to JSONPath $.outputs[N].field.
+ * Builds step_id->index mapping from non-evaluator steps.
+ */
+export declare function convertShorthandRefs(steps: EvalStep[]): EvalStep[];
+/**
+ * Inject default values:
+ * - use_agent_api=true on agent.create_session if neither use_agent_api nor planner_id present
+ */
+export declare function injectDefaults(steps: EvalStep[]): EvalStep[];
+/**
+ * Strip unrecognized fields from steps based on type-specific whitelists.
+ */
+export declare function stripUnrecognizedFields(steps: EvalStep[]): EvalStep[];
+/**
+ * Split tests array into chunks of batchSize.
+ */
+export declare function splitIntoBatches(tests: EvalTest[], batchSize: number): EvalTest[][];