@salesforce/plugin-agent 1.30.10 → 1.31.0

package/lib/evalFormatter.js

@@ -0,0 +1,263 @@
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function formatResults(results, format) {
+    switch (format) {
+        case 'human':
+            return formatHuman(results);
+        case 'json':
+            return JSON.stringify(results, null, 2);
+        case 'junit':
+            return formatJunit(results);
+        case 'tap':
+            return formatTap(results);
+        default:
+            return formatHuman(results);
+    }
+}
+// --- formatHuman helpers ---
+function formatOutputLines(outputs) {
+    const lines = [];
+    for (const output of outputs) {
+        const stepType = output.type ?? '';
+        const stepId = output.id ?? '';
+        if (stepType === 'agent.create_session') {
+            const sessionId = output.session_id ?? 'N/A';
+            lines.push(`- **Create Session**: ${sessionId}`);
+        }
+        else if (stepType === 'agent.send_message') {
+            let agentMsg = output.response;
+            if (agentMsg !== null && typeof agentMsg === 'object' && !Array.isArray(agentMsg)) {
+                const msgObj = agentMsg;
+                const msgs = msgObj.messages;
+                agentMsg = msgs?.[0]?.message ?? String(agentMsg);
+            }
+            const msgStr = String(agentMsg ?? '');
+            const displayMsg = msgStr.length > 200 ? msgStr.substring(0, 200) + '...' : msgStr;
+            lines.push(`- **Agent Response** (${stepId}): ${displayMsg}`);
+        }
+        else if (stepType === 'agent.get_state') {
+            const respData = output.response;
+            if (respData !== null && typeof respData === 'object') {
+                const resp = respData;
+                const planner = resp.planner_response;
+                const lastExec = planner?.lastExecution;
+                const topic = lastExec?.topic ?? 'N/A';
+                const latency = lastExec?.latency ?? 'N/A';
+                lines.push(`- **Topic Selected**: ${String(topic)}`);
+                lines.push(`- **Response Latency**: ${String(latency)}ms`);
+            }
+            else {
+                lines.push(`- **State**: ${String(respData).substring(0, 200)}`);
+            }
+        }
+    }
+    return lines;
+}
+function formatEvaluationTable(evalResults) {
+    const lines = [];
+    if (evalResults.length > 0) {
+        lines.push('### Evaluation Results\n');
+        lines.push('| Metric | Score | Pass | Actual | Expected |');
+        lines.push('|--------|-------|------|--------|----------|');
+        for (const evalR of evalResults) {
+            const metricId = evalR.id ?? 'unknown';
+            const score = evalR.score;
+            const scoreStr = score != null ? score.toFixed(3) : 'N/A';
+            const isPass = evalR.is_pass;
+            const passStr = isPass === true ? 'PASS' : isPass === false ? 'FAIL' : 'N/A';
+            const actual = String(evalR.actual_value ?? '').substring(0, 60);
+            const expected = String(evalR.expected_value ?? '').substring(0, 60);
+            const error = evalR.error_message;
+            if (error) {
+                lines.push(`| ${metricId} | ERROR | - | ${error.substring(0, 80)} | - |`);
+            }
+            else {
+                lines.push(`| ${metricId} | ${scoreStr} | ${passStr} | ${actual} | ${expected} |`);
+            }
+        }
+        lines.push('');
+    }
+    return lines;
+}
+function formatErrorLines(errors) {
+    const lines = [];
+    if (errors.length > 0) {
+        lines.push('### Errors\n');
+        for (const error of errors) {
+            const errorId = error.id ?? 'unknown';
+            const errorMsg = error.error_message ?? String(error);
+            lines.push(`- **${errorId}**: ${errorMsg}`);
+        }
+        lines.push('');
+    }
+    return lines;
+}
+function formatTestSummaryLines(evalResults, errors) {
+    const lines = [];
+    const totalEvals = evalResults.length;
+    const passed = evalResults.filter((e) => e.is_pass === true).length;
+    const failed = evalResults.filter((e) => e.is_pass === false).length;
+    const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length;
+    lines.push(`**Summary**: ${totalEvals} evaluations`);
+    if (passed || failed) {
+        lines.push(`  - Passed: ${passed}, Failed: ${failed}`);
+    }
+    if (scored) {
+        lines.push(`  - Scored (no threshold): ${scored}`);
+    }
+    if (errors.length > 0) {
+        lines.push(`  - Errors: ${errors.length}`);
+    }
+    lines.push('');
+    return lines;
+}
+function formatHuman(results) {
+    const lines = ['# Agent Evaluation Results\n'];
+    for (const testResult of results.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        const errors = testResult.errors ?? [];
+        const evalResults = testResult.evaluation_results ?? [];
+        const outputs = testResult.outputs ?? [];
+        lines.push(`## Test: ${testId}\n`);
+        lines.push(...formatOutputLines(outputs));
+        lines.push('');
+        lines.push(...formatEvaluationTable(evalResults));
+        lines.push(...formatErrorLines(errors));
+        lines.push(...formatTestSummaryLines(evalResults, errors));
+    }
+    return lines.join('\n');
+}
+function formatJunit(results) {
+    const allTests = [];
+    for (const testResult of results.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        for (const evalR of testResult.evaluation_results ?? []) {
+            const stepId = evalR.id ?? 'unknown';
+            const name = `${testId}.${stepId}`;
+            const score = evalR.score;
+            const isPass = evalR.is_pass;
+            const error = evalR.error_message;
+            allTests.push({
+                name,
+                classname: 'agent-eval-labs',
+                failed: isPass === false,
+                errored: !!error,
+                message: error
+                    ? error
+                    : isPass === false
+                        ? `Expected ${String(evalR.expected_value ?? '')} but got ${String(evalR.actual_value ?? '')}`
+                        : '',
+                score: score != null ? score.toFixed(3) : 'N/A',
+            });
+        }
+        for (const err of testResult.errors ?? []) {
+            const stepId = err.id ?? 'unknown';
+            allTests.push({
+                name: `${testId}.${stepId}`,
+                classname: 'agent-eval-labs',
+                failed: false,
+                errored: true,
+                message: err.error_message ?? 'Unknown error',
+                score: 'N/A',
+            });
+        }
+    }
+    const totalTests = allTests.length;
+    const failures = allTests.filter((t) => t.failed).length;
+    const errors = allTests.filter((t) => t.errored).length;
+    const lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<testsuites>',
+        `  <testsuite name="agent-eval-labs" tests="${totalTests}" failures="${failures}" errors="${errors}">`,
+    ];
+    for (const tc of allTests) {
+        lines.push(`    <testcase name="${escapeXml(tc.name)}" classname="${escapeXml(tc.classname)}">`);
+        if (tc.errored) {
+            lines.push(`      <error message="${escapeXml(tc.message)}">${escapeXml(tc.message)}</error>`);
+        }
+        else if (tc.failed) {
+            lines.push(`      <failure message="${escapeXml(tc.message)}">Score: ${tc.score}</failure>`);
+        }
+        lines.push('    </testcase>');
+    }
+    lines.push('  </testsuite>');
+    lines.push('</testsuites>');
+    return lines.join('\n');
+}
+function buildTapEntries(results) {
+    const entries = [];
+    for (const testResult of results.results ?? []) {
+        const testId = testResult.id ?? 'unknown';
+        for (const evalR of testResult.evaluation_results ?? []) {
+            const stepId = evalR.id ?? 'unknown';
+            const name = `${testId}.${stepId}`;
+            const score = evalR.score;
+            const isPass = evalR.is_pass;
+            const error = evalR.error_message;
+            entries.push({
+                ok: isPass !== false && !error,
+                name,
+                score: score != null ? score.toFixed(3) : 'N/A',
+                expected: evalR.expected_value != null ? String(evalR.expected_value) : undefined,
+                actual: evalR.actual_value != null ? String(evalR.actual_value) : undefined,
+                error: error ?? undefined,
+            });
+        }
+        for (const err of testResult.errors ?? []) {
+            const stepId = err.id ?? 'unknown';
+            entries.push({
+                ok: false,
+                name: `${testId}.${stepId}`,
+                score: 'N/A',
+                error: err.error_message ?? 'Unknown error',
+            });
+        }
+    }
+    return entries;
+}
+function formatTap(results) {
+    const entries = buildTapEntries(results);
+    const lines = ['TAP version 13', `1..${entries.length}`];
+    for (let i = 0; i < entries.length; i++) {
+        const e = entries[i];
+        const num = i + 1;
+        const prefix = e.ok ? 'ok' : 'not ok';
+        lines.push(`${prefix} ${num} - ${e.name} (score: ${e.score})`);
+        if (!e.ok) {
+            lines.push('  ---');
+            if (e.expected !== undefined) {
+                lines.push(`  expected: "${e.expected}"`);
+            }
+            if (e.actual !== undefined) {
+                lines.push(`  actual: "${e.actual}"`);
+            }
+            if (e.error) {
+                lines.push(`  error: "${e.error}"`);
+            }
+            lines.push('  ...');
+        }
+    }
+    return lines.join('\n');
+}
+function escapeXml(str) {
+    return str
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+//# sourceMappingURL=evalFormatter.js.map

package/lib/evalFormatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalFormatter.js","sourceRoot":"","sources":["../src/evalFormatter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAoCH,MAAM,UAAU,aAAa,CAAC,OAAwB,EAAE,MAAoB;IAC1E,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;QAC9B,KAAK,MAAM;YACT,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;QAC9B,KAAK,KAAK;YACR,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;QAC5B;YACE,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED,8BAA8B;AAE9B,SAAS,iBAAiB,CAAC,OAAqB;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;QAE/B,IAAI,QAAQ,KAAK,sBAAsB,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QACnD,CAAC;aAAM,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;YAC7C,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC/B,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAClF,MAAM,MAAM,GAAG,QAAmC,CAAC;gBACnD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAsD,CAAC;gBAC3E,QAAQ,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;YACtC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;YACnF,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,MAAM,UAAU,EAAE,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,QAAQ,KAAK,iBAAiB,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACjC,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACtD,MAAM,IAAI,GAAG,QAAmC,CAAC;gBACjD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAuD,CAAC;gBAC7E,MAAM,QAAQ,GAAG,OAAO,EAAE,aAAoD,CAAC;gBAC/E,MAAM,KAAK,GAAG,QAAQ,EAAE,KAAK,IAAI,KAAK,CAAC;gBACvC,MAAM,OAAO,GAAG,QAAQ,EAAE,OAAO,IAAI,KAAK,CAAC;gBAC3C,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACrD,KAAK,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,WAAyB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC1D,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7E,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YAElC,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,kBAAkB,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,MAAM,QAAQ,MAAM,OAAO,MAAM,MAAM,MAAM,QAAQ,IAAI,CAAC,CAAC;YACrF,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAmB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,OAAO,QAAQ,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,sBAAsB,CAAC,WAAyB,EAAE,MAAmB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;IAEtF,KAAK,CAAC,IAAI,CAAC,gBAAgB,UAAU,cAAc,CAAC,CAAC;IACrD,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,aAAa,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,8BAA8B,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAwB;IAC3C,MAAM,KAAK,GAAa,CAAC,8BAA8B,CAAC,CAAC;IAEzD,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,UAAU,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC;QAEzC,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,IAAI,CAAC,CAAC;QAEnC,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,WAAW,CAAC,OAAwB;IAC3C,MAAM,QAAQ,GAOT,EAAE,CAAC;IAER,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YAElC,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI;gBACJ,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,MAAM,KAAK,KAAK;gBACxB,OAAO,EAAE,CAAC,CAAC,KAAK;gBAChB,OAAO,EAAE,KAAK;oBACZ,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,MAAM,KAAK,KAAK;wBAClB,CAAC,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE;wBAC9F,CAAC,CAAC,EAAE;gBACN,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;aAChD,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,IAAI,SAAS,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE;gBAC3B,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,GAAG,CAAC,aAAa,IAAI,eAAe;gBAC7C,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAExD,MAAM,KAAK,GAAa;QACtB,wCAAwC;QACxC,cAAc;QACd,8CAA8C,UAAU,eAAe,QAAQ,aAAa,MAAM,IAAI;KACvG,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,uBAAuB,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACjG,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACjG,CAAC;aAAM,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,2BAA2B,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,YAAY,CAAC,CAAC;QAC/F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAE5B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAaD,SAAS,eAAe,CAAC,OAAwB;IAC/C,MAAM,OAAO,GAAe,EAAE,CAAC;IAE/B,KAAK,MAAM,UAAU,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,SAAS,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YAElC,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC,KAAK;gBAC9B,IAAI;gBACJ,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC/C,QAAQ,EAAE,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;gBACjF,MAAM,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC3E,KAAK,EAAE,KAAK,IAAI,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,IAAI,SAAS,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,KAAK;gBACT,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE;gBAC3B,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,GAAG,CAAC,aAAa,IAAI,eAAe;aAC5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,OAAwB;IACzC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAEzC,MAAM,KAAK,GAAa,CAAC,gBAAgB,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;QAE/D,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACZ,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YACtC,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,GAAG;SACP,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC"}

package/lib/evalNormalizer.d.ts

@@ -0,0 +1,57 @@
+export type EvalPayload = {
+    tests: EvalTest[];
+};
+export type EvalTest = {
+    id: string;
+    steps: EvalStep[];
+};
+export type EvalStep = {
+    [key: string]: unknown;
+    type: string;
+    id: string;
+};
+/**
+ * Apply all normalizations to a test payload.
+ * Passes run in order: mcp-shorthand -> auto-correct -> camelCase -> evaluator fields -> shorthand refs -> defaults -> strip.
+ */
+export declare function normalizePayload(payload: EvalPayload): EvalPayload;
+/**
+ * Convert MCP shorthand format to raw Eval API format.
+ * MCP uses type="evaluator" + evaluator_type, raw API uses type="evaluator.xxx".
+ * Also maps `field` to `actual` with proper JSONPath and auto-generates missing `id` fields.
+ */
+export declare function normalizeMcpShorthand(steps: EvalStep[]): EvalStep[];
+/**
+ * Auto-correct common field name mistakes.
+ * Maps wrong field names to correct ones (agentId->agent_id, text->utterance, etc.)
+ */
+export declare function autoCorrectFields(steps: EvalStep[]): EvalStep[];
+/**
+ * Normalize camelCase agent field names to snake_case.
+ * useAgentApi->use_agent_api, plannerDefinitionId->planner_id, etc.
+ */
+export declare function normalizeCamelCase(steps: EvalStep[]): EvalStep[];
+/**
+ * Normalize evaluator field names based on evaluator category.
+ * Maps actual/expected <-> generated_output/reference_answer.
+ * Also auto-lowercases operator values and auto-injects metric_name.
+ */
+export declare function normalizeEvaluatorFields(steps: EvalStep[]): EvalStep[];
+/**
+ * Convert {step_id.field} shorthand references to JSONPath $.outputs[N].field.
+ * Builds step_id->index mapping from non-evaluator steps.
+ */
+export declare function convertShorthandRefs(steps: EvalStep[]): EvalStep[];
+/**
+ * Inject default values:
+ * - use_agent_api=true on agent.create_session if neither use_agent_api nor planner_id present
+ */
+export declare function injectDefaults(steps: EvalStep[]): EvalStep[];
+/**
+ * Strip unrecognized fields from steps based on type-specific whitelists.
+ */
+export declare function stripUnrecognizedFields(steps: EvalStep[]): EvalStep[];
+/**
+ * Split tests array into chunks of batchSize.
+ */
+export declare function splitIntoBatches(tests: EvalTest[], batchSize: number): EvalTest[][];