@salesforce/agents 1.5.1 → 1.6.0

package/lib/evalNormalizer.js

@@ -0,0 +1,442 @@
+"use strict";
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizePayload = normalizePayload;
+exports.normalizeMcpShorthand = normalizeMcpShorthand;
+exports.autoCorrectFields = autoCorrectFields;
+exports.normalizeCamelCase = normalizeCamelCase;
+exports.normalizeEvaluatorFields = normalizeEvaluatorFields;
+exports.convertShorthandRefs = convertShorthandRefs;
+exports.injectDefaults = injectDefaults;
+exports.stripUnrecognizedFields = stripUnrecognizedFields;
+exports.splitIntoBatches = splitIntoBatches;
+// --- Evaluator classification ---
+const SCORING_EVALUATORS = new Set([
+    'evaluator.text_alignment',
+    'evaluator.hallucination_detection',
+    'evaluator.citation_recall',
+    'evaluator.answer_faithfulness',
+]);
+const ASSERTION_EVALUATORS = new Set(['evaluator.string_assertion', 'evaluator.json_assertion']);
+const DEFAULT_METRIC_NAMES = {
+    'evaluator.text_alignment': 'base.cosine_similarity',
+    'evaluator.hallucination_detection': 'hallucination_detection',
+    'evaluator.citation_recall': 'citation_recall',
+    'evaluator.answer_faithfulness': 'answer_faithfulness',
+};
+const SCORING_VALID_FIELDS = new Set([
+    'type',
+    'id',
+    'generated_output',
+    'reference_answer',
+    'metric_name',
+    'threshold',
+]);
+const ASSERTION_VALID_FIELDS = new Set([
+    'type',
+    'id',
+    'actual',
+    'expected',
+    'operator',
+    'threshold',
+    'json_path',
+    'json_schema',
+    'metric_name',
+]);
+const VALID_AGENT_FIELDS = {
+    'agent.create_session': new Set([
+        'type',
+        'id',
+        'agent_id',
+        'agent_version_id',
+        'use_agent_api',
+        'planner_id',
+        'state',
+        'setupSessionContext',
+        'context_variables',
+    ]),
+    'agent.send_message': new Set(['type', 'id', 'session_id', 'utterance']),
+    'agent.get_state': new Set(['type', 'id', 'session_id']),
+};
+// --- Auto-correction maps ---
+const AGENT_CORRECTIONS = {
+    agentId: 'agent_id',
+    agentVersionId: 'agent_version_id',
+    sessionId: 'session_id',
+    text: 'utterance',
+    message: 'utterance',
+    input: 'utterance',
+    prompt: 'utterance',
+    user_message: 'utterance',
+    userMessage: 'utterance',
+};
+const EVALUATOR_CORRECTIONS = {
+    subject: 'actual',
+    expectedValue: 'expected',
+    expected_value: 'expected',
+    actualValue: 'actual',
+    actual_value: 'actual',
+    assertionType: 'operator',
+    assertion_type: 'operator',
+    comparator: 'operator',
+};
+// --- camelCase alias maps for agent.create_session ---
+const AGENT_FIELD_ALIASES = {
+    useAgentApi: 'use_agent_api',
+    plannerId: 'planner_id',
+    plannerDefinitionId: 'planner_id',
+    planner_definition_id: 'planner_id',
+    planner_version_id: 'planner_id',
+    plannerVersionId: 'planner_id',
+};
+// --- Scoring evaluator field aliases ---
+const SCORING_FIELD_ALIASES = {
+    actual: 'generated_output',
+    expected: 'reference_answer',
+    actual_value: 'generated_output',
+    expected_value: 'reference_answer',
+    actual_output: 'generated_output',
+    expected_output: 'reference_answer',
+    response: 'generated_output',
+    ground_truth: 'reference_answer',
+};
+// --- Assertion evaluator field aliases ---
+const ASSERTION_FIELD_ALIASES = {
+    actual_value: 'actual',
+    expected_value: 'expected',
+    generated_output: 'actual',
+    reference_answer: 'expected',
+    actual_output: 'actual',
+    expected_output: 'expected',
+    response: 'actual',
+    ground_truth: 'expected',
+};
+// --- MCP shorthand field mapping ---
+// MCP uses `field: "gs1.planner_state.topic"` — map to Eval API `actual` with correct JSONPath
+const MCP_FIELD_MAP = {
+    'planner_state.topic': 'response.planner_response.lastExecution.topic',
+    'planner_state.invokedActions': 'response.planner_response.lastExecution.invokedActions',
+    'planner_state.actionsSequence': 'response.planner_response.lastExecution.invokedActions',
+    response: 'response',
+    'response.messages': 'response',
+};
+// --- Main entry point ---
+/**
+ * Apply all normalizations to a test payload.
+ * Passes run in order: mcp-shorthand -> auto-correct -> camelCase -> evaluator fields -> shorthand refs -> defaults -> strip.
+ */
+function normalizePayload(payload) {
+    const normalized = {
+        tests: payload.tests.map((test) => {
+            let steps = [...test.steps];
+            steps = normalizeMcpShorthand(steps);
+            steps = autoCorrectFields(steps);
+            steps = normalizeCamelCase(steps);
+            steps = normalizeEvaluatorFields(steps);
+            steps = convertShorthandRefs(steps);
+            steps = injectDefaults(steps);
+            steps = stripUnrecognizedFields(steps);
+            return { ...test, steps };
+        }),
+    };
+    return normalized;
+}
+// --- Individual normalization passes ---
+/**
+ * Convert MCP shorthand format to raw Eval API format.
+ * MCP uses type="evaluator" + evaluator_type, raw API uses type="evaluator.xxx".
+ * Also maps `field` to `actual` with proper JSONPath and auto-generates missing `id` fields.
+ */
+function normalizeMcpShorthand(steps) {
+    let evalCounter = 0;
+    return steps.map((step) => {
+        const evaluator_type = step.evaluator_type;
+        // Only applies to MCP shorthand: type="evaluator" with evaluator_type field
+        if (step.type !== 'evaluator' || !evaluator_type)
+            return step;
+        const normalized = { ...step };
+        // Merge type: "evaluator" + evaluator_type: "xxx" → type: "evaluator.xxx"
+        normalized.type = `evaluator.${evaluator_type}`;
+        delete normalized.evaluator_type;
+        // Convert `field` to `actual` with proper shorthand ref format
+        if ('field' in normalized) {
+            if (!('actual' in normalized)) {
+                const fieldValue = normalized.field;
+                // Parse "gs1.planner_state.topic" → stepId="gs1", fieldPath="planner_state.topic"
+                const dotIdx = fieldValue.indexOf('.');
+                if (dotIdx > 0) {
+                    const stepId = fieldValue.substring(0, dotIdx);
+                    const fieldPath = fieldValue.substring(dotIdx + 1);
+                    const mappedPath = MCP_FIELD_MAP[fieldPath] ?? fieldPath;
+                    normalized.actual = `{${stepId}.${mappedPath}}`;
+                }
+                else {
+                    normalized.actual = fieldValue;
+                }
+            }
+            delete normalized.field;
+        }
+        // Auto-generate id if missing
+        if (!normalized.id || normalized.id === '') {
+            normalized.id = `eval_${evalCounter}`;
+            evalCounter++;
+        }
+        return normalized;
+    });
+}
+/**
+ * Auto-correct common field name mistakes.
+ * Maps wrong field names to correct ones (agentId->agent_id, text->utterance, etc.)
+ */
+function autoCorrectFields(steps) {
+    return steps.map((step) => {
+        const corrected = { ...step };
+        const stepType = corrected.type ?? '';
+        if (stepType.startsWith('agent.')) {
+            for (const [wrong, correct] of Object.entries(AGENT_CORRECTIONS)) {
+                if (wrong in corrected && !(correct in corrected)) {
+                    corrected[correct] = corrected[wrong];
+                    delete corrected[wrong];
+                }
+            }
+        }
+        else if (stepType.startsWith('evaluator.')) {
+            for (const [wrong, correct] of Object.entries(EVALUATOR_CORRECTIONS)) {
+                if (wrong in corrected && !(correct in corrected)) {
+                    corrected[correct] = corrected[wrong];
+                    delete corrected[wrong];
+                }
+            }
+        }
+        return corrected;
+    });
+}
+/**
+ * Normalize camelCase agent field names to snake_case.
+ * useAgentApi->use_agent_api, plannerDefinitionId->planner_id, etc.
+ */
+function normalizeCamelCase(steps) {
+    return steps.map((step) => {
+        if (step.type !== 'agent.create_session')
+            return step;
+        const normalized = { ...step };
+        for (const [alias, canonical] of Object.entries(AGENT_FIELD_ALIASES)) {
+            if (alias in normalized) {
+                if (!(canonical in normalized)) {
+                    normalized[canonical] = normalized[alias];
+                }
+                delete normalized[alias];
+            }
+        }
+        return normalized;
+    });
+}
+/**
+ * Apply field aliases: remap alias keys to canonical keys, removing duplicates.
+ */
+function applyFieldAliases(step, aliases) {
+    for (const [alias, canonical] of Object.entries(aliases)) {
+        if (alias in step && !(canonical in step)) {
+            step[canonical] = step[alias];
+            delete step[alias];
+        }
+        else if (alias in step && canonical in step) {
+            delete step[alias];
+        }
+    }
+}
+/**
+ * Normalize a scoring evaluator step (field aliases + metric_name injection).
+ */
+function normalizeScoringEvaluator(normalized, evalType) {
+    applyFieldAliases(normalized, SCORING_FIELD_ALIASES);
+    // Auto-inject or correct metric_name
+    if (!('metric_name' in normalized)) {
+        const defaultMetric = DEFAULT_METRIC_NAMES[evalType];
+        if (defaultMetric) {
+            normalized.metric_name = defaultMetric;
+        }
+    }
+    else if (normalized.metric_name === evalType.split('.')[1]) {
+        const defaultMetric = DEFAULT_METRIC_NAMES[evalType];
+        if (defaultMetric) {
+            normalized.metric_name = defaultMetric;
+        }
+    }
+}
+/**
+ * Normalize an assertion evaluator step (field aliases + operator lowercase + metric_name).
+ */
+function normalizeAssertionEvaluator(normalized, evalType) {
+    applyFieldAliases(normalized, ASSERTION_FIELD_ALIASES);
+    // Auto-lowercase operator
+    if ('operator' in normalized && typeof normalized.operator === 'string') {
+        normalized.operator = normalized.operator.toLowerCase();
+    }
+    // Auto-inject metric_name for assertion evaluators
+    if (!('metric_name' in normalized)) {
+        normalized.metric_name = evalType.split('.')[1];
+    }
+}
+/**
+ * Normalize evaluator field names based on evaluator category.
+ * Maps actual/expected <-> generated_output/reference_answer.
+ * Also auto-lowercases operator values and auto-injects metric_name.
+ */
+function normalizeEvaluatorFields(steps) {
+    return steps.map((step) => {
+        const evalType = step.type ?? '';
+        if (!evalType.startsWith('evaluator.'))
+            return step;
+        const normalized = { ...step };
+        if (SCORING_EVALUATORS.has(evalType)) {
+            normalizeScoringEvaluator(normalized, evalType);
+        }
+        else if (ASSERTION_EVALUATORS.has(evalType)) {
+            normalizeAssertionEvaluator(normalized, evalType);
+        }
+        // Don't inject metric_name for unknown evaluator types to avoid API validation errors
+        // Unknown evaluators like bot_response_rating and planner_topic_assertion don't use metric_name
+        return normalized;
+    });
+}
+/**
+ * Convert {step_id.field} shorthand references to JSONPath $.outputs[N].field.
+ * Builds step_id->index mapping from non-evaluator steps.
+ */
+function convertShorthandRefs(steps) {
+    // Build step_id -> output-array index mapping
+    const stepIdToIdx = {};
+    let outputIdx = 0;
+    for (const step of steps) {
+        const sid = step.id;
+        const stype = step.type ?? '';
+        if (sid && !stype.startsWith('evaluator.')) {
+            stepIdToIdx[sid] = outputIdx;
+            outputIdx += 1;
+        }
+    }
+    const refPattern = /\{([^}]+)\}/g;
+    function replaceValue(value) {
+        if (typeof value !== 'string')
+            return value;
+        return value.replace(refPattern, (match, ref) => {
+            const dotIdx = ref.indexOf('.');
+            if (dotIdx < 0)
+                return match;
+            const sid = ref.substring(0, dotIdx);
+            let field = ref.substring(dotIdx + 1);
+            if (!(sid in stepIdToIdx))
+                return match;
+            const idx = stepIdToIdx[sid];
+            // Normalize legacy nested-response path to flat response
+            if (field.startsWith('response.messages')) {
+                field = 'response';
+            }
+            return `$.outputs[${idx}].${field}`;
+        });
+    }
+    return steps.map((step) => {
+        const newStep = {};
+        for (const [key, val] of Object.entries(step)) {
+            if (typeof val === 'string') {
+                newStep[key] = replaceValue(val);
+            }
+            else if (val !== null && typeof val === 'object' && !Array.isArray(val)) {
+                const newObj = {};
+                for (const [k, v] of Object.entries(val)) {
+                    newObj[k] = typeof v === 'string' ? replaceValue(v) : v;
+                }
+                newStep[key] = newObj;
+            }
+            else if (Array.isArray(val)) {
+                newStep[key] = val.map((item) => typeof item === 'string' ? replaceValue(item) : item);
+            }
+            else {
+                newStep[key] = val;
+            }
+        }
+        return newStep;
+    });
+}
+/**
+ * Inject default values:
+ * - use_agent_api=true on agent.create_session if neither use_agent_api nor planner_id present
+ */
+function injectDefaults(steps) {
+    return steps.map((step) => {
+        if (step.type === 'agent.create_session') {
+            if (!('use_agent_api' in step) && !('planner_id' in step)) {
+                return { ...step, use_agent_api: true };
+            }
+        }
+        return step;
+    });
+}
+/**
+ * Strip unrecognized fields from steps based on type-specific whitelists.
+ */
+function stripUnrecognizedFields(steps) {
+    return steps.map((step) => {
+        const stepType = step.type ?? '';
+        // Agent steps
+        if (stepType in VALID_AGENT_FIELDS) {
+            const validFields = VALID_AGENT_FIELDS[stepType];
+            const stripped = {};
+            for (const [key, val] of Object.entries(step)) {
+                if (validFields.has(key)) {
+                    stripped[key] = val;
+                }
+            }
+            return stripped;
+        }
+        // Scoring evaluators
+        if (SCORING_EVALUATORS.has(stepType)) {
+            const stripped = {};
+            for (const [key, val] of Object.entries(step)) {
+                if (SCORING_VALID_FIELDS.has(key)) {
+                    stripped[key] = val;
+                }
+            }
+            return stripped;
+        }
+        // Assertion evaluators
+        if (ASSERTION_EVALUATORS.has(stepType)) {
+            const stripped = {};
+            for (const [key, val] of Object.entries(step)) {
+                if (ASSERTION_VALID_FIELDS.has(key)) {
+                    stripped[key] = val;
+                }
+            }
+            return stripped;
+        }
+        // Unknown types: don't strip (to avoid breaking future evaluator types)
+        return step;
+    });
+}
+// --- Batch splitting ---
+/**
+ * Split tests array into chunks of batchSize.
+ */
+function splitIntoBatches(tests, batchSize) {
+    const batches = [];
+    for (let i = 0; i < tests.length; i += batchSize) {
+        batches.push(tests.slice(i, i + batchSize));
+    }
+    return batches;
+}
+//# sourceMappingURL=evalNormalizer.js.map

package/lib/evalNormalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalNormalizer.js","sourceRoot":"","sources":["../src/evalNormalizer.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AA2JH,4CAeC;AASD,sDA0CC;AAMD,8CAuBC;AAMD,gDAeC;AA0DD,4DAiBC;AAMD,oDA2DC;AAMD,wCASC;AAKD,0DAyCC;AAOD,4CAMC;AAhdD,mCAAmC;AAEnC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,0BAA0B;IAC1B,mCAAmC;IACnC,2BAA2B;IAC3B,+BAA+B;CAChC,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAC,CAAC;AAEjG,MAAM,oBAAoB,GAA2B;IACnD,0BAA0B,EAAE,wBAAwB;IACpD,mCAAmC,EAAE,yBAAyB;IAC9D,2BAA2B,EAAE,iBAAiB;IAC9C,+BAA+B,EAAE,qBAAqB;CACvD,CAAC;AAEF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,MAAM;IACN,IAAI;IACJ,kBAAkB;IAClB,kBAAkB;IAClB,aAAa;IACb,WAAW;CACZ,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,MAAM;IACN,IAAI;IACJ,QAAQ;IACR,UAAU;IACV,UAAU;IACV,WAAW;IACX,WAAW;IACX,aAAa;IACb,aAAa;CACd,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAgC;IACtD,sBAAsB,EAAE,IAAI,GAAG,CAAC;QAC9B,MAAM;QACN,IAAI;QACJ,UAAU;QACV,kBAAkB;QAClB,eAAe;QACf,YAAY;QACZ,OAAO;QACP,qBAAqB;QACrB,mBAAmB;KACpB,CAAC;IACF,oBAAoB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IACxE,iBAAiB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;CACzD,CAAC;AAEF,+BAA+B;AAE/B,MAAM,iBAAiB,GAA2B;IAChD,OAAO,EAAE,UAAU;IACnB,cAAc,EAAE,kBAAkB;IAClC,SAAS,EAAE,YAAY;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,WAAW;IAClB,MAAM,EAAE,WAAW;IACnB,YAAY,EAAE,WAAW;IACzB,WAAW,EAAE,WAAW;CACzB,CAAC;AAEF,MAAM,qBAAqB,GAA2B;IACpD,OAAO,EAAE,QAAQ;IACjB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,UAAU;IAC1B,WAAW,EAAE,QAAQ;IACrB,YAAY,EAAE,QAAQ;IACtB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,UAAU;IAC1B,UAAU,EAAE,UAAU;CACvB,CAAC;AAEF,wDAAwD;AAExD,MAAM,mBAAmB,GAA2B;IAClD,WAAW,EAAE,eAAe;IAC5B,SAAS,EAAE,YAAY;IACvB,mBAAmB,EAAE,YAAY;IACjC,qBAAqB,EAAE,YAAY;IACnC,kBAAkB,EAAE,YAAY;IAChC,gBAAgB,EAAE,YAAY;CAC/B,CAAC;AAEF,0CAA0C;AAE1C,MAAM,qBAAqB,GAA2B;IACpD,MAAM,EAAE,kBAAkB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,YAAY,EAAE,kBAAkB;IAChC,cAAc,EAAE,kBAAkB;IAClC,aAAa,EAAE,kBAAkB;IACjC,eAAe,EAAE,kBAAkB;IACnC,QAAQ,EAAE,kBAAkB;IAC5B,YAAY,EAAE,kBAAkB;CACjC,CAAC;AAEF,4CAA4C;AAE5C,MAAM,uBAAuB,GAA2B;IACtD,YAAY,EAAE,QAAQ;IACtB,cAAc,EAAE,UAAU;IAC1B,gBAAgB,EAAE,QAAQ;IAC1B,gBAAgB,EAAE,UAAU;IAC5B,aAAa,EAAE,QAAQ;IACvB,eAAe,EAAE,UAAU;IAC3B,QAAQ,EAAE,QAAQ;IAClB,YAAY,EAAE,UAAU;CACzB,CAAC;AAEF,sCAAsC;AAEtC,+FAA+F;AAC/F,MAAM,aAAa,GAA2B;IAC5C,qBAAqB,EAAE,+CAA+C;IACtE,8BAA8B,EAAE,wDAAwD;IACxF,+BAA+B,EAAE,wDAAwD;IACzF,QAAQ,EAAE,UAAU;IACpB,mBAAmB,EAAE,UAAU;CAChC,CAAC;AAEF,2BAA2B;AAE3B;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,OAAoB;IACnD,MAAM,UAAU,GAAgB;QAC9B,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAChC,IAAI,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YACrC,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAClC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;YACxC,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;YACpC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAC9B,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC,CAAC;KACH,CAAC;IACF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,0CAA0C;AAE1C;;;;GAIG;AACH,SAAgB,qBAAqB,CAAC,KAAiB;IACrD,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAoC,CAAC;QAEjE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAE9D,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAE/B,0EAA0E;QAC1E,UAAU,CAAC,IAAI,GAAG,aAAa,cAAc,EAAE,CAAC;QAChD,OAAO,UAAU,CAAC,cAAc,CAAC;QAEjC,+DAA+D;QAC/D,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,QAAQ,IAAI,UAAU,CAAC,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,UAAU,CAAC,KAAe,CAAC;gBAE9C,kFAAkF;gBAClF,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACvC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACnD,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;oBACzD,UAAU,CAAC,MAAM,GAAG,IAAI,MAAM,IAAI,UAAU,GAAG,CAAC;gBAClD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC;gBACjC,CAAC;YACH,CAAC;YACD,OAAO,UAAU,CAAC,KAAK,CAAC;QAC1B,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3C,UAAU,CAAC,EAAE,GAAG,QAAQ,WAAW,EAAE,CAAC;YACtC,WAAW,EAAE,CAAC;QAChB,CAAC;QAED,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,KAAiB;IACjD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;QAEtC,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACjE,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,EAAE,CAAC;oBAClD,SAAS,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBACrE,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,EAAE,CAAC;oBAClD,SAAS,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAqB,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAgB,kBAAkB,CAAC,KAAiB;IAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB;YAAE,OAAO,IAAI,CAAC;QAEtD,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACrE,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;gBACxB,IAAI,CAAC,CAAC,SAAS,IAAI,UAAU,CAAC,EAAE,CAAC;oBAC/B,UAAU,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC5C,CAAC;gBACD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAc,EAAE,OAA+B;IACxE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,KAAK,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,UAAoB,EAAE,QAAgB;IACvE,iBAAiB,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAC;IAErD,qCAAqC;IACrC,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,WAAW,GAAG,aAAa,CAAC;QACzC,CAAC;IACH,CAAC;SAAM,IAAI,UAAU,CAAC,WAAW,KAAK,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,WAAW,GAAG,aAAa,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,UAAoB,EAAE,QAAgB;IACzE,iBAAiB,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,UAAU,IAAI,UAAU,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxE,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC1D,CAAC;IAED,mDAAmD;IACnD,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,EAAE,CAAC;QACnC,UAAU,CAAC,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAgB,wBAAwB,CAAC,KAAiB;IACxD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpD,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAE/B,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,yBAAyB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9C,2BAA2B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,CAAC;QACD,sFAAsF;QACtF,gGAAgG;QAEhG,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,KAAiB;IACpD,8CAA8C;IAC9C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3C,WAAW,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YAC7B,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,cAAc,CAAC;IAElC,SAAS,YAAY,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE5C,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;YACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7B,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEtC,IAAI,CAAC,CAAC,GAAG,IAAI,WAAW,CAAC;gBAAE,OAAO,KAAK,CAAC;YAExC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YAE7B,yDAAyD;YACzD,IAAI,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBAC1C,KAAK,GAAG,UAAU,CAAC;YACrB,CAAC;YAED,OAAO,aAAa,GAAG,KAAK,KAAK,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,OAAO,GAA4B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;iBAAM,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1E,MAAM,MAAM,GAA4B,EAAE,CAAC;gBAC3C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;oBACpE,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,GAAI,GAAiB,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CACtD,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CACrD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,OAAmB,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAAC,KAAiB;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACzC,IAAI,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC1D,OAAO,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;YAC1C,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAgB,uBAAuB,CAAC,KAAiB;IACvD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAEjC,cAAc;QACd,IAAI,QAAQ,IAAI,kBAAkB,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,qBAAqB;QACrB,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,uBAAuB;QACvB,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,wEAAwE;QACxE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0BAA0B;AAE1B;;GAEG;AACH,SAAgB,gBAAgB,CAAC,KAAiB,EAAE,SAAiB;IACnE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/lib/index.d.ts

@@ -1,5 +1,5 @@
-export { type BaseAgentConfig, type AgentPreviewStartResponse, type AgentPreviewSendResponse, type AgentPreviewEndResponse, type EndReason, type ApiStatus, type AgentJson, type AgentCompilationSuccess, type AgentScriptContent, type AgentCreateConfig, type AgentCreateResponse, type AgentJobSpec, type AgentJobSpecCreateConfig, type AgentOptions, type AgentTone, type AgentType, type BotMetadata, type BotVersionMetadata, type PreviewableAgent, type CompilationError, type DraftAgentTopics, type DraftAgentTopicsBody, type DraftAgentTopicsResponse, type AvailableDefinition, type AgentPreviewMessageLinks, type AgentPreviewMessage, type AgentPreviewError, AgentSource, type ScriptAgentType, type ProductionAgentType, type AgentTestResultsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, type TestCaseResult, type TestStatus, type AgentTestConfig, type TestCase, type TestSpec, type MetadataMetric, type MetadataExpectation, type MetadataCustomEvaluation, type AiEvaluationDefinition, type AgentforceStudioTestStartResponse, type AgentforceStudioTestStatusResponse, type AgentforceStudioTestResultsResponse, type TestScorerResult, type AgentforceStudioTestCaseResult, type AgentTraceResponse, type AgentTraceStep, type UserInputStep, type LLMExecutionStep, type UpdateTopicStep, type EventStep, type ReasoningStep, type PlannerResponseStep, COMPILATION_API_EXIT_CODES, } from './types';
-export { metric, findAuthoringBundle, readTranscriptEntries, determineTestRunner, detectTestRunnerFromId, type TestRunnerType, createPreviewSessionCache, validatePreviewSession, removePreviewSessionCache, getCachedPreviewSessionIds, getCurrentPreviewSessionId, listCachedPreviewSessions, type SessionType, type PreviewSessionMeta, type CachedPreviewSessionInfo, type CachedPreviewSessionEntry, } from './utils';
+export { type BaseAgentConfig, type AgentPreviewStartResponse, type AgentPreviewSendResponse, type AgentPreviewEndResponse, type EndReason, type ApiStatus, type AgentJson, type AgentCompilationSuccess, type AgentScriptContent, type AgentCreateConfig, type AgentCreateResponse, type AgentJobSpec, type AgentJobSpecCreateConfig, type AgentOptions, type AgentTone, type AgentType, type BotMetadata, type BotVersionMetadata, type PreviewableAgent, type CompilationError, type DraftAgentTopics, type DraftAgentTopicsBody, type DraftAgentTopicsResponse, type AvailableDefinition, type AgentPreviewMessageLinks, type AgentPreviewMessage, type AgentPreviewError, AgentSource, type ScriptAgentType, type ProductionAgentType, type AgentTestResultsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, type TestCaseResult, type TestStatus, type AgentTestConfig, type TestCase, type TestSpec, type MetadataMetric, type MetadataExpectation, type MetadataCustomEvaluation, type AiEvaluationDefinition, type AgentforceStudioTestStartResponse, type AgentforceStudioTestStatusResponse, type AgentforceStudioTestResultsResponse, type TestScorerResult, type AgentforceStudioTestCaseResult, type AgentTraceResponse, type AgentTraceStep, type PlannerResponse, type PlanStep, type UserInputStep, type LLMExecutionStep, type UpdateTopicStep, type EventStep, type FunctionStep, type ReasoningStep, type PlannerResponseStep, COMPILATION_API_EXIT_CODES, } from './types';
+export { metric, findAuthoringBundle, readTranscriptEntries, determineTestRunner, detectTestRunnerFromId, type TestRunnerType, createPreviewSessionCache, validatePreviewSession, removePreviewSessionCache, getCachedPreviewSessionIds, getCurrentPreviewSessionId, listCachedPreviewSessions, listSessionTraces, readSessionTrace, readTurnIndex, type TraceFileInfo, type TurnIndex, type TurnIndexEntry, type SessionType, type PreviewSessionMeta, type CachedPreviewSessionInfo, type CachedPreviewSessionEntry, } from './utils';
 export { Agent, AgentCreateLifecycleStages, type AgentInstance } from './agent';
 export { AgentTester } from './agentTester';
 export { AgentforceStudioTester, normalizeAgentforceStudioResults } from './agentforceStudioTester';
@@ -11,3 +11,7 @@ export { AgentBase } from './agents/agentBase';
 export { convertTestResultsToFormat, humanFriendlyName } from './agentTestResults';
 export { writeDebugLog } from './apexUtils';
 export { applyStringReplacements, applyStringReplacementsToAgent, type ReplacementConfig, type ReplacementResult, } from './stringReplacements';
+export { normalizePayload, normalizeMcpShorthand, autoCorrectFields, normalizeCamelCase, normalizeEvaluatorFields, convertShorthandRefs, injectDefaults, stripUnrecognizedFields, splitIntoBatches, type EvalPayload, type EvalTest, type EvalStep, } from './evalNormalizer';
+export { formatResults, type EvalApiResponse, type EvalOutput, type EvalResult, type TestError, type TestResult, type ResultFormat, } from './evalFormatter';
+export { isYamlTestSpec, parseTestSpec, translateTestSpec, translateTestCase, } from './yamlSpecTranslator';
+export { resolveAgent, executeBatches, buildResultSummary, type AgentEvalRunResult, } from './agentEvalRunner';

package/lib/index.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyStringReplacementsToAgent = exports.applyStringReplacements = exports.writeDebugLog = exports.humanFriendlyName = exports.convertTestResultsToFormat = exports.AgentBase = exports.ScriptAgent = exports.ProductionAgent = exports.AgentTestCreateLifecycleStages = exports.AgentTest = exports.createAgentTester = exports.normalizeAgentforceStudioResults = exports.AgentforceStudioTester = exports.AgentTester = exports.AgentCreateLifecycleStages = exports.Agent = exports.listCachedPreviewSessions = exports.getCurrentPreviewSessionId = exports.getCachedPreviewSessionIds = exports.removePreviewSessionCache = exports.validatePreviewSession = exports.createPreviewSessionCache = exports.detectTestRunnerFromId = exports.determineTestRunner = exports.readTranscriptEntries = exports.findAuthoringBundle = exports.metric = exports.COMPILATION_API_EXIT_CODES = exports.AgentSource = void 0;
+exports.buildResultSummary = exports.executeBatches = exports.resolveAgent = exports.translateTestCase = exports.translateTestSpec = exports.parseTestSpec = exports.isYamlTestSpec = exports.formatResults = exports.splitIntoBatches = exports.stripUnrecognizedFields = exports.injectDefaults = exports.convertShorthandRefs = exports.normalizeEvaluatorFields = exports.normalizeCamelCase = exports.autoCorrectFields = exports.normalizeMcpShorthand = exports.normalizePayload = exports.applyStringReplacementsToAgent = exports.applyStringReplacements = exports.writeDebugLog = exports.humanFriendlyName = exports.convertTestResultsToFormat = exports.AgentBase = exports.ScriptAgent = exports.ProductionAgent = exports.AgentTestCreateLifecycleStages = exports.AgentTest = exports.createAgentTester = exports.normalizeAgentforceStudioResults = exports.AgentforceStudioTester = exports.AgentTester = exports.AgentCreateLifecycleStages = exports.Agent = exports.readTurnIndex = exports.readSessionTrace = exports.listSessionTraces = exports.listCachedPreviewSessions = exports.getCurrentPreviewSessionId = exports.getCachedPreviewSessionIds = exports.removePreviewSessionCache = exports.validatePreviewSession = exports.createPreviewSessionCache = exports.detectTestRunnerFromId = exports.determineTestRunner = exports.readTranscriptEntries = exports.findAuthoringBundle = exports.metric = exports.COMPILATION_API_EXIT_CODES = exports.AgentSource = void 0;
 var types_1 = require("./types");
 Object.defineProperty(exports, "AgentSource", { enumerable: true, get: function () { return types_1.AgentSource; } });
 // Compilation API exit codes (CLI contract)
@@ -32,6 +32,9 @@ Object.defineProperty(exports, "removePreviewSessionCache", { enumerable: true,
 Object.defineProperty(exports, "getCachedPreviewSessionIds", { enumerable: true, get: function () { return utils_1.getCachedPreviewSessionIds; } });
 Object.defineProperty(exports, "getCurrentPreviewSessionId", { enumerable: true, get: function () { return utils_1.getCurrentPreviewSessionId; } });
 Object.defineProperty(exports, "listCachedPreviewSessions", { enumerable: true, get: function () { return utils_1.listCachedPreviewSessions; } });
+Object.defineProperty(exports, "listSessionTraces", { enumerable: true, get: function () { return utils_1.listSessionTraces; } });
+Object.defineProperty(exports, "readSessionTrace", { enumerable: true, get: function () { return utils_1.readSessionTrace; } });
+Object.defineProperty(exports, "readTurnIndex", { enumerable: true, get: function () { return utils_1.readTurnIndex; } });
 var agent_1 = require("./agent");
 Object.defineProperty(exports, "Agent", { enumerable: true, get: function () { return agent_1.Agent; } });
 Object.defineProperty(exports, "AgentCreateLifecycleStages", { enumerable: true, get: function () { return agent_1.AgentCreateLifecycleStages; } });
@@ -59,4 +62,25 @@ Object.defineProperty(exports, "writeDebugLog", { enumerable: true, get: functio
 var stringReplacements_1 = require("./stringReplacements");
 Object.defineProperty(exports, "applyStringReplacements", { enumerable: true, get: function () { return stringReplacements_1.applyStringReplacements; } });
 Object.defineProperty(exports, "applyStringReplacementsToAgent", { enumerable: true, get: function () { return stringReplacements_1.applyStringReplacementsToAgent; } });
+var evalNormalizer_1 = require("./evalNormalizer");
+Object.defineProperty(exports, "normalizePayload", { enumerable: true, get: function () { return evalNormalizer_1.normalizePayload; } });
+Object.defineProperty(exports, "normalizeMcpShorthand", { enumerable: true, get: function () { return evalNormalizer_1.normalizeMcpShorthand; } });
+Object.defineProperty(exports, "autoCorrectFields", { enumerable: true, get: function () { return evalNormalizer_1.autoCorrectFields; } });
+Object.defineProperty(exports, "normalizeCamelCase", { enumerable: true, get: function () { return evalNormalizer_1.normalizeCamelCase; } });
+Object.defineProperty(exports, "normalizeEvaluatorFields", { enumerable: true, get: function () { return evalNormalizer_1.normalizeEvaluatorFields; } });
+Object.defineProperty(exports, "convertShorthandRefs", { enumerable: true, get: function () { return evalNormalizer_1.convertShorthandRefs; } });
+Object.defineProperty(exports, "injectDefaults", { enumerable: true, get: function () { return evalNormalizer_1.injectDefaults; } });
+Object.defineProperty(exports, "stripUnrecognizedFields", { enumerable: true, get: function () { return evalNormalizer_1.stripUnrecognizedFields; } });
+Object.defineProperty(exports, "splitIntoBatches", { enumerable: true, get: function () { return evalNormalizer_1.splitIntoBatches; } });
+var evalFormatter_1 = require("./evalFormatter");
+Object.defineProperty(exports, "formatResults", { enumerable: true, get: function () { return evalFormatter_1.formatResults; } });
+var yamlSpecTranslator_1 = require("./yamlSpecTranslator");
+Object.defineProperty(exports, "isYamlTestSpec", { enumerable: true, get: function () { return yamlSpecTranslator_1.isYamlTestSpec; } });
+Object.defineProperty(exports, "parseTestSpec", { enumerable: true, get: function () { return yamlSpecTranslator_1.parseTestSpec; } });
+Object.defineProperty(exports, "translateTestSpec", { enumerable: true, get: function () { return yamlSpecTranslator_1.translateTestSpec; } });
+Object.defineProperty(exports, "translateTestCase", { enumerable: true, get: function () { return yamlSpecTranslator_1.translateTestCase; } });
+var agentEvalRunner_1 = require("./agentEvalRunner");
+Object.defineProperty(exports, "resolveAgent", { enumerable: true, get: function () { return agentEvalRunner_1.resolveAgent; } });
+Object.defineProperty(exports, "executeBatches", { enumerable: true, get: function () { return agentEvalRunner_1.executeBatches; } });
+Object.defineProperty(exports, "buildResultSummary", { enumerable: true, get: function () { return agentEvalRunner_1.buildResultSummary; } });
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,iCAsEiB;AArCf,oGAAA,WAAW,OAAA;AAmCX,4CAA4C;AAC5C,mHAAA,0BAA0B,OAAA;AAG5B,iCAiBiB;AAhBf,+FAAA,MAAM,OAAA;AACN,4GAAA,mBAAmB,OAAA;AACnB,8GAAA,qBAAqB,OAAA;AACrB,4GAAA,mBAAmB,OAAA;AACnB,+GAAA,sBAAsB,OAAA;AAEtB,kHAAA,yBAAyB,OAAA;AACzB,+GAAA,sBAAsB,OAAA;AACtB,kHAAA,yBAAyB,OAAA;AACzB,mHAAA,0BAA0B,OAAA;AAC1B,mHAAA,0BAA0B,OAAA;AAC1B,kHAAA,yBAAyB,OAAA;AAM3B,iCAAgF;AAAvE,8FAAA,KAAK,OAAA;AAAE,mHAAA,0BAA0B,OAAA;AAC1C,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,mEAAoG;AAA3F,gIAAA,sBAAsB,OAAA;AAAE,0IAAA,gCAAgC,OAAA;AACjE,2DAAsH;AAA7G,uHAAA,iBAAiB,OAAA;AAC1B,yCAAwE;AAA/D,sGAAA,SAAS,OAAA;AAAE,2HAAA,8BAA8B,OAAA;AAClD,4DAA2D;AAAlD,kHAAA,eAAe,OAAA;AACxB,oDAAmD;AAA1C,0GAAA,WAAW,OAAA;AACpB,gDAA+C;AAAtC,sGAAA,SAAS,OAAA;AAClB,uDAAmF;AAA1E,8HAAA,0BAA0B,OAAA;AAAE,qHAAA,iBAAiB,OAAA;AACtD,yCAA4C;AAAnC,0GAAA,aAAa,OAAA;AACtB,2DAK8B;AAJ5B,6HAAA,uBAAuB,OAAA;AACvB,oIAAA,8BAA8B,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,iCAyEiB;AAxCf,oGAAA,WAAW,OAAA;AAsCX,4CAA4C;AAC5C,mHAAA,0BAA0B,OAAA;AAG5B,iCAuBiB;AAtBf,+FAAA,MAAM,OAAA;AACN,4GAAA,mBAAmB,OAAA;AACnB,8GAAA,qBAAqB,OAAA;AACrB,4GAAA,mBAAmB,OAAA;AACnB,+GAAA,sBAAsB,OAAA;AAEtB,kHAAA,yBAAyB,OAAA;AACzB,+GAAA,sBAAsB,OAAA;AACtB,kHAAA,yBAAyB,OAAA;AACzB,mHAAA,0BAA0B,OAAA;AAC1B,mHAAA,0BAA0B,OAAA;AAC1B,kHAAA,yBAAyB,OAAA;AACzB,0GAAA,iBAAiB,OAAA;AACjB,yGAAA,gBAAgB,OAAA;AAChB,sGAAA,aAAa,OAAA;AASf,iCAAgF;AAAvE,8FAAA,KAAK,OAAA;AAAE,mHAAA,0BAA0B,OAAA;AAC1C,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,mEAAoG;AAA3F,gIAAA,sBAAsB,OAAA;AAAE,0IAAA,gCAAgC,OAAA;AACjE,2DAAsH;AAA7G,uHAAA,iBAAiB,OAAA;AAC1B,yCAAwE;AAA/D,sGAAA,SAAS,OAAA;AAAE,2HAAA,8BAA8B,OAAA;AAClD,4DAA2D;AAAlD,kHAAA,eAAe,OAAA;AACxB,oDAAmD;AAA1C,0GAAA,WAAW,OAAA;AACpB,gDAA+C;AAAtC,sGAAA,SAAS,OAAA;AAClB,uDAAmF;AAA1E,8HAAA,0BAA0B,OAAA;AAAE,qHAAA,iBAAiB,OAAA;AACtD,yCAA4C;AAAnC,0GAAA,aAAa,OAAA;AACtB,2DAK8B;AAJ5B,6HAAA,uBAAuB,OAAA;AACvB,oIAAA,8BAA8B,OAAA;AAIhC,mDAa0B;AAZxB,kHAAA,gBAAgB,OAAA;AAChB,uHAAA,qBAAqB,OAAA;AACrB,mHAAA,iBAAiB,OAAA;AACjB,oHAAA,kBAAkB,OAAA;AAClB,0HAAA,wBAAwB,OAAA;AACxB,sHAAA,oBAAoB,OAAA;AACpB,gHAAA,cAAc,OAAA;AACd,yHAAA,uBAAuB,OAAA;AACvB,kHAAA,gBAAgB,OAAA;AAKlB,iDAQyB;AAPvB,8GAAA,aAAa,OAAA;AAQf,2DAK8B;AAJ5B,oHAAA,cAAc,OAAA;AACd,mHAAA,aAAa,OAAA;AACb,uHAAA,iBAAiB,OAAA;AACjB,uHAAA,iBAAiB,OAAA;AAEnB,qDAK2B;AAJzB,+GAAA,YAAY,OAAA;AACZ,iHAAA,cAAc,OAAA;AACd,qHAAA,kBAAkB,OAAA"}

package/lib/utils.d.ts

@@ -115,6 +115,27 @@ export declare const appendTranscriptToHistory: (entry: TranscriptEntry, session
  * @returns {Promise<void>}
  */
 export declare const writeTraceToHistory: (planId: string, trace: PlannerResponse | undefined, historyDir: string) => Promise<void>;
+export type TraceFileInfo = {
+    planId: string;
+    path: string;
+    size: number;
+    mtime: Date;
+};
+/**
+ * List trace files for a given agent session.
+ *
+ * Returns one entry per .json file in the session's traces/ directory.
+ * File path is absolute. Returns an empty array if the traces directory does not exist.
+ */
+export declare const listSessionTraces: (agentId: string, sessionId: string) => Promise<TraceFileInfo[]>;
+/**
+ * Read a single trace file by planId. Returns null if the file does not exist or cannot be parsed.
+ */
+export declare const readSessionTrace: (agentId: string, sessionId: string, planId: string) => Promise<PlannerResponse | null>;
+/**
+ * Read the turn-index.json for a session. Returns null if not found.
+ */
+export declare const readTurnIndex: (agentId: string, sessionId: string) => Promise<TurnIndex | null>;
 /**
  * Write or append a session line to .sfdx/agents/<agentId>/index.md.
  * If the file does not exist, creates it with a header and the session line.

package/lib/utils.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readTranscriptEntries = exports.getAllHistory = exports.recordTraceForTurn = exports.logTurnToHistory = exports.SessionHistoryBuffer = exports.logSessionToIndex = exports.writeTraceToHistory = exports.appendTranscriptToHistory = exports.getAgentIndexDir = exports.getHistoryDir = exports.useNamedUserJwt = exports.findLocalAgents = exports.findAuthoringBundle = exports.decodeHtmlEntities = exports.sanitizeFilename = exports.metric = void 0;
+exports.readTranscriptEntries = exports.getAllHistory = exports.recordTraceForTurn = exports.logTurnToHistory = exports.SessionHistoryBuffer = exports.logSessionToIndex = exports.readTurnIndex = exports.readSessionTrace = exports.listSessionTraces = exports.writeTraceToHistory = exports.appendTranscriptToHistory = exports.getAgentIndexDir = exports.getHistoryDir = exports.useNamedUserJwt = exports.findLocalAgents = exports.findAuthoringBundle = exports.decodeHtmlEntities = exports.sanitizeFilename = exports.metric = void 0;
 exports.getHttpStatusCode = getHttpStatusCode;
 exports.requestWithEndpointFallback = requestWithEndpointFallback;
 exports.detectTestRunnerFromId = detectTestRunnerFromId;
@@ -381,6 +381,60 @@ const writeTraceToHistory = async (planId, trace, historyDir) => {
     await (0, promises_1.writeFile)(tracePath, JSON.stringify(trace ?? {}, null, 2), 'utf-8');
 };
 exports.writeTraceToHistory = writeTraceToHistory;
+/**
+ * List trace files for a given agent session.
+ *
+ * Returns one entry per .json file in the session's traces/ directory.
+ * File path is absolute. Returns an empty array if the traces directory does not exist.
+ */
+const listSessionTraces = async (agentId, sessionId) => {
+    const historyDir = await (0, exports.getHistoryDir)(agentId, sessionId);
+    const tracesDir = path.join(historyDir, 'traces');
+    try {
+        const files = await (0, promises_1.readdir)(tracesDir);
+        return await Promise.all(files
+            .filter((f) => f.endsWith('.json'))
+            .map(async (f) => {
+            const filePath = path.join(tracesDir, f);
+            const s = await (0, promises_1.stat)(filePath);
+            return { planId: path.basename(f, '.json'), path: filePath, size: s.size, mtime: s.mtime };
+        }));
+    }
+    catch {
+        return [];
+    }
+};
+exports.listSessionTraces = listSessionTraces;
+/**
+ * Read a single trace file by planId. Returns null if the file does not exist or cannot be parsed.
+ */
+const readSessionTrace = async (agentId, sessionId, planId) => {
+    const historyDir = await (0, exports.getHistoryDir)(agentId, sessionId);
+    const tracePath = path.join(historyDir, 'traces', `${planId}.json`);
+    try {
+        const raw = await (0, promises_1.readFile)(tracePath, 'utf-8');
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+};
+exports.readSessionTrace = readSessionTrace;
+/**
+ * Read the turn-index.json for a session. Returns null if not found.
+ */
+const readTurnIndex = async (agentId, sessionId) => {
+    const historyDir = await (0, exports.getHistoryDir)(agentId, sessionId);
+    const turnIndexPath = path.join(historyDir, 'turn-index.json');
+    try {
+        const raw = await (0, promises_1.readFile)(turnIndexPath, 'utf-8');
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+};
+exports.readTurnIndex = readTurnIndex;
 /**
  * Write or append a session line to .sfdx/agents/<agentId>/index.md.
  * If the file does not exist, creates it with a header and the session line.