@salesforce/agents 1.5.1 → 1.5.2

package/lib/yamlSpecTranslator.js

@@ -0,0 +1,234 @@
+"use strict";
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isYamlTestSpec = isYamlTestSpec;
+exports.parseTestSpec = parseTestSpec;
+exports.translateTestSpec = translateTestSpec;
+exports.translateTestCase = translateTestCase;
+/* eslint-disable camelcase */
+const core_1 = require("@salesforce/core");
+const yaml_1 = require("yaml");
+// --- JSONPath mappings from org model to Eval API refs ---
+const ACTUAL_PATH_MAP = {
+    '$.generatedData.outcome': '{sm.response}',
+    '$.generatedData.topic': '{gs.response.planner_response.lastExecution.topic}',
+    '$.generatedData.invokedActions': '{gs.response.planner_response.lastExecution.invokedActions}',
+    '$.generatedData.actionsSequence': '{gs.response.planner_response.lastExecution.invokedActions}',
+};
+// --- Custom evaluation name to evaluator type mapping ---
+const CUSTOM_EVAL_TYPE_MAP = {
+    string_comparison: 'evaluator.string_assertion',
+    numeric_comparison: 'evaluator.numeric_assertion',
+};
+// JSONPaths that require the get_state step
+const PLANNER_PATHS = new Set([
+    '$.generatedData.topic',
+    '$.generatedData.invokedActions',
+    '$.generatedData.actionsSequence',
+]);
+// --- Public API ---
+/**
+ * Returns true if the content looks like a YAML TestSpec (has testCases + subjectName).
+ * Returns false for JSON EvalPayload, invalid content, or YAML missing required fields.
+ */
+function isYamlTestSpec(content) {
+    try {
+        const parsed = (0, yaml_1.parse)(content);
+        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            return false;
+        }
+        const obj = parsed;
+        return Array.isArray(obj.testCases) && typeof obj.subjectName === 'string';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Parse a YAML string into a TestSpec.
+ * Throws if the content is not valid YAML or is missing required fields.
+ */
+function parseTestSpec(content) {
+    const parsed = (0, yaml_1.parse)(content);
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new core_1.SfError('Invalid TestSpec: expected a YAML object');
+    }
+    const obj = parsed;
+    if (!Array.isArray(obj.testCases)) {
+        throw new core_1.SfError('Invalid TestSpec: missing testCases array');
+    }
+    if (typeof obj.subjectName !== 'string') {
+        throw new core_1.SfError('Invalid TestSpec: missing subjectName');
+    }
+    if (typeof obj.name !== 'string') {
+        throw new core_1.SfError('Invalid TestSpec: missing name');
+    }
+    return parsed;
+}
+/**
+ * Translate a full TestSpec into an EvalPayload.
+ */
+function translateTestSpec(spec) {
+    return {
+        tests: spec.testCases.map((tc, idx) => translateTestCase(tc, idx, spec.name)),
+    };
+}
+/**
+ * Translate a single TestCase into an EvalTest with ordered steps.
+ */
+function translateTestCase(testCase, index, specName) {
+    const id = specName ? `${specName}_case_${index}` : `test_case_${index}`;
+    const steps = [];
+    // 1. agent.create_session
+    const createSessionStep = {
+        type: 'agent.create_session',
+        id: 'cs',
+        use_agent_api: true,
+    };
+    if (testCase.contextVariables && testCase.contextVariables.length > 0) {
+        // Validate for duplicate names
+        const names = testCase.contextVariables.map((cv) => cv.name);
+        const duplicates = names.filter((name, idx) => names.indexOf(name) !== idx);
+        if (duplicates.length > 0) {
+            throw new core_1.SfError(`Duplicate contextVariable names found in test case ${index}: ${[...new Set(duplicates)].join(', ')}. Each contextVariable name must be unique.`);
+        }
+        createSessionStep.context_variables = Object.fromEntries(testCase.contextVariables.map((cv) => [cv.name, cv.value]));
+    }
+    steps.push(createSessionStep);
+    // 2. Conversation history — only user messages become send_message steps
+    let historyIdx = 0;
+    if (testCase.conversationHistory) {
+        for (const entry of testCase.conversationHistory) {
+            if (entry.role === 'user') {
+                steps.push({
+                    type: 'agent.send_message',
+                    id: `history_${historyIdx}`,
+                    session_id: '{cs.session_id}',
+                    utterance: entry.message,
+                });
+                historyIdx++;
+            }
+        }
+    }
+    // 3. Test utterance
+    steps.push({
+        type: 'agent.send_message',
+        id: 'sm',
+        session_id: '{cs.session_id}',
+        utterance: testCase.utterance,
+    });
+    // 4. Determine if get_state is needed
+    const needsGetState = needsPlannerState(testCase);
+    if (needsGetState) {
+        steps.push({
+            type: 'agent.get_state',
+            id: 'gs',
+            session_id: '{cs.session_id}',
+        });
+    }
+    // 5. Evaluators
+    if (testCase.expectedTopic !== undefined) {
+        steps.push({
+            type: 'evaluator.planner_topic_assertion',
+            id: 'check_topic',
+            expected: testCase.expectedTopic,
+            actual: '{gs.response.planner_response.lastExecution.topic}',
+            operator: 'contains',
+        });
+    }
+    if (testCase.expectedActions !== undefined && testCase.expectedActions.length > 0) {
+        steps.push({
+            type: 'evaluator.planner_actions_assertion',
+            id: 'check_actions',
+            expected: testCase.expectedActions,
+            actual: '{gs.response.planner_response.lastExecution.invokedActions}',
+            operator: 'includes_items',
+        });
+    }
+    if (testCase.expectedOutcome !== undefined) {
+        steps.push({
+            type: 'evaluator.bot_response_rating',
+            id: 'check_outcome',
+            utterance: testCase.utterance,
+            expected: testCase.expectedOutcome,
+            actual: '{sm.response}',
+            threshold: 3.0,
+        });
+    }
+    if (testCase.customEvaluations) {
+        testCase.customEvaluations.forEach((customEval, customIdx) => {
+            const step = translateCustomEvaluation(customEval, customIdx);
+            steps.push(step);
+        });
+    }
+    return { id, steps };
+}
+// --- Internal helpers ---
+/**
+ * Determine whether the get_state step is needed for this test case.
+ */
+function needsPlannerState(testCase) {
+    if (testCase.expectedTopic !== undefined)
+        return true;
+    if (testCase.expectedActions !== undefined && testCase.expectedActions.length > 0)
+        return true;
+    if (testCase.customEvaluations) {
+        for (const customEval of testCase.customEvaluations) {
+            for (const param of customEval.parameters) {
+                if (param.name === 'actual' && PLANNER_PATHS.has(param.value)) {
+                    return true;
+                }
+            }
+        }
+    }
+    return false;
+}
+/**
+ * Translate a single customEvaluation entry into an EvalStep.
+ */
+function translateCustomEvaluation(customEval, index) {
+    const evalType = CUSTOM_EVAL_TYPE_MAP[customEval.name] ?? `evaluator.${customEval.name}`;
+    let operator = '';
+    let actual = '';
+    let expected = '';
+    for (const param of customEval.parameters) {
+        if (param.name === 'operator') {
+            operator = param.value;
+        }
+        else if (param.name === 'actual') {
+            actual = mapActualPath(param.value);
+        }
+        else if (param.name === 'expected') {
+            expected = param.value;
+        }
+    }
+    return {
+        type: evalType,
+        id: `custom_${index}`,
+        operator,
+        actual,
+        expected,
+    };
+}
+/**
+ * Map an org-model JSONPath to the Eval API shorthand ref.
+ * Unknown paths are returned as-is.
+ */
+function mapActualPath(path) {
+    return ACTUAL_PATH_MAP[path] ?? path;
+}
+//# sourceMappingURL=yamlSpecTranslator.js.map

package/lib/yamlSpecTranslator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yamlSpecTranslator.js","sourceRoot":"","sources":["../src/yamlSpecTranslator.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAsCH,wCAWC;AAMD,sCAgBC;AAKD,8CAIC;AAKD,8CAwGC;AA3LD,8BAA8B;AAE9B,2CAA2C;AAC3C,+BAA0C;AAI1C,4DAA4D;AAE5D,MAAM,eAAe,GAA2B;IAC9C,yBAAyB,EAAE,eAAe;IAC1C,uBAAuB,EAAE,oDAAoD;IAC7E,gCAAgC,EAAE,6DAA6D;IAC/F,iCAAiC,EAAE,6DAA6D;CACjG,CAAC;AAEF,2DAA2D;AAE3D,MAAM,oBAAoB,GAA2B;IACnD,iBAAiB,EAAE,4BAA4B;IAC/C,kBAAkB,EAAE,6BAA6B;CAClD,CAAC;AAEF,4CAA4C;AAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,uBAAuB;IACvB,gCAAgC;IAChC,iCAAiC;CAClC,CAAC,CAAC;AAEH,qBAAqB;AAErB;;;GAGG;AACH,SAAgB,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAA,YAAS,EAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAC9C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAgB,aAAa,CAAC,OAAe;IAC3C,MAAM,MAAM,GAAY,IAAA,YAAS,EAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,cAAO,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,cAAO,CAAC,2CAA2C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,cAAO,CAAC,uCAAuC,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,cAAO,CAAC,gCAAgC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB,CAAC,IAAc;IAC9C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB,CAAC,QAAkB,EAAE,KAAa,EAAE,QAAiB;IACpF,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,EAAE,CAAC;IACzE,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,0BAA0B;IAC1B,MAAM,iBAAiB,GAAa;QAClC,IAAI,EAAE,sBAAsB;QAC5B,EAAE,EAAE,IAAI;QACR,aAAa,EAAE,IAAI;KACpB,CAAC;IAEF,IAAI,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtE,+BAA+B;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QAC5E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,cAAO,CACf,sDAAsD,KAAK,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAC3F,IAAI,CACL,6CAA6C,CAC/C,CAAC;QACJ,CAAC;QAED,iBAAiB,CAAC,iBAAiB,GAAG,MAAM,CAAC,WAAW,CACtD,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAE9B,yEAAyE;IACzE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,oBAAoB;oBAC1B,EAAE,EAAE,WAAW,UAAU,EAAE;oBAC3B,UAAU,EAAE,iBAAiB;oBAC7B,SAAS,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,oBAAoB;QAC1B,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,iBAAiB;QAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,iBAAiB;YACvB,EAAE,EAAE,IAAI;YACR,UAAU,EAAE,iBAAiB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB;IAChB,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,mCAAmC;YACzC,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,QAAQ,CAAC,aAAa;YAChC,MAAM,EAAE,oDAAoD;YAC5D,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,qCAAqC;YAC3C,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,6DAA6D;YACrE,QAAQ,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,+BAA+B;YACrC,EAAE,EAAE,eAAe;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,eAAe;YACvB,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,yBAAyB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,2BAA2B;AAE3B;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAkB;IAC3C,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtD,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/F,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAChC,UAA8D,EAC9D,KAAa;IAEb,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,UAAU,CAAC,IAAI,EAAE,CAAC;IAEzF,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9B,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,EAAE,EAAE,UAAU,KAAK,EAAE;QACrB,QAAQ;QACR,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@salesforce/agents",
   "description": "Client side APIs for working with Salesforce agents",
-  "version": "1.5.1",
+  "version": "1.5.2",
   "license": "Apache-2.0",
   "author": "Salesforce",
   "main": "lib/index",