@salesforce/plugin-agent 1.36.1 → 1.37.0

package/lib/yamlSpecTranslator.js

@@ -1,227 +0,0 @@
-/*
- * Copyright 2026, Salesforce, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/* eslint-disable camelcase */
-import { parse as parseYaml } from 'yaml';
-// --- JSONPath mappings from org model to Eval API refs ---
-const ACTUAL_PATH_MAP = {
-    '$.generatedData.outcome': '{sm.response}',
-    '$.generatedData.topic': '{gs.response.planner_response.lastExecution.topic}',
-    '$.generatedData.invokedActions': '{gs.response.planner_response.lastExecution.invokedActions}',
-    '$.generatedData.actionsSequence': '{gs.response.planner_response.lastExecution.invokedActions}',
-};
-// --- Custom evaluation name to evaluator type mapping ---
-const CUSTOM_EVAL_TYPE_MAP = {
-    string_comparison: 'evaluator.string_assertion',
-    numeric_comparison: 'evaluator.numeric_assertion',
-};
-// JSONPaths that require the get_state step
-const PLANNER_PATHS = new Set([
-    '$.generatedData.topic',
-    '$.generatedData.invokedActions',
-    '$.generatedData.actionsSequence',
-]);
-// --- Public API ---
-/**
- * Returns true if the content looks like a YAML TestSpec (has testCases + subjectName).
- * Returns false for JSON EvalPayload, invalid content, or YAML missing required fields.
- */
-export function isYamlTestSpec(content) {
-    try {
-        const parsed = parseYaml(content);
-        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
-            return false;
-        }
-        const obj = parsed;
-        return Array.isArray(obj.testCases) && typeof obj.subjectName === 'string';
-    }
-    catch {
-        return false;
-    }
-}
-/**
- * Parse a YAML string into a TestSpec.
- * Throws if the content is not valid YAML or is missing required fields.
- */
-export function parseTestSpec(content) {
-    const parsed = parseYaml(content);
-    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
-        throw new Error('Invalid TestSpec: expected a YAML object');
-    }
-    const obj = parsed;
-    if (!Array.isArray(obj.testCases)) {
-        throw new Error('Invalid TestSpec: missing testCases array');
-    }
-    if (typeof obj.subjectName !== 'string') {
-        throw new Error('Invalid TestSpec: missing subjectName');
-    }
-    if (typeof obj.name !== 'string') {
-        throw new Error('Invalid TestSpec: missing name');
-    }
-    return parsed;
-}
-/**
- * Translate a full TestSpec into an EvalPayload.
- */
-export function translateTestSpec(spec) {
-    return {
-        tests: spec.testCases.map((tc, idx) => translateTestCase(tc, idx, spec.name)),
-    };
-}
-/**
- * Translate a single TestCase into an EvalTest with ordered steps.
- */
-export function translateTestCase(testCase, index, specName) {
-    const id = specName ? `${specName}_case_${index}` : `test_case_${index}`;
-    const steps = [];
-    // 1. agent.create_session
-    const createSessionStep = {
-        type: 'agent.create_session',
-        id: 'cs',
-        use_agent_api: true,
-    };
-    if (testCase.contextVariables && testCase.contextVariables.length > 0) {
-        // Validate for duplicate names
-        const names = testCase.contextVariables.map((cv) => cv.name);
-        const duplicates = names.filter((name, idx) => names.indexOf(name) !== idx);
-        if (duplicates.length > 0) {
-            throw new Error(`Duplicate contextVariable names found in test case ${index}: ${[...new Set(duplicates)].join(', ')}. Each contextVariable name must be unique.`);
-        }
-        createSessionStep.context_variables = Object.fromEntries(testCase.contextVariables.map((cv) => [cv.name, cv.value]));
-    }
-    steps.push(createSessionStep);
-    // 2. Conversation history — only user messages become send_message steps
-    let historyIdx = 0;
-    if (testCase.conversationHistory) {
-        for (const entry of testCase.conversationHistory) {
-            if (entry.role === 'user') {
-                steps.push({
-                    type: 'agent.send_message',
-                    id: `history_${historyIdx}`,
-                    session_id: '{cs.session_id}',
-                    utterance: entry.message,
-                });
-                historyIdx++;
-            }
-        }
-    }
-    // 3. Test utterance
-    steps.push({
-        type: 'agent.send_message',
-        id: 'sm',
-        session_id: '{cs.session_id}',
-        utterance: testCase.utterance,
-    });
-    // 4. Determine if get_state is needed
-    const needsGetState = needsPlannerState(testCase);
-    if (needsGetState) {
-        steps.push({
-            type: 'agent.get_state',
-            id: 'gs',
-            session_id: '{cs.session_id}',
-        });
-    }
-    // 5. Evaluators
-    if (testCase.expectedTopic !== undefined) {
-        steps.push({
-            type: 'evaluator.planner_topic_assertion',
-            id: 'check_topic',
-            expected: testCase.expectedTopic,
-            actual: '{gs.response.planner_response.lastExecution.topic}',
-            operator: 'contains',
-        });
-    }
-    if (testCase.expectedActions !== undefined && testCase.expectedActions.length > 0) {
-        steps.push({
-            type: 'evaluator.planner_actions_assertion',
-            id: 'check_actions',
-            expected: testCase.expectedActions,
-            actual: '{gs.response.planner_response.lastExecution.invokedActions}',
-            operator: 'includes_items',
-        });
-    }
-    if (testCase.expectedOutcome !== undefined) {
-        steps.push({
-            type: 'evaluator.bot_response_rating',
-            id: 'check_outcome',
-            utterance: testCase.utterance,
-            expected: testCase.expectedOutcome,
-            actual: '{sm.response}',
-            threshold: 3.0,
-        });
-    }
-    if (testCase.customEvaluations) {
-        testCase.customEvaluations.forEach((customEval, customIdx) => {
-            const step = translateCustomEvaluation(customEval, customIdx);
-            steps.push(step);
-        });
-    }
-    return { id, steps };
-}
-// --- Internal helpers ---
-/**
- * Determine whether the get_state step is needed for this test case.
- */
-function needsPlannerState(testCase) {
-    if (testCase.expectedTopic !== undefined)
-        return true;
-    if (testCase.expectedActions !== undefined && testCase.expectedActions.length > 0)
-        return true;
-    if (testCase.customEvaluations) {
-        for (const customEval of testCase.customEvaluations) {
-            for (const param of customEval.parameters) {
-                if (param.name === 'actual' && PLANNER_PATHS.has(param.value)) {
-                    return true;
-                }
-            }
-        }
-    }
-    return false;
-}
-/**
- * Translate a single customEvaluation entry into an EvalStep.
- */
-function translateCustomEvaluation(customEval, index) {
-    const evalType = CUSTOM_EVAL_TYPE_MAP[customEval.name] ?? `evaluator.${customEval.name}`;
-    let operator = '';
-    let actual = '';
-    let expected = '';
-    for (const param of customEval.parameters) {
-        if (param.name === 'operator') {
-            operator = param.value;
-        }
-        else if (param.name === 'actual') {
-            actual = mapActualPath(param.value);
-        }
-        else if (param.name === 'expected') {
-            expected = param.value;
-        }
-    }
-    return {
-        type: evalType,
-        id: `custom_${index}`,
-        operator,
-        actual,
-        expected,
-    };
-}
-/**
- * Map an org-model JSONPath to the Eval API shorthand ref.
- * Unknown paths are returned as-is.
- */
-function mapActualPath(path) {
-    return ACTUAL_PATH_MAP[path] ?? path;
-}
-//# sourceMappingURL=yamlSpecTranslator.js.map

package/lib/yamlSpecTranslator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"yamlSpecTranslator.js","sourceRoot":"","sources":["../src/yamlSpecTranslator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8BAA8B;AAE9B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAI1C,4DAA4D;AAE5D,MAAM,eAAe,GAA2B;IAC9C,yBAAyB,EAAE,eAAe;IAC1C,uBAAuB,EAAE,oDAAoD;IAC7E,gCAAgC,EAAE,6DAA6D;IAC/F,iCAAiC,EAAE,6DAA6D;CACjG,CAAC;AAEF,2DAA2D;AAE3D,MAAM,oBAAoB,GAA2B;IACnD,iBAAiB,EAAE,4BAA4B;IAC/C,kBAAkB,EAAE,6BAA6B;CAClD,CAAC;AAEF,4CAA4C;AAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,uBAAuB;IACvB,gCAAgC;IAChC,iCAAiC;CAClC,CAAC,CAAC;AAEH,qBAAqB;AAErB;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAC9C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB,EAAE,KAAa,EAAE,QAAiB;IACpF,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,EAAE,CAAC;IACzE,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,0BAA0B;IAC1B,MAAM,iBAAiB,GAAa;QAClC,IAAI,EAAE,sBAAsB;QAC5B,EAAE,EAAE,IAAI;QACR,aAAa,EAAE,IAAI;KACpB,CAAC;IAEF,IAAI,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtE,+BAA+B;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QAC5E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACb,sDAAsD,KAAK,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAC3F,IAAI,CACL,6CAA6C,CAC/C,CAAC;QACJ,CAAC;QAED,iBAAiB,CAAC,iBAAiB,GAAG,MAAM,CAAC,WAAW,CACtD,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAE9B,yEAAyE;IACzE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,oBAAoB;oBAC1B,EAAE,EAAE,WAAW,UAAU,EAAE;oBAC3B,UAAU,EAAE,iBAAiB;oBAC7B,SAAS,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,oBAAoB;QAC1B,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,iBAAiB;QAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,iBAAiB;YACvB,EAAE,EAAE,IAAI;YACR,UAAU,EAAE,iBAAiB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB;IAChB,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,mCAAmC;YACzC,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,QAAQ,CAAC,aAAa;YAChC,MAAM,EAAE,oDAAoD;YAC5D,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,qCAAqC;YAC3C,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,6DAA6D;YACrE,QAAQ,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,+BAA+B;YACrC,EAAE,EAAE,eAAe;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,eAAe;YACvB,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,yBAAyB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,2BAA2B;AAE3B;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAkB;IAC3C,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtD,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/F,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAChC,UAA8D,EAC9D,KAAa;IAEb,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,UAAU,CAAC,IAAI,EAAE,CAAC;IAEzF,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9B,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,EAAE,EAAE,UAAU,KAAK,EAAE;QACrB,QAAQ;QACR,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC"}