@salesforce/agents 0.14.6 → 0.14.7-preview.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agentTest.d.ts +1 -1
- package/lib/agentTest.js +9 -0
- package/lib/agentTest.js.map +1 -1
- package/lib/index.d.ts +2 -1
- package/lib/index.js +3 -1
- package/lib/index.js.map +1 -1
- package/lib/types.d.ts +11 -1
- package/lib/utils.d.ts +1 -0
- package/lib/utils.js +9 -1
- package/lib/utils.js.map +1 -1
- package/package.json +1 -1
package/lib/agentTest.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Connection } from '@salesforce/core';
|
|
2
2
|
import { DeployResult } from '@salesforce/source-deploy-retrieve';
|
|
3
|
-
import {
|
|
3
|
+
import type { AvailableDefinition, AgentTestConfig, AiEvaluationDefinition, TestSpec } from './types.js';
|
|
4
4
|
/**
|
|
5
5
|
* Events emitted during agent test creation for consumers to listen to and keep track of progress.
|
|
6
6
|
*/
|
package/lib/agentTest.js
CHANGED
|
@@ -14,6 +14,7 @@ const kit_1 = require("@salesforce/kit");
|
|
|
14
14
|
const source_deploy_retrieve_1 = require("@salesforce/source-deploy-retrieve");
|
|
15
15
|
const yaml_1 = require("yaml");
|
|
16
16
|
const fast_xml_parser_1 = require("fast-xml-parser");
|
|
17
|
+
const utils_1 = require("./utils");
|
|
17
18
|
;
|
|
18
19
|
const messages = new core_1.Messages('@salesforce/agents', 'agentTest', new Map([["invalidAgentTestConfig", "Must provide one of: [ name, mdPath, specPath, specData ] when instantiating an AgentTest."], ["missingConnection", "Must provide an org connection to get agent test data based on an AiEvaluationDefinition API name."], ["missingTestSpecData", "The agent test is missing the required data to provide a test spec."]]));
|
|
19
20
|
/**
|
|
@@ -222,6 +223,8 @@ class AgentTest {
|
|
|
222
223
|
*/
|
|
223
224
|
async writeTestSpec(outputFile) {
|
|
224
225
|
const spec = await this.getTestSpec();
|
|
226
|
+
// by default, add the OOTB metrics to the spec, so generated MD will have it
|
|
227
|
+
spec.testCases.forEach((tc) => (tc.metrics = tc.metrics ?? Array.from(utils_1.metric)));
|
|
225
228
|
// strip out undefined values and empty strings
|
|
226
229
|
const clean = Object.entries(spec).reduce((acc, [key, value]) => {
|
|
227
230
|
if (value !== undefined && value !== '')
|
|
@@ -258,12 +261,16 @@ const convertToSpec = (data) => ({
|
|
|
258
261
|
const expectations = (0, kit_1.ensureArray)(tc.expectation);
|
|
259
262
|
return {
|
|
260
263
|
utterance: tc.inputs.utterance,
|
|
264
|
+
contextVariable: tc.inputs.contextVariable,
|
|
261
265
|
// TODO: remove old names once removed in 258 (topic_sequence_match, action_sequence_match, bot_response_rating)
|
|
262
266
|
expectedTopic: expectations.find((e) => e.name === 'topic_sequence_match' || e.name === 'topic_assertion')
|
|
263
267
|
?.expectedValue,
|
|
264
268
|
expectedActions: transformStringToArray(expectations.find((e) => e.name === 'action_sequence_match' || e.name === 'actions_assertion')?.expectedValue),
|
|
265
269
|
expectedOutcome: expectations.find((e) => e.name === 'bot_response_rating' || e.name === 'output_validation')
|
|
266
270
|
?.expectedValue,
|
|
271
|
+
metrics: expectations
|
|
272
|
+
.filter((e) => utils_1.metric.includes(e.name))
|
|
273
|
+
.map((e) => e.name),
|
|
267
274
|
};
|
|
268
275
|
}),
|
|
269
276
|
});
|
|
@@ -288,9 +295,11 @@ const convertToMetadata = (spec) => ({
|
|
|
288
295
|
expectedValue: tc.expectedOutcome,
|
|
289
296
|
name: 'bot_response_rating',
|
|
290
297
|
},
|
|
298
|
+
...(0, kit_1.ensureArray)(tc.metrics).map((m) => ({ name: m })),
|
|
291
299
|
],
|
|
292
300
|
inputs: {
|
|
293
301
|
utterance: tc.utterance,
|
|
302
|
+
contextVariable: tc.contextVariables?.map((cv) => ({ variableName: cv.name, variableValue: cv.value })),
|
|
294
303
|
},
|
|
295
304
|
number: spec.testCases.indexOf(tc) + 1,
|
|
296
305
|
})),
|
package/lib/agentTest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agentTest.js","sourceRoot":"","sources":["../src/agentTest.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEH,+CAA8D;AAC9D,yCAA0C;AAC1C,2CAA4E;AAC5E,yCAAwD;AACxD,+EAAsG;AACtG,+BAAwC;AACxC,qDAAwD;;
|
|
1
|
+
{"version":3,"file":"agentTest.js","sourceRoot":"","sources":["../src/agentTest.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEH,+CAA8D;AAC9D,yCAA0C;AAC1C,2CAA4E;AAC5E,yCAAwD;AACxD,+EAAsG;AACtG,+BAAwC;AACxC,qDAAwD;AAExD,mCAAiC;;AAGjC,MAAM,QAAQ,OAAG,eAAQ,CAAc,oBAAoB,EAAE,WAAW,mWAAC,CAAC;AAE1E;;GAEG;AACU,QAAA,8BAA8B,GAAG;IAC5C,qBAAqB,EAAE,yBAAyB;IAChD,OAAO,EAAE,gCAAgC;IACzC,iBAAiB,EAAE,oBAAoB;IACvC,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAa,SAAS;IAcO;IAbnB,QAAQ,CAAY;IACpB,IAAI,CAA0B;IAEtC;;;;;;;;;OASG;IACH,YAA2B,MAAuB;QAAvB,WAAM,GAAN,MAAM,CAAiB;QAChD,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAEpD,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,QAAQ,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAsB;QAC7C,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CACxB,UAAsB,EACtB,OAAe,EACf,YAAoB,EACpB,OAAiD;QAEjD,MAAM,aAAa,GAAG,IAAA,YAAK,EAAC,MAAM,IAAA,mBAAQ,EAAC,YAAY,EAAE,OAAO,CAAC,CAAa,CAAC;QAC/E,MAAM,SAAS,GAAG,gBAAS,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACzC,6CAA6C;QAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;QAC9D,MAAM,QAAQ,GAAG,OAAO;YACtB,CAAC,CAAC,GAAG,OAAO,YAAY,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,MAAM;YACtD,CAAC,CAAC,GAAG,OAAO,kCAAkC,CAAC;QACjD,MAAM,cAAc,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEjD,MAAM,GAAG,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC;QAC/D,MAAM,IAAA,gBAAK,EAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAA,oBAAS,EAAC,cAAc,EAAE,GAAG,CAAC,CAAC;QAErC,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,4CAAmB,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC7E,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,oBAAoB,EAAE,UAAU,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE;YACzB,IAAI,MAAM,CAAC,MAAM,KAAK,sCAAa,CAAC,OAAO,EAAE,CAAC;gBAC5C,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACtE,CAAC;iBAAM,CAAC;gBACN,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAChF,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE;YACzB,oFAAoF;YACpF,6EAA6E;YAC7E,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAC9E,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,cAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,cAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9G,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,cAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,IAAI,oBAAoB,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9F,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,WAAW,CAAC,UAAuB;QAC9C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,GAAG,IAAA,YAAK,EAAC,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAa,CAAC;YACjF,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,oEAAoE;QACpE,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,UAAU,EAAE,CAAC;gBACf,6EAA6E;gBAC7E,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CACzC,wBAAwB,EACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CACjB,CAA2B,CAAC;gBAC7B,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzC,OAAO,IAAI,CAAC,QAAQ,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,WAAW,CAAC,UAAuB;QAC9C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,GAAG,IAAA,YAAK,EAAC,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAa,CAAC;YACjF,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,oEAAoE;QACpE,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,UAAU,EAAE,CAAC;gBACf,6EAA6E;gBAC7E,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CACzC,wBAAwB,EACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CACjB,CAA2B,CAAC;gBAC7B,OAAO,IAAI,CAAC,IAAI,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,aAAa,CAAC,UAAkB;QAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAEtC,6EAA6E;QAC7E,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,cAAM,CAAC,CAAC,CAAC,CAAC;QAChF,+CAA+C;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CAAoB,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACjF,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC;YACzE,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAE,CAAC,CAAC;QAEP,MAAM,GAAG,GAAG,IAAA,gBAAS,EAAC,KAAK,EAAE,SAAS,EAAE;YACtC,eAAe,EAAE,CAAC;YAClB,SAAS,EAAE,CAAC;SACb,CAAC,CAAC;QACH,MAAM,IAAA,gBAAK,EAAC,IAAA,mBAAO,EAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,IAAA,oBAAS,EAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,aAAa,CAAC,UAAkB;QAC3C,MAAM,GAAG,GAAG,gBAAgB,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACvD,MAAM,IAAA,gBAAK,EAAC,IAAA,mBAAO,EAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,IAAA,oBAAS,EAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACnC,CAAC;CACF;AAjOD,8BAiOC;AAED,kFAAkF;AAClF,MAAM,aAAa,GAAG,CAAC,IAA4B,EAAY,EAAE,CAAC,CAAC;IACjE,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,cAAc,EAAE,IAAI,CAAC,cAAc;IACnC,SAAS,EAAE,IAAA,iBAAW,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/C,MAAM,YAAY,GAAG,IAAA,iBAAW,EAAC,EAAE,CAAC,WAAW,CAAC,CAAC;QACjD,OAAO;YACL,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS;YAC9B,eAAe,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe;YAC1C,gHAAgH;YAChH,aAAa,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,sBAAsB,IAAI,CAAC,CAAC,IAAI,KAAK,iBAAiB,CAAC;gBACxG,EAAE,aAAa;YACjB,eAAe,EAAE,sBAAsB,CACrC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,uBAAuB,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAC,EAAE,aAAa,CAC9G;YACD,eAAe,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,qBAAqB,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAC;gBAC3G,EAAE,aAAa;YACjB,OAAO,EAAE,YAAY;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAA+B,CAAC,CAAC;iBACjE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAA+B,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC;CACH,CAAC,CAAC;AAEH,kFAAkF;AAClF,MAAM,iBAAiB,GAAG,CAAC,IAAc,EAA0B,EAAE,CAAC,CAAC;IACrE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC;IACnE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACpC,WAAW,EAAE;YACX;gBACE,aAAa,EAAE,EAAE,CAAC,aAAuB;gBACzC,IAAI,EAAE,sBAAsB;aAC7B;YACD;gBACE,aAAa,EAAE,IAAI,CAAC,EAAE,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;gBAC/E,IAAI,EAAE,uBAAuB;aAC9B;YACD;gBACE,aAAa,EAAE,EAAE,CAAC,eAAyB;gBAC3C,IAAI,EAAE,qBAAqB;aAC5B;YACD,GAAG,IAAA,iBAAW,EAAC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;SACrD;QACD,MAAM,EAAE;YACN,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,eAAe,EAAE,EAAE,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC,IAAI,EAAE,aAAa,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;SACxG;QACD,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC;KACvC,CAAC,CAAC;CACJ,CAAC,CAAC;AAEH,SAAS,sBAAsB,CAAC,GAAuB;IACrD,IAAI,CAAC;QACH,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,sDAAsD;QACtD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAKD,MAAM,iBAAiB,GAAG,KAAK,EAAE,MAAc,EAAmC,EAAE;IAClF,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAQ,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,2BAAS,EAAE,CAAC;IAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAA8B,CAAC;IAClE,OAAO,UAAU,CAAC,sBAAsB,CAAC;AAC3C,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,IAA4B,EAAU,EAAE;IAChE,MAAM,SAAS,GAAG;QAChB,sBAAsB,EAAE;YACtB,MAAM,EAAE,yCAAyC;YACjD,GAAG,IAAI;SACR;KACF,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,4BAAU,CAAC;QAC7B,MAAM,EAAE,IAAI;QACZ,mBAAmB,EAAE,GAAG;QACxB,QAAQ,EAAE,MAAM;QAChB,gBAAgB,EAAE,KAAK;KACxB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAW,CAAC;IAE/C,OAAO,2CAA2C,GAAG,EAAE,CAAC;AAC1D,CAAC,CAAC"}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export { type AgentCreateConfig, type AgentCreateResponse, type AgentJobSpec, type AgentJobSpecCreateConfig, type AgentOptions, type AgentTone, type AgentType, type AgentPreviewMessageLinks, type AgentPreviewMessage, type AgentPreviewStartResponse, type AgentPreviewSendResponse, type AgentPreviewEndResponse, type DraftAgentTopics, type DraftAgentTopicsBody, type DraftAgentTopicsResponse, type AvailableDefinition, type AgentTestResultsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, type TestCaseResult, type TestStatus, } from './types';
|
|
1
|
+
export { type AgentCreateConfig, type AgentCreateResponse, type AgentJobSpec, type AgentJobSpecCreateConfig, type AgentOptions, type AgentTone, type AgentType, type AgentPreviewMessageLinks, type AgentPreviewMessage, type AgentPreviewStartResponse, type AgentPreviewSendResponse, type AgentPreviewEndResponse, type DraftAgentTopics, type DraftAgentTopicsBody, type DraftAgentTopicsResponse, type AvailableDefinition, type AgentTestResultsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, type TestCaseResult, type TestStatus, } from './types.js';
|
|
2
|
+
export { metric } from './utils';
|
|
2
3
|
export { Agent, AgentCreateLifecycleStages, generateAgentApiName } from './agent';
|
|
3
4
|
export { AgentTester } from './agentTester';
|
|
4
5
|
export { AgentTest, AgentTestCreateLifecycleStages } from './agentTest';
|
package/lib/index.js
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
|
|
7
7
|
*/
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.AgentPreview = exports.humanFriendlyName = exports.convertTestResultsToFormat = exports.AgentTestCreateLifecycleStages = exports.AgentTest = exports.AgentTester = exports.generateAgentApiName = exports.AgentCreateLifecycleStages = exports.Agent = void 0;
|
|
9
|
+
exports.AgentPreview = exports.humanFriendlyName = exports.convertTestResultsToFormat = exports.AgentTestCreateLifecycleStages = exports.AgentTest = exports.AgentTester = exports.generateAgentApiName = exports.AgentCreateLifecycleStages = exports.Agent = exports.metric = void 0;
|
|
10
|
+
var utils_1 = require("./utils");
|
|
11
|
+
Object.defineProperty(exports, "metric", { enumerable: true, get: function () { return utils_1.metric; } });
|
|
10
12
|
var agent_1 = require("./agent");
|
|
11
13
|
Object.defineProperty(exports, "Agent", { enumerable: true, get: function () { return agent_1.Agent; } });
|
|
12
14
|
Object.defineProperty(exports, "AgentCreateLifecycleStages", { enumerable: true, get: function () { return agent_1.AgentCreateLifecycleStages; } });
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAyBH,iCAAkF;AAAzE,8FAAA,KAAK,OAAA;AAAE,mHAAA,0BAA0B,OAAA;AAAE,6GAAA,oBAAoB,OAAA;AAChE,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,yCAAwE;AAA/D,sGAAA,SAAS,OAAA;AAAE,2HAAA,8BAA8B,OAAA;AAClD,uDAAmF;AAA1E,8HAAA,0BAA0B,OAAA;AAAE,qHAAA,iBAAiB,OAAA;AACtD,+CAA8C;AAArC,4GAAA,YAAY,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAyBH,iCAAiC;AAAxB,+FAAA,MAAM,OAAA;AACf,iCAAkF;AAAzE,8FAAA,KAAK,OAAA;AAAE,mHAAA,0BAA0B,OAAA;AAAE,6GAAA,oBAAoB,OAAA;AAChE,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,yCAAwE;AAA/D,sGAAA,SAAS,OAAA;AAAE,2HAAA,8BAA8B,OAAA;AAClD,uDAAmF;AAA1E,8HAAA,0BAA0B,OAAA;AAAE,qHAAA,iBAAiB,OAAA;AACtD,+CAA8C;AAArC,4GAAA,YAAY,OAAA"}
|
package/lib/types.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Connection, SfProject } from '@salesforce/core';
|
|
2
2
|
import { FileProperties } from '@salesforce/source-deploy-retrieve';
|
|
3
|
+
import { metric } from './utils';
|
|
3
4
|
/**
|
|
4
5
|
* Options for creating instances of agents from an org.
|
|
5
6
|
*/
|
|
@@ -260,6 +261,11 @@ export type TestCase = {
|
|
|
260
261
|
expectedActions: string[] | undefined;
|
|
261
262
|
expectedOutcome: string | undefined;
|
|
262
263
|
expectedTopic: string | undefined;
|
|
264
|
+
metrics?: Array<(typeof metric)[number]>;
|
|
265
|
+
contextVariables?: Array<{
|
|
266
|
+
name: string;
|
|
267
|
+
value: string;
|
|
268
|
+
}>;
|
|
263
269
|
};
|
|
264
270
|
export type TestSpec = {
|
|
265
271
|
name: string;
|
|
@@ -278,9 +284,13 @@ export type AiEvaluationDefinition = {
|
|
|
278
284
|
testCase: Array<{
|
|
279
285
|
expectation: Array<{
|
|
280
286
|
name: string;
|
|
281
|
-
expectedValue
|
|
287
|
+
expectedValue?: string;
|
|
282
288
|
}>;
|
|
283
289
|
inputs: {
|
|
290
|
+
contextVariable?: Array<{
|
|
291
|
+
variableName: string;
|
|
292
|
+
variableValue: string;
|
|
293
|
+
}>;
|
|
284
294
|
utterance: string;
|
|
285
295
|
};
|
|
286
296
|
}>;
|
package/lib/utils.d.ts
CHANGED
package/lib/utils.js
CHANGED
|
@@ -6,7 +6,15 @@
|
|
|
6
6
|
* For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
|
|
7
7
|
*/
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.decodeHtmlEntities = void 0;
|
|
9
|
+
exports.decodeHtmlEntities = exports.metric = void 0;
|
|
10
|
+
exports.metric = [
|
|
11
|
+
'completeness',
|
|
12
|
+
'coherence',
|
|
13
|
+
'conciseness',
|
|
14
|
+
'output_latency_milliseconds',
|
|
15
|
+
'instruction_following',
|
|
16
|
+
'factuality',
|
|
17
|
+
];
|
|
10
18
|
/**
|
|
11
19
|
* Clean a string by replacing HTML entities with their respective characters.
|
|
12
20
|
*
|
package/lib/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,MAAM,GAAG;IACpB,cAAc;IACd,WAAW;IACX,aAAa;IACb,6BAA6B;IAC7B,uBAAuB;IACvB,YAAY;CACJ,CAAC;AAEX;;;;;GAKG;AACI,MAAM,kBAAkB,GAAG,CAAC,MAAc,EAAE,EAAU,EAAE;IAC7D,MAAM,QAAQ,GAA8B;QAC1C,QAAQ,EAAE,GAAG;QACb,QAAQ,EAAE,GAAG;QACb,OAAO,EAAE,GAAG;QACZ,MAAM,EAAE,GAAG;QACX,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,GAAG;QACZ,OAAO,EAAE,GAAG;QACZ,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,GAAG;QACd,QAAQ,EAAE,GAAG;QACb,OAAO,EAAE,GAAG;QACZ,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,GAAG;QACd,OAAO,EAAE,GAAG;QACZ,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,GAAG;QACd,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,GAAG;QACd,QAAQ,EAAE,GAAG;QACb,QAAQ,EAAE,GAAG;QACb,QAAQ,EAAE,GAAG;QACb,UAAU,EAAE,GAAG;KAChB,CAAC;IAEF,OAAO,GAAG,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,CAAC;AACjF,CAAC,CAAC;AAnCW,QAAA,kBAAkB,sBAmC7B"}
|
package/package.json
CHANGED