@salesforce/agents 0.14.6 → 0.14.7-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agentTest.d.ts +1 -1
- package/lib/agentTest.js +7 -0
- package/lib/agentTest.js.map +1 -1
- package/lib/index.d.ts +2 -1
- package/lib/index.js +3 -1
- package/lib/index.js.map +1 -1
- package/lib/types.d.ts +3 -1
- package/lib/utils.d.ts +1 -0
- package/lib/utils.js +9 -1
- package/lib/utils.js.map +1 -1
- package/package.json +1 -1
package/lib/agentTest.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Connection } from '@salesforce/core';
|
|
2
2
|
import { DeployResult } from '@salesforce/source-deploy-retrieve';
|
|
3
|
-
import {
|
|
3
|
+
import type { AvailableDefinition, AgentTestConfig, AiEvaluationDefinition, TestSpec } from './types.js';
|
|
4
4
|
/**
|
|
5
5
|
* Events emitted during agent test creation for consumers to listen to and keep track of progress.
|
|
6
6
|
*/
|
package/lib/agentTest.js
CHANGED
|
@@ -14,6 +14,7 @@ const kit_1 = require("@salesforce/kit");
|
|
|
14
14
|
const source_deploy_retrieve_1 = require("@salesforce/source-deploy-retrieve");
|
|
15
15
|
const yaml_1 = require("yaml");
|
|
16
16
|
const fast_xml_parser_1 = require("fast-xml-parser");
|
|
17
|
+
const utils_1 = require("./utils");
|
|
17
18
|
;
|
|
18
19
|
const messages = new core_1.Messages('@salesforce/agents', 'agentTest', new Map([["invalidAgentTestConfig", "Must provide one of: [ name, mdPath, specPath, specData ] when instantiating an AgentTest."], ["missingConnection", "Must provide an org connection to get agent test data based on an AiEvaluationDefinition API name."], ["missingTestSpecData", "The agent test is missing the required data to provide a test spec."]]));
|
|
19
20
|
/**
|
|
@@ -222,6 +223,8 @@ class AgentTest {
|
|
|
222
223
|
*/
|
|
223
224
|
async writeTestSpec(outputFile) {
|
|
224
225
|
const spec = await this.getTestSpec();
|
|
226
|
+
// by default, add the OOTB metrics to the spec, so generated MD will have it
|
|
227
|
+
spec.testCases.forEach((tc) => (tc.metrics = tc.metrics ?? Array.from(utils_1.metric)));
|
|
225
228
|
// strip out undefined values and empty strings
|
|
226
229
|
const clean = Object.entries(spec).reduce((acc, [key, value]) => {
|
|
227
230
|
if (value !== undefined && value !== '')
|
|
@@ -264,6 +267,9 @@ const convertToSpec = (data) => ({
|
|
|
264
267
|
expectedActions: transformStringToArray(expectations.find((e) => e.name === 'action_sequence_match' || e.name === 'actions_assertion')?.expectedValue),
|
|
265
268
|
expectedOutcome: expectations.find((e) => e.name === 'bot_response_rating' || e.name === 'output_validation')
|
|
266
269
|
?.expectedValue,
|
|
270
|
+
metrics: expectations
|
|
271
|
+
.filter((e) => utils_1.metric.includes(e.name))
|
|
272
|
+
.map((e) => e.name),
|
|
267
273
|
};
|
|
268
274
|
}),
|
|
269
275
|
});
|
|
@@ -288,6 +294,7 @@ const convertToMetadata = (spec) => ({
|
|
|
288
294
|
expectedValue: tc.expectedOutcome,
|
|
289
295
|
name: 'bot_response_rating',
|
|
290
296
|
},
|
|
297
|
+
...(0, kit_1.ensureArray)(tc.metrics).map((m) => ({ name: m })),
|
|
291
298
|
],
|
|
292
299
|
inputs: {
|
|
293
300
|
utterance: tc.utterance,
|
package/lib/agentTest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agentTest.js","sourceRoot":"","sources":["../src/agentTest.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEH,+CAA8D;AAC9D,yCAA0C;AAC1C,2CAA4E;AAC5E,yCAAwD;AACxD,+EAAsG;AACtG,+BAAwC;AACxC,qDAAwD;;
|
|
1
|
+
{"version":3,"file":"agentTest.js","sourceRoot":"","sources":["../src/agentTest.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEH,+CAA8D;AAC9D,yCAA0C;AAC1C,2CAA4E;AAC5E,yCAAwD;AACxD,+EAAsG;AACtG,+BAAwC;AACxC,qDAAwD;AAExD,mCAAiC;;AAGjC,MAAM,QAAQ,OAAG,eAAQ,CAAc,oBAAoB,EAAE,WAAW,mWAAC,CAAC;AAE1E;;GAEG;AACU,QAAA,8BAA8B,GAAG;IAC5C,qBAAqB,EAAE,yBAAyB;IAChD,OAAO,EAAE,gCAAgC;IACzC,iBAAiB,EAAE,oBAAoB;IACvC,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAa,SAAS;IAcO;IAbnB,QAAQ,CAAY;IACpB,IAAI,CAA0B;IAEtC;;;;;;;;;OASG;IACH,YAA2B,MAAuB;QAAvB,WAAM,GAAN,MAAM,CAAiB;QAChD,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAEpD,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,QAAQ,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAsB;QAC7C,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CACxB,UAAsB,EACtB,OAAe,EACf,YAAoB,EACpB,OAAiD;QAEjD,MAAM,aAAa,GAAG,IAAA,YAAK,EAAC,MAAM,IAAA,mBAAQ,EAAC,YAAY,EAAE,OAAO,CAAC,CAAa,CAAC;QAC/E,MAAM,SAAS,GAAG,gBAAS,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACzC,6CAA6C;QAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;QAC9D,MAAM,QAAQ,GAAG,OAAO;YACtB,CAAC,CAAC,GAAG,OAAO,YAAY,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,MAAM;YACtD,CAAC,CAAC,GAAG,OAAO,kCAAkC,CAAC;QACjD,MAAM,cAAc,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEjD,MAAM,GAAG,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC;QAC/D,MAAM,IAAA,gBAAK,EAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAA,oBAAS,EAAC,cAAc,EAAE,GAAG,CAAC,CAAC;QAErC,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,4CAAmB,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC7E,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,oBAAoB,EAAE,UAAU,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE;YACzB,IAAI,MAAM,CAAC,MAAM,KAAK,sCAAa,CAAC,OAAO,EAAE,CAAC;gBAC5C,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACtE,CAAC;iBAAM,CAAC;gBACN,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAChF,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE;YACzB,oFAAoF;YACpF,6EAA6E;YAC7E,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAC9E,KAAK,SAAS,CAAC,IAAI,CAAC,sCAA8B,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,cAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,cAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9G,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,cAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,IAAI,oBAAoB,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9F,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,WAAW,CAAC,UAAuB;QAC9C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,GAAG,IAAA,YAAK,EAAC,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAa,CAAC;YACjF,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,oEAAoE;QACpE,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,UAAU,EAAE,CAAC;gBACf,6EAA6E;gBAC7E,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CACzC,wBAAwB,EACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CACjB,CAA2B,CAAC;gBAC7B,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzC,OAAO,IAAI,CAAC,QAAQ,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,WAAW,CAAC,UAAuB;QAC9C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,GAAG,IAAA,YAAK,EAAC,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAa,CAAC;YACjF,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,oEAAoE;QACpE,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,IAAI,UAAU,EAAE,CAAC;gBACf,6EAA6E;gBAC7E,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CACzC,wBAAwB,EACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CACjB,CAA2B,CAAC;gBAC7B,OAAO,IAAI,CAAC,IAAI,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,aAAa,CAAC,UAAkB;QAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAEtC,6EAA6E;QAC7E,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,cAAM,CAAC,CAAC,CAAC,CAAC;QAChF,+CAA+C;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CAAoB,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACjF,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC;YACzE,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAE,CAAC,CAAC;QAEP,MAAM,GAAG,GAAG,IAAA,gBAAS,EAAC,KAAK,EAAE,SAAS,EAAE;YACtC,eAAe,EAAE,CAAC;YAClB,SAAS,EAAE,CAAC;SACb,CAAC,CAAC;QACH,MAAM,IAAA,gBAAK,EAAC,IAAA,mBAAO,EAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,IAAA,oBAAS,EAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,aAAa,CAAC,UAAkB;QAC3C,MAAM,GAAG,GAAG,gBAAgB,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACvD,MAAM,IAAA,gBAAK,EAAC,IAAA,mBAAO,EAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,IAAA,oBAAS,EAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACnC,CAAC;CACF;AAjOD,8BAiOC;AAED,kFAAkF;AAClF,MAAM,aAAa,GAAG,CAAC,IAA4B,EAAY,EAAE,CAAC,CAAC;IACjE,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,cAAc,EAAE,IAAI,CAAC,cAAc;IACnC,SAAS,EAAE,IAAA,iBAAW,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/C,MAAM,YAAY,GAAG,IAAA,iBAAW,EAAC,EAAE,CAAC,WAAW,CAAC,CAAC;QACjD,OAAO;YACL,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS;YAC9B,gHAAgH;YAChH,aAAa,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,sBAAsB,IAAI,CAAC,CAAC,IAAI,KAAK,iBAAiB,CAAC;gBACxG,EAAE,aAAa;YACjB,eAAe,EAAE,sBAAsB,CACrC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,uBAAuB,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAC,EAAE,aAAa,CAC9G;YACD,eAAe,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,qBAAqB,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAC;gBAC3G,EAAE,aAAa;YACjB,OAAO,EAAE,YAAY;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAA+B,CAAC,CAAC;iBACjE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAA+B,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC;CACH,CAAC,CAAC;AAEH,kFAAkF;AAClF,MAAM,iBAAiB,GAAG,CAAC,IAAc,EAA0B,EAAE,CAAC,CAAC;IACrE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC;IACnE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACpC,WAAW,EAAE;YACX;gBACE,aAAa,EAAE,EAAE,CAAC,aAAuB;gBACzC,IAAI,EAAE,sBAAsB;aAC7B;YACD;gBACE,aAAa,EAAE,IAAI,CAAC,EAAE,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;gBAC/E,IAAI,EAAE,uBAAuB;aAC9B;YACD;gBACE,aAAa,EAAE,EAAE,CAAC,eAAyB;gBAC3C,IAAI,EAAE,qBAAqB;aAC5B;YACD,GAAG,IAAA,iBAAW,EAAC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;SACrD;QACD,MAAM,EAAE;YACN,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB;QACD,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC;KACvC,CAAC,CAAC;CACJ,CAAC,CAAC;AAEH,SAAS,sBAAsB,CAAC,GAAuB;IACrD,IAAI,CAAC;QACH,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,sDAAsD;QACtD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAKD,MAAM,iBAAiB,GAAG,KAAK,EAAE,MAAc,EAAmC,EAAE;IAClF,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAQ,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,2BAAS,EAAE,CAAC;IAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAA8B,CAAC;IAClE,OAAO,UAAU,CAAC,sBAAsB,CAAC;AAC3C,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,IAA4B,EAAU,EAAE;IAChE,MAAM,SAAS,GAAG;QAChB,sBAAsB,EAAE;YACtB,MAAM,EAAE,yCAAyC;YACjD,GAAG,IAAI;SACR;KACF,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,4BAAU,CAAC;QAC7B,MAAM,EAAE,IAAI;QACZ,mBAAmB,EAAE,GAAG;QACxB,QAAQ,EAAE,MAAM;QAChB,gBAAgB,EAAE,KAAK;KACxB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAW,CAAC;IAE/C,OAAO,2CAA2C,GAAG,EAAE,CAAC;AAC1D,CAAC,CAAC"}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export { type AgentCreateConfig, type AgentCreateResponse, type AgentJobSpec, type AgentJobSpecCreateConfig, type AgentOptions, type AgentTone, type AgentType, type AgentPreviewMessageLinks, type AgentPreviewMessage, type AgentPreviewStartResponse, type AgentPreviewSendResponse, type AgentPreviewEndResponse, type DraftAgentTopics, type DraftAgentTopicsBody, type DraftAgentTopicsResponse, type AvailableDefinition, type AgentTestResultsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, type TestCaseResult, type TestStatus, } from './types';
|
|
1
|
+
export { type AgentCreateConfig, type AgentCreateResponse, type AgentJobSpec, type AgentJobSpecCreateConfig, type AgentOptions, type AgentTone, type AgentType, type AgentPreviewMessageLinks, type AgentPreviewMessage, type AgentPreviewStartResponse, type AgentPreviewSendResponse, type AgentPreviewEndResponse, type DraftAgentTopics, type DraftAgentTopicsBody, type DraftAgentTopicsResponse, type AvailableDefinition, type AgentTestResultsResponse, type AgentTestStartResponse, type AgentTestStatusResponse, type TestCaseResult, type TestStatus, } from './types.js';
|
|
2
|
+
export { metric } from './utils';
|
|
2
3
|
export { Agent, AgentCreateLifecycleStages, generateAgentApiName } from './agent';
|
|
3
4
|
export { AgentTester } from './agentTester';
|
|
4
5
|
export { AgentTest, AgentTestCreateLifecycleStages } from './agentTest';
|
package/lib/index.js
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
|
|
7
7
|
*/
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.AgentPreview = exports.humanFriendlyName = exports.convertTestResultsToFormat = exports.AgentTestCreateLifecycleStages = exports.AgentTest = exports.AgentTester = exports.generateAgentApiName = exports.AgentCreateLifecycleStages = exports.Agent = void 0;
|
|
9
|
+
exports.AgentPreview = exports.humanFriendlyName = exports.convertTestResultsToFormat = exports.AgentTestCreateLifecycleStages = exports.AgentTest = exports.AgentTester = exports.generateAgentApiName = exports.AgentCreateLifecycleStages = exports.Agent = exports.metric = void 0;
|
|
10
|
+
var utils_1 = require("./utils");
|
|
11
|
+
Object.defineProperty(exports, "metric", { enumerable: true, get: function () { return utils_1.metric; } });
|
|
10
12
|
var agent_1 = require("./agent");
|
|
11
13
|
Object.defineProperty(exports, "Agent", { enumerable: true, get: function () { return agent_1.Agent; } });
|
|
12
14
|
Object.defineProperty(exports, "AgentCreateLifecycleStages", { enumerable: true, get: function () { return agent_1.AgentCreateLifecycleStages; } });
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAyBH,iCAAkF;AAAzE,8FAAA,KAAK,OAAA;AAAE,mHAAA,0BAA0B,OAAA;AAAE,6GAAA,oBAAoB,OAAA;AAChE,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,yCAAwE;AAA/D,sGAAA,SAAS,OAAA;AAAE,2HAAA,8BAA8B,OAAA;AAClD,uDAAmF;AAA1E,8HAAA,0BAA0B,OAAA;AAAE,qHAAA,iBAAiB,OAAA;AACtD,+CAA8C;AAArC,4GAAA,YAAY,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAyBH,iCAAiC;AAAxB,+FAAA,MAAM,OAAA;AACf,iCAAkF;AAAzE,8FAAA,KAAK,OAAA;AAAE,mHAAA,0BAA0B,OAAA;AAAE,6GAAA,oBAAoB,OAAA;AAChE,6CAA4C;AAAnC,0GAAA,WAAW,OAAA;AACpB,yCAAwE;AAA/D,sGAAA,SAAS,OAAA;AAAE,2HAAA,8BAA8B,OAAA;AAClD,uDAAmF;AAA1E,8HAAA,0BAA0B,OAAA;AAAE,qHAAA,iBAAiB,OAAA;AACtD,+CAA8C;AAArC,4GAAA,YAAY,OAAA"}
|
package/lib/types.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Connection, SfProject } from '@salesforce/core';
|
|
2
2
|
import { FileProperties } from '@salesforce/source-deploy-retrieve';
|
|
3
|
+
import { metric } from './utils';
|
|
3
4
|
/**
|
|
4
5
|
* Options for creating instances of agents from an org.
|
|
5
6
|
*/
|
|
@@ -260,6 +261,7 @@ export type TestCase = {
|
|
|
260
261
|
expectedActions: string[] | undefined;
|
|
261
262
|
expectedOutcome: string | undefined;
|
|
262
263
|
expectedTopic: string | undefined;
|
|
264
|
+
metrics?: Array<(typeof metric)[number]>;
|
|
263
265
|
};
|
|
264
266
|
export type TestSpec = {
|
|
265
267
|
name: string;
|
|
@@ -278,7 +280,7 @@ export type AiEvaluationDefinition = {
|
|
|
278
280
|
testCase: Array<{
|
|
279
281
|
expectation: Array<{
|
|
280
282
|
name: string;
|
|
281
|
-
expectedValue
|
|
283
|
+
expectedValue?: string;
|
|
282
284
|
}>;
|
|
283
285
|
inputs: {
|
|
284
286
|
utterance: string;
|
package/lib/utils.d.ts
CHANGED
package/lib/utils.js
CHANGED
|
@@ -6,7 +6,15 @@
|
|
|
6
6
|
* For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
|
|
7
7
|
*/
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.decodeHtmlEntities = void 0;
|
|
9
|
+
exports.decodeHtmlEntities = exports.metric = void 0;
|
|
10
|
+
exports.metric = [
|
|
11
|
+
'completeness',
|
|
12
|
+
'coherence',
|
|
13
|
+
'conciseness',
|
|
14
|
+
'output_latency_milliseconds',
|
|
15
|
+
'instruction_following',
|
|
16
|
+
'factuality',
|
|
17
|
+
];
|
|
10
18
|
/**
|
|
11
19
|
* Clean a string by replacing HTML entities with their respective characters.
|
|
12
20
|
*
|
package/lib/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,MAAM,GAAG;IACpB,cAAc;IACd,WAAW;IACX,aAAa;IACb,6BAA6B;IAC7B,uBAAuB;IACvB,YAAY;CACJ,CAAC;AAEX;;;;;GAKG;AACI,MAAM,kBAAkB,GAAG,CAAC,MAAc,EAAE,EAAU,EAAE;IAC7D,MAAM,QAAQ,GAA8B;QAC1C,QAAQ,EAAE,GAAG;QACb,QAAQ,EAAE,GAAG;QACb,OAAO,EAAE,GAAG;QACZ,MAAM,EAAE,GAAG;QACX,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,GAAG;QACZ,OAAO,EAAE,GAAG;QACZ,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,GAAG;QACd,QAAQ,EAAE,GAAG;QACb,OAAO,EAAE,GAAG;QACZ,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,GAAG;QACd,OAAO,EAAE,GAAG;QACZ,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,GAAG;QACd,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,GAAG;QACd,QAAQ,EAAE,GAAG;QACb,QAAQ,EAAE,GAAG;QACb,QAAQ,EAAE,GAAG;QACb,UAAU,EAAE,GAAG;KAChB,CAAC;IAEF,OAAO,GAAG,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,CAAC;AACjF,CAAC,CAAC;AAnCW,QAAA,kBAAkB,sBAmC7B"}
|
package/package.json
CHANGED