@salesforce/plugin-agent 1.36.0 → 1.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +129 -24
  2. package/lib/agentTestCache.d.ts +4 -1
  3. package/lib/agentTestCache.js +2 -2
  4. package/lib/agentTestCache.js.map +1 -1
  5. package/lib/commands/agent/test/list.js +6 -1
  6. package/lib/commands/agent/test/list.js.map +1 -1
  7. package/lib/commands/agent/test/results.d.ts +3 -2
  8. package/lib/commands/agent/test/results.js +5 -3
  9. package/lib/commands/agent/test/results.js.map +1 -1
  10. package/lib/commands/agent/test/resume.d.ts +1 -0
  11. package/lib/commands/agent/test/resume.js +12 -4
  12. package/lib/commands/agent/test/resume.js.map +1 -1
  13. package/lib/commands/agent/test/run-eval.d.ts +1 -5
  14. package/lib/commands/agent/test/run-eval.js +54 -149
  15. package/lib/commands/agent/test/run-eval.js.map +1 -1
  16. package/lib/commands/agent/test/run.d.ts +1 -0
  17. package/lib/commands/agent/test/run.js +21 -8
  18. package/lib/commands/agent/test/run.js.map +1 -1
  19. package/lib/flags.d.ts +6 -1
  20. package/lib/flags.js +21 -4
  21. package/lib/flags.js.map +1 -1
  22. package/lib/handleTestResults.d.ts +4 -2
  23. package/lib/handleTestResults.js +137 -5
  24. package/lib/handleTestResults.js.map +1 -1
  25. package/lib/testRunnerFactory.d.ts +7 -0
  26. package/lib/testRunnerFactory.js +30 -0
  27. package/lib/testRunnerFactory.js.map +1 -0
  28. package/lib/testStages.d.ts +4 -3
  29. package/lib/testStages.js.map +1 -1
  30. package/messages/agent.test.list.md +1 -1
  31. package/messages/agent.test.run-eval.md +0 -4
  32. package/messages/shared.md +8 -0
  33. package/oclif.manifest.json +506 -481
  34. package/package.json +5 -5
  35. package/schemas/agent-activate.json +5 -2
  36. package/schemas/agent-create.json +39 -8
  37. package/schemas/agent-deactivate.json +5 -2
  38. package/schemas/agent-generate-agent__spec.json +18 -4
  39. package/schemas/agent-generate-authoring__bundle.json +6 -2
  40. package/schemas/agent-generate-template.json +5 -2
  41. package/schemas/agent-preview-end.json +5 -2
  42. package/schemas/agent-preview-send.json +6 -2
  43. package/schemas/agent-preview-sessions.json +10 -3
  44. package/schemas/agent-preview-start.json +5 -2
  45. package/schemas/agent-preview.json +1 -1
  46. package/schemas/agent-test-create.json +5 -2
  47. package/schemas/agent-test-list.json +1 -1
  48. package/schemas/agent-test-results.json +111 -10
  49. package/schemas/agent-test-resume.json +113 -10
  50. package/schemas/agent-test-run.json +113 -10
  51. package/schemas/agent-test-run__eval.json +17 -4
  52. package/schemas/agent-validate-authoring__bundle.json +4 -2
  53. package/lib/evalFormatter.d.ts +0 -30
  54. package/lib/evalFormatter.js +0 -263
  55. package/lib/evalFormatter.js.map +0 -1
  56. package/lib/evalNormalizer.d.ts +0 -57
  57. package/lib/evalNormalizer.js +0 -431
  58. package/lib/evalNormalizer.js.map +0 -1
  59. package/lib/yamlSpecTranslator.d.ts +0 -20
  60. package/lib/yamlSpecTranslator.js +0 -227
  61. package/lib/yamlSpecTranslator.js.map +0 -1
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@salesforce/plugin-agent",
3
3
  "description": "Commands to interact with Salesforce agents",
4
- "version": "1.36.0",
4
+ "version": "1.37.0",
5
5
  "author": "Salesforce",
6
6
  "bugs": "https://github.com/forcedotcom/cli/issues",
7
7
  "enableO11y": true,
@@ -12,11 +12,11 @@
12
12
  "@inquirer/prompts": "^7.10.1",
13
13
  "@oclif/core": "^4",
14
14
  "@oclif/multi-stage-output": "^0.8.36",
15
- "@salesforce/agents": "^1.4.0",
15
+ "@salesforce/agents": "^1.5.2",
16
16
  "@salesforce/core": "^8.28.3",
17
17
  "@salesforce/kit": "^3.2.6",
18
18
  "@salesforce/sf-plugins-core": "^12.2.6",
19
- "@salesforce/source-deploy-retrieve": "^12.32.8",
19
+ "@salesforce/source-deploy-retrieve": "^12.35.3",
20
20
  "@salesforce/types": "^1.7.1",
21
21
  "ansis": "^3.3.2",
22
22
  "fast-xml-parser": "^5.7.1",
@@ -240,7 +240,7 @@
240
240
  "exports": "./lib/index.js",
241
241
  "type": "module",
242
242
  "sfdx": {
243
- "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.36.0.crt",
244
- "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.36.0.sig"
243
+ "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.37.0.crt",
244
+ "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.37.0.sig"
245
245
  }
246
246
  }
@@ -12,8 +12,11 @@
12
12
  "type": "number"
13
13
  }
14
14
  },
15
- "required": ["success", "version"],
15
+ "required": [
16
+ "success",
17
+ "version"
18
+ ],
16
19
  "additionalProperties": false
17
20
  }
18
21
  }
19
- }
22
+ }
@@ -28,7 +28,11 @@
28
28
  "type": "string"
29
29
  }
30
30
  },
31
- "required": ["botId", "botVersionId", "plannerId"],
31
+ "required": [
32
+ "botId",
33
+ "botVersionId",
34
+ "plannerId"
35
+ ],
32
36
  "additionalProperties": false,
33
37
  "description": "If the agent was created with saveAgent=true, these are the IDs that make up an agent; Bot, BotVersion, and GenAiPlanner metadata."
34
38
  },
@@ -78,7 +82,11 @@
78
82
  "type": "string"
79
83
  }
80
84
  },
81
- "required": ["inputName", "inputDataType", "inputDescription"],
85
+ "required": [
86
+ "inputName",
87
+ "inputDataType",
88
+ "inputDescription"
89
+ ],
82
90
  "additionalProperties": false
83
91
  },
84
92
  "minItems": 1,
@@ -99,14 +107,24 @@
99
107
  "type": "string"
100
108
  }
101
109
  },
102
- "required": ["outputName", "outputDataType", "outputDescription"],
110
+ "required": [
111
+ "outputName",
112
+ "outputDataType",
113
+ "outputDescription"
114
+ ],
103
115
  "additionalProperties": false
104
116
  },
105
117
  "minItems": 1,
106
118
  "maxItems": 1
107
119
  }
108
120
  },
109
- "required": ["actionName", "exampleOutput", "actionDescription", "inputs", "outputs"],
121
+ "required": [
122
+ "actionName",
123
+ "exampleOutput",
124
+ "actionDescription",
125
+ "inputs",
126
+ "outputs"
127
+ ],
110
128
  "additionalProperties": false
111
129
  },
112
130
  "minItems": 1,
@@ -122,7 +140,13 @@
122
140
  "type": "string"
123
141
  }
124
142
  },
125
- "required": ["scope", "topic", "actions", "instructions", "classificationDescription"],
143
+ "required": [
144
+ "scope",
145
+ "topic",
146
+ "actions",
147
+ "instructions",
148
+ "classificationDescription"
149
+ ],
126
150
  "additionalProperties": false
127
151
  },
128
152
  "minItems": 1,
@@ -135,11 +159,18 @@
135
159
  }
136
160
  }
137
161
  },
138
- "required": ["agentDescription", "topics", "sampleUtterances"],
162
+ "required": [
163
+ "agentDescription",
164
+ "topics",
165
+ "sampleUtterances"
166
+ ],
139
167
  "additionalProperties": false
140
168
  }
141
169
  },
142
- "required": ["agentDefinition", "isSuccess"]
170
+ "required": [
171
+ "agentDefinition",
172
+ "isSuccess"
173
+ ]
143
174
  }
144
175
  }
145
- }
176
+ }
@@ -12,8 +12,11 @@
12
12
  "type": "number"
13
13
  }
14
14
  },
15
- "required": ["success", "version"],
15
+ "required": [
16
+ "success",
17
+ "version"
18
+ ],
16
19
  "additionalProperties": false
17
20
  }
18
21
  }
19
- }
22
+ }
@@ -47,7 +47,14 @@
47
47
  "type": "string"
48
48
  }
49
49
  },
50
- "required": ["agentType", "companyDescription", "companyName", "isSuccess", "role", "topics"]
50
+ "required": [
51
+ "agentType",
52
+ "companyDescription",
53
+ "companyName",
54
+ "isSuccess",
55
+ "role",
56
+ "topics"
57
+ ]
51
58
  },
52
59
  "DraftAgentTopics": {
53
60
  "type": "array",
@@ -61,7 +68,10 @@
61
68
  "type": "string"
62
69
  }
63
70
  },
64
- "required": ["name", "description"],
71
+ "required": [
72
+ "name",
73
+ "description"
74
+ ],
65
75
  "additionalProperties": false
66
76
  },
67
77
  "minItems": 1,
@@ -69,7 +79,11 @@
69
79
  },
70
80
  "AgentType": {
71
81
  "type": "string",
72
- "enum": ["customer", "internal", "AGENT"]
82
+ "enum": [
83
+ "customer",
84
+ "internal",
85
+ "AGENT"
86
+ ]
73
87
  }
74
88
  }
75
- }
89
+ }
@@ -15,8 +15,12 @@
15
15
  "type": "string"
16
16
  }
17
17
  },
18
- "required": ["agentPath", "metaXmlPath", "outputDir"],
18
+ "required": [
19
+ "agentPath",
20
+ "metaXmlPath",
21
+ "outputDir"
22
+ ],
19
23
  "additionalProperties": false
20
24
  }
21
25
  }
22
- }
26
+ }
@@ -12,8 +12,11 @@
12
12
  "type": "string"
13
13
  }
14
14
  },
15
- "required": ["genAiPlannerBundlePath", "botTemplatePath"],
15
+ "required": [
16
+ "genAiPlannerBundlePath",
17
+ "botTemplatePath"
18
+ ],
16
19
  "additionalProperties": false
17
20
  }
18
21
  }
19
- }
22
+ }
@@ -12,8 +12,11 @@
12
12
  "type": "string"
13
13
  }
14
14
  },
15
- "required": ["sessionId", "tracesPath"],
15
+ "required": [
16
+ "sessionId",
17
+ "tracesPath"
18
+ ],
16
19
  "additionalProperties": false
17
20
  }
18
21
  }
19
- }
22
+ }
@@ -27,8 +27,12 @@
27
27
  "type": "string"
28
28
  }
29
29
  },
30
- "required": ["messages", "agentApiName", "sessionId"],
30
+ "required": [
31
+ "messages",
32
+ "agentApiName",
33
+ "sessionId"
34
+ ],
31
35
  "additionalProperties": false
32
36
  }
33
37
  }
34
- }
38
+ }
@@ -23,13 +23,20 @@
23
23
  "$ref": "#/definitions/SessionType"
24
24
  }
25
25
  },
26
- "required": ["agentId", "sessionId"],
26
+ "required": [
27
+ "agentId",
28
+ "sessionId"
29
+ ],
27
30
  "additionalProperties": false
28
31
  }
29
32
  },
30
33
  "SessionType": {
31
34
  "type": "string",
32
- "enum": ["simulated", "live", "published"]
35
+ "enum": [
36
+ "simulated",
37
+ "live",
38
+ "published"
39
+ ]
33
40
  }
34
41
  }
35
- }
42
+ }
@@ -12,8 +12,11 @@
12
12
  "type": "string"
13
13
  }
14
14
  },
15
- "required": ["sessionId", "agentApiName"],
15
+ "required": [
16
+ "sessionId",
17
+ "agentApiName"
18
+ ],
16
19
  "additionalProperties": false
17
20
  }
18
21
  }
19
- }
22
+ }
@@ -6,4 +6,4 @@
6
6
  "type": "null"
7
7
  }
8
8
  }
9
- }
9
+ }
@@ -12,8 +12,11 @@
12
12
  "type": "string"
13
13
  }
14
14
  },
15
- "required": ["path", "contents"],
15
+ "required": [
16
+ "path",
17
+ "contents"
18
+ ],
16
19
  "additionalProperties": false
17
20
  }
18
21
  }
19
- }
22
+ }
@@ -57,4 +57,4 @@
57
57
  "additionalProperties": false
58
58
  }
59
59
  }
60
- }
60
+ }
@@ -3,7 +3,14 @@
3
3
  "$ref": "#/definitions/AgentTestResultsResult",
4
4
  "definitions": {
5
5
  "AgentTestResultsResult": {
6
- "$ref": "#/definitions/AgentTestResultsResponse"
6
+ "anyOf": [
7
+ {
8
+ "$ref": "#/definitions/AgentTestResultsResponse"
9
+ },
10
+ {
11
+ "$ref": "#/definitions/AgentforceStudioTestResultsResponse"
12
+ }
13
+ ]
7
14
  },
8
15
  "AgentTestResultsResponse": {
9
16
  "type": "object",
@@ -30,12 +37,23 @@
30
37
  }
31
38
  }
32
39
  },
33
- "required": ["status", "startTime", "subjectName", "testCases"],
40
+ "required": [
41
+ "status",
42
+ "startTime",
43
+ "subjectName",
44
+ "testCases"
45
+ ],
34
46
  "additionalProperties": false
35
47
  },
36
48
  "TestStatus": {
37
49
  "type": "string",
38
- "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR", "TERMINATED"]
50
+ "enum": [
51
+ "NEW",
52
+ "IN_PROGRESS",
53
+ "COMPLETED",
54
+ "ERROR",
55
+ "TERMINATED"
56
+ ]
39
57
  },
40
58
  "TestCaseResult": {
41
59
  "type": "object",
@@ -56,7 +74,9 @@
56
74
  "type": "string"
57
75
  }
58
76
  },
59
- "required": ["utterance"],
77
+ "required": [
78
+ "utterance"
79
+ ],
60
80
  "additionalProperties": false
61
81
  },
62
82
  "generatedData": {
@@ -81,7 +101,13 @@
81
101
  "type": "string"
82
102
  }
83
103
  },
84
- "required": ["actionsSequence", "invokedActions", "sessionId", "outcome", "topic"],
104
+ "required": [
105
+ "actionsSequence",
106
+ "invokedActions",
107
+ "sessionId",
108
+ "outcome",
109
+ "topic"
110
+ ],
85
111
  "additionalProperties": false
86
112
  },
87
113
  "testResults": {
@@ -102,12 +128,22 @@
102
128
  "type": "number"
103
129
  },
104
130
  "result": {
105
- "type": ["null", "string"],
106
- "enum": [null, "PASS", "FAILURE"]
131
+ "type": [
132
+ "null",
133
+ "string"
134
+ ],
135
+ "enum": [
136
+ null,
137
+ "PASS",
138
+ "FAILURE"
139
+ ]
107
140
  },
108
141
  "metricLabel": {
109
142
  "type": "string",
110
- "enum": ["Accuracy", "Precision"]
143
+ "enum": [
144
+ "Accuracy",
145
+ "Precision"
146
+ ]
111
147
  },
112
148
  "metricExplainability": {
113
149
  "type": "string"
@@ -146,8 +182,73 @@
146
182
  "type": "number"
147
183
  }
148
184
  },
149
- "required": ["status", "startTime", "inputs", "generatedData", "testResults", "testNumber"],
185
+ "required": [
186
+ "status",
187
+ "startTime",
188
+ "inputs",
189
+ "generatedData",
190
+ "testResults",
191
+ "testNumber"
192
+ ],
193
+ "additionalProperties": false
194
+ },
195
+ "AgentforceStudioTestResultsResponse": {
196
+ "type": "object",
197
+ "properties": {
198
+ "status": {
199
+ "type": "string"
200
+ },
201
+ "testCases": {
202
+ "type": "array",
203
+ "items": {
204
+ "$ref": "#/definitions/AgentforceStudioTestCaseResult"
205
+ }
206
+ }
207
+ },
208
+ "required": [
209
+ "status",
210
+ "testCases"
211
+ ],
212
+ "additionalProperties": false
213
+ },
214
+ "AgentforceStudioTestCaseResult": {
215
+ "type": "object",
216
+ "properties": {
217
+ "subjectResponse": {
218
+ "type": "string"
219
+ },
220
+ "testNumber": {
221
+ "type": "number"
222
+ },
223
+ "testScorerResults": {
224
+ "type": "array",
225
+ "items": {
226
+ "$ref": "#/definitions/TestScorerResult"
227
+ }
228
+ }
229
+ },
230
+ "required": [
231
+ "subjectResponse",
232
+ "testNumber",
233
+ "testScorerResults"
234
+ ],
235
+ "additionalProperties": false
236
+ },
237
+ "TestScorerResult": {
238
+ "type": "object",
239
+ "properties": {
240
+ "scorerName": {
241
+ "type": "string"
242
+ },
243
+ "scorerResponse": {
244
+ "type": "string"
245
+ }
246
+ },
247
+ "required": [
248
+ "scorerName",
249
+ "scorerResponse"
250
+ ],
150
251
  "additionalProperties": false
151
252
  }
152
253
  }
153
- }
254
+ }
@@ -33,7 +33,10 @@
33
33
  }
34
34
  }
35
35
  },
36
- "required": ["runId", "status"]
36
+ "required": [
37
+ "runId",
38
+ "status"
39
+ ]
37
40
  },
38
41
  {
39
42
  "type": "object",
@@ -64,13 +67,49 @@
64
67
  }
65
68
  }
66
69
  },
67
- "required": ["runId", "startTime", "status", "subjectName", "testCases"]
70
+ "required": [
71
+ "runId",
72
+ "startTime",
73
+ "status",
74
+ "subjectName",
75
+ "testCases"
76
+ ]
77
+ },
78
+ {
79
+ "type": "object",
80
+ "additionalProperties": false,
81
+ "properties": {
82
+ "status": {
83
+ "type": "string",
84
+ "const": "COMPLETED"
85
+ },
86
+ "runId": {
87
+ "type": "string"
88
+ },
89
+ "testCases": {
90
+ "type": "array",
91
+ "items": {
92
+ "$ref": "#/definitions/AgentforceStudioTestCaseResult"
93
+ }
94
+ }
95
+ },
96
+ "required": [
97
+ "runId",
98
+ "status",
99
+ "testCases"
100
+ ]
68
101
  }
69
102
  ]
70
103
  },
71
104
  "TestStatus": {
72
105
  "type": "string",
73
- "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR", "TERMINATED"]
106
+ "enum": [
107
+ "NEW",
108
+ "IN_PROGRESS",
109
+ "COMPLETED",
110
+ "ERROR",
111
+ "TERMINATED"
112
+ ]
74
113
  },
75
114
  "TestCaseResult": {
76
115
  "type": "object",
@@ -91,7 +130,9 @@
91
130
  "type": "string"
92
131
  }
93
132
  },
94
- "required": ["utterance"],
133
+ "required": [
134
+ "utterance"
135
+ ],
95
136
  "additionalProperties": false
96
137
  },
97
138
  "generatedData": {
@@ -116,7 +157,13 @@
116
157
  "type": "string"
117
158
  }
118
159
  },
119
- "required": ["actionsSequence", "invokedActions", "sessionId", "outcome", "topic"],
160
+ "required": [
161
+ "actionsSequence",
162
+ "invokedActions",
163
+ "sessionId",
164
+ "outcome",
165
+ "topic"
166
+ ],
120
167
  "additionalProperties": false
121
168
  },
122
169
  "testResults": {
@@ -137,12 +184,22 @@
137
184
  "type": "number"
138
185
  },
139
186
  "result": {
140
- "type": ["null", "string"],
141
- "enum": [null, "PASS", "FAILURE"]
187
+ "type": [
188
+ "null",
189
+ "string"
190
+ ],
191
+ "enum": [
192
+ null,
193
+ "PASS",
194
+ "FAILURE"
195
+ ]
142
196
  },
143
197
  "metricLabel": {
144
198
  "type": "string",
145
- "enum": ["Accuracy", "Precision"]
199
+ "enum": [
200
+ "Accuracy",
201
+ "Precision"
202
+ ]
146
203
  },
147
204
  "metricExplainability": {
148
205
  "type": "string"
@@ -181,8 +238,54 @@
181
238
  "type": "number"
182
239
  }
183
240
  },
184
- "required": ["status", "startTime", "inputs", "generatedData", "testResults", "testNumber"],
241
+ "required": [
242
+ "status",
243
+ "startTime",
244
+ "inputs",
245
+ "generatedData",
246
+ "testResults",
247
+ "testNumber"
248
+ ],
249
+ "additionalProperties": false
250
+ },
251
+ "AgentforceStudioTestCaseResult": {
252
+ "type": "object",
253
+ "properties": {
254
+ "subjectResponse": {
255
+ "type": "string"
256
+ },
257
+ "testNumber": {
258
+ "type": "number"
259
+ },
260
+ "testScorerResults": {
261
+ "type": "array",
262
+ "items": {
263
+ "$ref": "#/definitions/TestScorerResult"
264
+ }
265
+ }
266
+ },
267
+ "required": [
268
+ "subjectResponse",
269
+ "testNumber",
270
+ "testScorerResults"
271
+ ],
272
+ "additionalProperties": false
273
+ },
274
+ "TestScorerResult": {
275
+ "type": "object",
276
+ "properties": {
277
+ "scorerName": {
278
+ "type": "string"
279
+ },
280
+ "scorerResponse": {
281
+ "type": "string"
282
+ }
283
+ },
284
+ "required": [
285
+ "scorerName",
286
+ "scorerResponse"
287
+ ],
185
288
  "additionalProperties": false
186
289
  }
187
290
  }
188
- }
291
+ }