@salesforce/plugin-agent 1.3.3 → 1.4.0

package/oclif.manifest.json

@@ -130,8 +130,8 @@
           "hasDynamicHelp": false,
           "multiple": false,
           "options": [
-            "customer_facing",
-            "employee_facing"
+            "customer",
+            "internal"
           ],
           "type": "option"
         },
@@ -216,7 +216,7 @@
       "args": {},
       "description": "Cancel a running test for an Agent, providing the AiEvaluation ID.",
       "examples": [
-        "Cancel a test for an Agent:\n<%= config.bin %> <%= command.id %> --id AiEvalId"
+        "Cancel a test for an Agent:\n<%= config.bin %> <%= command.id %> --job-id AiEvalId"
       ],
       "flags": {
         "json": {
@@ -244,10 +244,16 @@
           "multiple": false,
           "type": "option"
         },
+        "api-version": {
+          "description": "Override the api version used for api requests made by this command",
+          "name": "api-version",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
         "job-id": {
           "char": "i",
           "name": "job-id",
-          "required": true,
           "summary": "The AiEvaluation ID.",
           "hasDynamicHelp": false,
           "multiple": false,
@@ -289,12 +295,208 @@
         "cancel:test:agent"
       ]
     },
+    "agent:test:results": {
+      "aliases": [],
+      "args": {},
+      "description": "Provide the AiEvaluation ID to get the results of a test evaluation.",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> --job-id AiEvalId"
+      ],
+      "flags": {
+        "json": {
+          "description": "Format output as json.",
+          "helpGroup": "GLOBAL",
+          "name": "json",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "flags-dir": {
+          "helpGroup": "GLOBAL",
+          "name": "flags-dir",
+          "summary": "Import flag values from a directory.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "target-org": {
+          "char": "o",
+          "name": "target-org",
+          "noCacheDefault": true,
+          "required": true,
+          "summary": "Username or alias of the target org. Not required if the `target-org` configuration variable is already set.",
+          "hasDynamicHelp": true,
+          "multiple": false,
+          "type": "option"
+        },
+        "api-version": {
+          "description": "Override the api version used for api requests made by this command",
+          "name": "api-version",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "job-id": {
+          "char": "i",
+          "name": "job-id",
+          "required": true,
+          "summary": "The AiEvaluation ID.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "result-format": {
+          "name": "result-format",
+          "summary": "Format of the test run results.",
+          "default": "human",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "json",
+            "human"
+          ],
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": true,
+      "hiddenAliases": [],
+      "id": "agent:test:results",
+      "pluginAlias": "@salesforce/plugin-agent",
+      "pluginName": "@salesforce/plugin-agent",
+      "pluginType": "core",
+      "state": "beta",
+      "strict": true,
+      "summary": "Get the results of a test evaluation.",
+      "enableJsonFlag": true,
+      "isESM": true,
+      "relativePath": [
+        "lib",
+        "commands",
+        "agent",
+        "test",
+        "results.js"
+      ],
+      "aliasPermutations": [],
+      "permutations": [
+        "agent:test:results",
+        "test:agent:results",
+        "test:results:agent",
+        "agent:results:test",
+        "results:agent:test",
+        "results:test:agent"
+      ]
+    },
+    "agent:test:resume": {
+      "aliases": [],
+      "args": {},
+      "description": "Resume a running test for an Agent, providing the AiEvaluation ID.",
+      "examples": [
+        "Resume a test for an Agent:\n<%= config.bin %> <%= command.id %> --job-id AiEvalId"
+      ],
+      "flags": {
+        "json": {
+          "description": "Format output as json.",
+          "helpGroup": "GLOBAL",
+          "name": "json",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "flags-dir": {
+          "helpGroup": "GLOBAL",
+          "name": "flags-dir",
+          "summary": "Import flag values from a directory.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "target-org": {
+          "char": "o",
+          "name": "target-org",
+          "noCacheDefault": true,
+          "required": true,
+          "summary": "Username or alias of the target org. Not required if the `target-org` configuration variable is already set.",
+          "hasDynamicHelp": true,
+          "multiple": false,
+          "type": "option"
+        },
+        "api-version": {
+          "description": "Override the api version used for api requests made by this command",
+          "name": "api-version",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "job-id": {
+          "char": "i",
+          "name": "job-id",
+          "summary": "The AiEvaluation ID.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "use-most-recent": {
+          "char": "r",
+          "name": "use-most-recent",
+          "summary": "Use the job ID of the most recent test evaluation.",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "wait": {
+          "char": "w",
+          "description": "If the command continues to run after the wait period, the CLI returns control of the terminal window to you.",
+          "name": "wait",
+          "summary": "Number of minutes to wait for the command to complete and display results to the terminal window.",
+          "default": "5 minutes",
+          "hasDynamicHelp": true,
+          "multiple": false,
+          "type": "option"
+        },
+        "result-format": {
+          "name": "result-format",
+          "summary": "Format of the test run results.",
+          "default": "human",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "json",
+            "human"
+          ],
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": true,
+      "hiddenAliases": [],
+      "id": "agent:test:resume",
+      "pluginAlias": "@salesforce/plugin-agent",
+      "pluginName": "@salesforce/plugin-agent",
+      "pluginType": "core",
+      "state": "beta",
+      "strict": true,
+      "summary": "Resume a running test for an Agent.",
+      "enableJsonFlag": true,
+      "isESM": true,
+      "relativePath": [
+        "lib",
+        "commands",
+        "agent",
+        "test",
+        "resume.js"
+      ],
+      "aliasPermutations": [],
+      "permutations": [
+        "agent:test:resume",
+        "test:agent:resume",
+        "test:resume:agent",
+        "agent:resume:test",
+        "resume:agent:test",
+        "resume:test:agent"
+      ]
+    },
     "agent:test:run": {
       "aliases": [],
       "args": {},
       "description": "Start a test for an Agent, providing the AiEvalDefinitionVersion ID. Returns the job ID.",
       "examples": [
-        "Start a test for an Agent:\n<%= config.bin %> <%= command.id %> --id AiEvalDefVerId"
+        "Start a test for an Agent:\n<%= config.bin %> <%= command.id %> --name AiEvalDefVerId"
       ],
       "flags": {
         "json": {
@@ -322,12 +524,19 @@
           "multiple": false,
           "type": "option"
         },
-        "id": {
-          "char": "i",
-          "description": "The AiEvalDefinitionVersion ID.",
-          "name": "id",
+        "api-version": {
+          "description": "Override the api version used for api requests made by this command",
+          "name": "api-version",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "name": {
+          "char": "n",
+          "description": "The name of the AiEvaluationDefinition to start.",
+          "name": "name",
           "required": true,
-          "summary": "The AiEvalDefinitionVersion ID.",
+          "summary": "The name of the AiEvaluationDefinition to start.",
           "hasDynamicHelp": false,
           "multiple": false,
           "type": "option"
@@ -341,12 +550,16 @@
           "multiple": false,
           "type": "option"
         },
-        "output-dir": {
-          "char": "d",
-          "name": "output-dir",
-          "summary": "Directory in which to store test run files.",
+        "result-format": {
+          "name": "result-format",
+          "summary": "Format of the test run results.",
+          "default": "human",
           "hasDynamicHelp": false,
           "multiple": false,
+          "options": [
+            "json",
+            "human"
+          ],
           "type": "option"
         }
       },
@@ -379,5 +592,5 @@
       ]
     }
   },
-  "version": "1.3.3"
+  "version": "1.4.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@salesforce/plugin-agent",
     "description": "Commands to interact with Salesforce agents",
-    "version": "1.3.3",
+    "version": "1.4.0",
     "author": "Salesforce",
     "bugs": "https://github.com/forcedotcom/cli/issues",
     "dependencies": {
@@ -9,15 +9,16 @@
         "@inquirer/input": "^4.0.1",
         "@inquirer/select": "^4.0.1",
         "@oclif/core": "^4",
-        "@salesforce/agents": "^0.2.3",
         "@oclif/multi-stage-output": "^0.7.12",
-        "@salesforce/core": "^8.5.2",
+        "@salesforce/agents": "^0.3.0",
+        "@salesforce/core": "^8.8.0",
         "@salesforce/kit": "^3.2.1",
-        "@salesforce/sf-plugins-core": "^12",
+        "@salesforce/sf-plugins-core": "^12.1.0",
         "ansis": "^3.3.2"
     },
     "devDependencies": {
         "@oclif/plugin-command-snapshot": "^5.2.19",
+        "@oclif/test": "^4.1.0",
         "@salesforce/cli-plugins-testkit": "^5.3.35",
         "@salesforce/dev-scripts": "^10.2.10",
         "@salesforce/plugin-command-reference": "^3.1.29",
@@ -59,7 +60,16 @@
         ],
         "topics": {
             "agent": {
-                "description": "Commands to work with agents."
+                "description": "Commands to work with agents.",
+                "external": true,
+                "subtopics": {
+                    "test": {
+                        "external": true
+                    },
+                    "generate": {
+                        "external": true
+                    }
+                }
             }
         },
         "flexibleTaxonomy": true
@@ -203,7 +213,7 @@
     "exports": "./lib/index.js",
     "type": "module",
     "sfdx": {
-        "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.3.3.crt",
-        "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.3.3.sig"
+        "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.4.0.crt",
+        "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.4.0.sig"
     }
 }

package/schemas/agent-create.json

@@ -12,10 +12,8 @@
           "type": "string"
         }
       },
-      "required": [
-        "isSuccess"
-      ],
+      "required": ["isSuccess"],
       "additionalProperties": false
     }
   }
-}
+}

package/schemas/agent-generate-spec.json

@@ -15,10 +15,8 @@
           "type": "string"
         }
       },
-      "required": [
-        "isSuccess"
-      ],
+      "required": ["isSuccess"],
       "additionalProperties": false
     }
   }
-}
+}

package/schemas/agent-test-cancel.json

@@ -5,7 +5,7 @@
     "AgentTestCancelResult": {
       "type": "object",
       "properties": {
-        "jobId": {
+        "aiEvaluationId": {
           "type": "string"
         },
         "success": {
@@ -18,11 +18,8 @@
           "type": "string"
         }
       },
-      "required": [
-        "jobId",
-        "success"
-      ],
+      "required": ["aiEvaluationId", "success"],
       "additionalProperties": false
     }
   }
-}
+}

package/schemas/agent-test-results.json

@@ -0,0 +1,145 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentTestResultsResult",
+  "definitions": {
+    "AgentTestResultsResult": {
+      "$ref": "#/definitions/AgentTestDetailsResponse"
+    },
+    "AgentTestDetailsResponse": {
+      "type": "object",
+      "properties": {
+        "status": {
+          "$ref": "#/definitions/TestStatus"
+        },
+        "startTime": {
+          "type": "string"
+        },
+        "endTime": {
+          "type": "string"
+        },
+        "errorMessage": {
+          "type": "string"
+        },
+        "testCases": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TestCaseResult"
+          }
+        }
+      },
+      "required": ["status", "startTime", "testCases"],
+      "additionalProperties": false
+    },
+    "TestStatus": {
+      "type": "string",
+      "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR"]
+    },
+    "TestCaseResult": {
+      "type": "object",
+      "properties": {
+        "status": {
+          "$ref": "#/definitions/TestStatus"
+        },
+        "number": {
+          "type": "string"
+        },
+        "startTime": {
+          "type": "string"
+        },
+        "endTime": {
+          "type": "string"
+        },
+        "generatedData": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "const": "AGENT"
+            },
+            "actionsSequence": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "outcome": {
+              "type": "string",
+              "enum": ["Success", "Failure"]
+            },
+            "topic": {
+              "type": "string"
+            },
+            "inputTokensCount": {
+              "type": "string"
+            },
+            "outputTokensCount": {
+              "type": "string"
+            }
+          },
+          "required": ["type", "actionsSequence", "outcome", "topic", "inputTokensCount", "outputTokensCount"],
+          "additionalProperties": false
+        },
+        "expectationResults": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "actualValue": {
+                "type": "string"
+              },
+              "expectedValue": {
+                "type": "string"
+              },
+              "score": {
+                "type": "number"
+              },
+              "result": {
+                "type": "string",
+                "enum": ["Passed", "Failed"]
+              },
+              "metricLabel": {
+                "type": "string",
+                "enum": ["Accuracy", "Precision"]
+              },
+              "metricExplainability": {
+                "type": "string"
+              },
+              "status": {
+                "$ref": "#/definitions/TestStatus"
+              },
+              "startTime": {
+                "type": "string"
+              },
+              "endTime": {
+                "type": "string"
+              },
+              "errorCode": {
+                "type": "string"
+              },
+              "errorMessage": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "name",
+              "actualValue",
+              "expectedValue",
+              "score",
+              "result",
+              "metricLabel",
+              "metricExplainability",
+              "status",
+              "startTime"
+            ],
+            "additionalProperties": false
+          }
+        }
+      },
+      "required": ["status", "number", "startTime", "generatedData", "expectationResults"],
+      "additionalProperties": false
+    }
+  }
+}

package/schemas/agent-test-resume.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentTestResumeResult",
+  "definitions": {
+    "AgentTestResumeResult": {
+      "type": "object",
+      "properties": {
+        "aiEvaluationId": {
+          "type": "string"
+        },
+        "status": {
+          "type": "string"
+        }
+      },
+      "required": ["aiEvaluationId", "status"],
+      "additionalProperties": false
+    }
+  }
+}

package/schemas/agent-test-run.json

@@ -5,24 +5,15 @@
     "AgentTestRunResult": {
       "type": "object",
       "properties": {
-        "jobId": {
+        "aiEvaluationId": {
           "type": "string"
         },
-        "success": {
-          "type": "boolean"
-        },
-        "errorCode": {
-          "type": "string"
-        },
-        "message": {
+        "status": {
           "type": "string"
         }
       },
-      "required": [
-        "jobId",
-        "success"
-      ],
+      "required": ["aiEvaluationId", "status"],
       "additionalProperties": false
     }
   }
-}
+}