@salesforce/plugin-agent 1.30.11 → 1.31.1

package/lib/yamlSpecTranslator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yamlSpecTranslator.js","sourceRoot":"","sources":["../src/yamlSpecTranslator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8BAA8B;AAE9B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAI1C,4DAA4D;AAE5D,MAAM,eAAe,GAA2B;IAC9C,yBAAyB,EAAE,eAAe;IAC1C,uBAAuB,EAAE,oDAAoD;IAC7E,gCAAgC,EAAE,6DAA6D;IAC/F,iCAAiC,EAAE,6DAA6D;CACjG,CAAC;AAEF,2DAA2D;AAE3D,MAAM,oBAAoB,GAA2B;IACnD,iBAAiB,EAAE,4BAA4B;IAC/C,kBAAkB,EAAE,6BAA6B;CAClD,CAAC;AAEF,4CAA4C;AAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,uBAAuB;IACvB,gCAAgC;IAChC,iCAAiC;CAClC,CAAC,CAAC;AAEH,qBAAqB;AAErB;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAC9C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB,EAAE,KAAa,EAAE,QAAiB;IACpF,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,EAAE,CAAC;IACzE,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,0BAA0B;IAC1B,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,sBAAsB;QAC5B,EAAE,EAAE,IAAI;QACR,aAAa,EAAE,IAAI;KACpB,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,oBAAoB;oBAC1B,EAAE,EAAE,WAAW,UAAU,EAAE;oBAC3B,UAAU,EAAE,iBAAiB;oBAC7B,SAAS,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,oBAAoB;QAC1B,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,iBAAiB;QAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,iBAAiB;YACvB,EAAE,EAAE,IAAI;YACR,UAAU,EAAE,iBAAiB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB;IAChB,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,mCAAmC;YACzC,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,QAAQ,CAAC,aAAa;YAChC,MAAM,EAAE,oDAAoD;YAC5D,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,qCAAqC;YAC3C,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,6DAA6D;YACrE,QAAQ,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,+BAA+B;YACrC,EAAE,EAAE,eAAe;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,eAAe;YACvB,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,yBAAyB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,2BAA2B;AAE3B;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAkB;IAC3C,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtD,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/F,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAChC,UAA8D,EAC9D,KAAa;IAEb,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,UAAU,CAAC,IAAI,EAAE,CAAC;IAEzF,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9B,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,EAAE,EAAE,UAAU,KAAK,EAAE;QACrB,QAAQ;QACR,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC"}

package/messages/agent.test.run-eval.md

@@ -0,0 +1,91 @@
+# summary
+
+Run evaluation tests against an Agentforce agent.
+
+# description
+
+Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test specs (same format as `sf agent generate test-spec`) and JSON payloads.
+
+When you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`.
+
+When you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization.
+
+Supports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings.
+
+# flags.spec.summary
+
+Path to test spec file (YAML or JSON). Use `-` for stdin.
+
+# flags.api-name.summary
+
+Agent DeveloperName (also called API name) to resolve agent_id and agent_version_id. Auto-inferred from the YAML spec's subjectName.
+
+# flags.wait.summary
+
+Number of minutes to wait for results.
+
+# flags.result-format.summary
+
+Format of the agent test results.
+
+# flags.batch-size.summary
+
+Number of tests per API request (max 5).
+
+# flags.no-normalize.summary
+
+Disable auto-normalization of field names and shorthand references.
+
+# examples
+
+- Run tests using a YAML test spec on the org with alias "my-org":
+
+  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org
+
+- Run tests using a YAML spec with explicit agent name override; use your default org:
+
+  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --api-name My_Agent --target-org my-org
+
+- Run tests using a JSON payload:
+
+  <%= config.bin %> <%= command.id %> --spec tests/eval-payload.json --target-org my-org
+
+- Run tests and output results in JUnit format; useful for continuous integration and deployment (CI/CD):
+
+  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org --result-format junit
+
+- Pipe JSON payload from stdin:
+
+  $ echo '{"tests":[...]}' | <%= config.bin %> <%= command.id %> --spec - --target-org my-org
+
+# info.batchProgress
+
+Running batch %s of %s (%s tests)...
+
+# info.testComplete
+
+Test %s: %s.
+
+# info.summary
+
+Results: %s passed, %s failed, %s scored, %s errors.
+
+# info.yamlDetected
+
+Detected YAML test spec for agent '%s' with %s test case(s). Translating to Evaluation API format.
+
+# error.invalidPayload
+
+Invalid test payload: %s.
+
+# error.apiError
+
+Einstein Eval API error (HTTP %s): %s
+
+# error.agentNotFound
+
+No agent found with DeveloperName (also API name) '%s'. Verify that the agent exists in the target org.
+
+# error.agentVersionNotFound
+
+No published version found for agent '%s'. Make sure the agent has been published at least once.

package/oclif.manifest.json

@@ -791,6 +791,96 @@
         "test-spec:generate:agent"
       ]
     },
+    "agent:publish:authoring-bundle": {
+      "aliases": [],
+      "args": {},
+      "description": "An authoring bundle is a metadata type (named aiAuthoringBundle) that provides the blueprint for an agent. The metadata type contains two files: the standard metatada XML file and an Agent Script file (extension \".agent\") that fully describes the agent using the Agent Script language.\n\nWhen you publish an authoring bundle to your org, a number of things happen. First, this command validates that the Agent Script file successfully compiles. If there are compilation errors, the command exits and you must fix the Agent Script file to continue. Once the Agent Script file compiles, then it's published to the org, which in turn creates new associated metadata (Bot, BotVersion, GenAiX), or new versions of the metadata if the agent already exists. The new or updated metadata is retrieved back to your DX project; specify the --skip-retrieve flag to skip this step. Finally, the authoring bundle metadata (AiAuthoringBundle) is deployed to your org.\n\nThis command uses the API name of the authoring bundle.",
+      "examples": [
+        "Publish an authoring bundle by being prompted for its API name; use your default org:\n<%= config.bin %> <%= command.id %>",
+        "Publish an authoring bundle with API name MyAuthoringBundle to the org with alias \"my-dev-org\":\n<%= config.bin %> <%= command.id %> --api-name MyAuthoringbundle --target-org my-dev-org"
+      ],
+      "flags": {
+        "json": {
+          "description": "Format output as json.",
+          "helpGroup": "GLOBAL",
+          "name": "json",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "flags-dir": {
+          "helpGroup": "GLOBAL",
+          "name": "flags-dir",
+          "summary": "Import flag values from a directory.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "target-org": {
+          "char": "o",
+          "name": "target-org",
+          "noCacheDefault": true,
+          "required": true,
+          "summary": "Username or alias of the target org. Not required if the `target-org` configuration variable is already set.",
+          "hasDynamicHelp": true,
+          "multiple": false,
+          "type": "option"
+        },
+        "api-version": {
+          "description": "Override the api version used for api requests made by this command",
+          "name": "api-version",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "api-name": {
+          "char": "n",
+          "name": "api-name",
+          "summary": "API name of the authoring bundle you want to publish; if not specified, the command provides a list that you can choose from.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "skip-retrieve": {
+          "name": "skip-retrieve",
+          "summary": "Don't retrieve the metadata associated with the agent to your DX project.",
+          "allowNo": false,
+          "type": "boolean"
+        }
+      },
+      "hasDynamicHelp": true,
+      "hiddenAliases": [],
+      "id": "agent:publish:authoring-bundle",
+      "pluginAlias": "@salesforce/plugin-agent",
+      "pluginName": "@salesforce/plugin-agent",
+      "pluginType": "core",
+      "strict": true,
+      "summary": "Publish an authoring bundle to your org, which results in a new agent or a new version of an existing agent.",
+      "enableJsonFlag": true,
+      "requiresProject": true,
+      "FLAGGABLE_PROMPTS": {
+        "api-name": {
+          "message": "API name of the authoring bundle you want to publish; if not specified, the command provides a list that you can choose from.",
+          "promptMessage": "API name of the authoring bundle to publish"
+        }
+      },
+      "isESM": true,
+      "relativePath": [
+        "lib",
+        "commands",
+        "agent",
+        "publish",
+        "authoring-bundle.js"
+      ],
+      "aliasPermutations": [],
+      "permutations": [
+        "agent:publish:authoring-bundle",
+        "publish:agent:authoring-bundle",
+        "publish:authoring-bundle:agent",
+        "agent:authoring-bundle:publish",
+        "authoring-bundle:agent:publish",
+        "authoring-bundle:publish:agent"
+      ]
+    },
     "agent:preview:end": {
       "aliases": [],
       "args": {},
@@ -1136,96 +1226,6 @@
         "start:preview:agent"
       ]
     },
-    "agent:publish:authoring-bundle": {
-      "aliases": [],
-      "args": {},
-      "description": "An authoring bundle is a metadata type (named aiAuthoringBundle) that provides the blueprint for an agent. The metadata type contains two files: the standard metatada XML file and an Agent Script file (extension \".agent\") that fully describes the agent using the Agent Script language.\n\nWhen you publish an authoring bundle to your org, a number of things happen. First, this command validates that the Agent Script file successfully compiles. If there are compilation errors, the command exits and you must fix the Agent Script file to continue. Once the Agent Script file compiles, then it's published to the org, which in turn creates new associated metadata (Bot, BotVersion, GenAiX), or new versions of the metadata if the agent already exists. The new or updated metadata is retrieved back to your DX project; specify the --skip-retrieve flag to skip this step. Finally, the authoring bundle metadata (AiAuthoringBundle) is deployed to your org.\n\nThis command uses the API name of the authoring bundle.",
-      "examples": [
-        "Publish an authoring bundle by being prompted for its API name; use your default org:\n<%= config.bin %> <%= command.id %>",
-        "Publish an authoring bundle with API name MyAuthoringBundle to the org with alias \"my-dev-org\":\n<%= config.bin %> <%= command.id %> --api-name MyAuthoringbundle --target-org my-dev-org"
-      ],
-      "flags": {
-        "json": {
-          "description": "Format output as json.",
-          "helpGroup": "GLOBAL",
-          "name": "json",
-          "allowNo": false,
-          "type": "boolean"
-        },
-        "flags-dir": {
-          "helpGroup": "GLOBAL",
-          "name": "flags-dir",
-          "summary": "Import flag values from a directory.",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "target-org": {
-          "char": "o",
-          "name": "target-org",
-          "noCacheDefault": true,
-          "required": true,
-          "summary": "Username or alias of the target org. Not required if the `target-org` configuration variable is already set.",
-          "hasDynamicHelp": true,
-          "multiple": false,
-          "type": "option"
-        },
-        "api-version": {
-          "description": "Override the api version used for api requests made by this command",
-          "name": "api-version",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "api-name": {
-          "char": "n",
-          "name": "api-name",
-          "summary": "API name of the authoring bundle you want to publish; if not specified, the command provides a list that you can choose from.",
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "skip-retrieve": {
-          "name": "skip-retrieve",
-          "summary": "Don't retrieve the metadata associated with the agent to your DX project.",
-          "allowNo": false,
-          "type": "boolean"
-        }
-      },
-      "hasDynamicHelp": true,
-      "hiddenAliases": [],
-      "id": "agent:publish:authoring-bundle",
-      "pluginAlias": "@salesforce/plugin-agent",
-      "pluginName": "@salesforce/plugin-agent",
-      "pluginType": "core",
-      "strict": true,
-      "summary": "Publish an authoring bundle to your org, which results in a new agent or a new version of an existing agent.",
-      "enableJsonFlag": true,
-      "requiresProject": true,
-      "FLAGGABLE_PROMPTS": {
-        "api-name": {
-          "message": "API name of the authoring bundle you want to publish; if not specified, the command provides a list that you can choose from.",
-          "promptMessage": "API name of the authoring bundle to publish"
-        }
-      },
-      "isESM": true,
-      "relativePath": [
-        "lib",
-        "commands",
-        "agent",
-        "publish",
-        "authoring-bundle.js"
-      ],
-      "aliasPermutations": [],
-      "permutations": [
-        "agent:publish:authoring-bundle",
-        "publish:agent:authoring-bundle",
-        "publish:authoring-bundle:agent",
-        "agent:authoring-bundle:publish",
-        "authoring-bundle:agent:publish",
-        "authoring-bundle:publish:agent"
-      ]
-    },
     "agent:test:create": {
       "aliases": [],
       "args": {},
@@ -1624,6 +1624,134 @@
         "resume:test:agent"
       ]
     },
+    "agent:test:run-eval": {
+      "aliases": [],
+      "args": {},
+      "description": "Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test specs (same format as `sf agent generate test-spec`) and JSON payloads.\n\nWhen you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`.\n\nWhen you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization.\n\nSupports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings.",
+      "examples": [
+        "Run tests using a YAML test spec on the org with alias \"my-org\":\n<%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org",
+        "Run tests using a YAML spec with explicit agent name override; use your default org:\n<%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --api-name My_Agent --target-org my-org",
+        "Run tests using a JSON payload:\n<%= config.bin %> <%= command.id %> --spec tests/eval-payload.json --target-org my-org",
+        "Run tests and output results in JUnit format; useful for continuous integration and deployment (CI/CD):\n<%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org --result-format junit",
+        "Pipe JSON payload from stdin:\n$ echo '{\"tests\":[...]}' | <%= config.bin %> <%= command.id %> --spec - --target-org my-org"
+      ],
+      "flags": {
+        "json": {
+          "description": "Format output as json.",
+          "helpGroup": "GLOBAL",
+          "name": "json",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "flags-dir": {
+          "helpGroup": "GLOBAL",
+          "name": "flags-dir",
+          "summary": "Import flag values from a directory.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "target-org": {
+          "char": "o",
+          "name": "target-org",
+          "noCacheDefault": true,
+          "required": true,
+          "summary": "Username or alias of the target org. Not required if the `target-org` configuration variable is already set.",
+          "hasDynamicHelp": true,
+          "multiple": false,
+          "type": "option"
+        },
+        "api-version": {
+          "description": "Override the api version used for api requests made by this command",
+          "name": "api-version",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "spec": {
+          "char": "s",
+          "name": "spec",
+          "required": true,
+          "summary": "Path to test spec file (YAML or JSON). Use `-` for stdin.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "api-name": {
+          "char": "n",
+          "name": "api-name",
+          "summary": "Agent DeveloperName (also called API name) to resolve agent_id and agent_version_id. Auto-inferred from the YAML spec's subjectName.",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "wait": {
+          "char": "w",
+          "name": "wait",
+          "summary": "Number of minutes to wait for results.",
+          "default": 10,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "result-format": {
+          "name": "result-format",
+          "summary": "Format of the agent test run results.",
+          "default": "human",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "options": [
+            "json",
+            "human",
+            "junit",
+            "tap"
+          ],
+          "type": "option"
+        },
+        "batch-size": {
+          "name": "batch-size",
+          "summary": "Number of tests per API request (max 5).",
+          "default": 5,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "no-normalize": {
+          "name": "no-normalize",
+          "summary": "Disable auto-normalization of field names and shorthand references.",
+          "allowNo": false,
+          "type": "boolean"
+        }
+      },
+      "hasDynamicHelp": true,
+      "hidden": true,
+      "hiddenAliases": [],
+      "id": "agent:test:run-eval",
+      "pluginAlias": "@salesforce/plugin-agent",
+      "pluginName": "@salesforce/plugin-agent",
+      "pluginType": "core",
+      "state": "beta",
+      "strict": true,
+      "summary": "Run evaluation tests against an Agentforce agent.",
+      "enableJsonFlag": true,
+      "isESM": true,
+      "relativePath": [
+        "lib",
+        "commands",
+        "agent",
+        "test",
+        "run-eval.js"
+      ],
+      "aliasPermutations": [],
+      "permutations": [
+        "agent:test:run-eval",
+        "test:agent:run-eval",
+        "test:run-eval:agent",
+        "agent:run-eval:test",
+        "run-eval:agent:test",
+        "run-eval:test:agent"
+      ]
+    },
     "agent:test:run": {
       "aliases": [],
       "args": {},
@@ -1825,5 +1953,5 @@
       ]
     }
   },
-  "version": "1.30.11"
+  "version": "1.31.1"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@salesforce/plugin-agent",
     "description": "Commands to interact with Salesforce agents",
-    "version": "1.30.11",
+    "version": "1.31.1",
     "author": "Salesforce",
     "bugs": "https://github.com/forcedotcom/cli/issues",
     "enableO11y": true,
@@ -12,7 +12,7 @@
         "@inquirer/prompts": "^7.10.1",
         "@oclif/core": "^4",
         "@oclif/multi-stage-output": "^0.8.29",
-        "@salesforce/agents": "^0.23.4",
+        "@salesforce/agents": "^0.24.1",
         "@salesforce/core": "^8.26.2",
         "@salesforce/kit": "^3.2.4",
         "@salesforce/sf-plugins-core": "^12.2.6",
@@ -28,11 +28,11 @@
         "yaml": "^2.8.2"
     },
     "devDependencies": {
-        "@oclif/plugin-command-snapshot": "^5.3.8",
+        "@oclif/plugin-command-snapshot": "^5.3.10",
         "@oclif/test": "^4.1.15",
         "@salesforce/cli-plugins-testkit": "^5.3.41",
         "@salesforce/dev-scripts": "^11.0.4",
-        "@salesforce/plugin-command-reference": "^3.1.80",
+        "@salesforce/plugin-command-reference": "^3.1.81",
         "@types/inquirer": "^9.0.9",
         "@types/react": "^18.3.3",
         "eslint-config-xo": "^0.50.0",
@@ -240,7 +240,7 @@
     "exports": "./lib/index.js",
     "type": "module",
     "sfdx": {
-        "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.30.11.crt",
-        "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.30.11.sig"
+        "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.31.1.crt",
+        "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.31.1.sig"
     }
 }

package/schemas/agent-test-run__eval.json

@@ -0,0 +1,52 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/RunEvalResult",
+  "definitions": {
+    "RunEvalResult": {
+      "type": "object",
+      "properties": {
+        "tests": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string"
+              },
+              "status": {
+                "type": "string"
+              },
+              "evaluations": {
+                "type": "array",
+                "items": {}
+              }
+            },
+            "required": ["id", "status", "evaluations"],
+            "additionalProperties": false
+          }
+        },
+        "summary": {
+          "type": "object",
+          "properties": {
+            "passed": {
+              "type": "number"
+            },
+            "failed": {
+              "type": "number"
+            },
+            "scored": {
+              "type": "number"
+            },
+            "errors": {
+              "type": "number"
+            }
+          },
+          "required": ["passed", "failed", "scored", "errors"],
+          "additionalProperties": false
+        }
+      },
+      "required": ["tests", "summary"],
+      "additionalProperties": false
+    }
+  }
+}