@salesforce/plugin-agent 1.36.1 → 1.37.0

package/README.md

@@ -77,6 +77,7 @@ sf plugins
 - [`sf agent test results`](#sf-agent-test-results)
 - [`sf agent test resume`](#sf-agent-test-resume)
 - [`sf agent test run`](#sf-agent-test-run)
+- [`sf agent test run-eval`](#sf-agent-test-run-eval)
 - [`sf agent validate authoring-bundle`](#sf-agent-validate-authoring-bundle)
 
 ## `sf agent activate`
@@ -126,7 +127,7 @@ EXAMPLES
     $ sf agent activate --api-name Resort_Manager --version 2 --target-org my-org
 ```
 
-_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/activate.ts)_
+_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/activate.ts)_
 
 ## `sf agent create`
 
@@ -193,7 +194,7 @@ EXAMPLES
     $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview
 ```
 
-_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/create.ts)_
+_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/create.ts)_
 
 ## `sf agent deactivate`
 
@@ -234,7 +235,7 @@ EXAMPLES
     $ sf agent deactivate --api-name Resort_Manager --target-org my-org
 ```
 
-_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/deactivate.ts)_
+_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/deactivate.ts)_
 
 ## `sf agent generate agent-spec`
 
@@ -341,7 +342,7 @@ EXAMPLES
     $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com
 ```
 
-_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/agent-spec.ts)_
+_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/generate/agent-spec.ts)_
 
 ## `sf agent generate authoring-bundle`
 
@@ -418,7 +419,7 @@ EXAMPLES
       other-package-dir/main/default --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/authoring-bundle.ts)_
+_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/generate/authoring-bundle.ts)_
 
 ## `sf agent generate template`
 
@@ -480,7 +481,7 @@ EXAMPLES
       my-package --source-org my-scratch-org
 ```
 
-_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/template.ts)_
+_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/generate/template.ts)_
 
 ## `sf agent generate test-spec`
 
@@ -545,7 +546,7 @@ EXAMPLES
       force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml
 ```
 
-_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/generate/test-spec.ts)_
+_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/generate/test-spec.ts)_
 
 ## `sf agent preview`
 
@@ -618,7 +619,7 @@ EXAMPLES
     $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview
 ```
 
-_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview.ts)_
+_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/preview.ts)_
 
 ## `sf agent preview end`
 
@@ -673,7 +674,7 @@ EXAMPLES
     $ sf agent preview end --authoring-bundle My_Local_Agent
 ```
 
-_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/end.ts)_
+_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/preview/end.ts)_
 
 ## `sf agent preview send`
 
@@ -731,7 +732,7 @@ EXAMPLES
     $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent
 ```
 
-_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/send.ts)_
+_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/preview/send.ts)_
 
 ## `sf agent preview sessions`
 
@@ -764,7 +765,7 @@ EXAMPLES
     $ sf agent preview sessions
 ```
 
-_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/sessions.ts)_
+_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/preview/sessions.ts)_
 
 ## `sf agent preview start`
 
@@ -829,7 +830,7 @@ EXAMPLES
     $ sf agent preview start --api-name My_Published_Agent
 ```
 
-_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/preview/start.ts)_
+_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/preview/start.ts)_
 
 ## `sf agent publish authoring-bundle`
 
@@ -889,7 +890,7 @@ EXAMPLES
     $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --concise
 ```
 
-_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/publish/authoring-bundle.ts)_
+_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/publish/authoring-bundle.ts)_
 
 ## `sf agent test create`
 
@@ -944,7 +945,7 @@ EXAMPLES
     $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview
 ```
 
-_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/create.ts)_
+_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/test/create.ts)_
 
 ## `sf agent test list`
 
@@ -979,7 +980,7 @@ EXAMPLES
     $ sf agent test list --target-org my-org
 ```
 
-_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/list.ts)_
+_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/test/list.ts)_
 
 ## `sf agent test results`
 
@@ -1055,7 +1056,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/results.ts)_
+_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/test/results.ts)_
 
 ## `sf agent test resume`
 
@@ -1139,7 +1140,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/resume.ts)_
+_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/test/resume.ts)_
 
 ## `sf agent test run`
 
@@ -1223,7 +1224,80 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/test/run.ts)_
+_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/test/run.ts)_
+
+## `sf agent test run-eval`
+
+Run evaluation tests against an Agentforce agent.
+
+```
+USAGE
+  $ sf agent test run-eval -o <value> -s <value> [--json] [--flags-dir <value>] [--api-version <value>] [-n <value>]
+    [--result-format json|human|junit|tap] [--batch-size <value>] [--no-normalize]
+
+FLAGS
+  -n, --api-name=<value>        Agent DeveloperName (also called API name) to resolve agent_id and agent_version_id.
+                                Auto-inferred from the YAML spec's subjectName.
+  -o, --target-org=<value>      (required) Username or alias of the target org. Not required if the `target-org`
+                                configuration variable is already set.
+  -s, --spec=<value>            (required) Path to test spec file (YAML or JSON). Supports reading from stdin when
+                                piping content.
+      --api-version=<value>     Override the api version used for api requests made by this command
+      --batch-size=<value>      [default: 5] Number of tests per API request (max 5).
+      --no-normalize            Disable auto-normalization of field names and shorthand references.
+      --result-format=<option>  [default: human] Format of the agent test run results.
+                                <options: json|human|junit|tap>
+
+GLOBAL FLAGS
+  --flags-dir=<value>  Import flag values from a directory.
+  --json               Format output as json.
+
+DESCRIPTION
+  Run evaluation tests against an Agentforce agent.
+
+  Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test
+  specs (same format as `sf agent generate test-spec`) and JSON payloads.
+
+  When you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and
+  infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf
+  agent test run` and `sf agent test run-eval`. YAML test specs also support contextVariables, which allow you to inject
+  contextual data (such as CaseId or RoutableId) into agent sessions for testing with different contexts.
+
+  When you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer
+  auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use
+  `--no-normalize` to disable this auto-normalization. JSON payloads can also include context_variables on
+  agent.create_session steps for the same contextual testing capabilities.
+
+  Supports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions,
+  semantic similarity scoring, and LLM-based quality ratings.
+
+EXAMPLES
+  Run tests using a YAML test spec on the org with alias "my-org":
+
+    $ sf agent test run-eval --spec tests/my-agent-testSpec.yaml --target-org my-org
+
+  Run tests using a YAML spec with explicit agent name override; use your default org:
+
+    $ sf agent test run-eval --spec tests/my-agent-testSpec.yaml --api-name My_Agent --target-org my-org
+
+  Run tests using a JSON payload:
+
+    $ sf agent test run-eval --spec tests/eval-payload.json --target-org my-org
+
+  Run tests and output results in JUnit format; useful for continuous integration and deployment (CI/CD):
+
+    $ sf agent test run-eval --spec tests/my-agent-testSpec.yaml --target-org my-org --result-format junit
+
+  Run tests with contextVariables to inject contextual data into agent sessions (add contextVariables to test cases in
+  your YAML spec):
+
+    $ sf agent test run-eval --spec tests/agent-with-context.yaml --target-org my-org
+
+  Pipe JSON payload from stdin (--spec flag is automatically populated from stdin):
+  $ echo '{"tests":[...]}' | sf agent test run-eval --spec --target-org my-org
+```
+
+_See code: [src/commands/agent/test/run-eval.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/test/run-eval.ts)_
 
 ## `sf agent validate authoring-bundle`
 
@@ -1270,6 +1344,6 @@ EXAMPLES
     $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.36.1/src/commands/agent/validate/authoring-bundle.ts)_
+_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.37.0/src/commands/agent/validate/authoring-bundle.ts)_
 
 <!-- commandsstop -->

package/lib/commands/agent/test/run-eval.d.ts

@@ -1,5 +1,4 @@
 import { SfCommand } from '@salesforce/sf-plugins-core';
-import { Org } from '@salesforce/core';
 export type RunEvalResult = {
     tests: Array<{
         id: string;
@@ -18,16 +17,13 @@ export default class AgentTestRunEval extends SfCommand<RunEvalResult> {
     static readonly summary: string;
     static readonly description: string;
     static readonly examples: string[];
-    static state: string;
-    static readonly hidden = true;
     static readonly envVariablesSection: import("@oclif/core").HelpSection;
     static readonly errorCodes: import("@oclif/core").HelpSection;
     static readonly flags: {
-        'target-org': import("@oclif/core/interfaces").OptionFlag<Org, import("@oclif/core/interfaces").CustomOptions>;
+        'target-org': import("@oclif/core/interfaces").OptionFlag<import("@salesforce/core").Org, import("@oclif/core/interfaces").CustomOptions>;
         'api-version': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         spec: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
         'api-name': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        wait: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
         'result-format': import("@oclif/core/interfaces").OptionFlag<"json" | "human" | "junit" | "tap", import("@oclif/core/interfaces").CustomOptions>;
         'batch-size': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
         'no-normalize': import("@oclif/core/interfaces").BooleanFlag<boolean>;

package/lib/commands/agent/test/run-eval.js

@@ -16,100 +16,35 @@
 import { readFile } from 'node:fs/promises';
 import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core';
 import { EnvironmentVariable, Messages, SfError } from '@salesforce/core';
-import { normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js';
-import { formatResults } from '../../../evalFormatter.js';
+import { normalizePayload, splitIntoBatches, formatResults, isYamlTestSpec, parseTestSpec, translateTestSpec, resolveAgent, executeBatches, buildResultSummary, } from '@salesforce/agents';
 import { resultFormatFlag } from '../../../flags.js';
-import { isYamlTestSpec, parseTestSpec, translateTestSpec } from '../../../yamlSpecTranslator.js';
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
 const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run-eval');
-async function getApiHeaders(org) {
-    const conn = org.getConnection();
-    const userInfo = await conn.request(`${conn.instanceUrl}/services/oauth2/userinfo`);
-    return {
-        orgId: org.getOrgId(),
-        userId: userInfo.user_id,
-        instanceUrl: conn.instanceUrl,
-    };
-}
-async function callEvalApi(org, payload, headers) {
-    const conn = org.getConnection();
-    return conn.request({
-        url: 'https://api.salesforce.com/einstein/evaluation/v1/tests',
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json',
-            'x-sfdc-core-tenant-id': `core/prod/${headers.orgId}`,
-            'x-org-id': headers.orgId,
-            'x-sfdc-core-instance-url': headers.instanceUrl,
-            'x-sfdc-user-id': headers.userId,
-            'x-client-feature-id': 'AIPlatformEvaluation',
-            'x-sfdc-app-context': 'EinsteinGPT',
-        },
-        body: JSON.stringify(payload),
-    });
-}
-async function resolveAgent(org, apiName) {
-    const conn = org.getConnection();
-    // Escape single quotes to prevent SOQL injection
-    const escapedApiName = apiName.replace(/'/g, "\\'");
-    const botResult = await conn.query(`SELECT Id FROM BotDefinition WHERE DeveloperName = '${escapedApiName}'`);
-    if (!botResult.records.length) {
-        throw messages.createError('error.agentNotFound', [apiName]);
+async function resolveAndInjectAgent(org, agentApiName, payload) {
+    let agentId;
+    let versionId;
+    try {
+        ({ agentId, versionId } = await resolveAgent(org, agentApiName));
     }
-    const agentId = botResult.records[0].Id;
-    // Filter to published/active versions only
-    const versionResult = await conn.query(`SELECT Id FROM BotVersion WHERE BotDefinitionId = '${agentId}'  ORDER BY VersionNumber DESC LIMIT 1`);
-    if (!versionResult.records.length) {
-        throw messages.createError('error.agentVersionNotFound', [apiName]);
+    catch (e) {
+        const wrapped = SfError.wrap(e);
+        throw new SfError(`Agent '${agentApiName}' not found.`, 'AgentNotFound', [], 2, wrapped);
     }
-    const versionId = versionResult.records[0].Id;
-    return { agentId, versionId };
-}
-async function executeBatches(org, batches, log) {
-    // Pre-calculate headers once to avoid redundant API calls
-    const headers = await getApiHeaders(org);
-    // Execute all batches in parallel for better performance
-    if (batches.length > 1) {
-        log(messages.getMessage('info.batchProgress', [batches.length, batches.length, 'total']));
-    }
-    const batchPromises = batches.map(async (batch) => {
-        const batchPayload = { tests: batch };
-        const resultObj = await callEvalApi(org, batchPayload, headers);
-        return resultObj.results ?? [];
-    });
-    const batchResults = await Promise.all(batchPromises);
-    return batchResults.flat();
-}
-function buildResultSummary(mergedResponse) {
-    const summary = { passed: 0, failed: 0, scored: 0, errors: 0 };
-    const testSummaries = [];
-    for (const testResult of mergedResponse.results ?? []) {
-        const tr = testResult;
-        const testId = tr.id ?? 'unknown';
-        const evalResults = tr.evaluation_results ?? [];
-        const testErrors = tr.errors ?? [];
-        const passed = evalResults.filter((e) => e.is_pass === true).length;
-        const failed = evalResults.filter((e) => e.is_pass === false).length;
-        const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length;
-        summary.passed += passed;
-        summary.failed += failed;
-        summary.scored += scored;
-        summary.errors += testErrors.length;
-        testSummaries.push({
-            id: testId,
-            status: failed > 0 || testErrors.length > 0 ? 'failed' : 'passed',
-            evaluations: evalResults,
-            outputs: tr.outputs ?? [],
-        });
+    for (const test of payload.tests) {
+        for (const step of test.steps) {
+            if (step.type === 'agent.create_session') {
+                // eslint-disable-next-line camelcase
+                step.agent_id = agentId;
+                // eslint-disable-next-line camelcase
+                step.agent_version_id = versionId;
+            }
+        }
     }
-    return { summary, testSummaries };
 }
 export default class AgentTestRunEval extends SfCommand {
     static summary = messages.getMessage('summary');
     static description = messages.getMessage('description');
     static examples = messages.getMessages('examples');
-    static state = 'beta';
-    static hidden = true;
     static envVariablesSection = toHelpSection('ENVIRONMENT VARIABLES', EnvironmentVariable.SF_TARGET_ORG);
     static errorCodes = toHelpSection('ERROR CODES', {
         'Succeeded (0)': 'Tests completed successfully. Test results (passed/failed) are in the JSON output.',
@@ -130,11 +65,6 @@ export default class AgentTestRunEval extends SfCommand {
             char: 'n',
             summary: messages.getMessage('flags.api-name.summary'),
         }),
-        wait: Flags.integer({
-            char: 'w',
-            default: 10,
-            summary: messages.getMessage('flags.wait.summary'),
-        }),
         'result-format': resultFormatFlag(),
         'batch-size': Flags.integer({
             default: 5,
@@ -150,40 +80,39 @@ export default class AgentTestRunEval extends SfCommand {
         const org = flags['target-org'];
         // 1. Get spec content (from file or stdin via allowStdin)
         let rawContent = flags.spec;
-        // If spec looks like it might be a file path (not parseable content), read the file
-        try {
-            // Try to detect if it's actual content vs a file path
-            // If it's a valid YAML/JSON, it's content; otherwise treat as file path
-            if (!isYamlTestSpec(rawContent)) {
-                JSON.parse(rawContent);
-            }
-            // If we got here, it's valid content
-        }
-        catch {
-            // Not valid content, must be a file path - read it
+        let isYaml = isYamlTestSpec(rawContent);
+        if (!isYaml) {
             try {
-                rawContent = await readFile(flags.spec, 'utf-8');
+                JSON.parse(rawContent);
             }
-            catch (e) {
-                const wrapped = SfError.wrap(e);
-                throw new SfError(`Spec file not found: ${flags.spec}`, 'SpecFileNotFound', [], 2, wrapped);
+            catch {
+                try {
+                    rawContent = await readFile(flags.spec, 'utf-8');
+                }
+                catch (e) {
+                    const wrapped = SfError.wrap(e);
+                    throw new SfError(`Spec file not found: ${flags.spec}`, 'SpecFileNotFound', [], 2, wrapped);
+                }
+                isYaml = isYamlTestSpec(rawContent);
             }
         }
         // 2. Detect format and parse
         let payload;
         let agentApiName = flags['api-name'];
-        if (isYamlTestSpec(rawContent)) {
-            // YAML TestSpec detected — translate to EvalPayload
-            const spec = parseTestSpec(rawContent);
-            payload = translateTestSpec(spec);
-            // Auto-infer api-name from subjectName if not explicitly provided
-            if (!agentApiName) {
-                agentApiName = spec.subjectName;
-                this.log(messages.getMessage('info.yamlDetected', [spec.subjectName, spec.testCases.length.toString()]));
+        if (isYaml) {
+            try {
+                const spec = parseTestSpec(rawContent);
+                payload = translateTestSpec(spec);
+                if (!agentApiName) {
+                    agentApiName = spec.subjectName;
+                    this.log(messages.getMessage('info.yamlDetected', [spec.subjectName, spec.testCases.length.toString()]));
+                }
+            }
+            catch (e) {
+                throw messages.createError('error.invalidPayload', [e.message]);
             }
         }
         else {
-            // JSON EvalPayload (original behavior)
             try {
                 payload = JSON.parse(rawContent);
             }
@@ -194,56 +123,32 @@ export default class AgentTestRunEval extends SfCommand {
         if (!payload.tests || !Array.isArray(payload.tests) || payload.tests.length === 0) {
             throw messages.createError('error.invalidPayload', ['missing or empty "tests" array']);
         }
+        for (const test of payload.tests) {
+            if (!Array.isArray(test.steps)) {
+                throw messages.createError('error.invalidPayload', [`test '${test.id}' has missing or invalid 'steps' array`]);
+            }
+        }
         // 3. If --api-name (or auto-inferred from YAML), resolve IDs and inject
         if (agentApiName) {
-            let agentId;
-            let versionId;
-            try {
-                const resolved = await resolveAgent(org, agentApiName);
-                agentId = resolved.agentId;
-                versionId = resolved.versionId;
-            }
-            catch (e) {
-                const wrapped = SfError.wrap(e);
-                throw new SfError(`Agent '${agentApiName}' not found.`, 'AgentNotFound', [], 2, wrapped);
-            }
-            for (const test of payload.tests) {
-                for (const step of test.steps) {
-                    if (step.type === 'agent.create_session') {
-                        // eslint-disable-next-line camelcase
-                        step.agent_id = agentId;
-                        // eslint-disable-next-line camelcase
-                        step.agent_version_id = versionId;
-                    }
-                }
-            }
+            await resolveAndInjectAgent(org, agentApiName, payload);
         }
         // 4. Normalize payload unless --no-normalize
         if (!flags['no-normalize']) {
             payload = normalizePayload(payload);
         }
-        // 5. Clamp batch size
+        // 5. Clamp batch size and split into batches
         const batchSize = Math.min(Math.max(flags['batch-size'], 1), 5);
-        // 6. Split into batches
         const batches = splitIntoBatches(payload.tests, batchSize);
-        // 7. Execute batches
-        let allResults;
-        try {
-            allResults = await executeBatches(org, batches, (msg) => this.log(msg));
-        }
-        catch (e) {
+        // 6. Execute batches
+        const allResults = await executeBatches(org, batches, (msg) => this.log(msg)).catch((e) => {
             const wrapped = SfError.wrap(e);
             throw new SfError(`Failed to execute tests: ${wrapped.message}`, 'TestExecutionFailed', [wrapped.message], 4, wrapped);
-        }
+        });
         const mergedResponse = { results: allResults };
-        // 9. Format output
-        const resultFormat = (flags['result-format'] ?? 'human');
-        const formatted = formatResults(mergedResponse, resultFormat);
-        this.log(formatted);
-        // 10. Build structured result for --json
+        // 7. Format output
+        this.log(formatResults(mergedResponse, (flags['result-format'] ?? 'human')));
+        // 8. Build structured result for --json
         const { summary, testSummaries } = buildResultSummary(mergedResponse);
-        // Set exit code to 1 only for execution errors (tests couldn't run)
-        // Test failures (assertions failed) are business logic and should not affect exit code
         if (summary.errors > 0) {
             process.exitCode = 1;
         }

package/lib/commands/agent/test/run-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-eval.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAO,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAoB,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAwB,aAAa,EAAqB,MAAM,2BAA2B,CAAC;AACnG,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAElG,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;AAe1F,KAAK,UAAU,aAAa,CAAC,GAAQ;IACnC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAsB,GAAG,IAAI,CAAC,WAAW,2BAA2B,CAAC,CAAC;IAEzG,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE;QACrB,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,OAAoB,EAAE,OAAmB;IAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,OAAO,IAAI,CAAC,OAAO,CAA0B;QAC3C,GAAG,EAAE,yDAAyD;QAC9D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,uBAAuB,EAAE,aAAa,OAAO,CAAC,KAAK,EAAE;YACrD,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,0BAA0B,EAAE,OAAO,CAAC,WAAW;YAC/C,gBAAgB,EAAE,OAAO,CAAC,MAAM;YAChC,qBAAqB,EAAE,sBAAsB;YAC7C,oBAAoB,EAAE,aAAa;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAAe;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,iDAAiD;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAEpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAChC,uDAAuD,cAAc,GAAG,CACzE,CAAC;IACF,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,2CAA2C;IAC3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CACpC,sDAAsD,OAAO,wCAAwC,CACtG,CAAC;IACF,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,QAAQ,CAAC,WAAW,CAAC,4BAA4B,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAQ,EACR,OAAoC,EACpC,GAA0B;IAE1B,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAEzC,yDAAyD;IACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,MAAM,YAAY,GAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACtD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,kBAAkB,CAAC,cAA+B;IAIzD,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,aAAa,GAAsF,EAAE,CAAC;IAE5G,KAAK,MAAM,UAAU,IAAI,cAAc,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,EAAE,GAAG,UAAqC,CAAC;QACjD,MAAM,MAAM,GAAI,EAAE,CAAC,EAAa,IAAI,SAAS,CAAC;QAC9C,MAAM,WAAW,GAAI,EAAE,CAAC,kBAAqD,IAAI,EAAE,CAAC;QACpF,MAAM,UAAU,GAAI,EAAE,CAAC,MAAoB,IAAI,EAAE,CAAC;QAElD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;QAEtF,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAEpC,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjE,WAAW,EAAE,WAAW;YACxB,OAAO,EAAG,EAAE,CAAC,OAAqB,IAAI,EAAE;SACzC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,SAAwB;IAC7D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,MAAM,CAAU,MAAM,GAAG,IAAI,CAAC;IAE9B,MAAM,CAAU,mBAAmB,GAAG,aAAa,CACxD,uBAAuB,EACvB,mBAAmB,CAAC,aAAa,CAClC,CAAC;IAEK,MAAM,CAAU,UAAU,GAAG,aAAa,CAAC,aAAa,EAAE;QAC/D,eAAe,EAAE,oFAAoF;QACrG,YAAY,EAAE,mEAAmE;QACjF,cAAc,EAAE,8DAA8D;QAC9E,qBAAqB,EAAE,uDAAuD;KAC/E,CAAC,CAAC;IAEI,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;QACpC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,UAAU,EAAE,IAAI;SACjB,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC;YACvB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SACvD,CAAC;QACF,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;YAClB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;SACnD,CAAC;QACF,eAAe,EAAE,gBAAgB,EAAE;QACnC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;SACzD,CAAC;QACF,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC;YAC5B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,4BAA4B,CAAC;SAC3D,CAAC;KACH,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QAEhC,0DAA0D;QAC1D,IAAI,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAE5B,oFAAoF;QACpF,IAAI,CAAC;YACH,sDAAsD;YACtD,wEAAwE;YACxE,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;YACD,qCAAqC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;YACnD,IAAI,CAAC;gBACH,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,OAAO,CAAC,wBAAwB,KAAK,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAoB,CAAC;QACzB,IAAI,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAErC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,oDAAoD;YACpD,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAElC,kEAAkE;YAClE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAgB,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClF,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,wEAAwE;QACxE,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,OAAO,CAAC;YACZ,IAAI,SAAS,CAAC;YACd,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;gBACvD,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAC3B,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;YACjC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,OAAO,CAAC,UAAU,YAAY,cAAc,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;YAC3F,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;wBACzC,qCAAqC;wBACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;wBACxB,qCAAqC;wBACrC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,sBAAsB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEhE,wBAAwB;QACxB,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3D,qBAAqB;QACrB,IAAI,UAAU,CAAC;QACf,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,IAAI,OAAO,CACf,4BAA4B,OAAO,CAAC,OAAO,EAAE,EAC7C,qBAAqB,EACrB,CAAC,OAAO,CAAC,OAAO,CAAC,EACjB,CAAC,EACD,OAAO,CACR,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAoB,EAAE,OAAO,EAAE,UAAwC,EAAE,CAAC;QAE9F,mBAAmB;QACnB,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,OAAO,CAAiB,CAAC;QACzE,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEpB,yCAAyC;QACzC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAEtE,oEAAoE;QACpE,uFAAuF;QACvF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC"}
+{"version":3,"file":"run-eval.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAEL,gBAAgB,EAChB,gBAAgB,EAEhB,aAAa,EAEb,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,kBAAkB,GACnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;AAO1F,KAAK,UAAU,qBAAqB,CAClC,GAAuC,EACvC,YAAoB,EACpB,OAAoB;IAEpB,IAAI,OAAe,CAAC;IACpB,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACH,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC;IACnE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,IAAI,OAAO,CAAC,UAAU,YAAY,cAAc,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAC3F,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBACzC,qCAAqC;gBACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,SAAwB;IAC7D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAE5D,MAAM,CAAU,mBAAmB,GAAG,aAAa,CACxD,uBAAuB,EACvB,mBAAmB,CAAC,aAAa,CAClC,CAAC;IAEK,MAAM,CAAU,UAAU,GAAG,aAAa,CAAC,aAAa,EAAE;QAC/D,eAAe,EAAE,oFAAoF;QACrG,YAAY,EAAE,mEAAmE;QACjF,cAAc,EAAE,8DAA8D;QAC9E,qBAAqB,EAAE,uDAAuD;KAC/E,CAAC,CAAC;IAEI,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;QACpC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,UAAU,EAAE,IAAI;SACjB,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC;YACvB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SACvD,CAAC;QACF,eAAe,EAAE,gBAAgB,EAAE;QACnC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;SACzD,CAAC;QACF,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC;YAC5B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,4BAA4B,CAAC;SAC3D,CAAC;KACH,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QAEhC,0DAA0D;QAC1D,IAAI,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAC5B,IAAI,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;QAExC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBACH,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,IAAI,OAAO,CAAC,wBAAwB,KAAK,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC9F,CAAC;gBACD,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAqB,CAAC;QAC1B,IAAI,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAErC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;gBACvC,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAElC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;oBAChC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC3G,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAgB,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClF,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC,SAAS,IAAI,CAAC,EAAE,wCAAwC,CAAC,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;QAED,wEAAwE;QACxE,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,qBAAqB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAC1D,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,6CAA6C;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3D,qBAAqB;QACrB,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACxF,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,IAAI,OAAO,CACf,4BAA4B,OAAO,CAAC,OAAO,EAAE,EAC7C,qBAAqB,EACrB,CAAC,OAAO,CAAC,OAAO,CAAC,EACjB,CAAC,EACD,OAAO,CACR,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,cAAc,GAAoB,EAAE,OAAO,EAAE,UAAwC,EAAE,CAAC;QAE9F,mBAAmB;QACnB,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,OAAO,CAAiB,CAAC,CAAC,CAAC;QAE7F,wCAAwC;QACxC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAEtE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC"}

package/messages/agent.test.run-eval.md

@@ -20,10 +20,6 @@ Path to test spec file (YAML or JSON). Supports reading from stdin when piping c
 
 Agent DeveloperName (also called API name) to resolve agent_id and agent_version_id. Auto-inferred from the YAML spec's subjectName.
 
-# flags.wait.summary
-
-Number of minutes to wait for results.
-
 # flags.result-format.summary
 
 Format of the agent test results.