@salesforce/plugin-agent 1.30.11 → 1.31.0

package/README.md

@@ -116,7 +116,7 @@ EXAMPLES
     $ sf agent activate --api-name Resort_Manager --target-org my-org
 ```
 
-_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/activate.ts)_
+_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/activate.ts)_
 
 ## `sf agent create`
 
@@ -183,7 +183,7 @@ EXAMPLES
     $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview
 ```
 
-_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/create.ts)_
+_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/create.ts)_
 
 ## `sf agent deactivate`
 
@@ -223,7 +223,7 @@ EXAMPLES
     $ sf agent deactivate --api-name Resort_Manager --target-org my-org
 ```
 
-_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/deactivate.ts)_
+_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/deactivate.ts)_
 
 ## `sf agent generate agent-spec`
 
@@ -330,7 +330,7 @@ EXAMPLES
     $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com
 ```
 
-_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/agent-spec.ts)_
+_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/generate/agent-spec.ts)_
 
 ## `sf agent generate authoring-bundle`
 
@@ -407,7 +407,7 @@ EXAMPLES
       other-package-dir/main/default --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/authoring-bundle.ts)_
+_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/generate/authoring-bundle.ts)_
 
 ## `sf agent generate template`
 
@@ -455,7 +455,7 @@ EXAMPLES
       force-app/main/default/bots/My_Awesome_Agent/My_Awesome_Agent.bot-meta.xml --agent-version 1
 ```
 
-_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/template.ts)_
+_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/generate/template.ts)_
 
 ## `sf agent generate test-spec`
 
@@ -516,7 +516,7 @@ EXAMPLES
       force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml
 ```
 
-_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/test-spec.ts)_
+_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/generate/test-spec.ts)_
 
 ## `sf agent preview`
 
@@ -589,7 +589,7 @@ EXAMPLES
     $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview
 ```
 
-_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview.ts)_
+_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/preview.ts)_
 
 ## `sf agent preview end`
 
@@ -644,7 +644,7 @@ EXAMPLES
     $ sf agent preview end --authoring-bundle My_Local_Agent
 ```
 
-_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/end.ts)_
+_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/preview/end.ts)_
 
 ## `sf agent preview send`
 
@@ -702,7 +702,7 @@ EXAMPLES
     $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent
 ```
 
-_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/send.ts)_
+_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/preview/send.ts)_
 
 ## `sf agent preview sessions`
 
@@ -735,7 +735,7 @@ EXAMPLES
     $ sf agent preview sessions
 ```
 
-_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/sessions.ts)_
+_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/preview/sessions.ts)_
 
 ## `sf agent preview start`
 
@@ -792,7 +792,7 @@ EXAMPLES
     $ sf agent preview start --api-name My_Published_Agent
 ```
 
-_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/start.ts)_
+_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/preview/start.ts)_
 
 ## `sf agent publish authoring-bundle`
 
@@ -841,7 +841,7 @@ EXAMPLES
     $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/publish/authoring-bundle.ts)_
+_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/publish/authoring-bundle.ts)_
 
 ## `sf agent test create`
 
@@ -896,7 +896,7 @@ EXAMPLES
     $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview
 ```
 
-_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/create.ts)_
+_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/test/create.ts)_
 
 ## `sf agent test list`
 
@@ -931,7 +931,7 @@ EXAMPLES
     $ sf agent test list --target-org my-org
 ```
 
-_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/list.ts)_
+_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/test/list.ts)_
 
 ## `sf agent test results`
 
@@ -997,7 +997,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/results.ts)_
+_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/test/results.ts)_
 
 ## `sf agent test resume`
 
@@ -1070,7 +1070,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/resume.ts)_
+_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/test/resume.ts)_
 
 ## `sf agent test run`
 
@@ -1144,7 +1144,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/run.ts)_
+_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/test/run.ts)_
 
 ## `sf agent validate authoring-bundle`
 
@@ -1191,6 +1191,6 @@ EXAMPLES
     $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/validate/authoring-bundle.ts)_
+_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.0/src/commands/agent/validate/authoring-bundle.ts)_
 
 <!-- commandsstop -->

package/lib/commands/agent/test/run-eval.d.ts

@@ -0,0 +1,33 @@
+import { SfCommand } from '@salesforce/sf-plugins-core';
+import { Org } from '@salesforce/core';
+export type RunEvalResult = {
+    tests: Array<{
+        id: string;
+        status: string;
+        evaluations: unknown[];
+    }>;
+    summary: {
+        passed: number;
+        failed: number;
+        scored: number;
+        errors: number;
+    };
+};
+export default class AgentTestRunEval extends SfCommand<RunEvalResult> {
+    static readonly summary: string;
+    static readonly description: string;
+    static readonly examples: string[];
+    static state: string;
+    static readonly hidden = true;
+    static readonly flags: {
+        'target-org': import("@oclif/core/interfaces").OptionFlag<Org, import("@oclif/core/interfaces").CustomOptions>;
+        'api-version': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        spec: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'api-name': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        wait: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        'result-format': import("@oclif/core/interfaces").OptionFlag<"json" | "human" | "junit" | "tap", import("@oclif/core/interfaces").CustomOptions>;
+        'batch-size': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        'no-normalize': import("@oclif/core/interfaces").BooleanFlag<boolean>;
+    };
+    run(): Promise<RunEvalResult>;
+}

package/lib/commands/agent/test/run-eval.js

@@ -0,0 +1,221 @@
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { readFile } from 'node:fs/promises';
+import { Flags, SfCommand } from '@salesforce/sf-plugins-core';
+import { Messages } from '@salesforce/core';
+import { normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js';
+import { formatResults } from '../../../evalFormatter.js';
+import { resultFormatFlag } from '../../../flags.js';
+import { isYamlTestSpec, parseTestSpec, translateTestSpec } from '../../../yamlSpecTranslator.js';
+Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
+const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run-eval');
+async function getApiHeaders(org) {
+    const conn = org.getConnection();
+    const userInfo = await conn.request(`${conn.instanceUrl}/services/oauth2/userinfo`);
+    return {
+        orgId: org.getOrgId(),
+        userId: userInfo.user_id,
+        instanceUrl: conn.instanceUrl,
+    };
+}
+async function callEvalApi(org, payload, headers) {
+    const conn = org.getConnection();
+    return conn.request({
+        url: 'https://api.salesforce.com/einstein/evaluation/v1/tests',
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-sfdc-core-tenant-id': `core/prod/${headers.orgId}`,
+            'x-org-id': headers.orgId,
+            'x-sfdc-core-instance-url': headers.instanceUrl,
+            'x-sfdc-user-id': headers.userId,
+            'x-client-feature-id': 'AIPlatformEvaluation',
+            'x-sfdc-app-context': 'EinsteinGPT',
+        },
+        body: JSON.stringify(payload),
+    });
+}
+async function resolveAgent(org, apiName) {
+    const conn = org.getConnection();
+    // Escape single quotes to prevent SOQL injection
+    const escapedApiName = apiName.replace(/'/g, "\\'");
+    const botResult = await conn.query(`SELECT Id FROM BotDefinition WHERE DeveloperName = '${escapedApiName}'`);
+    if (!botResult.records.length) {
+        throw messages.createError('error.agentNotFound', [apiName]);
+    }
+    const agentId = botResult.records[0].Id;
+    // Filter to published/active versions only
+    const versionResult = await conn.query(`SELECT Id FROM BotVersion WHERE BotDefinitionId = '${agentId}'  ORDER BY VersionNumber DESC LIMIT 1`);
+    if (!versionResult.records.length) {
+        throw messages.createError('error.agentVersionNotFound', [apiName]);
+    }
+    const versionId = versionResult.records[0].Id;
+    return { agentId, versionId };
+}
+async function executeBatches(org, batches, log) {
+    // Pre-calculate headers once to avoid redundant API calls
+    const headers = await getApiHeaders(org);
+    // Execute all batches in parallel for better performance
+    if (batches.length > 1) {
+        log(messages.getMessage('info.batchProgress', [batches.length, batches.length, 'total']));
+    }
+    const batchPromises = batches.map(async (batch) => {
+        const batchPayload = { tests: batch };
+        const resultObj = await callEvalApi(org, batchPayload, headers);
+        return resultObj.results ?? [];
+    });
+    const batchResults = await Promise.all(batchPromises);
+    return batchResults.flat();
+}
+function buildResultSummary(mergedResponse) {
+    const summary = { passed: 0, failed: 0, scored: 0, errors: 0 };
+    const testSummaries = [];
+    for (const testResult of mergedResponse.results ?? []) {
+        const tr = testResult;
+        const testId = tr.id ?? 'unknown';
+        const evalResults = tr.evaluation_results ?? [];
+        const testErrors = tr.errors ?? [];
+        const passed = evalResults.filter((e) => e.is_pass === true).length;
+        const failed = evalResults.filter((e) => e.is_pass === false).length;
+        const scored = evalResults.filter((e) => e.score != null && e.is_pass == null).length;
+        summary.passed += passed;
+        summary.failed += failed;
+        summary.scored += scored;
+        summary.errors += testErrors.length;
+        testSummaries.push({
+            id: testId,
+            status: failed > 0 || testErrors.length > 0 ? 'failed' : 'passed',
+            evaluations: evalResults,
+        });
+    }
+    return { summary, testSummaries };
+}
+export default class AgentTestRunEval extends SfCommand {
+    static summary = messages.getMessage('summary');
+    static description = messages.getMessage('description');
+    static examples = messages.getMessages('examples');
+    static state = 'beta';
+    static hidden = true;
+    static flags = {
+        'target-org': Flags.requiredOrg(),
+        'api-version': Flags.orgApiVersion(),
+        spec: Flags.string({
+            char: 's',
+            required: true,
+            summary: messages.getMessage('flags.spec.summary'),
+            allowStdin: true,
+        }),
+        'api-name': Flags.string({
+            char: 'n',
+            summary: messages.getMessage('flags.api-name.summary'),
+        }),
+        wait: Flags.integer({
+            char: 'w',
+            default: 10,
+            summary: messages.getMessage('flags.wait.summary'),
+        }),
+        'result-format': resultFormatFlag(),
+        'batch-size': Flags.integer({
+            default: 5,
+            summary: messages.getMessage('flags.batch-size.summary'),
+        }),
+        'no-normalize': Flags.boolean({
+            default: false,
+            summary: messages.getMessage('flags.no-normalize.summary'),
+        }),
+    };
+    async run() {
+        const { flags } = await this.parse(AgentTestRunEval);
+        const org = flags['target-org'];
+        // 1. Get spec content (from file or stdin via allowStdin)
+        let rawContent = flags.spec;
+        // If spec looks like it might be a file path (not parseable content), read the file
+        try {
+            // Try to detect if it's actual content vs a file path
+            // If it's a valid YAML/JSON, it's content; otherwise treat as file path
+            if (!isYamlTestSpec(rawContent)) {
+                JSON.parse(rawContent);
+            }
+            // If we got here, it's valid content
+        }
+        catch {
+            // Not valid content, must be a file path - read it
+            rawContent = await readFile(flags.spec, 'utf-8');
+        }
+        // 2. Detect format and parse
+        let payload;
+        let agentApiName = flags['api-name'];
+        if (isYamlTestSpec(rawContent)) {
+            // YAML TestSpec detected — translate to EvalPayload
+            const spec = parseTestSpec(rawContent);
+            payload = translateTestSpec(spec);
+            // Auto-infer api-name from subjectName if not explicitly provided
+            if (!agentApiName) {
+                agentApiName = spec.subjectName;
+                this.log(messages.getMessage('info.yamlDetected', [spec.subjectName, spec.testCases.length.toString()]));
+            }
+        }
+        else {
+            // JSON EvalPayload (original behavior)
+            try {
+                payload = JSON.parse(rawContent);
+            }
+            catch (e) {
+                throw messages.createError('error.invalidPayload', [e.message]);
+            }
+        }
+        if (!payload.tests || !Array.isArray(payload.tests) || payload.tests.length === 0) {
+            throw messages.createError('error.invalidPayload', ['missing or empty "tests" array']);
+        }
+        // 3. If --api-name (or auto-inferred from YAML), resolve IDs and inject
+        if (agentApiName) {
+            const { agentId, versionId } = await resolveAgent(org, agentApiName);
+            for (const test of payload.tests) {
+                for (const step of test.steps) {
+                    if (step.type === 'agent.create_session') {
+                        // eslint-disable-next-line camelcase
+                        step.agent_id = agentId;
+                        // eslint-disable-next-line camelcase
+                        step.agent_version_id = versionId;
+                    }
+                }
+            }
+        }
+        // 4. Normalize payload unless --no-normalize
+        if (!flags['no-normalize']) {
+            payload = normalizePayload(payload);
+        }
+        // 5. Clamp batch size
+        const batchSize = Math.min(Math.max(flags['batch-size'], 1), 5);
+        // 6. Split into batches
+        const batches = splitIntoBatches(payload.tests, batchSize);
+        // 7. Execute batches
+        const allResults = await executeBatches(org, batches, (msg) => this.log(msg));
+        const mergedResponse = { results: allResults };
+        // 9. Format output
+        const resultFormat = (flags['result-format'] ?? 'human');
+        const formatted = formatResults(mergedResponse, resultFormat);
+        this.log(formatted);
+        // 10. Build structured result for --json
+        const { summary, testSummaries } = buildResultSummary(mergedResponse);
+        // Set exit code to 1 if any tests failed
+        if (summary.failed > 0 || summary.errors > 0) {
+            process.exitCode = 1;
+        }
+        return { tests: testSummaries, summary };
+    }
+}
+//# sourceMappingURL=run-eval.js.map

package/lib/commands/agent/test/run-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAO,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAoB,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAwB,aAAa,EAAqB,MAAM,2BAA2B,CAAC;AACnG,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAElG,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;AAe1F,KAAK,UAAU,aAAa,CAAC,GAAQ;IACnC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAsB,GAAG,IAAI,CAAC,WAAW,2BAA2B,CAAC,CAAC;IAEzG,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE;QACrB,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,OAAoB,EAAE,OAAmB;IAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,OAAO,IAAI,CAAC,OAAO,CAA0B;QAC3C,GAAG,EAAE,yDAAyD;QAC9D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,uBAAuB,EAAE,aAAa,OAAO,CAAC,KAAK,EAAE;YACrD,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,0BAA0B,EAAE,OAAO,CAAC,WAAW;YAC/C,gBAAgB,EAAE,OAAO,CAAC,MAAM;YAChC,qBAAqB,EAAE,sBAAsB;YAC7C,oBAAoB,EAAE,aAAa;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAAe;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,iDAAiD;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAEpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAChC,uDAAuD,cAAc,GAAG,CACzE,CAAC;IACF,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,2CAA2C;IAC3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CACpC,sDAAsD,OAAO,wCAAwC,CACtG,CAAC;IACF,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,QAAQ,CAAC,WAAW,CAAC,4BAA4B,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAQ,EACR,OAAoC,EACpC,GAA0B;IAE1B,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAEzC,yDAAyD;IACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,MAAM,YAAY,GAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACtD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,kBAAkB,CAAC,cAA+B;IAIzD,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,aAAa,GAAkE,EAAE,CAAC;IAExF,KAAK,MAAM,UAAU,IAAI,cAAc,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,EAAE,GAAG,UAAqC,CAAC;QACjD,MAAM,MAAM,GAAI,EAAE,CAAC,EAAa,IAAI,SAAS,CAAC;QAC9C,MAAM,WAAW,GAAI,EAAE,CAAC,kBAAqD,IAAI,EAAE,CAAC;QACpF,MAAM,UAAU,GAAI,EAAE,CAAC,MAAoB,IAAI,EAAE,CAAC;QAElD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;QAEtF,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAEpC,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjE,WAAW,EAAE,WAAW;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,SAAwB;IAC7D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,MAAM,CAAU,MAAM,GAAG,IAAI,CAAC;IAE9B,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;QACpC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,UAAU,EAAE,IAAI;SACjB,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC;YACvB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SACvD,CAAC;QACF,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;YAClB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;SACnD,CAAC;QACF,eAAe,EAAE,gBAAgB,EAAE;QACnC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;SACzD,CAAC;QACF,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC;YAC5B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,4BAA4B,CAAC;SAC3D,CAAC;KACH,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QAEhC,0DAA0D;QAC1D,IAAI,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAE5B,oFAAoF;QACpF,IAAI,CAAC;YACH,sDAAsD;YACtD,wEAAwE;YACxE,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;YACD,qCAAqC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;YACnD,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAoB,CAAC;QACzB,IAAI,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAErC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,oDAAoD;YACpD,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAElC,kEAAkE;YAClE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAgB,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClF,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,wEAAwE;QACxE,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACrE,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;wBACzC,qCAAqC;wBACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;wBACxB,qCAAqC;wBACrC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,sBAAsB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEhE,wBAAwB;QACxB,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3D,qBAAqB;QACrB,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAE9E,MAAM,cAAc,GAAoB,EAAE,OAAO,EAAE,UAAwC,EAAE,CAAC;QAE9F,mBAAmB;QACnB,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,OAAO,CAAiB,CAAC;QACzE,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEpB,yCAAyC;QACzC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAEtE,yCAAyC;QACzC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC"}

package/lib/evalFormatter.d.ts

@@ -0,0 +1,30 @@
+export type ResultFormat = 'human' | 'json' | 'junit' | 'tap';
+type EvalOutput = {
+    type?: string;
+    id?: string;
+    session_id?: string;
+    response?: unknown;
+};
+type EvalResult = {
+    id?: string;
+    score?: number | null;
+    is_pass?: boolean | null;
+    actual_value?: string;
+    expected_value?: string;
+    error_message?: string;
+};
+type TestError = {
+    id?: string;
+    error_message?: string;
+};
+type TestResult = {
+    id?: string;
+    outputs?: EvalOutput[];
+    evaluation_results?: EvalResult[];
+    errors?: TestError[];
+};
+export type EvalApiResponse = {
+    results?: TestResult[];
+};
+export declare function formatResults(results: EvalApiResponse, format: ResultFormat): string;
+export {};