@salesforce/plugin-agent 1.3.4 → 1.5.0

package/lib/commands/agent/test/run.d.ts

@@ -1,19 +1,19 @@
 import { SfCommand } from '@salesforce/sf-plugins-core';
 export type AgentTestRunResult = {
-    jobId: string;
-    success: boolean;
-    errorCode?: string;
-    message?: string;
+    aiEvaluationId: string;
+    status: string;
 };
 export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
     static readonly summary: string;
     static readonly description: string;
     static readonly examples: string[];
-    static state: string;
+    static readonly state = "beta";
     static readonly flags: {
         'target-org': import("@oclif/core/interfaces").OptionFlag<import("@salesforce/core").Org, import("@oclif/core/interfaces").CustomOptions>;
-        id: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'api-version': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        name: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
         wait: import("@oclif/core/interfaces").OptionFlag<import("@salesforce/kit").Duration, import("@oclif/core/interfaces").CustomOptions>;
+        'result-format': import("@oclif/core/interfaces").OptionFlag<"json" | "human" | "junit", import("@oclif/core/interfaces").CustomOptions>;
         'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
     };
     run(): Promise<AgentTestRunResult>;

package/lib/commands/agent/test/run.js

@@ -6,6 +6,12 @@
  */
 import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
 import { Messages } from '@salesforce/core';
+import { AgentTester } from '@salesforce/agents';
+import { colorize } from '@oclif/core/ux';
+import { resultFormatFlag, testOutputDirFlag } from '../../../flags.js';
+import { AgentTestCache } from '../../../agentTestCache.js';
+import { TestStages } from '../../../testStages.js';
+import { handleTestResults } from '../../../handleTestResults.js';
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
 const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run');
 export default class AgentTestRun extends SfCommand {
@@ -15,13 +21,15 @@ export default class AgentTestRun extends SfCommand {
     static state = 'beta';
     static flags = {
         'target-org': Flags.requiredOrg(),
-        // AiEvalDefinitionVersion.Id -- This should really be "test-name"
-        id: Flags.string({
-            char: 'i',
+        'api-version': Flags.orgApiVersion(),
+        name: Flags.string({
+            char: 'n',
             required: true,
-            summary: messages.getMessage('flags.id.summary'),
-            description: messages.getMessage('flags.id.description'),
+            summary: messages.getMessage('flags.name.summary'),
+            description: messages.getMessage('flags.name.description'),
         }),
+        // we want to pass `undefined` to the API
+        // eslint-disable-next-line sf-plugin/flag-min-max-default
         wait: Flags.duration({
             char: 'w',
             unit: 'minutes',
@@ -29,27 +37,40 @@ export default class AgentTestRun extends SfCommand {
             summary: messages.getMessage('flags.wait.summary'),
             description: messages.getMessage('flags.wait.description'),
         }),
-        'output-dir': Flags.directory({
-            char: 'd',
-            summary: messages.getMessage('flags.output-dir.summary'),
-        }),
-        //
-        // Future flags:
-        //   result-format [csv, json, table, junit, TAP]
-        //   suites [array of suite names]
-        //   verbose [boolean]
-        //   ??? api-version or build-version ???
+        'result-format': resultFormatFlag(),
+        'output-dir': testOutputDirFlag(),
     };
     async run() {
         const { flags } = await this.parse(AgentTestRun);
-        this.log(`Starting tests for AiEvalDefinitionVersion: ${flags.id}`);
-        // Call SF Eval Connect API passing AiEvalDefinitionVersion.Id
-        // POST to /einstein/ai-evaluations/{aiEvalDefinitionVersionId}/start
-        // Returns: AiEvaluation.Id
-        return {
-            success: true,
-            jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id; needed for getting status and stopping
-        };
+        const mso = new TestStages({ title: `Agent Test Run: ${flags.name}`, jsonEnabled: this.jsonEnabled() });
+        mso.start();
+        const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version']));
+        const response = await agentTester.start(flags.name);
+        mso.update({ id: response.aiEvaluationId });
+        const agentTestCache = await AgentTestCache.create();
+        await agentTestCache.createCacheEntry(response.aiEvaluationId, flags.name);
+        if (flags.wait?.minutes) {
+            const { completed, response: detailsResponse } = await mso.poll(agentTester, response.aiEvaluationId, flags.wait);
+            if (completed)
+                await agentTestCache.removeCacheEntry(response.aiEvaluationId);
+            mso.stop();
+            await handleTestResults({
+                id: response.aiEvaluationId,
+                format: flags['result-format'],
+                results: detailsResponse,
+                jsonEnabled: this.jsonEnabled(),
+                outputDir: flags['output-dir'],
+            });
+            return {
+                status: 'COMPLETED',
+                aiEvaluationId: response.aiEvaluationId,
+            };
+        }
+        else {
+            mso.stop();
+            this.log(`Run ${colorize('dim', `sf agent test resume --job-id ${response.aiEvaluationId}`)} to resuming watching this test.`);
+        }
+        return response;
     }
 }
 //# sourceMappingURL=run.js.map

package/lib/commands/agent/test/run.js.map

@@ -1 +1 @@
-{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,gBAAgB,CAAC,CAAC;AASrF,MAAM,CAAC,OAAO,OAAO,YAAa,SAAQ,SAA6B;IAC9D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;IAEtB,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,kEAAkE;QAClE,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC;YACf,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;YAChD,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,sBAAsB,CAAC;SACzD,CAAC;QACF,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;YACnB,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,SAAS;YACf,GAAG,EAAE,CAAC;YACN,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SAC3D,CAAC;QACF,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC;YAC5B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;SACzD,CAAC;QACF,EAAE;QACF,gBAAgB;QAChB,iDAAiD;QACjD,kCAAkC;QAClC,sBAAsB;QACtB,yCAAyC;KAC1C,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEjD,IAAI,CAAC,GAAG,CAAC,+CAA+C,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QAEpE,8DAA8D;QAC9D,qEAAqE;QAErE,2BAA2B;QAE3B,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,oBAAoB,EAAE,0DAA0D;SACxF,CAAC;IACJ,CAAC"}
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAElE,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,gBAAgB,CAAC,CAAC;AAQrF,MAAM,CAAC,OAAO,OAAO,YAAa,SAAQ,SAA6B;IAC9D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,CAAU,KAAK,GAAG,MAAM,CAAC;IAE/B,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;QACpC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SAC3D,CAAC;QACF,yCAAyC;QACzC,0DAA0D;QAC1D,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;YACnB,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,SAAS;YACf,GAAG,EAAE,CAAC;YACN,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SAC3D,CAAC;QACF,eAAe,EAAE,gBAAgB,EAAE;QACnC,YAAY,EAAE,iBAAiB,EAAE;KAClC,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEjD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,EAAE,KAAK,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACxG,GAAG,CAAC,KAAK,EAAE,CAAC;QAEZ,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAErD,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;QAE5C,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,cAAc,CAAC,gBAAgB,CAAC,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3E,IAAI,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;YACxB,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAClH,IAAI,SAAS;gBAAE,MAAM,cAAc,CAAC,gBAAgB,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;YAE9E,GAAG,CAAC,IAAI,EAAE,CAAC;YAEX,MAAM,iBAAiB,CAAC;gBACtB,EAAE,EAAE,QAAQ,CAAC,cAAc;gBAC3B,MAAM,EAAE,KAAK,CAAC,eAAe,CAAC;gBAC9B,OAAO,EAAE,eAAe;gBACxB,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE;gBAC/B,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC;aAC/B,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE,WAAW;gBACnB,cAAc,EAAE,QAAQ,CAAC,cAAc;aACxC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CACN,OAAO,QAAQ,CACb,KAAK,EACL,iCAAiC,QAAQ,CAAC,cAAc,EAAE,CAC3D,kCAAkC,CACpC,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC"}

package/lib/components/agent-preview-react.d.ts

@@ -0,0 +1,9 @@
+import React from 'react';
+/**
+ * Ideas:
+ * - Limit height based on terminal height
+ * - Add keystroke to clear chat
+ * - Add keystroke to scroll up
+ * - Add keystroke to scroll down
+ */
+export declare function AgentPreviewReact(): React.ReactNode;

package/lib/components/agent-preview-react.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2024, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import React from 'react';
+import { Box, Text } from 'ink';
+import figures from '@inquirer/figures';
+import TextInput from 'ink-text-input';
+import { sleep } from '@salesforce/kit';
+/**
+ * Ideas:
+ * - Limit height based on terminal height
+ * - Add keystroke to clear chat
+ * - Add keystroke to scroll up
+ * - Add keystroke to scroll down
+ */
+export function AgentPreviewReact() {
+    const [comments, setComments] = React.useState([]);
+    const [query, setQuery] = React.useState('');
+    return (React.createElement(Box, { flexDirection: "column" },
+        comments.length > 0 && (React.createElement(Box, { flexDirection: "column" },
+            comments.map(({ timestamp, role, content }, idx) => (React.createElement(Box, { key: role + '__' + timestamp.toISOString() + '__' + idx.toString(), alignItems: role === 'user' ? 'flex-end' : 'flex-start', flexDirection: "column" },
+                React.createElement(Box, { flexDirection: "row", columnGap: 1 },
+                    React.createElement(Text, null, role === 'user' ? 'You' : role),
+                    React.createElement(Text, { color: "gray" }, timestamp.toLocaleString())),
+                React.createElement(Box, { width: Math.min(process.stdout.columns - 4, content.length + 4), borderStyle: "round", paddingLeft: 1, paddingRight: 1 },
+                    React.createElement(Text, null, content))))),
+            React.createElement(Box, { paddingLeft: 1, paddingRight: 1 },
+                React.createElement(Text, { dimColor: true }, '─'.repeat(process.stdout.columns - 2))))),
+        React.createElement(Box, null,
+            React.createElement(Text, null,
+                figures.pointer,
+                " "),
+            React.createElement(TextInput, { showCursor: true, value: query, placeholder: "Start typing\u2026", onChange: setQuery, 
+                // eslint-disable-next-line @typescript-eslint/no-misused-promises
+                onSubmit: async (content) => {
+                    if (!content)
+                        return;
+                    setQuery('');
+                    setComments((prev) => [...prev, { role: 'user', content, timestamp: new Date() }]);
+                    await sleep(1000);
+                    setComments((prev) => {
+                        const lastComment = prev[prev.length - 1];
+                        // TODO - use agents library for generations
+                        return (Math.random() * 2) % 2 > 1
+                            ? [
+                                ...prev,
+                                { role: 'system', content: "I'm sorry, I can't help with this request", timestamp: new Date() },
+                            ]
+                            : [...prev, { ...lastComment, role: 'system', content: 'We have a scuba class at 9:30 AM' }];
+                    });
+                } }))));
+}
+//# sourceMappingURL=agent-preview-react.js.map

package/lib/components/agent-preview-react.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-preview-react.js","sourceRoot":"","sources":["../../src/components/agent-preview-react.tsx"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,KAAK,CAAC;AAChC,OAAO,OAAO,MAAM,mBAAmB,CAAC;AACxC,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAExC;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,KAAK,CAAC,QAAQ,CAC5C,EAAE,CACH,CAAC;IACF,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC7C,OAAO,CACL,oBAAC,GAAG,IAAC,aAAa,EAAC,QAAQ;QACxB,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CACtB,oBAAC,GAAG,IAAC,aAAa,EAAC,QAAQ;YACxB,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CACnD,oBAAC,GAAG,IACF,GAAG,EAAE,IAAI,GAAG,IAAI,GAAG,SAAS,CAAC,WAAW,EAAE,GAAG,IAAI,GAAG,GAAG,CAAC,QAAQ,EAAE,EAClE,UAAU,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,EACvD,aAAa,EAAC,QAAQ;gBAEtB,oBAAC,GAAG,IAAC,aAAa,EAAC,KAAK,EAAC,SAAS,EAAE,CAAC;oBACnC,oBAAC,IAAI,QAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAQ;oBAC7C,oBAAC,IAAI,IAAC,KAAK,EAAC,MAAM,IAAE,SAAS,CAAC,cAAc,EAAE,CAAQ,CAClD;gBACN,oBAAC,GAAG,IACF,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAC/D,WAAW,EAAC,OAAO,EACnB,WAAW,EAAE,CAAC,EACd,YAAY,EAAE,CAAC;oBAEf,oBAAC,IAAI,QAAE,OAAO,CAAQ,CAClB,CACF,CACP,CAAC;YACF,oBAAC,GAAG,IAAC,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC;gBAClC,oBAAC,IAAI,IAAC,QAAQ,UAAE,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAQ,CAC1D,CACF,CACP;QAED,oBAAC,GAAG;YACF,oBAAC,IAAI;gBAAE,OAAO,CAAC,OAAO;oBAAS;YAC/B,oBAAC,SAAS,IACR,UAAU,QACV,KAAK,EAAE,KAAK,EACZ,WAAW,EAAC,oBAAe,EAC3B,QAAQ,EAAE,QAAQ;gBAClB,kEAAkE;gBAClE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBAC1B,IAAI,CAAC,OAAO;wBAAE,OAAO;oBACrB,QAAQ,CAAC,EAAE,CAAC,CAAC;oBAEb,WAAW,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;oBACnF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;oBAElB,WAAW,CAAC,CAAC,IAAI,EAAE,EAAE;wBACnB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC1C,4CAA4C;wBAC5C,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;4BAChC,CAAC,CAAC;gCACE,GAAG,IAAI;gCACP,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,2CAA2C,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE;6BAChG;4BACH,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,GAAG,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;oBACjG,CAAC,CAAC,CAAC;gBACL,CAAC,GACD,CACE,CACF,CACP,CAAC;AACJ,CAAC"}

package/lib/flags.d.ts

@@ -0,0 +1,8 @@
+export declare const resultFormatFlag: import("@oclif/core/interfaces").FlagDefinition<"json" | "human" | "junit", import("@oclif/core/interfaces").CustomOptions, {
+    multiple: false;
+    requiredOrDefaulted: true;
+}>;
+export declare const testOutputDirFlag: import("@oclif/core/interfaces").FlagDefinition<string, import("@oclif/core/interfaces").CustomOptions, {
+    multiple: false;
+    requiredOrDefaulted: false;
+}>;

package/lib/flags.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2024, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import { Flags } from '@salesforce/sf-plugins-core';
+import { Messages } from '@salesforce/core';
+Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
+const messages = Messages.loadMessages('@salesforce/plugin-agent', 'shared');
+export const resultFormatFlag = Flags.option({
+    options: [
+        'json',
+        'human',
+        'junit',
+        // 'tap',
+    ],
+    default: 'human',
+    summary: messages.getMessage('flags.result-format.summary'),
+});
+export const testOutputDirFlag = Flags.custom({
+    char: 'f',
+    description: messages.getMessage('flags.output-dir.description'),
+    summary: messages.getMessage('flags.output-dir.summary'),
+});
+//# sourceMappingURL=flags.js.map

package/lib/flags.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flags.js","sourceRoot":"","sources":["../src/flags.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,QAAQ,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3C,OAAO,EAAE;QACP,MAAM;QACN,OAAO;QACP,OAAO;QACP,SAAS;KACD;IACV,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,6BAA6B,CAAC;CAC5D,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAC,MAAM,CAAS;IACpD,IAAI,EAAE,GAAG;IACT,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,8BAA8B,CAAC;IAChE,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;CACzD,CAAC,CAAC"}

package/lib/handleTestResults.d.ts

@@ -0,0 +1,8 @@
+import { AgentTestDetailsResponse } from '@salesforce/agents';
+export declare function handleTestResults({ id, format, results, jsonEnabled, outputDir, }: {
+    id: string;
+    format: 'human' | 'json' | 'junit';
+    results: AgentTestDetailsResponse | undefined;
+    jsonEnabled: boolean;
+    outputDir?: string;
+}): Promise<void>;

package/lib/handleTestResults.js

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2024, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import { join } from 'node:path';
+import { writeFile, mkdir } from 'node:fs/promises';
+import { jsonFormat, humanFormat, junitFormat } from '@salesforce/agents';
+import { Ux } from '@salesforce/sf-plugins-core/Ux';
+async function writeFileToDir(outputDir, fileName, content) {
+    // if directory doesn't exist, create it
+    await mkdir(outputDir, { recursive: true });
+    await writeFile(join(outputDir, fileName), content);
+}
+export async function handleTestResults({ id, format, results, jsonEnabled, outputDir, }) {
+    if (!results) {
+        // do nothing since there are no results to handle
+        return;
+    }
+    const ux = new Ux({ jsonEnabled });
+    if (format === 'human') {
+        const formatted = await humanFormat(results);
+        ux.log(formatted);
+        if (outputDir) {
+            await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted);
+        }
+    }
+    if (format === 'json') {
+        const formatted = await jsonFormat(results);
+        ux.log(formatted);
+        if (outputDir) {
+            await writeFileToDir(outputDir, `test-result-${id}.json`, formatted);
+        }
+    }
+    if (format === 'junit') {
+        const formatted = await junitFormat(results);
+        ux.log(formatted);
+        if (outputDir) {
+            await writeFileToDir(outputDir, `test-result-${id}.xml`, formatted);
+        }
+    }
+}
+//# sourceMappingURL=handleTestResults.js.map

package/lib/handleTestResults.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handleTestResults.js","sourceRoot":"","sources":["../src/handleTestResults.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAA4B,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACpG,OAAO,EAAE,EAAE,EAAE,MAAM,gCAAgC,CAAC;AAEpD,KAAK,UAAU,cAAc,CAAC,SAAiB,EAAE,QAAgB,EAAE,OAAe;IAChF,wCAAwC;IACxC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,EAAE,EACF,MAAM,EACN,OAAO,EACP,WAAW,EACX,SAAS,GAOV;IACC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,kDAAkD;QAClD,OAAO;IACT,CAAC;IAED,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;IAEnC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClB,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,cAAc,CAAC,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAC5C,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClB,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,cAAc,CAAC,SAAS,EAAE,eAAe,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClB,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,cAAc,CAAC,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;AACH,CAAC"}

package/lib/testStages.d.ts

@@ -0,0 +1,27 @@
+import { AgentTestDetailsResponse, AgentTester } from '@salesforce/agents';
+import { Duration } from '@salesforce/kit';
+type Data = {
+    id: string;
+    status: string;
+    totalTestCases: number;
+    passingTestCases: number;
+    failingTestCases: number;
+};
+export declare class TestStages {
+    private mso;
+    private ux;
+    constructor({ title, jsonEnabled }: {
+        title: string;
+        jsonEnabled: boolean;
+    });
+    start(data?: Partial<Data>): void;
+    poll(agentTester: AgentTester, id: string, wait: Duration): Promise<{
+        completed: boolean;
+        response?: AgentTestDetailsResponse;
+    }>;
+    update(data: Partial<Data>): void;
+    stop(finalStatus?: 'async'): void;
+    error(): void;
+    done(data?: Partial<Data>): void;
+}
+export {};

package/lib/testStages.js

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2024, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import { colorize } from '@oclif/core/ux';
+import { MultiStageOutput } from '@oclif/multi-stage-output';
+import { Lifecycle } from '@salesforce/core';
+import { Ux } from '@salesforce/sf-plugins-core';
+const isTimeoutError = (e) => e?.name === 'PollingClientTimeout';
+export class TestStages {
+    mso;
+    ux;
+    constructor({ title, jsonEnabled }) {
+        this.ux = new Ux({ jsonEnabled });
+        this.mso = new MultiStageOutput({
+            title,
+            jsonEnabled,
+            stages: ['Starting Tests', 'Polling for Test Results'],
+            stageSpecificBlock: [
+                {
+                    stage: 'Polling for Test Results',
+                    type: 'dynamic-key-value',
+                    label: 'Status',
+                    get: (data) => data?.status,
+                },
+                {
+                    stage: 'Polling for Test Results',
+                    type: 'dynamic-key-value',
+                    label: 'Completed Tests',
+                    get: (data) => data?.totalTestCases && data?.passingTestCases && data?.failingTestCases
+                        ? `${data?.passingTestCases + data?.failingTestCases}/${data?.totalTestCases}`
+                        : undefined,
+                },
+                {
+                    stage: 'Polling for Test Results',
+                    type: 'dynamic-key-value',
+                    label: 'Passing Tests',
+                    get: (data) => data?.passingTestCases?.toString(),
+                },
+                {
+                    stage: 'Polling for Test Results',
+                    type: 'dynamic-key-value',
+                    label: 'Failing Tests',
+                    get: (data) => data?.failingTestCases?.toString(),
+                },
+            ],
+            postStagesBlock: [
+                {
+                    type: 'dynamic-key-value',
+                    label: 'Job ID',
+                    get: (data) => data?.id,
+                },
+            ],
+        });
+    }
+    start(data) {
+        this.mso.skipTo('Starting Tests', data);
+    }
+    async poll(agentTester, id, wait) {
+        this.mso.skipTo('Polling for Test Results');
+        const lifecycle = Lifecycle.getInstance();
+        lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event) => Promise.resolve(this.update(event)));
+        try {
+            const response = await agentTester.poll(id, { timeout: wait });
+            this.stop();
+            return { completed: true, response };
+        }
+        catch (e) {
+            if (isTimeoutError(e)) {
+                this.stop('async');
+                this.ux.log(`Client timed out after ${wait.minutes} minutes.`);
+                this.ux.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`);
+                return { completed: true };
+            }
+            else {
+                this.error();
+                throw e;
+            }
+        }
+    }
+    update(data) {
+        this.mso.updateData(data);
+    }
+    stop(finalStatus) {
+        this.mso.stop(finalStatus);
+    }
+    error() {
+        this.mso.error();
+    }
+    done(data) {
+        this.mso.skipTo('Done', data);
+    }
+}
+//# sourceMappingURL=testStages.js.map

package/lib/testStages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testStages.js","sourceRoot":"","sources":["../src/testStages.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,EAAE,EAAE,MAAM,6BAA6B,CAAC;AAUjD,MAAM,cAAc,GAAG,CAAC,CAAU,EAAyC,EAAE,CAC1E,CAAsB,EAAE,IAAI,KAAK,sBAAsB,CAAC;AAE3D,MAAM,OAAO,UAAU;IACb,GAAG,CAAyB;IAC5B,EAAE,CAAK;IAEf,YAAmB,EAAE,KAAK,EAAE,WAAW,EAA2C;QAChF,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,gBAAgB,CAAO;YACpC,KAAK;YACL,WAAW;YACX,MAAM,EAAE,CAAC,gBAAgB,EAAE,0BAA0B,CAAC;YACtD,kBAAkB,EAAE;gBAClB;oBACE,KAAK,EAAE,0BAA0B;oBACjC,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,QAAQ;oBACf,GAAG,EAAE,CAAC,IAAI,EAAsB,EAAE,CAAC,IAAI,EAAE,MAAM;iBAChD;gBACD;oBACE,KAAK,EAAE,0BAA0B;oBACjC,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,iBAAiB;oBACxB,GAAG,EAAE,CAAC,IAAI,EAAsB,EAAE,CAChC,IAAI,EAAE,cAAc,IAAI,IAAI,EAAE,gBAAgB,IAAI,IAAI,EAAE,gBAAgB;wBACtE,CAAC,CAAC,GAAG,IAAI,EAAE,gBAAgB,GAAG,IAAI,EAAE,gBAAgB,IAAI,IAAI,EAAE,cAAc,EAAE;wBAC9E,CAAC,CAAC,SAAS;iBAChB;gBACD;oBACE,KAAK,EAAE,0BAA0B;oBACjC,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,eAAe;oBACtB,GAAG,EAAE,CAAC,IAAI,EAAsB,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE;iBACtE;gBACD;oBACE,KAAK,EAAE,0BAA0B;oBACjC,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,eAAe;oBACtB,GAAG,EAAE,CAAC,IAAI,EAAsB,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE;iBACtE;aACF;YACD,eAAe,EAAE;gBACf;oBACE,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,QAAQ;oBACf,GAAG,EAAE,CAAC,IAAI,EAAsB,EAAE,CAAC,IAAI,EAAE,EAAE;iBAC5C;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAoB;QAC/B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;IAC1C,CAAC;IAEM,KAAK,CAAC,IAAI,CACf,WAAwB,EACxB,EAAU,EACV,IAAc;QAEd,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAC1C,SAAS,CAAC,EAAE,CACV,0BAA0B,EAC1B,KAAK,EAAE,KAMN,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1C,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACvC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACnB,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,OAAO,WAAW,CAAC,CAAC;gBAC/D,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,QAAQ,CAAC,KAAK,EAAE,iCAAiC,EAAE,EAAE,CAAC,kCAAkC,CAAC,CAAC;gBAC7G,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,EAAE,CAAC;gBACb,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;IACH,CAAC;IAEM,MAAM,CAAC,IAAmB;QAC/B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAEM,IAAI,CAAC,WAAqB;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC;IAEM,KAAK;QACV,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;IAEM,IAAI,CAAC,IAAoB;QAC9B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;CACF"}

package/messages/agent.preview.md

@@ -0,0 +1,20 @@
+# summary
+
+Interact with an active agent, as a user would, to preview responses
+
+# description
+
+XXX
+
+# flags.name.summary
+
+The name of the agent you want to preview
+
+# flags.name.description
+
+the API name of the agent? (TBD based on agents library)
+
+# examples
+
+- <%= config.bin %> <%= command.id %> --agent HelpDeskAgent
+- <%= config.bin %> <%= command.id %> --agent ConciergeAgent --target-org production

package/messages/agent.test.cancel.md

@@ -6,7 +6,7 @@ Cancel a running test for an Agent.
 
 Cancel a running test for an Agent, providing the AiEvaluation ID.
 
-# flags.id.summary
+# flags.job-id.summary
 
 The AiEvaluation ID.
 
@@ -18,4 +18,4 @@ Use the job ID of the most recent test evaluation.
 
 - Cancel a test for an Agent:
 
-  <%= config.bin %> <%= command.id %> --id AiEvalId
+  <%= config.bin %> <%= command.id %> --job-id AiEvalId

package/messages/agent.test.results.md

@@ -0,0 +1,19 @@
+# summary
+
+Get the results of a test evaluation.
+
+# description
+
+Provide the AiEvaluation ID to get the results of a test evaluation.
+
+# flags.job-id.summary
+
+The AiEvaluation ID.
+
+# flags.use-most-recent.summary
+
+Use the job ID of the most recent test evaluation.
+
+# examples
+
+- <%= config.bin %> <%= command.id %> --job-id AiEvalId

package/messages/agent.test.resume.md

@@ -0,0 +1,29 @@
+# summary
+
+Resume a running test for an Agent.
+
+# description
+
+Resume a running test for an Agent, providing the AiEvaluation ID.
+
+# flags.job-id.summary
+
+The AiEvaluation ID.
+
+# flags.use-most-recent.summary
+
+Use the job ID of the most recent test evaluation.
+
+# flags.wait.summary
+
+Number of minutes to wait for the command to complete and display results to the terminal window.
+
+# flags.wait.description
+
+If the command continues to run after the wait period, the CLI returns control of the terminal window to you.
+
+# examples
+
+- Resume a test for an Agent:
+
+  <%= config.bin %> <%= command.id %> --job-id AiEvalId

package/messages/agent.test.run.md

@@ -6,13 +6,13 @@ Start a test for an Agent.
 
 Start a test for an Agent, providing the AiEvalDefinitionVersion ID. Returns the job ID.
 
-# flags.id.summary
+# flags.name.summary
 
-The AiEvalDefinitionVersion ID.
+The name of the AiEvaluationDefinition to start.
 
-# flags.id.description
+# flags.name.description
 
-The AiEvalDefinitionVersion ID.
+The name of the AiEvaluationDefinition to start.
 
 # flags.wait.summary
 
@@ -22,12 +22,8 @@ Number of minutes to wait for the command to complete and display results to the
 
 If the command continues to run after the wait period, the CLI returns control of the terminal window to you.
 
-# flags.output-dir.summary
-
-Directory in which to store test run files.
-
 # examples
 
 - Start a test for an Agent:
 
-  <%= config.bin %> <%= command.id %> --id AiEvalDefVerId
+  <%= config.bin %> <%= command.id %> --name AiEvalDefVerId

package/messages/shared.md

@@ -0,0 +1,11 @@
+# flags.result-format.summary
+
+Format of the test run results.
+
+# flags.output-dir.summary
+
+Directory to write the test results to.
+
+# flags.output-dir.description
+
+If test run is complete, write the results to the specified directory. If the tests are still running, the test results will not be written.