chrome-devtools-frontend 1.0.1376716 → 1.0.1386602

package/front_end/panels/freestyler/FreestylerAgent.test.ts

@@ -517,19 +517,20 @@ c`;
         serverSideLoggingEnabled: true,
       });
       sinon.stub(agent, 'preamble').value('preamble');
-      agent.chatNewHistoryForTesting = new Map([[
-        0,
-        [
-          {
-            type: Freestyler.ResponseType.QUERYING,
-            query: 'question',
-          },
-          {
-            type: Freestyler.ResponseType.ANSWER,
-            text: 'answer',
-          },
-        ],
-      ]]);
+      agent.chatNewHistoryForTesting = [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'question',
+        },
+        {
+          type: Freestyler.ResponseType.QUERYING,
+          query: 'question',
+        },
+        {
+          type: Freestyler.ResponseType.ANSWER,
+          text: 'answer',
+        },
+      ];
       assert.deepStrictEqual(
           agent.buildRequest({
             input: 'test input',
@@ -565,7 +566,8 @@ c`;
   });
 
   function mockAidaClient(
-      fetch: (_: unknown, options?: {signal: AbortSignal}) => AsyncGenerator<Host.AidaClient.AidaResponse, void, void>,
+      fetch: (_: Host.AidaClient.AidaRequest, options?: {signal: AbortSignal}) =>
+          AsyncGenerator<Host.AidaClient.AidaResponse, void, void>,
       ): Host.AidaClient.AidaClient {
     return {
       fetch,
@@ -616,7 +618,7 @@ c`;
         });
 
         promise.resolve(true);
-        await Array.fromAsync(agent.run('test', {selected: element}));
+        await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
 
         sinon.assert.match(execJs.getCall(0).args[1], sinon.match({throwOnSideEffect: true}));
       });
@@ -655,7 +657,7 @@ c`;
 
         });
         promise.resolve(true);
-        await Array.fromAsync(agent.run('test', {selected: element}));
+        await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
 
         assert.strictEqual(execJs.getCalls().length, 2);
         sinon.assert.match(execJs.getCall(1).args[1], sinon.match({throwOnSideEffect: false}));
@@ -694,7 +696,7 @@ c`;
 
         });
         promise.resolve(false);
-        const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+        const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
 
         const actionStep = responses.find(response => response.type === Freestyler.ResponseType.ACTION)!;
 
@@ -732,7 +734,7 @@ c`;
 
         });
 
-        const result = await Array.fromAsync(agent.run('test', {selected: element}));
+        const result = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
         const actionSteps = result.filter(step => {
           return step.type === Freestyler.ResponseType.ACTION;
         });
@@ -757,7 +759,7 @@ c`;
         execJs,
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
       assert.deepStrictEqual(responses, [
         {
           type: Freestyler.ResponseType.USER_QUERY,
@@ -797,6 +799,63 @@ c`;
       ]);
     });
 
+    it('correctly handles chat_history in AIDA requests', async () => {
+      const requests: Host.AidaClient.AidaRequest[] = [];
+
+      let i = 0;
+      async function* generateAnswer(request: Host.AidaClient.AidaRequest) {
+        requests.push(request);
+        if (i !== 0) {
+          yield {
+            explanation: 'ANSWER: this is the actual answer',
+            metadata: {},
+            completed: true,
+          };
+          return;
+        }
+        yield {
+          explanation: `THOUGHT: I am thinking.
+TITLE: thinking
+ACTION
+const data = {"test": "observation"};
+STOP`,
+          metadata: {},
+          completed: false,
+        };
+        i++;
+      }
+
+      const execJs = sinon.mock().once();
+      execJs.onCall(0).returns('test data');
+      const agent = new FreestylerAgent({
+        aidaClient: mockAidaClient(generateAnswer),
+        createExtensionScope,
+        execJs,
+      });
+
+      await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
+
+      assert.lengthOf(requests, 2, 'Unexpected number of AIDA requests');
+      assert.isUndefined(requests[0].chat_history, 'Unexpected chat history in the initial request');
+      assert.strictEqual(
+          requests[0].input, '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
+          'Unexpected input in the initial request');
+      assert.deepStrictEqual(
+          requests[1].chat_history,
+          [
+            {
+              entity: 1,
+              text: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
+            },
+            {
+              entity: 2,
+              text: 'THOUGHT: I am thinking.\nTITLE: thinking\nACTION\nconst data = {\"test\": \"observation\"};\nSTOP',
+            },
+          ],
+          'Unexpected chat history in the follow-up request');
+      assert.strictEqual(requests[1].input, 'OBSERVATION: test data', 'Unexpected input in the follow-up request');
+    });
+
     it('generates an rpcId for the answer', async () => {
       async function* generateAnswer() {
         yield {
@@ -814,7 +873,7 @@ c`;
 
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
       assert.deepStrictEqual(responses, [
         {
           type: Freestyler.ResponseType.USER_QUERY,
@@ -864,7 +923,7 @@ c`;
 
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
       assert.deepStrictEqual(responses, [
         {
           type: Freestyler.ResponseType.USER_QUERY,
@@ -913,7 +972,7 @@ c`;
 
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
       assert.deepStrictEqual(responses, [
         {
           type: Freestyler.ResponseType.USER_QUERY,
@@ -942,6 +1001,43 @@ c`;
       ]);
     });
 
+    it('should execute an action only once even when the partial response contains an action', async () => {
+      const execJs = sinon.spy();
+      async function* generatePartialAndFullAction() {
+        yield {
+          explanation: `THOUGHT: I am thinking.
+
+ACTION
+console.log('hel
+          `,
+          metadata: {},
+          completed: false,
+        };
+
+        sinon.assert.notCalled(execJs);
+        yield {
+          explanation: `THOUGHT: I am thinking.
+
+ACTION
+console.log('hello');
+STOP
+          `,
+          metadata: {},
+          completed: true,
+        };
+      }
+
+      const agent = new FreestylerAgent({
+        aidaClient: mockAidaClient(generatePartialAndFullAction),
+        createExtensionScope,
+        execJs,
+      });
+      await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
+
+      sinon.assert.calledOnce(execJs);
+      assert.include(execJs.lastCall.args[0], 'console.log(\'hello\');');
+    });
+
     it('generates a response if nothing is returned', async () => {
       async function* generateNothing() {
         yield {
@@ -956,7 +1052,7 @@ c`;
         aidaClient: mockAidaClient(generateNothing),
         execJs,
       });
-      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
       assert.deepStrictEqual(responses, [
         {
           type: Freestyler.ResponseType.USER_QUERY,
@@ -1019,7 +1115,7 @@ ANSWER: this is the answer`,
         execJs,
 
       });
-      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
       assert.deepStrictEqual(responses, [
         {
           type: Freestyler.ResponseType.USER_QUERY,
@@ -1092,7 +1188,7 @@ ANSWER: this is the answer`,
 
       });
 
-      await Array.fromAsync(agent.run('test', {selected: element}));
+      await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
 
       assert.deepStrictEqual(agent.chatHistoryForTesting, [
         {
@@ -1161,12 +1257,98 @@ ANSWER: this is the answer`,
 
       const controller = new AbortController();
       controller.abort();
-      await Array.fromAsync(agent.run('test', {selected: element, signal: controller.signal}));
+      await Array.fromAsync(
+          agent.run('test', {selected: new Freestyler.NodeContext(element), signal: controller.signal}));
 
       assert.deepStrictEqual(agent.chatHistoryForTesting, []);
     });
   });
 
+  describe('history', () => {
+    let element: sinon.SinonStubbedInstance<SDK.DOMModel.DOMNode>;
+    beforeEach(() => {
+      mockHostConfig();
+      element = sinon.createStubInstance(SDK.DOMModel.DOMNode);
+      // @ts-ignore
+      setAiAssistancePersistentHistory(true);
+    });
+
+    it('stores history via AiHistoryStorage', async () => {
+      let count = 0;
+      async function* generateMultipleTimes() {
+        if (count === 1) {
+          yield {
+            explanation: 'ANSWER: this is the answer',
+            metadata: {},
+            completed: true,
+          };
+          return;
+        }
+        count++;
+        yield {
+          explanation: `THOUGHT: thought ${count}\nTITLE:test\nACTION\nconsole.log('test')\nSTOP\n`,
+          metadata: {},
+          completed: false,
+        };
+      }
+      const historyStub = sinon.stub(Freestyler.AiHistoryStorage.instance(), 'upsertHistoryEntry');
+      const execJs = sinon.spy(async () => 'undefined');
+      const agent = new FreestylerAgent({
+        aidaClient: mockAidaClient(generateMultipleTimes),
+        createExtensionScope,
+        execJs,
+      });
+
+      await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
+
+      assert.deepStrictEqual(historyStub.lastCall.args[0].history, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [{
+            text: '* Its selector is `undefined`',
+            title: 'Data used',
+          }],
+        },
+        {
+          type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
+        },
+        {
+          type: Freestyler.ResponseType.TITLE,
+          title: 'test',
+          rpcId: undefined,
+        },
+        {
+          type: Freestyler.ResponseType.THOUGHT,
+          thought: 'thought 1',
+          rpcId: undefined,
+        },
+        {
+          type: Freestyler.ResponseType.ACTION,
+          code: 'console.log(\'test\')',
+          output: 'undefined',
+          canceled: false,
+          rpcId: undefined,
+        },
+        {
+          type: Freestyler.ResponseType.QUERYING,
+          query: 'OBSERVATION: undefined',
+        },
+        {
+          type: Freestyler.ResponseType.ANSWER,
+          text: 'this is the answer',
+          suggestions: undefined,
+          rpcId: undefined,
+        },
+      ]);
+    });
+  });
+
   describe('HostConfigFreestylerExecutionMode', () => {
     let element: sinon.SinonStubbedInstance<SDK.DOMModel.DOMNode>;
     beforeEach(() => {
@@ -1209,7 +1391,7 @@ ANSWER: this is the answer`,
           createExtensionScope,
           execJs,
         });
-        const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+        const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
         const actionStep = responses.find(response => response.type === Freestyler.ResponseType.ACTION)!;
         assert.strictEqual(actionStep.output, 'Error: JavaScript execution is currently disabled.');
         assert.strictEqual(execJs.getCalls().length, 0);
@@ -1231,7 +1413,7 @@ ANSWER: this is the answer`,
           createExtensionScope,
           execJs,
         });
-        const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+        const responses = await Array.fromAsync(agent.run('test', {selected: new Freestyler.NodeContext(element)}));
         const actionStep = responses.find(response => response.type === Freestyler.ResponseType.ACTION)!;
         assert.strictEqual(
             actionStep.output, 'Error: JavaScript execution that modifies the page is currently disabled.');

package/front_end/panels/freestyler/FreestylerAgent.ts

@@ -9,16 +9,19 @@ import * as Platform from '../../core/platform/platform.js';
 import * as Root from '../../core/root/root.js';
 import * as SDK from '../../core/sdk/sdk.js';
 import * as UI from '../../ui/legacy/legacy.js';
+import * as LitHtml from '../../ui/lit-html/lit-html.js';
 
 import {
   type ActionResponse,
+  type AgentOptions as BaseAgentOptions,
   AgentType,
   AiAgent,
-  type AidaRequestOptions,
   type ContextResponse,
+  ConversationContext,
   debugLog,
-  isDebugMode,
   type ParsedResponse,
+  type ParsedResponseAnswer,
+  type RequestOptions,
   ResponseType,
   type SideEffectResponse,
 } from './AiAgent.js';
@@ -193,26 +196,57 @@ async function executeJsCode(
 }
 
 const MAX_OBSERVATION_BYTE_LENGTH = 25_000;
+const OBSERVATION_TIMEOUT = 5_000;
 
 type CreateExtensionScopeFunction = (changes: ChangeManager) => {
   install(): Promise<void>, uninstall(): Promise<void>,
 };
 
-type AgentOptions = {
-  aidaClient: Host.AidaClient.AidaClient,
-  changeManager?: ChangeManager,
-  confirmSideEffectForTest?: typeof Promise.withResolvers,
-  serverSideLoggingEnabled?: boolean,
-  createExtensionScope?: CreateExtensionScopeFunction,
-  execJs?: typeof executeJsCode,
-};
+interface AgentOptions extends BaseAgentOptions {
+  changeManager?: ChangeManager;
+  confirmSideEffectForTest?: typeof Promise.withResolvers;
+  createExtensionScope?: CreateExtensionScopeFunction;
+  execJs?: typeof executeJsCode;
+}
+
+export class NodeContext extends ConversationContext<SDK.DOMModel.DOMNode> {
+  #node: SDK.DOMModel.DOMNode;
+
+  constructor(node: SDK.DOMModel.DOMNode) {
+    super();
+    this.#node = node;
+  }
+
+  getOrigin(): string {
+    const ownerDocument = this.#node.ownerDocument;
+    if (!ownerDocument) {
+      // The node is detached from a document.
+      return 'detached';
+    }
+    return new URL(ownerDocument.documentURL).origin;
+  }
+
+  getItem(): SDK.DOMModel.DOMNode {
+    return this.#node;
+  }
+
+  override getIcon(): HTMLElement {
+    return document.createElement('span');
+  }
+
+  override getTitle(): string|ReturnType<typeof LitHtml.Directives.until> {
+    return LitHtml.Directives.until(
+        Common.Linkifier.Linkifier.linkify(this.#node),
+    );
+  }
+}
 
 /**
  * One agent instance handles one conversation. Create a new agent
  * instance for a new conversation.
  */
 export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
-  override type = AgentType.FREESTYLER;
+  override readonly type = AgentType.FREESTYLER;
 
   readonly preamble = preamble;
   readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_FREESTYLER;
@@ -227,7 +261,7 @@ export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
     return config.devToolsFreestyler?.executionMode ?? Root.Runtime.HostConfigFreestylerExecutionMode.ALL_SCRIPTS;
   }
 
-  get options(): AidaRequestOptions {
+  get options(): RequestOptions {
     const config = Common.Settings.Settings.instance().getHostConfig();
     const temperature = config.devToolsFreestyler?.temperature;
     const modelId = config.devToolsFreestyler?.modelId;
@@ -430,10 +464,16 @@ export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
           canceled: true,
         };
       }
-      const result = await this.#execJs(
-          functionDeclaration,
-          {throwOnSideEffect},
-      );
+      const result = await Promise.race([
+        this.#execJs(
+            functionDeclaration,
+            {throwOnSideEffect},
+            ),
+        new Promise<never>((_, reject) => {
+          setTimeout(
+              () => reject(new Error('Script execution exceeded the maximum allowed time.')), OBSERVATION_TIMEOUT);
+        }),
+      ]);
       const byteCount = Platform.StringUtilities.countWtf8Bytes(result);
       Host.userMetrics.freestylerEvalResponseSize(byteCount);
       if (byteCount > MAX_OBSERVATION_BYTE_LENGTH) {
@@ -570,10 +610,6 @@ export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
         }
 
         const sideEffectConfirmationPromiseWithResolvers = this.#confirmSideEffect<boolean>();
-        if (isDebugMode()) {
-          window.dispatchEvent(new CustomEvent(
-              'freestylersideeffect', {detail: {confirm: sideEffectConfirmationPromiseWithResolvers.resolve}}));
-        }
 
         yield {
           type: ResponseType.SIDE_EFFECT,
@@ -606,7 +642,8 @@ export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
   }
 
   override async *
-      handleContextDetails(selectedElement: SDK.DOMModel.DOMNode|null): AsyncGenerator<ContextResponse, void, void> {
+      handleContextDetails(selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null):
+          AsyncGenerator<ContextResponse, void, void> {
     if (!selectedElement) {
       return;
     }
@@ -615,19 +652,21 @@ export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
       title: lockedString(UIStringsNotTranslate.analyzingThePrompt),
       details: [{
         title: lockedString(UIStringsNotTranslate.dataUsed),
-        text: await FreestylerAgent.describeElement(selectedElement),
+        text: await FreestylerAgent.describeElement(selectedElement.getItem()),
       }],
     };
   }
 
-  override async enhanceQuery(query: string, selectedElement: SDK.DOMModel.DOMNode|null): Promise<string> {
+  override async enhanceQuery(query: string, selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null):
+      Promise<string> {
     const elementEnchantmentQuery = selectedElement ?
-        `# Inspected element\n\n${await FreestylerAgent.describeElement(selectedElement)}\n\n# User request\n\n` :
+        `# Inspected element\n\n${
+            await FreestylerAgent.describeElement(selectedElement.getItem())}\n\n# User request\n\n` :
         '';
     return `${elementEnchantmentQuery}QUERY: ${query}`;
   }
 
-  override formatHistoryChunkAnswer(text: string): string {
-    return `ANSWER: ${text}`;
+  override formatParsedResponseAnswer(step: ParsedResponseAnswer): string {
+    return `ANSWER: ${step.answer}`;
   }
 }