chrome-devtools-frontend 1.0.1362775 → 1.0.1376716

package/front_end/panels/freestyler/FreestylerAgent.test.ts

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import * as Host from '../../core/host/host.js';
+import * as Root from '../../core/root/root.js';
 import * as SDK from '../../core/sdk/sdk.js';
 import {
   describeWithEnvironment,
@@ -14,12 +15,15 @@ import * as Freestyler from './freestyler.js';
 const {FreestylerAgent} = Freestyler;
 
 describeWithEnvironment('FreestylerAgent', () => {
-  function mockHostConfig(modelId?: string, temperature?: number, userTier?: string) {
+  function mockHostConfig(
+      modelId?: string, temperature?: number, userTier?: string,
+      executionMode?: Root.Runtime.HostConfigFreestylerExecutionMode) {
     getGetHostConfigStub({
-      devToolsFreestylerDogfood: {
+      devToolsFreestyler: {
         modelId,
         temperature,
         userTier,
+        executionMode,
       },
     });
   }
@@ -35,24 +39,28 @@ describeWithEnvironment('FreestylerAgent', () => {
     };
   }
   describe('parseResponse', () => {
+    const agent = new FreestylerAgent({
+      aidaClient: {} as Host.AidaClient.AidaClient,
+    });
+
     it('parses a thought', async () => {
       const payload = 'some response';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`THOUGHT: ${payload}`),
+          agent.parseResponse(`THOUGHT: ${payload}`),
           {
             title: undefined,
             thought: payload,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`   THOUGHT: ${payload}`),
+          agent.parseResponse(`   THOUGHT: ${payload}`),
           {
             title: undefined,
             thought: payload,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`Something\n   THOUGHT: ${payload}`),
+          agent.parseResponse(`Something\n   THOUGHT: ${payload}`),
           {
             title: undefined,
             thought: payload,
@@ -62,24 +70,24 @@ describeWithEnvironment('FreestylerAgent', () => {
     it('parses a answer', async () => {
       const payload = 'some response';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ANSWER: ${payload}`),
+          agent.parseResponse(`ANSWER: ${payload}`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`   ANSWER: ${payload}`),
+          agent.parseResponse(`   ANSWER: ${payload}`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`Something\n   ANSWER: ${payload}`),
+          agent.parseResponse(`Something\n   ANSWER: ${payload}`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
     });
@@ -88,44 +96,44 @@ describeWithEnvironment('FreestylerAgent', () => {
 b
 c`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ANSWER: ${payload}`),
+          agent.parseResponse(`ANSWER: ${payload}`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`   ANSWER: ${payload}`),
+          agent.parseResponse(`   ANSWER: ${payload}`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`Something\n   ANSWER: ${payload}`),
+          agent.parseResponse(`Something\n   ANSWER: ${payload}`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ANSWER: ${payload}\nTHOUGHT: thought`),
+          agent.parseResponse(`ANSWER: ${payload}\nTHOUGHT: thought`),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `ANSWER: ${payload}\nOBSERVATION: observation`,
               ),
           {
             answer: payload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `ANSWER: ${payload}\nACTION\naction\nSTOP`,
               ),
           {
@@ -140,7 +148,7 @@ c`;
   someKey: "value",
 }`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n${payload}\nSTOP`),
+          agent.parseResponse(`ACTION\n${payload}\nSTOP`),
           {
             action: payload,
             title: undefined,
@@ -148,7 +156,7 @@ c`;
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n${payload}`),
+          agent.parseResponse(`ACTION\n${payload}`),
           {
             action: payload,
             title: undefined,
@@ -156,7 +164,7 @@ c`;
           },
       );
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n\n${payload}\n\nSTOP`),
+          agent.parseResponse(`ACTION\n\n${payload}\n\nSTOP`),
           {
             action: payload,
             title: undefined,
@@ -165,7 +173,7 @@ c`;
       );
 
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n\n${payload}\n\nANSWER: answer`),
+          agent.parseResponse(`ACTION\n\n${payload}\n\nANSWER: answer`),
           {
             action: payload,
             title: undefined,
@@ -179,7 +187,7 @@ c`;
           styles
         };`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n${payload}STOP`),
+          agent.parseResponse(`ACTION\n${payload}STOP`),
           {
             action: payload,
             title: undefined,
@@ -191,7 +199,7 @@ c`;
       const payload = 'some response';
       const title = 'this is the title';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`THOUGHT: ${payload}\nTITLE: ${title}`),
+          agent.parseResponse(`THOUGHT: ${payload}\nTITLE: ${title}`),
           {
             thought: payload,
             title,
@@ -204,7 +212,7 @@ c`;
   someKey: "value",
 }`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `ACTION\n\`\`\`\n${payload}\n\`\`\`\nSTOP`,
               ),
           {
@@ -220,7 +228,7 @@ c`;
   someKey: "value",
 }`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `ACTION\n\`\`\`\`\`\njs\n${payload}\n\`\`\`\`\`\nSTOP`,
               ),
           {
@@ -237,7 +245,7 @@ c`;
 }`;
       const thoughtPayload = 'thought';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `THOUGHT:${thoughtPayload}\nACTION\n${actionPayload}\nSTOP`,
               ),
           {
@@ -252,22 +260,22 @@ c`;
       const answerPayload = 'answer';
       const thoughtPayload = 'thought';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `THOUGHT:${thoughtPayload}\nANSWER:${answerPayload}`,
               ),
           {
             answer: answerPayload,
-            suggestions: [],
+            suggestions: undefined,
           },
       );
     });
 
     it('parses an answer and suggestions', async () => {
       const answerPayload = 'answer';
-      const suggestions = ['suggestion'];
+      const suggestions = ['suggestion'] as [string];
       const suggestionsText = JSON.stringify(suggestions);
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `ANSWER:${answerPayload}\nSUGGESTIONS: ${suggestionsText}`,
               ),
           {
@@ -285,7 +293,7 @@ c`;
 }`;
       const title = 'title';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `THOUGHT: ${thoughtPayload}\nTITLE: ${title}\nACTION\n${actionPayload}\nSTOP\nANSWER:${answerPayload}`,
               ),
           {
@@ -303,7 +311,7 @@ c`;
           styles
         };`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               `ACTION\n${payload}STOP\nANSWER:${answerPayload}\nSUGGESTIONS: ${JSON.stringify(suggestions)}`),
           {
             action: payload,
@@ -318,7 +326,7 @@ c`;
           styles
         };`;
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n${payload}STOP\nOBSERVATION:{styles: {}}`),
+          agent.parseResponse(`ACTION\n${payload}STOP\nOBSERVATION:{styles: {}}`),
           {
             action: payload,
             thought: undefined,
@@ -333,7 +341,7 @@ c`;
         };`;
       const thoughtPayload = 'thought';
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(`ACTION\n${payload}STOP\nTHOUGHT:${thoughtPayload}`),
+          agent.parseResponse(`ACTION\n${payload}STOP\nTHOUGHT:${thoughtPayload}`),
           {
             action: payload,
             thought: thoughtPayload,
@@ -344,19 +352,19 @@ c`;
 
     it('parses a response as an answer', async () => {
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               'This is also an answer',
               ),
           {
             answer: 'This is also an answer',
-            suggestions: [],
+            suggestions: undefined,
           },
       );
     });
 
     it('parses a response with no instruction tags as an answer and correctly parses suggestions', async () => {
       assert.deepStrictEqual(
-          FreestylerAgent.parseResponse(
+          agent.parseResponse(
               'This is also an answer\nSUGGESTIONS: [\"suggestion\"]',
               ),
           {
@@ -365,6 +373,17 @@ c`;
           },
       );
     });
+
+    it('parses multi line thoughts', () => {
+      const thoughtText = 'first line\nsecond line';
+      assert.deepStrictEqual(
+          agent.parseResponse(`THOUGHT: ${thoughtText}`),
+          {
+            thought: thoughtText,
+            title: undefined,
+          },
+      );
+    });
   });
 
   describe('describeElement', () => {
@@ -498,20 +517,16 @@ c`;
         serverSideLoggingEnabled: true,
       });
       sinon.stub(agent, 'preamble').value('preamble');
-      agent.chatHistoryForTesting = new Map([[
+      agent.chatNewHistoryForTesting = new Map([[
         0,
         [
           {
-            text: 'first',
-            entity: Host.AidaClient.Entity.UNKNOWN,
+            type: Freestyler.ResponseType.QUERYING,
+            query: 'question',
           },
           {
-            text: 'second',
-            entity: Host.AidaClient.Entity.SYSTEM,
-          },
-          {
-            text: 'third',
-            entity: Host.AidaClient.Entity.USER,
+            type: Freestyler.ResponseType.ANSWER,
+            text: 'answer',
           },
         ],
       ]]);
@@ -525,16 +540,12 @@ c`;
             preamble: 'preamble',
             chat_history: [
               {
-                entity: 0,
-                text: 'first',
+                entity: 1,
+                text: 'question',
               },
               {
                 entity: 2,
-                text: 'second',
-              },
-              {
-                entity: 1,
-                text: 'third',
+                text: 'ANSWER: answer',
               },
             ],
             metadata: {
@@ -553,6 +564,15 @@ c`;
     });
   });
 
+  function mockAidaClient(
+      fetch: (_: unknown, options?: {signal: AbortSignal}) => AsyncGenerator<Host.AidaClient.AidaResponse, void, void>,
+      ): Host.AidaClient.AidaClient {
+    return {
+      fetch,
+      registerClientEvent: () => Promise.resolve({}),
+    };
+  }
+
   describe('run', () => {
     let element: sinon.SinonStubbedInstance<SDK.DOMModel.DOMNode>;
     beforeEach(() => {
@@ -560,16 +580,6 @@ c`;
       element = sinon.createStubInstance(SDK.DOMModel.DOMNode);
     });
 
-    function mockAidaClient(
-        fetch: (_: unknown, options?: {signal: AbortSignal}) =>
-            AsyncGenerator<Host.AidaClient.AidaResponse, void, void>,
-        ): Host.AidaClient.AidaClient {
-      return {
-        fetch,
-        registerClientEvent: () => Promise.resolve({}),
-      };
-    }
-
     describe('side effect handling', () => {
       it('calls confirmSideEffect when the code execution contains a side effect', async () => {
         const promise = Promise.withResolvers();
@@ -606,7 +616,7 @@ c`;
         });
 
         promise.resolve(true);
-        await Array.fromAsync(agent.run('test', {selectedElement: element}));
+        await Array.fromAsync(agent.run('test', {selected: element}));
 
         sinon.assert.match(execJs.getCall(0).args[1], sinon.match({throwOnSideEffect: true}));
       });
@@ -645,7 +655,7 @@ c`;
 
         });
         promise.resolve(true);
-        await Array.fromAsync(agent.run('test', {selectedElement: element}));
+        await Array.fromAsync(agent.run('test', {selected: element}));
 
         assert.strictEqual(execJs.getCalls().length, 2);
         sinon.assert.match(execJs.getCall(1).args[1], sinon.match({throwOnSideEffect: false}));
@@ -684,7 +694,7 @@ c`;
 
         });
         promise.resolve(false);
-        const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+        const responses = await Array.fromAsync(agent.run('test', {selected: element}));
 
         const actionStep = responses.find(response => response.type === Freestyler.ResponseType.ACTION)!;
 
@@ -722,7 +732,7 @@ c`;
 
         });
 
-        const result = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+        const result = await Array.fromAsync(agent.run('test', {selected: element}));
         const actionSteps = result.filter(step => {
           return step.type === Freestyler.ResponseType.ACTION;
         });
@@ -745,19 +755,33 @@ c`;
       const agent = new FreestylerAgent({
         aidaClient: mockAidaClient(generateAnswer),
         execJs,
-
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
       assert.deepStrictEqual(responses, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [
+            {
+              text: '* Its selector is `undefined`',
+              title: 'Data used',
+            },
+          ],
+        },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
         },
         {
           type: Freestyler.ResponseType.ANSWER,
           text: 'this is the answer',
+          suggestions: undefined,
           rpcId: undefined,
-          suggestions: [],
         },
       ]);
       sinon.assert.notCalled(execJs);
@@ -790,16 +814,31 @@ c`;
 
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
       assert.deepStrictEqual(responses, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [
+            {
+              text: '* Its selector is `undefined`',
+              title: 'Data used',
+            },
+          ],
+        },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
         },
         {
           type: Freestyler.ResponseType.ANSWER,
           text: 'this is the answer',
+          suggestions: undefined,
           rpcId: 123,
-          suggestions: [],
         },
       ]);
     });
@@ -825,10 +864,25 @@ c`;
 
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
       assert.deepStrictEqual(responses, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [
+            {
+              text: '* Its selector is `undefined`',
+              title: 'Data used',
+            },
+          ],
+        },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
         },
         {
           rpcId: undefined,
@@ -859,16 +913,31 @@ c`;
 
       });
 
-      const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
       assert.deepStrictEqual(responses, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [
+            {
+              text: '* Its selector is `undefined`',
+              title: 'Data used',
+            },
+          ],
+        },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
         },
         {
           type: Freestyler.ResponseType.ANSWER,
           text: 'this is the answer',
+          suggestions: undefined,
           rpcId: 123,
-          suggestions: [],
         },
       ]);
     });
@@ -887,10 +956,25 @@ c`;
         aidaClient: mockAidaClient(generateNothing),
         execJs,
       });
-      const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
       assert.deepStrictEqual(responses, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [
+            {
+              text: '* Its selector is `undefined`',
+              title: 'Data used',
+            },
+          ],
+        },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
         },
         {
           type: Freestyler.ResponseType.ERROR,
@@ -935,10 +1019,25 @@ ANSWER: this is the answer`,
         execJs,
 
       });
-      const responses = await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      const responses = await Array.fromAsync(agent.run('test', {selected: element}));
       assert.deepStrictEqual(responses, [
+        {
+          type: Freestyler.ResponseType.USER_QUERY,
+          query: 'test',
+        },
+        {
+          type: Freestyler.ResponseType.CONTEXT,
+          title: 'Analyzing the prompt',
+          details: [
+            {
+              text: '* Its selector is `undefined`',
+              title: 'Data used',
+            },
+          ],
+        },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: '# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test',
         },
         {
           type: Freestyler.ResponseType.THOUGHT,
@@ -954,12 +1053,13 @@ ANSWER: this is the answer`,
         },
         {
           type: Freestyler.ResponseType.QUERYING,
+          query: 'OBSERVATION: hello',
         },
         {
           type: Freestyler.ResponseType.ANSWER,
           text: 'this is the actual answer',
+          suggestions: undefined,
           rpcId: undefined,
-          suggestions: [],
         },
       ]);
       sinon.assert.calledOnce(execJs);
@@ -992,7 +1092,7 @@ ANSWER: this is the answer`,
 
       });
 
-      await Array.fromAsync(agent.run('test', {selectedElement: element}));
+      await Array.fromAsync(agent.run('test', {selected: element}));
 
       assert.deepStrictEqual(agent.chatHistoryForTesting, [
         {
@@ -1061,9 +1161,82 @@ ANSWER: this is the answer`,
 
       const controller = new AbortController();
       controller.abort();
-      await Array.fromAsync(agent.run('test', {selectedElement: element, signal: controller.signal}));
+      await Array.fromAsync(agent.run('test', {selected: element, signal: controller.signal}));
 
       assert.deepStrictEqual(agent.chatHistoryForTesting, []);
     });
   });
+
+  describe('HostConfigFreestylerExecutionMode', () => {
+    let element: sinon.SinonStubbedInstance<SDK.DOMModel.DOMNode>;
+    beforeEach(() => {
+      element = sinon.createStubInstance(SDK.DOMModel.DOMNode);
+    });
+
+    function getMockClient() {
+      let count = 0;
+      async function* generateActionAndAnswer() {
+        if (count === 0) {
+          yield {
+            explanation: `ACTION
+            $0.style.backgroundColor = 'red'
+            STOP`,
+            metadata: {},
+            completed: true,
+          };
+        } else {
+          yield {
+            explanation: 'ANSWER: This is the answer',
+            metadata: {},
+            completed: true,
+          };
+        }
+
+        count++;
+      }
+      return mockAidaClient(generateActionAndAnswer);
+    }
+
+    describe('NO_SCRIPTS', () => {
+      beforeEach(() => {
+        mockHostConfig(undefined, undefined, undefined, Root.Runtime.HostConfigFreestylerExecutionMode.NO_SCRIPTS);
+      });
+
+      it('returns an error if scripts are disabled', async () => {
+        const execJs = sinon.mock();
+        const agent = new FreestylerAgent({
+          aidaClient: getMockClient(),
+          createExtensionScope,
+          execJs,
+        });
+        const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+        const actionStep = responses.find(response => response.type === Freestyler.ResponseType.ACTION)!;
+        assert.strictEqual(actionStep.output, 'Error: JavaScript execution is currently disabled.');
+        assert.strictEqual(execJs.getCalls().length, 0);
+      });
+    });
+
+    describe('SIDE_EFFECT_FREE_SCRIPTS_ONLY', () => {
+      beforeEach(() => {
+        mockHostConfig(
+            undefined, undefined, undefined,
+            Root.Runtime.HostConfigFreestylerExecutionMode.SIDE_EFFECT_FREE_SCRIPTS_ONLY);
+      });
+
+      it('returns an error if a script causes a side effect', async () => {
+        const execJs =
+            sinon.mock().throws(new Freestyler.SideEffectError('EvalError: Possible side-effect in debug-evaluate'));
+        const agent = new FreestylerAgent({
+          aidaClient: getMockClient(),
+          createExtensionScope,
+          execJs,
+        });
+        const responses = await Array.fromAsync(agent.run('test', {selected: element}));
+        const actionStep = responses.find(response => response.type === Freestyler.ResponseType.ACTION)!;
+        assert.strictEqual(
+            actionStep.output, 'Error: JavaScript execution that modifies the page is currently disabled.');
+        assert.strictEqual(execJs.getCalls().length, 1);
+      });
+    });
+  });
 });