chrome-devtools-frontend 1.0.1367881 → 1.0.1376716

package/front_end/panels/freestyler/FreestylerAgent.ts

@@ -12,16 +12,15 @@ import * as UI from '../../ui/legacy/legacy.js';
 
 import {
   type ActionResponse,
+  AgentType,
   AiAgent,
   type AidaRequestOptions,
+  type ContextResponse,
   debugLog,
-  ErrorType,
   isDebugMode,
-  type ResponseData,
+  type ParsedResponse,
   ResponseType,
   type SideEffectResponse,
-  type ThoughtResponse,
-  type TitleResponse,
 } from './AiAgent.js';
 import {ChangeManager} from './ChangeManager.js';
 import {ExtensionScope, FREESTYLER_WORLD_NAME} from './ExtensionScope.js';
@@ -139,7 +138,8 @@ SUGGESTIONS: ["What is a stacking context?", "How can I change the stacking orde
 `;
 /* clang-format on */
 
-async function executeJsCode(code: string, {throwOnSideEffect}: {throwOnSideEffect: boolean}): Promise<string> {
+async function executeJsCode(
+    functionDeclaration: string, {throwOnSideEffect}: {throwOnSideEffect: boolean}): Promise<string> {
   const selectedNode = UI.Context.Context.instance().flavor(SDK.DOMModel.DOMNode);
   const target = selectedNode?.domModel().target() ?? UI.Context.Context.instance().flavor(SDK.Target.Target);
 
@@ -164,8 +164,25 @@ async function executeJsCode(code: string, {throwOnSideEffect}: {throwOnSideEffe
     throw new Error('Execution context is not found for executing code');
   }
 
+  if (executionContext.debuggerModel.selectedCallFrame()) {
+    throw new ExecutionError('Cannot evaluate JavaScript because the execution is paused on a breakpoint.');
+  }
+
+  const result = await executionContext.evaluate(
+      {
+        expression: '$0',
+        returnByValue: false,
+        includeCommandLineAPI: true,
+      },
+      false, false);
+
+  if ('error' in result) {
+    throw new ExecutionError('Cannot find $0');
+  }
+
   try {
-    return await FreestylerEvaluateAction.execute(code, executionContext, {throwOnSideEffect});
+    return await FreestylerEvaluateAction.execute(
+        functionDeclaration, [result.object], executionContext, {throwOnSideEffect});
   } catch (err) {
     if (err instanceof ExecutionError) {
       return `Error: ${err.message}`;
@@ -175,7 +192,6 @@ async function executeJsCode(code: string, {throwOnSideEffect}: {throwOnSideEffe
   }
 }
 
-const MAX_STEPS = 10;
 const MAX_OBSERVATION_BYTE_LENGTH = 25_000;
 
 type CreateExtensionScopeFunction = (changes: ChangeManager) => {
@@ -195,7 +211,9 @@ type AgentOptions = {
  * One agent instance handles one conversation. Create a new agent
  * instance for a new conversation.
  */
-export class FreestylerAgent extends AiAgent {
+export class FreestylerAgent extends AiAgent<SDK.DOMModel.DOMNode> {
+  override type = AgentType.FREESTYLER;
+
   readonly preamble = preamble;
   readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_FREESTYLER;
   get userTier(): string|undefined {
@@ -211,27 +229,19 @@ export class FreestylerAgent extends AiAgent {
 
   get options(): AidaRequestOptions {
     const config = Common.Settings.Settings.instance().getHostConfig();
-    const temperature = AiAgent.validTemperature(config.devToolsFreestyler?.temperature);
+    const temperature = config.devToolsFreestyler?.temperature;
     const modelId = config.devToolsFreestyler?.modelId;
 
     return {
       temperature,
-      model_id: modelId,
+      modelId,
     };
   }
 
-  static parseResponse(response: string): {
-    thought?: string,
-    title?: string,
-    action?: string,
-  }|{
-  answer:
-    string, suggestions: string[],
-  }
-  {
+  override parseResponse(response: string): ParsedResponse {
     // We're returning an empty answer to denote the erroneous case.
     if (!response) {
-      return {answer: '', suggestions: []};
+      return {answer: ''};
     }
 
     const lines = response.split('\n');
@@ -239,7 +249,7 @@ export class FreestylerAgent extends AiAgent {
     let title: string|undefined;
     let action: string|undefined;
     let answer: string|undefined;
-    let suggestions: string[] = [];
+    let suggestions: [string, ...string[]]|undefined;
     let i = 0;
 
     // If one of these is present, it means we're going to follow the instruction tags
@@ -262,7 +272,7 @@ export class FreestylerAgent extends AiAgent {
     // The block below ensures that the response we parse always contains a defining instruction tag.
     const hasDefiningInstruction = lines.some(line => isDefiningInstructionStart(line));
     if (!hasDefiningInstruction) {
-      return FreestylerAgent.parseResponse(`ANSWER: ${response}`);
+      return this.parseResponse(`ANSWER: ${response}`);
     }
 
     while (i < lines.length) {
@@ -328,8 +338,7 @@ export class FreestylerAgent extends AiAgent {
         try {
           // TODO: Do basic validation this is an array with strings
           suggestions = JSON.parse(trimmed.substring('SUGGESTIONS:'.length).trim());
-        } catch (err) {
-          suggestions = [];
+        } catch {
         }
 
         i++;
@@ -407,12 +416,10 @@ export class FreestylerAgent extends AiAgent {
     sideEffect: boolean,
     canceled: boolean,
   }> {
-    const actionExpression = `{
-      const scope = {$0, $1, getEventListeners};
-      with (scope) {
-        ${action}
-        ;((typeof data !== "undefined") ? data : undefined)
-      }
+    const functionDeclaration = `async function ($0) {
+      ${action}
+      ;
+      return ((typeof data !== "undefined") ? data : undefined);
     }`;
     try {
       const runConfirmed = await confirm ?? Promise.resolve(true);
@@ -424,7 +431,7 @@ export class FreestylerAgent extends AiAgent {
         };
       }
       const result = await this.#execJs(
-          actionExpression,
+          functionDeclaration,
           {throwOnSideEffect},
       );
       const byteCount = Platform.StringUtilities.countWtf8Bytes(result);
@@ -530,23 +537,25 @@ export class FreestylerAgent extends AiAgent {
       }
     }
 
-    return output;
+    return output.trim();
   }
 
-  async * handleAction(action: string, rpcId?: number): AsyncGenerator<SideEffectResponse, ActionResponse, void> {
+  override async *
+      handleAction(action: string, rpcId?: number): AsyncGenerator<SideEffectResponse, ActionResponse, void> {
     debugLog(`Action to execute: ${action}`);
+    if (this.executionMode === Root.Runtime.HostConfigFreestylerExecutionMode.NO_SCRIPTS) {
+      return {
+        type: ResponseType.ACTION,
+        code: action,
+        output: 'Error: JavaScript execution is currently disabled.',
+        canceled: true,
+        rpcId,
+      };
+    }
+
     const scope = this.#createExtensionScope(this.#changes);
     await scope.install();
     try {
-      if (this.executionMode === Root.Runtime.HostConfigFreestylerExecutionMode.NO_SCRIPTS) {
-        return {
-          type: ResponseType.ACTION,
-          code: action,
-          output: 'Error: JavaScript execution is currently disabled.',
-          canceled: true,
-          rpcId,
-        };
-      }
       let result = await this.#generateObservation(action, {throwOnSideEffect: true});
       debugLog(`Action result: ${JSON.stringify(result)}`);
       if (result.sideEffect) {
@@ -569,7 +578,13 @@ export class FreestylerAgent extends AiAgent {
         yield {
           type: ResponseType.SIDE_EFFECT,
           code: action,
-          confirm: sideEffectConfirmationPromiseWithResolvers.resolve,
+          confirm: (result: boolean) => {
+            sideEffectConfirmationPromiseWithResolvers.resolve(result);
+            Host.userMetrics.actionTaken(
+                result ? Host.UserMetrics.Action.AiAssistanceSideEffectConfirmed :
+                         Host.UserMetrics.Action.AiAssistanceSideEffectRejected,
+            );
+          },
           rpcId,
         };
 
@@ -590,162 +605,29 @@ export class FreestylerAgent extends AiAgent {
     }
   }
 
-  async *
-      handleContextDetails(selectedElement: SDK.DOMModel.DOMNode|null):
-          AsyncGenerator<ThoughtResponse|TitleResponse, void, void> {
+  override async *
+      handleContextDetails(selectedElement: SDK.DOMModel.DOMNode|null): AsyncGenerator<ContextResponse, void, void> {
     if (!selectedElement) {
       return;
     }
-
     yield {
-      type: ResponseType.TITLE,
+      type: ResponseType.CONTEXT,
       title: lockedString(UIStringsNotTranslate.analyzingThePrompt),
-    };
-    yield {
-      type: ResponseType.THOUGHT,
-      contextDetails: [{
+      details: [{
         title: lockedString(UIStringsNotTranslate.dataUsed),
         text: await FreestylerAgent.describeElement(selectedElement),
       }],
     };
   }
 
-  async enhanceQuery(query: string, selectedElement: SDK.DOMModel.DOMNode|null): Promise<string> {
+  override async enhanceQuery(query: string, selectedElement: SDK.DOMModel.DOMNode|null): Promise<string> {
     const elementEnchantmentQuery = selectedElement ?
         `# Inspected element\n\n${await FreestylerAgent.describeElement(selectedElement)}\n\n# User request\n\n` :
         '';
     return `${elementEnchantmentQuery}QUERY: ${query}`;
   }
 
-  #runId = 0;
-  async * run(query: string, options: {
-    signal?: AbortSignal, selectedElement: SDK.DOMModel.DOMNode|null,
-  }): AsyncGenerator<ResponseData, void, void> {
-    yield* this.handleContextDetails(options.selectedElement);
-
-    query = await this.enhanceQuery(query, options.selectedElement);
-    const currentRunId = ++this.#runId;
-    Host.userMetrics.freestylerQueryLength(query.length);
-    for (let i = 0; i < MAX_STEPS; i++) {
-      yield {
-        type: ResponseType.QUERYING,
-      };
-
-      let response: string;
-      let rpcId: number|undefined;
-      try {
-        const fetchResult = await this.aidaFetch(
-            query,
-            {signal: options.signal},
-        );
-        response = fetchResult.response;
-        rpcId = fetchResult.rpcId;
-      } catch (err) {
-        debugLog('Error calling the AIDA API', err);
-
-        if (err instanceof Host.AidaClient.AidaAbortError) {
-          this.removeHistoryRun(currentRunId);
-          yield {
-            type: ResponseType.ERROR,
-            error: ErrorType.ABORT,
-            rpcId,
-          };
-          break;
-        }
-
-        yield {
-          type: ResponseType.ERROR,
-          error: ErrorType.UNKNOWN,
-          rpcId,
-        };
-        break;
-      }
-
-      const parsedResponse = FreestylerAgent.parseResponse(response);
-
-      if ('answer' in parsedResponse) {
-        const {
-          answer,
-          suggestions,
-        } = parsedResponse;
-        if (answer) {
-          this.addToHistory({
-            id: currentRunId,
-            query,
-            output: `ANSWER: ${answer}`,
-          });
-          yield {
-            type: ResponseType.ANSWER,
-            text: answer,
-            rpcId,
-            suggestions,
-          };
-        } else {
-          yield {
-            type: ResponseType.ERROR,
-            error: ErrorType.UNKNOWN,
-            rpcId,
-          };
-        }
-
-        break;
-      }
-
-      const {
-        title,
-        thought,
-        action,
-      } = parsedResponse;
-
-      if (title) {
-        yield {
-          type: ResponseType.TITLE,
-          title,
-          rpcId,
-        };
-      }
-
-      if (thought) {
-        this.addToHistory({
-          id: currentRunId,
-          query,
-          output: `THOUGHT: ${thought}
-TITLE: ${title}
-ACTION
-${action}
-STOP`,
-        });
-        yield {
-          type: ResponseType.THOUGHT,
-          thought,
-          rpcId,
-        };
-      } else {
-        this.addToHistory({
-          id: currentRunId,
-          query,
-          output: `ACTION
-${action}
-STOP`,
-        });
-      }
-
-      if (action) {
-        const result = yield* this.handleAction(action, rpcId);
-        yield result;
-        query = `OBSERVATION: ${result.output}`;
-      }
-
-      if (i === MAX_STEPS - 1) {
-        yield {
-          type: ResponseType.ERROR,
-          error: ErrorType.MAX_STEPS,
-        };
-        break;
-      }
-    }
-    if (isDebugMode()) {
-      window.dispatchEvent(new CustomEvent('freestylerdone'));
-    }
+  override formatHistoryChunkAnswer(text: string): string {
+    return `ANSWER: ${text}`;
   }
 }

package/front_end/panels/freestyler/FreestylerEvaluateAction.test.ts

@@ -19,9 +19,9 @@ describe('FreestylerEvaluateAction', () => {
           return sinon.createStubInstance(SDK.DebuggerModel.CallFrame);
         };
       }
-      executionContextStub.evaluate.resolves(mockResult);
+      executionContextStub.callFunctionOn.resolves(mockResult);
       executionContextStub.runtimeModel = sinon.createStubInstance(SDK.RuntimeModel.RuntimeModel);
-      return Freestyler.FreestylerEvaluateAction.execute('', executionContextStub, {throwOnSideEffect: false});
+      return Freestyler.FreestylerEvaluateAction.execute('', [], executionContextStub, {throwOnSideEffect: false});
     }
 
     function mockRemoteObject(overrides: Partial<SDK.RemoteObject.RemoteObject> = {}): SDK.RemoteObject.RemoteObject {
@@ -106,99 +106,103 @@ describe('FreestylerEvaluateAction', () => {
     }
 
     async function executeForTest(code: string) {
-      const actionExpression = `const scope = {$0, $1, getEventListeners}; with (scope) {${code}}`;
+      const actionExpression = `async function ($0) {
+        ${code}
+        ;
+        return ((typeof data !== "undefined") ? data : undefined);
+      }`;
       return Freestyler.FreestylerEvaluateAction.execute(
-          actionExpression, await executionContextForTest(), {throwOnSideEffect: false});
+          actionExpression, [], await executionContextForTest(), {throwOnSideEffect: false});
     }
 
     it('should serialize primitive values correctly', async () => {
-      assert.strictEqual(await executeForTest('"string"'), '\'string\'');
-      assert.strictEqual(await executeForTest('999n'), '999n');
-      assert.strictEqual(await executeForTest('true'), 'true');
-      assert.strictEqual(await executeForTest('undefined'), 'undefined');
-      assert.strictEqual(await executeForTest('42'), '42');
-      assert.strictEqual(await executeForTest('Symbol("sym")'), 'Symbol(sym)');
+      assert.strictEqual(await executeForTest('const data = "string"'), '\'string\'');
+      assert.strictEqual(await executeForTest('const data = 999n'), '999n');
+      assert.strictEqual(await executeForTest('const data = true'), 'true');
+      assert.strictEqual(await executeForTest('const data = undefined'), 'undefined');
+      assert.strictEqual(await executeForTest('const data = 42'), '42');
+      assert.strictEqual(await executeForTest('const data = Symbol("sym")'), 'Symbol(sym)');
     });
 
     describe('HTMLElement', () => {
       it('should work with plain nodes', async () => {
-        const serializedElement = await executeForTest(`{
+        const serializedElement = await executeForTest(`
           const el = document.createElement('div');
 
-          el;
-        }`);
+          const data = el;
+        `);
         assert.strictEqual(serializedElement, '"<div></div>"');
       });
 
       it('should serialize node with classes', async () => {
-        const serializedElement = await executeForTest(`{
+        const serializedElement = await executeForTest(`
           const el = document.createElement('div');
           el.classList.add('section');
           el.classList.add('section-main');
 
-          el;
-        }`);
+          const data = el;
+        `);
         assert.strictEqual(serializedElement, '"<div class=\\"section section-main\\"></div>"');
       });
 
       it('should serialize node with id', async () => {
-        const serializedElement = await executeForTest(`{
+        const serializedElement = await executeForTest(`
           const el = document.createElement('div');
           el.id = 'promotion-section';
 
-          el;
-        }`);
+          const data = el;
+        `);
         assert.strictEqual(serializedElement, '"<div id=\\"promotion-section\\"></div>"');
       });
       it('should serialize node with class and id', async () => {
-        const serializedElement = await executeForTest(`{
+        const serializedElement = await executeForTest(`
           const el = document.createElement('div');
           el.id = 'promotion-section';
           el.classList.add('section');
 
-          el;
-        }`);
+          const data = el;
+        `);
         assert.strictEqual(serializedElement, '"<div id=\\"promotion-section\\" class=\\"section\\"></div>"');
       });
       it('should serialize node with children', async () => {
-        const serializedElement = await executeForTest(`{
+        const serializedElement = await executeForTest(`
           const el = document.createElement('div');
           const p = document.createElement('p');
           el.appendChild(p);
 
-          el;
-        }`);
+          const data = el;
+        `);
         assert.strictEqual(serializedElement, '"<div>...</div>"');
       });
     });
 
     it('should serialize arrays correctly', async () => {
-      assert.strictEqual(await executeForTest('[]'), '[]');
-      assert.strictEqual(await executeForTest('[1]'), '[1]');
-      assert.strictEqual(await executeForTest('[1, 2]'), '[1,2]');
-      assert.strictEqual(await executeForTest('[{key: 1}]'), '[{"key":1}]');
+      assert.strictEqual(await executeForTest('const data = []'), '[]');
+      assert.strictEqual(await executeForTest('const data = [1]'), '[1]');
+      assert.strictEqual(await executeForTest('const data = [1, 2]'), '[1,2]');
+      assert.strictEqual(await executeForTest('const data = [{key: 1}]'), '[{"key":1}]');
     });
 
     it('should serialize objects correctly', async () => {
-      assert.strictEqual(await executeForTest('const object = {key: "str"}; object;'), '{"key":"str"}');
+      assert.strictEqual(await executeForTest('const object = {key: "str"}; const data = object;'), '{"key":"str"}');
       assert.strictEqual(
-          await executeForTest('const object = {key: "str", secondKey: "str2"}; object;'),
+          await executeForTest('const object = {key: "str", secondKey: "str2"}; const data = object;'),
           '{"key":"str","secondKey":"str2"}');
-      assert.strictEqual(await executeForTest('const object = {key: 1}; object;'), '{"key":1}');
+      assert.strictEqual(await executeForTest('const object = {key: 1}; const data = object;'), '{"key":1}');
     });
 
     it('should not continue serializing cycles', async () => {
       assert.strictEqual(
-          await executeForTest(`{
+          await executeForTest(`
         const obj = { a: 1 };
         obj.itself = obj;
-        obj;
-      }`),
+        const data = obj;
+      `),
           '{"a":1,"itself":"(cycle)"}');
     });
 
     it('should not include number keys for CSSStyleDeclaration', async () => {
-      const result = await executeForTest('getComputedStyle(document.body)');
+      const result = await executeForTest('const data = getComputedStyle(document.body)');
       const parsedResult = JSON.parse(result);
       assert.isUndefined(parsedResult[0]);
     });

package/front_end/panels/freestyler/FreestylerEvaluateAction.ts

@@ -70,23 +70,24 @@ export interface Options {
   throwOnSideEffect: boolean;
 }
 export class FreestylerEvaluateAction {
-  static async execute(code: string, executionContext: SDK.RuntimeModel.ExecutionContext, {throwOnSideEffect}: Options):
-      Promise<string> {
+  static async execute(
+      functionDeclaration: string, args: Array<SDK.RemoteObject.RemoteObject>,
+      executionContext: SDK.RuntimeModel.ExecutionContext, {throwOnSideEffect}: Options): Promise<string> {
     if (executionContext.debuggerModel.selectedCallFrame()) {
       throw new ExecutionError('Cannot evaluate JavaScript because the execution is paused on a breakpoint.');
     }
-    const response = await executionContext.evaluate(
-        {
-          expression: code,
-          replMode: true,
-          includeCommandLineAPI: true,
-          returnByValue: false,
-          silent: false,
-          generatePreview: true,
-          allowUnsafeEvalBlockedByCSP: false,
-          throwOnSideEffect,
-        },
-        /* userGesture */ false, /* awaitPromise */ true);
+    const response = await executionContext.callFunctionOn({
+      functionDeclaration,
+      includeCommandLineAPI: false,
+      returnByValue: false,
+      allowUnsafeEvalBlockedByCSP: false,
+      throwOnSideEffect,
+      userGesture: true,
+      awaitPromise: true,
+      arguments: args.map(remoteObject => {
+        return {objectId: remoteObject.objectId};
+      }),
+    });
 
     try {
       if (!response) {