chrome-devtools-frontend 1.0.1642845 → 1.0.1643099

package/front_end/models/ai_assistance/agents/StylingAgent.ts

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 import * as Host from '../../../core/host/host.js';
-import * as i18n from '../../../core/i18n/i18n.js';
 import * as Root from '../../../core/root/root.js';
 import * as SDK from '../../../core/sdk/sdk.js';
 import * as Protocol from '../../../generated/protocol.js';
@@ -11,16 +10,15 @@ import * as Greendev from '../../../models/greendev/greendev.js';
 import * as Annotations from '../../annotations/annotations.js';
 import * as Emulation from '../../emulation/emulation.js';
 import {ChangeManager} from '../ChangeManager.js';
-import {debugLog} from '../debug.js';
 import {ExtensionScope} from '../ExtensionScope.js';
 import {AI_ASSISTANCE_CSS_CLASS_NAME} from '../injected.js';
+import {ToolName} from '../tools/Tool.js';
+import {ToolRegistry} from '../tools/ToolRegistry.js';
 
 import {
   AiAgent,
-  type ComputedStyleAiWidget,
   type ContextResponse,
-  ConversationContext,
-  type ConversationSuggestions,
+  type ConversationContext,
   type FunctionCallHandlerResult,
   type MultimodalInput,
   MultimodalInputType,
@@ -29,24 +27,10 @@ import {
 } from './AiAgent.js';
 import {
   type CreateExtensionScopeFunction,
-  executeJavaScriptFunction,
   type ExecuteJsAgentOptions,
   executeJsCode,
-  JavascriptExecutor
 } from './ExecuteJavascript.js';
 
-/*
-* Strings that don't need to be translated at this time.
-*/
-const UIStringsNotTranslate = {
-  /**
-   * @description Heading text for context details of Freestyler agent.
-   */
-  dataUsed: 'Data used',
-} as const;
-
-const lockedString = i18n.i18n.lockedString;
-
 const preamble = `You are the most advanced CSS/DOM/HTML debugging assistant integrated into Chrome DevTools.
 You always suggest considering the best web development practices and the newest platform features such as view transitions.
 The user selected a DOM element in the browser's DevTools and sends a query about the page or the selected DOM element.
@@ -152,78 +136,6 @@ const MULTIMODAL_ENHANCEMENT_PROMPTS: Record<MultimodalInputType, string> = {
 
 export const AI_ASSISTANCE_FILTER_REGEX = `\\.${AI_ASSISTANCE_CSS_CLASS_NAME}-.*&`;
 
-export class NodeContext extends ConversationContext<SDK.DOMModel.DOMNode> {
-  #node: SDK.DOMModel.DOMNode;
-
-  constructor(node: SDK.DOMModel.DOMNode) {
-    super();
-    this.#node = node;
-  }
-
-  override getURL(): string {
-    const ownerDocument = this.#node.ownerDocument;
-    if (!ownerDocument) {
-      // The node is detached from a document.
-      return 'detached';
-    }
-    return ownerDocument.documentURL;
-  }
-
-  getItem(): SDK.DOMModel.DOMNode {
-    return this.#node;
-  }
-
-  override getTitle(): string {
-    throw new Error('Not implemented');
-  }
-
-  override async getSuggestions(): Promise<ConversationSuggestions|undefined> {
-    const layoutProps = await this.#node.domModel().cssModel().getLayoutPropertiesFromComputedStyle(this.#node.id);
-
-    if (!layoutProps) {
-      return;
-    }
-
-    if (layoutProps.isFlex) {
-      return [
-        {title: 'How can I make flex items wrap?', jslogContext: 'flex-wrap'},
-        {title: 'How do I distribute flex items evenly?', jslogContext: 'flex-distribute'},
-        {title: 'What is flexbox?', jslogContext: 'flex-what'},
-      ];
-    }
-    if (layoutProps.isSubgrid) {
-      return [
-        {title: 'Where is this grid defined?', jslogContext: 'subgrid-where'},
-        {title: 'How to overwrite parent grid properties?', jslogContext: 'subgrid-override'},
-        {title: 'How do subgrids work? ', jslogContext: 'subgrid-how'},
-      ];
-    }
-    if (layoutProps.isGrid) {
-      return [
-        {title: 'How do I align items in a grid?', jslogContext: 'grid-align'},
-        {title: 'How to add spacing between grid items?', jslogContext: 'grid-gap'},
-        {title: 'How does grid layout work?', jslogContext: 'grid-how'},
-      ];
-    }
-    if (layoutProps.hasScroll) {
-      return [
-        {title: 'How do I remove scrollbars for this element?', jslogContext: 'scroll-remove'},
-        {title: 'How can I style a scrollbar?', jslogContext: 'scroll-style'},
-        {title: 'Why does this element scroll?', jslogContext: 'scroll-why'},
-      ];
-    }
-    if (layoutProps.containerType) {
-      return [
-        {title: 'What are container queries?', jslogContext: 'container-what'},
-        {title: 'How do I use container-type?', jslogContext: 'container-how'},
-        {title: 'What\'s the container context for this element?', jslogContext: 'container-context'},
-      ];
-    }
-
-    return;
-  }
-}
-
 /**
  * One agent instance handles one conversation. Create a new agent
  * instance for a new conversation.
@@ -255,88 +167,55 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
   }
 
   #execJs: typeof executeJsCode;
-  #javascriptExecutor: JavascriptExecutor;
 
   #changes: ChangeManager;
   #createExtensionScope: CreateExtensionScopeFunction;
   #greenDevEmulationScreenshot: string|null = null;
   #greenDevEmulationAxTree: string|null = null;
   #hasAddedEmulationInstructions = false;
-  #currentTurnId = 0;
 
   constructor(opts: ExecuteJsAgentOptions) {
     super(opts);
 
     this.#changes = opts.changeManager || new ChangeManager();
     this.#execJs = opts.execJs ?? executeJsCode;
-    this.#createExtensionScope =
-        opts.createExtensionScope ?? ((changes: ChangeManager) => {
-          return new ExtensionScope(changes, this.sessionId, this.context?.getItem() ?? null, this.#currentTurnId);
-        });
-    this.#javascriptExecutor = new JavascriptExecutor(
-        {
-          executionMode: this.executionMode,
-          getContextNode: () => this.#getSelectedNode(),
-          createExtensionScope: this.#createExtensionScope.bind(this),
-          changes: this.#changes,
-        },
-        this.#execJs);
-
-    this.declareFunction<{
-      elements: number[],
-      styleProperties: string[],
-      explanation: string,
-    }>('getStyles', {
-      description:
-          `Get computed and source styles for one or multiple elements on the inspected page for multiple elements at once by uid.
+    this.#createExtensionScope = opts.createExtensionScope ?? ((changes: ChangeManager) => {
+                                   return new ExtensionScope(changes, this.sessionId, this.context?.getItem() ?? null);
+                                 });
+
+    const getStylesTool = ToolRegistry.get(ToolName.GET_STYLES);
+    if (!getStylesTool) {
+      throw new Error('Required tool "getStyles" not found');
+    }
+    this.declareFunction(ToolName.GET_STYLES, {
+      description: getStylesTool.description,
+      parameters: getStylesTool.parameters,
+      displayInfoFromArgs: getStylesTool.displayInfoFromArgs,
+      handler: args => getStylesTool.handler(args, {
+        conversationContext: this.context ?? null,
+      }),
+    });
 
-**CRITICAL** An element uid is a number, not a selector.
-**CRITICAL** Use selectors to refer to elements in the text output. Do not use uids.
-**CRITICAL** Always provide the explanation argument to explain what and why you query.
-**CRITICAL** You MUST provide a specific list of CSS property names. Do not use generic values like "all" or "*".`,
-      parameters: {
-        type: Host.AidaClient.ParametersTypes.OBJECT,
-        description: '',
-        nullable: false,
-        properties: {
-          explanation: {
-            type: Host.AidaClient.ParametersTypes.STRING,
-            description: 'Explain why you want to get styles',
-            nullable: false,
-          },
-          elements: {
-            type: Host.AidaClient.ParametersTypes.ARRAY,
-            description: 'A list of element uids to get data for. These are numbers, not selectors.',
-            items: {type: Host.AidaClient.ParametersTypes.INTEGER, description: `An element uid.`},
-            nullable: false,
+    const executeJsTool = ToolRegistry.get(ToolName.EXECUTE_JAVASCRIPT);
+    if (!executeJsTool) {
+      throw new Error('Required tool "executeJavaScript" not found');
+    }
+    this.declareFunction(ToolName.EXECUTE_JAVASCRIPT, {
+      description: executeJsTool.description,
+      parameters: executeJsTool.parameters,
+      displayInfoFromArgs: executeJsTool.displayInfoFromArgs,
+      handler: (args, options) => executeJsTool.handler(
+          args,
+          {
+            conversationContext: this.context ?? null,
+            changeManager: this.#changes,
+            createExtensionScope: this.#createExtensionScope.bind(this),
+            execJs: this.#execJs,
           },
-          styleProperties: {
-            type: Host.AidaClient.ParametersTypes.ARRAY,
-            description:
-                'One or more specific CSS style property names to fetch. Generic values like "all" or "*" are not supported.',
-            nullable: false,
-            items: {
-              type: Host.AidaClient.ParametersTypes.STRING,
-              description: 'A CSS style property name to retrieve. For example, \'background-color\'.'
-            }
-          },
-        },
-        required: ['explanation', 'elements', 'styleProperties']
-      },
-      displayInfoFromArgs: params => {
-        return {
-          title: 'Reading computed and source styles',
-          thought: params.explanation,
-          action: `getStyles(${JSON.stringify(params.elements)}, ${JSON.stringify(params.styleProperties)})`,
-        };
-      },
-      handler: async params => {
-        return await this.#getStyles(params.elements, params.styleProperties);
-      },
+          options,
+          ),
     });
 
-    this.declareFunction('executeJavaScript', executeJavaScriptFunction(this.#javascriptExecutor));
-
     if (Annotations.AnnotationRepository.annotationsEnabled()) {
       this.declareFunction<{
         elementId: string,
@@ -401,162 +280,10 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
     });
   }
 
-  static async describeElement(element: SDK.DOMModel.DOMNode): Promise<string> {
-    let output = `* Element's uid is ${element.backendNodeId()}.
-* Its selector is \`${element.simpleSelector()}\``;
-    const childNodes = await element.getChildNodesPromise();
-    if (childNodes) {
-      const textChildNodes = childNodes.filter(childNode => childNode.nodeType() === Node.TEXT_NODE);
-      const elementChildNodes = childNodes.filter(childNode => childNode.nodeType() === Node.ELEMENT_NODE);
-      switch (elementChildNodes.length) {
-        case 0:
-          output += '\n* It doesn\'t have any child element nodes';
-          break;
-        case 1:
-          output += `\n* It only has 1 child element node: \`${elementChildNodes[0].simpleSelector()}\``;
-          break;
-        default:
-          output += `\n* It has ${elementChildNodes.length} child element nodes: ${
-              elementChildNodes.map(node => `\`${node.simpleSelector()}\` (uid=${node.backendNodeId()})`).join(', ')}`;
-      }
-
-      switch (textChildNodes.length) {
-        case 0:
-          output += '\n* It doesn\'t have any child text nodes';
-          break;
-        case 1:
-          output += '\n* It only has 1 child text node';
-          break;
-        default:
-          output += `\n* It has ${textChildNodes.length} child text nodes`;
-      }
-    }
-
-    if (element.nextSibling) {
-      const elementOrNodeElementNodeText = element.nextSibling.nodeType() === Node.ELEMENT_NODE ?
-          `an element (uid=${element.nextSibling.backendNodeId()})` :
-          'a non element';
-      output += `\n* It has a next sibling and it is ${elementOrNodeElementNodeText} node`;
-    }
-
-    if (element.previousSibling) {
-      const elementOrNodeElementNodeText = element.previousSibling.nodeType() === Node.ELEMENT_NODE ?
-          `an element (uid=${element.previousSibling.backendNodeId()})` :
-          'a non element';
-      output += `\n* It has a previous sibling and it is ${elementOrNodeElementNodeText} node`;
-    }
-
-    if (element.isInShadowTree()) {
-      output += '\n* It is in a shadow DOM tree.';
-    }
-
-    const parentNode = element.parentNode;
-    if (parentNode) {
-      const parentChildrenNodes = await parentNode.getChildNodesPromise();
-      output += `\n* Its parent's selector is \`${parentNode.simpleSelector()}\` (uid=${parentNode.backendNodeId()})`;
-      const elementOrNodeElementNodeText = parentNode.nodeType() === Node.ELEMENT_NODE ? 'an element' : 'a non element';
-      output += `\n* Its parent is ${elementOrNodeElementNodeText} node`;
-      if (parentNode.isShadowRoot()) {
-        output += '\n* Its parent is a shadow root.';
-      }
-      if (parentChildrenNodes) {
-        const childElementNodes =
-            parentChildrenNodes.filter(siblingNode => siblingNode.nodeType() === Node.ELEMENT_NODE);
-        switch (childElementNodes.length) {
-          case 0:
-            break;
-          case 1:
-            output += '\n* Its parent has only 1 child element node';
-            break;
-          default:
-            output += `\n* Its parent has ${childElementNodes.length} child element nodes: ${
-                childElementNodes.map(node => `\`${node.simpleSelector()}\` (uid=${node.backendNodeId()})`)
-                    .join(', ')}`;
-            break;
-        }
-
-        const siblingTextNodes = parentChildrenNodes.filter(siblingNode => siblingNode.nodeType() === Node.TEXT_NODE);
-        switch (siblingTextNodes.length) {
-          case 0:
-            break;
-          case 1:
-            output += '\n* Its parent has only 1 child text node';
-            break;
-          default:
-            output += `\n* Its parent has ${siblingTextNodes.length} child text nodes: ${
-                siblingTextNodes.map(node => `\`${node.simpleSelector()}\``).join(', ')}`;
-            break;
-        }
-      }
-    }
-
-    return output.trim();
-  }
-
   #getSelectedNode(): SDK.DOMModel.DOMNode|null {
     return this.context?.getItem() ?? null;
   }
 
-  async #getStyles(elements: number[], properties: string[]): Promise<FunctionCallHandlerResult<unknown>> {
-    const widgets: ComputedStyleAiWidget[] = [];
-
-    const result:
-        Record<string, {computed: Record<string, string|undefined>, authored: Record<string, string|undefined>}> = {};
-    for (const uid of elements) {
-      result[uid] = {computed: {}, authored: {}};
-      debugLog(`Action to execute: uid=${uid}`);
-      const selectedNode = this.#getSelectedNode();
-      if (!selectedNode) {
-        return {error: 'Error: Could not find the currently selected element.'};
-      }
-      const node = new SDK.DOMModel.DeferredDOMNode(
-          selectedNode.domModel().target(), Number(uid) as unknown as Protocol.DOM.BackendNodeId);
-      const resolved = await node.resolvePromise();
-      if (!resolved) {
-        return {error: 'Error: Could not find the element with uid=' + uid};
-      }
-      const newContext = new NodeContext(resolved);
-      if (this.context?.getOrigin() !== newContext.getOrigin()) {
-        return {error: 'Error: Node does not belong to the current origin.'};
-      }
-      const styles = await resolved.domModel().cssModel().getComputedStyle(resolved.id);
-      if (!styles) {
-        return {error: 'Error: Could not get computed styles.'};
-      }
-      const matchedStyles = await resolved.domModel().cssModel().getMatchedStyles(resolved.id);
-      if (!matchedStyles) {
-        return {error: 'Error: Could not get authored styles.'};
-      }
-      widgets.push({
-        name: 'COMPUTED_STYLES',
-        data: {
-          computedStyles: styles,
-          backendNodeId: node.backendNodeId(),
-          matchedCascade: matchedStyles,
-          properties,
-        }
-      });
-      for (const prop of properties) {
-        result[uid].computed[prop] = styles.get(prop);
-      }
-      for (const style of matchedStyles.nodeStyles()) {
-        for (const property of style.allProperties()) {
-          if (!properties.includes(property.name)) {
-            continue;
-          }
-          const state = matchedStyles.propertyState(property);
-          if (state === SDK.CSSMatchedStyles.PropertyState.ACTIVE) {
-            result[uid].authored[property.name] = property.value;
-          }
-        }
-      }
-    }
-    return {
-      result: JSON.stringify(result, null, 2),
-      widgets,
-    };
-  }
-
   async addElementAnnotation(elementId: string, annotationMessage: string):
       Promise<FunctionCallHandlerResult<unknown>> {
     if (!Annotations.AnnotationRepository.annotationsEnabled()) {
@@ -782,20 +509,15 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
   override async *
       handleContextDetails(selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null):
           AsyncGenerator<ContextResponse, void, void> {
-    if (!selectedElement) {
-      return;
+    if (selectedElement) {
+      const details = await selectedElement.getUserFacingDetails();
+      if (details) {
+        yield {
+          type: ResponseType.CONTEXT,
+          details,
+        };
+      }
     }
-    yield {
-      type: ResponseType.CONTEXT,
-      details: [{
-        title: lockedString(UIStringsNotTranslate.dataUsed),
-        text: await StylingAgent.describeElement(selectedElement.getItem()),
-      }],
-    };
-  }
-
-  protected override async preRun(): Promise<void> {
-    this.#currentTurnId++;
   }
 
   override async enhanceQuery(
@@ -814,10 +536,8 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
       this.#hasAddedEmulationInstructions = true;
     }
 
-    const elementEnchancementQuery = selectedElement ?
-        `# Inspected element\n\n${
-            await StylingAgent.describeElement(selectedElement.getItem())}\n\n# User request\n\n` :
-        '';
+    const promptDetails = selectedElement ? await selectedElement.getPromptDetails() : null;
+    const elementEnchancementQuery = promptDetails ? `${promptDetails}\n\n# User request\n\n` : '';
     return `${multimodalInputEnhancementQuery}${elementEnchancementQuery}QUERY: ${query}`;
   }
 }

package/front_end/models/ai_assistance/ai_assistance.ts

@@ -6,7 +6,6 @@ import * as AgentProject from './AgentProject.js';
 import * as AccessibilityAgent from './agents/AccessibilityAgent.js';
 import * as AiAgent from './agents/AiAgent.js';
 import * as ContextSelectionAgent from './agents/ContextSelectionAgent.js';
-import * as ConversationSummaryAgent from './agents/ConversationSummaryAgent.js';
 import * as FileAgent from './agents/FileAgent.js';
 import * as GreenDevAgent from './agents/GreenDevAgent.js';
 import * as GreenDevAgentAntigravityCliSocketClient from './agents/GreenDevAgentAntigravityCliSocketClient.js';
@@ -14,7 +13,6 @@ import * as GreenDevAgentGeminiCliSocketClient from './agents/GreenDevAgentGemin
 import * as NetworkAgent from './agents/NetworkAgent.js';
 import * as PatchAgent from './agents/PatchAgent.js';
 import * as PerformanceAgent from './agents/PerformanceAgent.js';
-import * as PerformanceAnnotationsAgent from './agents/PerformanceAnnotationsAgent.js';
 import * as StorageAgent from './agents/StorageAgent.js';
 import * as StylingAgent from './agents/StylingAgent.js';
 import * as AiAgent2 from './AiAgent2.js';
@@ -24,6 +22,8 @@ import * as AiOrigins from './AiOrigins.js';
 import * as AiUtils from './AiUtils.js';
 import * as BuiltInAi from './BuiltInAi.js';
 import * as ChangeManager from './ChangeManager.js';
+import * as DOMNodeContext from './contexts/DOMNodeContext.js';
+import * as ConversationSummary from './ConversationSummary.js';
 import * as FileFormatter from './data_formatters/FileFormatter.js';
 import * as LighthouseFormatter from './data_formatters/LighthouseFormatter.js';
 import * as NetworkRequestFormatter from './data_formatters/NetworkRequestFormatter.js';
@@ -37,7 +37,12 @@ import * as Injected from './injected.js';
 import * as AICallTree from './performance/AICallTree.js';
 import * as AIContext from './performance/AIContext.js';
 import * as AIQueries from './performance/AIQueries.js';
+import * as PerformanceAnnotations from './PerformanceAnnotations.js';
 import * as StorageItem from './StorageItem.js';
+import * as ExecuteJavaScript from './tools/ExecuteJavaScript.js';
+import * as GetStyles from './tools/GetStyles.js';
+import * as Tool from './tools/Tool.js';
+import * as ToolRegistry from './tools/ToolRegistry.js';
 
 export {
   AccessibilityAgent,
@@ -54,12 +59,15 @@ export {
   BuiltInAi,
   ChangeManager,
   ContextSelectionAgent,
-  ConversationSummaryAgent,
+  ConversationSummary,
   Debug,
+  DOMNodeContext,
   EvaluateAction,
+  ExecuteJavaScript,
   ExtensionScope,
   FileAgent,
   FileFormatter,
+  GetStyles,
   GreenDevAgent,
   GreenDevAgentAntigravityCliSocketClient,
   GreenDevAgentGeminiCliSocketClient,
@@ -69,11 +77,13 @@ export {
   NetworkRequestFormatter,
   PatchAgent,
   PerformanceAgent,
-  PerformanceAnnotationsAgent,
+  PerformanceAnnotations,
   PerformanceInsightFormatter,
   PerformanceTraceFormatter,
   StorageAgent,
   StorageItem,
   StylingAgent,
+  Tool,
+  ToolRegistry,
   UnitFormatters,
 };

package/front_end/models/ai_assistance/contexts/DOMNodeContext.snapshot.txt

@@ -0,0 +1,51 @@
+Title: DOMNodeContext getPromptDetails describes the node correctly
+Content:
+# Inspected element
+
+* Element's uid is 99.
+* Its selector is `div#myElement`
+=== end content
+
+Title: DOMNodeContext getUserFacingDetails returns details with Data Used title
+Content:
+[
+  {
+    "title": "Data used",
+    "text": "* Element's uid is 99.\n* Its selector is `div#myElement`"
+  }
+]
+=== end content
+
+Title: DOMNodeContext describes an element with child nodes not loaded
+Content:
+* Element's uid is 99.
+* Its selector is `div#myElement`
+=== end content
+
+Title: DOMNodeContext describes an element with no children, siblings, or parent
+Content:
+* Element's uid is 99.
+* Its selector is `div#myElement`
+* It doesn't have any child element nodes
+* It doesn't have any child text nodes
+=== end content
+
+Title: DOMNodeContext describes an element with child element and text nodes
+Content:
+* Element's uid is 99.
+* Its selector is `div#parentElement`
+* It has 2 child element nodes: `span.child1` (uid=undefined), `span.child2` (uid=undefined)
+* It only has 1 child text node
+=== end content
+
+Title: DOMNodeContext describes an element with siblings and a parent
+Content:
+* Element's uid is 99.
+* Its selector is `div#parentElement`
+* It has a next sibling and it is an element (uid=undefined) node
+* It has a previous sibling and it is a non element node
+* Its parent's selector is `div#grandparentElement` (uid=undefined)
+* Its parent is a non element node
+* Its parent has only 1 child element node
+* Its parent has only 1 child text node
+=== end content