chrome-devtools-frontend 1.0.1521880 → 1.0.1522585

package/front_end/models/ai_assistance/agents/StylingAgent.snapshot.txt

@@ -0,0 +1,559 @@
+Title: StylingAgent describeElement should describe an element with no children, siblings, or parent
+Content:
+* Its selector is `div#myElement`
+=== end content
+
+Title: StylingAgent describeElement should describe an element with child element and text nodes
+Content:
+* Its selector is `div#parentElement`
+* It has 2 child element nodes: `span.child1`, `span.child2`
+* It only has 1 child text node
+=== end content
+
+Title: StylingAgent describeElement should describe an element with siblings and a parent
+Content:
+* Its selector is `div#parentElement`
+* It has a next sibling and it is an element node
+* It has a previous sibling and it is a non element node
+* Its parent's selector is `div#grandparentElement`
+* Its parent is a non element node
+* Its parent has only 1 child element node
+* Its parent has only 1 child text node
+=== end content
+
+Title: StylingAgent buildRequest structure matches the snapshot
+Content:
+{
+  "client": "CHROME_DEVTOOLS",
+  "current_message": {
+    "parts": [
+      {
+        "text": "test input"
+      }
+    ],
+    "role": 1
+  },
+  "historical_contexts": [
+    {
+      "parts": [
+        {
+          "text": "QUERY: question"
+        }
+      ],
+      "role": 1
+    },
+    {
+      "parts": [
+        {
+          "text": "answer"
+        }
+      ],
+      "role": 2
+    }
+  ],
+  "function_declarations": [
+    {
+      "name": "executeJavaScript",
+      "description": "This function allows you to run JavaScript code on the inspected page to access the element styles and page content.\nCall this function to gather additional information or modify the page state. Call this function enough times to investigate the user request.",
+      "parameters": {
+        "type": 6,
+        "description": "",
+        "nullable": false,
+        "properties": {
+          "code": {
+            "type": 1,
+            "description": "JavaScript code snippet to run on the inspected page. Make sure the code is formatted for readability.\n\n# Instructions\n\n* To return data, define a top-level `data` variable and populate it with data you want to get. Only JSON-serializable objects can be assigned to `data`.\n* If you modify styles on an element, ALWAYS call the pre-defined global `async setElementStyles(el: Element, styles: object)` function. This function is an internal mechanism for you and should never be presented as a command/advice to the user.\n* Use `window.getComputedStyle` to gather **computed** styles and make sure that you take the distinction between authored styles and computed styles into account.\n* **CRITICAL** Only get styles that might be relevant to the user request.\n* **CRITICAL** Call `window.getComputedStyle` only once per element and store results into a local variable. Never try to return all the styles of the element in `data`.\n* **CRITICAL** Never assume a selector for the elements unless you verified your knowledge.\n* **CRITICAL** Consider that `data` variable from the previous function calls are not available in a new function call.\n\nFor example, the code to return basic styles:\n\n```\nconst styles = window.getComputedStyle($0);\nconst data = {\n    display: styles['display'],\n    visibility: styles['visibility'],\n    position: styles['position'],\n    left: styles['right'],\n    top: styles['top'],\n    width: styles['width'],\n    height: styles['height'],\n    zIndex: styles['z-index']\n};\n```\n\nFor example, the code to change element styles:\n\n```\nawait setElementStyles($0, {\n  color: 'blue',\n});\n```\n\nFor example, the code to get current and parent styles at once:\n\n```\nconst styles = window.getComputedStyle($0);\nconst parentStyles = window.getComputedStyle($0.parentElement);\nconst data = {\n    currentElementStyles: {\n      display: styles['display'],\n      visibility: styles['visibility'],\n      position: styles['position'],\n      left: styles['right'],\n      top: styles['top'],\n      width: styles['width'],\n      height: styles['height'],\n      zIndex: styles['z-index'],\n    },\n    parentElementStyles: {\n      display: parentStyles['display'],\n      visibility: parentStyles['visibility'],\n      position: parentStyles['position'],\n      left: parentStyles['right'],\n      top: parentStyles['top'],\n      width: parentStyles['width'],\n      height: parentStyles['height'],\n      zIndex: parentStyles['z-index'],\n    },\n};\n```\n\nFor example, the code to get check siblings and overlapping elements:\n\n```\nconst computedStyles = window.getComputedStyle($0);\nconst parentComputedStyles = window.getComputedStyle($0.parentElement);\nconst data = {\n  numberOfChildren: $0.children.length,\n  numberOfSiblings: $0.parentElement.children.length,\n  hasPreviousSibling: !!$0.previousElementSibling,\n  hasNextSibling: !!$0.nextElementSibling,\n  elementStyles: {\n    display: computedStyles['display'],\n    visibility: computedStyles['visibility'],\n    position: computedStyles['position'],\n    clipPath: computedStyles['clip-path'],\n    zIndex: computedStyles['z-index']\n  },\n  parentStyles: {\n    display: parentComputedStyles['display'],\n    visibility: parentComputedStyles['visibility'],\n    position: parentComputedStyles['position'],\n    clipPath: parentComputedStyles['clip-path'],\n    zIndex: parentComputedStyles['z-index']\n  },\n  overlappingElements: Array.from(document.querySelectorAll('*'))\n    .filter(el => {\n      const rect = el.getBoundingClientRect();\n      const popupRect = $0.getBoundingClientRect();\n      return (\n        el !== $0 &&\n        rect.left < popupRect.right &&\n        rect.right > popupRect.left &&\n        rect.top < popupRect.bottom &&\n        rect.bottom > popupRect.top\n      );\n    })\n    .map(el => ({\n      tagName: el.tagName,\n      id: el.id,\n      className: el.className,\n      zIndex: window.getComputedStyle(el)['z-index']\n    }))\n};\n```\n"
+          },
+          "thought": {
+            "type": 1,
+            "description": "Explain why you want to run this code"
+          },
+          "title": {
+            "type": 1,
+            "description": "Provide a summary of what the code does. For example, \"Checking related element styles\"."
+          }
+        }
+      }
+    }
+  ],
+  "options": {
+    "model_id": "test model"
+  },
+  "metadata": {
+    "disable_user_content_logging": false,
+    "string_session_id": "sessionId",
+    "user_tier": 2,
+    "client_version": "unit_test+function_calling"
+  },
+  "functionality_type": 5,
+  "client_feature": 2
+}
+=== end content
+
+Title: StylingAgent buildRequest builds a request with aborted query in history before a real request
+Content:
+[
+  {
+    "parts": [
+      {
+        "text": "# Inspected element\n\nelement-description\n\n# User request\n\nQUERY: test2"
+      }
+    ],
+    "role": 1
+  },
+  {
+    "parts": [
+      {
+        "functionCall": {
+          "name": "executeJavaScript",
+          "args": {
+            "title": "title2",
+            "thought": "thought2",
+            "code": "action2"
+          }
+        }
+      }
+    ],
+    "role": 2
+  },
+  {
+    "parts": [
+      {
+        "functionResponse": {
+          "name": "executeJavaScript",
+          "response": {
+            "result": "result2"
+          }
+        }
+      }
+    ],
+    "role": 0
+  },
+  {
+    "parts": [
+      {
+        "text": "answer2"
+      }
+    ],
+    "role": 2
+  }
+]
+=== end content
+
+Title: StylingAgent run generates an answer immediately
+Content:
+[
+  {
+    "type": "user-query",
+    "query": "test"
+  },
+  {
+    "type": "context",
+    "title": "Analyzing the prompt",
+    "details": [
+      {
+        "title": "Data used",
+        "text": "* Its selector is `undefined`"
+      }
+    ]
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "answer",
+    "text": "this is the answer",
+    "complete": true
+  }
+]
+=== end content
+
+Title: StylingAgent run generates an answer immediately with correct historical contexts in the new request
+Content:
+[
+  {
+    "parts": [
+      {
+        "text": "# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test"
+      }
+    ],
+    "role": 1
+  },
+  {
+    "parts": [
+      {
+        "text": "this is the answer"
+      }
+    ],
+    "role": 2
+  }
+]
+=== end content
+
+Title: StylingAgent run correctly handles historical_contexts in AIDA requests
+Content:
+[
+  {
+    "text": "# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test"
+  },
+  [
+    {
+      "parts": [
+        {
+          "text": "# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test"
+        }
+      ],
+      "role": 1
+    },
+    {
+      "parts": [
+        {
+          "functionCall": {
+            "name": "executeJavaScript",
+            "args": {
+              "code": "const data = {\"test\": \"observation\"}",
+              "thought": "I am thinking.",
+              "title": "thinking"
+            }
+          }
+        }
+      ],
+      "role": 2
+    }
+  ]
+]
+=== end content
+
+Title: StylingAgent run generates an rpcId for the answer
+Content:
+[
+  {
+    "type": "user-query",
+    "query": "test"
+  },
+  {
+    "type": "context",
+    "title": "Analyzing the prompt",
+    "details": [
+      {
+        "title": "Data used",
+        "text": "* Its selector is `undefined`"
+      }
+    ]
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "answer",
+    "text": "this is the answer",
+    "complete": true,
+    "rpcId": 123
+  }
+]
+=== end content
+
+Title: StylingAgent run throws an error based on the attribution metadata including RecitationAction.BLOCK
+Content:
+[
+  {
+    "type": "user-query",
+    "query": "test"
+  },
+  {
+    "type": "context",
+    "title": "Analyzing the prompt",
+    "details": [
+      {
+        "title": "Data used",
+        "text": "* Its selector is `undefined`"
+      }
+    ]
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "answer",
+    "text": "this is the partial answer",
+    "complete": false
+  },
+  {
+    "type": "error",
+    "error": "block"
+  }
+]
+=== end content
+
+Title: StylingAgent run does not throw an error based on attribution metadata not including RecitationAction.BLOCK
+Content:
+[
+  {
+    "type": "user-query",
+    "query": "test"
+  },
+  {
+    "type": "context",
+    "title": "Analyzing the prompt",
+    "details": [
+      {
+        "title": "Data used",
+        "text": "* Its selector is `undefined`"
+      }
+    ]
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "answer",
+    "text": "this is the answer",
+    "complete": true,
+    "rpcId": 123
+  }
+]
+=== end content
+
+Title: StylingAgent run generates a response if nothing is returned
+Content:
+[
+  {
+    "type": "user-query",
+    "query": "test"
+  },
+  {
+    "type": "context",
+    "title": "Analyzing the prompt",
+    "details": [
+      {
+        "title": "Data used",
+        "text": "* Its selector is `undefined`"
+      }
+    ]
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "error",
+    "error": "unknown"
+  }
+]
+=== end content
+
+Title: StylingAgent run generates an action response if action and answer both present
+Content:
+[
+  {
+    "type": "user-query",
+    "query": "test"
+  },
+  {
+    "type": "context",
+    "title": "Analyzing the prompt",
+    "details": [
+      {
+        "title": "Data used",
+        "text": "* Its selector is `undefined`"
+      }
+    ]
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "thought",
+    "thought": "I am thinking."
+  },
+  {
+    "type": "action",
+    "code": "console.log('hello');",
+    "output": "hello",
+    "canceled": false
+  },
+  {
+    "type": "querying"
+  },
+  {
+    "type": "answer",
+    "text": "this is the actual answer",
+    "complete": true
+  }
+]
+=== end content
+
+Title: StylingAgent run generates history for multiple actions
+Content:
+[
+  {
+    "parts": [
+      {
+        "text": "# Inspected element\n\n* Its selector is `undefined`\n\n# User request\n\nQUERY: test"
+      }
+    ],
+    "role": 1
+  },
+  {
+    "parts": [
+      {
+        "functionCall": {
+          "name": "executeJavaScript",
+          "args": {
+            "thought": "thought 1",
+            "title": "test",
+            "code": "console.log('test')"
+          }
+        }
+      }
+    ],
+    "role": 2
+  },
+  {
+    "parts": [
+      {
+        "functionResponse": {
+          "name": "executeJavaScript",
+          "response": {
+            "result": "undefined"
+          }
+        }
+      }
+    ],
+    "role": 0
+  },
+  {
+    "parts": [
+      {
+        "functionCall": {
+          "name": "executeJavaScript",
+          "args": {
+            "thought": "thought 2",
+            "title": "test",
+            "code": "console.log('test')"
+          }
+        }
+      }
+    ],
+    "role": 2
+  },
+  {
+    "parts": [
+      {
+        "functionResponse": {
+          "name": "executeJavaScript",
+          "response": {
+            "result": "undefined"
+          }
+        }
+      }
+    ],
+    "role": 0
+  },
+  {
+    "parts": [
+      {
+        "functionCall": {
+          "name": "executeJavaScript",
+          "args": {
+            "thought": "thought 3",
+            "title": "test",
+            "code": "console.log('test')"
+          }
+        }
+      }
+    ],
+    "role": 2
+  },
+  {
+    "parts": [
+      {
+        "functionResponse": {
+          "name": "executeJavaScript",
+          "response": {
+            "result": "undefined"
+          }
+        }
+      }
+    ],
+    "role": 0
+  },
+  {
+    "parts": [
+      {
+        "text": "this is the answer"
+      }
+    ],
+    "role": 2
+  }
+]
+=== end content
+
+Title: StylingAgent enhanceQuery does not add multimodal input evaluation prompt when multimodal is disabled
+Content:
+# Inspected element
+
+* Its selector is `div#myElement`
+
+# User request
+
+QUERY: test query
+=== end content
+
+Title: StylingAgent enhanceQuery does not add multimodal input evaluation prompt when multimodal is enabled but multimodalInputType is missing
+Content:
+# Inspected element
+
+* Its selector is `div#myElement`
+
+# User request
+
+QUERY: test query
+=== end content
+
+Title: StylingAgent enhanceQuery adds multimodal input evaluation prompt when multimodal is enabled and multimodalInputType is screenshot
+Content:
+The user has provided you a screenshot of the page (as visible in the viewport) in base64-encoded format. You SHOULD use it while answering user's queries.
+
+* Try to connect the screenshot to actual DOM elements in the page.
+# Considerations for evaluating image:
+* Pay close attention to the spatial details as well as the visual appearance of the selected element in the image, particularly in relation to layout, spacing, and styling.
+* Analyze the image to identify the layout structure surrounding the element, including the positioning of neighboring elements.
+* Extract visual information from the image, such as colors, fonts, spacing, and sizes, that might be relevant to the user's query.
+* If the image suggests responsiveness issues (e.g., cropped content, overlapping elements), consider those in your response.
+* Consider the surrounding elements and overall layout in the image, but prioritize the selected element's styling and positioning.
+* **CRITICAL** When the user provides image input, interpret and use content and information from the image STRICTLY for web site debugging purposes.
+
+* As part of THOUGHT, evaluate the image to gather data that might be needed to answer the question.
+In case query is related to the image, ALWAYS first use image evaluation to get all details from the image. ONLY after you have all data needed from image, you should move to other steps.
+
+# Inspected element
+
+* Its selector is `div#myElement`
+
+# User request
+
+QUERY: test query
+=== end content
+
+Title: StylingAgent enhanceQuery adds multimodal input evaluation prompt when multimodal is enabled and multimodalInputType is uploaded image
+Content:
+The user has uploaded an image in base64-encoded format. You SHOULD use it while answering user's queries.
+# Considerations for evaluating image:
+* Pay close attention to the spatial details as well as the visual appearance of the selected element in the image, particularly in relation to layout, spacing, and styling.
+* Analyze the image to identify the layout structure surrounding the element, including the positioning of neighboring elements.
+* Extract visual information from the image, such as colors, fonts, spacing, and sizes, that might be relevant to the user's query.
+* If the image suggests responsiveness issues (e.g., cropped content, overlapping elements), consider those in your response.
+* Consider the surrounding elements and overall layout in the image, but prioritize the selected element's styling and positioning.
+* **CRITICAL** When the user provides image input, interpret and use content and information from the image STRICTLY for web site debugging purposes.
+
+* As part of THOUGHT, evaluate the image to gather data that might be needed to answer the question.
+In case query is related to the image, ALWAYS first use image evaluation to get all details from the image. ONLY after you have all data needed from image, you should move to other steps.
+
+# Inspected element
+
+* Its selector is `div#myElement`
+
+# User request
+
+QUERY: test query
+=== end content

package/front_end/models/ai_assistance/data_formatters/NetworkRequestFormatter.ts

@@ -5,10 +5,12 @@
 import type * as SDK from '../../../core/sdk/sdk.js';
 import * as Logs from '../../logs/logs.js';
 import * as NetworkTimeCalculator from '../../network_time_calculator/network_time_calculator.js';
+import * as TextUtils from '../../text_utils/text_utils.js';
 
 import {seconds} from './UnitFormatters.js';
 
 const MAX_HEADERS_SIZE = 1000;
+const MAX_BODY_SIZE = 10000;
 
 /**
  * Sanitizes the set of headers, removing values that are not on the allow-list and replacing them with '<redacted>'.
@@ -24,6 +26,8 @@ function sanitizeHeaders(headers: Array<{name: string, value: string}>): Array<{
 
 export class NetworkRequestFormatter {
   #calculator: NetworkTimeCalculator.NetworkTransferTimeCalculator;
+  #request: SDK.NetworkRequest.NetworkRequest;
+
   static allowHeader(headerName: string): boolean {
     return allowedHeaders.has(headerName.toLowerCase().trim());
   }
@@ -37,6 +41,31 @@ export class NetworkRequestFormatter {
         MAX_HEADERS_SIZE);
   }
 
+  static async formatBody(title: string, request: SDK.NetworkRequest.NetworkRequest, maxBodySize: number):
+      Promise<string> {
+    const data = await request.requestContentData();
+
+    if (TextUtils.ContentData.ContentData.isError(data)) {
+      return '';
+    }
+
+    if (data.isEmpty) {
+      return `${title}\n<empty response>`;
+    }
+
+    if (data.isTextContent) {
+      const dataAsText = data.text;
+
+      if (dataAsText.length > maxBodySize) {
+        return `${title}\n${dataAsText.substring(0, maxBodySize) + '... <truncated>'}`;
+      }
+
+      return `${title}\n${dataAsText}`;
+    }
+
+    return `${title}\n<binary data>`;
+  }
+
   static formatInitiatorUrl(initiatorUrl: string, allowedOrigin: string): string {
     const initiatorOrigin = new URL(initiatorUrl).origin;
     if (initiatorOrigin === allowedOrigin) {
@@ -45,8 +74,6 @@ export class NetworkRequestFormatter {
     return '<redacted cross-origin initiator URL>';
   }
 
-  #request: SDK.NetworkRequest.NetworkRequest;
-
   constructor(
       request: SDK.NetworkRequest.NetworkRequest, calculator: NetworkTimeCalculator.NetworkTransferTimeCalculator) {
     this.#request = request;
@@ -61,16 +88,27 @@ export class NetworkRequestFormatter {
     return NetworkRequestFormatter.formatHeaders('Response headers:', this.#request.responseHeaders);
   }
 
+  async formatResponseBody(): Promise<string> {
+    return await NetworkRequestFormatter.formatBody('Response body:', this.#request, MAX_BODY_SIZE);
+  }
+
   /**
    * Note: nothing here should include information from origins other than
    * the request's origin.
    */
-  formatNetworkRequest(): string {
+  async formatNetworkRequest(): Promise<string> {
+    let responseBody = await this.formatResponseBody();
+
+    if (responseBody) {
+      // if we have a response then we add 2 new line to follow same structure of the context
+      responseBody = `\n\n${responseBody}`;
+    }
+
     return `Request: ${this.#request.url()}
 
 ${this.formatRequestHeaders()}
 
-${this.formatResponseHeaders()}
+${this.formatResponseHeaders()}${responseBody}
 
 Response status: ${this.#request.statusCode} ${this.#request.statusText}