chrome-devtools-frontend 1.0.1631386 → 1.0.1635648

package/front_end/models/ai_assistance/StorageItem.ts

@@ -2,15 +2,35 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+interface StorageItemOriginOnly {
+  origin: string;
+  storageKey?: string;
+  storageType?: never;
+  key?: never;
+}
+
+interface StorageItemWithKey {
+  origin: string;
+  storageKey?: string;
+  storageType: 'cookie'|'localStorage'|'sessionStorage';
+  key: string;
+}
+
+export type StorageItemData = StorageItemOriginOnly|StorageItemWithKey;
+
 // The StorageItem is used as context for the Ai Assistance.
 // If the user selects a row in e.g. the cookies table, the storageType and key
-// will be populated. An empty {} StorageItem is used if the user asks general questions.
-export type StorageItem = {
-  origin: string,
-}&({
-  storageType?: never,
-  key?: never,
-}|{
-  storageType: 'cookie' | 'localStorage' | 'sessionStorage',
-  key: string,
-});
+// will be populated.
+export class StorageItem {
+  readonly origin: string;
+  readonly storageKey?: string;
+  readonly storageType?: 'cookie'|'localStorage'|'sessionStorage';
+  readonly key?: string;
+
+  constructor(data: StorageItemData) {
+    this.origin = data.origin;
+    this.storageKey = data.storageKey;
+    this.storageType = data.storageType;
+    this.key = data.key;
+  }
+}

package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts

@@ -29,6 +29,25 @@ import {
   JavascriptExecutor
 } from './ExecuteJavascript.js';
 
+/**
+ * The subset of computed CSS properties relevant to accessibility audits.
+ * These are used to filter the Computed Styles widget in the AI Chat panel to keep it focused and minimal.
+ */
+const ACCESSIBILITY_CSS_PROPERTIES = [
+  'color',
+  'background-color',
+  'display',
+  'visibility',
+  'opacity',
+  'clip',
+  'clip-path',
+  'font-size',
+  'font-weight',
+  'line-height',
+  'letter-spacing',
+  'text-transform',
+];
+
 /**
  * WARNING: preamble defined in code is only used when userTier is
  * TESTERS. Otherwise, a server-side preamble is used (see
@@ -217,7 +236,18 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
     if (!nodeId) {
       return null;
     }
-    return domModel.nodeForId(nodeId);
+    const node = domModel.nodeForId(nodeId);
+    if (!node) {
+      return null;
+    }
+
+    const resourceTreeModel = target.model(SDK.ResourceTreeModel.ResourceTreeModel);
+    const mainFrameId = resourceTreeModel?.mainFrame?.id;
+    if (node.frameId() !== mainFrameId) {
+      return null;
+    }
+
+    return node;
   }
 
   #declareFunctions(): void {
@@ -260,7 +290,10 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
           return {error: 'Failed to run accessibility audits.'};
         }
         const audits = new LighthouseFormatter().audits(report, 'accessibility');
-        return {result: {audits}};
+        return {
+          result: {audits},
+          widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report, snapshotReport: true}}],
+        };
       }
     });
 
@@ -294,7 +327,10 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
           return {error: 'No Lighthouse report available.'};
         }
         const audits = new LighthouseFormatter().audits(report, params.categoryId);
-        return {result: {audits}};
+        return {
+          result: {audits},
+          widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
+        };
       }
     });
 
@@ -449,7 +485,26 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
           backendNodeId: node.backendNodeId(),
         };
 
-        return {result: JSON.stringify(result, null, 2)};
+        const widgets: AiWidget[] = [];
+        const cssModel = node.domModel().cssModel();
+        const styles = await cssModel.getComputedStyle(node.id);
+        const matchedStyles = await cssModel.getMatchedStyles(node.id);
+        if (styles && matchedStyles) {
+          widgets.push({
+            name: 'COMPUTED_STYLES',
+            data: {
+              computedStyles: styles,
+              backendNodeId: node.backendNodeId(),
+              matchedCascade: matchedStyles,
+              properties: ACCESSIBILITY_CSS_PROPERTIES,
+            },
+          });
+        }
+
+        return {
+          result: JSON.stringify(result, null, 2),
+          widgets: widgets.length > 0 ? widgets : undefined,
+        };
       },
     });
   }

package/front_end/models/ai_assistance/agents/AiAgent.ts

@@ -10,8 +10,11 @@ import * as Greendev from '../../greendev/greendev.js';
 import type * as LHModel from '../../lighthouse/lighthouse.js';
 import type * as TextUtils from '../../text_utils/text_utils.js';
 import type * as Trace from '../../trace/trace.js';
+import type * as Workspace from '../../workspace/workspace.js';
 import {debugLog, isStructuredLogEnabled} from '../debug.js';
 
+const MAX_SUGGESTION_LENGTH = 200;
+
 export const enum ResponseType {
   CONTEXT = 'context',
   TITLE = 'title',
@@ -34,6 +37,21 @@ export const enum ErrorType {
   CROSS_ORIGIN = 'cross-origin'
 }
 
+/**
+ * Returns true if the origin is considered opaque and should be blocked from
+ * AI assistance to prevent potential data leakage.
+ *
+ * @see https://crbug.com/513732588
+ */
+export function isOpaqueOrigin(origin: string): boolean {
+  /**
+   * Origins starting with 'about' (like about:blank or about:srcdoc) are
+   * considered opaque. 'about://' is the sentinel used by DevTools
+   * ParsedURL.securityOrigin() for these.
+   */
+  return origin === 'null' || origin === 'data:' || origin.startsWith('about') || origin.startsWith('detached');
+}
+
 export const enum MultimodalInputType {
   SCREENSHOT = 'screenshot',
   UPLOADED_IMAGE = 'uploaded-image',
@@ -145,6 +163,12 @@ export interface RequestOptions {
   modelId?: string;
 }
 
+export type AllowedOriginResult = {
+  origin: string|undefined,
+}|{
+  blocked: true,
+};
+
 export interface AgentOptions {
   aidaClient: Host.AidaClient.AidaClient;
   serverSideLoggingEnabled?: boolean;
@@ -152,7 +176,7 @@ export interface AgentOptions {
   confirmSideEffectForTest?: typeof Promise.withResolvers;
   onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
   history?: Host.AidaClient.Content[];
-  allowedOrigin?: () => string | undefined;
+  allowedOrigin?: () => AllowedOriginResult;
   lighthouseRecording?: (overrides?: LHModel.RunTypes.RunOverrides) => Promise<LHModel.ReporterTypes.ReportJSON|null>;
 }
 
@@ -178,15 +202,30 @@ export abstract class ConversationContext<T> {
   abstract getItem(): T;
   abstract getTitle(): string;
 
-  isOriginAllowed(agentOrigin: string|undefined): boolean {
-    if (!agentOrigin) {
+  /**
+   * Returns true if this data context (e.g., a DOM node or Network Request) is
+   * allowed to be included in a conversation that is locked to the provided
+   * `establishedOrigin`.
+   *
+   * A conversation is "locked" to an origin once the first query is made.
+   * This method ensures that we don't mix data from different origins in the
+   * same conversation.
+   *
+   * @param establishedOrigin The origin that the current conversation is locked to.
+   * If undefined, the conversation has not yet been locked to an origin.
+   */
+  isOriginAllowed(establishedOrigin: string|undefined): boolean {
+    const dataOrigin = this.getOrigin();
+    // Opaque origins are never allowed to be used as context.
+    if (isOpaqueOrigin(dataOrigin)) {
+      return false;
+    }
+    // If no origin is established yet, this context will be the one to lock the conversation.
+    if (!establishedOrigin) {
       return true;
     }
-    // Currently does not handle opaque origins because they
-    // are not available to DevTools, instead checks
-    // that serialization of the origin is the same
-    // https://html.spec.whatwg.org/#ascii-serialisation-of-an-origin.
-    return this.getOrigin() === agentOrigin;
+    // Only allow data that matches the origin the conversation is already locked to.
+    return dataOrigin === establishedOrigin;
   }
 
   /**
@@ -276,9 +315,25 @@ export interface BottomUpTreeAiWidget {
   };
 }
 
+export interface SourceFileAiWidget {
+  name: 'SOURCE_FILE';
+  data: {
+    uiSourceCode: Workspace.UISourceCode.UISourceCode,
+  };
+}
+
+export interface LighthouseReportAiWidget {
+  name: 'LIGHTHOUSE_REPORT';
+  data: {
+    report: LHModel.ReporterTypes.ReportJSON,
+    snapshotReport?: boolean,
+  };
+}
+
 // This type will grow as we add more widgets.
-export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget|DomTreeAiWidget|
-    PerformanceTraceAiWidget|PerfInsightAiWidget|TimelineRangeSummaryAiWidget|BottomUpTreeAiWidget;
+export type AiWidget =
+    ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget|DomTreeAiWidget|PerformanceTraceAiWidget|
+    PerfInsightAiWidget|TimelineRangeSummaryAiWidget|BottomUpTreeAiWidget|SourceFileAiWidget|LighthouseReportAiWidget;
 
 export type FunctionCallHandlerResult<Result> = {
   requiresApproval: true,
@@ -336,7 +391,7 @@ export interface FunctionDeclaration<Args extends Record<string, unknown>, Retur
 
 interface AidaFetchResult {
   text?: string;
-  functionCall?: Host.AidaClient.AidaFunctionCallResponse;
+  functionCall?: Host.AidaClient.AidaFunctionCall;
   completed: boolean;
   rpcId?: Host.AidaClient.RpcGlobalId;
 }
@@ -368,6 +423,7 @@ export abstract class AiAgent<T> {
   readonly #serverSideLoggingEnabled: boolean;
   readonly confirmSideEffect: typeof Promise.withResolvers;
   readonly #functionDeclarations = new Map<string, FunctionDeclaration<Record<string, unknown>, unknown>>();
+  readonly #allowedOrigin?: () => AllowedOriginResult;
 
   /**
    * Used in the debug mode and evals.
@@ -400,6 +456,7 @@ export abstract class AiAgent<T> {
     this.#sessionId = opts.sessionId ?? crypto.randomUUID();
     this.confirmSideEffect = opts.confirmSideEffectForTest ?? (() => Promise.withResolvers());
     this.#history = opts.history ?? [];
+    this.#allowedOrigin = opts.allowedOrigin;
   }
 
   async enhanceQuery(query: string, selected: ConversationContext<T>|null, multimodalInputType?: MultimodalInputType):
@@ -521,8 +578,7 @@ export abstract class AiAgent<T> {
       const trimmed = line.trim();
       if (trimmed.startsWith('SUGGESTIONS:')) {
         try {
-          // TODO: Do basic validation this is an array with strings
-          suggestions = JSON.parse(trimmed.substring('SUGGESTIONS:'.length).trim());
+          suggestions = sanitizeSuggestions(trimmed.substring('SUGGESTIONS:'.length).trim());
         } catch {
         }
       } else {
@@ -535,8 +591,7 @@ export abstract class AiAgent<T> {
     if (!suggestions && answerLines.at(-1)?.includes('SUGGESTIONS:')) {
       const [answer, suggestionsText] = answerLines[answerLines.length - 1].split('SUGGESTIONS:', 2);
       try {
-        // TODO: Do basic validation this is an array with strings
-        suggestions = JSON.parse(suggestionsText.trim().substring('SUGGESTIONS:'.length).trim());
+        suggestions = sanitizeSuggestions(suggestionsText.trim());
       } catch {
       }
       answerLines[answerLines.length - 1] = answer;
@@ -631,7 +686,7 @@ export abstract class AiAgent<T> {
 
       let rpcId: Host.AidaClient.RpcGlobalId|undefined;
       let textResponse = '';
-      let functionCall: Host.AidaClient.AidaFunctionCallResponse|undefined = undefined;
+      let functionCall: Host.AidaClient.AidaFunctionCall|undefined = undefined;
       try {
         for await (const fetchResult of this.#aidaFetch(request, {signal: options.signal})) {
           rpcId = fetchResult.rpcId;
@@ -695,11 +750,19 @@ export abstract class AiAgent<T> {
       }
 
       if (functionCall) {
+        const allowedOriginResult = this.#allowedOrigin?.();
+        if (allowedOriginResult && 'blocked' in allowedOriginResult) {
+          // Abort immediately if the page navigated before we could lock the origin.
+          // This prevents the AI from accessing data from the new page.
+          yield this.#createErrorResponse(ErrorType.CROSS_ORIGIN);
+          break;
+        }
         try {
           const result = yield*
               this.#callFunction(
                   functionCall.name,
                   functionCall.args,
+                  functionCall.thoughtSignature,
                   {
                     ...options,
                     explanation: textResponse,
@@ -751,6 +814,7 @@ export abstract class AiAgent<T> {
       #callFunction(
           name: string,
           args: Record<string, unknown>,
+          thoughtSignature?: string,
           options?: FunctionHandlerOptions&{explanation?: string},
           ): AsyncGenerator<FunctionCallResponseData, {
         result: unknown,
@@ -766,12 +830,14 @@ export abstract class AiAgent<T> {
         text: options.explanation,
       });
     }
-    parts.push({
-      functionCall: {
-        name,
-        args,
-      },
-    });
+    const functionCall: Host.AidaClient.AidaFunctionCall = {
+      name,
+      args,
+    };
+    if (thoughtSignature) {
+      functionCall.thoughtSignature = thoughtSignature;
+    }
+    parts.push({functionCall});
     this.#history.push({
       parts,
       role: Host.AidaClient.Role.MODEL,
@@ -932,3 +998,26 @@ export abstract class AiAgent<T> {
     };
   }
 }
+
+function sanitizeSuggestions(suggestions: string): [string, ...string[]]|undefined {
+  const parsed = JSON.parse(suggestions);
+  if (!Array.isArray(parsed)) {
+    return undefined;
+  }
+  const sanitized: string[] = [];
+  for (const item of parsed) {
+    if (typeof item !== 'string') {
+      continue;
+    }
+    // Collapse multiple whitespace/newlines into a single space.
+    const noExtraWhitespace = item.replace(/\s+/g, ' ').trim();
+    if (noExtraWhitespace.length === 0) {
+      continue;
+    }
+    sanitized.push(noExtraWhitespace.substring(0, MAX_SUGGESTION_LENGTH));
+  }
+  if (sanitized.length === 0) {
+    return undefined;
+  }
+  return sanitized as [string, ...string[]];
+}

package/front_end/models/ai_assistance/agents/ContextSelectionAgent.ts

@@ -18,7 +18,9 @@ import {AccessibilityContext} from './AccessibilityAgent.js';
 import {
   type AgentOptions,
   AiAgent,
+  type AllowedOriginResult,
   type ContextResponse,
+  isOpaqueOrigin,
   type RequestOptions,
 } from './AiAgent.js';
 import {FileContext} from './FileAgent.js';
@@ -87,7 +89,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
   readonly #networkTimeCalculator?: NetworkTimeCalculator.NetworkTransferTimeCalculator;
   readonly #lighthouseRecording?:
       (overrides?: LHModel.RunTypes.RunOverrides) => Promise<LHModel.ReporterTypes.ReportJSON|null>;
-  #allowedOrigin: () => string | undefined;
+  #allowedOrigin: () => AllowedOriginResult;
 
   constructor(opts: AgentOptions&{
     performanceRecordAndReload?: () => Promise<Trace.TraceModel.ParsedTrace>,
@@ -99,7 +101,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
     this.#lighthouseRecording = opts.lighthouseRecording;
     this.#onInspectElement = opts.onInspectElement;
     this.#networkTimeCalculator = opts.networkTimeCalculator;
-    this.#allowedOrigin = opts.allowedOrigin ?? (() => undefined);
+    this.#allowedOrigin = opts.allowedOrigin ?? (() => ({origin: undefined}));
 
     this.declareFunction<Record<string, never>>('listNetworkRequests', {
       description: `Gives a list of network requests including URL, status code, and duration.`,
@@ -118,7 +120,18 @@ export class ContextSelectionAgent extends AiAgent<never> {
       },
       handler: async () => {
         const requests = [];
-        const origin = this.#allowedOrigin();
+        const allowedOriginResult = this.#allowedOrigin();
+        if ('blocked' in allowedOriginResult) {
+          return {
+            error: 'Cross-origin access blocked due to navigation. Please start a new chat.',
+          };
+        }
+        const origin = allowedOriginResult.origin;
+        if (origin && isOpaqueOrigin(origin)) {
+          return {
+            error: 'No requests recorded by DevTools',
+          };
+        }
 
         let hasCrossOriginRequest = false;
         for (const request of Logs.NetworkLog.NetworkLog.instance().requests()) {
@@ -182,7 +195,18 @@ export class ContextSelectionAgent extends AiAgent<never> {
         };
       },
       handler: async ({id}) => {
-        const origin = this.#allowedOrigin();
+        const allowedOriginResult = this.#allowedOrigin();
+        if ('blocked' in allowedOriginResult) {
+          return {
+            error: 'Cross-origin access blocked due to navigation. Please start a new chat.',
+          };
+        }
+        const origin = allowedOriginResult.origin;
+        if (origin && isOpaqueOrigin(origin)) {
+          return {
+            error: 'No request found',
+          };
+        }
         const request = Logs.NetworkLog.NetworkLog.instance().requests().find(req => {
           if (req.requestId() !== id) {
             return false;
@@ -222,8 +246,23 @@ export class ContextSelectionAgent extends AiAgent<never> {
         };
       },
       handler: async () => {
+        const allowedOriginResult = this.#allowedOrigin();
+        if ('blocked' in allowedOriginResult) {
+          return {
+            error: 'Cross-origin access blocked due to navigation. Please start a new chat.',
+          };
+        }
+        const origin = allowedOriginResult.origin;
+
         const files: Array<{file: string, id: number | undefined}> = [];
         for (const file of ContextSelectionAgent.getUISourceCodes()) {
+          const fileUrl = file.url();
+          const fileOrigin = Common.ParsedURL.ParsedURL.extractOrigin(fileUrl);
+
+          if (origin && fileOrigin !== origin) {
+            continue;
+          }
+
           files.push({
             file: file.fullDisplayName(),
             id: ContextSelectionAgent.uiSourceCodeId.get(file),
@@ -259,8 +298,22 @@ export class ContextSelectionAgent extends AiAgent<never> {
         };
       },
       handler: async params => {
-        const file = ContextSelectionAgent.getUISourceCodes().find(
-            file => ContextSelectionAgent.uiSourceCodeId.get(file) === params.id);
+        const allowedOriginResult = this.#allowedOrigin();
+        if ('blocked' in allowedOriginResult) {
+          return {
+            error: 'Cross-origin access blocked due to navigation. Please start a new chat.',
+          };
+        }
+        const origin = allowedOriginResult.origin;
+
+        const file = ContextSelectionAgent.getUISourceCodes().find(file => {
+          if (ContextSelectionAgent.uiSourceCodeId.get(file) !== params.id) {
+            return false;
+          }
+          const fileUrl = file.url();
+          const fileOrigin = Common.ParsedURL.ParsedURL.extractOrigin(fileUrl);
+          return !origin || fileOrigin === origin;
+        });
 
         if (!file) {
           return {
@@ -271,6 +324,12 @@ export class ContextSelectionAgent extends AiAgent<never> {
         return {
           context: new FileContext(file),
           description: 'User selected a source file',
+          widgets: [{
+            name: 'SOURCE_FILE',
+            data: {
+              uiSourceCode: file,
+            },
+          }],
         };
       },
     });
@@ -352,6 +411,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
         return {
           context: new AccessibilityContext(result),
           description: 'User has selected a Lighthouse report',
+          widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report: result}}],
         };
       }
     });

package/front_end/models/ai_assistance/agents/PerformanceAgent.ts

@@ -14,6 +14,7 @@ import * as Logs from '../../logs/logs.js';
 import * as SourceMapScopes from '../../source_map_scopes/source_map_scopes.js';
 import * as TextUtils from '../../text_utils/text_utils.js';
 import * as Trace from '../../trace/trace.js';
+import {sanitizeHeaders} from '../data_formatters/NetworkRequestFormatter.js';
 import {
   PerformanceInsightFormatter,
 } from '../data_formatters/PerformanceInsightFormatter.js';
@@ -1060,7 +1061,35 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
         }
 
         // TODO(b/425270067): Format in the same way that "Summary" detail tab does.
-        const details = JSON.stringify(event);
+        let details;
+        if (Trace.Types.Events.isSyntheticNetworkRequest(event)) {
+          const eventToSerialize = {
+            ...event,
+            args: {
+              ...event.args,
+              data: {
+                ...event.args.data,
+                responseHeaders: event.args.data.responseHeaders ? sanitizeHeaders(event.args.data.responseHeaders) :
+                                                                   null,
+              },
+            },
+          };
+          details = JSON.stringify(eventToSerialize);
+        } else if (Trace.Types.Events.isResourceReceiveResponse(event)) {
+          const eventToSerialize = {
+            ...event,
+            args: {
+              ...event.args,
+              data: {
+                ...event.args.data,
+                headers: event.args.data.headers ? sanitizeHeaders(event.args.data.headers) : undefined,
+              },
+            },
+          };
+          details = JSON.stringify(eventToSerialize);
+        } else {
+          details = JSON.stringify(event);
+        }
 
         const key = `getEventByKey('${params.eventKey}')`;
         this.#cacheFunctionResult(focus, key, details);