chrome-devtools-frontend 1.0.1630574 → 1.0.1632065

package/front_end/models/ai_assistance/StorageItem.ts

@@ -2,15 +2,35 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+interface StorageItemOriginOnly {
+  origin: string;
+  storageKey?: string;
+  storageType?: never;
+  key?: never;
+}
+
+interface StorageItemWithKey {
+  origin: string;
+  storageKey?: string;
+  storageType: 'cookie'|'localStorage'|'sessionStorage';
+  key: string;
+}
+
+export type StorageItemData = StorageItemOriginOnly|StorageItemWithKey;
+
 // The StorageItem is used as context for the Ai Assistance.
 // If the user selects a row in e.g. the cookies table, the storageType and key
-// will be populated. An empty {} StorageItem is used if the user asks general questions.
-export type StorageItem = {
-  origin: string,
-}&({
-  storageType?: never,
-  key?: never,
-}|{
-  storageType: 'cookie' | 'localStorage' | 'sessionStorage',
-  key: string,
-});
+// will be populated.
+export class StorageItem {
+  readonly origin: string;
+  readonly storageKey?: string;
+  readonly storageType?: 'cookie'|'localStorage'|'sessionStorage';
+  readonly key?: string;
+
+  constructor(data: StorageItemData) {
+    this.origin = data.origin;
+    this.storageKey = data.storageKey;
+    this.storageType = data.storageType;
+    this.key = data.key;
+  }
+}

package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts

@@ -29,6 +29,25 @@ import {
   JavascriptExecutor
 } from './ExecuteJavascript.js';
 
+/**
+ * The subset of computed CSS properties relevant to accessibility audits.
+ * These are used to filter the Computed Styles widget in the AI Chat panel to keep it focused and minimal.
+ */
+const ACCESSIBILITY_CSS_PROPERTIES = [
+  'color',
+  'background-color',
+  'display',
+  'visibility',
+  'opacity',
+  'clip',
+  'clip-path',
+  'font-size',
+  'font-weight',
+  'line-height',
+  'letter-spacing',
+  'text-transform',
+];
+
 /**
  * WARNING: preamble defined in code is only used when userTier is
  * TESTERS. Otherwise, a server-side preamble is used (see
@@ -260,7 +279,10 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
           return {error: 'Failed to run accessibility audits.'};
         }
         const audits = new LighthouseFormatter().audits(report, 'accessibility');
-        return {result: {audits}};
+        return {
+          result: {audits},
+          widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
+        };
       }
     });
 
@@ -294,7 +316,10 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
           return {error: 'No Lighthouse report available.'};
         }
         const audits = new LighthouseFormatter().audits(report, params.categoryId);
-        return {result: {audits}};
+        return {
+          result: {audits},
+          widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
+        };
       }
     });
 
@@ -449,7 +474,26 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
           backendNodeId: node.backendNodeId(),
         };
 
-        return {result: JSON.stringify(result, null, 2)};
+        const widgets: AiWidget[] = [];
+        const cssModel = node.domModel().cssModel();
+        const styles = await cssModel.getComputedStyle(node.id);
+        const matchedStyles = await cssModel.getMatchedStyles(node.id);
+        if (styles && matchedStyles) {
+          widgets.push({
+            name: 'COMPUTED_STYLES',
+            data: {
+              computedStyles: styles,
+              backendNodeId: node.backendNodeId(),
+              matchedCascade: matchedStyles,
+              properties: ACCESSIBILITY_CSS_PROPERTIES,
+            },
+          });
+        }
+
+        return {
+          result: JSON.stringify(result, null, 2),
+          widgets: widgets.length > 0 ? widgets : undefined,
+        };
       },
     });
   }

package/front_end/models/ai_assistance/agents/AiAgent.ts

@@ -10,6 +10,7 @@ import * as Greendev from '../../greendev/greendev.js';
 import type * as LHModel from '../../lighthouse/lighthouse.js';
 import type * as TextUtils from '../../text_utils/text_utils.js';
 import type * as Trace from '../../trace/trace.js';
+import type * as Workspace from '../../workspace/workspace.js';
 import {debugLog, isStructuredLogEnabled} from '../debug.js';
 
 export const enum ResponseType {
@@ -145,6 +146,12 @@ export interface RequestOptions {
   modelId?: string;
 }
 
+export type AllowedOriginResult = {
+  origin: string|undefined,
+}|{
+  blocked: true,
+};
+
 export interface AgentOptions {
   aidaClient: Host.AidaClient.AidaClient;
   serverSideLoggingEnabled?: boolean;
@@ -152,7 +159,7 @@ export interface AgentOptions {
   confirmSideEffectForTest?: typeof Promise.withResolvers;
   onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
   history?: Host.AidaClient.Content[];
-  allowedOrigin?: () => string | undefined;
+  allowedOrigin?: () => AllowedOriginResult;
   lighthouseRecording?: (overrides?: LHModel.RunTypes.RunOverrides) => Promise<LHModel.ReporterTypes.ReportJSON|null>;
 }
 
@@ -276,9 +283,24 @@ export interface BottomUpTreeAiWidget {
   };
 }
 
+export interface SourceFileAiWidget {
+  name: 'SOURCE_FILE';
+  data: {
+    uiSourceCode: Workspace.UISourceCode.UISourceCode,
+  };
+}
+
+export interface LighthouseReportAiWidget {
+  name: 'LIGHTHOUSE_REPORT';
+  data: {
+    report: LHModel.ReporterTypes.ReportJSON,
+  };
+}
+
 // This type will grow as we add more widgets.
-export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget|DomTreeAiWidget|
-    PerformanceTraceAiWidget|PerfInsightAiWidget|TimelineRangeSummaryAiWidget|BottomUpTreeAiWidget;
+export type AiWidget =
+    ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget|DomTreeAiWidget|PerformanceTraceAiWidget|
+    PerfInsightAiWidget|TimelineRangeSummaryAiWidget|BottomUpTreeAiWidget|SourceFileAiWidget|LighthouseReportAiWidget;
 
 export type FunctionCallHandlerResult<Result> = {
   requiresApproval: true,
@@ -336,7 +358,7 @@ export interface FunctionDeclaration<Args extends Record<string, unknown>, Retur
 
 interface AidaFetchResult {
   text?: string;
-  functionCall?: Host.AidaClient.AidaFunctionCallResponse;
+  functionCall?: Host.AidaClient.AidaFunctionCall;
   completed: boolean;
   rpcId?: Host.AidaClient.RpcGlobalId;
 }
@@ -368,6 +390,7 @@ export abstract class AiAgent<T> {
   readonly #serverSideLoggingEnabled: boolean;
   readonly confirmSideEffect: typeof Promise.withResolvers;
   readonly #functionDeclarations = new Map<string, FunctionDeclaration<Record<string, unknown>, unknown>>();
+  readonly #allowedOrigin?: () => AllowedOriginResult;
 
   /**
    * Used in the debug mode and evals.
@@ -400,6 +423,7 @@ export abstract class AiAgent<T> {
     this.#sessionId = opts.sessionId ?? crypto.randomUUID();
     this.confirmSideEffect = opts.confirmSideEffectForTest ?? (() => Promise.withResolvers());
     this.#history = opts.history ?? [];
+    this.#allowedOrigin = opts.allowedOrigin;
   }
 
   async enhanceQuery(query: string, selected: ConversationContext<T>|null, multimodalInputType?: MultimodalInputType):
@@ -631,7 +655,7 @@ export abstract class AiAgent<T> {
 
       let rpcId: Host.AidaClient.RpcGlobalId|undefined;
       let textResponse = '';
-      let functionCall: Host.AidaClient.AidaFunctionCallResponse|undefined = undefined;
+      let functionCall: Host.AidaClient.AidaFunctionCall|undefined = undefined;
       try {
         for await (const fetchResult of this.#aidaFetch(request, {signal: options.signal})) {
           rpcId = fetchResult.rpcId;
@@ -695,11 +719,19 @@ export abstract class AiAgent<T> {
       }
 
       if (functionCall) {
+        const allowedOriginResult = this.#allowedOrigin?.();
+        if (allowedOriginResult && 'blocked' in allowedOriginResult) {
+          // Abort immediately if the page navigated before we could lock the origin.
+          // This prevents the AI from accessing data from the new page.
+          yield this.#createErrorResponse(ErrorType.CROSS_ORIGIN);
+          break;
+        }
         try {
           const result = yield*
               this.#callFunction(
                   functionCall.name,
                   functionCall.args,
+                  functionCall.thoughtSignature,
                   {
                     ...options,
                     explanation: textResponse,
@@ -751,6 +783,7 @@ export abstract class AiAgent<T> {
       #callFunction(
           name: string,
           args: Record<string, unknown>,
+          thoughtSignature?: string,
           options?: FunctionHandlerOptions&{explanation?: string},
           ): AsyncGenerator<FunctionCallResponseData, {
         result: unknown,
@@ -766,12 +799,14 @@ export abstract class AiAgent<T> {
         text: options.explanation,
       });
     }
-    parts.push({
-      functionCall: {
-        name,
-        args,
-      },
-    });
+    const functionCall: Host.AidaClient.AidaFunctionCall = {
+      name,
+      args,
+    };
+    if (thoughtSignature) {
+      functionCall.thoughtSignature = thoughtSignature;
+    }
+    parts.push({functionCall});
     this.#history.push({
       parts,
       role: Host.AidaClient.Role.MODEL,

package/front_end/models/ai_assistance/agents/ContextSelectionAgent.ts

@@ -18,6 +18,7 @@ import {AccessibilityContext} from './AccessibilityAgent.js';
 import {
   type AgentOptions,
   AiAgent,
+  type AllowedOriginResult,
   type ContextResponse,
   type RequestOptions,
 } from './AiAgent.js';
@@ -87,7 +88,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
   readonly #networkTimeCalculator?: NetworkTimeCalculator.NetworkTransferTimeCalculator;
   readonly #lighthouseRecording?:
       (overrides?: LHModel.RunTypes.RunOverrides) => Promise<LHModel.ReporterTypes.ReportJSON|null>;
-  #allowedOrigin: () => string | undefined;
+  #allowedOrigin: () => AllowedOriginResult;
 
   constructor(opts: AgentOptions&{
     performanceRecordAndReload?: () => Promise<Trace.TraceModel.ParsedTrace>,
@@ -99,7 +100,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
     this.#lighthouseRecording = opts.lighthouseRecording;
     this.#onInspectElement = opts.onInspectElement;
     this.#networkTimeCalculator = opts.networkTimeCalculator;
-    this.#allowedOrigin = opts.allowedOrigin ?? (() => undefined);
+    this.#allowedOrigin = opts.allowedOrigin ?? (() => ({origin: undefined}));
 
     this.declareFunction<Record<string, never>>('listNetworkRequests', {
       description: `Gives a list of network requests including URL, status code, and duration.`,
@@ -118,7 +119,13 @@ export class ContextSelectionAgent extends AiAgent<never> {
       },
       handler: async () => {
         const requests = [];
-        const origin = this.#allowedOrigin();
+        const allowedOriginResult = this.#allowedOrigin();
+        if ('blocked' in allowedOriginResult) {
+          return {
+            error: 'Cross-origin access blocked due to navigation. Please start a new chat.',
+          };
+        }
+        const origin = allowedOriginResult.origin;
 
         let hasCrossOriginRequest = false;
         for (const request of Logs.NetworkLog.NetworkLog.instance().requests()) {
@@ -182,7 +189,13 @@ export class ContextSelectionAgent extends AiAgent<never> {
         };
       },
       handler: async ({id}) => {
-        const origin = this.#allowedOrigin();
+        const allowedOriginResult = this.#allowedOrigin();
+        if ('blocked' in allowedOriginResult) {
+          return {
+            error: 'Cross-origin access blocked due to navigation. Please start a new chat.',
+          };
+        }
+        const origin = allowedOriginResult.origin;
         const request = Logs.NetworkLog.NetworkLog.instance().requests().find(req => {
           if (req.requestId() !== id) {
             return false;
@@ -271,6 +284,12 @@ export class ContextSelectionAgent extends AiAgent<never> {
         return {
           context: new FileContext(file),
           description: 'User selected a source file',
+          widgets: [{
+            name: 'SOURCE_FILE',
+            data: {
+              uiSourceCode: file,
+            },
+          }],
         };
       },
     });
@@ -352,6 +371,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
         return {
           context: new AccessibilityContext(result),
           description: 'User has selected a Lighthouse report',
+          widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report: result}}],
         };
       }
     });

package/front_end/models/ai_assistance/agents/StorageAgent.ts

@@ -3,25 +3,52 @@
 // found in the LICENSE file.
 
 import * as Host from '../../../core/host/host.js';
+import * as i18n from '../../../core/i18n/i18n.js';
 import * as Root from '../../../core/root/root.js';
+import * as SDK from '../../../core/sdk/sdk.js';
 import type {StorageItem} from '../StorageItem.js';
 
 import {
+  type AgentOptions,
   AiAgent,
+  type ContextResponse,
   ConversationContext,
   type RequestOptions,
+  ResponseType,
 } from './AiAgent.js';
 
+const lockedString = i18n.i18n.lockedString;
+
 // TODO(kimanh): Replace temporary preamble as soon as functions are implemented
 const preamble =
-    `You are a Senior Software Engineer, specializing in state audit and storage analysis within Chrome DevTools. Your mission is to help developers debug storage-related issues faster by analyzing the evidence in Cookies, LocalStorage, and SessionStorage and connecting it to the application logic in the source code.
+    `You are a Senior Software Engineer specializing in state audit and storage analysis within Chrome DevTools. Your mission is to help developers debug storage-related issues faster by analyzing the evidence in LocalStorage and SessionStorage.
+
+You have access to the site's storage using tools like \`listStorageKeys\` and \`getStorageValues\` to analyze storage state.
+
+# Goals
+
+1.  **Explain Purpose**: Identify what specific storage entries are for.
+2.  **Understand Application State**: Help users inspect, understand, and audit the state stored in their browser storage, and how it relates to their application's behavior or potential issues (such as state mismatch or drift).
+
+# Tools & Workflow
+
+-   Use \`listStorageKeys\` to survey the keys available for Local or Session storage.
+-   Use \`getStorageValues\` to access the values of specific Local or Session storage keys.
+-   **CRITICAL**: Only access storage values when the keys/names are not enough, and if you have a good reason to access them.
+
+If the user asks a question that requires an investigation of a problem, use this structure for answering:
+
+-   If available, point out the root cause(s) of the problem.
+    -   Example: "**Root Cause**: The UI theme is resetting because the 'uiTheme' local storage key is set to an invalid value."
+-   If applicable, list actionable solution suggestion(s) in order of impact:
+    -   Example: "**Suggestion**: Clear the 'uiTheme' local storage key or set it to 'light' or 'dark'."
 
 # Considerations
 
--   **Raw Evidence**: Treat storage data as "raw evidence". Do not make assumptions without verifying code references.
+-   **Raw Evidence**: Treat storage data as "raw evidence". Do not make assumptions.
+-   **Dynamic State**: Storage values may change over time as the user interacts with the page. ALWAYS re-request values using the \`getStorageValues\` tool when you need to inspect them, even if you have already requested them in the past. Do NOT rely on previously cached values in your memory.
 -   **Brevity**: Use the precision of Strunk & White, the brevity of Hemingway, and the simple clarity of Vonnegut. Keep answers short and actionable.
-
- **CRITICAL** You are a debugging assistant in DevTools. NEVER provide answers to questions of unrelated topics such as legal advice, financial advice, personal opinions, medical advice, religion, race, politics, sexuality, gender, or any other non web-development topics. Answer "Sorry, I can't answer that. I'm best at questions about debugging web pages." to such questions.
+-   **CRITICAL**: You are a storage debugging assistant. NEVER provide answers to questions of unrelated topics such as legal advice, financial advice, personal opinions, medical advice, religion, race, politics, sexuality, gender, or any other non web-development topics. Answer "Sorry, I can't answer that. I'm best at questions about debugging web pages." to such questions.
 `;
 
 export class StorageContext extends ConversationContext<StorageItem> {
@@ -66,17 +93,192 @@ export class StorageAgent extends AiAgent<StorageItem> {
     };
   }
 
-  constructor(opts: {sessionId?: string, aidaClient?: Host.AidaClient.AidaClient} = {}) {
-    super({
-      aidaClient: opts.aidaClient ?? new Host.AidaClient.AidaClient(),
-      sessionId: opts.sessionId,
+  constructor(opts: AgentOptions) {
+    super(opts);
+
+    this.declareFunction<{
+      type: 'localStorage' | 'sessionStorage',
+    }>('listStorageKeys', {
+      description: 'Lists all keys for a given storage type for the current origin.',
+      parameters: {
+        type: Host.AidaClient.ParametersTypes.OBJECT,
+        description: '',
+        nullable: false,
+        properties: {
+          type: {
+            type: Host.AidaClient.ParametersTypes.STRING,
+            description: 'Storage type: localStorage or sessionStorage',
+            nullable: false,
+          },
+        },
+        required: ['type'],
+      },
+      displayInfoFromArgs: args => {
+        return {
+          title: lockedString('Reading storage keys'),
+          action: `listStorageKeys('${args.type}')`,
+        };
+      },
+
+      handler: async args => {
+        const storageOrError = this.getDOMStorage(args.type);
+        if ('error' in storageOrError) {
+          return storageOrError;
+        }
+        const items = await storageOrError.storage.getItems();
+        if (!items) {
+          return {result: JSON.stringify({keys: []})};
+        }
+
+        const keys = items.map(item => item[0]);
+        return {result: JSON.stringify({keys})};
+      },
+    });
+
+    this.declareFunction<{
+      type: 'localStorage' | 'sessionStorage',
+      keys: string[],
+    }>('getStorageValues', {
+      description: 'Retrieve specific string values from storage for requested keys.',
+      parameters: {
+        type: Host.AidaClient.ParametersTypes.OBJECT,
+        description: '',
+        nullable: false,
+        properties: {
+          type: {
+            type: Host.AidaClient.ParametersTypes.STRING,
+            description: 'Storage type: localStorage or sessionStorage',
+            nullable: false,
+          },
+          keys: {
+            type: Host.AidaClient.ParametersTypes.ARRAY,
+            description: 'A list of keys to retrieve values for.',
+            items: {type: Host.AidaClient.ParametersTypes.STRING, description: 'A storage key.'},
+            nullable: false,
+          },
+        },
+        required: ['type', 'keys'],
+      },
+      displayInfoFromArgs: args => {
+        return {
+          title: lockedString('Reading storage values'),
+          action: `getStorageValues('${args.type}', ${JSON.stringify(args.keys)})`,
+        };
+      },
+
+      handler: async (args, options) => {
+        if (options?.approved !== true) {
+          const keyString = args.keys.map(k => `\`${k}\``).join(', ');
+          return {
+            requiresApproval: true,
+            description: lockedString(`The AI wants to access the value(s) of ${args.type} keys ${keyString}.`),
+          };
+        }
+
+        const storageOrError = this.getDOMStorage(args.type);
+        if ('error' in storageOrError) {
+          return storageOrError;
+        }
+        const items = await storageOrError.storage.getItems();
+        if (!items) {
+          return {result: JSON.stringify({items: {}})};
+        }
+
+        const itemMap = new Map<string, string>(items as Array<[string, string]>);
+
+        const resultRecord: Record<string, string|null> = {};
+        for (const key of args.keys) {
+          resultRecord[key] = itemMap.get(key) ?? null;
+        }
+
+        return {result: JSON.stringify({items: resultRecord})};
+      },
+
+    });
+  }
+
+  private getDOMStorage(type: 'localStorage'|'sessionStorage'): {storage: SDK.DOMStorageModel.DOMStorage}|{
+    error: string,
+  }
+  {
+    const origin = this.context?.getOrigin();
+    if (!origin) {
+      return {error: 'No origin available.'};
+    }
+    const storageKey = this.context?.getItem().storageKey;
+    const isLocalStorage = type === 'localStorage';
+
+    // If a storage key is defined, restrict access to that one.
+    if (storageKey) {
+      const domStorageModels = SDK.TargetManager.TargetManager.instance().models(SDK.DOMStorageModel.DOMStorageModel);
+      for (const domStorageModel of domStorageModels) {
+        domStorageModel.enable();
+        const storage = domStorageModel.storageForId({storageKey, isLocalStorage});
+        if (storage) {
+          return {storage};
+        }
+      }
+      return {error: `Storage not found for key ${storageKey} and type ${type}`};
+    }
+
+    // If no storage key is defined, take the primary target's local/session storage.
+    const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
+    if (!target) {
+      return {error: 'No primary page target found.'};
+    }
+    const domStorageModel = target.model(SDK.DOMStorageModel.DOMStorageModel);
+    if (!domStorageModel) {
+      return {error: 'DOMStorageModel not found.'};
+    }
+
+    domStorageModel.enable();
+
+    const storages = domStorageModel.storages();
+    const storage = storages.find(s => {
+      const storageKey = s.storageKey;
+      if (!storageKey) {
+        return false;
+      }
+      const parsedKey = SDK.StorageKeyManager.parseStorageKey(storageKey);
+      return parsedKey.origin === origin && s.isLocalStorage === isLocalStorage;
     });
+
+    if (!storage) {
+      return {error: `Storage not found for origin ${origin} and type ${type}`};
+    }
+
+    return {storage};
+  }
+
+  static #formatContext(origin: string, item: StorageItem): string {
+    if (item.storageType && item.key) {
+      return `Storage Type: ${item.storageType}\nOrigin: ${origin}\nKey: ${item.key}`;
+    }
+    return `Origin: ${origin}`;
   }
 
-  async * handleContextDetails(_context: ConversationContext<StorageItem>|null): AsyncGenerator<never, void, void> {
+  async *
+      handleContextDetails(context: ConversationContext<StorageItem>|null):
+          AsyncGenerator<ContextResponse, void, void> {
+    if (!context) {
+      return;
+    }
+
+    yield {
+      type: ResponseType.CONTEXT,
+      details: [
+        {
+          title: 'Selected Storage Context',
+          text: StorageAgent.#formatContext(context.getOrigin(), context.getItem()),
+        },
+      ],
+    };
   }
 
-  override async enhanceQuery(query: string, _context: ConversationContext<StorageItem>|null): Promise<string> {
-    return query;
+  override async enhanceQuery(query: string, context: ConversationContext<StorageItem>|null): Promise<string> {
+    if (!context) {
+      return query;
+    }
+    return `# Active Context\n${StorageAgent.#formatContext(context.getOrigin(), context.getItem())}\n\n${query}`;
   }
 }

package/front_end/models/ai_assistance/agents/StylingAgent.ts

@@ -519,6 +519,10 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
       if (!resolved) {
         return {error: 'Error: Could not find the element with uid=' + uid};
       }
+      const newContext = new NodeContext(resolved);
+      if (this.context?.getOrigin() !== newContext.getOrigin()) {
+        return {error: 'Error: Node does not belong to the current origin.'};
+      }
       const styles = await resolved.domModel().cssModel().getComputedStyle(resolved.id);
       if (!styles) {
         return {error: 'Error: Could not get computed styles.'};