chrome-devtools-frontend 1.0.1596535 → 1.0.1597624

package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts

@@ -0,0 +1,127 @@
+// Copyright 2026 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as Host from '../../../core/host/host.js';
+import * as i18n from '../../../core/i18n/i18n.js';
+import * as Root from '../../../core/root/root.js';
+import type * as LHModel from '../../lighthouse/lighthouse.js';
+
+import {
+  AiAgent,
+  type ContextDetail,
+  type ContextResponse,
+  ConversationContext,
+  type RequestOptions,
+  ResponseType,
+} from './AiAgent.js';
+
+/**
+ * WARNING: preamble defined in code is only used when userTier is
+ * TESTERS. Otherwise, a server-side preamble is used (see
+ * chrome_preambles.gcl). Sync local changes with the server-side.
+ */
+const preamble = `You are an accessibility agent.
+
+# Considerations
+* Keep your analysis concise and focused, highlighting only the most critical aspects for a software engineer.
+* Answer questions directly, using the provided links whenever relevant.
+* Always double-check links to make sure they are complete and correct.
+* **CRITICAL** You are an accessibility agent. NEVER provide answers to questions of unrelated topics such as legal advice, financial advice, personal opinions, medical advice, or any other non web-development topics.
+`;
+
+/*
+* Strings that don't need to be translated at this time.
+*/
+const UIStringsNotTranslate = {
+  /**
+   * @description Title for thinking step of the accessibility agent.
+   */
+  inspectingAudits: 'Inspecting audits',
+} as const;
+
+const lockedString = i18n.i18n.lockedString;
+
+export class Context extends ConversationContext<LHModel.ReporterTypes.ReportJSON> {
+  #lh: LHModel.ReporterTypes.ReportJSON;
+
+  constructor(report: LHModel.ReporterTypes.ReportJSON) {
+    super();
+    this.#lh = report;
+  }
+
+  #url(): string {
+    return this.#lh.finalUrl ?? this.#lh.finalDisplayedUrl;
+  }
+
+  override getOrigin(): string {
+    return new URL(this.#url()).origin;
+  }
+
+  override getItem(): LHModel.ReporterTypes.ReportJSON {
+    return this.#lh;
+  }
+
+  override getTitle(): string {
+    return `Lighthouse report: ${this.#url()}`;
+  }
+}
+
+/**
+ * One agent instance handles one conversation. Create a new agent
+ * instance for a new conversation.
+ */
+export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON> {
+  readonly preamble = preamble;
+  readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_ACCESSIBILITY_AGENT;
+
+  get userTier(): string|undefined {
+    // TODO(b/491772868): tidy up userTier & feature flags in the backend.
+    return Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
+  }
+
+  get options(): RequestOptions {
+    // TODO(b/491772868): tidy up userTier & feature flags in the backend.
+    const temperature = Root.Runtime.hostConfig.devToolsAiAssistanceFileAgent?.temperature;
+    const modelId = Root.Runtime.hostConfig.devToolsAiAssistanceFileAgent?.modelId;
+
+    return {
+      temperature,
+      modelId,
+    };
+  }
+
+  async *
+      handleContextDetails(selectedFile: ConversationContext<LHModel.ReporterTypes.ReportJSON>|null):
+          AsyncGenerator<ContextResponse, void, void> {
+    if (!selectedFile) {
+      return;
+    }
+
+    yield {
+      type: ResponseType.CONTEXT,
+      title: lockedString(UIStringsNotTranslate.inspectingAudits),
+      details: createContextDetails(selectedFile),
+    };
+  }
+
+  override async enhanceQuery(query: string, lhr: ConversationContext<LHModel.ReporterTypes.ReportJSON>|null):
+      Promise<string> {
+    const enhancedQuery = lhr ?
+        // TODO: formatter for LH report.
+        `# Lighthouse Report\n${JSON.stringify(lhr.getItem(), null, 2)}\n\n# User request\n\n` :
+        '';
+    return `${enhancedQuery}${query}`;
+  }
+}
+
+function createContextDetails(_lhr: ConversationContext<LHModel.ReporterTypes.ReportJSON>):
+    [ContextDetail, ...ContextDetail[]] {
+  return [
+    {
+      title: 'Lighthouse report',
+      // TODO(b/491772868);
+      text: ''
+    },
+  ];
+}

package/front_end/models/ai_assistance/agents/AiAgent.ts

@@ -29,6 +29,7 @@ export const enum ErrorType {
   ABORT = 'abort',
   MAX_STEPS = 'max-steps',
   BLOCK = 'block',
+  CROSS_ORIGIN = 'cross-origin'
 }
 
 export const enum MultimodalInputType {
@@ -49,6 +50,7 @@ export interface AnswerResponse {
   complete: boolean;
   rpcId?: Host.AidaClient.RpcGlobalId;
   suggestions?: [string, ...string[]];
+  widgets?: AiWidget[];
 }
 
 export interface SuggestionsResponse {
@@ -148,6 +150,7 @@ export interface AgentOptions {
   confirmSideEffectForTest?: typeof Promise.withResolvers;
   onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
   history?: Host.AidaClient.Content[];
+  allowedOrigin?: () => string | undefined;
 }
 
 export interface ParsedAnswer {
@@ -238,8 +241,16 @@ export interface CoreVitalsAiWidget {
   };
 }
 
+export interface StylePropertiesAiWidget {
+  name: 'STYLE_PROPERTIES';
+  data: {
+    backendNodeId: Protocol.DOM.BackendNodeId,
+    selector?: string,
+  };
+}
+
 // This type will grow as we add more widgets.
-export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget;
+export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget;
 
 export type FunctionCallHandlerResult<Result> = {
   requiresApproval: true,
@@ -394,6 +405,10 @@ export abstract class AiAgent<T> {
     this.#facts.clear();
   }
 
+  popPendingMultimodalInput(): MultimodalInput|undefined {
+    return undefined;
+  }
+
   preambleFeatures(): string[] {
     return [];
   }
@@ -517,6 +532,10 @@ export abstract class AiAgent<T> {
     return this.parseTextResponseForSuggestions(response.trim());
   }
 
+  protected async finalizeAnswer(answer: AnswerResponse): Promise<AnswerResponse> {
+    return answer;
+  }
+
   /**
    * Declare a function that the AI model can call.
    * @param name The name of the function
@@ -542,6 +561,9 @@ export abstract class AiAgent<T> {
     this.#functionDeclarations.clear();
   }
 
+  protected async preRun(): Promise<void> {
+  }
+
   async *
       run(
           initialQuery: string,
@@ -551,6 +573,7 @@ export abstract class AiAgent<T> {
           },
           multimodalInput?: MultimodalInput,
           ): AsyncGenerator<ResponseData, void, void> {
+    await this.preRun();
     await options.selected?.refresh();
     if (options.selected) {
       this.context = options.selected;
@@ -628,13 +651,13 @@ export abstract class AiAgent<T> {
           });
         }
         Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceAnswerReceived);
-        yield {
+        yield await this.finalizeAnswer({
           type: ResponseType.ANSWER,
           text: parsedResponse.answer,
           suggestions: parsedResponse.suggestions,
           complete: true,
           rpcId,
-        };
+        });
         if (!functionCall) {
           break;
         }

package/front_end/models/ai_assistance/agents/ContextSelectionAgent.snapshot.txt

@@ -120,6 +120,6 @@ Content:
     "client_version": "unit_test"
   },
   "functionality_type": 5,
-  "client_feature": 9
+  "client_feature": 25
 }
 === end content

package/front_end/models/ai_assistance/agents/ContextSelectionAgent.ts

@@ -2,11 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import * as Common from '../../../core/common/common.js';
 import * as Host from '../../../core/host/host.js';
 import * as i18n from '../../../core/i18n/i18n.js';
 import * as Root from '../../../core/root/root.js';
-import * as SDK from '../../../core/sdk/sdk.js';
+import type * as SDK from '../../../core/sdk/sdk.js';
 import * as Logs from '../../logs/logs.js';
 import * as NetworkTimeCalculator from '../../network_time_calculator/network_time_calculator.js';
 import type * as Trace from '../../trace/trace.js';
@@ -63,7 +62,7 @@ You aim to help developers of all levels, prioritizing teaching web concepts as
  */
 export class ContextSelectionAgent extends AiAgent<never> {
   readonly preamble = preamble;
-  readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_FILE_AGENT;
+  readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_CONTEXT_SELECTION_AGENT;
   get userTier(): string|undefined {
     // TODO: Make this depend on variable.
     return Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
@@ -81,6 +80,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
   readonly #performanceRecordAndReload?: () => Promise<Trace.TraceModel.ParsedTrace>;
   readonly #onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
   readonly #networkTimeCalculator?: NetworkTimeCalculator.NetworkTransferTimeCalculator;
+  #allowedOrigin: () => string | undefined;
 
   constructor(opts: AgentOptions&{
     performanceRecordAndReload?: () => Promise<Trace.TraceModel.ParsedTrace>,
@@ -91,6 +91,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
     this.#performanceRecordAndReload = opts.performanceRecordAndReload;
     this.#onInspectElement = opts.onInspectElement;
     this.#networkTimeCalculator = opts.networkTimeCalculator;
+    this.#allowedOrigin = opts.allowedOrigin ?? (() => undefined);
 
     this.declareFunction<Record<string, never>>('listNetworkRequests', {
       description: `Gives a list of network requests including URL, status code, and duration.`,
@@ -109,12 +110,12 @@ export class ContextSelectionAgent extends AiAgent<never> {
       },
       handler: async () => {
         const requests = [];
-        const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
-        const inspectedURL = target?.inspectedURL();
-        const mainSecurityOrigin = inspectedURL ? new Common.ParsedURL.ParsedURL(inspectedURL).securityOrigin() : null;
+        const origin = this.#allowedOrigin();
 
+        let hasCrossOriginRequest = false;
         for (const request of Logs.NetworkLog.NetworkLog.instance().requests()) {
-          if (mainSecurityOrigin && request.securityOrigin() !== mainSecurityOrigin) {
+          if (origin && request.securityOrigin() !== origin) {
+            hasCrossOriginRequest = true;
             continue;
           }
 
@@ -129,7 +130,9 @@ export class ContextSelectionAgent extends AiAgent<never> {
 
         if (requests.length === 0) {
           return {
-            error: 'No requests recorded by DevTools',
+            error: hasCrossOriginRequest ?
+                `No requests showing with origin ${origin}. Tell the user to start a new chat` :
+                'No requests recorded by DevTools',
           };
         }
 

package/front_end/models/ai_assistance/agents/StylingAgent.snapshot.txt

@@ -121,6 +121,30 @@ Content:
           "title"
         ]
       }
+    },
+    {
+      "name": "activateDeviceEmulation",
+      "description": "Sets emulation viewing mode for a specific device and optionally enables vision deficiency emulation.",
+      "parameters": {
+        "type": 6,
+        "description": "",
+        "nullable": false,
+        "properties": {
+          "deviceName": {
+            "type": 1,
+            "description": "The name of the device to emulate. Allowed values: Pixel 3 XL, Pixel 7, Samsung Galaxy S8+, Samsung Galaxy S20 Ultra, Surface Pro 7, Surface Duo, Galaxy Z Fold 5, Asus Zenbook Fold, Samsung Galaxy A51/71, Nest Hub Max, Nest Hub, iPhone 4, iPhone 5/SE, iPhone 6/7/8, iPhone SE, iPhone XR, iPhone 12 Pro, iPhone 14 Pro Max, iPad Mini, iPad Air, iPad Pro.",
+            "nullable": false
+          },
+          "visionDeficiency": {
+            "type": 1,
+            "description": "Optional vision deficiency to emulate. Allowed values: blurredVision, reducedContrast, achromatopsia, deuteranopia, protanopia, tritanopia.",
+            "nullable": true
+          }
+        },
+        "required": [
+          "deviceName"
+        ]
+      }
     }
   ],
   "options": {