chrome-devtools-frontend 1.0.1597448 → 1.0.1598808

package/docs/ui_engineering.md

@@ -44,7 +44,7 @@ To embed another presenter (`UI.Widget`) in the lit-html template, use `widget(<
 
 This will instantiate a `Widget` class with the web component as its `element` and, optionally, will set the properties provided in the second parameter. The widget won’t be re-instantiated on the subsequent template renders, but the properties would be updated. For this to work, the widget needs to accept `HTMLElement` as a sole constructor parameter and properties need to be public members or setters.
 
-For backwards compatibility, the first argument to `widgetConfig` can also be a factory function: `widget(element => new MyWidget(foo, bar, element))`. Similar to the class constructor version, `element` is the actual `<devtools-widget>` so the following two invocations of `widgetConfig` are equivalent: `widget(MyWidget)` and `widget(element => new MyWidget(element))`.
+For backwards compatibility, the first argument to `widget` can also be a factory function: `widget(element => new MyWidget(foo, bar, element))`. Similar to the class constructor version, `element` is the actual `<devtools-widget>` so the following two invocations of `widget` are equivalent: `widget(MyWidget)` and `widget(element => new MyWidget(element))`.
 
 ## Styling
 To prevent style conflicts in widgets without relying on shadow DOM, we use the CSS [`@scope`](https://developer.mozilla.org/en-US/docs/Web/CSS/@scope) at-rule for style encapsulation. This ensures that styles defined for a widget do not leak out and affect other components.
@@ -93,9 +93,9 @@ In this example, the `.title` style will apply within the parent widget but will
 ## Examples
 
 ```html
-<devtools-widget .widgetConfig=${widgetConfig(ElementsPanel)}>
+<devtools-widget ${widget(ElementsPanel)}>
   <devtools-split-view>
-    <devtools-widget slot="main" .widgetConfig=${widgetConfig(ElementsTree)}></devtools-widget>
+    <devtools-widget slot="main" ${widget(ElementsTree)}></devtools-widget>
     <devtools-tab-pane slot="sidebar">
       ${widget(StylesPane, {element: input.element})}
       ${widget(ComputedPane, {element: input.element})}
@@ -1169,12 +1169,12 @@ export const DEFAULT_VIEW = (input, _output, target) => {
     <div>
       <devtools-split-view direction=${this.vertical ? 'column' : 'row'} sidebar-position="first"
           sidebar-initial-size="200">
-        <devtools-widget slot="sidebar" .widgetConfig=${widgetConfig(SidebarPanel,
+        <devtools-widget slot="sidebar" ${widget(SidebarPanel,
             {minimumSize: {width: 100, height: 25}})}></devtools-widget>
         <devtools-split-view direction="column" sidebar-position="second" slot="main"
             direction="row" sidebar-position="$this.dockedLeft ? 'second' : 'first'}">
-          <devtools-widget slot="main" .widgetConfig=${widgetConfig(UI.Widget.EmptyWidget)}></devtools-widget>
-          <devtools-widget slot="sidebar" .widgetConfig=${widgetConfig(DetailsView)}></devtools-widget>
+          <devtools-widget slot="main" ${widget(UI.Widget.EmptyWidget)}></devtools-widget>
+          <devtools-widget slot="sidebar" ${widget(DetailsView)}></devtools-widget>
         </devtools-split-view>
       </devtools-split-view>
     </div>`,

package/front_end/core/host/AidaClient.ts

@@ -133,6 +133,12 @@ export enum ClientFeature {
   CHROME_PATCH_AGENT = 12,
   // Chrome AI Assistance Performance Agent.
   CHROME_PERFORMANCE_FULL_AGENT = 24,
+  // Chrome Context Selection Agent.
+  CHROME_CONTEXT_SELECTION_AGENT = 25,
+  // Chrome Accessibility Agent
+  CHROME_ACCESSIBILITY_AGENT = 26,
+  // Chrome AI Assistance Conversation Summary Agent.
+  CHROME_CONVERSATION_SUMMARY_AGENT = 27,
 
   // Removed features (for reference).
   // Chrome AI Assistance Performance Insights Agent.

package/front_end/core/sdk/CPUThrottlingManager.ts

@@ -35,7 +35,7 @@ const str_ = i18n.i18n.registerUIStrings('core/sdk/CPUThrottlingManager.ts', UIS
 const i18nString = i18n.i18n.getLocalizedString.bind(undefined, str_);
 const i18nLazyString = i18n.i18n.getLazilyComputedLocalizedString.bind(undefined, str_);
 
-let throttlingManagerInstance: CPUThrottlingManager;
+let throttlingManagerInstance: CPUThrottlingManager|undefined;
 
 export class CPUThrottlingManager extends Common.ObjectWrapper.ObjectWrapper<EventTypes> implements
     SDKModelObserver<EmulationModel> {
@@ -62,6 +62,10 @@ export class CPUThrottlingManager extends Common.ObjectWrapper.ObjectWrapper<Eve
     return throttlingManagerInstance;
   }
 
+  static removeInstance(): void {
+    throttlingManagerInstance = undefined;
+  }
+
   cpuThrottlingRate(): number {
     return this.#cpuThrottlingOption.rate();
   }

package/front_end/core/sdk/CSSMatchedStyles.ts

@@ -21,6 +21,7 @@ import {
   BinOpMatcher,
   ColorMatcher,
   ColorMixMatcher,
+  ContrastColorMatcher,
   CustomFunctionMatcher,
   defaultValueForCSSType,
   EnvFunctionMatcher,
@@ -932,6 +933,7 @@ export class CSSMatchedStyles {
       new VariableMatcher(this, style),
       new ColorMatcher(() => computedStyles?.get('color') ?? null),
       new ColorMixMatcher(),
+      new ContrastColorMatcher(),
       new URLMatcher(),
       new AngleMatcher(),
       new LinkableNameMatcher(),

package/front_end/core/sdk/CSSPropertyParserMatchers.ts

@@ -433,6 +433,34 @@ export class ColorMixMatcher extends matcherBase(ColorMixMatch) {
   }
 }
 
+export class ContrastColorMatch implements Match {
+  constructor(readonly text: string, readonly node: CodeMirror.SyntaxNode, readonly color: CodeMirror.SyntaxNode[]) {
+  }
+}
+
+// clang-format off
+export class ContrastColorMatcher extends matcherBase(ContrastColorMatch) {
+  // clang-format on
+  override accepts(propertyName: string): boolean {
+    return cssMetadata().isColorAwareProperty(propertyName);
+  }
+  override matches(node: CodeMirror.SyntaxNode, matching: BottomUpTreeMatching): ContrastColorMatch|null {
+    if (node.name !== 'CallExpression' || matching.ast.text(node.getChild('Callee')) !== 'contrast-color') {
+      return null;
+    }
+
+    if (matching.getComputedText(node) === '') {
+      return null;
+    }
+
+    const args = ASTUtils.callArgs(node);
+    if (args.length !== 1) {
+      return null;
+    }
+    return new ContrastColorMatch(matching.ast.text(node), node, args[0]);
+  }
+}
+
 // clang-format off
 export class URLMatch implements Match {
   constructor(

package/front_end/models/ai_assistance/AiConversation.ts

@@ -1,10 +1,10 @@
 // Copyright 2024 The Chromium Authors
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-
+import * as Common from '../../core/common/common.js';
 import * as Host from '../../core/host/host.js';
 import * as Root from '../../core/root/root.js';
-import type * as SDK from '../../core/sdk/sdk.js';
+import * as SDK from '../../core/sdk/sdk.js';
 import type * as Trace from '../../models/trace/trace.js';
 import * as Greendev from '../greendev/greendev.js';
 import type * as NetworkTimeCalculator from '../network_time_calculator/network_time_calculator.js';
@@ -13,6 +13,7 @@ import {
   type AiAgent,
   type ContextDetail,
   type ConversationContext,
+  ErrorType,
   type MultimodalInput,
   type ResponseData,
   ResponseType,
@@ -264,21 +265,23 @@ export class AiConversation {
       id: this.id,
       history: this.history
                    .map(item => {
-                     if (item.type === ResponseType.CONTEXT_CHANGE) {
-                       return null;
-                     }
-
-                     if (item.type === ResponseType.USER_QUERY) {
-                       return {...item, imageInput: undefined};
-                     }
-                     // Remove the `confirm()`-function because `structuredClone()` throws on functions
-                     if (item.type === ResponseType.SIDE_EFFECT) {
-                       return {...item, confirm: undefined};
-                     }
-                     if (item.type === ResponseType.CONTEXT && item.widgets) {
-                       return {...item, widgets: undefined};
+                     switch (item.type) {
+                       case ResponseType.CONTEXT_CHANGE: {
+                         return null;
+                       }
+                       case ResponseType.USER_QUERY: {
+                         return {...item, imageInput: undefined};
+                       }
+                       case ResponseType.SIDE_EFFECT: {
+                         return {...item, confirm: undefined};
+                       }
+                       case ResponseType.CONTEXT:
+                       case ResponseType.ACTION: {
+                         return {...item, widgets: undefined};
+                       }
+                       default:
+                         return item;
                      }
-                     return item;
                    })
                    .filter(history => !!history),
       type: this.#type,
@@ -313,6 +316,7 @@ export class AiConversation {
       performanceRecordAndReload: this.#performanceRecordAndReload,
       onInspectElement: this.#onInspectElement,
       networkTimeCalculator: this.#networkTimeCalculator,
+      allowedOrigin: this.allowedOrigin,
       history,
     };
     switch (type) {
@@ -369,12 +373,6 @@ export class AiConversation {
     void this.addHistoryItem(userQuery);
     yield userQuery;
 
-    this.#setOriginIfEmpty(this.selectedContext?.getOrigin());
-
-    if (this.isBlockedByOrigin) {
-      throw new Error('Cross-origin context data should not be included');
-    }
-
     yield* this.#runAgent(initialQuery, options);
   }
 
@@ -390,6 +388,15 @@ export class AiConversation {
             multimodalInput?: MultimodalInput,
           } = {},
           ): AsyncGenerator<ResponseData, void, void> {
+    this.#setOriginIfEmpty(this.selectedContext?.getOrigin());
+    if (this.isBlockedByOrigin) {
+      yield {
+        type: ResponseType.ERROR,
+        error: ErrorType.CROSS_ORIGIN,
+      };
+      return;
+    }
+
     function shouldAddToHistory(data: ResponseData): boolean {
       if (data.type === ResponseType.CONTEXT_CHANGE) {
         return false;
@@ -445,6 +452,17 @@ export class AiConversation {
   get type(): ConversationType {
     return this.#type;
   }
+
+  allowedOrigin = (): string|undefined => {
+    if (this.#origin) {
+      return this.#origin;
+    }
+    const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
+    const inspectedURL = target?.inspectedURL();
+    this.#origin = inspectedURL ? new Common.ParsedURL.ParsedURL(inspectedURL).securityOrigin() : undefined;
+
+    return this.#origin;
+  };
 }
 
 function isAiAssistanceServerSideLoggingEnabled(): boolean {

package/front_end/models/ai_assistance/ChangeManager.ts

@@ -9,6 +9,8 @@ import type * as Protocol from '../../generated/protocol.js';
 
 export interface Change {
   groupId: string;
+  // Optional turn ID to group changes from the same turn.
+  turnId?: number;
   // Optional about where in the source the selector was defined.
   sourceLocation?: string;
   // Selector used by the page or a simple selector as the fallback.
@@ -17,6 +19,7 @@ export interface Change {
   simpleSelector?: string;
   className: string;
   styles: Record<string, string>;
+  backendNodeId?: Protocol.DOM.BackendNodeId;
 }
 
 function formatStyles(styles: Record<string, string>, indent = 2): string {
@@ -101,6 +104,7 @@ export class ChangeManager {
       // it currently causes crashes in the Styles tab when duplicate selectors exist (crbug.com/393515428).
       // This workaround avoids that crash.
       existingChange.groupId = change.groupId;
+      existingChange.turnId = change.turnId;
     } else {
       changes.push({
         ...change,
@@ -122,6 +126,18 @@ export class ChangeManager {
         .join('\n\n');
   }
 
+  getChangedNodesForGroupId(groupId: string, turnId?: number): Protocol.DOM.BackendNodeId[] {
+    const nodes = new Set<Protocol.DOM.BackendNodeId>();
+    for (const changes of this.#stylesheetChanges.values()) {
+      for (const change of changes) {
+        if (change.groupId === groupId && change.backendNodeId && (turnId === undefined || change.turnId === turnId)) {
+          nodes.add(change.backendNodeId);
+        }
+      }
+    }
+    return Array.from(nodes);
+  }
+
   #formatChangesForInspectorStylesheet(changes: Change[]): string {
     return changes
         .map(change => {

package/front_end/models/ai_assistance/ExtensionScope.ts

@@ -22,6 +22,7 @@ interface ElementContext {
   selector: string;
   simpleSelector?: string;
   sourceLocation?: string;
+  backendNodeId?: Protocol.DOM.BackendNodeId;
 }
 
 /**
@@ -33,6 +34,7 @@ export class ExtensionScope {
                     }) => Promise<void>> = [];
   #changeManager: ChangeManager;
   #agentId: string;
+  #turnId?: number;
   /** Don't use directly use the getter */
   #frameId?: Protocol.Page.FrameId|null;
   /** Don't use directly use the getter */
@@ -40,15 +42,15 @@ export class ExtensionScope {
 
   readonly #bindingMutex = new Common.Mutex.Mutex();
 
-  constructor(changes: ChangeManager, agentId: string, selectedNode: SDK.DOMModel.DOMNode|null) {
+  constructor(changes: ChangeManager, agentId: string, selectedNode: SDK.DOMModel.DOMNode|null, turnId?: number) {
     this.#changeManager = changes;
     const frameId = selectedNode?.frameId();
     const target = selectedNode?.domModel().target();
     this.#agentId = agentId;
+    this.#turnId = turnId;
     this.#target = target;
     this.#frameId = frameId;
   }
-
   get target(): SDK.Target.Target {
     if (!this.#target) {
       throw new Error('Target is not found for executing code');
@@ -274,6 +276,7 @@ export class ExtensionScope {
       throw new Error('Node is not found');
     }
 
+    const backendNodeId = node.backendNodeId();
     try {
       const matchedStyles = await cssModel.getMatchedStyles(node.id);
 
@@ -297,6 +300,7 @@ export class ExtensionScope {
         selector,
         simpleSelector: ExtensionScope.getSelectorForNode(node),
         sourceLocation: ExtensionScope.getSourceLocation(styleRule),
+        backendNodeId,
       };
     } catch {
       // no-op to allow the fallback below to run.
@@ -305,6 +309,7 @@ export class ExtensionScope {
     // Fallback
     return {
       selector: ExtensionScope.getSelectorForNode(node),
+      backendNodeId,
     };
   }
 
@@ -337,7 +342,8 @@ export class ExtensionScope {
 
       let context: ElementContext = {
         // TODO: Should this a be a *?
-        selector: ''
+        selector: '',
+        backendNodeId: undefined,
       };
       try {
         context = await this.#computeContextFromElement(element.object);
@@ -351,11 +357,13 @@ export class ExtensionScope {
         const sanitizedStyles = await this.sanitizedStyleChanges(context.selector, arg.styles);
         const styleChanges = await this.#changeManager.addChange(cssModel, this.frameId, {
           groupId: this.#agentId,
+          turnId: this.#turnId,
           sourceLocation: context.sourceLocation,
           selector: context.selector,
           simpleSelector: context.simpleSelector,
           className: arg.className,
           styles: sanitizedStyles,
+          backendNodeId: context.backendNodeId,
         });
         await this.#simpleEval(executionContext, `freestyler.respond(${id}, ${JSON.stringify(styleChanges)})`);
       } catch (error) {

package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts

@@ -0,0 +1,127 @@
+// Copyright 2026 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as Host from '../../../core/host/host.js';
+import * as i18n from '../../../core/i18n/i18n.js';
+import * as Root from '../../../core/root/root.js';
+import type * as LHModel from '../../lighthouse/lighthouse.js';
+
+import {
+  AiAgent,
+  type ContextDetail,
+  type ContextResponse,
+  ConversationContext,
+  type RequestOptions,
+  ResponseType,
+} from './AiAgent.js';
+
+/**
+ * WARNING: preamble defined in code is only used when userTier is
+ * TESTERS. Otherwise, a server-side preamble is used (see
+ * chrome_preambles.gcl). Sync local changes with the server-side.
+ */
+const preamble = `You are an accessibility agent.
+
+# Considerations
+* Keep your analysis concise and focused, highlighting only the most critical aspects for a software engineer.
+* Answer questions directly, using the provided links whenever relevant.
+* Always double-check links to make sure they are complete and correct.
+* **CRITICAL** You are an accessibility agent. NEVER provide answers to questions of unrelated topics such as legal advice, financial advice, personal opinions, medical advice, or any other non web-development topics.
+`;
+
+/*
+* Strings that don't need to be translated at this time.
+*/
+const UIStringsNotTranslate = {
+  /**
+   * @description Title for thinking step of the accessibility agent.
+   */
+  inspectingAudits: 'Inspecting audits',
+} as const;
+
+const lockedString = i18n.i18n.lockedString;
+
+export class Context extends ConversationContext<LHModel.ReporterTypes.ReportJSON> {
+  #lh: LHModel.ReporterTypes.ReportJSON;
+
+  constructor(report: LHModel.ReporterTypes.ReportJSON) {
+    super();
+    this.#lh = report;
+  }
+
+  #url(): string {
+    return this.#lh.finalUrl ?? this.#lh.finalDisplayedUrl;
+  }
+
+  override getOrigin(): string {
+    return new URL(this.#url()).origin;
+  }
+
+  override getItem(): LHModel.ReporterTypes.ReportJSON {
+    return this.#lh;
+  }
+
+  override getTitle(): string {
+    return `Lighthouse report: ${this.#url()}`;
+  }
+}
+
+/**
+ * One agent instance handles one conversation. Create a new agent
+ * instance for a new conversation.
+ */
+export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON> {
+  readonly preamble = preamble;
+  readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_ACCESSIBILITY_AGENT;
+
+  get userTier(): string|undefined {
+    // TODO(b/491772868): tidy up userTier & feature flags in the backend.
+    return Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
+  }
+
+  get options(): RequestOptions {
+    // TODO(b/491772868): tidy up userTier & feature flags in the backend.
+    const temperature = Root.Runtime.hostConfig.devToolsAiAssistanceFileAgent?.temperature;
+    const modelId = Root.Runtime.hostConfig.devToolsAiAssistanceFileAgent?.modelId;
+
+    return {
+      temperature,
+      modelId,
+    };
+  }
+
+  async *
+      handleContextDetails(selectedFile: ConversationContext<LHModel.ReporterTypes.ReportJSON>|null):
+          AsyncGenerator<ContextResponse, void, void> {
+    if (!selectedFile) {
+      return;
+    }
+
+    yield {
+      type: ResponseType.CONTEXT,
+      title: lockedString(UIStringsNotTranslate.inspectingAudits),
+      details: createContextDetails(selectedFile),
+    };
+  }
+
+  override async enhanceQuery(query: string, lhr: ConversationContext<LHModel.ReporterTypes.ReportJSON>|null):
+      Promise<string> {
+    const enhancedQuery = lhr ?
+        // TODO: formatter for LH report.
+        `# Lighthouse Report\n${JSON.stringify(lhr.getItem(), null, 2)}\n\n# User request\n\n` :
+        '';
+    return `${enhancedQuery}${query}`;
+  }
+}
+
+function createContextDetails(_lhr: ConversationContext<LHModel.ReporterTypes.ReportJSON>):
+    [ContextDetail, ...ContextDetail[]] {
+  return [
+    {
+      title: 'Lighthouse report',
+      // TODO(b/491772868);
+      text: ''
+    },
+  ];
+}

package/front_end/models/ai_assistance/agents/AiAgent.ts

@@ -29,6 +29,7 @@ export const enum ErrorType {
   ABORT = 'abort',
   MAX_STEPS = 'max-steps',
   BLOCK = 'block',
+  CROSS_ORIGIN = 'cross-origin'
 }
 
 export const enum MultimodalInputType {
@@ -49,6 +50,7 @@ export interface AnswerResponse {
   complete: boolean;
   rpcId?: Host.AidaClient.RpcGlobalId;
   suggestions?: [string, ...string[]];
+  widgets?: AiWidget[];
 }
 
 export interface SuggestionsResponse {
@@ -148,6 +150,7 @@ export interface AgentOptions {
   confirmSideEffectForTest?: typeof Promise.withResolvers;
   onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
   history?: Host.AidaClient.Content[];
+  allowedOrigin?: () => string | undefined;
 }
 
 export interface ParsedAnswer {
@@ -238,8 +241,23 @@ export interface CoreVitalsAiWidget {
   };
 }
 
+export interface StylePropertiesAiWidget {
+  name: 'STYLE_PROPERTIES';
+  data: {
+    backendNodeId: Protocol.DOM.BackendNodeId,
+    selector?: string,
+  };
+}
+
+export interface DomTreeAiWidget {
+  name: 'DOM_TREE';
+  data: {
+    root: SDK.DOMModel.DOMNodeSnapshot,
+  };
+}
+
 // This type will grow as we add more widgets.
-export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget;
+export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget|DomTreeAiWidget;
 
 export type FunctionCallHandlerResult<Result> = {
   requiresApproval: true,
@@ -521,6 +539,10 @@ export abstract class AiAgent<T> {
     return this.parseTextResponseForSuggestions(response.trim());
   }
 
+  protected async finalizeAnswer(answer: AnswerResponse): Promise<AnswerResponse> {
+    return answer;
+  }
+
   /**
    * Declare a function that the AI model can call.
    * @param name The name of the function
@@ -546,6 +568,9 @@ export abstract class AiAgent<T> {
     this.#functionDeclarations.clear();
   }
 
+  protected async preRun(): Promise<void> {
+  }
+
   async *
       run(
           initialQuery: string,
@@ -555,6 +580,7 @@ export abstract class AiAgent<T> {
           },
           multimodalInput?: MultimodalInput,
           ): AsyncGenerator<ResponseData, void, void> {
+    await this.preRun();
     await options.selected?.refresh();
     if (options.selected) {
       this.context = options.selected;
@@ -632,13 +658,13 @@ export abstract class AiAgent<T> {
           });
         }
         Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceAnswerReceived);
-        yield {
+        yield await this.finalizeAnswer({
           type: ResponseType.ANSWER,
           text: parsedResponse.answer,
           suggestions: parsedResponse.suggestions,
           complete: true,
           rpcId,
-        };
+        });
         if (!functionCall) {
           break;
         }
@@ -670,10 +696,12 @@ export abstract class AiAgent<T> {
 
             return;
           }
+
           query = {
             functionResponse: {
               name: functionCall.name,
-              response: result,
+              // Widgets are not sent back to the LLM
+              response: {...result, widgets: undefined},
             },
           };
           request = this.buildRequest(query, Host.AidaClient.Role.ROLE_UNSPECIFIED);
@@ -701,7 +729,8 @@ export abstract class AiAgent<T> {
           options?: FunctionHandlerOptions&{explanation?: string},
           ): AsyncGenerator<FunctionCallResponseData, {
         result: unknown,
-      }|{context: ConversationContext<unknown>, description: string}> {
+        widgets?: AiWidget[],
+      }|{context: ConversationContext<unknown>, description: string, widgets?: AiWidget[]}> {
     const call = this.#functionDeclarations.get(name);
     if (!call) {
       throw new Error(`Function ${name} is not found.`);
@@ -818,7 +847,7 @@ export abstract class AiAgent<T> {
       return result;
     }
 
-    return result as {result: unknown};
+    return result as {result: unknown, widgets?: AiWidget[]};
   }
 
   async *

package/front_end/models/ai_assistance/agents/ContextSelectionAgent.snapshot.txt

@@ -120,6 +120,6 @@ Content:
     "client_version": "unit_test"
   },
   "functionality_type": 5,
-  "client_feature": 9
+  "client_feature": 25
 }
 === end content