chrome-devtools-frontend 1.0.1596260 → 1.0.1597448

package/front_end/generated/protocol.ts

@@ -20584,6 +20584,77 @@ export namespace WebAuthn {
   }
 }
 
+export namespace WebMCP {
+
+  /**
+   * Tool annotations
+   */
+  export interface Annotation {
+    /**
+     * A hint indicating that the tool does not modify any state.
+     */
+    readOnly?: boolean;
+    /**
+     * If the declarative tool was declared with the autosubmit attribute.
+     */
+    autosubmit?: boolean;
+  }
+
+  /**
+   * Definition of a tool that can be invoked.
+   */
+  export interface Tool {
+    /**
+     * Tool name.
+     */
+    name: string;
+    /**
+     * Tool description.
+     */
+    description: string;
+    /**
+     * Schema for the tool's input parameters.
+     */
+    inputSchema?: any;
+    /**
+     * Optional annotations for the tool.
+     */
+    annotations?: Annotation;
+    /**
+     * Frame identifier associated with the tool registration.
+     */
+    frameId: Page.FrameId;
+    /**
+     * Optional node ID for declarative tools.
+     */
+    backendNodeId?: DOM.BackendNodeId;
+    /**
+     * The stack trace at the time of the registration.
+     */
+    stackTrace?: Runtime.StackTrace;
+  }
+
+  /**
+   * Event fired when new tools are added.
+   */
+  export interface ToolsAddedEvent {
+    /**
+     * Array of tools that were added.
+     */
+    tools: Tool[];
+  }
+
+  /**
+   * Event fired when tools are removed.
+   */
+  export interface ToolsRemovedEvent {
+    /**
+     * Array of tools that were removed.
+     */
+    tools: Tool[];
+  }
+}
+
 /**
  * Debugger domain exposes JavaScript debugging capabilities. It allows setting and removing
  * breakpoints, stepping through execution, exploring stack traces, etc.

package/front_end/models/ai_assistance/AiConversation.ts

@@ -4,10 +4,10 @@
 
 import * as Host from '../../core/host/host.js';
 import * as Root from '../../core/root/root.js';
-import * as SDK from '../../core/sdk/sdk.js';
-import * as Trace from '../../models/trace/trace.js';
+import type * as SDK from '../../core/sdk/sdk.js';
+import type * as Trace from '../../models/trace/trace.js';
 import * as Greendev from '../greendev/greendev.js';
-import * as NetworkTimeCalculator from '../network_time_calculator/network_time_calculator.js';
+import type * as NetworkTimeCalculator from '../network_time_calculator/network_time_calculator.js';
 
 import {
   type AiAgent,
@@ -26,10 +26,6 @@ import {PerformanceAgent, PerformanceTraceContext} from './agents/PerformanceAge
 import {NodeContext, StylingAgent} from './agents/StylingAgent.js';
 import {AiHistoryStorage, ConversationType, type SerializedConversation} from './AiHistoryStorage.js';
 import type {ChangeManager} from './ChangeManager.js';
-import {NetworkRequestFormatter} from './data_formatters/NetworkRequestFormatter.js';
-import {PerformanceInsightFormatter} from './data_formatters/PerformanceInsightFormatter.js';
-import {micros} from './data_formatters/UnitFormatters.js';
-import {AgentFocus} from './performance/AIContext.js';
 
 export const NOT_FOUND_IMAGE_DATA = '';
 const MAX_TITLE_LENGTH = 80;
@@ -166,6 +162,11 @@ export class AiConversation {
     return this.#contexts.at(0);
   }
 
+  getPendingMultimodalInput(): MultimodalInput|undefined {
+    const greenDevEmulationEnabled = Greendev.Prototypes.instance().isEnabled('emulationCapabilities');
+    return greenDevEmulationEnabled ? this.#agent.popPendingMultimodalInput() : undefined;
+  }
+
   #reconstructHistory(historyWithoutImages: ResponseData[]): ResponseData[] {
     const imageHistory = AiHistoryStorage.instance().getImageHistory();
     if (imageHistory && imageHistory.length > 0) {
@@ -274,6 +275,9 @@ export class AiConversation {
                      if (item.type === ResponseType.SIDE_EFFECT) {
                        return {...item, confirm: undefined};
                      }
+                     if (item.type === ResponseType.CONTEXT && item.widgets) {
+                       return {...item, widgets: undefined};
+                     }
                      return item;
                    })
                    .filter(history => !!history),
@@ -342,88 +346,11 @@ export class AiConversation {
     }
   }
 
-  #factsCache = new Map<ExtraContext, Host.AidaClient.RequestFact>();
-
-  async #createFactsForExtraContext(contexts: ExtraContext[]): Promise<void> {
-    for (const context of contexts) {
-      const cached = this.#factsCache.get(context);
-      if (cached) {
-        this.#agent.addFact(cached);
-        continue;
-      }
-
-      if (context instanceof SDK.DOMModel.DOMNode) {
-        const desc = await StylingAgent.describeElement(context);
-
-        const fact = {
-          text: `Relevant HTML element:\n${desc}`,
-          metadata: {
-            source: 'devtools-floaty',
-            score: 1,
-          }
-        };
-        this.#factsCache.set(context, fact);
-        this.#agent.addFact(fact);
-      } else if (context instanceof SDK.NetworkRequest.NetworkRequest) {
-        const calculator = new NetworkTimeCalculator.NetworkTransferTimeCalculator();
-        calculator.updateBoundaries(context);
-        const formatter = new NetworkRequestFormatter(context, calculator);
-        const desc = await formatter.formatNetworkRequest();
-
-        const fact = {
-          text: `Relevant network request:\n${desc}`,
-          metadata: {
-            source: 'devtools-floaty',
-            score: 1,
-          }
-        };
-        this.#factsCache.set(context, fact);
-        this.#agent.addFact(fact);
-      } else if ('insight' in context) {
-        const focus = AgentFocus.fromInsight(context.trace, context.insight);
-        const formatter = new PerformanceInsightFormatter(
-            focus,
-            context.insight,
-        );
-
-        const text = `Relevant Performance Insight:\n${formatter.formatInsight()}`;
-        const fact = {
-          text,
-          metadata: {
-            source: 'devtools-floaty',
-            score: 1,
-          }
-        };
-        this.#factsCache.set(context, fact);
-        this.#agent.addFact(fact);
-      } else {
-        // Must be a trace event
-        const time = Trace.Types.Timing.Micro(
-            context.event.ts - context.traceStartTime,
-        );
-
-        const desc = `Trace event: ${context.event.name}
-Time: ${micros(time)}`;
-
-        const fact = {
-          text: `Relevant trace event:\n${desc}`,
-          metadata: {
-            source: 'devtools-floaty',
-            score: 1,
-          }
-        };
-        this.#factsCache.set(context, fact);
-        this.#agent.addFact(fact);
-      }
-    }
-  }
-
   async *
       run(
           initialQuery: string,
           options: {
             signal?: AbortSignal,
-            extraContext?: ExtraContext[],
             multimodalInput?: MultimodalInput,
           } = {},
           ): AsyncGenerator<ResponseData, void, void> {
@@ -442,9 +369,6 @@ Time: ${micros(time)}`;
     void this.addHistoryItem(userQuery);
     yield userQuery;
 
-    if (options.extraContext) {
-      await this.#createFactsForExtraContext(options.extraContext);
-    }
     this.#setOriginIfEmpty(this.selectedContext?.getOrigin());
 
     if (this.isBlockedByOrigin) {
@@ -463,7 +387,6 @@ Time: ${micros(time)}`;
           initialQuery: string,
           options: {
             signal?: AbortSignal,
-            extraContext?: ExtraContext[],
             multimodalInput?: MultimodalInput,
           } = {},
           ): AsyncGenerator<ResponseData, void, void> {
@@ -528,13 +451,6 @@ function isAiAssistanceServerSideLoggingEnabled(): boolean {
   return !Root.Runtime.hostConfig.aidaAvailability?.disallowLogging;
 }
 
-// TODO: this is the same as the type in UI.Floaty but we cannot use UI
-// here. This is fine for prototyping but if we take this further we can
-// rearchitect.
-type ExtraContext = SDK.DOMModel.DOMNode|SDK.NetworkRequest.NetworkRequest|
-                    {event: Trace.Types.Events.Event, traceStartTime: Trace.Types.Timing.Micro}|
-                    {insight: Trace.Insights.Types.InsightModel, trace: Trace.TraceModel.ParsedTrace};
-
 function isAiAssistanceContextSelectionAgentEnabled(): boolean {
   return Boolean(Root.Runtime.hostConfig.devToolsAiAssistanceContextSelectionAgent?.enabled);
 }

package/front_end/models/ai_assistance/agents/AiAgent.ts

@@ -7,6 +7,7 @@ import * as Root from '../../../core/root/root.js';
 import type * as SDK from '../../../core/sdk/sdk.js';
 import type * as Protocol from '../../../generated/protocol.js';
 import * as Greendev from '../../greendev/greendev.js';
+import type * as Trace from '../../trace/trace.js';
 import {debugLog, isStructuredLogEnabled} from '../debug.js';
 
 export const enum ResponseType {
@@ -69,6 +70,7 @@ export interface ContextResponse {
   type: ResponseType.CONTEXT;
   title: string;
   details: [ContextDetail, ...ContextDetail[]];
+  widgets?: AiWidget[];
 }
 
 export interface TitleResponse {
@@ -228,8 +230,16 @@ export interface ComputedStyleAiWidget {
     properties: string[],
   };
 }
+export interface CoreVitalsAiWidget {
+  name: 'CORE_VITALS';
+  data: {
+    insightSetKey: string,
+    parsedTrace: Trace.TraceModel.ParsedTrace,
+  };
+}
+
 // This type will grow as we add more widgets.
-export type AiWidget = ComputedStyleAiWidget;
+export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget;
 
 export type FunctionCallHandlerResult<Result> = {
   requiresApproval: true,
@@ -384,6 +394,10 @@ export abstract class AiAgent<T> {
     this.#facts.clear();
   }
 
+  popPendingMultimodalInput(): MultimodalInput|undefined {
+    return undefined;
+  }
+
   preambleFeatures(): string[] {
     return [];
   }

package/front_end/models/ai_assistance/agents/PerformanceAgent.ts

@@ -22,6 +22,7 @@ import {AgentFocus} from '../performance/AIContext.js';
 
 import {
   AiAgent,
+  type AiWidget,
   type ContextResponse,
   ConversationContext,
   type ConversationSuggestions,
@@ -381,6 +382,18 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
       return;
     }
 
+    const widgets: AiWidget[] = [];
+    const primaryInsightSet = context.getItem().primaryInsightSet;
+    if (primaryInsightSet) {
+      widgets.push({
+        name: 'CORE_VITALS',
+        data: {
+          parsedTrace: context.getItem().parsedTrace,
+          insightSetKey: primaryInsightSet.id,
+        },
+      });
+    }
+
     yield {
       type: ResponseType.CONTEXT,
       title: lockedString(UIStringsNotTranslated.analyzingTrace),
@@ -390,6 +403,7 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
           text: this.#formatter?.formatTraceSummary() ?? '',
         },
       ],
+      widgets,
     };
 
     this.#hasShownAnalyzeTraceContext = true;

package/front_end/models/ai_assistance/agents/StylingAgent.snapshot.txt

@@ -121,6 +121,30 @@ Content:
           "title"
         ]
       }
+    },
+    {
+      "name": "activateDeviceEmulation",
+      "description": "Sets emulation viewing mode for a specific device and optionally enables vision deficiency emulation.",
+      "parameters": {
+        "type": 6,
+        "description": "",
+        "nullable": false,
+        "properties": {
+          "deviceName": {
+            "type": 1,
+            "description": "The name of the device to emulate. Allowed values: Pixel 3 XL, Pixel 7, Samsung Galaxy S8+, Samsung Galaxy S20 Ultra, Surface Pro 7, Surface Duo, Galaxy Z Fold 5, Asus Zenbook Fold, Samsung Galaxy A51/71, Nest Hub Max, Nest Hub, iPhone 4, iPhone 5/SE, iPhone 6/7/8, iPhone SE, iPhone XR, iPhone 12 Pro, iPhone 14 Pro Max, iPad Mini, iPad Air, iPad Pro.",
+            "nullable": false
+          },
+          "visionDeficiency": {
+            "type": 1,
+            "description": "Optional vision deficiency to emulate. Allowed values: blurredVision, reducedContrast, achromatopsia, deuteranopia, protanopia, tritanopia.",
+            "nullable": true
+          }
+        },
+        "required": [
+          "deviceName"
+        ]
+      }
     }
   ],
   "options": {