chrome-devtools-frontend 1.0.1473514 → 1.0.1506453

package/scripts/ai_assistance/suite/helpers/evaluators.ts

@@ -0,0 +1,183 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import assert from 'node:assert';
+
+import {loadInstructions} from '../instructions/load.ts';
+import type {Conversation} from '../types';
+
+import {generateGeminiContent} from './gemini.ts';
+import {getMarkdownConversation, getOutputs, type Output} from './outputs.ts';
+
+abstract class Evaluator {}
+
+export class FunctionCalled extends Evaluator {
+  static nameOnly(example: Conversation, funcName: string): boolean {
+    return example.queries.some(q => {
+      return q.response.functionCallRequests?.some(call => call.name === funcName);
+    });
+  }
+}
+
+export class LLMComparison extends Evaluator {
+  static async judge(example: Conversation, prompt: string): Promise<{score: number, reasons: string}> {
+    const scoringInstructions = loadInstructions('scoring');
+    const exampleAsMarkdown = getMarkdownConversation(example);
+    const response = await generateGeminiContent(
+        `${scoringInstructions}
+
+        ${prompt}.
+
+## Conversation to score:
+${exampleAsMarkdown}`,
+        'gemini-2.5-flash', {
+          type: 'object',
+          properties: {
+            score: {type: 'number', description: 'A numerical score assigned by the AI.'},
+            reasons: {type: 'string', description: 'A string containing the reasons for the assigned score.'}
+          },
+          required: ['score', 'reasons']
+        });
+    const r = JSON.parse(response) as {score: number, reasons: string};
+    return {score: r.score, reasons: r.reasons};
+  }
+}
+
+interface GroupTestState {
+  store: ResultStore;
+  outputsByDate: Partial<Record<string, Output[]>>;
+}
+
+let state: GroupTestState|null = null;
+
+export type ItEval = {
+  test: string,
+}&({
+  succeed: (example: Conversation) => boolean,
+}|{
+  judge: (example: Conversation) => Promise<{score: number, reasons: string}>,
+});
+
+export async function itEval(config: ItEval): Promise<void> {
+  assert.ok(state);
+  if ('succeed' in config) {
+    for (const [date, outputs] of Object.entries(state.outputsByDate)) {
+      if (!outputs) {
+        continue;
+      }
+
+      const allDevToolsConversations = outputs.flatMap(o => o.contents.conversations);
+
+      let total = 0;
+      let succeeded = 0;
+      for (const conversation of allDevToolsConversations) {
+        total++;
+        if (config.succeed(conversation)) {
+          succeeded++;
+        }
+        state.store.saveResult(config.test, date, {type: 'BINARY', success: succeeded, total});
+      }
+    }
+  } else if ('judge' in config) {
+    for (const [date, outputs] of Object.entries(state.outputsByDate)) {
+      if (!outputs) {
+        continue;
+      }
+      const allDevToolsConversations = outputs.flatMap(o => o.contents.conversations);
+      const scores = await Promise.all(allDevToolsConversations.map(async example => {
+        const result = await config.judge(example);
+        return result.score;
+      }));
+      const totalOfAllScores = scores.reduce((acc: number, score: number) => acc + score, 0);
+      const average = totalOfAllScores / scores.length;
+      state.store.saveResult(config.test, date, {type: 'JUDGE', average, allScores: scores, total: totalOfAllScores});
+    }
+  }
+}
+
+export interface GroupConfig {
+  type: string;
+  label: string;
+}
+
+export async function evalGroup(config: GroupConfig, cb: (() => Promise<void>)): Promise<void> {
+  const store = new ResultStore(config.type, config.label);
+  const outputs = await getOutputs(config.type, config.label);
+  const outputsByDate = Object.groupBy(outputs, o => o.dateFolder);
+  state = {
+    store,
+    outputsByDate,
+  };
+
+  await cb();
+  printResults(state.store);
+}
+
+function log(indentation: number, message: string): void {
+  console.log(`${' '.repeat(indentation)}${message}`);
+}
+
+function printResults(store: ResultStore): void {
+  log(0, `Results for: ${store.type}/${store.label}`);
+
+  // Structures the results in Date => <Test Name, Test Output>.
+  const dataForTable: Record<string, Record<string, string>> = {};
+
+  for (const [test, dateToResult] of store.results) {
+    for (const [date, result] of dateToResult) {
+      dataForTable[date] ??= {};
+      switch (result.type) {
+        case 'BINARY':
+          dataForTable[date][test] = `${result.success} / ${result.total} passed`;
+          break;
+        case 'JUDGE':
+          dataForTable[date][test] = `${result.average.toFixed(1)} average from ${result.allScores.length} inputs.`;
+          break;
+        default:
+          throw new Error('Unknown result type!');
+      }
+    }
+  }
+  console.table(dataForTable);
+}
+
+type Result = {
+  type: 'BINARY',
+  total: number,
+  success: number,
+}|{
+  type: 'JUDGE',
+  average: number,
+  total: number,
+  allScores: number[],
+};
+
+class ResultStore {
+  // Map of testName => YYYY-MM-DD => Result
+  #results = new Map<string, Map<string, Result>>();
+  #type: string;
+  #label: string;
+
+  constructor(type: string, label: string) {
+    this.#type = type;
+    this.#label = label;
+  }
+
+  get type(): string {
+    return this.#type;
+  }
+  get label(): string {
+    return this.#label;
+  }
+
+  get results(): ReadonlyMap<string, ReadonlyMap<string, Result>> {
+    return this.#results;
+  }
+
+  saveResult(testName: string, dateFolder: string, result: Result): void {
+    const forTest = this.#results.get(testName) ?? new Map<string, Result>();
+    forTest.set(dateFolder, result);
+    this.#results.set(testName, forTest);
+  }
+}

package/scripts/ai_assistance/suite/helpers/gemini.ts

@@ -0,0 +1,122 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * Makes requests to the Gemini API. Assumes process.env.GEMINI_API_KEY is available.
+ * Note: the reason we do not use any Google Gemini SDK here is to save on the
+ * node module install & committing it to the repo. Our API usage of Gemini is
+ * lightweight so it doesn't feel worth it vs just wrapping the XHR requests
+ * ourselves.
+ */
+
+// Define interfaces for better type safety
+interface Part {
+  text: string;
+}
+
+interface Content {
+  parts: Part[];
+}
+
+interface GenerateContentRequestBody {
+  contents: Content[];
+  generationConfig?: {
+    responseMimeType: string,
+    responseSchema: object,
+  };
+}
+
+interface CandidatePart {
+  text: string;
+}
+
+interface CandidateContent {
+  parts: CandidatePart[];
+  role: string;
+}
+
+interface Candidate {
+  content: CandidateContent;
+  finishReason: string;
+  index: number;
+}
+
+interface GenerateContentResponse {
+  candidates: Candidate[];
+}
+
+/**
+ * Helper function to construct the request body for the Gemini generateContent API.
+ *
+ * @param promptText The text prompt for the Gemini model.
+ * @param jsonSchema An optional JSON schema to define the expected response structure.
+ * @returns The constructed GenerateContentRequestBody object.
+ */
+function buildGeminiRequestBody(promptText: string, jsonSchema?: object): GenerateContentRequestBody {
+  const requestBody: GenerateContentRequestBody = {
+    contents: [
+      {
+        parts: [
+          {
+            text: promptText,
+          },
+        ],
+      },
+    ],
+  };
+
+  if (jsonSchema) {
+    requestBody.generationConfig = {
+      responseMimeType: 'application/json',
+      responseSchema: jsonSchema,
+    };
+  }
+
+  return requestBody;
+}
+
+/**
+ * Makes a request to the Gemini API's generateContent endpoint.
+ *
+ * @param promptText The text prompt to send to the Gemini model.
+ * @param modelName The name of the Gemini model to use (e.g., 'gemini-pro', 'gemini-1.5-flash').
+ * @returns A Promise that resolves to the generated text, or an error string.
+ */
+export async function generateGeminiContent(
+    promptText: string, modelName = 'gemini-2.5-flash', jsonSchema?: object): Promise<string> {
+  const apiKey = process.env.GEMINI_API_KEY;
+
+  if (!apiKey) {
+    throw new Error('GEMINI_API_KEY environment variable is not set. Please provide your API key.');
+  }
+
+  const apiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent`;
+
+  const requestBody = buildGeminiRequestBody(promptText, jsonSchema);
+
+  try {
+    const response = await fetch(apiUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-goog-api-key': apiKey,
+      },
+      body: JSON.stringify(requestBody),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.json();
+      throw new Error(`API error: ${response.status} ${response.statusText} - ${JSON.stringify(errorData)}`);
+    }
+
+    const data: GenerateContentResponse = await response.json();
+
+    // Safely access the generated content
+    return (
+        data.candidates?.[0]?.content?.parts?.[0]?.text || 'No content generated or content not in expected format.');
+  } catch (error) {
+    console.error('Error generating content:', error);
+    return `Error: Could not generate content. Details: ${error instanceof Error ? error.message : String(error)}`;
+  }
+}

package/scripts/ai_assistance/suite/helpers/outputs.ts

@@ -0,0 +1,112 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as fs from 'node:fs';
+import * as fsPromises from 'node:fs/promises';
+import * as path from 'node:path';
+
+import type {Conversation, EvalFileOutput} from '../types';
+
+const BASE_DIR = path.join(path.dirname(import.meta.filename), '..', 'outputs', 'outputs');
+
+export function getMarkdownConversation(example: Conversation): string {
+  let markdown = '';
+  for (const query of example.queries) {
+    markdown += '## REQUEST FROM CLIENT:\n';
+    if (query.request.prompt) {
+      markdown += `### QUERY:\n${query.request.prompt}\n`;
+    }
+    if (query.request.functionCallResponse) {
+      markdown += `### FUNCTION CALL RESPONSE:\n${query.request.functionCallResponse}\n`;
+    }
+
+    markdown += '## RESPONSE FROM SERVER:\n';
+    if (query.response.text) {
+      markdown += `### EXPLANATION:\n${query.response.text}\n`;
+    }
+    if (query.response.functionCallRequests?.length) {
+      markdown += `### FUNCTION CALL REQUESTS:\n${query.response.functionCallRequests.join(', ')}\n`;
+    }
+    markdown += '\n';
+  }
+  return markdown;
+}
+
+export interface Output {
+  absoluteFilePath: string;
+  fileName: string;
+  label: string;
+  dateFolder: string;
+  type: string;
+  contents: EvalFileOutput;
+}
+
+export async function getOutputs(type: string, label: string): Promise<Output[]> {
+  const folder = path.join(BASE_DIR, type);
+  const matchedFiles = await findNestedFiles(folder, fileName => {
+    return fileName.startsWith(label);
+  });
+  return matchedFiles.map(filePath => {
+    const absoluteFilePath = path.join(BASE_DIR, type, filePath);
+    const [parentDir, fileName] = filePath.split('/');
+    return {
+      absoluteFilePath,
+      fileName,
+      dateFolder: parentDir,
+      type,
+      label,
+      contents: JSON.parse(fs.readFileSync(absoluteFilePath, 'utf8')) as EvalFileOutput,
+    };
+  });
+}
+
+/**
+ * Finds all files within one level of nested folders in a given directory,
+ * returning a list of files prefixed with their parent folder's name.
+ *
+ * Example: if `baseDirPath` contains `folder1/fileA.txt` and `folder2/fileB.js`,
+ * and `folder1` contains `fileA.txt`, `folder2` contains `fileB.js`,
+ * the result would be `['folder1/fileA.txt', 'folder2/fileB.js']`.
+ * Files directly in `baseDirPath` are ignored.
+ *
+ * @param baseDirPath The path to the base directory to search within.
+ * @returns A promise that resolves to an array of file paths in 'foldername/filename' format.
+ */
+export async function findNestedFiles(
+    baseDirPath: string, predicate: (fileName: string) => boolean): Promise<string[]> {
+  const result: string[] = [];
+  try {
+    // Read entries in the base directory
+    const entries = await fsPromises.readdir(baseDirPath, {withFileTypes: true});
+
+    for (const entry of entries) {
+      // If the entry is a directory, it's a "nested folder"
+      if (entry.isDirectory()) {
+        const nestedFolderPath = path.join(baseDirPath, entry.name);
+        try {
+          // Read entries in the nested folder
+          const nestedFolderEntries = await fsPromises.readdir(nestedFolderPath, {withFileTypes: true});
+
+          for (const nestedEntry of nestedFolderEntries) {
+            // If the nested entry is a file, add it to the result
+            if (nestedEntry.isFile()) {
+              // Prefix with the nested folder's name
+              if (predicate(nestedEntry.name)) {
+                result.push(path.join(entry.name, nestedEntry.name));
+              }
+            }
+          }
+        } catch (nestedError) {
+          // Log error for a specific nested folder but continue with others
+          console.warn(`Could not read nested directory ${nestedFolderPath}: ${nestedError.message}`);
+        }
+      }
+    }
+  } catch (error) {
+    // Log error for the base directory and return empty array
+    console.error(`Error processing base directory ${baseDirPath}: ${error.message}`);
+    return [];  // Return empty array if base directory read fails
+  }
+  return result;
+}

package/scripts/ai_assistance/suite/instructions/lcp-breakdown.md

@@ -0,0 +1,12 @@
+## LCP Breakdown judge instructions
+
+You will be shown a conversation with a Web Performance AI Agent that focuses on the page load speed and the page's Largest Contentful Paint (LCP) score.
+
+A great conversation with this agent will include the following:
+
+- The LCP occurred at 1,113ms. Do not worry if the actual score is slightly different or at a different level of precision. For example, "1,113.94ms" is fine.
+- The LCP element was `lcp-image.jpg`.
+- "Element render delay" contributed significantly to LCP time, accounting for over 90%.
+- Main thread activity in `render-blocking-script.js` blocked the main thread for nearly 1 second, primarily with the `sleepFor` function.
+- Consider preloading the LCP image with a `link rel="preload"` tag.
+- Consider giving the LCP image a `fetchpriority="high"`.

package/scripts/ai_assistance/suite/instructions/load.ts

@@ -0,0 +1,13 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+const DIR = path.dirname(import.meta.filename);
+
+export function loadInstructions(name: string): string {
+  const content = fs.readFileSync(path.join(DIR, name + '.md'), 'utf8');
+  return content;
+}

package/scripts/ai_assistance/suite/instructions/scoring.md

@@ -0,0 +1,27 @@
+# You are an AI judge who specialises in scoring output from another AI system on a scale of 0 to 1 where:
+
+- 0 is a very bad response
+- 0.5 is a mediocre response
+- 1 is a great response
+
+When scoring you must consider the following criteria:
+
+## Correctness (judge on response only)
+
+Consider how correct the output is and if there are any hallucinations or incomplete answers in the response.
+
+## Relevance (judge on user queries and responses)
+
+Consider how relevant the responses are and how accurately they answer the user's prompt.
+
+## Conciseness (judge on response only)
+
+Consider how concise the output is. Does it contain repeated information or phrases that are long and could be shortened? Are all code examples or suggestions brief and to the point?
+
+## Readability (judge on response only)
+
+How easy is the response to digest? Does it use unnecessarily complex phrasing or vocabulary that could be shortened to make it more readable and less overwhelming? Remember that we want the tool to help beginners as well as experts and it is important that our output does not intidimate or make people feel silly.
+
+## Safety (judge on response only)
+
+Does the output contain any potentially unsafe content, or content on topics completely irrelevant?

package/scripts/ai_assistance/suite/performance.eval.ts

@@ -0,0 +1,18 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {evalGroup, FunctionCalled, itEval, LLMComparison} from './helpers/evaluators.ts';
+import {loadInstructions} from './instructions/load.ts';
+
+await evalGroup({type: 'performance', label: 'lcp-breakdown'}, async function() {
+  await itEval({
+    test: 'calls getMainThreadActivity',
+    succeed: example => FunctionCalled.nameOnly(example, 'getMainThreadActivity'),
+  });
+
+  await itEval({
+    test: 'is an accurate response',
+    judge: example => LLMComparison.judge(example, loadInstructions('lcp-breakdown')),
+  });
+});

package/scripts/ai_assistance/suite/to_eval_output.ts

@@ -0,0 +1,166 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as assert from 'node:assert';
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {hideBin} from 'yargs/helpers';
+import yargs from 'yargs/yargs';
+
+import type {Conversation, EvalFileOutput, ProcessedQuery} from './types';
+
+const userArgs =
+    yargs(hideBin(process.argv))
+        .option('file', {type: 'string', demandOption: true, description: 'The raw JSON file from Auto Run.'})
+        .option('label', {type: 'string', demandOption: true, desc: 'A human readable, short label to use.'})
+        .option('pretty', {
+          type: 'boolean',
+          demandOption: false,
+          default: false,
+          description: 'Output formatted JSON rather than minified.'
+        })
+        .parseSync();
+
+const inputPath = path.isAbsolute(userArgs.file) ? userArgs.file : path.join(process.cwd(), userArgs.file);
+const contents = fs.readFileSync(inputPath, 'utf8');
+
+const INPUT_HASH = hash(contents, null);
+
+// Note: non-exhaustive.
+interface RawOutput {
+  metadata: Array<{exampleId: string, explanation: string}>;
+  examples: Array<{
+    exampleId: string,
+    request: {
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      current_message: {
+        parts: Array<{
+          text?: string,
+          functionResponse?: {
+            name: string,
+            response: {result: Record<string, string>},
+          },
+        }>,
+      },
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      function_declarations: Array<{
+        name: string,
+        description: string,
+        parameters: {
+          properties?: Record<string, unknown>,
+        },
+      }>,
+      metadata: {
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        client_version: string,
+      },
+    },
+    aidaResponse: {
+      metadata: {
+        rcpGlobalId: string,
+        inferenceOptionMetadata: {
+          modelId: string,
+          modelVersion: string,
+        },
+      },
+      explanation?: string,
+      functionCalls?: Array<{name: string, args: Record<string, unknown>}>,
+      completed?: true,
+    },
+  }>;
+}
+
+const json = JSON.parse(contents) as RawOutput;
+
+const examples = json.metadata.map(m => m.exampleId);
+
+const processedExamples: Conversation[] =
+    examples
+        .map((exampleIdFromInput, index) => {
+          const data = json.examples.filter(e => e.exampleId === exampleIdFromInput);
+          if (!data.length) {
+            return null;
+          }
+
+          const exampleMetadata = json.metadata[index];
+
+          const id = INPUT_HASH + '-' + index;
+          const chromeVersion = data.at(0)?.request.metadata.client_version;
+          assert.ok(chromeVersion, 'No client_version');
+          const modelData = data.at(0)?.aidaResponse.metadata.inferenceOptionMetadata;
+          assert.ok(modelData, 'No inferenceOptionMetadata');
+          const processed: Conversation = {
+            id,
+            chromeVersion,
+            explanation: exampleMetadata?.explanation ?? '',
+            model: {
+              id: modelData?.modelId,
+              version: modelData?.modelVersion,
+            },
+            queries: [],
+          };
+
+          for (const {request, aidaResponse} of data) {
+            if (!aidaResponse.completed) {
+              continue;
+            }
+
+            const responseText = aidaResponse.explanation?.trim() ?? undefined;
+
+            const query: ProcessedQuery = {
+              request: {
+                prompt: request.current_message.parts[0].text,
+                functionCallResponse: request.current_message.parts[0].functionResponse?.name,
+                availableFunctionNames: request.function_declarations.map(dec => dec.name),
+              },
+              response: {
+                rpcGlobalId: aidaResponse.metadata.rcpGlobalId,
+                text: responseText,
+                functionCallRequests: aidaResponse.functionCalls?.map(call => {
+                  return {
+                    name: call.name,
+                    args: call.args,
+                  };
+                }),
+              }
+            };
+            processed.queries.push(query);
+          }
+          return processed;
+        })
+        .filter(x => x !== null);
+
+const finalOutput: EvalFileOutput = {
+  metadata: {
+    createdAt: new Date().toISOString(),
+    id: hash(processedExamples.map(x => x.id).join(''), 16),
+    label: userArgs.label,
+  },
+  conversations: processedExamples,
+};
+
+const stringified = userArgs.pretty ? JSON.stringify(finalOutput, null, 2) : JSON.stringify(finalOutput);
+
+const fileName = `${slug(userArgs.label)}-${finalOutput.metadata.id}.json`;
+fs.writeFileSync(path.join(process.cwd(), fileName), stringified, 'utf8');
+console.log(`Wrote ${fileName} to disk.`);
+
+function hash(str: string, length: number|null) {
+  const hash = crypto.createHash('md5').update(str).digest('hex');
+  if (length) {
+    return hash.substring(0, length);
+  }
+  return hash;
+}
+
+function slug(str: string): string {
+  str = str.replace(/^\s+|\s+$/g, '');  // Trim leading/trailing whitespace
+  str = str.toLowerCase();
+  str = str.replace(/[^a-z0-9 -]/g, '')  // Remove invalid chars
+            .replace(/\s+/g, '-')        // Collapse whitespace and replace with -
+            .replace(/-+/g, '-');        // Collapse dashes
+
+  return str;
+}

package/scripts/ai_assistance/suite/types.d.ts

@@ -0,0 +1,40 @@
+// Copyright 2025 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+export interface ProcessedQuery {
+  request: {
+    availableFunctionNames: string[],
+    // Populated if the user typed a query
+    prompt?: string,
+    // Populated if the model decided it needed to call a function; this is the
+    // frontend sending the data back to the server.
+    functionCallResponse?: string,
+  };
+  response: {
+    rpcGlobalId: string,
+    // Populated if the LLM returned a text explanation that was complete.
+    text?: string,
+    // Populated as the name of the functions if the LLM requested it to be called
+    functionCallRequests?: Array<{name: string, args: Record<string, unknown>}>,
+  };
+}
+
+export interface EvalFileOutput {
+  metadata: {
+    createdAt: string,
+    id: string,
+    label: string,
+  };
+  conversations: Conversation[];
+}
+
+export interface Conversation {
+  id: string;
+  chromeVersion: string;
+  // These are explanations found in the input example HTML that can be used to
+  // judge the AI's output.
+  explanation: string;
+  model: {id: string, version: string};
+  queries: ProcessedQuery[];
+}