lighthouse 11.4.0-dev.20240103 → 11.4.0-dev.20240105

package/core/gather/gatherers/root-causes.js

@@ -0,0 +1,133 @@
+/**
+ * @license
+ * Copyright 2023 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import BaseGatherer from '../base-gatherer.js';
+import Trace from './trace.js';
+import * as TraceEngine from '../../lib/trace-engine.js';
+import {TraceEngineResult} from '../../computed/trace-engine-result.js';
+
+class RootCauses extends BaseGatherer {
+  static symbol = Symbol('RootCauses');
+
+  /** @type {LH.Gatherer.GathererMeta<'Trace'>} */
+  meta = {
+    symbol: RootCauses.symbol,
+    supportedModes: ['timespan', 'navigation'],
+    dependencies: {Trace: Trace.symbol},
+  };
+
+  /**
+   * @param {LH.Gatherer.Driver} driver
+   * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+   * @return {Promise<LH.Artifacts.TraceEngineRootCauses>}
+   */
+  static async runRootCauseAnalysis(driver, traceEngineResult) {
+    await driver.defaultSession.sendCommand('DOM.enable');
+    await driver.defaultSession.sendCommand('CSS.enable');
+
+    // DOM.getDocument is necessary for pushNodesByBackendIdsToFrontend to properly retrieve
+    // nodeIds if the DOM domain was enabled before this gatherer, invoke it to be safe.
+    await driver.defaultSession.sendCommand('DOM.getDocument', {depth: -1, pierce: true});
+
+    const protocolInterface = {
+      /** @param {string} url */
+      // eslint-disable-next-line no-unused-vars
+      getInitiatorForRequest(url) {
+        return null;
+      },
+      /** @param {number[]} backendNodeIds */
+      async pushNodesByBackendIdsToFrontend(backendNodeIds) {
+        const response = await driver.defaultSession.sendCommand(
+          'DOM.pushNodesByBackendIdsToFrontend', {backendNodeIds});
+        return response.nodeIds;
+      },
+      /** @param {number} nodeId */
+      async getNode(nodeId) {
+        try {
+          const response = await driver.defaultSession.sendCommand('DOM.describeNode', {nodeId});
+          // This always zero, so let's fix it here.
+          // https://bugs.chromium.org/p/chromium/issues/detail?id=1515175
+          response.node.nodeId = nodeId;
+          return response.node;
+        } catch (err) {
+          if (err.message.includes('Could not find node with given id')) {
+            // TODO: when injecting an iframe, the engine gets the node of that frame's document element.
+            // But we don't have a way to access that frame. We just have our default session.
+            // Ex:
+            // node cli http://localhost:10503/shift-attribution.html --quiet --only-audits layout-shifts
+            // To fix we must:
+            // - Change trace engine `getNode` protocol interface to also give frame id
+            // - Expand our driver.targetManager to know how to talk to a session connected to a specific frame
+            // When this is fixed, remove this try/catch.
+            // Note: this could be buggy by giving the wrong node detail if a node id meant for a non-main frame
+            // happens to match one from the main frame ... which is pretty likely...
+            return null;
+          }
+          throw err;
+        }
+      },
+      /** @param {number} nodeId */
+      async getComputedStyleForNode(nodeId) {
+        try {
+          const response = await driver.defaultSession.sendCommand(
+            'CSS.getComputedStyleForNode', {nodeId});
+          return response.computedStyle;
+        } catch {
+          return [];
+        }
+      },
+      /** @param {number} nodeId */
+      async getMatchedStylesForNode(nodeId) {
+        try {
+          const response = await driver.defaultSession.sendCommand(
+            'CSS.getMatchedStylesForNode', {nodeId});
+          return response;
+        } catch {
+          return [];
+        }
+      },
+      /** @param {string} url */
+      // eslint-disable-next-line no-unused-vars
+      async fontFaceForSource(url) {
+        return null;
+      },
+    };
+
+    /** @type {LH.Artifacts.TraceEngineRootCauses} */
+    const rootCauses = {
+      layoutShifts: {},
+    };
+    const rootCausesEngine = new TraceEngine.RootCauses(protocolInterface);
+    const layoutShiftEvents = traceEngineResult.LayoutShifts.clusters.flatMap(c => c.events);
+    for (const event of layoutShiftEvents) {
+      const r = await rootCausesEngine.layoutShifts.rootCausesForEvent(traceEngineResult, event);
+      for (const cause of r.fontChanges) {
+        // TODO: why isn't trace engine unwrapping this promise ...
+        cause.fontFace = await cause.fontFace;
+      }
+      rootCauses.layoutShifts[layoutShiftEvents.indexOf(event)] = r;
+    }
+
+    // Yeah this gatherer enabled it, and so you'd think it should disable it too...
+    // ...but we don't give gatherers their own session so this stomps on others.
+    // await driver.defaultSession.sendCommand('DOM.disable');
+    await driver.defaultSession.sendCommand('CSS.disable');
+
+    return rootCauses;
+  }
+
+  /**
+   * @param {LH.Gatherer.Context<'Trace'>} context
+   * @return {Promise<LH.Artifacts.TraceEngineRootCauses>}
+   */
+  async getArtifact(context) {
+    const trace = context.dependencies.Trace;
+    const traceEngineResult = await TraceEngineResult.request({trace}, context);
+    return RootCauses.runRootCauseAnalysis(context.driver, traceEngineResult);
+  }
+}
+
+export default RootCauses;

package/core/gather/gatherers/trace-elements.d.ts

@@ -10,13 +10,38 @@ export type TraceElementData = {
 };
 declare class TraceElements extends BaseGatherer {
     /**
-     * This function finds the top (up to 15) elements that contribute to the CLS score of the page.
+     * This function finds the top (up to 15) elements that shift on the page.
      *
      * @param {LH.Trace} trace
      * @param {LH.Gatherer.Context} context
-     * @return {Promise<Array<TraceElementData>>}
+     * @return {Promise<Array<{nodeId: number}>>}
      */
-    static getTopLayoutShiftElements(trace: LH.Trace, context: LH.Gatherer.Context): Promise<Array<TraceElementData>>;
+    static getTopLayoutShiftElements(trace: LH.Trace, context: LH.Gatherer.Context): Promise<Array<{
+        nodeId: number;
+    }>>;
+    /**
+     * We want to a single representative node to represent the shift, so let's pick
+     * the one with the largest impact (size x distance moved).
+     *
+     * @param {LH.Artifacts.TraceImpactedNode[]} impactedNodes
+     * @param {Map<number, number>} impactByNodeId
+     * @return {number|undefined}
+     */
+    static getBiggestImpactNodeForShiftEvent(impactedNodes: LH.Artifacts.TraceImpactedNode[], impactByNodeId: Map<number, number>): number | undefined;
+    /**
+     * This function finds the top (up to 15) layout shifts on the page, and returns
+     * the id of the largest impacted node of each shift, along with any related nodes
+     * that may have caused the shift.
+     *
+     * @param {LH.Trace} trace
+     * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+     * @param {LH.Artifacts.TraceEngineRootCauses} rootCauses
+     * @param {LH.Gatherer.Context} context
+     * @return {Promise<Array<{nodeId: number}>>}
+     */
+    static getTopLayoutShifts(trace: LH.Trace, traceEngineResult: LH.Artifacts.TraceEngineResult, rootCauses: LH.Artifacts.TraceEngineRootCauses, context: LH.Gatherer.Context): Promise<Array<{
+        nodeId: number;
+    }>>;
     /**
      * @param {LH.Trace} trace
      * @param {LH.Gatherer.Context} context
@@ -32,8 +57,8 @@ declare class TraceElements extends BaseGatherer {
         nodeId: number;
         type: string;
     } | undefined>;
-    /** @type {LH.Gatherer.GathererMeta<'Trace'>} */
-    meta: LH.Gatherer.GathererMeta<'Trace'>;
+    /** @type {LH.Gatherer.GathererMeta<'Trace'|'RootCauses'>} */
+    meta: LH.Gatherer.GathererMeta<'Trace' | 'RootCauses'>;
     /** @type {Map<string, string>} */
     animationIdToName: Map<string, string>;
     /** @param {LH.Crdp.Animation.AnimationStartedEvent} args */
@@ -53,11 +78,17 @@ declare class TraceElements extends BaseGatherer {
      */
     stopInstrumentation(context: LH.Gatherer.Context): Promise<void>;
     /**
-     * @param {LH.Gatherer.Context<'Trace'>} context
+     * @param {LH.Gatherer.ProtocolSession} session
+     * @param {number} backendNodeId
+     */
+    getNodeDetails(session: LH.Gatherer.ProtocolSession, backendNodeId: number): Promise<import("devtools-protocol").Protocol.Runtime.CallFunctionOnResponse | null>;
+    /**
+     * @param {LH.Gatherer.Context<'Trace'|'RootCauses'>} context
      * @return {Promise<LH.Artifacts.TraceElement[]>}
      */
-    getArtifact(context: LH.Gatherer.Context<'Trace'>): Promise<LH.Artifacts.TraceElement[]>;
+    getArtifact(context: LH.Gatherer.Context<'Trace' | 'RootCauses'>): Promise<LH.Artifacts.TraceElement[]>;
 }
 import BaseGatherer from '../base-gatherer.js';
 import Trace from './trace.js';
+import { TraceEngineResult } from '../../computed/trace-engine-result.js';
 //# sourceMappingURL=trace-elements.d.ts.map

package/core/gather/gatherers/trace-elements.js

@@ -23,10 +23,13 @@ import {LighthouseError} from '../../lib/lh-error.js';
 import {Responsiveness} from '../../computed/metrics/responsiveness.js';
 import {CumulativeLayoutShift} from '../../computed/metrics/cumulative-layout-shift.js';
 import {ExecutionContext} from '../driver/execution-context.js';
+import RootCauses from './root-causes.js';
+import {TraceEngineResult} from '../../computed/trace-engine-result.js';
 
 /** @typedef {{nodeId: number, animations?: {name?: string, failureReasonsMask?: number, unsupportedProperties?: string[]}[], type?: string}} TraceElementData */
 
 const MAX_LAYOUT_SHIFT_ELEMENTS = 15;
+const MAX_LAYOUT_SHIFTS = 15;
 
 /**
  * @this {HTMLElement}
@@ -44,10 +47,10 @@ function getNodeDetailsData() {
 /* c8 ignore stop */
 
 class TraceElements extends BaseGatherer {
-  /** @type {LH.Gatherer.GathererMeta<'Trace'>} */
+  /** @type {LH.Gatherer.GathererMeta<'Trace'|'RootCauses'>} */
   meta = {
     supportedModes: ['timespan', 'navigation'],
-    dependencies: {Trace: Trace.symbol},
+    dependencies: {Trace: Trace.symbol, RootCauses: RootCauses.symbol},
   };
 
   /** @type {Map<string, string>} */
@@ -64,11 +67,11 @@ class TraceElements extends BaseGatherer {
   }
 
   /**
-   * This function finds the top (up to 15) elements that contribute to the CLS score of the page.
+   * This function finds the top (up to 15) elements that shift on the page.
    *
    * @param {LH.Trace} trace
    * @param {LH.Gatherer.Context} context
-   * @return {Promise<Array<TraceElementData>>}
+   * @return {Promise<Array<{nodeId: number}>>}
    */
   static async getTopLayoutShiftElements(trace, context) {
     const {impactByNodeId} = await CumulativeLayoutShift.request(trace, context);
@@ -79,6 +82,66 @@ class TraceElements extends BaseGatherer {
       .map(([nodeId]) => ({nodeId}));
   }
 
+  /**
+   * We want to a single representative node to represent the shift, so let's pick
+   * the one with the largest impact (size x distance moved).
+   *
+   * @param {LH.Artifacts.TraceImpactedNode[]} impactedNodes
+   * @param {Map<number, number>} impactByNodeId
+   * @return {number|undefined}
+   */
+  static getBiggestImpactNodeForShiftEvent(impactedNodes, impactByNodeId) {
+    let biggestImpactNodeId;
+    let biggestImpactNodeScore = Number.NEGATIVE_INFINITY;
+    for (const node of impactedNodes) {
+      const impactScore = impactByNodeId.get(node.node_id);
+      if (impactScore !== undefined && impactScore > biggestImpactNodeScore) {
+        biggestImpactNodeId = node.node_id;
+        biggestImpactNodeScore = impactScore;
+      }
+    }
+    return biggestImpactNodeId;
+  }
+
+  /**
+   * This function finds the top (up to 15) layout shifts on the page, and returns
+   * the id of the largest impacted node of each shift, along with any related nodes
+   * that may have caused the shift.
+   *
+   * @param {LH.Trace} trace
+   * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+   * @param {LH.Artifacts.TraceEngineRootCauses} rootCauses
+   * @param {LH.Gatherer.Context} context
+   * @return {Promise<Array<{nodeId: number}>>}
+   */
+  static async getTopLayoutShifts(trace, traceEngineResult, rootCauses, context) {
+    const {impactByNodeId} = await CumulativeLayoutShift.request(trace, context);
+    const clusters = traceEngineResult.LayoutShifts.clusters ?? [];
+    const layoutShiftEvents = clusters.flatMap(c => c.events);
+
+    return layoutShiftEvents
+      .sort((a, b) => b.args.data.weighted_score_delta - a.args.data.weighted_score_delta)
+      .slice(0, MAX_LAYOUT_SHIFTS)
+      .flatMap(event => {
+        const nodeIds = [];
+        const biggestImpactedNodeId =
+          this.getBiggestImpactNodeForShiftEvent(event.args.data.impacted_nodes, impactByNodeId);
+        if (biggestImpactedNodeId !== undefined) {
+          nodeIds.push(biggestImpactedNodeId);
+        }
+
+        const index = layoutShiftEvents.indexOf(event);
+        const shiftRootCauses = rootCauses.layoutShifts[index];
+        if (shiftRootCauses) {
+          for (const cause of shiftRootCauses.unsizedMedia) {
+            nodeIds.push(cause.node.backendNodeId);
+          }
+        }
+
+        return nodeIds.map(nodeId => ({nodeId}));
+      });
+  }
+
   /**
    * @param {LH.Trace} trace
    * @param {LH.Gatherer.Context} context
@@ -195,61 +258,77 @@ class TraceElements extends BaseGatherer {
   }
 
   /**
-   * @param {LH.Gatherer.Context<'Trace'>} context
+   * @param {LH.Gatherer.ProtocolSession} session
+   * @param {number} backendNodeId
+   */
+  async getNodeDetails(session, backendNodeId) {
+    try {
+      const objectId = await resolveNodeIdToObjectId(session, backendNodeId);
+      if (!objectId) return null;
+
+      const deps = ExecutionContext.serializeDeps([
+        pageFunctions.getNodeDetails,
+        getNodeDetailsData,
+      ]);
+      return await session.sendCommand('Runtime.callFunctionOn', {
+        objectId,
+        functionDeclaration: `function () {
+          ${deps}
+          return getNodeDetailsData.call(this);
+        }`,
+        returnByValue: true,
+        awaitPromise: true,
+      });
+    } catch (err) {
+      Sentry.captureException(err, {
+        tags: {gatherer: 'TraceElements'},
+        level: 'error',
+      });
+    }
+
+    return null;
+  }
+
+  /**
+   * @param {LH.Gatherer.Context<'Trace'|'RootCauses'>} context
    * @return {Promise<LH.Artifacts.TraceElement[]>}
    */
   async getArtifact(context) {
     const session = context.driver.defaultSession;
 
     const trace = context.dependencies.Trace;
-    if (!trace) {
-      throw new Error('Trace is missing!');
-    }
+    const traceEngineResult = await TraceEngineResult.request({trace}, context);
+    const rootCauses = context.dependencies.RootCauses;
 
     const processedTrace = await ProcessedTrace.request(trace, context);
     const {mainThreadEvents} = processedTrace;
 
     const lcpNodeData = await TraceElements.getLcpElement(trace, context);
-    const clsNodeData = await TraceElements.getTopLayoutShiftElements(trace, context);
+    const shiftElementsNodeData = await TraceElements.getTopLayoutShiftElements(trace, context);
+    const shiftsData = await TraceElements.getTopLayoutShifts(
+      trace, traceEngineResult, rootCauses, context);
     const animatedElementData = await this.getAnimatedElements(mainThreadEvents);
     const responsivenessElementData = await TraceElements.getResponsivenessElement(trace, context);
 
     /** @type {Map<string, TraceElementData[]>} */
     const backendNodeDataMap = new Map([
       ['largest-contentful-paint', lcpNodeData ? [lcpNodeData] : []],
-      ['layout-shift', clsNodeData],
+      ['layout-shift-element', shiftElementsNodeData],
+      ['layout-shift', shiftsData],
       ['animation', animatedElementData],
       ['responsiveness', responsivenessElementData ? [responsivenessElementData] : []],
     ]);
 
+    /** @type {Map<number, LH.Crdp.Runtime.CallFunctionOnResponse | null>} */
+    const callFunctionOnCache = new Map();
     const traceElements = [];
     for (const [traceEventType, backendNodeData] of backendNodeDataMap) {
       for (let i = 0; i < backendNodeData.length; i++) {
         const backendNodeId = backendNodeData[i].nodeId;
-        let response;
-        try {
-          const objectId = await resolveNodeIdToObjectId(session, backendNodeId);
-          if (!objectId) continue;
-
-          const deps = ExecutionContext.serializeDeps([
-            pageFunctions.getNodeDetails,
-            getNodeDetailsData,
-          ]);
-          response = await session.sendCommand('Runtime.callFunctionOn', {
-            objectId,
-            functionDeclaration: `function () {
-              ${deps}
-              return getNodeDetailsData.call(this);
-            }`,
-            returnByValue: true,
-            awaitPromise: true,
-          });
-        } catch (err) {
-          Sentry.captureException(err, {
-            tags: {gatherer: 'TraceElements'},
-            level: 'error',
-          });
-          continue;
+        let response = callFunctionOnCache.get(backendNodeId);
+        if (response === undefined) {
+          response = await this.getNodeDetails(session, backendNodeId);
+          callFunctionOnCache.set(backendNodeId, response);
         }
 
         if (response?.result?.value) {

package/core/gather/gatherers/trace.js

@@ -56,6 +56,9 @@ class Trace extends BaseGatherer {
       'disabled-by-default-devtools.timeline.frame',
       'latencyInfo',
 
+      // For CLS root causes.
+      'disabled-by-default-devtools.timeline.invalidationTracking',
+
       // Not used by Lighthouse (yet) but included for users that want JS samples when looking at
       // a trace collected by Lighthouse (e.g. "View Trace" workflow in DevTools)
       'disabled-by-default-v8.cpu_profiler',

package/core/gather/session.js

@@ -10,6 +10,17 @@ import {LighthouseError} from '../lib/lh-error.js';
 
 // Controls how long to wait for a response after sending a DevTools protocol command.
 const DEFAULT_PROTOCOL_TIMEOUT = 30000;
+const PPTR_BUFFER = 50;
+
+/**
+ * Puppeteer timeouts must fit into an int32 and the maximum timeout for `setTimeout` is a *signed*
+ * int32. However, this also needs to account for the puppeteer buffer we add to the timeout later.
+ *
+ * So this is defined as the max *signed* int32 minus PPTR_BUFFER.
+ *
+ * In human terms, this timeout is ~25 days which is as good as infinity for all practical purposes.
+ */
+const MAX_TIMEOUT = 2147483647 - PPTR_BUFFER;
 
 /** @typedef {LH.Protocol.StrictEventEmitterClass<LH.CrdpEvents>} CrdpEventMessageEmitter */
 const CrdpEventEmitter = /** @type {CrdpEventMessageEmitter} */ (EventEmitter);
@@ -70,6 +81,7 @@ class ProtocolSession extends CrdpEventEmitter {
    * @param {number} ms
    */
   setNextProtocolTimeout(ms) {
+    if (ms > MAX_TIMEOUT) ms = MAX_TIMEOUT;
     this._nextProtocolTimeout = ms;
   }
 
@@ -86,15 +98,16 @@ class ProtocolSession extends CrdpEventEmitter {
     /** @type {NodeJS.Timer|undefined} */
     let timeout;
     const timeoutPromise = new Promise((resolve, reject) => {
-      if (timeoutMs === Infinity) return;
-
       // eslint-disable-next-line max-len
       timeout = setTimeout(reject, timeoutMs, new LighthouseError(LighthouseError.errors.PROTOCOL_TIMEOUT, {
         protocolMethod: method,
       }));
     });
 
-    const resultPromise = this._cdpSession.send(method, ...params);
+    const resultPromise = this._cdpSession.send(method, ...params, {
+      // Add 50ms to the Puppeteer timeout to ensure the Lighthouse timeout finishes first.
+      timeout: timeoutMs + PPTR_BUFFER,
+    });
     const resultWithTimeoutPromise = Promise.race([resultPromise, timeoutPromise]);
 
     return resultWithTimeoutPromise.finally(() => {

package/core/lib/i18n/i18n.js

@@ -17,6 +17,7 @@ import {LH_ROOT} from '../../../shared/root.js';
 import {isIcuMessage, formatMessage, DEFAULT_LOCALE} from '../../../shared/localization/format.js';
 import {getModulePath} from '../../../shared/esm-utils.js';
 
+/* eslint-disable max-len */
 const UIStrings = {
   /** Used to show the duration in milliseconds that something lasted. The `{timeInMs}` placeholder will be replaced with the time duration, shown in milliseconds (e.g. 63 ms) */
   ms: '{timeInMs, number, milliseconds}\xa0ms',
@@ -113,6 +114,7 @@ const UIStrings = {
   /** Table item value for the severity of a high impact, or dangerous vulnerability. Part of a ranking scale in the form: low, medium, high. */
   itemSeverityHigh: 'High',
 };
+/* eslint-enable max-len */
 
 /**
  * Look up the best available locale for the requested language through these fall backs:

package/core/lib/trace-engine.d.ts

@@ -1,4 +1,7 @@
-export const TraceProcessor: any;
+/** @type {import('../../types/trace-engine.js').TraceProcessor & typeof import('../../types/trace-engine.js').TraceProcessor} */
+export const TraceProcessor: import('../../types/trace-engine.js').TraceProcessor & typeof import('../../types/trace-engine.js').TraceProcessor;
+/** @type {import('../../types/trace-engine.js').TraceHandlers} */
 export const TraceHandlers: any;
-export const RootCauses: any;
+/** @type {import('../../types/trace-engine.js').RootCauses & typeof import('../../types/trace-engine.js').RootCauses} */
+export const RootCauses: import('../../types/trace-engine.js').RootCauses & typeof import('../../types/trace-engine.js').RootCauses;
 //# sourceMappingURL=trace-engine.d.ts.map

package/core/lib/trace-engine.js

@@ -5,8 +5,11 @@ import {polyfillDOMRect} from './polyfill-dom-rect.js';
 
 polyfillDOMRect();
 
+/** @type {import('../../types/trace-engine.js').TraceProcessor & typeof import('../../types/trace-engine.js').TraceProcessor} */
 const TraceProcessor = TraceEngine.Processor.TraceProcessor;
+/** @type {import('../../types/trace-engine.js').TraceHandlers} */
 const TraceHandlers = TraceEngine.Handlers.ModelHandlers;
+/** @type {import('../../types/trace-engine.js').RootCauses & typeof import('../../types/trace-engine.js').RootCauses} */
 const RootCauses = TraceEngine.RootCauses.RootCauses.RootCauses;
 
 export {