lighthouse 9.5.0-dev.20230124 → 9.5.0-dev.20230126

package/core/audits/preload-lcp-image.js

@@ -11,6 +11,7 @@ import {MainResource} from '../computed/main-resource.js';
 import {LanternLargestContentfulPaint} from '../computed/metrics/lantern-largest-contentful-paint.js';
 import {LoadSimulator} from '../computed/load-simulator.js';
 import {ByteEfficiencyAudit} from './byte-efficiency/byte-efficiency-audit.js';
+import {ProcessedNavigation} from '../computed/processed-navigation.js';
 
 const UIStrings = {
   /** Title of a lighthouse audit that tells a user to preload an image in order to improve their LCP time. */
@@ -36,8 +37,7 @@ class PreloadLCPImageAudit extends Audit {
       title: str_(UIStrings.title),
       description: str_(UIStrings.description),
       supportedModes: ['navigation'],
-      requiredArtifacts: ['traces', 'devtoolsLogs', 'GatherContext', 'URL', 'TraceElements',
-        'ImageElements'],
+      requiredArtifacts: ['traces', 'devtoolsLogs', 'GatherContext', 'URL', 'TraceElements'],
       scoreDisplayMode: Audit.SCORING_MODES.NUMERIC,
     };
   }
@@ -87,17 +87,11 @@ class PreloadLCPImageAudit extends Audit {
   /**
    * @param {LH.Artifacts.NetworkRequest} mainResource
    * @param {LH.Gatherer.Simulation.GraphNode} graph
-   * @param {LH.Artifacts.TraceElement} lcpElement
-   * @param {Array<LH.Artifacts.ImageElement>} imageElements
+   * @param {string | undefined} lcpUrl
    * @return {{lcpNodeToPreload?: LH.Gatherer.Simulation.GraphNetworkNode, initiatorPath?: InitiatorPath}}
    */
-  static getLCPNodeToPreload(mainResource, graph, lcpElement, imageElements) {
-    const lcpImageElement = imageElements.find(elem => {
-      return elem.node.devtoolsNodePath === lcpElement.node.devtoolsNodePath;
-    });
-
-    if (!lcpImageElement) return {};
-    const lcpUrl = lcpImageElement.src;
+  static getLCPNodeToPreload(mainResource, graph, lcpUrl) {
+    if (!lcpUrl) return {};
     const {lcpNode, path} = PreloadLCPImageAudit.findLCPNode(graph, lcpUrl);
     if (!lcpNode || !path) return {};
 
@@ -116,6 +110,29 @@ class PreloadLCPImageAudit extends Audit {
     };
   }
 
+  /**
+   * Match the LCP event with the paint event to get the URL of the image actually painted.
+   * This could differ from the `ImageElement` associated with the nodeId if e.g. the LCP
+   * was a pseudo-element associated with a node containing a smaller background-image.
+   * @param {LH.Trace} trace
+   * @param {LH.Artifacts.ProcessedNavigation} processedNavigation
+   * @return {string | undefined}
+   */
+  static getLcpUrl(trace, processedNavigation) {
+    const lcpEvent = processedNavigation.largestContentfulPaintAllFramesEvt;
+    if (!lcpEvent) return;
+
+    const lcpImagePaintEvent = trace.traceEvents.filter(e => {
+      return e.name === 'LargestImagePaint::Candidate' &&
+          e.args.frame === lcpEvent.args.frame &&
+          e.args.data?.DOMNodeId === lcpEvent.args.data?.nodeId &&
+          e.args.data?.size === lcpEvent.args.data?.size;
+    // Get last candidate, in case there was more than one.
+    }).sort((a, b) => b.ts - a.ts)[0];
+
+    return lcpImagePaintEvent?.args.data?.imageUrl;
+  }
+
   /**
    * Computes the estimated effect of preloading the LCP image.
    * @param {LH.Artifacts.TraceElement} lcpElement
@@ -223,15 +240,17 @@ class PreloadLCPImageAudit extends Audit {
       return {score: null, notApplicable: true};
     }
 
-    const [mainResource, lanternLCP, simulator] = await Promise.all([
+    const [processedNavigation, mainResource, lanternLCP, simulator] = await Promise.all([
+      ProcessedNavigation.request(trace, context),
       MainResource.request({devtoolsLog, URL}, context),
       LanternLargestContentfulPaint.request(metricData, context),
       LoadSimulator.request({devtoolsLog, settings: context.settings}, context),
     ]);
 
+    const lcpUrl = PreloadLCPImageAudit.getLcpUrl(trace, processedNavigation);
     const graph = lanternLCP.pessimisticGraph;
     // eslint-disable-next-line max-len
-    const {lcpNodeToPreload, initiatorPath} = PreloadLCPImageAudit.getLCPNodeToPreload(mainResource, graph, lcpElement, artifacts.ImageElements);
+    const {lcpNodeToPreload, initiatorPath} = PreloadLCPImageAudit.getLCPNodeToPreload(mainResource, graph, lcpUrl);
 
     const {results, wastedMs} =
       PreloadLCPImageAudit.computeWasteWithGraph(lcpElement, lcpNodeToPreload, graph, simulator);

package/core/computed/entity-classification.js

@@ -6,7 +6,7 @@
 
 import {makeComputedArtifact} from './computed-artifact.js';
 import {NetworkRecords} from './network-records.js';
-import {Util} from '../util.cjs';
+import {Util} from '../../shared/util.js';
 import UrlUtils from '../lib/url-utils.js';
 import thirdPartyWeb from '../lib/third-party-web.js';
 

package/core/computed/metrics/responsiveness.js

@@ -87,8 +87,11 @@ class Responsiveness {
       return evt.name === 'EventTiming' && evt.ph !== 'e';
     });
 
-    if (candidates.length && !candidates.some(candidate => candidate.args.data?.frame)) {
-      // Full EventTiming data added in https://crrev.com/c/3632661
+    // If trace is from < m103, the timestamps cannot be trusted, so we craft a fallback
+    // <m103 traces (bad) had a   args.frame
+    // m103+ traces (good) have a args.data.frame (https://crrev.com/c/3632661)
+    // TODO(compat): remove FallbackTiming handling when we don't care about <m103
+    if (candidates.length && candidates.every(candidate => !candidate.args.data?.frame)) {
       return {
         name: 'FallbackTiming',
         duration: responsivenessEvent.args.data.maxDuration,

package/core/computed/page-dependency-graph.js

@@ -468,7 +468,7 @@ class PageDependencyGraph {
    * @return {URLArtifact}
    */
   static getDocumentUrls(devtoolsLog, networkRecords, processedTrace) {
-    const mainFrameId = processedTrace.mainFrameIds.frameId;
+    const mainFrameId = processedTrace.mainFrameInfo.frameId;
 
     /** @type {string|undefined} */
     let requestedUrl;

package/core/config/constants.d.ts

@@ -80,7 +80,7 @@ declare const MOTOG4_EMULATION_METRICS: Required<LH.SharedFlagsSettings['screenE
  * @type {Required<LH.SharedFlagsSettings['screenEmulation']>}
  */
 declare const DESKTOP_EMULATION_METRICS: Required<LH.SharedFlagsSettings['screenEmulation']>;
-declare const MOTOG4_USERAGENT: "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Mobile Safari/537.36 Chrome-Lighthouse";
-declare const DESKTOP_USERAGENT: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Safari/537.36 Chrome-Lighthouse";
+declare const MOTOG4_USERAGENT: "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Mobile Safari/537.36";
+declare const DESKTOP_USERAGENT: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36";
 export {};
 //# sourceMappingURL=constants.d.ts.map

package/core/config/constants.js

@@ -81,8 +81,8 @@ const screenEmulationMetrics = {
 };
 
 
-const MOTOG4_USERAGENT = 'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Mobile Safari/537.36 Chrome-Lighthouse'; // eslint-disable-line max-len
-const DESKTOP_USERAGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Safari/537.36 Chrome-Lighthouse'; // eslint-disable-line max-len
+const MOTOG4_USERAGENT = 'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Mobile Safari/537.36'; // eslint-disable-line max-len
+const DESKTOP_USERAGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36'; // eslint-disable-line max-len
 
 const userAgents = {
   mobile: MOTOG4_USERAGENT,

package/core/gather/gatherers/accessibility.js

@@ -61,10 +61,12 @@ async function runA11yChecks() {
       'frame-focusable-content': {enabled: false},
       'aria-roledescription': {enabled: false},
       'scrollable-region-focusable': {enabled: false},
-      // TODO(paulirish): create audits and enable these 3.
+      // TODO(paulirish): create audits and enable these 5.
       'input-button-name': {enabled: false},
       'role-img-alt': {enabled: false},
       'select-name': {enabled: false},
+      'link-in-text-block': {enabled: false},
+      'frame-title-unique': {enabled: false},
     },
   });
 

package/core/gather/gatherers/trace-elements.js

@@ -231,7 +231,7 @@ class TraceElements extends FRGatherer {
     }
 
     // These should exist, but trace types are loose.
-    const lcpData = processedNavigation.largestContentfulPaintEvt?.args?.data;
+    const lcpData = processedNavigation.largestContentfulPaintAllFramesEvt?.args?.data;
     if (lcpData?.nodeId === undefined || !lcpData.type) return;
 
     return {

package/core/lib/network-request.d.ts

@@ -86,6 +86,8 @@ export class NetworkRequest {
     isValid: boolean;
     parsedURL: ParsedURL;
     documentURL: string;
+    /** When the renderer process initially discovers a network request, in milliseconds. */
+    rendererStartTime: number;
     /**
      * When the network service is about to handle a request, ie. just before going to the
      * HTTP cache or going to the network for DNS/connection setup, in milliseconds.

package/core/lib/network-request.js

@@ -125,6 +125,8 @@ class NetworkRequest {
     this.parsedURL = /** @type {ParsedURL} */ ({scheme: ''});
     this.documentURL = '';
 
+    /** When the renderer process initially discovers a network request, in milliseconds. */
+    this.rendererStartTime = -1;
     /**
      * When the network service is about to handle a request, ie. just before going to the
      * HTTP cache or going to the network for DNS/connection setup, in milliseconds.
@@ -222,6 +224,7 @@ class NetworkRequest {
     };
     this.isSecure = UrlUtils.isSecureScheme(this.parsedURL.scheme);
 
+    this.rendererStartTime = data.timestamp * 1000;
     // Expected to be overriden with better value in `_recomputeTimesWithResourceTiming`.
     this.startTime = data.timestamp * 1000;
 

package/core/lib/stack-packs.js

@@ -19,6 +19,10 @@ const stackPacksToInclude = [
     packId: 'wordpress',
     requiredStacks: ['js:wordpress'],
   },
+  {
+    packId: 'wp-rocket',
+    requiredStacks: ['js:wp-rocket'],
+  },
   {
     packId: 'ezoic',
     requiredStacks: ['js:ezoic'],

package/core/lib/tracehouse/trace-processor.d.ts

@@ -148,13 +148,22 @@ export class TraceProcessor {
     static getMainThreadTopLevelEvents(trace: LH.Artifacts.ProcessedTrace, startTime?: number | undefined, endTime?: number | undefined): Array<ToplevelEvent>;
     /**
      * @param {LH.TraceEvent[]} events
-     * @return {{pid: number, tid: number, frameId: string}}
+     * @return {{startingPid: number, frameId: string}}
      */
     static findMainFrameIds(events: LH.TraceEvent[]): {
-        pid: number;
-        tid: number;
+        startingPid: number;
         frameId: string;
     };
+    /**
+     * If there were any cross-origin navigations, there'll be more than one pid returned
+     * @param {{startingPid: number, frameId: string}} mainFrameInfo
+     * @param {LH.TraceEvent[]} keyEvents
+     * @return {Map<number, number>} Map where keys are process IDs and their values are thread IDs
+     */
+    static findMainFramePidTids(mainFrameInfo: {
+        startingPid: number;
+        frameId: string;
+    }, keyEvents: LH.TraceEvent[]): Map<number, number>;
     /**
      * @param {LH.TraceEvent} evt
      * @return {boolean}
@@ -239,14 +248,14 @@ export class TraceProcessor {
      *      Can also be skewed by several hundred milliseconds or even seconds when the browser takes a long
      *      time to unload `about:blank`.
      *
-     * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameIds: {frameId: string}}} traceEventSubsets
+     * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameInfo: {frameId: string}}} traceEventSubsets
      * @param {TimeOriginDeterminationMethod} method
      * @return {LH.TraceEvent}
      */
     static computeTimeOrigin(traceEventSubsets: {
         keyEvents: Array<LH.TraceEvent>;
         frameEvents: Array<LH.TraceEvent>;
-        mainFrameIds: {
+        mainFrameInfo: {
             frameId: string;
         };
     }, method: TimeOriginDeterminationMethod): LH.TraceEvent;

package/core/lib/tracehouse/trace-processor.js

@@ -423,7 +423,7 @@ class TraceProcessor {
 
   /**
    * @param {LH.TraceEvent[]} events
-   * @return {{pid: number, tid: number, frameId: string}}
+   * @return {{startingPid: number, frameId: string}}
    */
   static findMainFrameIds(events) {
     // Prefer the newer TracingStartedInBrowser event first, if it exists
@@ -433,14 +433,9 @@ class TraceProcessor {
       const frameId = mainFrame?.frame;
       const pid = mainFrame?.processId;
 
-      const threadNameEvt = events.find(e => e.pid === pid && e.ph === 'M' &&
-        e.cat === '__metadata' && e.name === 'thread_name' && e.args.name === 'CrRendererMain');
-      const tid = threadNameEvt?.tid;
-
-      if (pid && tid && frameId) {
+      if (pid && frameId) {
         return {
-          pid,
-          tid,
+          startingPid: pid,
           frameId,
         };
       }
@@ -454,8 +449,7 @@ class TraceProcessor {
       const frameId = startedInPageEvt.args.data.page;
       if (frameId) {
         return {
-          pid: startedInPageEvt.pid,
-          tid: startedInPageEvt.tid,
+          startingPid: startedInPageEvt.pid,
           frameId,
         };
       }
@@ -478,8 +472,7 @@ class TraceProcessor {
       const frameId = navStartEvt.args.frame;
       if (frameId) {
         return {
-          pid: navStartEvt.pid,
-          tid: navStartEvt.tid,
+          startingPid: navStartEvt.pid,
           frameId,
         };
       }
@@ -488,6 +481,47 @@ class TraceProcessor {
     throw this.createNoTracingStartedError();
   }
 
+  /**
+   * If there were any cross-origin navigations, there'll be more than one pid returned
+   * @param {{startingPid: number, frameId: string}} mainFrameInfo
+   * @param {LH.TraceEvent[]} keyEvents
+   * @return {Map<number, number>} Map where keys are process IDs and their values are thread IDs
+   */
+  static findMainFramePidTids(mainFrameInfo, keyEvents) {
+    const frameCommittedEvts = keyEvents.filter(evt =>
+      evt.name === 'FrameCommittedInBrowser' &&
+      evt.args?.data?.frame === mainFrameInfo.frameId
+    );
+
+    // "Modern" traces with a navigation have a FrameCommittedInBrowser event
+    const mainFramePids = frameCommittedEvts.length
+      ? frameCommittedEvts.map(e => e?.args?.data?.processId)
+      // …But old traces and some timespan traces may not. In these situations, we'll assume the
+      // primary process ID remains constant (as there were no navigations).
+      : [mainFrameInfo.startingPid];
+
+    const pidToTid = new Map();
+
+    mainFramePids.forEach(pid => {
+      // While renderer tids are generally predictable, we'll doublecheck it
+      const threadNameEvt = keyEvents.find(e =>
+        e.cat === '__metadata' &&
+        e.pid === pid &&
+        e.ph === 'M' &&
+        e.name === 'thread_name' &&
+        e.args.name === 'CrRendererMain'
+      );
+      const tid = threadNameEvt?.tid;
+
+      if (!tid) {
+        throw new Error('Unable to determine tid for renderer process');
+      }
+
+      pidToTid.set(pid, tid);
+    });
+    return pidToTid;
+  }
+
   /**
    * @param {LH.TraceEvent} evt
    * @return {boolean}
@@ -583,6 +617,9 @@ class TraceProcessor {
       while (parentFrames.has(cur)) {
         cur = /** @type {string} */ (parentFrames.get(cur));
       }
+      if (cur === undefined) {
+        throw new Error('Unexpected undefined frameId');
+      }
       frameIdToRootFrameId.set(frame.id, cur);
     }
 
@@ -609,7 +646,15 @@ class TraceProcessor {
     });
 
     // Find the inspected frame
-    const mainFrameIds = this.findMainFrameIds(keyEvents);
+    const mainFrameInfo = this.findMainFrameIds(keyEvents);
+    const rendererPidToTid = this.findMainFramePidTids(mainFrameInfo, keyEvents);
+
+    // Subset all trace events to just our tab's process (incl threads other than main)
+    // stable-sort events to keep them correctly nested.
+    const processEvents = TraceProcessor
+      .filteredTraceSort(trace.traceEvents, e => rendererPidToTid.has(e.pid));
+
+    // TODO(paulirish): filter down frames (and subsequent actions) to the primary process tree & frame tree
 
     /** @type {Map<string, {id: string, url: string, parent?: string}>} */
     const framesById = new Map();
@@ -645,18 +690,32 @@ class TraceProcessor {
           parent: evt.args.data.parent,
         });
       });
+
     const frames = [...framesById.values()];
     const frameIdToRootFrameId = this.resolveRootFrames(frames);
 
+    const inspectedTreeFrameIds = [...frameIdToRootFrameId.entries()]
+      .filter(([, rootFrameId]) => rootFrameId === mainFrameInfo.frameId)
+      .map(([child]) => child);
+
     // Filter to just events matching the main frame ID, just to make sure.
-    const frameEvents = keyEvents.filter(e => e.args.frame === mainFrameIds.frameId);
+    /** @param {LH.TraceEvent} e */
+    function associatedToMainFrame(e) {
+      const frameId = e.args?.data?.frame || e.args.frame;
+      return frameId === mainFrameInfo.frameId;
+    }
+
+    /** @param {LH.TraceEvent} e */
+    function associatedToAllFrames(e) {
+      const frameId = e.args?.data?.frame || e.args.frame;
+      return frameId ? inspectedTreeFrameIds.includes(frameId) : false;
+    }
+    const frameEvents = keyEvents.filter(e => associatedToMainFrame(e));
 
     // Filter to just events matching the main frame ID or any child frame IDs.
     let frameTreeEvents = [];
-    if (frameIdToRootFrameId.has(mainFrameIds.frameId)) {
-      frameTreeEvents = keyEvents.filter(e => {
-        return e.args.frame && frameIdToRootFrameId.get(e.args.frame) === mainFrameIds.frameId;
-      });
+    if (frameIdToRootFrameId.has(mainFrameInfo.frameId)) {
+      frameTreeEvents = keyEvents.filter(e => associatedToAllFrames(e));
     } else {
       // In practice, there should always be TracingStartedInBrowser/FrameCommittedInBrowser events to
       // define the frame tree. Unfortunately, many test traces do not that frame info due to minification.
@@ -665,23 +724,17 @@ class TraceProcessor {
         'TraceProcessor',
         'frameTreeEvents may be incomplete, make sure the trace has frame events'
       );
-      frameIdToRootFrameId.set(mainFrameIds.frameId, mainFrameIds.frameId);
+      frameIdToRootFrameId.set(mainFrameInfo.frameId, mainFrameInfo.frameId);
       frameTreeEvents = frameEvents;
     }
 
     // Compute our time origin to use for all relative timings.
     const timeOriginEvt = this.computeTimeOrigin(
-      {keyEvents, frameEvents, mainFrameIds},
+      {keyEvents, frameEvents, mainFrameInfo: mainFrameInfo},
       timeOriginDeterminationMethod
     );
 
-    // Subset all trace events to just our tab's process (incl threads other than main)
-    // stable-sort events to keep them correctly nested.
-    const processEvents = TraceProcessor
-      .filteredTraceSort(trace.traceEvents, e => e.pid === mainFrameIds.pid);
-
-    const mainThreadEvents = processEvents
-      .filter(e => e.tid === mainFrameIds.tid);
+    const mainThreadEvents = processEvents.filter(e => e.tid === rendererPidToTid.get(e.pid));
 
     // Ensure our traceEnd reflects all page activity.
     const traceEnd = this.computeTraceEnd(trace.traceEvents, timeOriginEvt);
@@ -694,7 +747,7 @@ class TraceProcessor {
       frameEvents,
       frameTreeEvents,
       processEvents,
-      mainFrameIds,
+      mainFrameInfo,
       timeOriginEvt,
       timings: {
         timeOrigin: 0,
@@ -704,6 +757,8 @@ class TraceProcessor {
         timeOrigin: timeOriginEvt.ts,
         traceEnd: traceEnd.timestamp,
       },
+      _keyEvents: keyEvents,
+      _rendererPidToTid: rendererPidToTid,
     };
   }
 
@@ -807,7 +862,7 @@ class TraceProcessor {
    *      Can also be skewed by several hundred milliseconds or even seconds when the browser takes a long
    *      time to unload `about:blank`.
    *
-   * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameIds: {frameId: string}}} traceEventSubsets
+   * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameInfo: {frameId: string}}} traceEventSubsets
    * @param {TimeOriginDeterminationMethod} method
    * @return {LH.TraceEvent}
    */
@@ -833,7 +888,7 @@ class TraceProcessor {
         const fetchStart = traceEventSubsets.keyEvents.find(event => {
           if (event.name !== 'ResourceSendRequest') return false;
           const data = event.args.data || {};
-          return data.frame === traceEventSubsets.mainFrameIds.frameId;
+          return data.frame === traceEventSubsets.mainFrameInfo.frameId;
         });
         if (!fetchStart) throw this.createNoResourceSendRequestError();
         return fetchStart;

package/core/lib/traces/metric-trace-events.js

@@ -128,7 +128,7 @@ class MetricTraceEvents {
     if (!timeOriginMetric) return {errorMessage: 'timeorigin Metric not found in definitions'};
     try {
       const frameIds = TraceProcessor.findMainFrameIds(this._traceEvents);
-      return {pid: frameIds.pid, tid: frameIds.tid, ts: timeOriginMetric.ts};
+      return {pid: frameIds.startingPid, tid: 1, ts: timeOriginMetric.ts};
     } catch (err) {
       return {errorMessage: err.message};
     }