lighthouse 9.5.0-dev.20230123 → 9.5.0-dev.20230125

package/core/lib/network-request.d.ts

@@ -86,6 +86,8 @@ export class NetworkRequest {
     isValid: boolean;
     parsedURL: ParsedURL;
     documentURL: string;
+    /** When the renderer process initially discovers a network request, in milliseconds. */
+    rendererStartTime: number;
     /**
      * When the network service is about to handle a request, ie. just before going to the
      * HTTP cache or going to the network for DNS/connection setup, in milliseconds.

package/core/lib/network-request.js

@@ -125,6 +125,8 @@ class NetworkRequest {
     this.parsedURL = /** @type {ParsedURL} */ ({scheme: ''});
     this.documentURL = '';
 
+    /** When the renderer process initially discovers a network request, in milliseconds. */
+    this.rendererStartTime = -1;
     /**
      * When the network service is about to handle a request, ie. just before going to the
      * HTTP cache or going to the network for DNS/connection setup, in milliseconds.
@@ -222,6 +224,7 @@ class NetworkRequest {
     };
     this.isSecure = UrlUtils.isSecureScheme(this.parsedURL.scheme);
 
+    this.rendererStartTime = data.timestamp * 1000;
     // Expected to be overriden with better value in `_recomputeTimesWithResourceTiming`.
     this.startTime = data.timestamp * 1000;
 

package/core/lib/stack-packs.js

@@ -19,6 +19,10 @@ const stackPacksToInclude = [
     packId: 'wordpress',
     requiredStacks: ['js:wordpress'],
   },
+  {
+    packId: 'wp-rocket',
+    requiredStacks: ['js:wp-rocket'],
+  },
   {
     packId: 'ezoic',
     requiredStacks: ['js:ezoic'],

package/core/lib/tracehouse/trace-processor.d.ts

@@ -148,13 +148,22 @@ export class TraceProcessor {
     static getMainThreadTopLevelEvents(trace: LH.Artifacts.ProcessedTrace, startTime?: number | undefined, endTime?: number | undefined): Array<ToplevelEvent>;
     /**
      * @param {LH.TraceEvent[]} events
-     * @return {{pid: number, tid: number, frameId: string}}
+     * @return {{startingPid: number, frameId: string}}
      */
     static findMainFrameIds(events: LH.TraceEvent[]): {
-        pid: number;
-        tid: number;
+        startingPid: number;
         frameId: string;
     };
+    /**
+     * If there were any cross-origin navigations, there'll be more than one pid returned
+     * @param {{startingPid: number, frameId: string}} mainFrameInfo
+     * @param {LH.TraceEvent[]} keyEvents
+     * @return {Map<number, number>} Map where keys are process IDs and their values are thread IDs
+     */
+    static findMainFramePidTids(mainFrameInfo: {
+        startingPid: number;
+        frameId: string;
+    }, keyEvents: LH.TraceEvent[]): Map<number, number>;
     /**
      * @param {LH.TraceEvent} evt
      * @return {boolean}
@@ -239,14 +248,14 @@ export class TraceProcessor {
      *      Can also be skewed by several hundred milliseconds or even seconds when the browser takes a long
      *      time to unload `about:blank`.
      *
-     * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameIds: {frameId: string}}} traceEventSubsets
+     * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameInfo: {frameId: string}}} traceEventSubsets
      * @param {TimeOriginDeterminationMethod} method
      * @return {LH.TraceEvent}
      */
     static computeTimeOrigin(traceEventSubsets: {
         keyEvents: Array<LH.TraceEvent>;
         frameEvents: Array<LH.TraceEvent>;
-        mainFrameIds: {
+        mainFrameInfo: {
             frameId: string;
         };
     }, method: TimeOriginDeterminationMethod): LH.TraceEvent;

package/core/lib/tracehouse/trace-processor.js

@@ -423,7 +423,7 @@ class TraceProcessor {
 
   /**
    * @param {LH.TraceEvent[]} events
-   * @return {{pid: number, tid: number, frameId: string}}
+   * @return {{startingPid: number, frameId: string}}
    */
   static findMainFrameIds(events) {
     // Prefer the newer TracingStartedInBrowser event first, if it exists
@@ -433,14 +433,9 @@ class TraceProcessor {
       const frameId = mainFrame?.frame;
       const pid = mainFrame?.processId;
 
-      const threadNameEvt = events.find(e => e.pid === pid && e.ph === 'M' &&
-        e.cat === '__metadata' && e.name === 'thread_name' && e.args.name === 'CrRendererMain');
-      const tid = threadNameEvt?.tid;
-
-      if (pid && tid && frameId) {
+      if (pid && frameId) {
         return {
-          pid,
-          tid,
+          startingPid: pid,
           frameId,
         };
       }
@@ -454,8 +449,7 @@ class TraceProcessor {
       const frameId = startedInPageEvt.args.data.page;
       if (frameId) {
         return {
-          pid: startedInPageEvt.pid,
-          tid: startedInPageEvt.tid,
+          startingPid: startedInPageEvt.pid,
           frameId,
         };
       }
@@ -478,8 +472,7 @@ class TraceProcessor {
       const frameId = navStartEvt.args.frame;
       if (frameId) {
         return {
-          pid: navStartEvt.pid,
-          tid: navStartEvt.tid,
+          startingPid: navStartEvt.pid,
           frameId,
         };
       }
@@ -488,6 +481,47 @@ class TraceProcessor {
     throw this.createNoTracingStartedError();
   }
 
+  /**
+   * If there were any cross-origin navigations, there'll be more than one pid returned
+   * @param {{startingPid: number, frameId: string}} mainFrameInfo
+   * @param {LH.TraceEvent[]} keyEvents
+   * @return {Map<number, number>} Map where keys are process IDs and their values are thread IDs
+   */
+  static findMainFramePidTids(mainFrameInfo, keyEvents) {
+    const frameCommittedEvts = keyEvents.filter(evt =>
+      evt.name === 'FrameCommittedInBrowser' &&
+      evt.args?.data?.frame === mainFrameInfo.frameId
+    );
+
+    // "Modern" traces with a navigation have a FrameCommittedInBrowser event
+    const mainFramePids = frameCommittedEvts.length
+      ? frameCommittedEvts.map(e => e?.args?.data?.processId)
+      // …But old traces and some timespan traces may not. In these situations, we'll assume the
+      // primary process ID remains constant (as there were no navigations).
+      : [mainFrameInfo.startingPid];
+
+    const pidToTid = new Map();
+
+    mainFramePids.forEach(pid => {
+      // While renderer tids are generally predictable, we'll doublecheck it
+      const threadNameEvt = keyEvents.find(e =>
+        e.cat === '__metadata' &&
+        e.pid === pid &&
+        e.ph === 'M' &&
+        e.name === 'thread_name' &&
+        e.args.name === 'CrRendererMain'
+      );
+      const tid = threadNameEvt?.tid;
+
+      if (!tid) {
+        throw new Error('Unable to determine tid for renderer process');
+      }
+
+      pidToTid.set(pid, tid);
+    });
+    return pidToTid;
+  }
+
   /**
    * @param {LH.TraceEvent} evt
    * @return {boolean}
@@ -583,6 +617,9 @@ class TraceProcessor {
       while (parentFrames.has(cur)) {
         cur = /** @type {string} */ (parentFrames.get(cur));
       }
+      if (cur === undefined) {
+        throw new Error('Unexpected undefined frameId');
+      }
       frameIdToRootFrameId.set(frame.id, cur);
     }
 
@@ -609,7 +646,15 @@ class TraceProcessor {
     });
 
     // Find the inspected frame
-    const mainFrameIds = this.findMainFrameIds(keyEvents);
+    const mainFrameInfo = this.findMainFrameIds(keyEvents);
+    const rendererPidToTid = this.findMainFramePidTids(mainFrameInfo, keyEvents);
+
+    // Subset all trace events to just our tab's process (incl threads other than main)
+    // stable-sort events to keep them correctly nested.
+    const processEvents = TraceProcessor
+      .filteredTraceSort(trace.traceEvents, e => rendererPidToTid.has(e.pid));
+
+    // TODO(paulirish): filter down frames (and subsequent actions) to the primary process tree & frame tree
 
     /** @type {Map<string, {id: string, url: string, parent?: string}>} */
     const framesById = new Map();
@@ -645,18 +690,32 @@ class TraceProcessor {
           parent: evt.args.data.parent,
         });
       });
+
     const frames = [...framesById.values()];
     const frameIdToRootFrameId = this.resolveRootFrames(frames);
 
+    const inspectedTreeFrameIds = [...frameIdToRootFrameId.entries()]
+      .filter(([, rootFrameId]) => rootFrameId === mainFrameInfo.frameId)
+      .map(([child]) => child);
+
     // Filter to just events matching the main frame ID, just to make sure.
-    const frameEvents = keyEvents.filter(e => e.args.frame === mainFrameIds.frameId);
+    /** @param {LH.TraceEvent} e */
+    function associatedToMainFrame(e) {
+      const frameId = e.args?.data?.frame || e.args.frame;
+      return frameId === mainFrameInfo.frameId;
+    }
+
+    /** @param {LH.TraceEvent} e */
+    function associatedToAllFrames(e) {
+      const frameId = e.args?.data?.frame || e.args.frame;
+      return frameId ? inspectedTreeFrameIds.includes(frameId) : false;
+    }
+    const frameEvents = keyEvents.filter(e => associatedToMainFrame(e));
 
     // Filter to just events matching the main frame ID or any child frame IDs.
     let frameTreeEvents = [];
-    if (frameIdToRootFrameId.has(mainFrameIds.frameId)) {
-      frameTreeEvents = keyEvents.filter(e => {
-        return e.args.frame && frameIdToRootFrameId.get(e.args.frame) === mainFrameIds.frameId;
-      });
+    if (frameIdToRootFrameId.has(mainFrameInfo.frameId)) {
+      frameTreeEvents = keyEvents.filter(e => associatedToAllFrames(e));
     } else {
       // In practice, there should always be TracingStartedInBrowser/FrameCommittedInBrowser events to
       // define the frame tree. Unfortunately, many test traces do not that frame info due to minification.
@@ -665,23 +724,17 @@ class TraceProcessor {
         'TraceProcessor',
         'frameTreeEvents may be incomplete, make sure the trace has frame events'
       );
-      frameIdToRootFrameId.set(mainFrameIds.frameId, mainFrameIds.frameId);
+      frameIdToRootFrameId.set(mainFrameInfo.frameId, mainFrameInfo.frameId);
       frameTreeEvents = frameEvents;
     }
 
     // Compute our time origin to use for all relative timings.
     const timeOriginEvt = this.computeTimeOrigin(
-      {keyEvents, frameEvents, mainFrameIds},
+      {keyEvents, frameEvents, mainFrameInfo: mainFrameInfo},
       timeOriginDeterminationMethod
     );
 
-    // Subset all trace events to just our tab's process (incl threads other than main)
-    // stable-sort events to keep them correctly nested.
-    const processEvents = TraceProcessor
-      .filteredTraceSort(trace.traceEvents, e => e.pid === mainFrameIds.pid);
-
-    const mainThreadEvents = processEvents
-      .filter(e => e.tid === mainFrameIds.tid);
+    const mainThreadEvents = processEvents.filter(e => e.tid === rendererPidToTid.get(e.pid));
 
     // Ensure our traceEnd reflects all page activity.
     const traceEnd = this.computeTraceEnd(trace.traceEvents, timeOriginEvt);
@@ -694,7 +747,7 @@ class TraceProcessor {
       frameEvents,
       frameTreeEvents,
       processEvents,
-      mainFrameIds,
+      mainFrameInfo,
       timeOriginEvt,
       timings: {
         timeOrigin: 0,
@@ -704,6 +757,8 @@ class TraceProcessor {
         timeOrigin: timeOriginEvt.ts,
         traceEnd: traceEnd.timestamp,
       },
+      _keyEvents: keyEvents,
+      _rendererPidToTid: rendererPidToTid,
     };
   }
 
@@ -807,7 +862,7 @@ class TraceProcessor {
    *      Can also be skewed by several hundred milliseconds or even seconds when the browser takes a long
    *      time to unload `about:blank`.
    *
-   * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameIds: {frameId: string}}} traceEventSubsets
+   * @param {{keyEvents: Array<LH.TraceEvent>, frameEvents: Array<LH.TraceEvent>, mainFrameInfo: {frameId: string}}} traceEventSubsets
    * @param {TimeOriginDeterminationMethod} method
    * @return {LH.TraceEvent}
    */
@@ -833,7 +888,7 @@ class TraceProcessor {
         const fetchStart = traceEventSubsets.keyEvents.find(event => {
           if (event.name !== 'ResourceSendRequest') return false;
           const data = event.args.data || {};
-          return data.frame === traceEventSubsets.mainFrameIds.frameId;
+          return data.frame === traceEventSubsets.mainFrameInfo.frameId;
         });
         if (!fetchStart) throw this.createNoResourceSendRequestError();
         return fetchStart;

package/core/lib/traces/metric-trace-events.js

@@ -128,7 +128,7 @@ class MetricTraceEvents {
     if (!timeOriginMetric) return {errorMessage: 'timeorigin Metric not found in definitions'};
     try {
       const frameIds = TraceProcessor.findMainFrameIds(this._traceEvents);
-      return {pid: frameIds.pid, tid: frameIds.tid, ts: timeOriginMetric.ts};
+      return {pid: frameIds.startingPid, tid: 1, ts: timeOriginMetric.ts};
     } catch (err) {
       return {errorMessage: err.message};
     }

package/core/lib/url-utils.js

@@ -4,7 +4,7 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
-import {Util} from '../util.cjs';
+import {Util} from '../../shared/util.js';
 import {LighthouseError} from './lh-error.js';
 
 /** @typedef {import('./network-request.js').NetworkRequest} NetworkRequest */

package/core/runner.d.ts

@@ -16,6 +16,16 @@ export class Runner {
         driverMock?: Driver | undefined;
         computedCache: Map<string, ArbitraryEqualityMap>;
     }): Promise<LH.RunnerResult | undefined>;
+    /**
+     * @param {LH.Artifacts} artifacts
+     * @param {LH.Artifacts.ComputedContext} context
+     */
+    static getEntityClassification(artifacts: LH.Artifacts, context: LH.Artifacts.ComputedContext): Promise<{
+        list: import("../types/lhr/lhr.js").default.LhrEntity[];
+        firstParty: string | undefined;
+        entityIndexByOrigin: Record<string, number>;
+        entityIndexByName: Record<string, number>;
+    } | undefined>;
     /**
      * User can run -G solo, -A solo, or -GA together
      * -G and -A will run partial lighthouse pipelines,

package/core/runner.js

@@ -23,6 +23,8 @@ import {ReportGenerator} from '../report/generator/report-generator.js';
 import {LighthouseError} from './lib/lh-error.js';
 import {lighthouseVersion} from '../root.js';
 import {getModuleDirectory} from '../esm-utils.js';
+import {EntityClassification} from './computed/entity-classification.js';
+import UrlUtils from './lib/url-utils.js';
 
 const moduleDir = getModuleDirectory(import.meta);
 
@@ -110,6 +112,7 @@ class Runner {
         categories,
         categoryGroups: resolvedConfig.groups || undefined,
         stackPacks: stackPacks.getStackPacks(artifacts.Stacks),
+        entities: await Runner.getEntityClassification(artifacts, {computedCache}),
         fullPageScreenshot: resolvedConfig.settings.disableFullPageScreenshot ?
           undefined : artifacts.FullPageScreenshot,
         timing: this._getTiming(artifacts),
@@ -139,6 +142,51 @@ class Runner {
     }
   }
 
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @param {LH.Artifacts.ComputedContext} context
+   */
+  static async getEntityClassification(artifacts, context) {
+    const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
+    if (!devtoolsLog) return;
+    const classifiedEntities = await EntityClassification.request(
+      {URL: artifacts.URL, devtoolsLog}, context);
+
+    /** @type {Array<LH.Result.LhrEntity>} */
+    const entities = [];
+    /** @type {Record<string, number>} */
+    const entityIndexByOrigin = {};
+    /** @type {Record<string, number>} */
+    const entityIndexByName = {};
+
+    for (const [entity, entityUrls] of classifiedEntities.urlsByEntity) {
+      /** @type {LH.Result.LhrEntity} */
+      const shortEntity = {
+        name: entity.name,
+        homepage: entity.homepage,
+      };
+
+      // Reduce payload size in LHR JSON by omitting whats falsy.
+      if (entity === classifiedEntities.firstParty) shortEntity.isFirstParty = true;
+      if (entity.isUnrecognized) shortEntity.isUnrecognized = true;
+
+      const id = entities.push(shortEntity) - 1;
+      for (const url of entityUrls) {
+        const origin = UrlUtils.getOrigin(url);
+        if (!origin) continue;
+        entityIndexByOrigin[origin] = id;
+      }
+      entityIndexByName[shortEntity.name] = id;
+    }
+
+    return {
+      list: entities,
+      firstParty: classifiedEntities.firstParty?.name,
+      entityIndexByOrigin,
+      entityIndexByName,
+    };
+  }
+
   /**
    * User can run -G solo, -A solo, or -GA together
    * -G and -A will run partial lighthouse pipelines,