lighthouse 12.0.0 → 12.1.0-dev.20240619

package/core/lib/lantern/{network-node.js → NetworkNode.js}

@@ -4,9 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import * as Lantern from './types/lantern.js';
+import * as Lantern from './lantern.js';
 import {NetworkRequestTypes} from './lantern.js';
-import {BaseNode} from './base-node.js';
 
 const NON_NETWORK_SCHEMES = [
   'blob', // @see https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL
@@ -18,7 +17,6 @@ const NON_NETWORK_SCHEMES = [
 ];
 
 /**
- * Use `NetworkRequest.isNonNetworkRequest(req)` if working with a request.
  * Note: the `protocol` field from CDP can be 'h2', 'http', (not 'https'!) or it'll be url's scheme.
  *   https://source.chromium.org/chromium/chromium/src/+/main:content/browser/devtools/protocol/network_handler.cc;l=598-611;drc=56d4a9a9deb30be73adcee8737c73bcb2a5ab64f
  * However, a `new URL(href).protocol` has a colon suffix.
@@ -36,9 +34,9 @@ function isNonNetworkProtocol(protocol) {
 
 /**
  * @template [T=any]
- * @extends {BaseNode<T>}
+ * @extends {Lantern.BaseNode<T>}
  */
-class NetworkNode extends BaseNode {
+class NetworkNode extends Lantern.BaseNode {
   /**
    * @param {Lantern.NetworkRequest<T>} networkRequest
    */
@@ -49,7 +47,7 @@ class NetworkNode extends BaseNode {
   }
 
   get type() {
-    return BaseNode.TYPES.NETWORK;
+    return Lantern.BaseNode.TYPES.NETWORK;
   }
 
   /**
@@ -69,8 +67,8 @@ class NetworkNode extends BaseNode {
   /**
    * @return {Readonly<T>}
    */
-  get record() {
-    return /** @type {Required<T>} */ (this._request.record);
+  get rawRequest() {
+    return /** @type {Required<T>} */ (this._request.rawRequest);
   }
 
   /**
@@ -105,7 +103,7 @@ class NetworkNode extends BaseNode {
   }
 
   /**
-   * Returns whether this network record can be downloaded without a TCP connection.
+   * Returns whether this network request can be downloaded without a TCP connection.
    * During simulation we treat data coming in over a network connection separately from on-device data.
    * @return {boolean}
    */

package/core/lib/lantern/PageDependencyGraph.d.ts

@@ -0,0 +1,82 @@
+export type Node = import('./BaseNode.js').Node;
+export type NetworkNodeOutput = {
+    nodes: Array<NetworkNode>;
+    idToNodeMap: Map<string, NetworkNode>;
+    urlToNodeMap: Map<string, Array<NetworkNode>>;
+    frameIdToNodeMap: Map<string, NetworkNode | null>;
+};
+export class PageDependencyGraph {
+    /**
+     * @param {Lantern.NetworkRequest} request
+     * @return {Array<string>}
+     */
+    static getNetworkInitiators(request: Lantern.NetworkRequest): Array<string>;
+    /**
+     * @param {Array<Lantern.NetworkRequest>} networkRequests
+     * @return {NetworkNodeOutput}
+     */
+    static getNetworkNodeOutput(networkRequests: Array<Lantern.NetworkRequest>): NetworkNodeOutput;
+    /**
+     * @param {Lantern.TraceEvent} evt
+     * @return {boolean}
+     */
+    static isScheduleableTask(evt: Lantern.TraceEvent): boolean;
+    /**
+     * There should *always* be at least one top level event, having 0 typically means something is
+     * drastically wrong with the trace and we should just give up early and loudly.
+     *
+     * @param {Lantern.TraceEvent[]} events
+     */
+    static assertHasToplevelEvents(events: Lantern.TraceEvent[]): void;
+    /**
+     * @param {Lantern.TraceEvent[]} mainThreadEvents
+     * @return {Array<CPUNode>}
+     */
+    static getCPUNodes(mainThreadEvents: Lantern.TraceEvent[]): Array<CPUNode>;
+    /**
+     * @param {NetworkNode} rootNode
+     * @param {NetworkNodeOutput} networkNodeOutput
+     */
+    static linkNetworkNodes(rootNode: NetworkNode, networkNodeOutput: NetworkNodeOutput): void;
+    /**
+     * @param {Node} rootNode
+     * @param {NetworkNodeOutput} networkNodeOutput
+     * @param {Array<CPUNode>} cpuNodes
+     */
+    static linkCPUNodes(rootNode: Node, networkNodeOutput: NetworkNodeOutput, cpuNodes: Array<CPUNode>): void;
+    /**
+     * Removes the given node from the graph, but retains all paths between its dependencies and
+     * dependents.
+     * @param {Node} node
+     */
+    static _pruneNode(node: Node): void;
+    /**
+     * TODO(15841): remove when CDT backend is gone. until then, this is a useful debugging tool
+     * to find delta between using CDP or the trace to create the network requests.
+     *
+     * When a test fails using the trace backend, I enabled this debug method and copied the network
+     * requests when CDP was used, then when trace is used, and diff'd them. This method helped
+     * remove non-logical differences from the comparison (order of properties, slight rounding
+     * discrepancies, removing object cycles, etc).
+     *
+     * When using for a unit test, make sure to do `.only` so you are getting what you expect.
+     * @param {Lantern.NetworkRequest[]} lanternRequests
+     * @return {never}
+     */
+    static _debugNormalizeRequests(lanternRequests: Lantern.NetworkRequest[]): never;
+    /**
+     * @param {Lantern.TraceEvent[]} mainThreadEvents
+     * @param {Lantern.NetworkRequest[]} networkRequests
+     * @param {Lantern.Simulation.URL} URL
+     * @return {Node}
+     */
+    static createGraph(mainThreadEvents: Lantern.TraceEvent[], networkRequests: Lantern.NetworkRequest[], URL: Lantern.Simulation.URL): Node;
+    /**
+     * @param {Node} rootNode
+     */
+    static printGraph(rootNode: Node, widthInCharacters?: number): void;
+}
+import { NetworkNode } from './NetworkNode.js';
+import * as Lantern from './lantern.js';
+import { CPUNode } from './CpuNode.js';
+//# sourceMappingURL=PageDependencyGraph.d.ts.map

package/core/lib/lantern/{page-dependency-graph.js → PageDependencyGraph.js}

@@ -4,15 +4,20 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import * as Lantern from './types/lantern.js';
-import {NetworkRequestTypes} from './lantern.js';
-import {NetworkNode} from './network-node.js';
-import {CPUNode} from './cpu-node.js';
-import {TraceProcessor} from '../tracehouse/trace-processor.js';
-import {NetworkAnalyzer} from './simulator/network-analyzer.js';
+import * as Lantern from './lantern.js';
+import {NetworkNode} from './NetworkNode.js';
+import {CPUNode} from './CpuNode.js';
 
-/** @typedef {import('./base-node.js').Node} Node */
-/** @typedef {Omit<LH.Artifacts['URL'], 'finalDisplayedUrl'>} URLArtifact */
+// COMPAT: m71+ We added RunTask to `disabled-by-default-lighthouse`
+const SCHEDULABLE_TASK_TITLE_LH = 'RunTask';
+// m69-70 DoWork is different and we now need RunTask, see https://bugs.chromium.org/p/chromium/issues/detail?id=871204#c11
+const SCHEDULABLE_TASK_TITLE_ALT1 = 'ThreadControllerImpl::RunTask';
+// In m66-68 refactored to this task title, https://crrev.com/c/883346
+const SCHEDULABLE_TASK_TITLE_ALT2 = 'ThreadControllerImpl::DoWork';
+// m65 and earlier
+const SCHEDULABLE_TASK_TITLE_ALT3 = 'TaskQueueManager::ProcessTaskFromWorkQueue';
+
+/** @typedef {import('./BaseNode.js').Node} Node */
 
 /**
  * @typedef {Object} NetworkNodeOutput
@@ -31,19 +36,19 @@ const IGNORED_MIME_TYPES_REGEX = /^video/;
 
 class PageDependencyGraph {
   /**
-   * @param {Lantern.NetworkRequest} record
+   * @param {Lantern.NetworkRequest} request
    * @return {Array<string>}
    */
-  static getNetworkInitiators(record) {
-    if (!record.initiator) return [];
-    if (record.initiator.url) return [record.initiator.url];
-    if (record.initiator.type === 'script') {
+  static getNetworkInitiators(request) {
+    if (!request.initiator) return [];
+    if (request.initiator.url) return [request.initiator.url];
+    if (request.initiator.type === 'script') {
       // Script initiators have the stack of callFrames from all functions that led to this request.
       // If async stacks are enabled, then the stack will also have the parent functions that asynchronously
       // led to this request chained in the `parent` property.
       /** @type {Set<string>} */
       const scriptURLs = new Set();
-      let stack = record.initiator.stack;
+      let stack = request.initiator.stack;
       while (stack) {
         const callFrames = stack.callFrames || [];
         for (const frame of callFrames) {
@@ -60,10 +65,10 @@ class PageDependencyGraph {
   }
 
   /**
-   * @param {Array<Lantern.NetworkRequest>} networkRecords
+   * @param {Array<Lantern.NetworkRequest>} networkRequests
    * @return {NetworkNodeOutput}
    */
-  static getNetworkNodeOutput(networkRecords) {
+  static getNetworkNodeOutput(networkRequests) {
     /** @type {Array<NetworkNode>} */
     const nodes = [];
     /** @type {Map<string, NetworkNode>} */
@@ -73,35 +78,35 @@ class PageDependencyGraph {
     /** @type {Map<string, NetworkNode|null>} */
     const frameIdToNodeMap = new Map();
 
-    networkRecords.forEach(record => {
-      if (IGNORED_MIME_TYPES_REGEX.test(record.mimeType)) return;
-      if (record.sessionTargetType === 'worker') return;
+    networkRequests.forEach(request => {
+      if (IGNORED_MIME_TYPES_REGEX.test(request.mimeType)) return;
+      if (request.fromWorker) return;
 
-      // Network record requestIds can be duplicated for an unknown reason
-      // Suffix all subsequent records with `:duplicate` until it's unique
+      // Network requestIds can be duplicated for an unknown reason
+      // Suffix all subsequent requests with `:duplicate` until it's unique
       // NOTE: This should never happen with modern NetworkRequest library, but old fixtures
       // might still have this issue.
-      while (idToNodeMap.has(record.requestId)) {
-        record.requestId += ':duplicate';
+      while (idToNodeMap.has(request.requestId)) {
+        request.requestId += ':duplicate';
       }
 
-      const node = new NetworkNode(record);
+      const node = new NetworkNode(request);
       nodes.push(node);
 
-      const urlList = urlToNodeMap.get(record.url) || [];
+      const urlList = urlToNodeMap.get(request.url) || [];
       urlList.push(node);
 
-      idToNodeMap.set(record.requestId, node);
-      urlToNodeMap.set(record.url, urlList);
+      idToNodeMap.set(request.requestId, node);
+      urlToNodeMap.set(request.url, urlList);
 
       // If the request was for the root document of an iframe, save an entry in our
       // map so we can link up the task `args.data.frame` dependencies later in graph creation.
-      if (record.frameId &&
-          record.resourceType === NetworkRequestTypes.Document &&
-          record.documentURL === record.url) {
+      if (request.frameId &&
+          request.resourceType === Lantern.NetworkRequestTypes.Document &&
+          request.documentURL === request.url) {
         // If there's ever any ambiguity, permanently set the value to `false` to avoid loops in the graph.
-        const value = frameIdToNodeMap.has(record.frameId) ? null : node;
-        frameIdToNodeMap.set(record.frameId, value);
+        const value = frameIdToNodeMap.has(request.frameId) ? null : node;
+        frameIdToNodeMap.set(request.frameId, value);
       }
     });
 
@@ -109,7 +114,31 @@ class PageDependencyGraph {
   }
 
   /**
-   * @param {LH.TraceEvent[]} mainThreadEvents
+   * @param {Lantern.TraceEvent} evt
+   * @return {boolean}
+   */
+  static isScheduleableTask(evt) {
+    return evt.name === SCHEDULABLE_TASK_TITLE_LH ||
+      evt.name === SCHEDULABLE_TASK_TITLE_ALT1 ||
+      evt.name === SCHEDULABLE_TASK_TITLE_ALT2 ||
+      evt.name === SCHEDULABLE_TASK_TITLE_ALT3;
+  }
+
+  /**
+   * There should *always* be at least one top level event, having 0 typically means something is
+   * drastically wrong with the trace and we should just give up early and loudly.
+   *
+   * @param {Lantern.TraceEvent[]} events
+   */
+  static assertHasToplevelEvents(events) {
+    const hasToplevelTask = events.some(this.isScheduleableTask);
+    if (!hasToplevelTask) {
+      throw new Error('Could not find any top level events');
+    }
+  }
+
+  /**
+   * @param {Lantern.TraceEvent[]} mainThreadEvents
    * @return {Array<CPUNode>}
    */
   static getCPUNodes(mainThreadEvents) {
@@ -117,14 +146,14 @@ class PageDependencyGraph {
     const nodes = [];
     let i = 0;
 
-    TraceProcessor.assertHasToplevelEvents(mainThreadEvents);
+    PageDependencyGraph.assertHasToplevelEvents(mainThreadEvents);
 
     while (i < mainThreadEvents.length) {
       const evt = mainThreadEvents[i];
       i++;
 
       // Skip all trace events that aren't schedulable tasks with sizable duration
-      if (!TraceProcessor.isScheduleableTask(evt) || !evt.dur) {
+      if (!PageDependencyGraph.isScheduleableTask(evt) || !evt.dur) {
         continue;
       }
 
@@ -132,23 +161,25 @@ class PageDependencyGraph {
       let correctedEndTs = undefined;
 
       // Capture all events that occurred within the task
-      /** @type {Array<LH.TraceEvent>} */
+      /** @type {Array<Lantern.TraceEvent>} */
       const children = [];
       for (
         const endTime = evt.ts + evt.dur;
         i < mainThreadEvents.length && mainThreadEvents[i].ts < endTime;
         i++
       ) {
+        const event = mainThreadEvents[i];
+
         // Temporary fix for a Chrome bug where some RunTask events can be overlapping.
         // We correct that here be ensuring each RunTask ends at least 1 microsecond before the next
         // https://github.com/GoogleChrome/lighthouse/issues/15896
         // https://issues.chromium.org/issues/329678173
-        if (TraceProcessor.isScheduleableTask(mainThreadEvents[i]) && mainThreadEvents[i].dur) {
-          correctedEndTs = mainThreadEvents[i].ts - 1;
+        if (PageDependencyGraph.isScheduleableTask(event) && event.dur) {
+          correctedEndTs = event.ts - 1;
           break;
         }
 
-        children.push(mainThreadEvents[i]);
+        children.push(event);
       }
 
       nodes.push(new CPUNode(evt, children, correctedEndTs));
@@ -210,9 +241,11 @@ class PageDependencyGraph {
    * @param {Array<CPUNode>} cpuNodes
    */
   static linkCPUNodes(rootNode, networkNodeOutput, cpuNodes) {
-    /** @type {Set<LH.Crdp.Network.ResourceType|undefined>} */
+    /** @type {Set<Lantern.ResourceType|undefined>} */
     const linkableResourceTypes = new Set([
-      NetworkRequestTypes.XHR, NetworkRequestTypes.Fetch, NetworkRequestTypes.Script,
+      Lantern.NetworkRequestTypes.XHR,
+      Lantern.NetworkRequestTypes.Fetch,
+      Lantern.NetworkRequestTypes.Script,
     ]);
 
     /** @param {CPUNode} cpuNode @param {string} reqId */
@@ -405,25 +438,128 @@ class PageDependencyGraph {
   }
 
   /**
-   * @param {LH.TraceEvent[]} mainThreadEvents
-   * @param {Array<Lantern.NetworkRequest>} networkRecords
-   * @param {URLArtifact} URL
+   * TODO(15841): remove when CDT backend is gone. until then, this is a useful debugging tool
+   * to find delta between using CDP or the trace to create the network requests.
+   *
+   * When a test fails using the trace backend, I enabled this debug method and copied the network
+   * requests when CDP was used, then when trace is used, and diff'd them. This method helped
+   * remove non-logical differences from the comparison (order of properties, slight rounding
+   * discrepancies, removing object cycles, etc).
+   *
+   * When using for a unit test, make sure to do `.only` so you are getting what you expect.
+   * @param {Lantern.NetworkRequest[]} lanternRequests
+   * @return {never}
+   */
+  static _debugNormalizeRequests(lanternRequests) {
+    for (const request of lanternRequests) {
+      request.rendererStartTime = Math.round(request.rendererStartTime * 1000) / 1000;
+      request.networkRequestTime = Math.round(request.networkRequestTime * 1000) / 1000;
+      request.responseHeadersEndTime = Math.round(request.responseHeadersEndTime * 1000) / 1000;
+      request.networkEndTime = Math.round(request.networkEndTime * 1000) / 1000;
+    }
+
+    for (const r of lanternRequests) {
+      delete r.rawRequest;
+      if (r.initiatorRequest) {
+        // @ts-expect-error
+        r.initiatorRequest = {id: r.initiatorRequest.requestId};
+      }
+      if (r.redirectDestination) {
+        // @ts-expect-error
+        r.redirectDestination = {id: r.redirectDestination.requestId};
+      }
+      if (r.redirectSource) {
+        // @ts-expect-error
+        r.redirectSource = {id: r.redirectSource.requestId};
+      }
+      if (r.redirects) {
+        // @ts-expect-error
+        r.redirects = r.redirects.map(r2 => r2.requestId);
+      }
+    }
+    /** @type {Lantern.NetworkRequest[]} */
+    const requests = lanternRequests.map(r => ({
+      requestId: r.requestId,
+      connectionId: r.connectionId,
+      connectionReused: r.connectionReused,
+      url: r.url,
+      protocol: r.protocol,
+      parsedURL: r.parsedURL,
+      documentURL: r.documentURL,
+      rendererStartTime: r.rendererStartTime,
+      networkRequestTime: r.networkRequestTime,
+      responseHeadersEndTime: r.responseHeadersEndTime,
+      networkEndTime: r.networkEndTime,
+      transferSize: r.transferSize,
+      resourceSize: r.resourceSize,
+      fromDiskCache: r.fromDiskCache,
+      fromMemoryCache: r.fromMemoryCache,
+      finished: r.finished,
+      statusCode: r.statusCode,
+      redirectSource: r.redirectSource,
+      redirectDestination: r.redirectDestination,
+      redirects: r.redirects,
+      failed: r.failed,
+      initiator: r.initiator,
+      timing: r.timing ? {
+        requestTime: r.timing.requestTime,
+        proxyStart: r.timing.proxyStart,
+        proxyEnd: r.timing.proxyEnd,
+        dnsStart: r.timing.dnsStart,
+        dnsEnd: r.timing.dnsEnd,
+        connectStart: r.timing.connectStart,
+        connectEnd: r.timing.connectEnd,
+        sslStart: r.timing.sslStart,
+        sslEnd: r.timing.sslEnd,
+        workerStart: r.timing.workerStart,
+        workerReady: r.timing.workerReady,
+        workerFetchStart: r.timing.workerFetchStart,
+        workerRespondWithSettled: r.timing.workerRespondWithSettled,
+        sendStart: r.timing.sendStart,
+        sendEnd: r.timing.sendEnd,
+        pushStart: r.timing.pushStart,
+        pushEnd: r.timing.pushEnd,
+        receiveHeadersStart: r.timing.receiveHeadersStart,
+        receiveHeadersEnd: r.timing.receiveHeadersEnd,
+      } : r.timing,
+      resourceType: r.resourceType,
+      mimeType: r.mimeType,
+      priority: r.priority,
+      initiatorRequest: r.initiatorRequest,
+      frameId: r.frameId,
+      fromWorker: r.fromWorker,
+      isLinkPreload: r.isLinkPreload,
+      serverResponseTime: r.serverResponseTime,
+    })).filter(r => !r.fromWorker);
+    // eslint-disable-next-line no-unused-vars
+    const debug = requests;
+    // Set breakpoint here.
+    // Copy `debug` and compare with https://www.diffchecker.com/text-compare/
+    process.exit(1);
+  }
+
+  /**
+   * @param {Lantern.TraceEvent[]} mainThreadEvents
+   * @param {Lantern.NetworkRequest[]} networkRequests
+   * @param {Lantern.Simulation.URL} URL
    * @return {Node}
    */
-  static createGraph(mainThreadEvents, networkRecords, URL) {
-    const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
+  static createGraph(mainThreadEvents, networkRequests, URL) {
+    // This is for debugging trace/devtoolslog network records.
+    // const debug = PageDependencyGraph._debugNormalizeRequests(networkRequests);
+    const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRequests);
     const cpuNodes = PageDependencyGraph.getCPUNodes(mainThreadEvents);
     const {requestedUrl, mainDocumentUrl} = URL;
     if (!requestedUrl) throw new Error('requestedUrl is required to get the root request');
     if (!mainDocumentUrl) throw new Error('mainDocumentUrl is required to get the main resource');
 
-    const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl);
+    const rootRequest =
+      Lantern.Simulation.NetworkAnalyzer.findResourceForUrl(networkRequests, requestedUrl);
     if (!rootRequest) throw new Error('rootRequest not found');
     const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
     if (!rootNode) throw new Error('rootNode not found');
-
     const mainDocumentRequest =
-      NetworkAnalyzer.findLastDocumentForUrl(networkRecords, mainDocumentUrl);
+      Lantern.Simulation.NetworkAnalyzer.findLastDocumentForUrl(networkRequests, mainDocumentUrl);
     if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found');
     const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId);
     if (!mainDocumentNode) throw new Error('mainDocumentNode not found');
@@ -440,7 +576,6 @@ class PageDependencyGraph {
   }
 
   /**
-   *
    * @param {Node} rootNode
    */
   static printGraph(rootNode, widthInCharacters = 100) {