lighthouse 12.0.0-dev.20240603 → 12.0.0-dev.20240604

package/core/lib/lantern/page-dependency-graph.js

@@ -32,19 +32,19 @@ const IGNORED_MIME_TYPES_REGEX = /^video/;
 
 class PageDependencyGraph {
   /**
-   * @param {Lantern.NetworkRequest} record
+   * @param {Lantern.NetworkRequest} request
    * @return {Array<string>}
    */
-  static getNetworkInitiators(record) {
-    if (!record.initiator) return [];
-    if (record.initiator.url) return [record.initiator.url];
-    if (record.initiator.type === 'script') {
+  static getNetworkInitiators(request) {
+    if (!request.initiator) return [];
+    if (request.initiator.url) return [request.initiator.url];
+    if (request.initiator.type === 'script') {
       // Script initiators have the stack of callFrames from all functions that led to this request.
       // If async stacks are enabled, then the stack will also have the parent functions that asynchronously
       // led to this request chained in the `parent` property.
       /** @type {Set<string>} */
       const scriptURLs = new Set();
-      let stack = record.initiator.stack;
+      let stack = request.initiator.stack;
       while (stack) {
         const callFrames = stack.callFrames || [];
         for (const frame of callFrames) {
@@ -61,10 +61,10 @@ class PageDependencyGraph {
   }
 
   /**
-   * @param {Array<Lantern.NetworkRequest>} networkRecords
+   * @param {Array<Lantern.NetworkRequest>} networkRequests
    * @return {NetworkNodeOutput}
    */
-  static getNetworkNodeOutput(networkRecords) {
+  static getNetworkNodeOutput(networkRequests) {
     /** @type {Array<NetworkNode>} */
     const nodes = [];
     /** @type {Map<string, NetworkNode>} */
@@ -74,35 +74,35 @@ class PageDependencyGraph {
     /** @type {Map<string, NetworkNode|null>} */
     const frameIdToNodeMap = new Map();
 
-    networkRecords.forEach(record => {
-      if (IGNORED_MIME_TYPES_REGEX.test(record.mimeType)) return;
-      if (record.fromWorker) return;
+    networkRequests.forEach(request => {
+      if (IGNORED_MIME_TYPES_REGEX.test(request.mimeType)) return;
+      if (request.fromWorker) return;
 
-      // Network record requestIds can be duplicated for an unknown reason
-      // Suffix all subsequent records with `:duplicate` until it's unique
+      // Network requestIds can be duplicated for an unknown reason
+      // Suffix all subsequent requests with `:duplicate` until it's unique
       // NOTE: This should never happen with modern NetworkRequest library, but old fixtures
       // might still have this issue.
-      while (idToNodeMap.has(record.requestId)) {
-        record.requestId += ':duplicate';
+      while (idToNodeMap.has(request.requestId)) {
+        request.requestId += ':duplicate';
       }
 
-      const node = new NetworkNode(record);
+      const node = new NetworkNode(request);
       nodes.push(node);
 
-      const urlList = urlToNodeMap.get(record.url) || [];
+      const urlList = urlToNodeMap.get(request.url) || [];
       urlList.push(node);
 
-      idToNodeMap.set(record.requestId, node);
-      urlToNodeMap.set(record.url, urlList);
+      idToNodeMap.set(request.requestId, node);
+      urlToNodeMap.set(request.url, urlList);
 
       // If the request was for the root document of an iframe, save an entry in our
       // map so we can link up the task `args.data.frame` dependencies later in graph creation.
-      if (record.frameId &&
-          record.resourceType === NetworkRequestTypes.Document &&
-          record.documentURL === record.url) {
+      if (request.frameId &&
+          request.resourceType === NetworkRequestTypes.Document &&
+          request.documentURL === request.url) {
         // If there's ever any ambiguity, permanently set the value to `false` to avoid loops in the graph.
-        const value = frameIdToNodeMap.has(record.frameId) ? null : node;
-        frameIdToNodeMap.set(record.frameId, value);
+        const value = frameIdToNodeMap.has(request.frameId) ? null : node;
+        frameIdToNodeMap.set(request.frameId, value);
       }
     });
 
@@ -427,7 +427,7 @@ class PageDependencyGraph {
     }
 
     for (const r of lanternRequests) {
-      delete r.record;
+      delete r.rawRequest;
       if (r.initiatorRequest) {
         // @ts-expect-error
         r.initiatorRequest = {id: r.initiatorRequest.requestId};
@@ -508,25 +508,25 @@ class PageDependencyGraph {
 
   /**
    * @param {LH.TraceEvent[]} mainThreadEvents
-   * @param {Array<Lantern.NetworkRequest>} networkRecords
+   * @param {Lantern.NetworkRequest[]} networkRequests
    * @param {URLArtifact} URL
    * @return {Node}
    */
-  static createGraph(mainThreadEvents, networkRecords, URL) {
+  static createGraph(mainThreadEvents, networkRequests, URL) {
     // This is for debugging trace/devtoolslog network records.
-    // const debug = PageDependencyGraph._debugNormalizeRequests(networkRecords);
-    const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
+    // const debug = PageDependencyGraph._debugNormalizeRequests(networkRequests);
+    const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRequests);
     const cpuNodes = PageDependencyGraph.getCPUNodes(mainThreadEvents);
     const {requestedUrl, mainDocumentUrl} = URL;
     if (!requestedUrl) throw new Error('requestedUrl is required to get the root request');
     if (!mainDocumentUrl) throw new Error('mainDocumentUrl is required to get the main resource');
 
-    const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl);
+    const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRequests, requestedUrl);
     if (!rootRequest) throw new Error('rootRequest not found');
     const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
     if (!rootNode) throw new Error('rootNode not found');
     const mainDocumentRequest =
-      NetworkAnalyzer.findLastDocumentForUrl(networkRecords, mainDocumentUrl);
+      NetworkAnalyzer.findLastDocumentForUrl(networkRequests, mainDocumentUrl);
     if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found');
     const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId);
     if (!mainDocumentNode) throw new Error('mainDocumentNode not found');
@@ -543,20 +543,20 @@ class PageDependencyGraph {
   }
 
   /**
-   * @param {Lantern.NetworkRequest} record The record to find the initiator of
-   * @param {Map<string, Lantern.NetworkRequest[]>} recordsByURL
+   * @param {Lantern.NetworkRequest} request The request to find the initiator of
+   * @param {Map<string, Lantern.NetworkRequest[]>} requestsByURL
    * @return {Lantern.NetworkRequest|null}
    */
-  static chooseInitiatorRequest(record, recordsByURL) {
-    if (record.redirectSource) {
-      return record.redirectSource;
+  static chooseInitiatorRequest(request, requestsByURL) {
+    if (request.redirectSource) {
+      return request.redirectSource;
     }
 
-    const initiatorURL = PageDependencyGraph.getNetworkInitiators(record)[0];
-    let candidates = recordsByURL.get(initiatorURL) || [];
+    const initiatorURL = PageDependencyGraph.getNetworkInitiators(request)[0];
+    let candidates = requestsByURL.get(initiatorURL) || [];
     // The (valid) initiator must come before the initiated request.
     candidates = candidates.filter(c => {
-      return c.responseHeadersEndTime <= record.rendererStartTime &&
+      return c.responseHeadersEndTime <= request.rendererStartTime &&
           c.finished && !c.failed;
     });
     if (candidates.length > 1) {
@@ -570,12 +570,12 @@ class PageDependencyGraph {
     }
     if (candidates.length > 1) {
       // Disambiguate based on frame. It's likely that the initiator comes from the same frame.
-      const sameFrameCandidates = candidates.filter(cand => cand.frameId === record.frameId);
+      const sameFrameCandidates = candidates.filter(cand => cand.frameId === request.frameId);
       if (sameFrameCandidates.length) {
         candidates = sameFrameCandidates;
       }
     }
-    if (candidates.length > 1 && record.initiator.type === 'parser') {
+    if (candidates.length > 1 && request.initiator.type === 'parser') {
       // Filter to just Documents when initiator type is parser.
       const documentCandidates = candidates.filter(cand =>
         cand.resourceType === RESOURCE_TYPES.Document);
@@ -712,9 +712,23 @@ class PageDependencyGraph {
     if (request.args.data.initiator?.fetchType === 'xmlhttprequest') {
       // @ts-expect-error yes XHR is a valid ResourceType. TypeScript const enums are so unhelpful.
       resourceType = 'XHR';
+    } else if (request.args.data.initiator?.fetchType === 'fetch') {
+      // @ts-expect-error yes Fetch is a valid ResourceType. TypeScript const enums are so unhelpful.
+      resourceType = 'Fetch';
+    }
+
+    // TODO: set decodedBodyLength for data urls in Trace Engine.
+    let resourceSize = request.args.data.decodedBodyLength ?? 0;
+    if (url.protocol === 'data:' && resourceSize === 0) {
+      const needle = 'base64,';
+      const index = url.pathname.indexOf(needle);
+      if (index !== -1) {
+        resourceSize = atob(url.pathname.substring(index + needle.length)).length;
+      }
     }
 
     return {
+      rawRequest: request,
       requestId: request.args.data.requestId,
       connectionId: request.args.data.connectionId,
       connectionReused: request.args.data.connectionReused,
@@ -727,7 +741,7 @@ class PageDependencyGraph {
       responseHeadersEndTime: request.args.data.syntheticData.downloadStart / 1000,
       networkEndTime: request.args.data.syntheticData.finishTime / 1000,
       transferSize: request.args.data.encodedDataLength,
-      resourceSize: request.args.data.decodedBodyLength,
+      resourceSize,
       fromDiskCache: request.args.data.syntheticData.isDiskCached,
       fromMemoryCache: request.args.data.syntheticData.isMemoryCached,
       isLinkPreload: request.args.data.isLinkPreload,
@@ -741,7 +755,6 @@ class PageDependencyGraph {
       priority: request.args.data.priority,
       frameId: request.args.data.frame,
       fromWorker,
-      record: request,
       // Set later.
       redirects: undefined,
       redirectSource: undefined,
@@ -792,31 +805,50 @@ class PageDependencyGraph {
     // TraceEngine consolidates all redirects into a single request object, but lantern needs
     // an entry for each redirected request.
     for (const request of [...lanternRequests]) {
-      if (!request.record) continue;
+      if (!request.rawRequest) continue;
 
-      const redirects = request.record.args.data.redirects;
+      const redirects = request.rawRequest.args.data.redirects;
       if (!redirects.length) continue;
 
       const requestChain = [];
       for (const redirect of redirects) {
         const redirectedRequest = structuredClone(request);
-        if (redirectedRequest.timing) {
-          // TODO: These are surely wrong for when there is more than one redirect. Would be
-          // simpler if each redirect remembered it's `timing` object in this `redirects` array.
-          redirectedRequest.timing.requestTime = redirect.ts / 1000 / 1000;
-          redirectedRequest.timing.receiveHeadersStart -= redirect.dur / 1000 / 1000;
-          redirectedRequest.timing.receiveHeadersEnd -= redirect.dur / 1000 / 1000;
-          redirectedRequest.rendererStartTime = redirect.ts / 1000;
-          redirectedRequest.networkRequestTime = redirect.ts / 1000;
-          redirectedRequest.networkEndTime = (redirect.ts + redirect.dur) / 1000;
-          redirectedRequest.responseHeadersEndTime =
-            redirectedRequest.timing.requestTime * 1000 +
-            redirectedRequest.timing.receiveHeadersEnd;
-        }
+
+        redirectedRequest.networkRequestTime = redirect.ts / 1000;
+        redirectedRequest.rendererStartTime = redirectedRequest.networkRequestTime;
+
+        redirectedRequest.networkEndTime = (redirect.ts + redirect.dur) / 1000;
+        redirectedRequest.responseHeadersEndTime = redirectedRequest.networkEndTime;
+
+        redirectedRequest.timing = {
+          requestTime: redirectedRequest.networkRequestTime / 1000,
+          receiveHeadersStart: redirectedRequest.responseHeadersEndTime,
+          receiveHeadersEnd: redirectedRequest.responseHeadersEndTime,
+          proxyStart: -1,
+          proxyEnd: -1,
+          dnsStart: -1,
+          dnsEnd: -1,
+          connectStart: -1,
+          connectEnd: -1,
+          sslStart: -1,
+          sslEnd: -1,
+          sendStart: -1,
+          sendEnd: -1,
+          workerStart: -1,
+          workerReady: -1,
+          workerFetchStart: -1,
+          workerRespondWithSettled: -1,
+          pushStart: -1,
+          pushEnd: -1,
+        };
+
         redirectedRequest.url = redirect.url;
         redirectedRequest.parsedURL = this._createParsedUrl(redirect.url);
         // TODO: TraceEngine is not retaining the actual status code.
         redirectedRequest.statusCode = 302;
+        redirectedRequest.resourceType = undefined;
+        // TODO: TraceEngine is not retaining transfer size of redirected request.
+        redirectedRequest.transferSize = 400;
         requestChain.push(redirectedRequest);
         lanternRequests.push(redirectedRequest);
       }

package/core/lib/lantern/simulator/connection-pool.d.ts

@@ -9,7 +9,7 @@ export class ConnectionPool {
     /** @type {Map<string, TcpConnection[]>} */
     _connectionsByOrigin: Map<string, TcpConnection[]>;
     /** @type {Map<Lantern.NetworkRequest, TcpConnection>} */
-    _connectionsByRecord: Map<Lantern.NetworkRequest, TcpConnection>;
+    _connectionsByRequest: Map<Lantern.NetworkRequest, TcpConnection>;
     _connectionsInUse: Set<any>;
     _connectionReusedByRequestId: Map<string, boolean>;
     /**
@@ -22,26 +22,26 @@ export class ConnectionPool {
      */
     _findAvailableConnectionWithLargestCongestionWindow(connections: Array<TcpConnection>): TcpConnection | null;
     /**
-     * This method finds an available connection to the origin specified by the network record or null
+     * This method finds an available connection to the origin specified by the network request or null
      * if no connection was available. If returned, connection will not be available for other network
      * records until release is called.
      *
-     * @param {Lantern.NetworkRequest} record
+     * @param {Lantern.NetworkRequest} request
      * @return {?TcpConnection}
      */
-    acquire(record: Lantern.NetworkRequest): TcpConnection | null;
+    acquire(request: Lantern.NetworkRequest): TcpConnection | null;
     /**
-     * Return the connection currently being used to fetch a record. If no connection
-     * currently being used for this record, an error will be thrown.
+     * Return the connection currently being used to fetch a request. If no connection
+     * currently being used for this request, an error will be thrown.
      *
-     * @param {Lantern.NetworkRequest} record
+     * @param {Lantern.NetworkRequest} request
      * @return {TcpConnection}
      */
-    acquireActiveConnectionFromRecord(record: Lantern.NetworkRequest): TcpConnection;
+    acquireActiveConnectionFromRequest(request: Lantern.NetworkRequest): TcpConnection;
     /**
-     * @param {Lantern.NetworkRequest} record
+     * @param {Lantern.NetworkRequest} request
      */
-    release(record: Lantern.NetworkRequest): void;
+    release(request: Lantern.NetworkRequest): void;
 }
 import * as Lantern from '../types/lantern.js';
 import { TcpConnection } from './tcp-connection.js';

package/core/lib/lantern/simulator/connection-pool.js

@@ -27,7 +27,7 @@ export class ConnectionPool {
     /** @type {Map<string, TcpConnection[]>} */
     this._connectionsByOrigin = new Map();
     /** @type {Map<Lantern.NetworkRequest, TcpConnection>} */
-    this._connectionsByRecord = new Map();
+    this._connectionsByRequest = new Map();
     this._connectionsInUse = new Set();
     this._connectionReusedByRequestId = NetworkAnalyzer.estimateIfConnectionWasReused(records, {
       forceCoarseEstimates: true,
@@ -49,16 +49,16 @@ export class ConnectionPool {
     const serverResponseTimeByOrigin = this._options.serverResponseTimeByOrigin;
 
     const recordsByOrigin = NetworkAnalyzer.groupByOrigin(this._records);
-    for (const [origin, records] of recordsByOrigin.entries()) {
+    for (const [origin, requests] of recordsByOrigin.entries()) {
       const connections = [];
       const additionalRtt = additionalRttByOrigin.get(origin) || 0;
       const responseTime = serverResponseTimeByOrigin.get(origin) || DEFAULT_SERVER_RESPONSE_TIME;
 
-      for (const record of records) {
-        if (connectionReused.get(record.requestId)) continue;
+      for (const request of requests) {
+        if (connectionReused.get(request.requestId)) continue;
 
-        const isTLS = TLS_SCHEMES.includes(record.parsedURL.scheme);
-        const isH2 = record.protocol === 'h2';
+        const isTLS = TLS_SCHEMES.includes(request.parsedURL.scheme);
+        const isH2 = request.protocol === 'h2';
         const connection = new TcpConnection(
           this._options.rtt + additionalRtt,
           this._options.throughput,
@@ -106,47 +106,47 @@ export class ConnectionPool {
   }
 
   /**
-   * This method finds an available connection to the origin specified by the network record or null
+   * This method finds an available connection to the origin specified by the network request or null
    * if no connection was available. If returned, connection will not be available for other network
    * records until release is called.
    *
-   * @param {Lantern.NetworkRequest} record
+   * @param {Lantern.NetworkRequest} request
    * @return {?TcpConnection}
    */
-  acquire(record) {
-    if (this._connectionsByRecord.has(record)) throw new Error('Record already has a connection');
+  acquire(request) {
+    if (this._connectionsByRequest.has(request)) throw new Error('Record already has a connection');
 
-    const origin = record.parsedURL.securityOrigin;
+    const origin = request.parsedURL.securityOrigin;
     const connections = this._connectionsByOrigin.get(origin) || [];
     const connectionToUse = this._findAvailableConnectionWithLargestCongestionWindow(connections);
 
     if (!connectionToUse) return null;
 
     this._connectionsInUse.add(connectionToUse);
-    this._connectionsByRecord.set(record, connectionToUse);
+    this._connectionsByRequest.set(request, connectionToUse);
     return connectionToUse;
   }
 
   /**
-   * Return the connection currently being used to fetch a record. If no connection
-   * currently being used for this record, an error will be thrown.
+   * Return the connection currently being used to fetch a request. If no connection
+   * currently being used for this request, an error will be thrown.
    *
-   * @param {Lantern.NetworkRequest} record
+   * @param {Lantern.NetworkRequest} request
    * @return {TcpConnection}
    */
-  acquireActiveConnectionFromRecord(record) {
-    const activeConnection = this._connectionsByRecord.get(record);
-    if (!activeConnection) throw new Error('Could not find an active connection for record');
+  acquireActiveConnectionFromRequest(request) {
+    const activeConnection = this._connectionsByRequest.get(request);
+    if (!activeConnection) throw new Error('Could not find an active connection for request');
 
     return activeConnection;
   }
 
   /**
-   * @param {Lantern.NetworkRequest} record
+   * @param {Lantern.NetworkRequest} request
    */
-  release(record) {
-    const connection = this._connectionsByRecord.get(record);
-    this._connectionsByRecord.delete(record);
+  release(request) {
+    const connection = this._connectionsByRequest.get(request);
+    this._connectionsByRequest.delete(request);
     this._connectionsInUse.delete(connection);
   }
 }

package/core/lib/lantern/simulator/dns-cache.d.ts

@@ -25,7 +25,7 @@ export class DNSCache {
      */
     _updateCacheResolvedAtIfNeeded(request: Lantern.NetworkRequest, resolvedAt: number): void;
     /**
-     * Forcefully sets the DNS resolution time for a record.
+     * Forcefully sets the DNS resolution time for a request.
      * Useful for testing and alternate execution simulations.
      *
      * @param {string} domain

package/core/lib/lantern/simulator/dns-cache.js

@@ -58,7 +58,7 @@ class DNSCache {
   }
 
   /**
-   * Forcefully sets the DNS resolution time for a record.
+   * Forcefully sets the DNS resolution time for a request.
    * Useful for testing and alternate execution simulations.
    *
    * @param {string} domain

package/core/lib/lantern/simulator/network-analyzer.d.ts

@@ -46,14 +46,14 @@ export class NetworkAnalyzer {
      * @return {Map<string, Summary>}
      */
     static summarize(values: Map<string, number[]>): Map<string, Summary>;
-    /** @typedef {{record: Lantern.NetworkRequest, timing: LH.Crdp.Network.ResourceTiming, connectionReused?: boolean}} RequestInfo */
+    /** @typedef {{request: Lantern.NetworkRequest, timing: LH.Crdp.Network.ResourceTiming, connectionReused?: boolean}} RequestInfo */
     /**
-     * @param {Lantern.NetworkRequest[]} records
+     * @param {Lantern.NetworkRequest[]} requests
      * @param {(e: RequestInfo) => number | number[] | undefined} iteratee
      * @return {Map<string, number[]>}
      */
-    static _estimateValueByOrigin(records: Lantern.NetworkRequest[], iteratee: (e: {
-        record: Lantern.NetworkRequest;
+    static _estimateValueByOrigin(requests: Lantern.NetworkRequest[], iteratee: (e: {
+        request: Lantern.NetworkRequest;
         timing: LH.Crdp.Network.ResourceTiming;
         connectionReused?: boolean | undefined;
     }) => number | number[] | undefined): Map<string, number[]>;
@@ -69,7 +69,7 @@ export class NetworkAnalyzer {
      * @return {number[]|number|undefined}
      */
     static _estimateRTTViaConnectionTiming(info: {
-        record: Lantern.NetworkRequest;
+        request: Lantern.NetworkRequest;
         timing: LH.Crdp.Network.ResourceTiming;
         connectionReused?: boolean | undefined;
     }): number[] | number | undefined;
@@ -82,7 +82,7 @@ export class NetworkAnalyzer {
      * @return {number|undefined}
      */
     static _estimateRTTViaDownloadTiming(info: {
-        record: Lantern.NetworkRequest;
+        request: Lantern.NetworkRequest;
         timing: LH.Crdp.Network.ResourceTiming;
         connectionReused?: boolean | undefined;
     }): number | undefined;
@@ -96,7 +96,7 @@ export class NetworkAnalyzer {
      * @return {number|undefined}
      */
     static _estimateRTTViaSendStartTiming(info: {
-        record: Lantern.NetworkRequest;
+        request: Lantern.NetworkRequest;
         timing: LH.Crdp.Network.ResourceTiming;
         connectionReused?: boolean | undefined;
     }): number | undefined;
@@ -110,7 +110,7 @@ export class NetworkAnalyzer {
      * @return {number|undefined}
      */
     static _estimateRTTViaHeadersEndTiming(info: {
-        record: Lantern.NetworkRequest;
+        request: Lantern.NetworkRequest;
         timing: LH.Crdp.Network.ResourceTiming;
         connectionReused?: boolean | undefined;
     }): number | undefined;
@@ -123,10 +123,10 @@ export class NetworkAnalyzer {
      */
     static _estimateResponseTimeByOrigin(records: Lantern.NetworkRequest[], rttByOrigin: Map<string, number>): Map<string, number[]>;
     /**
-     * @param {Lantern.NetworkRequest[]} records
+     * @param {Lantern.NetworkRequest[]} requests
      * @return {boolean}
      */
-    static canTrustConnectionInformation(records: Lantern.NetworkRequest[]): boolean;
+    static canTrustConnectionInformation(requests: Lantern.NetworkRequest[]): boolean;
     /**
      * Returns a map of requestId -> connectionReused, estimating the information if the information
      * available in the records themselves appears untrustworthy.
@@ -160,9 +160,9 @@ export class NetworkAnalyzer {
         rttByOrigin?: Map<string, number> | undefined;
     }) | undefined): Map<string, Summary>;
     /**
-     * Computes the average throughput for the given records in bits/second.
+     * Computes the average throughput for the given requests in bits/second.
      * Excludes data URI, failed or otherwise incomplete, and cached requests.
-     * Returns Infinity if there were no analyzable network records.
+     * Returns Infinity if there were no analyzable network requests.
      *
      * @param {Lantern.NetworkRequest[]} records
      * @return {number}