lighthouse 11.3.0-dev.20231129 → 11.3.0-dev.20231201

package/core/audits/byte-efficiency/byte-efficiency-audit.d.ts

@@ -43,6 +43,19 @@ export class ByteEfficiencyAudit extends Audit {
      * @return {number}
      */
     static estimateTransferSize(networkRecord: LH.Artifacts.NetworkRequest | undefined, totalBytes: number, resourceType?: LH.Crdp.Network.ResourceType | undefined): number;
+    /**
+     * Estimates the number of bytes the content of this network record would have consumed on the network based on the
+     * uncompressed size (totalBytes). Uses the actual transfer size from the network record if applicable,
+     * minus the size of the response headers.
+     *
+     * This differs from `estimateTransferSize` only in that is subtracts the response headers from the estimate.
+     *
+     * @param {LH.Artifacts.NetworkRequest|undefined} networkRecord
+     * @param {number} totalBytes Uncompressed size of the resource
+     * @param {LH.Crdp.Network.ResourceType=} resourceType
+     * @return {number}
+     */
+    static estimateCompressedContentSize(networkRecord: LH.Artifacts.NetworkRequest | undefined, totalBytes: number, resourceType?: LH.Crdp.Network.ResourceType | undefined): number;
     /**
      * @param {LH.Artifacts} artifacts
      * @param {LH.Audit.Context} context
@@ -99,4 +112,5 @@ export class ByteEfficiencyAudit extends Audit {
     static audit_(artifacts: LH.Artifacts, networkRecords: Array<LH.Artifacts.NetworkRequest>, context: LH.Audit.Context): ByteEfficiencyProduct | Promise<ByteEfficiencyProduct>;
 }
 import { Audit } from '../audit.js';
+import { NetworkRequest } from '../../lib/network-request.js';
 //# sourceMappingURL=byte-efficiency-audit.d.ts.map

package/core/audits/byte-efficiency/byte-efficiency-audit.js

@@ -13,6 +13,7 @@ import {PageDependencyGraph} from '../../computed/page-dependency-graph.js';
 import {LanternLargestContentfulPaint} from '../../computed/metrics/lantern-largest-contentful-paint.js';
 import {LanternFirstContentfulPaint} from '../../computed/metrics/lantern-first-contentful-paint.js';
 import {LCPImageRecord} from '../../computed/lcp-image-record.js';
+import {NetworkRequest} from '../../lib/network-request.js';
 
 const str_ = i18n.createIcuMessageFn(import.meta.url, {});
 
@@ -100,6 +101,65 @@ class ByteEfficiencyAudit extends Audit {
     }
   }
 
+  /**
+   * Estimates the number of bytes the content of this network record would have consumed on the network based on the
+   * uncompressed size (totalBytes). Uses the actual transfer size from the network record if applicable,
+   * minus the size of the response headers.
+   *
+   * This differs from `estimateTransferSize` only in that is subtracts the response headers from the estimate.
+   *
+   * @param {LH.Artifacts.NetworkRequest|undefined} networkRecord
+   * @param {number} totalBytes Uncompressed size of the resource
+   * @param {LH.Crdp.Network.ResourceType=} resourceType
+   * @return {number}
+   */
+  static estimateCompressedContentSize(networkRecord, totalBytes, resourceType) {
+    if (!networkRecord) {
+      // We don't know how many bytes this asset used on the network, but we can guess it was
+      // roughly the size of the content gzipped.
+      // See https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/optimize-encoding-and-transfer for specific CSS/Script examples
+      // See https://discuss.httparchive.org/t/file-size-and-compression-savings/145 for fallback multipliers
+      switch (resourceType) {
+        case 'Stylesheet':
+          // Stylesheets tend to compress extremely well.
+          return Math.round(totalBytes * 0.2);
+        case 'Script':
+        case 'Document':
+          // Scripts and HTML compress fairly well too.
+          return Math.round(totalBytes * 0.33);
+        default:
+          // Otherwise we'll just fallback to the average savings in HTTPArchive
+          return Math.round(totalBytes * 0.5);
+      }
+    }
+
+    // Get the size of the response body on the network.
+    let contentTransferSize = networkRecord.transferSize || 0;
+    if (!NetworkRequest.isContentEncoded(networkRecord)) {
+      // This is not encoded, so we can use resourceSize directly.
+      // This would be equivalent to transfer size minus headers transfer size, but transfer size
+      // may also include bytes for SSL connection etc.
+      contentTransferSize = networkRecord.resourceSize;
+    } else if (networkRecord.responseHeadersTransferSize) {
+      // Subtract the size of the encoded headers.
+      contentTransferSize =
+        Math.max(0, contentTransferSize - networkRecord.responseHeadersTransferSize);
+    }
+
+    if (networkRecord.resourceType === resourceType) {
+      // This was a regular standalone asset, just use the transfer size.
+      return contentTransferSize;
+    } else {
+      // This was an asset that was inlined in a different resource type (e.g. HTML document).
+      // Use the compression ratio of the resource to estimate the total transferred bytes.
+      const resourceSize = networkRecord.resourceSize || 0;
+      // Get the compression ratio, if it's an invalid number, assume no compression.
+      const compressionRatio = Number.isFinite(resourceSize) && resourceSize > 0 ?
+        (contentTransferSize / resourceSize) : 1;
+      return Math.round(totalBytes * compressionRatio);
+    }
+  }
+
   /**
    * @param {LH.Artifacts} artifacts
    * @param {LH.Audit.Context} context

package/core/audits/byte-efficiency/legacy-javascript.d.ts

@@ -61,16 +61,18 @@ declare class LegacyJavascript extends ByteEfficiencyAudit {
      */
     static estimateWastedBytes(matches: PatternMatchResult[]): number;
     /**
-     * Utility function to estimate transfer size and cache calculation.
+     * Utility function to estimate the ratio of the compression on the resource.
+     * This excludes the size of the response headers.
+     * Also caches the calculation.
      *
      * Note: duplicated-javascript does this exact thing. In the future, consider
-     * making a generic estimator on ByteEfficienyAudit.
-     * @param {Map<string, number>} transferRatioByUrl
+     * making a generic estimator on ByteEfficiencyAudit.
+     * @param {Map<string, number>} compressionRatioByUrl
      * @param {string} url
      * @param {LH.Artifacts} artifacts
      * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
      */
-    static estimateTransferRatioForScript(transferRatioByUrl: Map<string, number>, url: string, artifacts: LH.Artifacts, networkRecords: Array<LH.Artifacts.NetworkRequest>): Promise<number>;
+    static estimateCompressionRatioForContent(compressionRatioByUrl: Map<string, number>, url: string, artifacts: LH.Artifacts, networkRecords: Array<LH.Artifacts.NetworkRequest>): Promise<number>;
     /**
      * @param {LH.Artifacts} artifacts
      * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords

package/core/audits/byte-efficiency/legacy-javascript.js

@@ -391,34 +391,37 @@ class LegacyJavascript extends ByteEfficiencyAudit {
   }
 
   /**
-   * Utility function to estimate transfer size and cache calculation.
+   * Utility function to estimate the ratio of the compression on the resource.
+   * This excludes the size of the response headers.
+   * Also caches the calculation.
    *
    * Note: duplicated-javascript does this exact thing. In the future, consider
-   * making a generic estimator on ByteEfficienyAudit.
-   * @param {Map<string, number>} transferRatioByUrl
+   * making a generic estimator on ByteEfficiencyAudit.
+   * @param {Map<string, number>} compressionRatioByUrl
    * @param {string} url
    * @param {LH.Artifacts} artifacts
    * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
    */
-  static async estimateTransferRatioForScript(transferRatioByUrl, url, artifacts, networkRecords) {
-    let transferRatio = transferRatioByUrl.get(url);
-    if (transferRatio !== undefined) return transferRatio;
+  static async estimateCompressionRatioForContent(compressionRatioByUrl, url,
+      artifacts, networkRecords) {
+    let compressionRatio = compressionRatioByUrl.get(url);
+    if (compressionRatio !== undefined) return compressionRatio;
 
     const script = artifacts.Scripts.find(script => script.url === url);
 
-    if (!script || script.content === null) {
+    if (!script) {
       // Can't find content, so just use 1.
-      transferRatio = 1;
+      compressionRatio = 1;
     } else {
       const networkRecord = getRequestForScript(networkRecords, script);
-      const contentLength = script.length || 0;
-      const transferSize =
-        ByteEfficiencyAudit.estimateTransferSize(networkRecord, contentLength, 'Script');
-      transferRatio = transferSize / contentLength;
+      const contentLength = networkRecord?.resourceSize || script.length || 0;
+      const compressedSize =
+        ByteEfficiencyAudit.estimateCompressedContentSize(networkRecord, contentLength, 'Script');
+      compressionRatio = compressedSize / contentLength;
     }
 
-    transferRatioByUrl.set(url, transferRatio);
-    return transferRatio;
+    compressionRatioByUrl.set(url, compressionRatio);
+    return compressionRatio;
   }
 
   /**
@@ -443,14 +446,14 @@ class LegacyJavascript extends ByteEfficiencyAudit {
     ]);
 
     /** @type {Map<string, number>} */
-    const transferRatioByUrl = new Map();
+    const compressionRatioByUrl = new Map();
 
     const scriptToMatchResults =
       this.detectAcrossScripts(matcher, artifacts.Scripts, networkRecords, bundles);
     for (const [script, matches] of scriptToMatchResults.entries()) {
-      const transferRatio = await this.estimateTransferRatioForScript(
-        transferRatioByUrl, script.url, artifacts, networkRecords);
-      const wastedBytes = Math.round(this.estimateWastedBytes(matches) * transferRatio);
+      const compressionRatio = await this.estimateCompressionRatioForContent(
+        compressionRatioByUrl, script.url, artifacts, networkRecords);
+      const wastedBytes = Math.round(this.estimateWastedBytes(matches) * compressionRatio);
       /** @type {typeof items[number]} */
       const item = {
         url: script.url,

package/core/computed/entity-classification.js

@@ -59,7 +59,7 @@ class EntityClassification {
     // Make up an entity only for valid http/https URLs.
     if (!parsedUrl.protocol.startsWith('http')) return;
 
-    const rootDomain = Util.getRootDomain(url);
+    const rootDomain = UrlUtils.getRootDomain(url);
     if (!rootDomain) return;
     if (entityCache.has(rootDomain)) return entityCache.get(rootDomain);
 

package/core/computed/resource-summary.d.ts

@@ -39,7 +39,6 @@ declare class ResourceSummary {
         budgets: LH.Util.ImmutableObject<LH.Budget[] | null>;
     }, context: LH.Artifacts.ComputedContext): Promise<Record<LH.Budget.ResourceType, ResourceEntry>>;
 }
-import { Util } from '../../shared/util.js';
 import { Budget } from '../config/budget.js';
 import { NetworkRequest } from '../lib/network-request.js';
 import { EntityClassification } from './entity-classification.js';

package/core/computed/resource-summary.js

@@ -9,7 +9,7 @@ import {makeComputedArtifact} from './computed-artifact.js';
 import {NetworkRecords} from './network-records.js';
 import {NetworkRequest} from '../lib/network-request.js';
 import {Budget} from '../config/budget.js';
-import {Util} from '../../shared/util.js';
+import UrlUtils from '../lib/url-utils.js';
 
 /** @typedef {{count: number, resourceSize: number, transferSize: number}} ResourceEntry */
 
@@ -59,7 +59,7 @@ class ResourceSummary {
       firstPartyHosts = budget.options.firstPartyHostnames;
     } else {
       firstPartyHosts = classifiedEntities.firstParty?.domains.map(domain => `*.${domain}`) ||
-        [`*.${Util.getRootDomain(URLArtifact.finalDisplayedUrl)}`];
+        [`*.${UrlUtils.getRootDomain(URLArtifact.finalDisplayedUrl)}`];
     }
 
     networkRecords.filter(record => {

package/core/gather/navigation-runner.js

@@ -220,7 +220,7 @@ async function _navigation(navigationContext) {
 
   // Every required url is initialized to an empty string in `getBaseArtifacts`.
   // If we haven't set all the required urls yet, set them here.
-  if (!Object.values(phaseState.baseArtifacts).every(Boolean)) {
+  if (!Object.values(phaseState.baseArtifacts.URL).every(Boolean)) {
     phaseState.baseArtifacts.URL = {
       requestedUrl: navigateResult.requestedUrl,
       mainDocumentUrl: navigateResult.mainDocumentUrl,

package/core/lib/network-recorder.d.ts

@@ -66,6 +66,14 @@ export class NetworkRecorder extends NetworkRecorder_base {
         targetType: LH.Protocol.TargetType;
         sessionId?: string;
     }): void;
+    /**
+     * @param {{params: LH.Crdp.Network.ResponseReceivedExtraInfoEvent, targetType: LH.Protocol.TargetType, sessionId?: string}} event
+     */
+    onResponseReceivedExtraInfo(event: {
+        params: LH.Crdp.Network.ResponseReceivedExtraInfoEvent;
+        targetType: LH.Protocol.TargetType;
+        sessionId?: string;
+    }): void;
     /**
      * @param {{params: LH.Crdp.Network.DataReceivedEvent, targetType: LH.Protocol.TargetType, sessionId?: string}} event
      */

package/core/lib/network-recorder.js

@@ -138,6 +138,18 @@ class NetworkRecorder extends RequestEventEmitter {
     request.onResponseReceived(data);
   }
 
+  /**
+   * @param {{params: LH.Crdp.Network.ResponseReceivedExtraInfoEvent, targetType: LH.Protocol.TargetType, sessionId?: string}} event
+   */
+  onResponseReceivedExtraInfo(event) {
+    const data = event.params;
+    const request = this._findRealRequestAndSetSession(
+      data.requestId, event.targetType, event.sessionId);
+    if (!request) return;
+    log.verbose('network', `${request.url} response received extra info`);
+    request.onResponseReceivedExtraInfo(data);
+  }
+
   /**
    * @param {{params: LH.Crdp.Network.DataReceivedEvent, targetType: LH.Protocol.TargetType, sessionId?: string}} event
    */
@@ -196,6 +208,7 @@ class NetworkRecorder extends RequestEventEmitter {
       case 'Network.requestWillBeSent': return this.onRequestWillBeSent(event);
       case 'Network.requestServedFromCache': return this.onRequestServedFromCache(event);
       case 'Network.responseReceived': return this.onResponseReceived(event);
+      case 'Network.responseReceivedExtraInfo': return this.onResponseReceivedExtraInfo(event);
       case 'Network.dataReceived': return this.onDataReceived(event);
       case 'Network.loadingFinished': return this.onLoadingFinished(event);
       case 'Network.loadingFailed': return this.onLoadingFailed(event);

package/core/lib/network-request.d.ts

@@ -66,6 +66,12 @@ export class NetworkRequest {
      * @return {boolean}
      */
     static isHstsRequest(record: NetworkRequest): boolean;
+    /**
+     * Returns whether the network request was sent encoded.
+     * @param {NetworkRequest} record
+     * @return {boolean}
+     */
+    static isContentEncoded(record: NetworkRequest): boolean;
     /**
      * Resource size is almost always the right one to be using because of the below:
      *     `transferSize = resourceSize + headers.length`.
@@ -98,6 +104,7 @@ export class NetworkRequest {
     /** When the last byte of the response body is received, in milliseconds. */
     networkEndTime: number;
     transferSize: number;
+    responseHeadersTransferSize: number;
     resourceSize: number;
     fromDiskCache: boolean;
     fromMemoryCache: boolean;
@@ -155,6 +162,10 @@ export class NetworkRequest {
      * @param {LH.Crdp.Network.ResponseReceivedEvent} data
      */
     onResponseReceived(data: LH.Crdp.Network.ResponseReceivedEvent): void;
+    /**
+     * @param {LH.Crdp.Network.ResponseReceivedExtraInfoEvent} data
+     */
+    onResponseReceivedExtraInfo(data: LH.Crdp.Network.ResponseReceivedExtraInfoEvent): void;
     /**
      * @param {LH.Crdp.Network.DataReceivedEvent} data
      */

package/core/lib/network-request.js

@@ -135,6 +135,7 @@ class NetworkRequest {
 
     // Go read the comment on _updateTransferSizeForLightrider.
     this.transferSize = 0;
+    this.responseHeadersTransferSize = 0;
     this.resourceSize = 0;
     this.fromDiskCache = false;
     this.fromMemoryCache = false;
@@ -246,6 +247,13 @@ class NetworkRequest {
     this.frameId = data.frameId;
   }
 
+  /**
+   * @param {LH.Crdp.Network.ResponseReceivedExtraInfoEvent} data
+   */
+  onResponseReceivedExtraInfo(data) {
+    this.responseHeadersText = data.headersText || '';
+  }
+
   /**
    * @param {LH.Crdp.Network.DataReceivedEvent} data
    */
@@ -344,6 +352,7 @@ class NetworkRequest {
     this.responseHeadersEndTime = timestamp * 1000;
 
     this.transferSize = response.encodedDataLength;
+    this.responseHeadersTransferSize = response.encodedDataLength;
     if (typeof response.fromDiskCache === 'boolean') this.fromDiskCache = response.fromDiskCache;
     if (typeof response.fromPrefetchCache === 'boolean') {
       this.fromPrefetchCache = response.fromPrefetchCache;
@@ -354,7 +363,6 @@ class NetworkRequest {
     this.timing = response.timing;
     if (resourceType) this.resourceType = RESOURCE_TYPES[resourceType];
     this.mimeType = response.mimeType;
-    this.responseHeadersText = response.headersText || '';
     this.responseHeaders = NetworkRequest._headersDictToHeadersArray(response.headers);
 
     this.fetchedViaServiceWorker = !!response.fromServiceWorker;
@@ -600,6 +608,15 @@ class NetworkRequest {
     return reason === 'HSTS' && NetworkRequest.isSecureRequest(destination);
   }
 
+  /**
+   * Returns whether the network request was sent encoded.
+   * @param {NetworkRequest} record
+   * @return {boolean}
+   */
+  static isContentEncoded(record) {
+    return record.responseHeaders.some(item => item.name === 'Content-Encoding');
+  }
+
   /**
    * Resource size is almost always the right one to be using because of the below:
    *     `transferSize = resourceSize + headers.length`.

package/core/lib/url-utils.d.ts

@@ -23,6 +23,12 @@ declare class UrlUtils {
      * @return {?string}
      */
     static getOrigin(url: string): string | null;
+    /**
+     * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+     * @param {string|URL} url hostname or URL object
+     * @return {string}
+     */
+    static getRootDomain(url: string | URL): string;
     /**
      * Check if rootDomains matches
      *

package/core/lib/url-utils.js

@@ -4,6 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import {getDomain} from 'tldts';
+
 import {Util} from '../../shared/util.js';
 import {LighthouseError} from './lh-error.js';
 
@@ -99,6 +101,16 @@ class UrlUtils {
     }
   }
 
+  /**
+   * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+   * @param {string|URL} url hostname or URL object
+   * @return {string}
+   */
+  static getRootDomain(url) {
+    const parsedUrl = Util.createOrReturnURL(url);
+    return getDomain(parsedUrl.href) || parsedUrl.hostname;
+  }
+
   /**
    * Check if rootDomains matches
    *
@@ -120,8 +132,8 @@ class UrlUtils {
     }
 
     // get the string before the tld
-    const urlARootDomain = Util.getRootDomain(urlAInfo);
-    const urlBRootDomain = Util.getRootDomain(urlBInfo);
+    const urlARootDomain = UrlUtils.getRootDomain(urlAInfo);
+    const urlBRootDomain = UrlUtils.getRootDomain(urlBInfo);
 
     return urlARootDomain === urlBRootDomain;
   }

package/core/scripts/download-chrome.sh

@@ -6,9 +6,22 @@
 ##
 
 # Download chrome inside of our CI env.
+# Takes one arg - the location to extract ToT chrome to. Defaults to .tmp/chrome-tot
+# If already exists, this script does nothing.
 
 set -euo pipefail
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+LH_ROOT_DIR="$SCRIPT_DIR/../.."
+
+chrome_out=${1:-"$LH_ROOT_DIR/.tmp/chrome-tot"}
+mkdir -p "$LH_ROOT_DIR/.tmp"
+
+if [ -e "$chrome_out" ]; then
+  echo "cached chrome found"
+  exit 0
+fi
+
 unameOut="$(uname -s)"
 case "${unameOut}" in
   Linux*)     machine=Linux;;
@@ -17,24 +30,49 @@ case "${unameOut}" in
   *)          machine="UNKNOWN:${unameOut}"
 esac
 
-if [ "$machine" == "MinGw" ]; then
-  url="https://download-chromium.appspot.com/dl/Win?type=snapshots"
-elif [ "$machine" == "Linux" ]; then
-  url="https://download-chromium.appspot.com/dl/Linux_x64?type=snapshots"
-elif [ "$machine" == "Mac" ]; then
-  arch="$(uname -m)"
-  if [ "$arch" == "arm64" ]; then
-    url="https://download-chromium.appspot.com/dl/Mac_Arm?type=snapshots"
+# Only set this to true when actual ToT is broken and we can't fix it yet.
+should_hardcode_ci=true
+
+if [[ "${CI:-}" ]] && [ "$should_hardcode_ci" == true ]; then
+  rev=1228630
+  if [ "$machine" == "MinGw" ]; then
+    url="http://commondatastorage.googleapis.com/chromium-browser-snapshots/Win_x64/$rev/chrome-win.zip"
+  elif [ "$machine" == "Linux" ]; then
+    url="http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux_x64/$rev/chrome-linux.zip"
+  elif [ "$machine" == "Mac" ]; then
+    arch="$(uname -m)"
+    if [ "$arch" == "arm64" ]; then
+      url="http://commondatastorage.googleapis.com/chromium-browser-snapshots/Mac_Arm/$rev/chrome-mac.zip"
+    else
+      url="http://commondatastorage.googleapis.com/chromium-browser-snapshots/Mac/$rev/chrome-mac.zip"
+    fi
   else
-    url="https://download-chromium.appspot.com/dl/Mac?type=snapshots"
+    echo "unsupported platform"
+    exit 1
   fi
 else
-  echo "unsupported platform"
-  exit 1
+  if [ "$machine" == "MinGw" ]; then
+    url="https://download-chromium.appspot.com/dl/Win?type=snapshots"
+  elif [ "$machine" == "Linux" ]; then
+    url="https://download-chromium.appspot.com/dl/Linux_x64?type=snapshots"
+  elif [ "$machine" == "Mac" ]; then
+    arch="$(uname -m)"
+    if [ "$arch" == "arm64" ]; then
+      url="https://download-chromium.appspot.com/dl/Mac_Arm?type=snapshots"
+    else
+      url="https://download-chromium.appspot.com/dl/Mac?type=snapshots"
+    fi
+  else
+    echo "unsupported platform"
+    exit 1
+  fi
 fi
 
-if [ -e "$CHROME_PATH" ]; then
-  echo "cached chrome found"
-else
-  curl "$url" -Lo chrome.zip && unzip -q chrome.zip
-fi
+echo "downloading $url"
+
+mkdir -p .tmp-download && cd .tmp-download
+curl "$url" -Lo chrome.zip && unzip -q chrome.zip && rm chrome.zip
+mv * "$chrome_out"
+cd - && rm -rf .tmp-download
+
+ls "$chrome_out"