lighthouse 11.4.0-dev.20231218 → 11.4.0-dev.20231220

package/cli/test/smokehouse/__snapshots__/report-assert-test.js.snap

@@ -3,7 +3,7 @@
 exports[`getAssertionReport works (multiple failing) 1`] = `
 "X difference at cumulative-layout-shift audit.details.items.length
               expected: []
-                 found: [{\\"cumulativeLayoutShiftMainFrame\\":0.13570762803819444}]
+                 found: [{\\"cumulativeLayoutShiftMainFrame\\":0.13570762803819444,\\"newEngineResult\\":{\\"cumulativeLayoutShift\\":0.13570762803819444,\\"cumulativeLayoutShiftMainFrame\\":0.13570762803819444},\\"newEngineResultDiffered\\":false}]
 
 
   X difference at cumulative-layout-shift audit.details.blah
@@ -28,7 +28,12 @@ exports[`getAssertionReport works (multiple failing) 1`] = `
           \\"type\\": \\"debugdata\\",
           \\"items\\": [
             {
-              \\"cumulativeLayoutShiftMainFrame\\": 0.13570762803819444
+              \\"cumulativeLayoutShiftMainFrame\\": 0.13570762803819444,
+              \\"newEngineResult\\": {
+                \\"cumulativeLayoutShift\\": 0.13570762803819444,
+                \\"cumulativeLayoutShiftMainFrame\\": 0.13570762803819444
+              },
+              \\"newEngineResultDiffered\\": false
             }
           ]
         }
@@ -38,7 +43,7 @@ exports[`getAssertionReport works (multiple failing) 1`] = `
 exports[`getAssertionReport works (trivial failing) 1`] = `
 "X difference at cumulative-layout-shift audit.details.items.length
               expected: []
-                 found: [{\\"cumulativeLayoutShiftMainFrame\\":0.13570762803819444}]
+                 found: [{\\"cumulativeLayoutShiftMainFrame\\":0.13570762803819444,\\"newEngineResult\\":{\\"cumulativeLayoutShift\\":0.13570762803819444,\\"cumulativeLayoutShiftMainFrame\\":0.13570762803819444},\\"newEngineResultDiffered\\":false}]
 
           found result:
       {
@@ -58,7 +63,12 @@ exports[`getAssertionReport works (trivial failing) 1`] = `
           \\"type\\": \\"debugdata\\",
           \\"items\\": [
             {
-              \\"cumulativeLayoutShiftMainFrame\\": 0.13570762803819444
+              \\"cumulativeLayoutShiftMainFrame\\": 0.13570762803819444,
+              \\"newEngineResult\\": {
+                \\"cumulativeLayoutShift\\": 0.13570762803819444,
+                \\"cumulativeLayoutShiftMainFrame\\": 0.13570762803819444
+              },
+              \\"newEngineResultDiffered\\": false
             }
           ]
         }

package/core/audits/byte-efficiency/render-blocking-resources.d.ts

@@ -6,15 +6,16 @@ declare class RenderBlockingResources extends Audit {
     /**
      * @param {LH.Artifacts} artifacts
      * @param {LH.Audit.Context} context
-     * @return {Promise<{wastedMs: number, results: Array<{url: string, totalBytes: number, wastedMs: number}>}>}
+     * @return {Promise<{fcpWastedMs: number, lcpWastedMs: number, results: Array<{url: string, totalBytes: number, wastedMs: number}>}>}
      */
     static computeResults(artifacts: LH.Artifacts, context: LH.Audit.Context): Promise<{
-        wastedMs: number;
-        results: {
+        fcpWastedMs: number;
+        lcpWastedMs: number;
+        results: Array<{
             url: string;
             totalBytes: number;
             wastedMs: number;
-        }[];
+        }>;
     }>;
     /**
      * Estimates how much faster this page would reach FCP if we inlined all the used CSS from the

package/core/audits/byte-efficiency/render-blocking-resources.js

@@ -17,6 +17,8 @@ import {NetworkRequest} from '../../lib/network-request.js';
 import {ProcessedNavigation} from '../../computed/processed-navigation.js';
 import {LoadSimulator} from '../../computed/load-simulator.js';
 import {FirstContentfulPaint} from '../../computed/metrics/first-contentful-paint.js';
+import {LCPImageRecord} from '../../computed/lcp-image-record.js';
+
 
 /** @typedef {import('../../lib/dependency-graph/simulator/simulator').Simulator} Simulator */
 /** @typedef {import('../../lib/dependency-graph/base-node.js').Node} Node */
@@ -125,7 +127,7 @@ class RenderBlockingResources extends Audit {
   /**
    * @param {LH.Artifacts} artifacts
    * @param {LH.Audit.Context} context
-   * @return {Promise<{wastedMs: number, results: Array<{url: string, totalBytes: number, wastedMs: number}>}>}
+   * @return {Promise<{fcpWastedMs: number, lcpWastedMs: number, results: Array<{url: string, totalBytes: number, wastedMs: number}>}>}
    */
   static async computeResults(artifacts, context) {
     const gatherContext = artifacts.GatherContext;
@@ -179,10 +181,10 @@ class RenderBlockingResources extends Audit {
     }
 
     if (!results.length) {
-      return {results, wastedMs: 0};
+      return {results, fcpWastedMs: 0, lcpWastedMs: 0};
     }
 
-    const wastedMs = RenderBlockingResources.estimateSavingsWithGraphs(
+    const fcpWastedMs = RenderBlockingResources.estimateSavingsWithGraphs(
       simulator,
       fcpSimulation.optimisticGraph,
       deferredNodeIds,
@@ -190,7 +192,10 @@ class RenderBlockingResources extends Audit {
       artifacts.Stacks
     );
 
-    return {results, wastedMs};
+    const lcpRecord = await LCPImageRecord.request(metricComputationData, context);
+
+    // In most cases if the LCP is an image, render blocking resources don't affect LCP. For these cases we should reduce its impact.
+    return {results, fcpWastedMs, lcpWastedMs: lcpRecord ? 0 : fcpWastedMs};
   }
 
   /**
@@ -276,11 +281,12 @@ class RenderBlockingResources extends Audit {
    * @return {Promise<LH.Audit.Product>}
    */
   static async audit(artifacts, context) {
-    const {results, wastedMs} = await RenderBlockingResources.computeResults(artifacts, context);
+    const {results, fcpWastedMs, lcpWastedMs} =
+      await RenderBlockingResources.computeResults(artifacts, context);
 
     let displayValue;
     if (results.length > 0) {
-      displayValue = str_(i18n.UIStrings.displayValueMsSavings, {wastedMs});
+      displayValue = str_(i18n.UIStrings.displayValueMsSavings, {wastedMs: fcpWastedMs});
     }
 
     /** @type {LH.Audit.Details.Opportunity['headings']} */
@@ -291,15 +297,15 @@ class RenderBlockingResources extends Audit {
     ];
 
     const details = Audit.makeOpportunityDetails(headings, results,
-      {overallSavingsMs: wastedMs});
+      {overallSavingsMs: fcpWastedMs});
 
     return {
       displayValue,
       score: results.length ? 0 : 1,
-      numericValue: wastedMs,
+      numericValue: fcpWastedMs,
       numericUnit: 'millisecond',
       details,
-      metricSavings: {FCP: wastedMs, LCP: wastedMs},
+      metricSavings: {FCP: fcpWastedMs, LCP: lcpWastedMs},
     };
   }
 }

package/core/computed/metrics/cumulative-layout-shift.d.ts

@@ -4,6 +4,7 @@ export type LayoutShiftEvent = {
     isMainFrame: boolean;
     weightedScore: number;
     impactedNodes?: LH.Artifacts.TraceImpactedNode[];
+    event: LH.TraceEvent;
 };
 declare const CumulativeLayoutShiftComputed: typeof CumulativeLayoutShift & {
     request: (dependencies: import("../../index.js").Trace, context: import("../../../types/utility-types.js").default.ImmutableObject<{
@@ -12,6 +13,11 @@ declare const CumulativeLayoutShiftComputed: typeof CumulativeLayoutShift & {
         cumulativeLayoutShift: number;
         cumulativeLayoutShiftMainFrame: number;
         impactByNodeId: Map<number, number>;
+        newEngineResult?: {
+            cumulativeLayoutShift: number;
+            cumulativeLayoutShiftMainFrame: number;
+        } | undefined;
+        newEngineResultDiffered: boolean;
     }>;
 };
 declare class CumulativeLayoutShift {
@@ -43,15 +49,28 @@ declare class CumulativeLayoutShift {
      * @return {number}
      */
     static calculate(layoutShiftEvents: Array<LayoutShiftEvent>): number;
+    /**
+     * @param {LayoutShiftEvent[]} allFrameShiftEvents
+     * @param {LayoutShiftEvent[]} mainFrameShiftEvents
+     */
+    static computeWithSharedTraceEngine(allFrameShiftEvents: LayoutShiftEvent[], mainFrameShiftEvents: LayoutShiftEvent[]): Promise<{
+        cumulativeLayoutShift: any;
+        cumulativeLayoutShiftMainFrame: any;
+    }>;
     /**
      * @param {LH.Trace} trace
      * @param {LH.Artifacts.ComputedContext} context
-     * @return {Promise<{cumulativeLayoutShift: number, cumulativeLayoutShiftMainFrame: number, impactByNodeId: Map<number, number>}>}
+     * @return {Promise<{cumulativeLayoutShift: number, cumulativeLayoutShiftMainFrame: number, impactByNodeId: Map<number, number>, newEngineResult?: {cumulativeLayoutShift: number, cumulativeLayoutShiftMainFrame: number}, newEngineResultDiffered: boolean}>}
      */
     static compute_(trace: LH.Trace, context: LH.Artifacts.ComputedContext): Promise<{
         cumulativeLayoutShift: number;
         cumulativeLayoutShiftMainFrame: number;
         impactByNodeId: Map<number, number>;
+        newEngineResult?: {
+            cumulativeLayoutShift: number;
+            cumulativeLayoutShiftMainFrame: number;
+        } | undefined;
+        newEngineResultDiffered: boolean;
     }>;
 }
 import { ProcessedTrace } from '../processed-trace.js';

package/core/computed/metrics/cumulative-layout-shift.js

@@ -7,8 +7,10 @@
 import {makeComputedArtifact} from '../computed-artifact.js';
 import {ProcessedTrace} from '../processed-trace.js';
 import * as RectHelpers from '../../lib/rect-helpers.js';
+import * as TraceEngine from '../../lib/trace-engine.js';
+import {Sentry} from '../../lib/sentry.js';
 
-/** @typedef {{ts: number, isMainFrame: boolean, weightedScore: number, impactedNodes?: LH.Artifacts.TraceImpactedNode[]}} LayoutShiftEvent */
+/** @typedef {{ts: number, isMainFrame: boolean, weightedScore: number, impactedNodes?: LH.Artifacts.TraceImpactedNode[], event: LH.TraceEvent}} LayoutShiftEvent */
 
 const RECENT_INPUT_WINDOW = 500;
 
@@ -67,6 +69,7 @@ class CumulativeLayoutShift {
         isMainFrame: event.args.data.is_main_frame,
         weightedScore: event.args.data.weighted_score_delta,
         impactedNodes: event.args.data.impacted_nodes,
+        event,
       });
     }
 
@@ -145,10 +148,29 @@ class CumulativeLayoutShift {
     return maxScore;
   }
 
+  /**
+   * @param {LayoutShiftEvent[]} allFrameShiftEvents
+   * @param {LayoutShiftEvent[]} mainFrameShiftEvents
+   */
+  static async computeWithSharedTraceEngine(allFrameShiftEvents, mainFrameShiftEvents) {
+    /** @param {LH.TraceEvent[]} events */
+    const run = async (events) => {
+      const processor = new TraceEngine.TraceProcessor({
+        LayoutShifts: TraceEngine.TraceHandlers.LayoutShifts,
+        Screenshots: TraceEngine.TraceHandlers.Screenshots,
+      });
+      await processor.parse(events);
+      return processor.data.LayoutShifts.sessionMaxScore;
+    };
+    const cumulativeLayoutShift = await run(allFrameShiftEvents.map(e => e.event));
+    const cumulativeLayoutShiftMainFrame = await run(mainFrameShiftEvents.map(e => e.event));
+    return {cumulativeLayoutShift, cumulativeLayoutShiftMainFrame};
+  }
+
   /**
    * @param {LH.Trace} trace
    * @param {LH.Artifacts.ComputedContext} context
-   * @return {Promise<{cumulativeLayoutShift: number, cumulativeLayoutShiftMainFrame: number, impactByNodeId: Map<number, number>}>}
+   * @return {Promise<{cumulativeLayoutShift: number, cumulativeLayoutShiftMainFrame: number, impactByNodeId: Map<number, number>, newEngineResult?: {cumulativeLayoutShift: number, cumulativeLayoutShiftMainFrame: number}, newEngineResultDiffered: boolean}>}
    */
   static async compute_(trace, context) {
     const processedTrace = await ProcessedTrace.request(trace, context);
@@ -157,11 +179,59 @@ class CumulativeLayoutShift {
         CumulativeLayoutShift.getLayoutShiftEvents(processedTrace);
     const impactByNodeId = CumulativeLayoutShift.getImpactByNodeId(allFrameShiftEvents);
     const mainFrameShiftEvents = allFrameShiftEvents.filter(e => e.isMainFrame);
+    const cumulativeLayoutShift = CumulativeLayoutShift.calculate(allFrameShiftEvents);
+    const cumulativeLayoutShiftMainFrame = CumulativeLayoutShift.calculate(mainFrameShiftEvents);
+
+    // Run with the new trace engine, and only throw an error if we are running our unit tests.
+    // Otherwise, simply report any differences or errors to Sentry.
+    // TODO: TraceEngine always drops `had_recent_input` events, but Lighthouse is more lenient.
+    //       See comment in `getLayoutShiftEvents`. We need to upstream this.
+    let newEngineResult;
+    let newEngineResultDiffered = false;
+    let tryNewTraceEngine = true;
+    if (allFrameShiftEvents.some(e => e.event.args.data?.had_recent_input)) {
+      tryNewTraceEngine = false;
+    }
+    if (tryNewTraceEngine) {
+      try {
+        newEngineResult =
+          await this.computeWithSharedTraceEngine(allFrameShiftEvents, mainFrameShiftEvents);
+        newEngineResultDiffered =
+          newEngineResult.cumulativeLayoutShift !== cumulativeLayoutShift ||
+          newEngineResult.cumulativeLayoutShiftMainFrame !== cumulativeLayoutShiftMainFrame;
+        if (newEngineResultDiffered) {
+          newEngineResultDiffered = true;
+          const expected = JSON.stringify({cumulativeLayoutShift, cumulativeLayoutShiftMainFrame});
+          const got = JSON.stringify(newEngineResult);
+          throw new Error(`new trace engine differed. expected: ${expected}, got: ${got}`);
+        }
+      } catch (err) {
+        console.error(err);
+        newEngineResultDiffered = true;
+
+        const error = new Error('Error when using new trace engine', {cause: err});
+        // @ts-expect-error Check for running from tests.
+        if (global.expect) {
+          throw error;
+        } else {
+          Sentry.captureException(error, {
+            tags: {computed: 'new-trace-engine'},
+            level: 'error',
+            extra: {
+              // Not sure if Sentry handles `cause`, so just in case add the info in a second place.
+              errorMsg: err.toString(),
+            },
+          });
+        }
+      }
+    }
 
     return {
-      cumulativeLayoutShift: CumulativeLayoutShift.calculate(allFrameShiftEvents),
-      cumulativeLayoutShiftMainFrame: CumulativeLayoutShift.calculate(mainFrameShiftEvents),
+      cumulativeLayoutShift,
+      cumulativeLayoutShiftMainFrame,
       impactByNodeId,
+      newEngineResult,
+      newEngineResultDiffered,
     };
   }
 }

package/core/gather/gatherers/dobetterweb/response-compression.js

@@ -23,12 +23,6 @@ import {fetchResponseBodyFromCache} from '../../driver/network.js';
 import {NetworkRecords} from '../../../computed/network-records.js';
 
 const CHROME_EXTENSION_PROTOCOL = 'chrome-extension:';
-const compressionHeaders = [
-  'content-encoding',
-  'x-original-content-encoding',
-  'x-content-encoding-over-network',
-];
-const compressionTypes = ['gzip', 'br', 'deflate'];
 const binaryMimeTypes = ['image', 'audio', 'video'];
 /** @type {LH.Crdp.Network.ResourceType[]} */
 const textResourceTypes = [
@@ -71,12 +65,7 @@ class ResponseCompression extends BaseGatherer {
         return;
       }
 
-      const isContentEncoded = (record.responseHeaders || []).find(header =>
-        compressionHeaders.includes(header.name.toLowerCase()) &&
-        compressionTypes.includes(header.value)
-      );
-
-      if (!isContentEncoded) {
+      if (!NetworkRequest.isContentEncoded(record)) {
         unoptimizedResponses.push({
           requestId: record.requestId,
           url: record.url,

package/core/gather/gatherers/trace.js

@@ -56,9 +56,9 @@ class Trace extends BaseGatherer {
       'disabled-by-default-devtools.timeline.frame',
       'latencyInfo',
 
-      // A bug introduced in M92 causes these categories to crash targets on Linux.
-      // See https://github.com/GoogleChrome/lighthouse/issues/12835 for full investigation.
-      // 'disabled-by-default-v8.cpu_profiler',
+      // Not used by Lighthouse (yet) but included for users that want JS samples when looking at
+      // a trace collected by Lighthouse (e.g. "View Trace" workflow in DevTools)
+      'disabled-by-default-v8.cpu_profiler',
     ];
   }
 

package/core/gather/navigation-runner.js

@@ -303,7 +303,7 @@ async function navigationGather(page, requestor, options = {}) {
         page,
         resolvedConfig,
         requestor: normalizedRequestor,
-        computedCache: new Map(),
+        computedCache,
       };
       const {baseArtifacts} = await _setup(context);
 

package/core/lib/network-request.js

@@ -616,8 +616,16 @@ class NetworkRequest {
   static isContentEncoded(record) {
     // FYI: older devtools logs (like our test fixtures) seems to be lower case, while modern logs
     // are Cased-Like-This.
-    const pattern = /^content-encoding$/i;
-    return record.responseHeaders.some(item => item.name.match(pattern));
+    const patterns = global.isLightrider ? [
+      /^x-original-content-encoding$/i,
+    ] : [
+      /^content-encoding$/i,
+      /^x-content-encoding-over-network$/i,
+    ];
+    const compressionTypes = ['gzip', 'br', 'deflate'];
+    return record.responseHeaders.some(header =>
+      patterns.some(p => header.name.match(p)) && compressionTypes.includes(header.value)
+    );
   }
 
   /**

package/core/lib/polyfill-dom-rect.d.ts

@@ -0,0 +1,2 @@
+export function polyfillDOMRect(): void;
+//# sourceMappingURL=polyfill-dom-rect.d.ts.map

package/core/lib/polyfill-dom-rect.js

@@ -0,0 +1,111 @@
+// @ts-nocheck
+/* eslint-disable */
+
+function polyfillDOMRect() {
+  // devtools assumes clientside :(
+
+  // Everything else in here is the DOMRect polyfill
+  // https://raw.githubusercontent.com/JakeChampion/polyfill-library/master/polyfills/DOMRect/polyfill.js
+
+  (function (global) {
+    function number(v) {
+      return v === undefined ? 0 : Number(v);
+    }
+
+    function different(u, v) {
+      return u !== v && !(isNaN(u) && isNaN(v));
+    }
+
+    function DOMRect(xArg, yArg, wArg, hArg) {
+      let x, y, width, height, left, right, top, bottom;
+
+      x = number(xArg);
+      y = number(yArg);
+      width = number(wArg);
+      height = number(hArg);
+
+      Object.defineProperties(this, {
+        x: {
+          get: function () { return x; },
+          set: function (newX) {
+            if (different(x, newX)) {
+              x = newX;
+              left = right = undefined;
+            }
+          },
+          enumerable: true
+        },
+        y: {
+          get: function () { return y; },
+          set: function (newY) {
+            if (different(y, newY)) {
+              y = newY;
+              top = bottom = undefined;
+            }
+          },
+          enumerable: true
+        },
+        width: {
+          get: function () { return width; },
+          set: function (newWidth) {
+            if (different(width, newWidth)) {
+              width = newWidth;
+              left = right = undefined;
+            }
+          },
+          enumerable: true
+        },
+        height: {
+          get: function () { return height; },
+          set: function (newHeight) {
+            if (different(height, newHeight)) {
+              height = newHeight;
+              top = bottom = undefined;
+            }
+          },
+          enumerable: true
+        },
+        left: {
+          get: function () {
+            if (left === undefined) {
+              left = x + Math.min(0, width);
+            }
+            return left;
+          },
+          enumerable: true
+        },
+        right: {
+          get: function () {
+            if (right === undefined) {
+              right = x + Math.max(0, width);
+            }
+            return right;
+          },
+          enumerable: true
+        },
+        top: {
+          get: function () {
+            if (top === undefined) {
+              top = y + Math.min(0, height);
+            }
+            return top;
+          },
+          enumerable: true
+        },
+        bottom: {
+          get: function () {
+            if (bottom === undefined) {
+              bottom = y + Math.max(0, height);
+            }
+            return bottom;
+          },
+          enumerable: true
+        }
+      });
+    }
+
+    globalThis.DOMRect = DOMRect;
+  })(globalThis);
+}
+
+export {polyfillDOMRect};

package/core/lib/trace-engine.d.ts

@@ -0,0 +1,4 @@
+export const TraceProcessor: any;
+export const TraceHandlers: any;
+export const RootCauses: any;
+//# sourceMappingURL=trace-engine.d.ts.map

package/core/lib/trace-engine.js

@@ -0,0 +1,16 @@
+// @ts-expect-error missing types
+import * as TraceEngine from '@paulirish/trace_engine';
+
+import {polyfillDOMRect} from './polyfill-dom-rect.js';
+
+polyfillDOMRect();
+
+const TraceProcessor = TraceEngine.Processor.TraceProcessor;
+const TraceHandlers = TraceEngine.Handlers.ModelHandlers;
+const RootCauses = TraceEngine.RootCauses.RootCauses.RootCauses;
+
+export {
+  TraceProcessor,
+  TraceHandlers,
+  RootCauses,
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lighthouse",
   "type": "module",
-  "version": "11.4.0-dev.20231218",
+  "version": "11.4.0-dev.20231220",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./core/index.js",
   "bin": {
@@ -180,6 +180,7 @@
     "webtreemap-cdt": "^3.2.1"
   },
   "dependencies": {
+    "@paulirish/trace_engine": "^0.0.7",
     "@sentry/node": "^6.17.4",
     "axe-core": "^4.8.1",
     "chrome-launcher": "^1.1.0",