lighthouse 12.0.0 → 12.1.0-dev.20240619

package/core/lib/lantern-trace-saver.d.ts

@@ -3,15 +3,15 @@ declare namespace _default {
     export { convertNodeTimingsToTrace };
 }
 export default _default;
-export type Node = import('./lantern/base-node.js').Node<LH.Artifacts.NetworkRequest>;
-export type CompleteNodeTiming = import('./lantern/simulator/simulator.js').CompleteNodeTiming;
+export type Node = import('./lantern/BaseNode.js').Node<LH.Artifacts.NetworkRequest>;
+export type CompleteNodeTiming = import('./lantern/simulation/Simulator.js').CompleteNodeTiming;
 /**
  * @license
  * Copyright 2018 Google LLC
  * SPDX-License-Identifier: Apache-2.0
  */
-/** @typedef {import('./lantern/base-node.js').Node<LH.Artifacts.NetworkRequest>} Node */
-/** @typedef {import('./lantern/simulator/simulator.js').CompleteNodeTiming} CompleteNodeTiming */
+/** @typedef {import('./lantern/BaseNode.js').Node<LH.Artifacts.NetworkRequest>} Node */
+/** @typedef {import('./lantern/simulation/Simulator.js').CompleteNodeTiming} CompleteNodeTiming */
 /**
  * @param {Map<Node, CompleteNodeTiming>} nodeTimings
  * @return {LH.Trace}

package/core/lib/lantern-trace-saver.js

@@ -4,8 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-/** @typedef {import('./lantern/base-node.js').Node<LH.Artifacts.NetworkRequest>} Node */
-/** @typedef {import('./lantern/simulator/simulator.js').CompleteNodeTiming} CompleteNodeTiming */
+/** @typedef {import('./lantern/BaseNode.js').Node<LH.Artifacts.NetworkRequest>} Node */
+/** @typedef {import('./lantern/simulation/Simulator.js').CompleteNodeTiming} CompleteNodeTiming */
 
 /**
  * @param {Map<Node, CompleteNodeTiming>} nodeTimings
@@ -33,7 +33,7 @@ function convertNodeTimingsToTrace(nodeTimings) {
       traceEvents.push(...createFakeTaskEvents(node, timing));
     } else {
       /** @type {LH.Artifacts.NetworkRequest} */
-      const record = node.record;
+      const record = node.rawRequest;
       // Ignore data URIs as they don't really add much value
       if (/^data/.test(record.url)) continue;
       traceEvents.push(...createFakeNetworkEvents(requestId, record, timing));
@@ -116,7 +116,7 @@ function convertNodeTimingsToTrace(nodeTimings) {
       const ts = eventTs + (event.ts - nestedBaseTs) * multiplier;
       const newEvent = {...event, ...{pid: baseEvent.pid, tid: baseEvent.tid}, ts};
       if (event.dur) newEvent.dur = event.dur * multiplier;
-      events.push(newEvent);
+      events.push(/** @type {LH.TraceEvent} */(newEvent));
     }
 
     return events;

package/core/lib/navigation-error.js

@@ -4,8 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import * as Lantern from './lantern/lantern.js';
 import {LighthouseError} from './lh-error.js';
-import {NetworkAnalyzer} from './lantern/simulator/network-analyzer.js';
 import {NetworkRequest} from './network-request.js';
 import * as i18n from '../lib/i18n/i18n.js';
 
@@ -131,7 +131,7 @@ function getNonHtmlError(finalRecord) {
 function getPageLoadError(navigationError, context) {
   const {url, networkRecords} = context;
   /** @type {LH.Artifacts.NetworkRequest|undefined} */
-  let mainRecord = NetworkAnalyzer.findResourceForUrl(networkRecords, url);
+  let mainRecord = Lantern.Simulation.NetworkAnalyzer.findResourceForUrl(networkRecords, url);
 
   // If the url doesn't give us a network request, it's possible we landed on a chrome-error:// page
   // In this case, just get the first document request.
@@ -149,7 +149,7 @@ function getPageLoadError(navigationError, context) {
   // MIME Type is only set on the final redirected document request. Use this for the HTML check instead of root.
   let finalRecord;
   if (mainRecord) {
-    finalRecord = NetworkAnalyzer.resolveRedirects(mainRecord);
+    finalRecord = Lantern.Simulation.NetworkAnalyzer.resolveRedirects(mainRecord);
   } else {
     // We have no network requests to process, use the navError
     return navigationError;

package/core/lib/network-recorder.js

@@ -9,8 +9,8 @@ import {EventEmitter} from 'events';
 import log from 'lighthouse-logger';
 
 import * as LH from '../../types/lh.js';
+import * as Lantern from './lantern/lantern.js';
 import {NetworkRequest} from './network-request.js';
-import {PageDependencyGraph} from '../lib/lantern/page-dependency-graph.js';
 
 /**
  * @typedef {{
@@ -253,7 +253,7 @@ class NetworkRecorder extends RequestEventEmitter {
       return record.redirectSource;
     }
 
-    const initiatorURL = PageDependencyGraph.getNetworkInitiators(record)[0];
+    const initiatorURL = Lantern.PageDependencyGraph.getNetworkInitiators(record)[0];
     let candidates = recordsByURL.get(initiatorURL) || [];
     // The (valid) initiator must come before the initiated request.
     candidates = candidates.filter(c => {

package/core/lib/network-request.d.ts

@@ -89,7 +89,7 @@ export class NetworkRequest {
      */
     static getResourceSizeOnNetwork(networkRecord: NetworkRequest): number;
     requestId: string;
-    connectionId: string;
+    connectionId: number;
     connectionReused: boolean;
     url: string;
     protocol: string;
@@ -104,7 +104,11 @@ export class NetworkRequest {
      * HTTP cache or going to the network for DNS/connection setup, in milliseconds.
      */
     networkRequestTime: number;
-    /** When the last byte of the response headers is received, in milliseconds. */
+    /**
+     * When the last byte of the response headers is received, in milliseconds.
+     * Equal to networkRequestTime if no data is recieved over the
+     * network (ex: cached requests or data urls).
+     */
     responseHeadersEndTime: number;
     /** When the last byte of the response body is received, in milliseconds. */
     networkEndTime: number;
@@ -149,6 +153,7 @@ export class NetworkRequest {
     sessionId: string | undefined;
     /** @type {LH.Protocol.TargetType|undefined} */
     sessionTargetType: LH.Protocol.TargetType | undefined;
+    fromWorker: boolean;
     isLinkPreload: boolean;
     /**
      * @return {boolean}
@@ -202,6 +207,7 @@ export class NetworkRequest {
      * @param {LH.Crdp.Network.ResponseReceivedEvent['type']=} resourceType
      */
     _onResponse(response: LH.Crdp.Network.Response, timestamp: number, resourceType?: LH.Crdp.Network.ResponseReceivedEvent['type'] | undefined): void;
+    responseTimestamp: number | undefined;
     /**
      * Resolve differences between conflicting timing signals. Based on the property setters in DevTools.
      * @see https://github.com/ChromeDevTools/devtools-frontend/blob/56a99365197b85c24b732ac92b0ac70feed80179/front_end/sdk/NetworkRequest.js#L485-L502
@@ -258,8 +264,28 @@ export namespace NetworkRequest {
     export { HEADER_FETCHED_SIZE };
     export { HEADER_PROTOCOL_IS_H2 };
 }
+/**
+ * @typedef HeaderEntry
+ * @property {string} name
+ * @property {string} value
+ */
+/**
+ * @typedef ParsedURL
+ * @property {string} scheme Equivalent to a `new URL(url).protocol` BUT w/o the trailing colon (:)
+ * @property {string} host Equivalent to a `new URL(url).hostname`
+ * @property {string} securityOrigin
+ */
+/**
+ * @typedef LightriderStatistics
+ * @property {number} endTimeDeltaMs The difference in networkEndTime between the observed Lighthouse networkEndTime and Lightrider's derived networkEndTime.
+ * @property {number} TCPMs The time spent making a TCP connection (connect + SSL). Note: this is poorly named.
+ * @property {number} requestMs The time spent requesting a resource from a remote server, we use this to approx RTT. Note: this is poorly names, it really should be "server response time".
+ * @property {number} responseMs Time to receive the entire response payload starting the clock on receiving the first fragment (first non-header byte).
+ */
+/** @type {LH.Util.SelfMap<LH.Crdp.Network.ResourceType>} */
+export const RESOURCE_TYPES: LH.Util.SelfMap<LH.Crdp.Network.ResourceType>;
 import * as LH from '../../types/lh.js';
-import * as Lantern from './lantern/types/lantern.js';
+import * as Lantern from './lantern/lantern.js';
 declare const HEADER_TCP: "X-TCPMs";
 declare const HEADER_SSL: "X-SSLMs";
 declare const HEADER_REQ: "X-RequestMs";

package/core/lib/network-request.js

@@ -53,7 +53,7 @@
  */
 
 import * as LH from '../../types/lh.js';
-import * as Lantern from './lantern/types/lantern.js';
+import * as Lantern from './lantern/lantern.js';
 import UrlUtils from './url-utils.js';
 
 // Lightrider X-Header names for timing information.
@@ -112,7 +112,7 @@ const RESOURCE_TYPES = {
 class NetworkRequest {
   constructor() {
     this.requestId = '';
-    this.connectionId = '0';
+    this.connectionId = 0;
     this.connectionReused = false;
 
     this.url = '';
@@ -129,7 +129,11 @@ class NetworkRequest {
      * HTTP cache or going to the network for DNS/connection setup, in milliseconds.
      */
     this.networkRequestTime = -1;
-    /** When the last byte of the response headers is received, in milliseconds. */
+    /**
+     * When the last byte of the response headers is received, in milliseconds.
+     * Equal to networkRequestTime if no data is recieved over the
+     * network (ex: cached requests or data urls).
+     */
     this.responseHeadersEndTime = -1;
     /** When the last byte of the response body is received, in milliseconds. */
     this.networkEndTime = -1;
@@ -180,6 +184,7 @@ class NetworkRequest {
     this.sessionId = undefined;
     /** @type {LH.Protocol.TargetType|undefined} */
     this.sessionTargetType = undefined;
+    this.fromWorker = false;
     this.isLinkPreload = false;
   }
 
@@ -221,8 +226,9 @@ class NetworkRequest {
     this.isSecure = UrlUtils.isSecureScheme(this.parsedURL.scheme);
 
     this.rendererStartTime = data.timestamp * 1000;
-    // Expected to be overridden with better value in `_recomputeTimesWithResourceTiming`.
+    // These are expected to be overridden with better value in `_recomputeTimesWithResourceTiming`.
     this.networkRequestTime = this.rendererStartTime;
+    this.responseHeadersEndTime = this.rendererStartTime;
 
     this.requestMethod = data.request.method;
 
@@ -344,13 +350,12 @@ class NetworkRequest {
   _onResponse(response, timestamp, resourceType) {
     this.url = response.url;
 
-    this.connectionId = String(response.connectionId);
+    this.connectionId = response.connectionId;
     this.connectionReused = response.connectionReused;
 
     if (response.protocol) this.protocol = response.protocol;
 
-    // This is updated in _recomputeTimesWithResourceTiming, if timings are present.
-    this.responseHeadersEndTime = timestamp * 1000;
+    this.responseTimestamp = timestamp * 1000;
 
     this.transferSize = response.encodedDataLength;
     this.responseHeadersTransferSize = response.encodedDataLength;
@@ -380,7 +385,7 @@ class NetworkRequest {
   _recomputeTimesWithResourceTiming(timing) {
     // Don't recompute times if the data is invalid. RequestTime should always be a thread timestamp.
     // If we don't have receiveHeadersEnd, we really don't have more accurate data.
-    if (timing.requestTime === 0 || timing.receiveHeadersEnd === -1) return;
+    if (timing.requestTime === -1 || timing.receiveHeadersEnd === -1) return;
 
     // Take networkRequestTime and responseHeadersEndTime from timing data for better accuracy.
     // Before this, networkRequestTime and responseHeadersEndTime were set to bogus values based on
@@ -391,15 +396,15 @@ class NetworkRequest {
     // See https://raw.githubusercontent.com/GoogleChrome/lighthouse/main/docs/Network-Timings.svg
     this.networkRequestTime = timing.requestTime * 1000;
     const headersReceivedTime = this.networkRequestTime + timing.receiveHeadersEnd;
-    // This was set in `_onResponse` as that event's timestamp.
-    const responseTimestamp = this.responseHeadersEndTime;
 
     // Update this.responseHeadersEndTime. All timing values from the netstack (timing) are well-ordered, and
     // so are the timestamps from CDP (which this.responseHeadersEndTime belongs to). It shouldn't be possible
     // that this timing from the netstack is greater than the onResponse timestamp, but just to ensure proper order
     // is maintained we bound the new timing by the network request time and the response timestamp.
     this.responseHeadersEndTime = headersReceivedTime;
-    this.responseHeadersEndTime = Math.min(this.responseHeadersEndTime, responseTimestamp);
+    if (this.responseTimestamp !== undefined) {
+      this.responseHeadersEndTime = Math.min(this.responseHeadersEndTime, this.responseTimestamp);
+    }
     this.responseHeadersEndTime = Math.max(this.responseHeadersEndTime, this.networkRequestTime);
 
     // We're only at responseReceived (_onResponse) at this point.
@@ -600,11 +605,13 @@ class NetworkRequest {
       }
     }
 
+    record.fromWorker = record.sessionTargetType === 'worker';
+
     return {
+      rawRequest: record,
       ...record,
       timing,
       serverResponseTime,
-      record,
     };
   }
 
@@ -691,4 +698,4 @@ NetworkRequest.HEADER_TOTAL = HEADER_TOTAL;
 NetworkRequest.HEADER_FETCHED_SIZE = HEADER_FETCHED_SIZE;
 NetworkRequest.HEADER_PROTOCOL_IS_H2 = HEADER_PROTOCOL_IS_H2;
 
-export {NetworkRequest};
+export {NetworkRequest, RESOURCE_TYPES};

package/core/lib/tracehouse/trace-processor.d.ts

@@ -281,11 +281,9 @@ export class TraceProcessor {
         timeOriginEvt: import("../..").TraceEvent;
         firstPaintEvt: import("../..").TraceEvent | undefined;
         firstContentfulPaintEvt: import("../..").TraceEvent;
-        firstMeaningfulPaintEvt: import("../..").TraceEvent | undefined;
         largestContentfulPaintEvt: LCPEvent | undefined;
         loadEvt: import("../..").TraceEvent | undefined;
         domContentLoadedEvt: import("../..").TraceEvent | undefined;
-        fmpFellBack: boolean;
         lcpInvalidated: boolean;
     };
 }

package/core/lib/tracehouse/trace-processor.js

@@ -9,7 +9,7 @@
  * various tools. This artifact will take a trace and then:
  *
  * 1. Find the TracingStartedInPage and navigationStart events of our intended tab & frame.
- * 2. Find the firstContentfulPaint and marked firstMeaningfulPaint events
+ * 2. Find the firstContentfulPaint and marked largestContentfulPaint events
  * 3. Isolate only the trace events from the tab's process (including all threads like compositor)
  *      * Sort those trace events in chronological order (as order isn't guaranteed)
  * 4. Return all those items in one handy bundle.
@@ -835,7 +835,6 @@ class TraceProcessor {
         firstPaint: frameTimings.timings.firstPaint,
         firstContentfulPaint: frameTimings.timings.firstContentfulPaint,
         firstContentfulPaintAllFrames: getTiming(fcpAllFramesEvt.ts),
-        firstMeaningfulPaint: frameTimings.timings.firstMeaningfulPaint,
         largestContentfulPaint: frameTimings.timings.largestContentfulPaint,
         largestContentfulPaintAllFrames: maybeGetTiming(lcpAllFramesEvt?.ts),
         load: frameTimings.timings.load,
@@ -847,7 +846,6 @@ class TraceProcessor {
         firstPaint: frameTimings.timestamps.firstPaint,
         firstContentfulPaint: frameTimings.timestamps.firstContentfulPaint,
         firstContentfulPaintAllFrames: fcpAllFramesEvt.ts,
-        firstMeaningfulPaint: frameTimings.timestamps.firstMeaningfulPaint,
         largestContentfulPaint: frameTimings.timestamps.largestContentfulPaint,
         largestContentfulPaintAllFrames: lcpAllFramesEvt?.ts,
         load: frameTimings.timestamps.load,
@@ -857,12 +855,10 @@ class TraceProcessor {
       firstPaintEvt: frameTimings.firstPaintEvt,
       firstContentfulPaintEvt: frameTimings.firstContentfulPaintEvt,
       firstContentfulPaintAllFramesEvt: fcpAllFramesEvt,
-      firstMeaningfulPaintEvt: frameTimings.firstMeaningfulPaintEvt,
       largestContentfulPaintEvt: frameTimings.largestContentfulPaintEvt,
       largestContentfulPaintAllFramesEvt: lcpAllFramesEvt,
       loadEvt: frameTimings.loadEvt,
       domContentLoadedEvt: frameTimings.domContentLoadedEvt,
-      fmpFellBack: frameTimings.fmpFellBack,
       lcpInvalidated: frameTimings.lcpInvalidated,
     };
   }
@@ -970,28 +966,6 @@ class TraceProcessor {
       throw this.createNoFirstContentfulPaintError();
     }
 
-    // fMP will follow at/after the FP
-    let firstMeaningfulPaint = frameEvents.find(
-      e => e.name === 'firstMeaningfulPaint' && e.ts > timeOriginEvt.ts
-    );
-    let fmpFellBack = false;
-
-    // If there was no firstMeaningfulPaint event found in the trace, the network idle detection
-    // may have not been triggered before Lighthouse finished tracing.
-    // In this case, we'll use the last firstMeaningfulPaintCandidate we can find.
-    // However, if no candidates were found (a bogus trace, likely), we fail.
-    if (!firstMeaningfulPaint) {
-      const fmpCand = 'firstMeaningfulPaintCandidate';
-      fmpFellBack = true;
-      log.verbose('TraceProcessor',
-        `No firstMeaningfulPaint found, falling back to last ${fmpCand}`);
-      const lastCandidate = frameEvents.filter(e => e.name === fmpCand).pop();
-      if (!lastCandidate) {
-        log.verbose('TraceProcessor', 'No `firstMeaningfulPaintCandidate` events found in trace');
-      }
-      firstMeaningfulPaint = lastCandidate;
-    }
-
     // This function accepts events spanning multiple frames, but this usage will only provide events from the main frame.
     const lcpResult = this.computeValidLCPAllFrames(frameEvents, timeOriginEvt);
 
@@ -1007,7 +981,6 @@ class TraceProcessor {
       timeOrigin: timeOriginEvt.ts,
       firstPaint: getTimestamp(firstPaint),
       firstContentfulPaint: firstContentfulPaint.ts,
-      firstMeaningfulPaint: getTimestamp(firstMeaningfulPaint),
       largestContentfulPaint: getTimestamp(lcpResult.lcp),
       load: getTimestamp(load),
       domContentLoaded: getTimestamp(domContentLoaded),
@@ -1022,7 +995,6 @@ class TraceProcessor {
       timeOrigin: 0,
       firstPaint: maybeGetTiming(timestamps.firstPaint),
       firstContentfulPaint: getTiming(timestamps.firstContentfulPaint),
-      firstMeaningfulPaint: maybeGetTiming(timestamps.firstMeaningfulPaint),
       largestContentfulPaint: maybeGetTiming(timestamps.largestContentfulPaint),
       load: maybeGetTiming(timestamps.load),
       domContentLoaded: maybeGetTiming(timestamps.domContentLoaded),
@@ -1034,11 +1006,9 @@ class TraceProcessor {
       timeOriginEvt: timeOriginEvt,
       firstPaintEvt: firstPaint,
       firstContentfulPaintEvt: firstContentfulPaint,
-      firstMeaningfulPaintEvt: firstMeaningfulPaint,
       largestContentfulPaintEvt: lcpResult.lcp,
       loadEvt: load,
       domContentLoadedEvt: domContentLoaded,
-      fmpFellBack,
       lcpInvalidated: lcpResult.invalidated,
     };
   }

package/core/lib/traces/metric-trace-events.js

@@ -45,11 +45,6 @@ class MetricTraceEvents {
         id: 'ttfcp',
         tsKey: 'observedFirstContentfulPaintTs',
       },
-      {
-        name: 'First Meaningful Paint',
-        id: 'ttfmp',
-        tsKey: 'observedFirstMeaningfulPaintTs',
-      },
       {
         name: 'Speed Index',
         id: 'si',