lighthouse 12.0.0-dev.20240602 → 12.0.0-dev.20240603

package/core/audits/byte-efficiency/render-blocking-resources.js

@@ -53,7 +53,7 @@ function getNodesAndTimingByRequestId(nodeTimings) {
   for (const [node, nodeTiming] of nodeTimings) {
     if (node.type !== 'network') continue;
 
-    requestIdToNode.set(node.record.requestId, {node, nodeTiming});
+    requestIdToNode.set(node.request.requestId, {node, nodeTiming});
   }
 
   return requestIdToNode;

package/core/audits/dobetterweb/uses-http2.js

@@ -89,7 +89,7 @@ class UsesHTTP2Audit extends Audit {
       if (node.type !== 'network') return;
       if (!urlsToChange.has(node.record.url)) return;
 
-      originalProtocols.set(node.record.requestId, node.record.protocol);
+      originalProtocols.set(node.request.requestId, node.record.protocol);
       node.request.protocol = 'h2';
     });
 
@@ -98,7 +98,7 @@ class UsesHTTP2Audit extends Audit {
     // Restore the original protocol after we've done our simulation
     graph.traverse(node => {
       if (node.type !== 'network') return;
-      const originalProtocol = originalProtocols.get(node.record.requestId);
+      const originalProtocol = originalProtocols.get(node.request.requestId);
       if (originalProtocol === undefined) return;
       node.request.protocol = originalProtocol;
     });

package/core/audits/prioritize-lcp-image.js

@@ -69,7 +69,7 @@ class PrioritizeLcpImage extends Audit {
   static findLCPNode(graph, lcpRecord) {
     for (const {node} of graph.traverseGenerator()) {
       if (node.type !== 'network') continue;
-      if (node.record.requestId === lcpRecord.requestId) {
+      if (node.request.requestId === lcpRecord.requestId) {
         return node;
       }
     }

package/core/audits/redirects.js

@@ -98,7 +98,7 @@ class Redirects extends Audit {
     const nodeTimingsById = new Map();
     for (const [node, timing] of metricResult.pessimisticEstimate.nodeTimings.entries()) {
       if (node.type === 'network') {
-        nodeTimingsById.set(node.record.requestId, timing);
+        nodeTimingsById.set(node.request.requestId, timing);
       }
     }
 

package/core/computed/metrics/lantern-metric.d.ts

@@ -2,11 +2,29 @@
  * @param {LH.Artifacts.MetricComputationDataInput} data
  * @param {LH.Artifacts.ComputedContext} context
  */
-export function getComputationDataParams(data: LH.Artifacts.MetricComputationDataInput, context: LH.Artifacts.ComputedContext): Promise<{
+export function getComputationDataParamsFromTrace(data: LH.Artifacts.MetricComputationDataInput, context: LH.Artifacts.ComputedContext): Promise<{
+    simulator: import("../../lib/lantern/simulator/simulator.js").Simulator<any>;
+    graph: import("../../lib/lantern/page-dependency-graph.js").Node;
+    processedNavigation: import("../../index.js").Artifacts.ProcessedNavigation;
+}>;
+/**
+ * @param {LH.Artifacts.MetricComputationDataInput} data
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+export function getComputationDataParamsFromDevtoolsLog(data: LH.Artifacts.MetricComputationDataInput, context: LH.Artifacts.ComputedContext): Promise<{
     simulator: import("../../lib/lantern/simulator/simulator.js").Simulator<any>;
     graph: import("../page-dependency-graph.js").Node;
     processedNavigation: import("../../index.js").Artifacts.ProcessedNavigation;
 }>;
+/**
+ * @param {LH.Artifacts.MetricComputationDataInput} data
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+export function getComputationDataParams(data: LH.Artifacts.MetricComputationDataInput, context: LH.Artifacts.ComputedContext): Promise<{
+    simulator: import("../../lib/lantern/simulator/simulator.js").Simulator<any>;
+    graph: import("../../lib/lantern/page-dependency-graph.js").Node;
+    processedNavigation: import("../../index.js").Artifacts.ProcessedNavigation;
+}>;
 /**
  * @param {unknown} err
  * @return {never}

package/core/computed/metrics/lantern-metric.js

@@ -5,16 +5,20 @@
  */
 
 import {LanternError} from '../../lib/lantern/lantern-error.js';
+import {PageDependencyGraph as LanternPageDependencyGraph} from '../../lib/lantern/page-dependency-graph.js';
 import {LighthouseError} from '../../lib/lh-error.js';
 import {LoadSimulator} from '../load-simulator.js';
-import {PageDependencyGraph} from '../page-dependency-graph.js';
 import {ProcessedNavigation} from '../processed-navigation.js';
+import {ProcessedTrace} from '../processed-trace.js';
+import {TraceEngineResult} from '../trace-engine-result.js';
+import {PageDependencyGraph} from '../page-dependency-graph.js';
 
+// TODO: we need to update all test traces (that use lantern) before this can be removed
 /**
  * @param {LH.Artifacts.MetricComputationDataInput} data
  * @param {LH.Artifacts.ComputedContext} context
  */
-async function getComputationDataParams(data, context) {
+async function getComputationDataParamsFromDevtoolsLog(data, context) {
   if (data.gatherContext.gatherMode !== 'navigation') {
     throw new Error(`Lantern metrics can only be computed on navigations`);
   }
@@ -26,6 +30,35 @@ async function getComputationDataParams(data, context) {
   return {simulator, graph, processedNavigation};
 }
 
+/**
+ * @param {LH.Artifacts.URL} theURL
+ * @param {LH.Trace} trace
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+async function createGraphFromTrace(theURL, trace, context) {
+  const {mainThreadEvents} = await ProcessedTrace.request(trace, context);
+  const traceEngineResult = await TraceEngineResult.request({trace}, context);
+  return LanternPageDependencyGraph.createGraphFromTrace(
+    mainThreadEvents, trace, traceEngineResult, theURL);
+}
+
+/**
+ * @param {LH.Artifacts.MetricComputationDataInput} data
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+async function getComputationDataParamsFromTrace(data, context) {
+  if (data.gatherContext.gatherMode !== 'navigation') {
+    throw new Error(`Lantern metrics can only be computed on navigations`);
+  }
+
+  const {trace, URL} = data;
+  const {graph} = await createGraphFromTrace(URL, trace, context);
+  const processedNavigation = await ProcessedNavigation.request(data.trace, context);
+  const simulator = data.simulator || (await LoadSimulator.request(data, context));
+
+  return {simulator, graph, processedNavigation};
+}
+
 /**
  * @param {unknown} err
  * @return {never}
@@ -43,7 +76,23 @@ function lanternErrorAdapter(err) {
   throw err;
 }
 
+/**
+ * @param {LH.Artifacts.MetricComputationDataInput} data
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+function getComputationDataParams(data, context) {
+  // TODO(15841): remove when all tests use the trace, and we want to remove CDT graph impl.
+  if (process.env.INTERNAL_LANTERN_USE_TRACE !== undefined) {
+    return getComputationDataParamsFromTrace(data, context);
+  } else {
+    // This is the default behavior.
+    return getComputationDataParamsFromDevtoolsLog(data, context);
+  }
+}
+
 export {
+  getComputationDataParamsFromTrace,
+  getComputationDataParamsFromDevtoolsLog,
   getComputationDataParams,
   lanternErrorAdapter,
 };

package/core/computed/navigation-insights.d.ts

@@ -11,6 +11,7 @@ declare const NavigationInsightsComputed: typeof NavigationInsights & {
         Samples: typeof import("@paulirish/trace_engine/models/trace/handlers/SamplesHandler.js");
         Screenshots: typeof import("@paulirish/trace_engine/models/trace/handlers/ScreenshotsHandler.js");
         PageLoadMetrics: typeof import("@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js");
+        Workers: typeof import("@paulirish/trace_engine/models/trace/handlers/WorkersHandler.js");
     }>>;
 };
 /**
@@ -31,6 +32,7 @@ declare class NavigationInsights {
         Samples: typeof import("@paulirish/trace_engine/models/trace/handlers/SamplesHandler.js");
         Screenshots: typeof import("@paulirish/trace_engine/models/trace/handlers/ScreenshotsHandler.js");
         PageLoadMetrics: typeof import("@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js");
+        Workers: typeof import("@paulirish/trace_engine/models/trace/handlers/WorkersHandler.js");
     }>>;
 }
 //# sourceMappingURL=navigation-insights.d.ts.map

package/core/computed/trace-engine-result.d.ts

@@ -16,6 +16,7 @@ declare namespace ENABLED_HANDLERS {
     const Samples: typeof import("@paulirish/trace_engine/models/trace/handlers/SamplesHandler.js");
     const Screenshots: typeof import("@paulirish/trace_engine/models/trace/handlers/ScreenshotsHandler.js");
     const PageLoadMetrics: typeof import("@paulirish/trace_engine/models/trace/handlers/PageLoadMetricsHandler.js");
+    const Workers: typeof import("@paulirish/trace_engine/models/trace/handlers/WorkersHandler.js");
 }
 /**
  * @fileoverview Processes trace with the shared trace engine.

package/core/computed/trace-engine-result.js

@@ -21,6 +21,7 @@ const ENABLED_HANDLERS = {
   Samples: TraceEngine.TraceHandlers.Samples,
   Screenshots: TraceEngine.TraceHandlers.Screenshots,
   PageLoadMetrics: TraceEngine.TraceHandlers.PageLoadMetrics,
+  Workers: TraceEngine.TraceHandlers.Workers,
 };
 
 /**

package/core/lib/lantern/metric.js

@@ -6,7 +6,7 @@
 
 import * as Lantern from './types/lantern.js';
 import {BaseNode} from '../../lib/lantern/base-node.js';
-import {NetworkRequest} from '../../lib/network-request.js';
+import {RESOURCE_TYPES} from '../../lib/network-request.js';
 
 /** @typedef {import('./base-node.js').Node} Node */
 /** @typedef {import('./network-node.js').NetworkNode} NetworkNode */
@@ -33,9 +33,9 @@ class Metric {
 
     dependencyGraph.traverse(node => {
       if (node.type !== BaseNode.TYPES.NETWORK) return;
-      if (node.record.resourceType !== NetworkRequest.TYPES.Script) return;
+      if (node.request.resourceType !== RESOURCE_TYPES.Script) return;
       if (treatNodeAsRenderBlocking?.(node)) {
-        scriptUrls.add(node.record.url);
+        scriptUrls.add(node.request.url);
       }
     });
 

package/core/lib/lantern/metrics/first-contentful-paint.js

@@ -153,7 +153,7 @@ class FirstContentfulPaint extends Metric {
         const endedAfterPaint = node.endTime > cutoffTimestamp || node.startTime > cutoffTimestamp;
         if (endedAfterPaint && !node.isMainDocument()) return false;
 
-        const url = node.record.url;
+        const url = node.request.url;
         // If the URL definitely wasn't render-blocking then we filter it out.
         if (definitelyNotRenderBlockingScriptUrls.has(url)) {
           return false;

package/core/lib/lantern/metrics/interactive.js

@@ -41,13 +41,13 @@ class Interactive extends Metric {
       }
 
       // Include all scripts and high priority requests, exclude all images
-      const isImage = node.record.resourceType === NetworkRequestTypes.Image;
-      const isScript = node.record.resourceType === NetworkRequestTypes.Script;
+      const isImage = node.request.resourceType === NetworkRequestTypes.Image;
+      const isScript = node.request.resourceType === NetworkRequestTypes.Script;
       return (
         !isImage &&
         (isScript ||
-          node.record.priority === 'High' ||
-          node.record.priority === 'VeryHigh')
+          node.request.priority === 'High' ||
+          node.request.priority === 'VeryHigh')
       );
     });
   }

package/core/lib/lantern/metrics/largest-contentful-paint.js

@@ -32,8 +32,8 @@ class LargestContentfulPaint extends Metric {
    */
   static isNotLowPriorityImageNode(node) {
     if (node.type !== 'network') return true;
-    const isImage = node.record.resourceType === 'Image';
-    const isLowPriority = node.record.priority === 'Low' || node.record.priority === 'VeryLow';
+    const isImage = node.request.resourceType === 'Image';
+    const isLowPriority = node.request.priority === 'Low' || node.request.priority === 'VeryLow';
     return !isImage || !isLowPriority;
   }
 

package/core/lib/lantern/page-dependency-graph.d.ts

@@ -39,6 +39,20 @@ export class PageDependencyGraph {
      * @param {Node} node
      */
     static _pruneNode(node: Node): void;
+    /**
+     * TODO(15841): remove when CDT backend is gone. until then, this is a useful debugging tool
+     * to find delta between using CDP or the trace to create the network requests.
+     *
+     * When a test fails using the trace backend, I enabled this debug method and copied the network
+     * requests when CDP was used, then when trace is used, and diff'd them. This method helped
+     * remove non-logical differences from the comparison (order of properties, slight rounding
+     * discrepancies, removing object cycles, etc).
+     *
+     * When using for a unit test, make sure to do `.only` so you are getting what you expect.
+     * @param {Lantern.NetworkRequest[]} lanternRequests
+     * @return {never}
+     */
+    static _debugNormalizeRequests(lanternRequests: Lantern.NetworkRequest[]): never;
     /**
      * @param {LH.TraceEvent[]} mainThreadEvents
      * @param {Array<Lantern.NetworkRequest>} networkRecords
@@ -46,6 +60,48 @@ export class PageDependencyGraph {
      * @return {Node}
      */
     static createGraph(mainThreadEvents: LH.TraceEvent[], networkRecords: Array<Lantern.NetworkRequest>, URL: URLArtifact): Node;
+    /**
+     * @param {Lantern.NetworkRequest} record The record to find the initiator of
+     * @param {Map<string, Lantern.NetworkRequest[]>} recordsByURL
+     * @return {Lantern.NetworkRequest|null}
+     */
+    static chooseInitiatorRequest(record: Lantern.NetworkRequest, recordsByURL: Map<string, Lantern.NetworkRequest[]>): Lantern.NetworkRequest | null;
+    /**
+     * Returns a map of `pid` -> `tid[]`.
+     * @param {LH.Trace} trace
+     * @return {Map<number, number[]>}
+     */
+    static _findWorkerThreads(trace: LH.Trace): Map<number, number[]>;
+    /**
+     * @param {URL|string} url
+     */
+    static _createParsedUrl(url: URL | string): {
+        scheme: string;
+        host: string;
+        securityOrigin: string;
+    };
+    /**
+     * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+     * @param {Map<number, number[]>} workerThreads
+     * @param {import('@paulirish/trace_engine/models/trace/types/TraceEvents.js').SyntheticNetworkRequest} request
+     * @return {Lantern.NetworkRequest=}
+     */
+    static _createLanternRequest(traceEngineResult: LH.Artifacts.TraceEngineResult, workerThreads: Map<number, number[]>, request: import('@paulirish/trace_engine/models/trace/types/TraceEvents.js').SyntheticNetworkRequest): Lantern.NetworkRequest | undefined;
+    /**
+     *
+     * @param {Lantern.NetworkRequest[]} lanternRequests
+     */
+    static _linkInitiators(lanternRequests: Lantern.NetworkRequest[]): void;
+    /**
+     * @param {LH.TraceEvent[]} mainThreadEvents
+     * @param {LH.Trace} trace
+     * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+     * @param {LH.Artifacts.URL} URL
+     */
+    static createGraphFromTrace(mainThreadEvents: LH.TraceEvent[], trace: LH.Trace, traceEngineResult: LH.Artifacts.TraceEngineResult, URL: LH.Artifacts.URL): Promise<{
+        graph: Node;
+        records: Lantern.NetworkRequest<any>[];
+    }>;
     /**
      *
      * @param {Node} rootNode

package/core/lib/lantern/page-dependency-graph.js

@@ -10,6 +10,7 @@ import {NetworkNode} from './network-node.js';
 import {CPUNode} from './cpu-node.js';
 import {TraceProcessor} from '../tracehouse/trace-processor.js';
 import {NetworkAnalyzer} from './simulator/network-analyzer.js';
+import {RESOURCE_TYPES} from '../network-request.js';
 
 /** @typedef {import('./base-node.js').Node} Node */
 /** @typedef {Omit<LH.Artifacts['URL'], 'finalDisplayedUrl'>} URLArtifact */
@@ -75,7 +76,7 @@ class PageDependencyGraph {
 
     networkRecords.forEach(record => {
       if (IGNORED_MIME_TYPES_REGEX.test(record.mimeType)) return;
-      if (record.sessionTargetType === 'worker') return;
+      if (record.fromWorker) return;
 
       // Network record requestIds can be duplicated for an unknown reason
       // Suffix all subsequent records with `:duplicate` until it's unique
@@ -404,6 +405,107 @@ class PageDependencyGraph {
     }
   }
 
+  /**
+   * TODO(15841): remove when CDT backend is gone. until then, this is a useful debugging tool
+   * to find delta between using CDP or the trace to create the network requests.
+   *
+   * When a test fails using the trace backend, I enabled this debug method and copied the network
+   * requests when CDP was used, then when trace is used, and diff'd them. This method helped
+   * remove non-logical differences from the comparison (order of properties, slight rounding
+   * discrepancies, removing object cycles, etc).
+   *
+   * When using for a unit test, make sure to do `.only` so you are getting what you expect.
+   * @param {Lantern.NetworkRequest[]} lanternRequests
+   * @return {never}
+   */
+  static _debugNormalizeRequests(lanternRequests) {
+    for (const request of lanternRequests) {
+      request.rendererStartTime = Math.round(request.rendererStartTime * 1000) / 1000;
+      request.networkRequestTime = Math.round(request.networkRequestTime * 1000) / 1000;
+      request.responseHeadersEndTime = Math.round(request.responseHeadersEndTime * 1000) / 1000;
+      request.networkEndTime = Math.round(request.networkEndTime * 1000) / 1000;
+    }
+
+    for (const r of lanternRequests) {
+      delete r.record;
+      if (r.initiatorRequest) {
+        // @ts-expect-error
+        r.initiatorRequest = {id: r.initiatorRequest.requestId};
+      }
+      if (r.redirectDestination) {
+        // @ts-expect-error
+        r.redirectDestination = {id: r.redirectDestination.requestId};
+      }
+      if (r.redirectSource) {
+        // @ts-expect-error
+        r.redirectSource = {id: r.redirectSource.requestId};
+      }
+      if (r.redirects) {
+        // @ts-expect-error
+        r.redirects = r.redirects.map(r2 => r2.requestId);
+      }
+    }
+    /** @type {Lantern.NetworkRequest[]} */
+    const requests = lanternRequests.map(r => ({
+      requestId: r.requestId,
+      connectionId: r.connectionId,
+      connectionReused: r.connectionReused,
+      url: r.url,
+      protocol: r.protocol,
+      parsedURL: r.parsedURL,
+      documentURL: r.documentURL,
+      rendererStartTime: r.rendererStartTime,
+      networkRequestTime: r.networkRequestTime,
+      responseHeadersEndTime: r.responseHeadersEndTime,
+      networkEndTime: r.networkEndTime,
+      transferSize: r.transferSize,
+      resourceSize: r.resourceSize,
+      fromDiskCache: r.fromDiskCache,
+      fromMemoryCache: r.fromMemoryCache,
+      finished: r.finished,
+      statusCode: r.statusCode,
+      redirectSource: r.redirectSource,
+      redirectDestination: r.redirectDestination,
+      redirects: r.redirects,
+      failed: r.failed,
+      initiator: r.initiator,
+      timing: r.timing ? {
+        requestTime: r.timing.requestTime,
+        proxyStart: r.timing.proxyStart,
+        proxyEnd: r.timing.proxyEnd,
+        dnsStart: r.timing.dnsStart,
+        dnsEnd: r.timing.dnsEnd,
+        connectStart: r.timing.connectStart,
+        connectEnd: r.timing.connectEnd,
+        sslStart: r.timing.sslStart,
+        sslEnd: r.timing.sslEnd,
+        workerStart: r.timing.workerStart,
+        workerReady: r.timing.workerReady,
+        workerFetchStart: r.timing.workerFetchStart,
+        workerRespondWithSettled: r.timing.workerRespondWithSettled,
+        sendStart: r.timing.sendStart,
+        sendEnd: r.timing.sendEnd,
+        pushStart: r.timing.pushStart,
+        pushEnd: r.timing.pushEnd,
+        receiveHeadersStart: r.timing.receiveHeadersStart,
+        receiveHeadersEnd: r.timing.receiveHeadersEnd,
+      } : r.timing,
+      resourceType: r.resourceType,
+      mimeType: r.mimeType,
+      priority: r.priority,
+      initiatorRequest: r.initiatorRequest,
+      frameId: r.frameId,
+      fromWorker: r.fromWorker,
+      isLinkPreload: r.isLinkPreload,
+      serverResponseTime: r.serverResponseTime,
+    })).filter(r => !r.fromWorker);
+    // eslint-disable-next-line no-unused-vars
+    const debug = requests;
+    // Set breakpoint here.
+    // Copy `debug` and compare with https://www.diffchecker.com/text-compare/
+    process.exit(1);
+  }
+
   /**
    * @param {LH.TraceEvent[]} mainThreadEvents
    * @param {Array<Lantern.NetworkRequest>} networkRecords
@@ -411,6 +513,8 @@ class PageDependencyGraph {
    * @return {Node}
    */
   static createGraph(mainThreadEvents, networkRecords, URL) {
+    // This is for debugging trace/devtoolslog network records.
+    // const debug = PageDependencyGraph._debugNormalizeRequests(networkRecords);
     const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
     const cpuNodes = PageDependencyGraph.getCPUNodes(mainThreadEvents);
     const {requestedUrl, mainDocumentUrl} = URL;
@@ -421,7 +525,6 @@ class PageDependencyGraph {
     if (!rootRequest) throw new Error('rootRequest not found');
     const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
     if (!rootNode) throw new Error('rootNode not found');
-
     const mainDocumentRequest =
       NetworkAnalyzer.findLastDocumentForUrl(networkRecords, mainDocumentUrl);
     if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found');
@@ -439,6 +542,314 @@ class PageDependencyGraph {
     return rootNode;
   }
 
+  /**
+   * @param {Lantern.NetworkRequest} record The record to find the initiator of
+   * @param {Map<string, Lantern.NetworkRequest[]>} recordsByURL
+   * @return {Lantern.NetworkRequest|null}
+   */
+  static chooseInitiatorRequest(record, recordsByURL) {
+    if (record.redirectSource) {
+      return record.redirectSource;
+    }
+
+    const initiatorURL = PageDependencyGraph.getNetworkInitiators(record)[0];
+    let candidates = recordsByURL.get(initiatorURL) || [];
+    // The (valid) initiator must come before the initiated request.
+    candidates = candidates.filter(c => {
+      return c.responseHeadersEndTime <= record.rendererStartTime &&
+          c.finished && !c.failed;
+    });
+    if (candidates.length > 1) {
+      // Disambiguate based on prefetch. Prefetch requests have type 'Other' and cannot
+      // initiate requests, so we drop them here.
+      const nonPrefetchCandidates = candidates.filter(
+          cand => cand.resourceType !== RESOURCE_TYPES.Other);
+      if (nonPrefetchCandidates.length) {
+        candidates = nonPrefetchCandidates;
+      }
+    }
+    if (candidates.length > 1) {
+      // Disambiguate based on frame. It's likely that the initiator comes from the same frame.
+      const sameFrameCandidates = candidates.filter(cand => cand.frameId === record.frameId);
+      if (sameFrameCandidates.length) {
+        candidates = sameFrameCandidates;
+      }
+    }
+    if (candidates.length > 1 && record.initiator.type === 'parser') {
+      // Filter to just Documents when initiator type is parser.
+      const documentCandidates = candidates.filter(cand =>
+        cand.resourceType === RESOURCE_TYPES.Document);
+      if (documentCandidates.length) {
+        candidates = documentCandidates;
+      }
+    }
+    if (candidates.length > 1) {
+      // If all real loads came from successful preloads (url preloaded and
+      // loads came from the cache), filter to link rel=preload request(s).
+      const linkPreloadCandidates = candidates.filter(c => c.isLinkPreload);
+      if (linkPreloadCandidates.length) {
+        const nonPreloadCandidates = candidates.filter(c => !c.isLinkPreload);
+        const allPreloaded = nonPreloadCandidates.every(c => c.fromDiskCache || c.fromMemoryCache);
+        if (nonPreloadCandidates.length && allPreloaded) {
+          candidates = linkPreloadCandidates;
+        }
+      }
+    }
+
+    // Only return an initiator if the result is unambiguous.
+    return candidates.length === 1 ? candidates[0] : null;
+  }
+
+  /**
+   * Returns a map of `pid` -> `tid[]`.
+   * @param {LH.Trace} trace
+   * @return {Map<number, number[]>}
+   */
+  static _findWorkerThreads(trace) {
+    // TODO: WorkersHandler in TraceEngine needs to be updated to also include `pid` (only had `tid`).
+    const workerThreads = new Map();
+    const workerCreationEvents = ['ServiceWorker thread', 'DedicatedWorker thread'];
+
+    for (const event of trace.traceEvents) {
+      if (event.name !== 'thread_name' || !event.args.name) {
+        continue;
+      }
+      if (!workerCreationEvents.includes(event.args.name)) {
+        continue;
+      }
+
+      const tids = workerThreads.get(event.pid);
+      if (tids) {
+        tids.push(event.tid);
+      } else {
+        workerThreads.set(event.pid, [event.tid]);
+      }
+    }
+
+    return workerThreads;
+  }
+
+  /**
+   * @param {URL|string} url
+   */
+  static _createParsedUrl(url) {
+    if (typeof url === 'string') {
+      url = new URL(url);
+    }
+    return {
+      scheme: url.protocol.split(':')[0],
+      // Intentional, DevTools uses different terminology
+      host: url.hostname,
+      securityOrigin: url.origin,
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+   * @param {Map<number, number[]>} workerThreads
+   * @param {import('@paulirish/trace_engine/models/trace/types/TraceEvents.js').SyntheticNetworkRequest} request
+   * @return {Lantern.NetworkRequest=}
+   */
+  static _createLanternRequest(traceEngineResult, workerThreads, request) {
+    if (request.args.data.connectionId === undefined ||
+        request.args.data.connectionReused === undefined) {
+      throw new Error('Trace is too old');
+    }
+
+    let url;
+    try {
+      url = new URL(request.args.data.url);
+    } catch (e) {
+      return;
+    }
+
+    const timing = request.args.data.timing ? {
+      // These two timings are not included in the trace.
+      workerFetchStart: -1,
+      workerRespondWithSettled: -1,
+      ...request.args.data.timing,
+    } : undefined;
+
+    const networkRequestTime = timing ?
+      timing.requestTime * 1000 :
+      request.args.data.syntheticData.downloadStart / 1000;
+
+    let fromWorker = false;
+    const tids = workerThreads.get(request.pid);
+    if (tids?.includes(request.tid)) {
+      fromWorker = true;
+    }
+
+    // TraceEngine collects worker thread ids in a different manner than `workerThreads` does.
+    // AFAIK these should be equivalent, but in case they are not let's also check this for now.
+    if (traceEngineResult.data.Workers.workerIdByThread.has(request.tid)) {
+      fromWorker = true;
+    }
+
+    // `initiator` in the trace does not contain the stack trace for JS-initiated
+    // requests. Instead, that is stored in the `stackTrace` property of the SyntheticNetworkRequest.
+    // There are some minor differences in the fields, accounted for here.
+    // Most importantly, there seems to be fewer frames in the trace than the equivalent
+    // events over the CDP. This results in less accuracy in determining the initiator request,
+    // which means less edges in the graph, which mean worse results.
+    // TODO: Should fix in Chromium.
+    /** @type {Lantern.NetworkRequest['initiator']} */
+    const initiator = request.args.data.initiator ?? {type: 'other'};
+    if (request.args.data.stackTrace) {
+      const callFrames = request.args.data.stackTrace.map(f => {
+        return {
+          scriptId: String(f.scriptId),
+          url: f.url,
+          lineNumber: f.lineNumber - 1,
+          columnNumber: f.columnNumber - 1,
+          functionName: f.functionName,
+        };
+      });
+      initiator.stack = {callFrames};
+    }
+
+    let resourceType = request.args.data.resourceType;
+    if (request.args.data.initiator?.fetchType === 'xmlhttprequest') {
+      // @ts-expect-error yes XHR is a valid ResourceType. TypeScript const enums are so unhelpful.
+      resourceType = 'XHR';
+    }
+
+    return {
+      requestId: request.args.data.requestId,
+      connectionId: request.args.data.connectionId,
+      connectionReused: request.args.data.connectionReused,
+      url: request.args.data.url,
+      protocol: request.args.data.protocol,
+      parsedURL: this._createParsedUrl(url),
+      documentURL: request.args.data.requestingFrameUrl,
+      rendererStartTime: request.ts / 1000,
+      networkRequestTime,
+      responseHeadersEndTime: request.args.data.syntheticData.downloadStart / 1000,
+      networkEndTime: request.args.data.syntheticData.finishTime / 1000,
+      transferSize: request.args.data.encodedDataLength,
+      resourceSize: request.args.data.decodedBodyLength,
+      fromDiskCache: request.args.data.syntheticData.isDiskCached,
+      fromMemoryCache: request.args.data.syntheticData.isMemoryCached,
+      isLinkPreload: request.args.data.isLinkPreload,
+      finished: request.args.data.finished,
+      failed: request.args.data.failed,
+      statusCode: request.args.data.statusCode,
+      initiator,
+      timing,
+      resourceType,
+      mimeType: request.args.data.mimeType,
+      priority: request.args.data.priority,
+      frameId: request.args.data.frame,
+      fromWorker,
+      record: request,
+      // Set later.
+      redirects: undefined,
+      redirectSource: undefined,
+      redirectDestination: undefined,
+      initiatorRequest: undefined,
+    };
+  }
+
+  /**
+   *
+   * @param {Lantern.NetworkRequest[]} lanternRequests
+   */
+  static _linkInitiators(lanternRequests) {
+    /** @type {Map<string, Lantern.NetworkRequest[]>} */
+    const requestsByURL = new Map();
+    for (const request of lanternRequests) {
+      const requests = requestsByURL.get(request.url) || [];
+      requests.push(request);
+      requestsByURL.set(request.url, requests);
+    }
+
+    for (const request of lanternRequests) {
+      const initiatorRequest = PageDependencyGraph.chooseInitiatorRequest(request, requestsByURL);
+      if (initiatorRequest) {
+        request.initiatorRequest = initiatorRequest;
+      }
+    }
+  }
+
+  /**
+   * @param {LH.TraceEvent[]} mainThreadEvents
+   * @param {LH.Trace} trace
+   * @param {LH.Artifacts.TraceEngineResult} traceEngineResult
+   * @param {LH.Artifacts.URL} URL
+   */
+  static async createGraphFromTrace(mainThreadEvents, trace, traceEngineResult, URL) {
+    const workerThreads = this._findWorkerThreads(trace);
+
+    /** @type {Lantern.NetworkRequest[]} */
+    const lanternRequests = [];
+    for (const request of traceEngineResult.data.NetworkRequests.byTime) {
+      const lanternRequest = this._createLanternRequest(traceEngineResult, workerThreads, request);
+      if (lanternRequest) {
+        lanternRequests.push(lanternRequest);
+      }
+    }
+
+    // TraceEngine consolidates all redirects into a single request object, but lantern needs
+    // an entry for each redirected request.
+    for (const request of [...lanternRequests]) {
+      if (!request.record) continue;
+
+      const redirects = request.record.args.data.redirects;
+      if (!redirects.length) continue;
+
+      const requestChain = [];
+      for (const redirect of redirects) {
+        const redirectedRequest = structuredClone(request);
+        if (redirectedRequest.timing) {
+          // TODO: These are surely wrong for when there is more than one redirect. Would be
+          // simpler if each redirect remembered it's `timing` object in this `redirects` array.
+          redirectedRequest.timing.requestTime = redirect.ts / 1000 / 1000;
+          redirectedRequest.timing.receiveHeadersStart -= redirect.dur / 1000 / 1000;
+          redirectedRequest.timing.receiveHeadersEnd -= redirect.dur / 1000 / 1000;
+          redirectedRequest.rendererStartTime = redirect.ts / 1000;
+          redirectedRequest.networkRequestTime = redirect.ts / 1000;
+          redirectedRequest.networkEndTime = (redirect.ts + redirect.dur) / 1000;
+          redirectedRequest.responseHeadersEndTime =
+            redirectedRequest.timing.requestTime * 1000 +
+            redirectedRequest.timing.receiveHeadersEnd;
+        }
+        redirectedRequest.url = redirect.url;
+        redirectedRequest.parsedURL = this._createParsedUrl(redirect.url);
+        // TODO: TraceEngine is not retaining the actual status code.
+        redirectedRequest.statusCode = 302;
+        requestChain.push(redirectedRequest);
+        lanternRequests.push(redirectedRequest);
+      }
+      requestChain.push(request);
+
+      for (let i = 0; i < requestChain.length; i++) {
+        const request = requestChain[i];
+        if (i > 0) {
+          request.redirectSource = requestChain[i - 1];
+          request.redirects = requestChain.slice(0, i);
+        }
+        if (i !== requestChain.length - 1) {
+          request.redirectDestination = requestChain[i + 1];
+        }
+      }
+
+      // Apply the `:redirect` requestId convention: only redirects[0].requestId is the actual
+      // requestId, all the rest have n occurences of `:redirect` as a suffix.
+      for (let i = 1; i < requestChain.length; i++) {
+        requestChain[i].requestId = `${requestChain[i - 1].requestId}:redirect`;
+      }
+    }
+
+    this._linkInitiators(lanternRequests);
+
+    // This would already be sorted by rendererStartTime, if not for the redirect unwrapping done
+    // above.
+    lanternRequests.sort((a, b) => a.rendererStartTime - b.rendererStartTime);
+
+    const graph = PageDependencyGraph.createGraph(mainThreadEvents, lanternRequests, URL);
+    return {graph, records: lanternRequests};
+  }
+
   /**
    *
    * @param {Node} rootNode

package/core/lib/lantern/simulator/simulator.d.ts

@@ -10,7 +10,7 @@ export class Simulator<T = any> {
     /**
      * @param {Lantern.Simulation.Settings} settings
      */
-    static createSimulator(settings: Lantern.Simulation.Settings): Promise<Simulator<any>>;
+    static createSimulator(settings: Lantern.Simulation.Settings): Simulator<any>;
     /** @return {Map<string, Map<Node, CompleteNodeTiming>>} */
     static get ALL_NODE_TIMINGS(): Map<string, Map<Node, CompleteNodeTiming>>;
     /**

package/core/lib/lantern/simulator/simulator.js

@@ -53,7 +53,7 @@ class Simulator {
   /**
    * @param {Lantern.Simulation.Settings} settings
    */
-  static async createSimulator(settings) {
+  static createSimulator(settings) {
     const {throttlingMethod, throttling, precomputedLanternData, networkAnalysis} = settings;
 
     /** @type {Lantern.Simulation.Options} */

package/core/lib/lantern/types/lantern.d.ts

@@ -67,11 +67,14 @@ export class NetworkRequest<T = any> {
     resourceSize: number;
     fromDiskCache: boolean;
     fromMemoryCache: boolean;
+    isLinkPreload: boolean;
     finished: boolean;
+    failed: boolean;
     statusCode: number;
+    /** The network request that redirected to this one */
+    redirectSource: NetworkRequest<T> | undefined;
     /** The network request that this one redirected to */
     redirectDestination: NetworkRequest<T> | undefined;
-    failed: boolean;
     initiator: LH.Crdp.Network.Initiator;
     initiatorRequest: NetworkRequest<T> | undefined;
     /** The chain of network requests that redirected to this one */
@@ -81,12 +84,12 @@ export class NetworkRequest<T = any> {
      * Optional value for how long the server took to respond to this request.
      * When not provided, the server response time is derived from the timing object.
      */
-    serverResponseTime: number | undefined;
+    serverResponseTime?: number;
     resourceType: LH.Crdp.Network.ResourceType | undefined;
     mimeType: string;
     priority: LH.Crdp.Network.ResourcePriority;
     frameId: string | undefined;
-    sessionTargetType: LH.Protocol.TargetType | undefined;
+    fromWorker: boolean;
 }
 
 export namespace Simulation {

package/core/lib/network-request.d.ts

@@ -149,6 +149,7 @@ export class NetworkRequest {
     sessionId: string | undefined;
     /** @type {LH.Protocol.TargetType|undefined} */
     sessionTargetType: LH.Protocol.TargetType | undefined;
+    fromWorker: boolean;
     isLinkPreload: boolean;
     /**
      * @return {boolean}
@@ -258,6 +259,26 @@ export namespace NetworkRequest {
     export { HEADER_FETCHED_SIZE };
     export { HEADER_PROTOCOL_IS_H2 };
 }
+/**
+ * @typedef HeaderEntry
+ * @property {string} name
+ * @property {string} value
+ */
+/**
+ * @typedef ParsedURL
+ * @property {string} scheme Equivalent to a `new URL(url).protocol` BUT w/o the trailing colon (:)
+ * @property {string} host Equivalent to a `new URL(url).hostname`
+ * @property {string} securityOrigin
+ */
+/**
+ * @typedef LightriderStatistics
+ * @property {number} endTimeDeltaMs The difference in networkEndTime between the observed Lighthouse networkEndTime and Lightrider's derived networkEndTime.
+ * @property {number} TCPMs The time spent making a TCP connection (connect + SSL). Note: this is poorly named.
+ * @property {number} requestMs The time spent requesting a resource from a remote server, we use this to approx RTT. Note: this is poorly names, it really should be "server response time".
+ * @property {number} responseMs Time to receive the entire response payload starting the clock on receiving the first fragment (first non-header byte).
+ */
+/** @type {LH.Util.SelfMap<LH.Crdp.Network.ResourceType>} */
+export const RESOURCE_TYPES: LH.Util.SelfMap<LH.Crdp.Network.ResourceType>;
 import * as LH from '../../types/lh.js';
 import * as Lantern from './lantern/types/lantern.js';
 declare const HEADER_TCP: "X-TCPMs";

package/core/lib/network-request.js

@@ -180,6 +180,7 @@ class NetworkRequest {
     this.sessionId = undefined;
     /** @type {LH.Protocol.TargetType|undefined} */
     this.sessionTargetType = undefined;
+    this.fromWorker = false;
     this.isLinkPreload = false;
   }
 
@@ -600,6 +601,8 @@ class NetworkRequest {
       }
     }
 
+    record.fromWorker = record.sessionTargetType === 'worker';
+
     return {
       ...record,
       timing,
@@ -691,4 +694,4 @@ NetworkRequest.HEADER_TOTAL = HEADER_TOTAL;
 NetworkRequest.HEADER_FETCHED_SIZE = HEADER_FETCHED_SIZE;
 NetworkRequest.HEADER_PROTOCOL_IS_H2 = HEADER_PROTOCOL_IS_H2;
 
-export {NetworkRequest};
+export {NetworkRequest, RESOURCE_TYPES};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lighthouse",
   "type": "module",
-  "version": "12.0.0-dev.20240602",
+  "version": "12.0.0-dev.20240603",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./core/index.js",
   "bin": {
@@ -51,17 +51,19 @@
     "test-proto": "yarn compile-proto && yarn build-proto-roundtrip",
     "unit-core": "yarn mocha core",
     "unit-cli": "yarn mocha --testMatch cli/**/*-test.js",
+    "unit-lantern-trace": "INTERNAL_LANTERN_USE_TRACE=1 yarn mocha core/test/computed/metrics core/test/audits",
     "unit-report": "yarn mocha --testMatch report/**/*-test.js",
     "unit-treemap": "yarn mocha --testMatch treemap/**/*-test.js",
     "unit-viewer": "yarn mocha --testMatch viewer/**/*-test.js",
     "unit-flow": "bash flow-report/test/run-flow-report-tests.sh",
-    "unit": "yarn unit-flow && yarn mocha",
+    "unit": "yarn unit-flow && yarn mocha && yarn unit-lantern-trace",
     "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit",
+    "unit-lantern-trace:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit-lantern-trace",
     "core-unit": "yarn unit-core",
     "cli-unit": "yarn unit-cli",
     "viewer-unit": "yarn unit-viewer",
     "watch": "yarn unit-core --watch",
-    "unit:cicoverage": "yarn c8 --all yarn unit:ci",
+    "unit:cicoverage": "yarn c8 --all yarn unit:ci && yarn c8 --all yarn unit-lantern-trace:ci",
     "coverage": "yarn unit:cicoverage && c8 report --reporter html",
     "coverage:smoke": "yarn c8 yarn smoke -j=1 && c8 report --reporter html",
     "devtools": "bash core/scripts/roll-to-devtools.sh",
@@ -179,7 +181,7 @@
     "webtreemap-cdt": "^3.2.1"
   },
   "dependencies": {
-    "@paulirish/trace_engine": "^0.0.19",
+    "@paulirish/trace_engine": "^0.0.23",
     "@sentry/node": "^6.17.4",
     "axe-core": "^4.9.0",
     "chrome-launcher": "^1.1.1",

package/tsconfig.json

@@ -109,5 +109,6 @@
     "core/test/computed/metrics/interactive-test.js",
     "core/test/computed/tbt-impact-tasks-test.js",
     "core/test/fixtures/config-plugins/lighthouse-plugin-simple/plugin-simple.js",
+    "core/test/lib/lantern/metrics/metric-test-utils.js",
   ],
 }

package/types/artifacts.d.ts

@@ -942,6 +942,22 @@ export interface TraceEvent {
       name?: string;
       duration?: number;
       blockingDuration?: number;
+      candidateIndex?: number;
+      priority?: string;
+      requestMethod?: string;
+      resourceType?: string;
+      fromCache?: boolean;
+      fromServiceWorker?: boolean;
+      mimeType?: string;
+      statusCode?: number;
+      timing?: any;
+      connectionId?: number;
+      connectionReused?: boolean;
+      encodedDataLength?: number;
+      decodedBodyLength?: number;
+      initiator?: {type: string, url?: string, stack?: any};
+      protocol?: string;
+      finishTime?: number;
     };
     frame?: string;
     name?: string;