lighthouse 9.5.0-dev.20220328 → 9.5.0-dev.20220331

package/lighthouse-core/computed/page-dependency-graph.js

@@ -8,13 +8,14 @@
 const makeComputedArtifact = require('./computed-artifact.js');
 const NetworkNode = require('../lib/dependency-graph/network-node.js');
 const CPUNode = require('../lib/dependency-graph/cpu-node.js');
-const NetworkAnalyzer = require('../lib/dependency-graph/simulator/network-analyzer.js');
 const TracingProcessor = require('../lib/tracehouse/trace-processor.js');
 const NetworkRequest = require('../lib/network-request.js');
 const ProcessedTrace = require('./processed-trace.js');
 const NetworkRecords = require('./network-records.js');
+const NetworkAnalyzer = require('../lib/dependency-graph/simulator/network-analyzer.js');
 
 /** @typedef {import('../lib/dependency-graph/base-node.js').Node} Node */
+/** @typedef {Omit<LH.Artifacts['URL'], 'initialUrl'|'finalUrl'>} URLArtifact */
 
 // Shorter tasks have negligible impact on simulation results.
 const SIGNIFICANT_DUR_THRESHOLD_MS = 10;
@@ -388,37 +389,24 @@ class PageDependencyGraph {
   /**
    * @param {LH.Artifacts.ProcessedTrace} processedTrace
    * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
+   * @param {URLArtifact} URL
    * @return {Node}
    */
-  static createGraph(processedTrace, networkRecords) {
+  static createGraph(processedTrace, networkRecords, URL) {
     const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
     const cpuNodes = PageDependencyGraph.getCPUNodes(processedTrace);
+    const {requestedUrl, mainDocumentUrl} = URL;
+    if (!mainDocumentUrl) throw new Error('mainDocumentUrl is required to get the main resource');
 
-    // TODO: Remove this usage of `findMainDocument` and create a new function to get the first document request.
-    // https://github.com/GoogleChrome/lighthouse/issues/8984
-    //
-    // The main document request is the earliest network request *of type document*.
-    // This will be different from the root request when there are server redirects.
-    const mainDocumentRequest = NetworkAnalyzer.findMainDocument(networkRecords);
-    const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId);
-    if (!mainDocumentNode) {
-      // mainDocumentNode should always be found.
-      throw new Error('mainDocumentNode not found.');
-    }
+    const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl);
+    if (!rootRequest) throw new Error('rootRequest not found');
+    const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
+    if (!rootNode) throw new Error('rootNode not found');
 
-    // The root request is the earliest request in the main document redirect chain.
-    // Will be undefined if there were no redirects.
-    const rootRequest = mainDocumentNode.record.redirects?.[0];
-
-    let rootNode;
-    if (rootRequest) {
-      // rootNode should always be found.
-      rootNode = rootRequest && networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
-      if (!rootNode) throw new Error('rootNode not found');
-    } else {
-      // Use main document as root if there were no redirects.
-      rootNode = mainDocumentNode;
-    }
+    const mainDocumentRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, mainDocumentUrl);
+    if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found');
+    const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId);
+    if (!mainDocumentNode) throw new Error('mainDocumentNode not found');
 
     PageDependencyGraph.linkNetworkNodes(rootNode, networkNodeOutput);
     PageDependencyGraph.linkCPUNodes(rootNode, networkNodeOutput, cpuNodes);
@@ -464,23 +452,58 @@ class PageDependencyGraph {
   }
 
   /**
-   * @param {{trace: LH.Trace, devtoolsLog: LH.DevtoolsLog}} data
+   * Recalculate `artifacts.URL` for clients that don't provide it.
+   *
+   * @param {LH.DevtoolsLog} devtoolsLog
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
+   * @param {LH.Artifacts.ProcessedTrace} processedTrace
+   * @return {URLArtifact}
+   */
+  static getDocumentUrls(devtoolsLog, networkRecords, processedTrace) {
+    const mainFrameId = processedTrace.mainFrameIds.frameId;
+
+    /** @type {string|undefined} */
+    let requestedUrl;
+    /** @type {string|undefined} */
+    let mainDocumentUrl;
+    for (const event of devtoolsLog) {
+      if (event.method === 'Page.frameNavigated' && event.params.frame.id === mainFrameId) {
+        const {url} = event.params.frame;
+        // Only set requestedUrl on the first main frame navigation.
+        if (!requestedUrl) requestedUrl = url;
+        mainDocumentUrl = url;
+      }
+    }
+    if (!requestedUrl || !mainDocumentUrl) throw new Error('No main frame navigations found');
+
+    const initialRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl);
+    if (initialRequest?.redirects?.length) requestedUrl = initialRequest.redirects[0].url;
+
+    return {requestedUrl, mainDocumentUrl};
+  }
+
+  /**
+   * @param {{trace: LH.Trace, devtoolsLog: LH.DevtoolsLog, URL: LH.Artifacts['URL']}} data
    * @param {LH.Artifacts.ComputedContext} context
    * @return {Promise<Node>}
    */
   static async compute_(data, context) {
-    const trace = data.trace;
-    const devtoolsLog = data.devtoolsLog;
+    const {trace, devtoolsLog} = data;
     const [processedTrace, networkRecords] = await Promise.all([
       ProcessedTrace.request(trace, context),
       NetworkRecords.request(devtoolsLog, context),
     ]);
 
-    return PageDependencyGraph.createGraph(processedTrace, networkRecords);
+    // COMPAT: Backport for pre-10.0 clients that don't pass the URL artifact here (e.g. pubads).
+    // Calculates the URL artifact from the processed trace and DT log.
+    const URL = data.URL ||
+      PageDependencyGraph.getDocumentUrls(devtoolsLog, networkRecords, processedTrace);
+
+    return PageDependencyGraph.createGraph(processedTrace, networkRecords, URL);
   }
 }
 
-module.exports = makeComputedArtifact(PageDependencyGraph, ['devtoolsLog', 'trace']);
+module.exports = makeComputedArtifact(PageDependencyGraph, ['devtoolsLog', 'trace', 'URL']);
 
 /**
  * @typedef {Object} NetworkNodeOutput

package/lighthouse-core/fraggle-rock/gather/base-artifacts.js

@@ -35,7 +35,12 @@ async function getBaseArtifacts(config, driver, context) {
     HostFormFactor: userAgent.includes('Android') || userAgent.includes('Mobile') ?
       'mobile' : 'desktop',
     // Contextual artifacts whose collection changes based on gather mode.
-    URL: {requestedUrl: '', finalUrl: ''},
+    // TODO: Make `requestedUrl` optional in timespan and snapshot modes.
+    URL: {
+      initialUrl: '',
+      requestedUrl: '',
+      finalUrl: '',
+    },
     PageLoadError: null,
     GatherContext: context,
     // Artifacts that have been replaced by regular gatherers in Fraggle Rock.
@@ -87,6 +92,8 @@ function finalizeArtifacts(baseArtifacts, gathererArtifacts) {
   }
 
   // Check that the runner remembered to mutate the special-case URL artifact.
+  // TODO: Make `requestedUrl` optional.
+  if (!artifacts.URL.initialUrl) throw new Error('Runner did not set initialUrl');
   if (!artifacts.URL.requestedUrl) throw new Error('Runner did not set requestedUrl');
   if (!artifacts.URL.finalUrl) throw new Error('Runner did not set finalUrl');
 

package/lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -44,17 +44,16 @@ const NetworkRecords = require('../../computed/network-records.js');
 /** @typedef {Omit<Parameters<typeof collectPhaseArtifacts>[0], 'phase'>} PhaseState */
 
 /**
- * @param {{driver: Driver, config: LH.Config.FRConfig, requestor: LH.NavigationRequestor, options?: InternalOptions}} args
+ * @param {{driver: Driver, config: LH.Config.FRConfig, options?: InternalOptions}} args
  * @return {Promise<{baseArtifacts: LH.FRBaseArtifacts}>}
  */
-async function _setup({driver, config, requestor, options}) {
+async function _setup({driver, config, options}) {
   await driver.connect();
   if (!options?.skipAboutBlank) {
     await gotoURL(driver, defaultNavigationConfig.blankPage, {waitUntil: ['navigated']});
   }
 
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'navigation'});
-  if (typeof requestor === 'string') baseArtifacts.URL.requestedUrl = requestor;
 
   await prepare.prepareTargetForNavigationMode(driver, config.settings);
 
@@ -90,25 +89,32 @@ async function _cleanupNavigation({driver}) {
 
 /**
  * @param {NavigationContext} navigationContext
- * @return {Promise<{requestedUrl: string, finalUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
  */
 async function _navigate(navigationContext) {
   const {driver, config, requestor} = navigationContext;
 
   try {
-    const {requestedUrl, finalUrl, warnings} = await gotoURL(driver, requestor, {
+    const {requestedUrl, mainDocumentUrl, warnings} = await gotoURL(driver, requestor, {
       ...navigationContext.navigation,
       debugNavigation: config.settings.debugNavigation,
       maxWaitForFcp: config.settings.maxWaitForFcp,
       maxWaitForLoad: config.settings.maxWaitForLoad,
       waitUntil: navigationContext.navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
     });
-    return {requestedUrl, finalUrl, navigationError: undefined, warnings};
+    return {requestedUrl, mainDocumentUrl, navigationError: undefined, warnings};
   } catch (err) {
     if (!(err instanceof LighthouseError)) throw err;
     if (err.code !== 'NO_FCP' && err.code !== 'PAGE_HUNG') throw err;
     if (typeof requestor !== 'string') throw err;
-    return {requestedUrl: requestor, finalUrl: requestor, navigationError: err, warnings: []};
+
+    // TODO: Make the urls optional here so we don't need to throw an error with a callback requestor.
+    return {
+      requestedUrl: requestor,
+      mainDocumentUrl: requestor,
+      navigationError: err,
+      warnings: [],
+    };
   }
 }
 
@@ -152,7 +158,7 @@ async function _collectDebugData(navigationContext, phaseState) {
  * @param {PhaseState} phaseState
  * @param {Awaited<ReturnType<typeof _setupNavigation>>} setupResult
  * @param {Awaited<ReturnType<typeof _navigate>>} navigateResult
- * @return {Promise<{requestedUrl: string, finalUrl: string, artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
+ * @return {Promise<{artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
  */
 async function _computeNavigationResult(
   navigationContext,
@@ -160,12 +166,12 @@ async function _computeNavigationResult(
   setupResult,
   navigateResult
 ) {
-  const {navigationError, finalUrl} = navigateResult;
+  const {navigationError, mainDocumentUrl} = navigateResult;
   const warnings = [...setupResult.warnings, ...navigateResult.warnings];
   const debugData = await _collectDebugData(navigationContext, phaseState);
   const pageLoadError = debugData.records
     ? getPageLoadError(navigationError, {
-      url: finalUrl,
+      url: mainDocumentUrl,
       loadFailureMode: navigationContext.navigation.loadFailureMode,
       networkRecords: debugData.records,
     })
@@ -183,8 +189,6 @@ async function _computeNavigationResult(
     if (debugData.trace) artifacts.traces = {[pageLoadErrorId]: debugData.trace};
 
     return {
-      requestedUrl: navigateResult.requestedUrl,
-      finalUrl,
       pageLoadError,
       artifacts,
       warnings: [...warnings, pageLoadError.friendlyMessage],
@@ -194,8 +198,6 @@ async function _computeNavigationResult(
 
     const artifacts = await awaitArtifacts(phaseState.artifactState);
     return {
-      requestedUrl: navigateResult.requestedUrl,
-      finalUrl,
       artifacts,
       warnings,
       pageLoadError: undefined,
@@ -209,8 +211,9 @@ async function _computeNavigationResult(
  */
 async function _navigation(navigationContext) {
   const artifactState = getEmptyArtifactState();
+  const initialUrl = await navigationContext.driver.url();
   const phaseState = {
-    url: await navigationContext.driver.url(),
+    url: initialUrl,
     gatherMode: /** @type {const} */ ('navigation'),
     driver: navigationContext.driver,
     computedCache: navigationContext.computedCache,
@@ -224,8 +227,20 @@ async function _navigation(navigationContext) {
   await collectPhaseArtifacts({phase: 'startInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'startSensitiveInstrumentation', ...phaseState});
   const navigateResult = await _navigate(navigationContext);
-  phaseState.baseArtifacts.URL.finalUrl = navigateResult.finalUrl;
-  phaseState.url = navigateResult.finalUrl;
+
+  // Every required url is initialized to an empty string in `getBaseArtifacts`.
+  // If we haven't set the required urls yet, set them here.
+  const {URL} = phaseState.baseArtifacts;
+  if (!URL.finalUrl || !URL.initialUrl) {
+    phaseState.baseArtifacts.URL = {
+      initialUrl,
+      requestedUrl: navigateResult.requestedUrl,
+      mainDocumentUrl: navigateResult.mainDocumentUrl,
+      finalUrl: navigateResult.mainDocumentUrl,
+    };
+  }
+  phaseState.url = navigateResult.mainDocumentUrl;
+
   await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseState});
   await _cleanupNavigation(navigationContext);
@@ -263,11 +278,6 @@ async function _navigations({driver, config, requestor, baseArtifacts, computedC
         artifacts.PageLoadError = navigationResult.pageLoadError;
         shouldHaltNavigations = true;
       }
-
-      artifacts.URL = {
-        requestedUrl: navigationResult.requestedUrl,
-        finalUrl: navigationResult.finalUrl,
-      };
     }
 
     LighthouseRunWarnings.push(...navigationResult.warnings);

package/lighthouse-core/fraggle-rock/gather/runner-helpers.js

@@ -15,7 +15,6 @@
  * @property {LH.Gatherer.GatherMode} gatherMode
  * @property {Map<string, LH.ArbitraryEqualityMap>} computedCache
  * @property {LH.Config.Settings} settings
- * @property {string} url
  */
 
 /** @typedef {Record<string, Promise<any>>} IntermediateArtifacts */
@@ -75,7 +74,6 @@ async function collectPhaseArtifacts(options) {
     gatherMode,
     computedCache,
     settings,
-    url,
   } = options;
   const priorPhase = phaseToPriorPhase[phase];
   const priorPhaseArtifacts = (priorPhase && artifactState[priorPhase]) || {};
@@ -92,7 +90,6 @@ async function collectPhaseArtifacts(options) {
         : /** @type {Dependencies} */ ({});
 
       return gatherer[phase]({
-        url,
         gatherMode,
         driver,
         baseArtifacts,

package/lighthouse-core/fraggle-rock/gather/snapshot-runner.js

@@ -36,13 +36,16 @@ async function snapshotGather(options) {
   const artifacts = await Runner.gather(
     async () => {
       const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'snapshot'});
-      baseArtifacts.URL.requestedUrl = url;
-      baseArtifacts.URL.finalUrl = url;
+      baseArtifacts.URL = {
+        initialUrl: url,
+        // TODO: Remove `requestedUrl` from snapshot mode.
+        requestedUrl: url,
+        finalUrl: url,
+      };
 
       const artifactDefinitions = config.artifacts || [];
       const artifactState = getEmptyArtifactState();
       await collectPhaseArtifacts({
-        url,
         phase: 'getArtifact',
         gatherMode: 'snapshot',
         driver,

package/lighthouse-core/fraggle-rock/gather/timespan-runner.js

@@ -32,12 +32,11 @@ async function startTimespanGather(options) {
   /** @type {Map<string, LH.ArbitraryEqualityMap>} */
   const computedCache = new Map();
   const artifactDefinitions = config.artifacts || [];
-  const requestedUrl = await driver.url();
+  const initialUrl = await driver.url();
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'timespan'});
   const artifactState = getEmptyArtifactState();
   /** @type {Omit<import('./runner-helpers.js').CollectPhaseArtifactOptions, 'phase'>} */
   const phaseOptions = {
-    url: requestedUrl,
     driver,
     artifactDefinitions,
     artifactState,
@@ -54,13 +53,16 @@ async function startTimespanGather(options) {
   return {
     async endTimespanGather() {
       const finalUrl = await driver.url();
-      phaseOptions.url = finalUrl;
 
       const runnerOptions = {config, computedCache};
       const artifacts = await Runner.gather(
         async () => {
-          baseArtifacts.URL.requestedUrl = requestedUrl;
-          baseArtifacts.URL.finalUrl = finalUrl;
+          baseArtifacts.URL = {
+            initialUrl,
+            // TODO: Remove requestedUrl from timespan mode
+            requestedUrl: initialUrl,
+            finalUrl,
+          };
 
           await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseOptions});
           await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseOptions});

package/lighthouse-core/gather/connections/cri.js

@@ -88,6 +88,7 @@ class CriConnection extends Connection {
         path: '/json/' + command,
       }, response => {
         let data = '';
+        response.setEncoding('utf8');
         response.on('data', chunk => {
           data += chunk;
         });

package/lighthouse-core/gather/driver/execution-context.js

@@ -97,7 +97,8 @@ class ExecutionContext {
             .catch(${pageFunctions.wrapRuntimeEvalErrorInBrowserString})
             .then(resolve);
         });
-      }())`,
+      }())
+      //# sourceURL=_lighthouse-eval.js`,
       includeCommandLineAPI: true,
       awaitPromise: true,
       returnByValue: true,
@@ -195,7 +196,8 @@ class ExecutionContext {
       ${ExecutionContext._cachedNativesPreamble};
       ${depsSerialized};
       (${mainFn})(${argsSerialized});
-    })()`;
+    })()
+    //# sourceURL=_lighthouse-eval.js`;
 
     await this._session.sendCommand('Page.addScriptToEvaluateOnNewDocument', {source: expression});
   }

package/lighthouse-core/gather/driver/navigation.js

@@ -80,7 +80,7 @@ function resolveWaitForFullyLoadedOptions(options) {
  * @param {LH.Gatherer.FRTransitionalDriver} driver
  * @param {LH.NavigationRequestor} requestor
  * @param {NavigationOptions} options
- * @return {Promise<{requestedUrl: string, finalUrl: string, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, warnings: Array<LH.IcuMessage>}>}
  */
 async function gotoURL(driver, requestor, options) {
   const status = typeof requestor === 'string' ?
@@ -130,14 +130,14 @@ async function gotoURL(driver, requestor, options) {
 
   let requestedUrl = navigationUrls.requestedUrl;
   if (typeof requestor === 'string') {
-    if (requestor !== requestedUrl) {
+    if (requestedUrl && !URL.equalWithExcludedFragments(requestor, requestedUrl)) {
       log.error('Navigation', 'Provided URL did not match initial navigation URL');
     }
     requestedUrl = requestor;
   }
   if (!requestedUrl) throw Error('No navigations detected when running user defined requestor.');
 
-  const finalUrl = navigationUrls.finalUrl || requestedUrl;
+  const mainDocumentUrl = navigationUrls.mainDocumentUrl || requestedUrl;
 
   // Bring `Page.navigate` errors back into the promise chain. See https://github.com/GoogleChrome/lighthouse/pull/6739.
   await waitForNavigationTriggered;
@@ -150,26 +150,26 @@ async function gotoURL(driver, requestor, options) {
   log.timeEnd(status);
   return {
     requestedUrl,
-    finalUrl,
-    warnings: getNavigationWarnings({timedOut, finalUrl, requestedUrl}),
+    mainDocumentUrl,
+    warnings: getNavigationWarnings({timedOut, mainDocumentUrl, requestedUrl}),
   };
 }
 
 /**
- * @param {{timedOut: boolean, requestedUrl: string, finalUrl: string; }} navigation
+ * @param {{timedOut: boolean, requestedUrl: string, mainDocumentUrl: string; }} navigation
  * @return {Array<LH.IcuMessage>}
  */
 function getNavigationWarnings(navigation) {
-  const {requestedUrl, finalUrl} = navigation;
+  const {requestedUrl, mainDocumentUrl} = navigation;
   /** @type {Array<LH.IcuMessage>} */
   const warnings = [];
 
   if (navigation.timedOut) warnings.push(str_(UIStrings.warningTimeout));
 
-  if (!URL.equalWithExcludedFragments(requestedUrl, finalUrl)) {
+  if (!URL.equalWithExcludedFragments(requestedUrl, mainDocumentUrl)) {
     warnings.push(str_(UIStrings.warningRedirected, {
       requested: requestedUrl,
-      final: finalUrl,
+      final: mainDocumentUrl,
     }));
   }
 

package/lighthouse-core/gather/driver/network-monitor.js

@@ -133,7 +133,7 @@ class NetworkMonitor {
     this._sessions = new Map();
   }
 
-  /** @return {Promise<{requestedUrl?: string, finalUrl?: string}>} */
+  /** @return {Promise<{requestedUrl?: string, mainDocumentUrl?: string}>} */
   async getNavigationUrls() {
     const frameNavigations = this._frameNavigations;
     if (!frameNavigations.length) return {};
@@ -143,9 +143,22 @@ class NetworkMonitor {
     const mainFrameNavigations = frameNavigations.filter(frame => frame.id === mainFrameId);
     if (!mainFrameNavigations.length) log.warn('NetworkMonitor', 'No detected navigations');
 
+    // The requested URL is the initiator request for the first frame navigation.
+    /** @type {string|undefined} */
+    let requestedUrl = mainFrameNavigations[0]?.url;
+    if (this._networkRecorder) {
+      const records = this._networkRecorder.getRawRecords();
+
+      let initialUrlRequest = records.find(record => record.url === requestedUrl);
+      while (initialUrlRequest?.redirectSource) {
+        initialUrlRequest = initialUrlRequest.redirectSource;
+        requestedUrl = initialUrlRequest.url;
+      }
+    }
+
     return {
-      requestedUrl: mainFrameNavigations[0]?.url,
-      finalUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
+      requestedUrl,
+      mainDocumentUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
     };
   }
 

package/lighthouse-core/gather/driver.js

@@ -470,6 +470,11 @@ class Driver {
     this._devtoolsLog.endRecording();
     return this._devtoolsLog.messages;
   }
+
+  async url() {
+    const {frameTree} = await this.sendCommand('Page.getFrameTree');
+    return `${frameTree.frame.url}${frameTree.frame.urlFragment || ''}`;
+  }
 }
 
 module.exports = Driver;

package/lighthouse-core/gather/gather-runner.js

@@ -71,7 +71,7 @@ class GatherRunner {
     log.time(status);
     try {
       const requestedUrl = passContext.url;
-      const {finalUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
+      const {mainDocumentUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
         waitUntil: passContext.passConfig.recordTrace ?
           ['load', 'fcp'] : ['load'],
         debugNavigation: passContext.settings.debugNavigation,
@@ -79,8 +79,12 @@ class GatherRunner {
         maxWaitForLoad: passContext.settings.maxWaitForLoad,
         ...passContext.passConfig,
       });
-      passContext.url = finalUrl;
-      passContext.baseArtifacts.URL.finalUrl = finalUrl;
+      passContext.url = mainDocumentUrl;
+      const {URL} = passContext.baseArtifacts;
+      if (!URL.finalUrl || !URL.mainDocumentUrl) {
+        URL.finalUrl = mainDocumentUrl;
+        URL.mainDocumentUrl = mainDocumentUrl;
+      }
       if (passContext.passConfig.loadFailureMode === 'fatal') {
         passContext.LighthouseRunWarnings.push(...warnings);
       }
@@ -408,7 +412,12 @@ class GatherRunner {
       devtoolsLogs: {},
       settings: options.settings,
       GatherContext: {gatherMode: 'navigation'},
-      URL: {requestedUrl: options.requestedUrl, finalUrl: options.requestedUrl},
+      URL: {
+        initialUrl: await options.driver.url(),
+        requestedUrl: options.requestedUrl,
+        mainDocumentUrl: '',
+        finalUrl: '',
+      },
       Timing: [],
       PageLoadError: null,
     };
@@ -426,9 +435,6 @@ class GatherRunner {
 
     const baseArtifacts = passContext.baseArtifacts;
 
-    // Copy redirected URL to artifact.
-    baseArtifacts.URL.finalUrl = passContext.url;
-
     // Fetch the manifest, if it exists.
     try {
       baseArtifacts.WebAppManifest = await WebAppManifest.getWebAppManifest(

package/lighthouse-core/gather/gatherers/js-usage.js

@@ -50,12 +50,13 @@ class JsUsage extends FRGatherer {
     const usageByScriptId = {};
 
     for (const scriptUsage of this._scriptUsages) {
-      // If `url` is blank, that means the script was anonymous (eval, new Function, onload, ...).
-      // Or, it's because it was code Lighthouse over the protocol via `Runtime.evaluate`.
-      // We currently don't consider coverage of anonymous scripts, and we definitely don't want
-      // coverage of code Lighthouse ran to inspect the page, so we ignore this ScriptCoverage if
-      // url is blank.
-      if (scriptUsage.url === '') {
+      // If `url` is blank, that means the script was dynamically
+      // created (eval, new Function, onload, ...)
+      if (scriptUsage.url === '' || scriptUsage.url === '_lighthouse-eval.js') {
+        // We currently don't consider coverage of dynamic scripts, and we definitely don't want
+        // coverage of code Lighthouse ran to inspect the page, so we ignore this ScriptCoverage.
+        // Audits would work the same without this, it is only an optimization (not tracking coverage
+        // for scripts we don't care about).
         continue;
       }
 

package/lighthouse-core/gather/gatherers/scripts.js

@@ -28,6 +28,23 @@ async function runInSeriesOrParallel(values, promiseMapper, runInSeries) {
   }
 }
 
+/**
+ * Returns true if the script was created via our own calls
+ * to Runtime.evaluate.
+ * @param {LH.Crdp.Debugger.ScriptParsedEvent} script
+ */
+function isLighthouseRuntimeEvaluateScript(script) {
+  // Scripts created by Runtime.evaluate that run on the main session/frame
+  // result in an empty string for the embedderName.
+  // Or, it means the script was dynamically created (eval, new Function, onload, ...)
+  if (!script.embedderName) return true;
+
+  // Otherwise, when running our own code inside other frames, the embedderName
+  // is set to the frame's url. In that case, we rely on the special sourceURL that
+  // we set.
+  return script.hasSourceURL && script.url === '_lighthouse-eval.js';
+}
+
 /**
  * @fileoverview Gets JavaScript file contents.
  */
@@ -68,8 +85,9 @@ class Scripts extends FRGatherer {
     // it also blocks scripts from the same origin but that happen to run in a different process,
     // like a worker.
     if (event.method === 'Debugger.scriptParsed' && !sessionId) {
-      // Events without an embedderName (read: a url) are for JS that we ran over the protocol.
-      if (event.params.embedderName) this._scriptParsedEvents.push(event.params);
+      if (!isLighthouseRuntimeEvaluateScript(event.params)) {
+        this._scriptParsedEvents.push(event.params);
+      }
     }
   }
 

package/lighthouse-core/gather/gatherers/seo/robots-txt.js

@@ -46,7 +46,8 @@ class RobotsTxt extends FRGatherer {
       });
     }
 
-    const robotsUrl = new URL('/robots.txt', passContext.url).href;
+    const {finalUrl} = passContext.baseArtifacts.URL;
+    const robotsUrl = new URL('/robots.txt', finalUrl).href;
     await passContext.driver.fetcher.enable();
     return passContext.driver.fetcher.fetchResource(robotsUrl)
       .catch(err => ({status: null, content: null, errorMessage: err.message}));

package/lighthouse-core/gather/gatherers/web-app-manifest.js

@@ -94,8 +94,8 @@ class WebAppManifest extends FRGatherer {
    */
   getArtifact(context) {
     const driver = context.driver;
-
-    return WebAppManifest.getWebAppManifest(driver.defaultSession, context.url);
+    const {finalUrl} = context.baseArtifacts.URL;
+    return WebAppManifest.getWebAppManifest(driver.defaultSession, finalUrl);
   }
 }