lighthouse 9.5.0-dev.20220607 → 9.6.2

package/lighthouse-core/computed/page-dependency-graph.js

@@ -8,14 +8,13 @@
 const makeComputedArtifact = require('./computed-artifact.js');
 const NetworkNode = require('../lib/dependency-graph/network-node.js');
 const CPUNode = require('../lib/dependency-graph/cpu-node.js');
+const NetworkAnalyzer = require('../lib/dependency-graph/simulator/network-analyzer.js');
 const TracingProcessor = require('../lib/tracehouse/trace-processor.js');
 const NetworkRequest = require('../lib/network-request.js');
 const ProcessedTrace = require('./processed-trace.js');
 const NetworkRecords = require('./network-records.js');
-const NetworkAnalyzer = require('../lib/dependency-graph/simulator/network-analyzer.js');
 
 /** @typedef {import('../lib/dependency-graph/base-node.js').Node} Node */
-/** @typedef {Omit<LH.Artifacts['URL'], 'initialUrl'|'finalUrl'>} URLArtifact */
 
 // Shorter tasks have negligible impact on simulation results.
 const SIGNIFICANT_DUR_THRESHOLD_MS = 10;
@@ -389,25 +388,34 @@ class PageDependencyGraph {
   /**
    * @param {LH.Artifacts.ProcessedTrace} processedTrace
    * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
-   * @param {URLArtifact} URL
    * @return {Node}
    */
-  static createGraph(processedTrace, networkRecords, URL) {
+  static createGraph(processedTrace, networkRecords) {
     const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
     const cpuNodes = PageDependencyGraph.getCPUNodes(processedTrace);
-    const {requestedUrl, mainDocumentUrl} = URL;
-    if (!requestedUrl) throw new Error('requestedUrl is required to get the root request');
-    if (!mainDocumentUrl) throw new Error('mainDocumentUrl is required to get the main resource');
-
-    const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl);
-    if (!rootRequest) throw new Error('rootRequest not found');
-    const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
-    if (!rootNode) throw new Error('rootNode not found');
 
-    const mainDocumentRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, mainDocumentUrl);
-    if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found');
+    // The main document request is the earliest network request *of type document*.
+    // This will be different from the root request when there are server redirects.
+    const mainDocumentRequest = NetworkAnalyzer.findMainDocument(networkRecords);
     const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId);
-    if (!mainDocumentNode) throw new Error('mainDocumentNode not found');
+    if (!mainDocumentNode) {
+      // mainDocumentNode should always be found.
+      throw new Error('mainDocumentNode not found.');
+    }
+
+    // The root request is the earliest request in the main document redirect chain.
+    // Will be undefined if there were no redirects.
+    const rootRequest = mainDocumentNode.record.redirects?.[0];
+
+    let rootNode;
+    if (rootRequest) {
+      // rootNode should always be found.
+      rootNode = rootRequest && networkNodeOutput.idToNodeMap.get(rootRequest.requestId);
+      if (!rootNode) throw new Error('rootNode not found');
+    } else {
+      // Use main document as root if there were no redirects.
+      rootNode = mainDocumentNode;
+    }
 
     PageDependencyGraph.linkNetworkNodes(rootNode, networkNodeOutput);
     PageDependencyGraph.linkCPUNodes(rootNode, networkNodeOutput, cpuNodes);
@@ -453,58 +461,23 @@ class PageDependencyGraph {
   }
 
   /**
-   * Recalculate `artifacts.URL` for clients that don't provide it.
-   *
-   * @param {LH.DevtoolsLog} devtoolsLog
-   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
-   * @param {LH.Artifacts.ProcessedTrace} processedTrace
-   * @return {URLArtifact}
-   */
-  static getDocumentUrls(devtoolsLog, networkRecords, processedTrace) {
-    const mainFrameId = processedTrace.mainFrameIds.frameId;
-
-    /** @type {string|undefined} */
-    let requestedUrl;
-    /** @type {string|undefined} */
-    let mainDocumentUrl;
-    for (const event of devtoolsLog) {
-      if (event.method === 'Page.frameNavigated' && event.params.frame.id === mainFrameId) {
-        const {url} = event.params.frame;
-        // Only set requestedUrl on the first main frame navigation.
-        if (!requestedUrl) requestedUrl = url;
-        mainDocumentUrl = url;
-      }
-    }
-    if (!requestedUrl || !mainDocumentUrl) throw new Error('No main frame navigations found');
-
-    const initialRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl);
-    if (initialRequest?.redirects?.length) requestedUrl = initialRequest.redirects[0].url;
-
-    return {requestedUrl, mainDocumentUrl};
-  }
-
-  /**
-   * @param {{trace: LH.Trace, devtoolsLog: LH.DevtoolsLog, URL: LH.Artifacts['URL']}} data
+   * @param {{trace: LH.Trace, devtoolsLog: LH.DevtoolsLog}} data
    * @param {LH.Artifacts.ComputedContext} context
    * @return {Promise<Node>}
    */
   static async compute_(data, context) {
-    const {trace, devtoolsLog} = data;
+    const trace = data.trace;
+    const devtoolsLog = data.devtoolsLog;
     const [processedTrace, networkRecords] = await Promise.all([
       ProcessedTrace.request(trace, context),
       NetworkRecords.request(devtoolsLog, context),
     ]);
 
-    // COMPAT: Backport for pre-10.0 clients that don't pass the URL artifact here (e.g. pubads).
-    // Calculates the URL artifact from the processed trace and DT log.
-    const URL = data.URL ||
-      PageDependencyGraph.getDocumentUrls(devtoolsLog, networkRecords, processedTrace);
-
-    return PageDependencyGraph.createGraph(processedTrace, networkRecords, URL);
+    return PageDependencyGraph.createGraph(processedTrace, networkRecords);
   }
 }
 
-module.exports = makeComputedArtifact(PageDependencyGraph, ['devtoolsLog', 'trace', 'URL']);
+module.exports = makeComputedArtifact(PageDependencyGraph, ['devtoolsLog', 'trace']);
 
 /**
  * @typedef {Object} NetworkNodeOutput

package/lighthouse-core/computed/resource-summary.js

@@ -35,11 +35,11 @@ class ResourceSummary {
 
   /**
    * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
-   * @param {LH.Artifacts.URL} URLArtifact
+   * @param {string} mainResourceURL
    * @param {ImmutableObject<LH.Budget[]|null>} budgets
    * @return {Record<LH.Budget.ResourceType, ResourceEntry>}
    */
-  static summarize(networkRecords, URLArtifact, budgets) {
+  static summarize(networkRecords, mainResourceURL, budgets) {
     /** @type {Record<LH.Budget.ResourceType, ResourceEntry>} */
     const resourceSummary = {
       'stylesheet': {count: 0, resourceSize: 0, transferSize: 0},
@@ -52,13 +52,13 @@ class ResourceSummary {
       'total': {count: 0, resourceSize: 0, transferSize: 0},
       'third-party': {count: 0, resourceSize: 0, transferSize: 0},
     };
-    const budget = Budget.getMatchingBudget(budgets, URLArtifact.mainDocumentUrl);
+    const budget = Budget.getMatchingBudget(budgets, mainResourceURL);
     /** @type {ReadonlyArray<string>} */
     let firstPartyHosts = [];
     if (budget?.options?.firstPartyHostnames) {
       firstPartyHosts = budget.options.firstPartyHostnames;
     } else {
-      const rootDomain = Util.getRootDomain(URLArtifact.finalUrl);
+      const rootDomain = Util.getRootDomain(mainResourceURL);
       firstPartyHosts = [`*.${rootDomain}`];
     }
 
@@ -107,7 +107,7 @@ class ResourceSummary {
    */
   static async compute_(data, context) {
     const networkRecords = await NetworkRecords.request(data.devtoolsLog, context);
-    return ResourceSummary.summarize(networkRecords, data.URL, data.budgets);
+    return ResourceSummary.summarize(networkRecords, data.URL.finalUrl, data.budgets);
   }
 }
 

package/lighthouse-core/computed/unused-css.js

@@ -120,11 +120,12 @@ class UnusedCSS {
 
   /**
    * @param {StyleSheetInfo} stylesheetInfo The stylesheetInfo object.
+   * @param {string} pageUrl The URL of the page, used to identify inline styles.
    * @return {LH.Audit.ByteEfficiencyItem}
    */
-  static mapSheetToResult(stylesheetInfo) {
+  static mapSheetToResult(stylesheetInfo, pageUrl) {
     let url = stylesheetInfo.header.sourceURL;
-    if (!url || stylesheetInfo.header.isInline) {
+    if (!url || url === pageUrl) {
       const contentPreview = UnusedCSS.determineContentPreview(stylesheetInfo.content);
       url = contentPreview;
     }
@@ -134,20 +135,20 @@ class UnusedCSS {
   }
 
   /**
-   * @param {{CSSUsage: LH.Artifacts['CSSUsage'], devtoolsLog: LH.DevtoolsLog}} data
+   * @param {{CSSUsage: LH.Artifacts['CSSUsage'], URL: LH.Artifacts['URL'], devtoolsLog: LH.DevtoolsLog}} data
    * @param {LH.Artifacts.ComputedContext} context
    * @return {Promise<LH.Audit.ByteEfficiencyItem[]>}
   */
   static async compute_(data, context) {
-    const {CSSUsage, devtoolsLog} = data;
+    const {CSSUsage, URL, devtoolsLog} = data;
     const networkRecords = await NetworkRecords.request(devtoolsLog, context);
     const indexedSheets = UnusedCSS.indexStylesheetsById(CSSUsage.stylesheets, networkRecords);
     UnusedCSS.indexUsedRules(CSSUsage.rules, indexedSheets);
 
     const items = Object.keys(indexedSheets)
-      .map(sheetId => UnusedCSS.mapSheetToResult(indexedSheets[sheetId]));
+      .map(sheetId => UnusedCSS.mapSheetToResult(indexedSheets[sheetId], URL.finalUrl));
     return items;
   }
 }
 
-module.exports = makeComputedArtifact(UnusedCSS, ['CSSUsage', 'devtoolsLog']);
+module.exports = makeComputedArtifact(UnusedCSS, ['CSSUsage', 'URL', 'devtoolsLog']);

package/lighthouse-core/computed/unused-javascript-summary.js

@@ -16,14 +16,14 @@ const makeComputedArtifact = require('./computed-artifact.js');
 
 /**
  * @typedef ComputeInput
- * @property {string} scriptId
- * @property {Omit<LH.Crdp.Profiler.ScriptCoverage, 'url'>} scriptCoverage
+ * @property {string} url
+ * @property {Array<Omit<LH.Crdp.Profiler.ScriptCoverage, 'url'>>} scriptCoverages
  * @property {LH.Artifacts.Bundle=} bundle
  */
 
 /**
  * @typedef Summary
- * @property {string} scriptId
+ * @property {string} url
  * @property {number} wastedBytes
  * @property {number} totalBytes
  * @property {number} wastedBytes
@@ -68,24 +68,43 @@ class UnusedJavascriptSummary {
   }
 
   /**
-   * @param {string} scriptId
-   * @param {WasteData} wasteData
+   * @param {string} url
+   * @param {ReturnType<typeof UnusedJavascriptSummary.determineLengths>} lengths
    * @return {Summary}
    */
-  static createItem(scriptId, wasteData) {
-    const wastedRatio = (wasteData.unusedLength / wasteData.contentLength) || 0;
-    const wastedBytes = Math.round(wasteData.contentLength * wastedRatio);
+  static createItem(url, lengths) {
+    const wastedRatio = (lengths.unused / lengths.content) || 0;
+    const wastedBytes = Math.round(lengths.content * wastedRatio);
 
     return {
-      scriptId,
-      totalBytes: wasteData.contentLength,
+      url,
+      totalBytes: lengths.content,
       wastedBytes,
       wastedPercent: 100 * wastedRatio,
     };
   }
 
   /**
-   * @param {WasteData} wasteData
+   * @param {WasteData[]} wasteData
+   */
+  static determineLengths(wasteData) {
+    let unused = 0;
+    let content = 0;
+    // TODO: this is right for multiple script tags in an HTML document,
+    // but may be wrong for multiple frames using the same script resource.
+    for (const usage of wasteData) {
+      unused += usage.unusedLength;
+      content += usage.contentLength;
+    }
+
+    return {
+      content,
+      unused,
+    };
+  }
+
+  /**
+   * @param {WasteData[]} wasteData
    * @param {LH.Artifacts.Bundle} bundle
    */
   static createSourceWastedBytes(wasteData, bundle) {
@@ -112,7 +131,7 @@ class UnusedJavascriptSummary {
         mapping.lastColumnNumber - 1 :
         lineLengths[mapping.lineNumber];
       for (let i = mapping.columnNumber; i <= lastColumnOfMapping; i++) {
-        if (wasteData.unusedByIndex[offset] === 1) {
+        if (wasteData.every(data => data.unusedByIndex[offset] === 1)) {
           const key = mapping.sourceURL || '(unmapped)';
           files[key] = (files[key] || 0) + 1;
         }
@@ -136,10 +155,11 @@ class UnusedJavascriptSummary {
    * @return {Promise<Summary>}
    */
   static async compute_(data) {
-    const {scriptId, scriptCoverage, bundle} = data;
+    const {url, scriptCoverages, bundle} = data;
 
-    const wasteData = UnusedJavascriptSummary.computeWaste(scriptCoverage);
-    const item = UnusedJavascriptSummary.createItem(scriptId, wasteData);
+    const wasteData = scriptCoverages.map(UnusedJavascriptSummary.computeWaste);
+    const lengths = UnusedJavascriptSummary.determineLengths(wasteData);
+    const item = UnusedJavascriptSummary.createItem(url, lengths);
     if (!bundle) return item;
 
     return {
@@ -151,5 +171,5 @@ class UnusedJavascriptSummary {
 
 module.exports = makeComputedArtifact(
   UnusedJavascriptSummary,
-  ['bundle', 'scriptCoverage', 'scriptId']
+  ['bundle', 'scriptCoverages', 'url']
 );

package/lighthouse-core/config/budget.js

@@ -137,11 +137,11 @@ class Budget {
    * If multiple budgets match based on thier 'path' property,
    * then the last-listed of those budgets is returned.
    * @param {Immutable<Array<LH.Budget>>|null} budgets
-   * @param {string|undefined} url
+   * @param {string} url
    * @return {Immutable<LH.Budget> | undefined} budget
    */
   static getMatchingBudget(budgets, url) {
-    if (budgets === null || url === undefined) return;
+    if (budgets === null) return;
 
     // Applies the LAST matching budget.
     for (let i = budgets.length - 1; i >= 0; i--) {

package/lighthouse-core/config/default-config.js

@@ -41,7 +41,7 @@ const UIStrings = {
   /** Description of the Accessibility category. This is displayed at the top of a list of audits focused on making web content accessible to all users. No character length limits. 'improve the accessibility of your web app' becomes link text to additional documentation. */
   a11yCategoryDescription: 'These checks highlight opportunities to [improve the accessibility of your web app](https://web.dev/lighthouse-accessibility/). Only a subset of accessibility issues can be automatically detected so manual testing is also encouraged.',
   /** Description of the Accessibility manual checks category. This description is displayed above a list of accessibility audits that currently have no automated test and so must be verified manually by the user. No character length limits. 'conducting an accessibility review' becomes link text to additional documentation. */
-  a11yCategoryManualDescription: 'These items address areas which an automated testing tool cannot cover. Learn more in our guide on [conducting an accessibility review](https://web.dev/how-to-review/).',
+  a11yCategoryManualDescription: 'These items address areas which an automated testing tool cannot cover. Learn more in our guide on [conducting an accessibility review](https://developers.google.com/web/fundamentals/accessibility/how-to-review).',
   /** Title of the best practices section of the Accessibility category. Within this section are audits with descriptive titles that highlight common accessibility best practices. */
   a11yBestPracticesGroupTitle: 'Best practices',
   /** Description of the best practices section within the Accessibility category. Within this section are audits with descriptive titles that highlight common accessibility best practices. */
@@ -143,7 +143,6 @@ const defaultConfig = {
       'link-elements',
       'meta-elements',
       'script-elements',
-      'scripts',
       'iframe-elements',
       'inputs',
       'main-document-content',

package/lighthouse-core/fraggle-rock/api.js

@@ -13,7 +13,7 @@ const {generateFlowReportHtml} = require('../../report/generator/report-generato
 const Runner = require('../runner.js');
 
 /**
- * @param {LH.Puppeteer.Page} page
+ * @param {import('puppeteer').Page} page
  * @param {ConstructorParameters<LH.UserFlow>[1]} [options]
  */
 async function startFlow(page, options) {

package/lighthouse-core/fraggle-rock/config/default-config.js

@@ -70,7 +70,6 @@ const artifacts = {
   RobotsTxt: '',
   ServiceWorker: '',
   ScriptElements: '',
-  Scripts: '',
   SourceMaps: '',
   Stacks: '',
   TagsBlockingFirstPaint: '',
@@ -120,7 +119,6 @@ const defaultConfig = {
     {id: artifacts.RobotsTxt, gatherer: 'seo/robots-txt'},
     {id: artifacts.ServiceWorker, gatherer: 'service-worker'},
     {id: artifacts.ScriptElements, gatherer: 'script-elements'},
-    {id: artifacts.Scripts, gatherer: 'scripts'},
     {id: artifacts.SourceMaps, gatherer: 'source-maps'},
     {id: artifacts.Stacks, gatherer: 'stacks'},
     {id: artifacts.TagsBlockingFirstPaint, gatherer: 'dobetterweb/tags-blocking-first-paint'},
@@ -175,7 +173,6 @@ const defaultConfig = {
         artifacts.RobotsTxt,
         artifacts.ServiceWorker,
         artifacts.ScriptElements,
-        artifacts.Scripts,
         artifacts.SourceMaps,
         artifacts.Stacks,
         artifacts.TagsBlockingFirstPaint,

package/lighthouse-core/fraggle-rock/gather/base-artifacts.js

@@ -35,10 +35,7 @@ async function getBaseArtifacts(config, driver, context) {
     HostFormFactor: userAgent.includes('Android') || userAgent.includes('Mobile') ?
       'mobile' : 'desktop',
     // Contextual artifacts whose collection changes based on gather mode.
-    URL: {
-      initialUrl: '',
-      finalUrl: '',
-    },
+    URL: {requestedUrl: '', finalUrl: ''},
     PageLoadError: null,
     GatherContext: context,
     // Artifacts that have been replaced by regular gatherers in Fraggle Rock.
@@ -86,11 +83,11 @@ function finalizeArtifacts(baseArtifacts, gathererArtifacts) {
   artifacts.LighthouseRunWarnings = deduplicateWarnings(warnings);
 
   if (artifacts.PageLoadError && !artifacts.URL.finalUrl) {
-    artifacts.URL.finalUrl = artifacts.URL.requestedUrl || artifacts.URL.initialUrl;
+    artifacts.URL.finalUrl = artifacts.URL.requestedUrl;
   }
 
   // Check that the runner remembered to mutate the special-case URL artifact.
-  if (!artifacts.URL.initialUrl) throw new Error('Runner did not set initialUrl');
+  if (!artifacts.URL.requestedUrl) throw new Error('Runner did not set requestedUrl');
   if (!artifacts.URL.finalUrl) throw new Error('Runner did not set finalUrl');
 
   return artifacts;

package/lighthouse-core/fraggle-rock/gather/driver.js

@@ -35,7 +35,7 @@ const defaultSession = {
 /** @implements {LH.Gatherer.FRTransitionalDriver} */
 class Driver {
   /**
-   * @param {LH.Puppeteer.Page} page
+   * @param {import('puppeteer').Page} page
    */
   constructor(page) {
     this._page = page;
@@ -74,7 +74,7 @@ class Driver {
     const session = await this._page.target().createCDPSession();
     this._session = this.defaultSession = new ProtocolSession(session);
     this._executionContext = new ExecutionContext(this._session);
-    this._fetcher = new Fetcher(this._session);
+    this._fetcher = new Fetcher(this._session, this._executionContext);
     log.timeEnd(status);
   }
 

package/lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -5,7 +5,6 @@
  */
 'use strict';
 
-const puppeteer = require('puppeteer-core');
 const log = require('lighthouse-logger');
 const Driver = require('./driver.js');
 const Runner = require('../../runner.js');
@@ -44,20 +43,18 @@ const NetworkRecords = require('../../computed/network-records.js');
 
 /** @typedef {Omit<Parameters<typeof collectPhaseArtifacts>[0], 'phase'>} PhaseState */
 
-const DEFAULT_HOSTNAME = '127.0.0.1';
-const DEFAULT_PORT = 9222;
-
 /**
- * @param {{driver: Driver, config: LH.Config.FRConfig, options?: InternalOptions}} args
+ * @param {{driver: Driver, config: LH.Config.FRConfig, requestor: LH.NavigationRequestor, options?: InternalOptions}} args
  * @return {Promise<{baseArtifacts: LH.FRBaseArtifacts}>}
  */
-async function _setup({driver, config, options}) {
+async function _setup({driver, config, requestor, options}) {
   await driver.connect();
   if (!options?.skipAboutBlank) {
     await gotoURL(driver, defaultNavigationConfig.blankPage, {waitUntil: ['navigated']});
   }
 
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'navigation'});
+  if (typeof requestor === 'string') baseArtifacts.URL.requestedUrl = requestor;
 
   await prepare.prepareTargetForNavigationMode(driver, config.settings);
 
@@ -93,32 +90,25 @@ async function _cleanupNavigation({driver}) {
 
 /**
  * @param {NavigationContext} navigationContext
- * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
  */
 async function _navigate(navigationContext) {
   const {driver, config, requestor} = navigationContext;
 
   try {
-    const {requestedUrl, mainDocumentUrl, warnings} = await gotoURL(driver, requestor, {
+    const {requestedUrl, finalUrl, warnings} = await gotoURL(driver, requestor, {
       ...navigationContext.navigation,
       debugNavigation: config.settings.debugNavigation,
       maxWaitForFcp: config.settings.maxWaitForFcp,
       maxWaitForLoad: config.settings.maxWaitForLoad,
       waitUntil: navigationContext.navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
     });
-    return {requestedUrl, mainDocumentUrl, navigationError: undefined, warnings};
+    return {requestedUrl, finalUrl, navigationError: undefined, warnings};
   } catch (err) {
     if (!(err instanceof LighthouseError)) throw err;
     if (err.code !== 'NO_FCP' && err.code !== 'PAGE_HUNG') throw err;
     if (typeof requestor !== 'string') throw err;
-
-    // TODO: Make the urls optional here so we don't need to throw an error with a callback requestor.
-    return {
-      requestedUrl: requestor,
-      mainDocumentUrl: requestor,
-      navigationError: err,
-      warnings: [],
-    };
+    return {requestedUrl: requestor, finalUrl: requestor, navigationError: err, warnings: []};
   }
 }
 
@@ -162,7 +152,7 @@ async function _collectDebugData(navigationContext, phaseState) {
  * @param {PhaseState} phaseState
  * @param {Awaited<ReturnType<typeof _setupNavigation>>} setupResult
  * @param {Awaited<ReturnType<typeof _navigate>>} navigateResult
- * @return {Promise<{artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
  */
 async function _computeNavigationResult(
   navigationContext,
@@ -170,12 +160,12 @@ async function _computeNavigationResult(
   setupResult,
   navigateResult
 ) {
-  const {navigationError, mainDocumentUrl} = navigateResult;
+  const {navigationError, finalUrl} = navigateResult;
   const warnings = [...setupResult.warnings, ...navigateResult.warnings];
   const debugData = await _collectDebugData(navigationContext, phaseState);
   const pageLoadError = debugData.records
     ? getPageLoadError(navigationError, {
-      url: mainDocumentUrl,
+      url: finalUrl,
       loadFailureMode: navigationContext.navigation.loadFailureMode,
       networkRecords: debugData.records,
     })
@@ -193,6 +183,8 @@ async function _computeNavigationResult(
     if (debugData.trace) artifacts.traces = {[pageLoadErrorId]: debugData.trace};
 
     return {
+      requestedUrl: navigateResult.requestedUrl,
+      finalUrl,
       pageLoadError,
       artifacts,
       warnings: [...warnings, pageLoadError.friendlyMessage],
@@ -202,6 +194,8 @@ async function _computeNavigationResult(
 
     const artifacts = await awaitArtifacts(phaseState.artifactState);
     return {
+      requestedUrl: navigateResult.requestedUrl,
+      finalUrl,
       artifacts,
       warnings,
       pageLoadError: undefined,
@@ -215,9 +209,8 @@ async function _computeNavigationResult(
  */
 async function _navigation(navigationContext) {
   const artifactState = getEmptyArtifactState();
-  const initialUrl = await navigationContext.driver.url();
   const phaseState = {
-    url: initialUrl,
+    url: await navigationContext.driver.url(),
     gatherMode: /** @type {const} */ ('navigation'),
     driver: navigationContext.driver,
     computedCache: navigationContext.computedCache,
@@ -231,19 +224,7 @@ async function _navigation(navigationContext) {
   await collectPhaseArtifacts({phase: 'startInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'startSensitiveInstrumentation', ...phaseState});
   const navigateResult = await _navigate(navigationContext);
-
-  // Every required url is initialized to an empty string in `getBaseArtifacts`.
-  // If we haven't set all the required urls yet, set them here.
-  if (!Object.values(phaseState.baseArtifacts).every(Boolean)) {
-    phaseState.baseArtifacts.URL = {
-      initialUrl,
-      requestedUrl: navigateResult.requestedUrl,
-      mainDocumentUrl: navigateResult.mainDocumentUrl,
-      finalUrl: navigateResult.mainDocumentUrl,
-    };
-  }
-  phaseState.url = navigateResult.mainDocumentUrl;
-
+  phaseState.url = navigateResult.finalUrl;
   await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseState});
   await _cleanupNavigation(navigationContext);
@@ -281,6 +262,11 @@ async function _navigations({driver, config, requestor, baseArtifacts, computedC
         artifacts.PageLoadError = navigationResult.pageLoadError;
         shouldHaltNavigations = true;
       }
+
+      artifacts.URL = {
+        requestedUrl: navigationResult.requestedUrl,
+        finalUrl: navigationResult.finalUrl,
+      };
     }
 
     LighthouseRunWarnings.push(...navigationResult.warnings);
@@ -302,12 +288,12 @@ async function _cleanup({requestedUrl, driver, config}) {
 }
 
 /**
- * @param {LH.NavigationRequestor|undefined} requestor
- * @param {{page?: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ * @param {LH.NavigationRequestor} requestor
+ * @param {{page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<LH.Gatherer.FRGatherResult>}
  */
 async function navigationGather(requestor, options) {
-  const {configContext = {}} = options;
+  const {page, configContext = {}} = options;
   log.setLevel(configContext.logLevel || 'error');
 
   const {config} = initializeConfig(options.config, {...configContext, gatherMode: 'navigation'});
@@ -317,29 +303,18 @@ async function navigationGather(requestor, options) {
   };
 
   // We can't trigger the navigation through user interaction if we reset the page before starting.
-  const isCallback = typeof requestor === 'function';
-  if (isCallback) {
+  if (typeof requestor !== 'string') {
     internalOptions.skipAboutBlank = true;
   }
 
   const runnerOptions = {config, computedCache};
   const artifacts = await Runner.gather(
     async () => {
-      let {page} = options;
-
-      // For navigation mode, we shouldn't connect to a browser in audit mode,
-      // therefore we connect to the browser in the gatherFn callback.
-      if (!page) {
-        const {hostname = DEFAULT_HOSTNAME, port = DEFAULT_PORT} = configContext;
-        const browser = await puppeteer.connect({browserURL: `http://${hostname}:${port}`});
-        page = await browser.newPage();
-      }
-
       const driver = new Driver(page);
       const context = {
         driver,
         config,
-        requestor: isCallback ? requestor : URL.normalizeUrl(requestor),
+        requestor: typeof requestor === 'string' ? URL.normalizeUrl(requestor) : requestor,
         options: internalOptions,
       };
       const {baseArtifacts} = await _setup(context);

package/lighthouse-core/fraggle-rock/gather/runner-helpers.js

@@ -15,6 +15,7 @@
  * @property {LH.Gatherer.GatherMode} gatherMode
  * @property {Map<string, LH.ArbitraryEqualityMap>} computedCache
  * @property {LH.Config.Settings} settings
+ * @property {string} url
  */
 
 /** @typedef {Record<string, Promise<any>>} IntermediateArtifacts */
@@ -74,6 +75,7 @@ async function collectPhaseArtifacts(options) {
     gatherMode,
     computedCache,
     settings,
+    url,
   } = options;
   const priorPhase = phaseToPriorPhase[phase];
   const priorPhaseArtifacts = (priorPhase && artifactState[priorPhase]) || {};
@@ -90,6 +92,7 @@ async function collectPhaseArtifacts(options) {
         : /** @type {Dependencies} */ ({});
 
       return gatherer[phase]({
+        url,
         gatherMode,
         driver,
         baseArtifacts,