lighthouse 9.5.0-dev.20220328 → 9.5.0-dev.20220329

package/lighthouse-core/fraggle-rock/gather/base-artifacts.js

@@ -35,7 +35,12 @@ async function getBaseArtifacts(config, driver, context) {
     HostFormFactor: userAgent.includes('Android') || userAgent.includes('Mobile') ?
       'mobile' : 'desktop',
     // Contextual artifacts whose collection changes based on gather mode.
-    URL: {requestedUrl: '', finalUrl: ''},
+    // TODO: Make `requestedUrl` optional in timespan and snapshot modes.
+    URL: {
+      initialUrl: '',
+      requestedUrl: '',
+      finalUrl: '',
+    },
     PageLoadError: null,
     GatherContext: context,
     // Artifacts that have been replaced by regular gatherers in Fraggle Rock.
@@ -87,6 +92,8 @@ function finalizeArtifacts(baseArtifacts, gathererArtifacts) {
   }
 
   // Check that the runner remembered to mutate the special-case URL artifact.
+  // TODO: Make `requestedUrl` optional.
+  if (!artifacts.URL.initialUrl) throw new Error('Runner did not set initialUrl');
   if (!artifacts.URL.requestedUrl) throw new Error('Runner did not set requestedUrl');
   if (!artifacts.URL.finalUrl) throw new Error('Runner did not set finalUrl');
 

package/lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -44,17 +44,16 @@ const NetworkRecords = require('../../computed/network-records.js');
 /** @typedef {Omit<Parameters<typeof collectPhaseArtifacts>[0], 'phase'>} PhaseState */
 
 /**
- * @param {{driver: Driver, config: LH.Config.FRConfig, requestor: LH.NavigationRequestor, options?: InternalOptions}} args
+ * @param {{driver: Driver, config: LH.Config.FRConfig, options?: InternalOptions}} args
  * @return {Promise<{baseArtifacts: LH.FRBaseArtifacts}>}
  */
-async function _setup({driver, config, requestor, options}) {
+async function _setup({driver, config, options}) {
   await driver.connect();
   if (!options?.skipAboutBlank) {
     await gotoURL(driver, defaultNavigationConfig.blankPage, {waitUntil: ['navigated']});
   }
 
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'navigation'});
-  if (typeof requestor === 'string') baseArtifacts.URL.requestedUrl = requestor;
 
   await prepare.prepareTargetForNavigationMode(driver, config.settings);
 
@@ -90,25 +89,32 @@ async function _cleanupNavigation({driver}) {
 
 /**
  * @param {NavigationContext} navigationContext
- * @return {Promise<{requestedUrl: string, finalUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
  */
 async function _navigate(navigationContext) {
   const {driver, config, requestor} = navigationContext;
 
   try {
-    const {requestedUrl, finalUrl, warnings} = await gotoURL(driver, requestor, {
+    const {requestedUrl, mainDocumentUrl, warnings} = await gotoURL(driver, requestor, {
       ...navigationContext.navigation,
       debugNavigation: config.settings.debugNavigation,
       maxWaitForFcp: config.settings.maxWaitForFcp,
       maxWaitForLoad: config.settings.maxWaitForLoad,
       waitUntil: navigationContext.navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
     });
-    return {requestedUrl, finalUrl, navigationError: undefined, warnings};
+    return {requestedUrl, mainDocumentUrl, navigationError: undefined, warnings};
   } catch (err) {
     if (!(err instanceof LighthouseError)) throw err;
     if (err.code !== 'NO_FCP' && err.code !== 'PAGE_HUNG') throw err;
     if (typeof requestor !== 'string') throw err;
-    return {requestedUrl: requestor, finalUrl: requestor, navigationError: err, warnings: []};
+
+    // TODO: Make the urls optional here so we don't need to throw an error with a callback requestor.
+    return {
+      requestedUrl: requestor,
+      mainDocumentUrl: requestor,
+      navigationError: err,
+      warnings: [],
+    };
   }
 }
 
@@ -152,7 +158,7 @@ async function _collectDebugData(navigationContext, phaseState) {
  * @param {PhaseState} phaseState
  * @param {Awaited<ReturnType<typeof _setupNavigation>>} setupResult
  * @param {Awaited<ReturnType<typeof _navigate>>} navigateResult
- * @return {Promise<{requestedUrl: string, finalUrl: string, artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
+ * @return {Promise<{artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
  */
 async function _computeNavigationResult(
   navigationContext,
@@ -160,12 +166,12 @@ async function _computeNavigationResult(
   setupResult,
   navigateResult
 ) {
-  const {navigationError, finalUrl} = navigateResult;
+  const {navigationError, mainDocumentUrl} = navigateResult;
   const warnings = [...setupResult.warnings, ...navigateResult.warnings];
   const debugData = await _collectDebugData(navigationContext, phaseState);
   const pageLoadError = debugData.records
     ? getPageLoadError(navigationError, {
-      url: finalUrl,
+      url: mainDocumentUrl,
       loadFailureMode: navigationContext.navigation.loadFailureMode,
       networkRecords: debugData.records,
     })
@@ -183,8 +189,6 @@ async function _computeNavigationResult(
     if (debugData.trace) artifacts.traces = {[pageLoadErrorId]: debugData.trace};
 
     return {
-      requestedUrl: navigateResult.requestedUrl,
-      finalUrl,
       pageLoadError,
       artifacts,
       warnings: [...warnings, pageLoadError.friendlyMessage],
@@ -194,8 +198,6 @@ async function _computeNavigationResult(
 
     const artifacts = await awaitArtifacts(phaseState.artifactState);
     return {
-      requestedUrl: navigateResult.requestedUrl,
-      finalUrl,
       artifacts,
       warnings,
       pageLoadError: undefined,
@@ -209,8 +211,9 @@ async function _computeNavigationResult(
  */
 async function _navigation(navigationContext) {
   const artifactState = getEmptyArtifactState();
+  const initialUrl = await navigationContext.driver.url();
   const phaseState = {
-    url: await navigationContext.driver.url(),
+    url: initialUrl,
     gatherMode: /** @type {const} */ ('navigation'),
     driver: navigationContext.driver,
     computedCache: navigationContext.computedCache,
@@ -224,8 +227,20 @@ async function _navigation(navigationContext) {
   await collectPhaseArtifacts({phase: 'startInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'startSensitiveInstrumentation', ...phaseState});
   const navigateResult = await _navigate(navigationContext);
-  phaseState.baseArtifacts.URL.finalUrl = navigateResult.finalUrl;
-  phaseState.url = navigateResult.finalUrl;
+
+  // Every required url is initialized to an empty string in `getBaseArtifacts`.
+  // If we haven't set the required urls yet, set them here.
+  const {URL} = phaseState.baseArtifacts;
+  if (!URL.finalUrl || !URL.initialUrl) {
+    phaseState.baseArtifacts.URL = {
+      initialUrl,
+      requestedUrl: navigateResult.requestedUrl,
+      mainDocumentUrl: navigateResult.mainDocumentUrl,
+      finalUrl: navigateResult.mainDocumentUrl,
+    };
+  }
+  phaseState.url = navigateResult.mainDocumentUrl;
+
   await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseState});
   await _cleanupNavigation(navigationContext);
@@ -263,11 +278,6 @@ async function _navigations({driver, config, requestor, baseArtifacts, computedC
         artifacts.PageLoadError = navigationResult.pageLoadError;
         shouldHaltNavigations = true;
       }
-
-      artifacts.URL = {
-        requestedUrl: navigationResult.requestedUrl,
-        finalUrl: navigationResult.finalUrl,
-      };
     }
 
     LighthouseRunWarnings.push(...navigationResult.warnings);

package/lighthouse-core/fraggle-rock/gather/snapshot-runner.js

@@ -36,8 +36,12 @@ async function snapshotGather(options) {
   const artifacts = await Runner.gather(
     async () => {
       const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'snapshot'});
-      baseArtifacts.URL.requestedUrl = url;
-      baseArtifacts.URL.finalUrl = url;
+      baseArtifacts.URL = {
+        initialUrl: url,
+        // TODO: Remove `requestedUrl` from snapshot mode.
+        requestedUrl: url,
+        finalUrl: url,
+      };
 
       const artifactDefinitions = config.artifacts || [];
       const artifactState = getEmptyArtifactState();

package/lighthouse-core/fraggle-rock/gather/timespan-runner.js

@@ -32,12 +32,12 @@ async function startTimespanGather(options) {
   /** @type {Map<string, LH.ArbitraryEqualityMap>} */
   const computedCache = new Map();
   const artifactDefinitions = config.artifacts || [];
-  const requestedUrl = await driver.url();
+  const initialUrl = await driver.url();
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'timespan'});
   const artifactState = getEmptyArtifactState();
   /** @type {Omit<import('./runner-helpers.js').CollectPhaseArtifactOptions, 'phase'>} */
   const phaseOptions = {
-    url: requestedUrl,
+    url: initialUrl,
     driver,
     artifactDefinitions,
     artifactState,
@@ -59,8 +59,12 @@ async function startTimespanGather(options) {
       const runnerOptions = {config, computedCache};
       const artifacts = await Runner.gather(
         async () => {
-          baseArtifacts.URL.requestedUrl = requestedUrl;
-          baseArtifacts.URL.finalUrl = finalUrl;
+          baseArtifacts.URL = {
+            initialUrl,
+            // TODO: Remove requestedUrl from timespan mode
+            requestedUrl: initialUrl,
+            finalUrl,
+          };
 
           await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseOptions});
           await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseOptions});

package/lighthouse-core/gather/driver/navigation.js

@@ -80,7 +80,7 @@ function resolveWaitForFullyLoadedOptions(options) {
  * @param {LH.Gatherer.FRTransitionalDriver} driver
  * @param {LH.NavigationRequestor} requestor
  * @param {NavigationOptions} options
- * @return {Promise<{requestedUrl: string, finalUrl: string, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, warnings: Array<LH.IcuMessage>}>}
  */
 async function gotoURL(driver, requestor, options) {
   const status = typeof requestor === 'string' ?
@@ -130,14 +130,14 @@ async function gotoURL(driver, requestor, options) {
 
   let requestedUrl = navigationUrls.requestedUrl;
   if (typeof requestor === 'string') {
-    if (requestor !== requestedUrl) {
+    if (requestedUrl && !URL.equalWithExcludedFragments(requestor, requestedUrl)) {
       log.error('Navigation', 'Provided URL did not match initial navigation URL');
     }
     requestedUrl = requestor;
   }
   if (!requestedUrl) throw Error('No navigations detected when running user defined requestor.');
 
-  const finalUrl = navigationUrls.finalUrl || requestedUrl;
+  const mainDocumentUrl = navigationUrls.mainDocumentUrl || requestedUrl;
 
   // Bring `Page.navigate` errors back into the promise chain. See https://github.com/GoogleChrome/lighthouse/pull/6739.
   await waitForNavigationTriggered;
@@ -150,26 +150,26 @@ async function gotoURL(driver, requestor, options) {
   log.timeEnd(status);
   return {
     requestedUrl,
-    finalUrl,
-    warnings: getNavigationWarnings({timedOut, finalUrl, requestedUrl}),
+    mainDocumentUrl,
+    warnings: getNavigationWarnings({timedOut, mainDocumentUrl, requestedUrl}),
   };
 }
 
 /**
- * @param {{timedOut: boolean, requestedUrl: string, finalUrl: string; }} navigation
+ * @param {{timedOut: boolean, requestedUrl: string, mainDocumentUrl: string; }} navigation
  * @return {Array<LH.IcuMessage>}
  */
 function getNavigationWarnings(navigation) {
-  const {requestedUrl, finalUrl} = navigation;
+  const {requestedUrl, mainDocumentUrl} = navigation;
   /** @type {Array<LH.IcuMessage>} */
   const warnings = [];
 
   if (navigation.timedOut) warnings.push(str_(UIStrings.warningTimeout));
 
-  if (!URL.equalWithExcludedFragments(requestedUrl, finalUrl)) {
+  if (!URL.equalWithExcludedFragments(requestedUrl, mainDocumentUrl)) {
     warnings.push(str_(UIStrings.warningRedirected, {
       requested: requestedUrl,
-      final: finalUrl,
+      final: mainDocumentUrl,
     }));
   }
 

package/lighthouse-core/gather/driver/network-monitor.js

@@ -133,7 +133,7 @@ class NetworkMonitor {
     this._sessions = new Map();
   }
 
-  /** @return {Promise<{requestedUrl?: string, finalUrl?: string}>} */
+  /** @return {Promise<{requestedUrl?: string, mainDocumentUrl?: string}>} */
   async getNavigationUrls() {
     const frameNavigations = this._frameNavigations;
     if (!frameNavigations.length) return {};
@@ -143,9 +143,22 @@ class NetworkMonitor {
     const mainFrameNavigations = frameNavigations.filter(frame => frame.id === mainFrameId);
     if (!mainFrameNavigations.length) log.warn('NetworkMonitor', 'No detected navigations');
 
+    // The requested URL is the initiator request for the first frame navigation.
+    /** @type {string|undefined} */
+    let requestedUrl = mainFrameNavigations[0]?.url;
+    if (this._networkRecorder) {
+      const records = this._networkRecorder.getRawRecords();
+
+      let initialUrlRequest = records.find(record => record.url === requestedUrl);
+      while (initialUrlRequest?.redirectSource) {
+        initialUrlRequest = initialUrlRequest.redirectSource;
+        requestedUrl = initialUrlRequest.url;
+      }
+    }
+
     return {
-      requestedUrl: mainFrameNavigations[0]?.url,
-      finalUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
+      requestedUrl,
+      mainDocumentUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
     };
   }
 

package/lighthouse-core/gather/driver.js

@@ -470,6 +470,11 @@ class Driver {
     this._devtoolsLog.endRecording();
     return this._devtoolsLog.messages;
   }
+
+  async url() {
+    const {frameTree} = await this.sendCommand('Page.getFrameTree');
+    return `${frameTree.frame.url}${frameTree.frame.urlFragment || ''}`;
+  }
 }
 
 module.exports = Driver;

package/lighthouse-core/gather/gather-runner.js

@@ -71,7 +71,7 @@ class GatherRunner {
     log.time(status);
     try {
       const requestedUrl = passContext.url;
-      const {finalUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
+      const {mainDocumentUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
         waitUntil: passContext.passConfig.recordTrace ?
           ['load', 'fcp'] : ['load'],
         debugNavigation: passContext.settings.debugNavigation,
@@ -79,8 +79,12 @@ class GatherRunner {
         maxWaitForLoad: passContext.settings.maxWaitForLoad,
         ...passContext.passConfig,
       });
-      passContext.url = finalUrl;
-      passContext.baseArtifacts.URL.finalUrl = finalUrl;
+      passContext.url = mainDocumentUrl;
+      const {URL} = passContext.baseArtifacts;
+      if (!URL.finalUrl || !URL.mainDocumentUrl) {
+        URL.finalUrl = mainDocumentUrl;
+        URL.mainDocumentUrl = mainDocumentUrl;
+      }
       if (passContext.passConfig.loadFailureMode === 'fatal') {
         passContext.LighthouseRunWarnings.push(...warnings);
       }
@@ -408,7 +412,12 @@ class GatherRunner {
       devtoolsLogs: {},
       settings: options.settings,
       GatherContext: {gatherMode: 'navigation'},
-      URL: {requestedUrl: options.requestedUrl, finalUrl: options.requestedUrl},
+      URL: {
+        initialUrl: await options.driver.url(),
+        requestedUrl: options.requestedUrl,
+        mainDocumentUrl: '',
+        finalUrl: '',
+      },
       Timing: [],
       PageLoadError: null,
     };
@@ -426,9 +435,6 @@ class GatherRunner {
 
     const baseArtifacts = passContext.baseArtifacts;
 
-    // Copy redirected URL to artifact.
-    baseArtifacts.URL.finalUrl = passContext.url;
-
     // Fetch the manifest, if it exists.
     try {
       baseArtifacts.WebAppManifest = await WebAppManifest.getWebAppManifest(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse",
-  "version": "9.5.0-dev.20220328",
+  "version": "9.5.0-dev.20220329",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./lighthouse-core/index.js",
   "bin": {

package/types/artifacts.d.ts

@@ -66,7 +66,7 @@ interface UniversalBaseArtifacts {
  */
 interface ContextualBaseArtifacts {
   /** The URL initially requested and the post-redirects URL that was actually loaded. */
-  URL: {requestedUrl: string, finalUrl: string};
+  URL: Artifacts.URL;
   /** If loading the page failed, value is the error that caused it. Otherwise null. */
   PageLoadError: LighthouseError | null;
 }
@@ -187,6 +187,28 @@ declare module Artifacts {
   type TaskNode = _TaskNode;
   type MetaElement = Artifacts['MetaElements'][0];
 
+  interface URL {
+    /** URL of the main frame before Lighthouse starts. */
+    initialUrl: string;
+    /**
+     * URL of the first document request during a Lighthouse navigation.
+     * Will be the same as `initialUrl` in timespan/snapshot.
+     * TODO: Make this property `undefined` in timespan/snapshot.
+     */
+    requestedUrl: string;
+    /**
+     * URL of the last document request during a Lighthouse navigation.
+     * Will be `undefined` in timespan/snapshot.
+     */
+    mainDocumentUrl?: string;
+    /**
+     * Will be the same as `mainDocumentUrl` in navigation mode.
+     * Wil be the URL of the main frame after Lighthouse finishes in timespan/snapshot.
+     * TODO: Use the main frame URL in navigation mode as well.
+     */
+    finalUrl: string;
+  }
+
   interface NodeDetails {
     lhId: string,
     devtoolsNodePath: string,
@@ -299,7 +321,7 @@ declare module Artifacts {
     url: string;
     content?: string;
   }
-  
+
   interface ScriptElement {
     type: string | null
     src: string | null

package/types/gatherer.d.ts

@@ -42,6 +42,7 @@ declare module Gatherer {
     defaultSession: FRProtocolSession;
     executionContext: ExecutionContext;
     fetcher: Fetcher;
+    url: () => Promise<string>;
   }
 
   /** The limited context interface shared between pre and post Fraggle Rock Lighthouse. */