lighthouse 9.5.0-dev.20220509 → 9.6.1

package/lighthouse-core/fraggle-rock/gather/base-artifacts.js

@@ -35,10 +35,7 @@ async function getBaseArtifacts(config, driver, context) {
     HostFormFactor: userAgent.includes('Android') || userAgent.includes('Mobile') ?
       'mobile' : 'desktop',
     // Contextual artifacts whose collection changes based on gather mode.
-    URL: {
-      initialUrl: '',
-      finalUrl: '',
-    },
+    URL: {requestedUrl: '', finalUrl: ''},
     PageLoadError: null,
     GatherContext: context,
     // Artifacts that have been replaced by regular gatherers in Fraggle Rock.
@@ -86,11 +83,11 @@ function finalizeArtifacts(baseArtifacts, gathererArtifacts) {
   artifacts.LighthouseRunWarnings = deduplicateWarnings(warnings);
 
   if (artifacts.PageLoadError && !artifacts.URL.finalUrl) {
-    artifacts.URL.finalUrl = artifacts.URL.requestedUrl || artifacts.URL.initialUrl;
+    artifacts.URL.finalUrl = artifacts.URL.requestedUrl;
   }
 
   // Check that the runner remembered to mutate the special-case URL artifact.
-  if (!artifacts.URL.initialUrl) throw new Error('Runner did not set initialUrl');
+  if (!artifacts.URL.requestedUrl) throw new Error('Runner did not set requestedUrl');
   if (!artifacts.URL.finalUrl) throw new Error('Runner did not set finalUrl');
 
   return artifacts;

package/lighthouse-core/fraggle-rock/gather/driver.js

@@ -35,7 +35,7 @@ const defaultSession = {
 /** @implements {LH.Gatherer.FRTransitionalDriver} */
 class Driver {
   /**
-   * @param {LH.Puppeteer.Page} page
+   * @param {import('puppeteer').Page} page
    */
   constructor(page) {
     this._page = page;
@@ -74,7 +74,7 @@ class Driver {
     const session = await this._page.target().createCDPSession();
     this._session = this.defaultSession = new ProtocolSession(session);
     this._executionContext = new ExecutionContext(this._session);
-    this._fetcher = new Fetcher(this._session);
+    this._fetcher = new Fetcher(this._session, this._executionContext);
     log.timeEnd(status);
   }
 

package/lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -5,7 +5,6 @@
  */
 'use strict';
 
-const puppeteer = require('puppeteer-core');
 const log = require('lighthouse-logger');
 const Driver = require('./driver.js');
 const Runner = require('../../runner.js');
@@ -44,20 +43,18 @@ const NetworkRecords = require('../../computed/network-records.js');
 
 /** @typedef {Omit<Parameters<typeof collectPhaseArtifacts>[0], 'phase'>} PhaseState */
 
-const DEFAULT_HOSTNAME = '127.0.0.1';
-const DEFAULT_PORT = 9222;
-
 /**
- * @param {{driver: Driver, config: LH.Config.FRConfig, options?: InternalOptions}} args
+ * @param {{driver: Driver, config: LH.Config.FRConfig, requestor: LH.NavigationRequestor, options?: InternalOptions}} args
  * @return {Promise<{baseArtifacts: LH.FRBaseArtifacts}>}
  */
-async function _setup({driver, config, options}) {
+async function _setup({driver, config, requestor, options}) {
   await driver.connect();
   if (!options?.skipAboutBlank) {
     await gotoURL(driver, defaultNavigationConfig.blankPage, {waitUntil: ['navigated']});
   }
 
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'navigation'});
+  if (typeof requestor === 'string') baseArtifacts.URL.requestedUrl = requestor;
 
   await prepare.prepareTargetForNavigationMode(driver, config.settings);
 
@@ -93,32 +90,25 @@ async function _cleanupNavigation({driver}) {
 
 /**
  * @param {NavigationContext} navigationContext
- * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
  */
 async function _navigate(navigationContext) {
   const {driver, config, requestor} = navigationContext;
 
   try {
-    const {requestedUrl, mainDocumentUrl, warnings} = await gotoURL(driver, requestor, {
+    const {requestedUrl, finalUrl, warnings} = await gotoURL(driver, requestor, {
       ...navigationContext.navigation,
       debugNavigation: config.settings.debugNavigation,
       maxWaitForFcp: config.settings.maxWaitForFcp,
       maxWaitForLoad: config.settings.maxWaitForLoad,
       waitUntil: navigationContext.navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
     });
-    return {requestedUrl, mainDocumentUrl, navigationError: undefined, warnings};
+    return {requestedUrl, finalUrl, navigationError: undefined, warnings};
   } catch (err) {
     if (!(err instanceof LighthouseError)) throw err;
     if (err.code !== 'NO_FCP' && err.code !== 'PAGE_HUNG') throw err;
     if (typeof requestor !== 'string') throw err;
-
-    // TODO: Make the urls optional here so we don't need to throw an error with a callback requestor.
-    return {
-      requestedUrl: requestor,
-      mainDocumentUrl: requestor,
-      navigationError: err,
-      warnings: [],
-    };
+    return {requestedUrl: requestor, finalUrl: requestor, navigationError: err, warnings: []};
   }
 }
 
@@ -162,7 +152,7 @@ async function _collectDebugData(navigationContext, phaseState) {
  * @param {PhaseState} phaseState
  * @param {Awaited<ReturnType<typeof _setupNavigation>>} setupResult
  * @param {Awaited<ReturnType<typeof _navigate>>} navigateResult
- * @return {Promise<{artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
  */
 async function _computeNavigationResult(
   navigationContext,
@@ -170,12 +160,12 @@ async function _computeNavigationResult(
   setupResult,
   navigateResult
 ) {
-  const {navigationError, mainDocumentUrl} = navigateResult;
+  const {navigationError, finalUrl} = navigateResult;
   const warnings = [...setupResult.warnings, ...navigateResult.warnings];
   const debugData = await _collectDebugData(navigationContext, phaseState);
   const pageLoadError = debugData.records
     ? getPageLoadError(navigationError, {
-      url: mainDocumentUrl,
+      url: finalUrl,
       loadFailureMode: navigationContext.navigation.loadFailureMode,
       networkRecords: debugData.records,
     })
@@ -193,6 +183,8 @@ async function _computeNavigationResult(
     if (debugData.trace) artifacts.traces = {[pageLoadErrorId]: debugData.trace};
 
     return {
+      requestedUrl: navigateResult.requestedUrl,
+      finalUrl,
       pageLoadError,
       artifacts,
       warnings: [...warnings, pageLoadError.friendlyMessage],
@@ -202,6 +194,8 @@ async function _computeNavigationResult(
 
     const artifacts = await awaitArtifacts(phaseState.artifactState);
     return {
+      requestedUrl: navigateResult.requestedUrl,
+      finalUrl,
       artifacts,
       warnings,
       pageLoadError: undefined,
@@ -215,9 +209,8 @@ async function _computeNavigationResult(
  */
 async function _navigation(navigationContext) {
   const artifactState = getEmptyArtifactState();
-  const initialUrl = await navigationContext.driver.url();
   const phaseState = {
-    url: initialUrl,
+    url: await navigationContext.driver.url(),
     gatherMode: /** @type {const} */ ('navigation'),
     driver: navigationContext.driver,
     computedCache: navigationContext.computedCache,
@@ -231,19 +224,7 @@ async function _navigation(navigationContext) {
   await collectPhaseArtifacts({phase: 'startInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'startSensitiveInstrumentation', ...phaseState});
   const navigateResult = await _navigate(navigationContext);
-
-  // Every required url is initialized to an empty string in `getBaseArtifacts`.
-  // If we haven't set all the required urls yet, set them here.
-  if (!Object.values(phaseState.baseArtifacts).every(Boolean)) {
-    phaseState.baseArtifacts.URL = {
-      initialUrl,
-      requestedUrl: navigateResult.requestedUrl,
-      mainDocumentUrl: navigateResult.mainDocumentUrl,
-      finalUrl: navigateResult.mainDocumentUrl,
-    };
-  }
-  phaseState.url = navigateResult.mainDocumentUrl;
-
+  phaseState.url = navigateResult.finalUrl;
   await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseState});
   await _cleanupNavigation(navigationContext);
@@ -281,6 +262,11 @@ async function _navigations({driver, config, requestor, baseArtifacts, computedC
         artifacts.PageLoadError = navigationResult.pageLoadError;
         shouldHaltNavigations = true;
       }
+
+      artifacts.URL = {
+        requestedUrl: navigationResult.requestedUrl,
+        finalUrl: navigationResult.finalUrl,
+      };
     }
 
     LighthouseRunWarnings.push(...navigationResult.warnings);
@@ -302,12 +288,12 @@ async function _cleanup({requestedUrl, driver, config}) {
 }
 
 /**
- * @param {LH.NavigationRequestor|undefined} requestor
- * @param {{page?: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ * @param {LH.NavigationRequestor} requestor
+ * @param {{page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<LH.Gatherer.FRGatherResult>}
  */
 async function navigationGather(requestor, options) {
-  const {configContext = {}} = options;
+  const {page, configContext = {}} = options;
   log.setLevel(configContext.logLevel || 'error');
 
   const {config} = initializeConfig(options.config, {...configContext, gatherMode: 'navigation'});
@@ -317,29 +303,18 @@ async function navigationGather(requestor, options) {
   };
 
   // We can't trigger the navigation through user interaction if we reset the page before starting.
-  const isCallback = typeof requestor === 'function';
-  if (isCallback) {
+  if (typeof requestor !== 'string') {
     internalOptions.skipAboutBlank = true;
   }
 
   const runnerOptions = {config, computedCache};
   const artifacts = await Runner.gather(
     async () => {
-      let {page} = options;
-
-      // For navigation mode, we shouldn't connect to a browser in audit mode,
-      // therefore we connect to the browser in the gatherFn callback.
-      if (!page) {
-        const {hostname = DEFAULT_HOSTNAME, port = DEFAULT_PORT} = configContext;
-        const browser = await puppeteer.connect({browserURL: `http://${hostname}:${port}`});
-        page = await browser.newPage();
-      }
-
       const driver = new Driver(page);
       const context = {
         driver,
         config,
-        requestor: isCallback ? requestor : URL.normalizeUrl(requestor),
+        requestor: typeof requestor === 'string' ? URL.normalizeUrl(requestor) : requestor,
         options: internalOptions,
       };
       const {baseArtifacts} = await _setup(context);

package/lighthouse-core/fraggle-rock/gather/runner-helpers.js

@@ -15,6 +15,7 @@
  * @property {LH.Gatherer.GatherMode} gatherMode
  * @property {Map<string, LH.ArbitraryEqualityMap>} computedCache
  * @property {LH.Config.Settings} settings
+ * @property {string} url
  */
 
 /** @typedef {Record<string, Promise<any>>} IntermediateArtifacts */
@@ -74,6 +75,7 @@ async function collectPhaseArtifacts(options) {
     gatherMode,
     computedCache,
     settings,
+    url,
   } = options;
   const priorPhase = phaseToPriorPhase[phase];
   const priorPhaseArtifacts = (priorPhase && artifactState[priorPhase]) || {};
@@ -90,6 +92,7 @@ async function collectPhaseArtifacts(options) {
         : /** @type {Dependencies} */ ({});
 
       return gatherer[phase]({
+        url,
         gatherMode,
         driver,
         baseArtifacts,

package/lighthouse-core/fraggle-rock/gather/session.js

@@ -14,7 +14,7 @@ const DEFAULT_PROTOCOL_TIMEOUT = 30000;
 /** @implements {LH.Gatherer.FRProtocolSession} */
 class ProtocolSession {
   /**
-   * @param {LH.Puppeteer.CDPSession} session
+   * @param {import('puppeteer').CDPSession} session
    */
   constructor(session) {
     this._session = session;
@@ -93,7 +93,7 @@ class ProtocolSession {
    * @param {(session: ProtocolSession) => void} callback
    */
   addSessionAttachedListener(callback) {
-    /** @param {LH.Puppeteer.CDPSession} session */
+    /** @param {import('puppeteer').CDPSession} session */
     const listener = session => callback(new ProtocolSession(session));
     this._callbackMap.set(callback, listener);
     this._getConnection().on('sessionattached', listener);

package/lighthouse-core/fraggle-rock/gather/snapshot-runner.js

@@ -17,7 +17,7 @@ const {initializeConfig} = require('../config/config.js');
 const {getBaseArtifacts, finalizeArtifacts} = require('./base-artifacts.js');
 
 /**
- * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ * @param {{page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<LH.Gatherer.FRGatherResult>}
  */
 async function snapshotGather(options) {
@@ -36,14 +36,13 @@ async function snapshotGather(options) {
   const artifacts = await Runner.gather(
     async () => {
       const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'snapshot'});
-      baseArtifacts.URL = {
-        initialUrl: url,
-        finalUrl: url,
-      };
+      baseArtifacts.URL.requestedUrl = url;
+      baseArtifacts.URL.finalUrl = url;
 
       const artifactDefinitions = config.artifacts || [];
       const artifactState = getEmptyArtifactState();
       await collectPhaseArtifacts({
+        url,
         phase: 'getArtifact',
         gatherMode: 'snapshot',
         driver,

package/lighthouse-core/fraggle-rock/gather/timespan-runner.js

@@ -18,7 +18,7 @@ const {initializeConfig} = require('../config/config.js');
 const {getBaseArtifacts, finalizeArtifacts} = require('./base-artifacts.js');
 
 /**
- * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ * @param {{page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<{endTimespanGather(): Promise<LH.Gatherer.FRGatherResult>}>}
  */
 async function startTimespanGather(options) {
@@ -32,11 +32,12 @@ async function startTimespanGather(options) {
   /** @type {Map<string, LH.ArbitraryEqualityMap>} */
   const computedCache = new Map();
   const artifactDefinitions = config.artifacts || [];
-  const initialUrl = await driver.url();
+  const requestedUrl = await driver.url();
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'timespan'});
   const artifactState = getEmptyArtifactState();
   /** @type {Omit<import('./runner-helpers.js').CollectPhaseArtifactOptions, 'phase'>} */
   const phaseOptions = {
+    url: requestedUrl,
     driver,
     artifactDefinitions,
     artifactState,
@@ -53,14 +54,13 @@ async function startTimespanGather(options) {
   return {
     async endTimespanGather() {
       const finalUrl = await driver.url();
+      phaseOptions.url = finalUrl;
 
       const runnerOptions = {config, computedCache};
       const artifacts = await Runner.gather(
         async () => {
-          baseArtifacts.URL = {
-            initialUrl,
-            finalUrl,
-          };
+          baseArtifacts.URL.requestedUrl = requestedUrl;
+          baseArtifacts.URL.finalUrl = finalUrl;
 
           await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseOptions});
           await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseOptions});

package/lighthouse-core/gather/connections/cri.js

@@ -88,7 +88,6 @@ class CriConnection extends Connection {
         path: '/json/' + command,
       }, response => {
         let data = '';
-        response.setEncoding('utf8');
         response.on('data', chunk => {
           data += chunk;
         });

package/lighthouse-core/gather/driver/execution-context.js

@@ -97,8 +97,7 @@ class ExecutionContext {
             .catch(${pageFunctions.wrapRuntimeEvalErrorInBrowserString})
             .then(resolve);
         });
-      }())
-      //# sourceURL=_lighthouse-eval.js`,
+      }())`,
       includeCommandLineAPI: true,
       awaitPromise: true,
       returnByValue: true,
@@ -196,8 +195,7 @@ class ExecutionContext {
       ${ExecutionContext._cachedNativesPreamble};
       ${depsSerialized};
       (${mainFn})(${argsSerialized});
-    })()
-    //# sourceURL=_lighthouse-eval.js`;
+    })()`;
 
     await this._session.sendCommand('Page.addScriptToEvaluateOnNewDocument', {source: expression});
   }

package/lighthouse-core/gather/driver/navigation.js

@@ -80,7 +80,7 @@ function resolveWaitForFullyLoadedOptions(options) {
  * @param {LH.Gatherer.FRTransitionalDriver} driver
  * @param {LH.NavigationRequestor} requestor
  * @param {NavigationOptions} options
- * @return {Promise<{requestedUrl: string, mainDocumentUrl: string, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, warnings: Array<LH.IcuMessage>}>}
  */
 async function gotoURL(driver, requestor, options) {
   const status = typeof requestor === 'string' ?
@@ -130,14 +130,14 @@ async function gotoURL(driver, requestor, options) {
 
   let requestedUrl = navigationUrls.requestedUrl;
   if (typeof requestor === 'string') {
-    if (requestedUrl && !URL.equalWithExcludedFragments(requestor, requestedUrl)) {
+    if (requestor !== requestedUrl) {
       log.error('Navigation', 'Provided URL did not match initial navigation URL');
     }
     requestedUrl = requestor;
   }
   if (!requestedUrl) throw Error('No navigations detected when running user defined requestor.');
 
-  const mainDocumentUrl = navigationUrls.mainDocumentUrl || requestedUrl;
+  const finalUrl = navigationUrls.finalUrl || requestedUrl;
 
   // Bring `Page.navigate` errors back into the promise chain. See https://github.com/GoogleChrome/lighthouse/pull/6739.
   await waitForNavigationTriggered;
@@ -150,26 +150,26 @@ async function gotoURL(driver, requestor, options) {
   log.timeEnd(status);
   return {
     requestedUrl,
-    mainDocumentUrl,
-    warnings: getNavigationWarnings({timedOut, mainDocumentUrl, requestedUrl}),
+    finalUrl,
+    warnings: getNavigationWarnings({timedOut, finalUrl, requestedUrl}),
   };
 }
 
 /**
- * @param {{timedOut: boolean, requestedUrl: string, mainDocumentUrl: string; }} navigation
+ * @param {{timedOut: boolean, requestedUrl: string, finalUrl: string; }} navigation
  * @return {Array<LH.IcuMessage>}
  */
 function getNavigationWarnings(navigation) {
-  const {requestedUrl, mainDocumentUrl} = navigation;
+  const {requestedUrl, finalUrl} = navigation;
   /** @type {Array<LH.IcuMessage>} */
   const warnings = [];
 
   if (navigation.timedOut) warnings.push(str_(UIStrings.warningTimeout));
 
-  if (!URL.equalWithExcludedFragments(requestedUrl, mainDocumentUrl)) {
+  if (!URL.equalWithExcludedFragments(requestedUrl, finalUrl)) {
     warnings.push(str_(UIStrings.warningRedirected, {
       requested: requestedUrl,
-      final: mainDocumentUrl,
+      final: finalUrl,
     }));
   }
 

package/lighthouse-core/gather/driver/network-monitor.js

@@ -133,7 +133,7 @@ class NetworkMonitor {
     this._sessions = new Map();
   }
 
-  /** @return {Promise<{requestedUrl?: string, mainDocumentUrl?: string}>} */
+  /** @return {Promise<{requestedUrl?: string, finalUrl?: string}>} */
   async getNavigationUrls() {
     const frameNavigations = this._frameNavigations;
     if (!frameNavigations.length) return {};
@@ -143,22 +143,9 @@ class NetworkMonitor {
     const mainFrameNavigations = frameNavigations.filter(frame => frame.id === mainFrameId);
     if (!mainFrameNavigations.length) log.warn('NetworkMonitor', 'No detected navigations');
 
-    // The requested URL is the initiator request for the first frame navigation.
-    /** @type {string|undefined} */
-    let requestedUrl = mainFrameNavigations[0]?.url;
-    if (this._networkRecorder) {
-      const records = this._networkRecorder.getRawRecords();
-
-      let initialUrlRequest = records.find(record => record.url === requestedUrl);
-      while (initialUrlRequest?.redirectSource) {
-        initialUrlRequest = initialUrlRequest.redirectSource;
-        requestedUrl = initialUrlRequest.url;
-      }
-    }
-
     return {
-      requestedUrl,
-      mainDocumentUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
+      requestedUrl: mainFrameNavigations[0]?.url,
+      finalUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
     };
   }
 

package/lighthouse-core/gather/driver.js

@@ -64,7 +64,7 @@ class Driver {
   defaultSession = this;
 
   // eslint-disable-next-line no-invalid-this
-  fetcher = new Fetcher(this.defaultSession);
+  fetcher = new Fetcher(this.defaultSession, this.executionContext);
 
   /**
    * @param {Connection} connection
@@ -470,11 +470,6 @@ class Driver {
     this._devtoolsLog.endRecording();
     return this._devtoolsLog.messages;
   }
-
-  async url() {
-    const {frameTree} = await this.sendCommand('Page.getFrameTree');
-    return `${frameTree.frame.url}${frameTree.frame.urlFragment || ''}`;
-  }
 }
 
 module.exports = Driver;