lighthouse 9.5.0-dev.20220509 → 9.6.1

package/lighthouse-core/gather/fetcher.js

@@ -6,55 +6,124 @@
 'use strict';
 
 /**
- * @fileoverview Fetcher is a utility for making requests to any arbitrary resource,
- * ignoring normal browser constraints such as CORS.
+ * @fileoverview Fetcher is a utility for making requests within the context of the page.
+ * Requests can circumvent CORS, and so are good for fetching source maps that may be hosted
+ * on a different origin.
  */
 
-/* global fetch */
+/* global document */
 
 /** @typedef {{content: string|null, status: number|null}} FetchResponse */
 
+const log = require('lighthouse-logger');
+const {getBrowserVersion} = require('./driver/environment.js');
+
 class Fetcher {
   /**
    * @param {LH.Gatherer.FRProtocolSession} session
+   * @param {import('./driver/execution-context.js')} executionContext
    */
-  constructor(session) {
+  constructor(session, executionContext) {
     this.session = session;
+    this.executionContext = executionContext;
+    /** @type {Map<string, (event: LH.Crdp.Fetch.RequestPausedEvent) => void>} */
+    this._onRequestPausedHandlers = new Map();
+    this._onRequestPaused = this._onRequestPaused.bind(this);
+    this._enabled = false;
   }
 
   /**
-   * Fetches any resource using the network directly.
+   * Chrome M92 and above:
+   * We use `Network.loadNetworkResource` to fetch each resource.
+   *
+   * Chrome <M92:
+   * The Fetch domain accepts patterns for controlling what requests are intercepted, but we
+   * enable the domain for all patterns and filter events at a lower level to support multiple
+   * concurrent usages. Reasons for this:
+   *
+   * 1) only one set of patterns may be applied for the entire domain.
+   * 2) every request that matches the patterns are paused and only resumes when certain Fetch
+   *    commands are sent. So a listener of the `Fetch.requestPaused` event must either handle
+   *    the requests it cares about, or explicitly allow them to continue.
+   * 3) if multiple commands to continue the same request are sent, protocol errors occur.
    *
+   * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific
+   * urls to be intercepted via `fetcher._setOnRequestPausedHandler`.
+   */
+  async enable() {
+    if (this._enabled) return;
+
+    this._enabled = true;
+    await this.session.sendCommand('Fetch.enable', {
+      patterns: [{requestStage: 'Request'}, {requestStage: 'Response'}],
+    });
+    await this.session.on('Fetch.requestPaused', this._onRequestPaused);
+  }
+
+  async disable() {
+    if (!this._enabled) return;
+
+    this._enabled = false;
+    await this.session.off('Fetch.requestPaused', this._onRequestPaused);
+    await this.session.sendCommand('Fetch.disable');
+    this._onRequestPausedHandlers.clear();
+  }
+
+  /**
    * @param {string} url
-   * @param {{timeout: number}=} options timeout is in ms
-   * @return {Promise<FetchResponse>}
+   * @param {(event: LH.Crdp.Fetch.RequestPausedEvent) => void} handler
    */
-  async fetchResource(url, options = {timeout: 2_000}) {
-    // In Lightrider, `Network.loadNetworkResource` is not implemented, but fetch
-    // is configured to work for any resource.
-    if (global.isLightrider) {
-      return this._wrapWithTimeout(this._fetchWithFetchApi(url), options.timeout);
+  async _setOnRequestPausedHandler(url, handler) {
+    this._onRequestPausedHandlers.set(url, handler);
+  }
+
+  /**
+   * @param {LH.Crdp.Fetch.RequestPausedEvent} event
+   */
+  _onRequestPaused(event) {
+    const handler = this._onRequestPausedHandlers.get(event.request.url);
+    if (handler) {
+      handler(event);
+    } else {
+      // Nothing cares about this URL, so continue.
+      this.session.sendCommand('Fetch.continueRequest', {requestId: event.requestId}).catch(err => {
+        log.error('Fetcher', `Failed to continueRequest: ${err.message}`);
+      });
     }
+  }
 
-    return this._fetchResourceOverProtocol(url, options);
+  /**
+   * `Network.loadNetworkResource` was introduced in M88.
+   * The long timeout bug with `IO.read` was fixed in M92:
+   * https://bugs.chromium.org/p/chromium/issues/detail?id=1191757
+   * Lightrider has a bug forcing us to use the old version for now:
+   * https://docs.google.com/document/d/1V-DxgsOFMPxUuFrdGPQpyiCqSljvgNlOqXCtqDtd0b8/edit?usp=sharing&resourcekey=0-aIaIqcHFKG-0dX4MAudBEw
+   * @return {Promise<boolean>}
+   */
+  async shouldUseLegacyFetcher() {
+    const {milestone} = await getBrowserVersion(this.session);
+    return milestone < 92 || Boolean(global.isLightrider);
   }
 
   /**
+   * Requires that `fetcher.enable` has been called.
+   *
+   * Fetches any resource in a way that circumvents CORS.
+   *
    * @param {string} url
+   * @param {{timeout: number}=} options timeout is in ms
    * @return {Promise<FetchResponse>}
    */
-  async _fetchWithFetchApi(url) {
-    const response = await fetch(url);
+  async fetchResource(url, options = {timeout: 2_000}) {
+    if (!this._enabled) {
+      throw new Error('Must call `enable` before using fetchResource');
+    }
 
-    let content = null;
-    try {
-      content = await response.text();
-    } catch {}
+    if (await this.shouldUseLegacyFetcher()) {
+      return this._fetchResourceIframe(url, options);
+    }
 
-    return {
-      content,
-      status: response.status,
-    };
+    return this._fetchResourceOverProtocol(url, options);
   }
 
   /**
@@ -103,6 +172,18 @@ class Fetcher {
     };
   }
 
+  /**
+   * @param {string} requestId
+   * @return {Promise<string>}
+   */
+  async _resolveResponseBody(requestId) {
+    const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId});
+    if (responseBody.base64Encoded) {
+      return Buffer.from(responseBody.body, 'base64').toString();
+    }
+    return responseBody.body;
+  }
+
   /**
    * @param {string} url
    * @param {{timeout: number}} options timeout is in ms
@@ -110,7 +191,18 @@ class Fetcher {
    */
   async _fetchResourceOverProtocol(url, options) {
     const startTime = Date.now();
-    const response = await this._wrapWithTimeout(this._loadNetworkResource(url), options.timeout);
+
+    /** @type {NodeJS.Timeout} */
+    let timeoutHandle;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(reject, options.timeout, new Error('Timed out fetching resource'));
+    });
+
+    const responsePromise = this._loadNetworkResource(url);
+
+    /** @type {{stream: LH.Crdp.IO.StreamHandle|null, status: number|null}} */
+    const response = await Promise.race([responsePromise, timeoutPromise])
+      .finally(() => clearTimeout(timeoutHandle));
 
     const isOk = response.status && response.status >= 200 && response.status <= 299;
     if (!response.stream || !isOk) return {status: response.status, content: null};
@@ -121,21 +213,105 @@ class Fetcher {
   }
 
   /**
-   * @template T
-   * @param {Promise<T>} promise
-   * @param {number} ms
+   * Fetches resource by injecting an iframe into the page.
+   * @param {string} url
+   * @param {{timeout: number}} options timeout is in ms
+   * @return {Promise<FetchResponse>}
    */
-  async _wrapWithTimeout(promise, ms) {
+  async _fetchResourceIframe(url, options) {
+    /** @type {Promise<FetchResponse>} */
+    const requestInterceptionPromise = new Promise((resolve, reject) => {
+      /** @param {LH.Crdp.Fetch.RequestPausedEvent} event */
+      const handlerAsync = async event => {
+        const {requestId, responseStatusCode} = event;
+
+        // The first requestPaused event is for the request stage. Continue it.
+        if (!responseStatusCode) {
+          // Remove cookies so we aren't buying stuff on Amazon.
+          const headers = Object.entries(event.request.headers)
+            .filter(([name]) => name !== 'Cookie')
+            .map(([name, value]) => {
+              return {name, value};
+            });
+
+          await this.session.sendCommand('Fetch.continueRequest', {
+            requestId,
+            headers,
+          });
+          return;
+        }
+
+        if (responseStatusCode >= 200 && responseStatusCode <= 299) {
+          resolve({
+            status: responseStatusCode,
+            content: await this._resolveResponseBody(requestId),
+          });
+        } else {
+          resolve({status: responseStatusCode, content: null});
+        }
+
+        // Fail the request (from the page's perspective) so that the iframe never loads.
+        await this.session.sendCommand('Fetch.failRequest', {requestId, errorReason: 'Aborted'});
+      };
+      this._setOnRequestPausedHandler(url, event => handlerAsync(event).catch(reject));
+    });
+
+    /**
+     * @param {string} src
+     */
+    /* c8 ignore start */
+    function injectIframe(src) {
+      const iframe = document.createElement('iframe');
+      // Try really hard not to affect the page.
+      iframe.style.display = 'none';
+      iframe.style.visibility = 'hidden';
+      iframe.style.position = 'absolute';
+      iframe.style.top = '-1000px';
+      iframe.style.left = '-1000px';
+      iframe.style.width = '1px';
+      iframe.style.height = '1px';
+      iframe.src = src;
+      iframe.onload = iframe.onerror = () => {
+        iframe.remove();
+        iframe.onload = null;
+        iframe.onerror = null;
+      };
+      document.body.appendChild(iframe);
+    }
+    /* c8 ignore stop */
+
     /** @type {NodeJS.Timeout} */
-    let timeoutHandle;
+    let asyncTimeout;
+    /** @type {Promise<never>} */
     const timeoutPromise = new Promise((_, reject) => {
-      timeoutHandle = setTimeout(reject, ms, new Error('Timed out fetching resource'));
+      asyncTimeout = setTimeout(reject, options.timeout, new Error('Timed out fetching resource.'));
     });
 
-    /** @type {Promise<T>} */
-    const wrappedPromise = await Promise.race([promise, timeoutPromise])
-      .finally(() => clearTimeout(timeoutHandle));
-    return wrappedPromise;
+    const racePromise = Promise.race([
+      timeoutPromise,
+      requestInterceptionPromise,
+    ]).finally(() => clearTimeout(asyncTimeout));
+
+    // Temporarily disable auto-attaching for this iframe.
+    await this.session.sendCommand('Target.setAutoAttach', {
+      autoAttach: false,
+      waitForDebuggerOnStart: false,
+    });
+
+    const injectionPromise = this.executionContext.evaluate(injectIframe, {
+      args: [url],
+      useIsolation: true,
+    });
+
+    const [fetchResult] = await Promise.all([racePromise, injectionPromise]);
+
+    await this.session.sendCommand('Target.setAutoAttach', {
+      flatten: true,
+      autoAttach: true,
+      waitForDebuggerOnStart: true,
+    });
+
+    return fetchResult;
   }
 }
 

package/lighthouse-core/gather/gather-runner.js

@@ -71,7 +71,7 @@ class GatherRunner {
     log.time(status);
     try {
       const requestedUrl = passContext.url;
-      const {mainDocumentUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
+      const {finalUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
         waitUntil: passContext.passConfig.recordTrace ?
           ['load', 'fcp'] : ['load'],
         debugNavigation: passContext.settings.debugNavigation,
@@ -79,12 +79,7 @@ class GatherRunner {
         maxWaitForLoad: passContext.settings.maxWaitForLoad,
         ...passContext.passConfig,
       });
-      passContext.url = mainDocumentUrl;
-      const {URL} = passContext.baseArtifacts;
-      if (!URL.finalUrl || !URL.mainDocumentUrl) {
-        URL.finalUrl = mainDocumentUrl;
-        URL.mainDocumentUrl = mainDocumentUrl;
-      }
+      passContext.url = finalUrl;
       if (passContext.passConfig.loadFailureMode === 'fatal') {
         passContext.LighthouseRunWarnings.push(...warnings);
       }
@@ -183,6 +178,11 @@ class GatherRunner {
       const resetStorage = !options.settings.disableStorageReset;
       if (resetStorage) await storage.clearDataForOrigin(session, options.requestedUrl);
 
+      // Disable fetcher, in case a gatherer enabled it.
+      // This cleanup should be removed once the only usage of
+      // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
+      await driver.fetcher.disable();
+
       await driver.disconnect();
     } catch (err) {
       // Ignore disconnecting error if browser was already closed.
@@ -407,12 +407,7 @@ class GatherRunner {
       devtoolsLogs: {},
       settings: options.settings,
       GatherContext: {gatherMode: 'navigation'},
-      URL: {
-        initialUrl: await options.driver.url(),
-        requestedUrl: options.requestedUrl,
-        mainDocumentUrl: '',
-        finalUrl: '',
-      },
+      URL: {requestedUrl: options.requestedUrl, finalUrl: options.requestedUrl},
       Timing: [],
       PageLoadError: null,
     };
@@ -430,6 +425,9 @@ class GatherRunner {
 
     const baseArtifacts = passContext.baseArtifacts;
 
+    // Copy redirected URL to artifact.
+    baseArtifacts.URL.finalUrl = passContext.url;
+
     // Fetch the manifest, if it exists.
     try {
       baseArtifacts.WebAppManifest = await WebAppManifest.getWebAppManifest(
@@ -519,6 +517,12 @@ class GatherRunner {
           await GatherRunner.populateBaseArtifacts(passContext);
           isFirstPass = false;
         }
+
+        // Disable fetcher for every pass, in case a gatherer enabled it.
+        // Noop if fetcher was never enabled.
+        // This cleanup should be removed once the only usage of
+        // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
+        await driver.fetcher.disable();
       }
 
       await GatherRunner.disposeDriver(driver, options);

package/lighthouse-core/gather/gatherers/accessibility.js

@@ -12,6 +12,9 @@ const axeLibSource = require('../../lib/axe.js').source;
 const pageFunctions = require('../../lib/page-functions.js');
 
 /**
+ * This is run in the page, not Lighthouse itself.
+ * axe.run returns a promise which fulfills with a results object
+ * containing any violations.
  * @return {Promise<LH.Artifacts.Accessibility>}
  */
 /* c8 ignore start */
@@ -35,11 +38,8 @@ async function runA11yChecks() {
         'wcag2aa',
       ],
     },
-    // resultTypes doesn't limit the output of the axeResults object. Instead, if it's defined,
-    // some expensive element identification is done only for the respective types. https://github.com/dequelabs/axe-core/blob/f62f0cf18f7b69b247b0b6362cf1ae71ffbf3a1b/lib/core/reporters/helpers/process-aggregate.js#L61-L97
     resultTypes: ['violations', 'inapplicable'],
     rules: {
-      // Consider http://go/prcpg for expert review of the aXe rules.
       'tabindex': {enabled: true},
       'accesskeys': {enabled: true},
       'heading-order': {enabled: true},
@@ -60,12 +60,6 @@ async function runA11yChecks() {
       // https://github.com/dequelabs/axe-core/issues/2958
       'nested-interactive': {enabled: false},
       'frame-focusable-content': {enabled: false},
-      'aria-roledescription': {enabled: false},
-      'scrollable-region-focusable': {enabled: false},
-      // TODO(paulirish): create audits and enable these 3.
-      'input-button-name': {enabled: false},
-      'role-img-alt': {enabled: false},
-      'select-name': {enabled: false},
     },
   });
 
@@ -76,8 +70,7 @@ async function runA11yChecks() {
   return {
     violations: axeResults.violations.map(createAxeRuleResultArtifact),
     incomplete: axeResults.incomplete.map(createAxeRuleResultArtifact),
-    notApplicable: axeResults.inapplicable.map(result => ({id: result.id})), // FYI: inapplicable => notApplicable!
-    passes: axeResults.passes.map(result => ({id: result.id})),
+    notApplicable: axeResults.inapplicable.map(result => ({id: result.id})),
     version: axeResults.testEngine.version,
   };
 }
@@ -157,11 +150,6 @@ class Accessibility extends FRGatherer {
     supportedModes: ['snapshot', 'navigation'],
   };
 
-  static pageFns = {
-    runA11yChecks,
-    createAxeRuleResultArtifact,
-  };
-
   /**
    * @param {LH.Gatherer.FRTransitionalContext} passContext
    * @return {Promise<LH.Artifacts.Accessibility>}

package/lighthouse-core/gather/gatherers/console-messages.js

@@ -101,7 +101,6 @@ class ConsoleMessages extends FRGatherer {
       stackTrace: event.exceptionDetails.stackTrace,
       timestamp: event.timestamp,
       url: event.exceptionDetails.url,
-      scriptId: event.exceptionDetails.scriptId,
       lineNumber: event.exceptionDetails.lineNumber,
       columnNumber: event.exceptionDetails.columnNumber,
     };
@@ -118,7 +117,7 @@ class ConsoleMessages extends FRGatherer {
 
     // JS events have a stack trace, which we use to get the column.
     // CSS/HTML events only expose a line number.
-    const firstStackFrame = event.entry.stackTrace?.callFrames[0];
+    const {columnNumber} = event.entry.stackTrace?.callFrames[0] || {};
 
     this._logEntries.push({
       eventType: 'protocolLog',
@@ -128,9 +127,8 @@ class ConsoleMessages extends FRGatherer {
       stackTrace,
       timestamp,
       url,
-      scriptId: firstStackFrame?.scriptId,
       lineNumber,
-      columnNumber: firstStackFrame?.columnNumber,
+      columnNumber,
     });
   }
 

package/lighthouse-core/gather/gatherers/dobetterweb/doctype.js

@@ -12,8 +12,7 @@ const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js');
 /**
  * Get and return `name`, `publicId`, `systemId` from
  * `document.doctype`
- * and `compatMode` from `document` to check `quirks-mode`
- * @return {{name: string, publicId: string, systemId: string, documentCompatMode: string} | null}
+ * @return {{name: string, publicId: string, systemId: string} | null}
  */
 function getDoctype() {
   // An example of this is warnerbros.com/archive/spacejam/movie/jam.htm
@@ -21,9 +20,8 @@ function getDoctype() {
     return null;
   }
 
-  const documentCompatMode = document.compatMode;
   const {name, publicId, systemId} = document.doctype;
-  return {name, publicId, systemId, documentCompatMode};
+  return {name, publicId, systemId};
 }
 
 class Doctype extends FRGatherer {

package/lighthouse-core/gather/gatherers/full-page-screenshot.js

@@ -15,7 +15,6 @@ const {waitForNetworkIdle} = require('../driver/wait-for-condition.js');
 
 // JPEG quality setting
 // Exploration and examples of reports using different quality settings: https://docs.google.com/document/d/1ZSffucIca9XDW2eEwfoevrk-OTl7WQFeMf0CgeJAA8M/edit#
-// Note: this analysis was done for JPEG, but now we use WEBP.
 const FULL_PAGE_SCREENSHOT_QUALITY = 30;
 
 /**
@@ -119,10 +118,10 @@ class FullPageScreenshot extends FRGatherer {
     await context.driver.executionContext.evaluate(waitForDoubleRaf, {args: []});
 
     const result = await session.sendCommand('Page.captureScreenshot', {
-      format: 'webp',
+      format: 'jpeg',
       quality: FULL_PAGE_SCREENSHOT_QUALITY,
     });
-    const data = 'data:image/webp;base64,' + result.data;
+    const data = 'data:image/jpeg;base64,' + result.data;
 
     return {
       data,

package/lighthouse-core/gather/gatherers/js-usage.js

@@ -18,8 +18,20 @@ class JsUsage extends FRGatherer {
 
   constructor() {
     super();
+    /** @type {LH.Crdp.Debugger.ScriptParsedEvent[]} */
+    this._scriptParsedEvents = [];
     /** @type {LH.Crdp.Profiler.ScriptCoverage[]} */
     this._scriptUsages = [];
+    this.onScriptParsed = this.onScriptParsed.bind(this);
+  }
+
+  /**
+   * @param {LH.Crdp.Debugger.ScriptParsedEvent} event
+   */
+  onScriptParsed(event) {
+    if (event.embedderName) {
+      this._scriptParsedEvents.push(event);
+    }
   }
 
   /**
@@ -43,27 +55,90 @@ class JsUsage extends FRGatherer {
   }
 
   /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   */
+  async startSensitiveInstrumentation(context) {
+    const session = context.driver.defaultSession;
+    session.on('Debugger.scriptParsed', this.onScriptParsed);
+    await session.sendCommand('Debugger.enable');
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   */
+  async stopSensitiveInstrumentation(context) {
+    const session = context.driver.defaultSession;
+    await session.sendCommand('Debugger.disable');
+    session.off('Debugger.scriptParsed', this.onScriptParsed);
+  }
+
+  /**
+   * Usages alone do not always generate an exhaustive list of scripts in timespan and snapshot.
+   * For audits which use this for url/scriptId mappings, we can include an empty usage object.
+   *
+   * @param {Record<string, Array<LH.Crdp.Profiler.ScriptCoverage>>} usageByUrl
+   */
+  _addMissingScriptIds(usageByUrl) {
+    for (const scriptParsedEvent of this._scriptParsedEvents) {
+      const url = scriptParsedEvent.embedderName;
+      if (!url) continue;
+
+      const scripts = usageByUrl[url] || [];
+      if (!scripts.find(s => s.scriptId === scriptParsedEvent.scriptId)) {
+        scripts.push({
+          url,
+          scriptId: scriptParsedEvent.scriptId,
+          functions: [],
+        });
+      }
+      usageByUrl[url] = scripts;
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
    * @return {Promise<LH.Artifacts['JsUsage']>}
    */
-  async getArtifact() {
-    /** @type {Record<string, LH.Crdp.Profiler.ScriptCoverage>} */
-    const usageByScriptId = {};
+  async getArtifact(context) {
+    /** @type {Record<string, Array<LH.Crdp.Profiler.ScriptCoverage>>} */
+    const usageByUrl = {};
+
+    // Force `Debugger.scriptParsed` events for url to scriptId mappings in snapshot mode.
+    if (context.gatherMode === 'snapshot') {
+      await this.startSensitiveInstrumentation(context);
+      await this.stopSensitiveInstrumentation(context);
+    }
 
     for (const scriptUsage of this._scriptUsages) {
-      // If `url` is blank, that means the script was dynamically
-      // created (eval, new Function, onload, ...)
-      if (scriptUsage.url === '' || scriptUsage.url === '_lighthouse-eval.js') {
-        // We currently don't consider coverage of dynamic scripts, and we definitely don't want
-        // coverage of code Lighthouse ran to inspect the page, so we ignore this ScriptCoverage.
-        // Audits would work the same without this, it is only an optimization (not tracking coverage
-        // for scripts we don't care about).
+      // `ScriptCoverage.url` can be overridden by a magic sourceURL comment.
+      // Get the associated ScriptParsedEvent and use embedderName, which is the original url.
+      // See https://chromium-review.googlesource.com/c/v8/v8/+/2317310
+      let url = scriptUsage.url;
+      const scriptParsedEvent =
+        this._scriptParsedEvents.find(e => e.scriptId === scriptUsage.scriptId);
+      if (scriptParsedEvent?.embedderName) {
+        url = scriptParsedEvent.embedderName;
+      }
+
+      // If `url` is blank, that means the script was anonymous (eval, new Function, onload, ...).
+      // Or, it's because it was code Lighthouse over the protocol via `Runtime.evaluate`.
+      // We currently don't consider coverage of anonymous scripts, and we definitely don't want
+      // coverage of code Lighthouse ran to inspect the page, so we ignore this ScriptCoverage if
+      // url is blank.
+      if (scriptUsage.url === '' || (scriptParsedEvent && scriptParsedEvent.embedderName === '')) {
         continue;
       }
 
-      usageByScriptId[scriptUsage.scriptId] = scriptUsage;
+      const scripts = usageByUrl[url] || [];
+      scripts.push(scriptUsage);
+      usageByUrl[url] = scripts;
+    }
+
+    if (context.gatherMode !== 'navigation') {
+      this._addMissingScriptIds(usageByUrl);
     }
 
-    return usageByScriptId;
+    return usageByUrl;
   }
 }