lighthouse 9.5.0-dev.20220607 → 9.6.2

package/lighthouse-core/gather/gather-runner.js

@@ -71,7 +71,7 @@ class GatherRunner {
     log.time(status);
     try {
       const requestedUrl = passContext.url;
-      const {mainDocumentUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
+      const {finalUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
         waitUntil: passContext.passConfig.recordTrace ?
           ['load', 'fcp'] : ['load'],
         debugNavigation: passContext.settings.debugNavigation,
@@ -79,12 +79,7 @@ class GatherRunner {
         maxWaitForLoad: passContext.settings.maxWaitForLoad,
         ...passContext.passConfig,
       });
-      passContext.url = mainDocumentUrl;
-      const {URL} = passContext.baseArtifacts;
-      if (!URL.finalUrl || !URL.mainDocumentUrl) {
-        URL.finalUrl = mainDocumentUrl;
-        URL.mainDocumentUrl = mainDocumentUrl;
-      }
+      passContext.url = finalUrl;
       if (passContext.passConfig.loadFailureMode === 'fatal') {
         passContext.LighthouseRunWarnings.push(...warnings);
       }
@@ -183,6 +178,11 @@ class GatherRunner {
       const resetStorage = !options.settings.disableStorageReset;
       if (resetStorage) await storage.clearDataForOrigin(session, options.requestedUrl);
 
+      // Disable fetcher, in case a gatherer enabled it.
+      // This cleanup should be removed once the only usage of
+      // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
+      await driver.fetcher.disable();
+
       await driver.disconnect();
     } catch (err) {
       // Ignore disconnecting error if browser was already closed.
@@ -407,12 +407,7 @@ class GatherRunner {
       devtoolsLogs: {},
       settings: options.settings,
       GatherContext: {gatherMode: 'navigation'},
-      URL: {
-        initialUrl: await options.driver.url(),
-        requestedUrl: options.requestedUrl,
-        mainDocumentUrl: '',
-        finalUrl: '',
-      },
+      URL: {requestedUrl: options.requestedUrl, finalUrl: options.requestedUrl},
       Timing: [],
       PageLoadError: null,
     };
@@ -430,6 +425,9 @@ class GatherRunner {
 
     const baseArtifacts = passContext.baseArtifacts;
 
+    // Copy redirected URL to artifact.
+    baseArtifacts.URL.finalUrl = passContext.url;
+
     // Fetch the manifest, if it exists.
     try {
       baseArtifacts.WebAppManifest = await WebAppManifest.getWebAppManifest(
@@ -519,6 +517,12 @@ class GatherRunner {
           await GatherRunner.populateBaseArtifacts(passContext);
           isFirstPass = false;
         }
+
+        // Disable fetcher for every pass, in case a gatherer enabled it.
+        // Noop if fetcher was never enabled.
+        // This cleanup should be removed once the only usage of
+        // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
+        await driver.fetcher.disable();
       }
 
       await GatherRunner.disposeDriver(driver, options);

package/lighthouse-core/gather/gatherers/accessibility.js

@@ -12,6 +12,9 @@ const axeLibSource = require('../../lib/axe.js').source;
 const pageFunctions = require('../../lib/page-functions.js');
 
 /**
+ * This is run in the page, not Lighthouse itself.
+ * axe.run returns a promise which fulfills with a results object
+ * containing any violations.
  * @return {Promise<LH.Artifacts.Accessibility>}
  */
 /* c8 ignore start */
@@ -35,11 +38,8 @@ async function runA11yChecks() {
         'wcag2aa',
       ],
     },
-    // resultTypes doesn't limit the output of the axeResults object. Instead, if it's defined,
-    // some expensive element identification is done only for the respective types. https://github.com/dequelabs/axe-core/blob/f62f0cf18f7b69b247b0b6362cf1ae71ffbf3a1b/lib/core/reporters/helpers/process-aggregate.js#L61-L97
     resultTypes: ['violations', 'inapplicable'],
     rules: {
-      // Consider http://go/prcpg for expert review of the aXe rules.
       'tabindex': {enabled: true},
       'accesskeys': {enabled: true},
       'heading-order': {enabled: true},
@@ -60,12 +60,6 @@ async function runA11yChecks() {
       // https://github.com/dequelabs/axe-core/issues/2958
       'nested-interactive': {enabled: false},
       'frame-focusable-content': {enabled: false},
-      'aria-roledescription': {enabled: false},
-      'scrollable-region-focusable': {enabled: false},
-      // TODO(paulirish): create audits and enable these 3.
-      'input-button-name': {enabled: false},
-      'role-img-alt': {enabled: false},
-      'select-name': {enabled: false},
     },
   });
 
@@ -76,8 +70,7 @@ async function runA11yChecks() {
   return {
     violations: axeResults.violations.map(createAxeRuleResultArtifact),
     incomplete: axeResults.incomplete.map(createAxeRuleResultArtifact),
-    notApplicable: axeResults.inapplicable.map(result => ({id: result.id})), // FYI: inapplicable => notApplicable!
-    passes: axeResults.passes.map(result => ({id: result.id})),
+    notApplicable: axeResults.inapplicable.map(result => ({id: result.id})),
     version: axeResults.testEngine.version,
   };
 }
@@ -157,11 +150,6 @@ class Accessibility extends FRGatherer {
     supportedModes: ['snapshot', 'navigation'],
   };
 
-  static pageFns = {
-    runA11yChecks,
-    createAxeRuleResultArtifact,
-  };
-
   /**
    * @param {LH.Gatherer.FRTransitionalContext} passContext
    * @return {Promise<LH.Artifacts.Accessibility>}

package/lighthouse-core/gather/gatherers/console-messages.js

@@ -101,7 +101,6 @@ class ConsoleMessages extends FRGatherer {
       stackTrace: event.exceptionDetails.stackTrace,
       timestamp: event.timestamp,
       url: event.exceptionDetails.url,
-      scriptId: event.exceptionDetails.scriptId,
       lineNumber: event.exceptionDetails.lineNumber,
       columnNumber: event.exceptionDetails.columnNumber,
     };
@@ -118,7 +117,7 @@ class ConsoleMessages extends FRGatherer {
 
     // JS events have a stack trace, which we use to get the column.
     // CSS/HTML events only expose a line number.
-    const firstStackFrame = event.entry.stackTrace?.callFrames[0];
+    const {columnNumber} = event.entry.stackTrace?.callFrames[0] || {};
 
     this._logEntries.push({
       eventType: 'protocolLog',
@@ -128,9 +127,8 @@ class ConsoleMessages extends FRGatherer {
       stackTrace,
       timestamp,
       url,
-      scriptId: firstStackFrame?.scriptId,
       lineNumber,
-      columnNumber: firstStackFrame?.columnNumber,
+      columnNumber,
     });
   }
 

package/lighthouse-core/gather/gatherers/dobetterweb/doctype.js

@@ -12,8 +12,7 @@ const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js');
 /**
  * Get and return `name`, `publicId`, `systemId` from
  * `document.doctype`
- * and `compatMode` from `document` to check `quirks-mode`
- * @return {{name: string, publicId: string, systemId: string, documentCompatMode: string} | null}
+ * @return {{name: string, publicId: string, systemId: string} | null}
  */
 function getDoctype() {
   // An example of this is warnerbros.com/archive/spacejam/movie/jam.htm
@@ -21,9 +20,8 @@ function getDoctype() {
     return null;
   }
 
-  const documentCompatMode = document.compatMode;
   const {name, publicId, systemId} = document.doctype;
-  return {name, publicId, systemId, documentCompatMode};
+  return {name, publicId, systemId};
 }
 
 class Doctype extends FRGatherer {

package/lighthouse-core/gather/gatherers/full-page-screenshot.js

@@ -15,7 +15,6 @@ const {waitForNetworkIdle} = require('../driver/wait-for-condition.js');
 
 // JPEG quality setting
 // Exploration and examples of reports using different quality settings: https://docs.google.com/document/d/1ZSffucIca9XDW2eEwfoevrk-OTl7WQFeMf0CgeJAA8M/edit#
-// Note: this analysis was done for JPEG, but now we use WEBP.
 const FULL_PAGE_SCREENSHOT_QUALITY = 30;
 
 /**
@@ -119,10 +118,10 @@ class FullPageScreenshot extends FRGatherer {
     await context.driver.executionContext.evaluate(waitForDoubleRaf, {args: []});
 
     const result = await session.sendCommand('Page.captureScreenshot', {
-      format: 'webp',
+      format: 'jpeg',
       quality: FULL_PAGE_SCREENSHOT_QUALITY,
     });
-    const data = 'data:image/webp;base64,' + result.data;
+    const data = 'data:image/jpeg;base64,' + result.data;
 
     return {
       data,

package/lighthouse-core/gather/gatherers/iframe-elements.js

@@ -17,13 +17,10 @@ const pageFunctions = require('../../lib/page-functions.js');
  */
 /* c8 ignore start */
 function collectIFrameElements() {
-  const realBoundingClientRect = window.__HTMLElementBoundingClientRect ||
-    window.HTMLElement.prototype.getBoundingClientRect;
-
   // @ts-expect-error - put into scope via stringification
   const iFrameElements = getElementsInDocument('iframe'); // eslint-disable-line no-undef
   return iFrameElements.map(/** @param {HTMLIFrameElement} node */ (node) => {
-    const clientRect = realBoundingClientRect.call(node);
+    const clientRect = node.getBoundingClientRect();
     const {top, bottom, left, right, width, height} = clientRect;
     return {
       id: node.id,

package/lighthouse-core/gather/gatherers/image-elements.js

@@ -258,7 +258,6 @@ class ImageElements extends FRGatherer {
         driver.defaultSession.setNextProtocolTimeout(250);
         size = await driver.executionContext.evaluate(determineNaturalSize, {
           args: [url],
-          useIsolation: true,
         });
         this._naturalSizeCache.set(url, size);
       } catch (_) {
@@ -341,7 +340,6 @@ class ImageElements extends FRGatherer {
 
     const elements = await executionContext.evaluate(collectImageElementInfo, {
       args: [],
-      useIsolation: true,
       deps: [
         pageFunctions.getElementsInDocumentString,
         pageFunctions.getBoundingClientRectString,

package/lighthouse-core/gather/gatherers/js-usage.js

@@ -18,8 +18,20 @@ class JsUsage extends FRGatherer {
 
   constructor() {
     super();
+    /** @type {LH.Crdp.Debugger.ScriptParsedEvent[]} */
+    this._scriptParsedEvents = [];
     /** @type {LH.Crdp.Profiler.ScriptCoverage[]} */
     this._scriptUsages = [];
+    this.onScriptParsed = this.onScriptParsed.bind(this);
+  }
+
+  /**
+   * @param {LH.Crdp.Debugger.ScriptParsedEvent} event
+   */
+  onScriptParsed(event) {
+    if (event.embedderName) {
+      this._scriptParsedEvents.push(event);
+    }
   }
 
   /**
@@ -43,27 +55,90 @@ class JsUsage extends FRGatherer {
   }
 
   /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   */
+  async startSensitiveInstrumentation(context) {
+    const session = context.driver.defaultSession;
+    session.on('Debugger.scriptParsed', this.onScriptParsed);
+    await session.sendCommand('Debugger.enable');
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   */
+  async stopSensitiveInstrumentation(context) {
+    const session = context.driver.defaultSession;
+    await session.sendCommand('Debugger.disable');
+    session.off('Debugger.scriptParsed', this.onScriptParsed);
+  }
+
+  /**
+   * Usages alone do not always generate an exhaustive list of scripts in timespan and snapshot.
+   * For audits which use this for url/scriptId mappings, we can include an empty usage object.
+   *
+   * @param {Record<string, Array<LH.Crdp.Profiler.ScriptCoverage>>} usageByUrl
+   */
+  _addMissingScriptIds(usageByUrl) {
+    for (const scriptParsedEvent of this._scriptParsedEvents) {
+      const url = scriptParsedEvent.embedderName;
+      if (!url) continue;
+
+      const scripts = usageByUrl[url] || [];
+      if (!scripts.find(s => s.scriptId === scriptParsedEvent.scriptId)) {
+        scripts.push({
+          url,
+          scriptId: scriptParsedEvent.scriptId,
+          functions: [],
+        });
+      }
+      usageByUrl[url] = scripts;
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
    * @return {Promise<LH.Artifacts['JsUsage']>}
    */
-  async getArtifact() {
-    /** @type {Record<string, LH.Crdp.Profiler.ScriptCoverage>} */
-    const usageByScriptId = {};
+  async getArtifact(context) {
+    /** @type {Record<string, Array<LH.Crdp.Profiler.ScriptCoverage>>} */
+    const usageByUrl = {};
+
+    // Force `Debugger.scriptParsed` events for url to scriptId mappings in snapshot mode.
+    if (context.gatherMode === 'snapshot') {
+      await this.startSensitiveInstrumentation(context);
+      await this.stopSensitiveInstrumentation(context);
+    }
 
     for (const scriptUsage of this._scriptUsages) {
-      // If `url` is blank, that means the script was dynamically
-      // created (eval, new Function, onload, ...)
-      if (scriptUsage.url === '' || scriptUsage.url === '_lighthouse-eval.js') {
-        // We currently don't consider coverage of dynamic scripts, and we definitely don't want
-        // coverage of code Lighthouse ran to inspect the page, so we ignore this ScriptCoverage.
-        // Audits would work the same without this, it is only an optimization (not tracking coverage
-        // for scripts we don't care about).
+      // `ScriptCoverage.url` can be overridden by a magic sourceURL comment.
+      // Get the associated ScriptParsedEvent and use embedderName, which is the original url.
+      // See https://chromium-review.googlesource.com/c/v8/v8/+/2317310
+      let url = scriptUsage.url;
+      const scriptParsedEvent =
+        this._scriptParsedEvents.find(e => e.scriptId === scriptUsage.scriptId);
+      if (scriptParsedEvent?.embedderName) {
+        url = scriptParsedEvent.embedderName;
+      }
+
+      // If `url` is blank, that means the script was anonymous (eval, new Function, onload, ...).
+      // Or, it's because it was code Lighthouse over the protocol via `Runtime.evaluate`.
+      // We currently don't consider coverage of anonymous scripts, and we definitely don't want
+      // coverage of code Lighthouse ran to inspect the page, so we ignore this ScriptCoverage if
+      // url is blank.
+      if (scriptUsage.url === '' || (scriptParsedEvent && scriptParsedEvent.embedderName === '')) {
         continue;
       }
 
-      usageByScriptId[scriptUsage.scriptId] = scriptUsage;
+      const scripts = usageByUrl[url] || [];
+      scripts.push(scriptUsage);
+      usageByUrl[url] = scripts;
+    }
+
+    if (context.gatherMode !== 'navigation') {
+      this._addMissingScriptIds(usageByUrl);
     }
 
-    return usageByScriptId;
+    return usageByUrl;
   }
 }
 

package/lighthouse-core/gather/gatherers/link-elements.js

@@ -8,9 +8,10 @@
 const LinkHeader = require('http-link-header');
 const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
 const {URL} = require('../../lib/url-shim.js');
+const NetworkRecords = require('../../computed/network-records.js');
+const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
 const pageFunctions = require('../../lib/page-functions.js');
 const DevtoolsLog = require('./devtools-log.js');
-const MainResource = require('../../computed/main-resource.js');
 
 /* globals HTMLLinkElement getNodeDetails */
 
@@ -113,12 +114,12 @@ class LinkElements extends FRGatherer {
 
   /**
    * @param {LH.Gatherer.FRTransitionalContext} context
-   * @param {LH.Artifacts['DevtoolsLog']} devtoolsLog
-   * @return {Promise<LH.Artifacts['LinkElements']>}
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
+   * @return {LH.Artifacts['LinkElements']}
    */
-  static async getLinkElementsInHeaders(context, devtoolsLog) {
-    const mainDocument =
-      await MainResource.request({devtoolsLog, URL: context.baseArtifacts.URL}, context);
+  static getLinkElementsInHeaders(context, networkRecords) {
+    const finalUrl = context.url;
+    const mainDocument = NetworkAnalyzer.findMainDocument(networkRecords, finalUrl);
 
     /** @type {LH.Artifacts['LinkElements']} */
     const linkElements = [];
@@ -129,7 +130,7 @@ class LinkElements extends FRGatherer {
       for (const link of LinkHeader.parse(header.value).refs) {
         linkElements.push({
           rel: link.rel || '',
-          href: normalizeUrlOrNull(link.uri, context.baseArtifacts.URL.finalUrl),
+          href: normalizeUrlOrNull(link.uri, finalUrl),
           hrefRaw: link.uri || '',
           hreflang: link.hreflang || '',
           as: link.as || '',
@@ -145,12 +146,12 @@ class LinkElements extends FRGatherer {
 
   /**
    * @param {LH.Gatherer.FRTransitionalContext} context
-   * @param {LH.Artifacts['DevtoolsLog']} devtoolsLog
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
    * @return {Promise<LH.Artifacts['LinkElements']>}
    */
-  async _getArtifact(context, devtoolsLog) {
+  async _getArtifact(context, networkRecords) {
     const fromDOM = await LinkElements.getLinkElementsInDOM(context);
-    const fromHeaders = await LinkElements.getLinkElementsInHeaders(context, devtoolsLog);
+    const fromHeaders = LinkElements.getLinkElementsInHeaders(context, networkRecords);
     const linkElements = fromDOM.concat(fromHeaders);
 
     for (const link of linkElements) {
@@ -167,7 +168,7 @@ class LinkElements extends FRGatherer {
    * @return {Promise<LH.Artifacts['LinkElements']>}
    */
   async afterPass(context, loadData) {
-    return this._getArtifact({...context, dependencies: {}}, loadData.devtoolsLog);
+    return this._getArtifact({...context, dependencies: {}}, loadData.networkRecords);
   }
 
   /**
@@ -175,7 +176,8 @@ class LinkElements extends FRGatherer {
    * @return {Promise<LH.Artifacts['LinkElements']>}
    */
   async getArtifact(context) {
-    return this._getArtifact(context, context.dependencies.DevtoolsLog);
+    const records = await NetworkRecords.request(context.dependencies.DevtoolsLog, context);
+    return this._getArtifact(context, records);
   }
 }
 

package/lighthouse-core/gather/gatherers/main-document-content.js

@@ -6,9 +6,10 @@
 'use strict';
 
 const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
+const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
+const NetworkRecords = require('../../computed/network-records.js');
 const DevtoolsLog = require('./devtools-log.js');
 const {fetchResponseBodyFromCache} = require('../driver/network.js');
-const MainResource = require('../../computed/main-resource.js');
 
 /**
  * Collects the content of the main html document.
@@ -23,16 +24,14 @@ class MainDocumentContent extends FRGatherer {
   /**
    *
    * @param {LH.Gatherer.FRTransitionalContext} context
-   * @param {LH.Artifacts['DevtoolsLog']} devtoolsLog
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
    * @return {Promise<LH.Artifacts['MainDocumentContent']>}
    */
-  async _getArtifact(context, devtoolsLog) {
-    const mainResource =
-      await MainResource.request({devtoolsLog, URL: context.baseArtifacts.URL}, context);
+  async _getArtifact(context, networkRecords) {
+    const mainResource = NetworkAnalyzer.findMainDocument(networkRecords, context.url);
     const session = context.driver.defaultSession;
     return fetchResponseBodyFromCache(session, mainResource.requestId);
   }
-
   /**
    *
    * @param {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} context
@@ -40,7 +39,8 @@ class MainDocumentContent extends FRGatherer {
    */
   async getArtifact(context) {
     const devtoolsLog = context.dependencies.DevtoolsLog;
-    return this._getArtifact(context, devtoolsLog);
+    const networkRecords = await NetworkRecords.request(devtoolsLog, context);
+    return this._getArtifact(context, networkRecords);
   }
 
   /**
@@ -49,7 +49,7 @@ class MainDocumentContent extends FRGatherer {
    * @return {Promise<LH.Artifacts['MainDocumentContent']>}
    */
   async afterPass(passContext, loadData) {
-    return this._getArtifact({...passContext, dependencies: {}}, loadData.devtoolsLog);
+    return this._getArtifact({...passContext, dependencies: {}}, loadData.networkRecords);
   }
 }
 

package/lighthouse-core/gather/gatherers/script-elements.js

@@ -7,8 +7,10 @@
 
 const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
 const NetworkRecords = require('../../computed/network-records.js');
+const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
 const NetworkRequest = require('../../lib/network-request.js');
 const pageFunctions = require('../../lib/page-functions.js');
+const {fetchResponseBodyFromCache} = require('../driver/network.js');
 const DevtoolsLog = require('./devtools-log.js');
 
 /* global getNodeDetails */
@@ -30,6 +32,8 @@ function collectAllScriptElements() {
       async: script.async,
       defer: script.defer,
       source: script.closest('head') ? 'head' : 'body',
+      content: script.src ? null : script.text,
+      requestId: null,
       // @ts-expect-error - getNodeDetails put into scope via stringification
       node: getNodeDetails(script),
     };
@@ -37,6 +41,27 @@ function collectAllScriptElements() {
 }
 /* c8 ignore stop */
 
+/**
+ * @template T, U
+ * @param {Array<T>} values
+ * @param {(value: T) => Promise<U>} promiseMapper
+ * @param {boolean} runInSeries
+ * @return {Promise<Array<U>>}
+ */
+async function runInSeriesOrParallel(values, promiseMapper, runInSeries) {
+  if (runInSeries) {
+    const results = [];
+    for (const value of values) {
+      const result = await promiseMapper(value);
+      results.push(result);
+    }
+    return results;
+  } else {
+    const promises = values.map(promiseMapper);
+    return await Promise.all(promises);
+  }
+}
+
 /**
  * @fileoverview Gets JavaScript file contents.
  */
@@ -50,10 +75,13 @@ class ScriptElements extends FRGatherer {
   /**
    * @param {LH.Gatherer.FRTransitionalContext} context
    * @param {LH.Artifacts.NetworkRequest[]} networkRecords
+   * @param {LH.Artifacts['HostFormFactor']} formFactor
    * @return {Promise<LH.Artifacts['ScriptElements']>}
    */
-  async _getArtifact(context, networkRecords) {
+  async _getArtifact(context, networkRecords, formFactor) {
+    const session = context.driver.defaultSession;
     const executionContext = context.driver.executionContext;
+    const mainResource = NetworkAnalyzer.findOptionalMainDocument(networkRecords, context.url);
 
     const scripts = await executionContext.evaluate(collectAllScriptElements, {
       args: [],
@@ -64,16 +92,34 @@ class ScriptElements extends FRGatherer {
       ],
     });
 
+    for (const script of scripts) {
+      if (mainResource && script.content) script.requestId = mainResource.requestId;
+    }
+
     const scriptRecords = networkRecords
-      .filter(record => record.resourceType === NetworkRequest.TYPES.Script)
       // Ignore records from OOPIFs
-      .filter(record => !record.sessionId);
+      .filter(record => !record.sessionId)
+      // Only get the content of script requests
+      .filter(record => record.resourceType === NetworkRequest.TYPES.Script);
+
+    // If run on a mobile device, be sensitive to memory limitations and only request one
+    // record at a time.
+    const scriptRecordContents = await runInSeriesOrParallel(
+      scriptRecords,
+      record => fetchResponseBodyFromCache(session, record.requestId).catch(() => ''),
+      formFactor === 'mobile' /* runInSeries */
+    );
 
     for (let i = 0; i < scriptRecords.length; i++) {
       const record = scriptRecords[i];
+      const content = scriptRecordContents[i];
+      if (!content) continue;
 
       const matchedScriptElement = scripts.find(script => script.src === record.url);
-      if (!matchedScriptElement) {
+      if (matchedScriptElement) {
+        matchedScriptElement.requestId = record.requestId;
+        matchedScriptElement.content = content;
+      } else {
         scripts.push({
           type: null,
           src: record.url,
@@ -81,11 +127,12 @@ class ScriptElements extends FRGatherer {
           async: false,
           defer: false,
           source: 'network',
+          requestId: record.requestId,
+          content,
           node: null,
         });
       }
     }
-
     return scripts;
   }
 
@@ -94,8 +141,9 @@ class ScriptElements extends FRGatherer {
    */
   async getArtifact(context) {
     const devtoolsLog = context.dependencies.DevtoolsLog;
+    const formFactor = context.baseArtifacts.HostFormFactor;
     const networkRecords = await NetworkRecords.request(devtoolsLog, context);
-    return this._getArtifact(context, networkRecords);
+    return this._getArtifact(context, networkRecords, formFactor);
   }
 
   /**
@@ -104,7 +152,8 @@ class ScriptElements extends FRGatherer {
    */
   async afterPass(passContext, loadData) {
     const networkRecords = loadData.networkRecords;
-    return this._getArtifact({...passContext, dependencies: {}}, networkRecords);
+    const formFactor = passContext.baseArtifacts.HostFormFactor;
+    return this._getArtifact({...passContext, dependencies: {}}, networkRecords, formFactor);
   }
 }
 

package/lighthouse-core/gather/gatherers/seo/robots-txt.js

@@ -7,6 +7,25 @@
 
 const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js');
 
+/* global fetch, location */
+
+/** @return {Promise<LH.Artifacts['RobotsTxt']>} */
+/* c8 ignore start */
+async function getRobotsTxtContent() {
+  try {
+    const response = await fetch(new URL('/robots.txt', location.href).href);
+    if (!response.ok) {
+      return {status: response.status, content: null};
+    }
+
+    const content = await response.text();
+    return {status: response.status, content};
+  } catch (err) {
+    return {status: null, content: null, errorMessage: err.message};
+  }
+}
+/* c8 ignore stop */
+
 class RobotsTxt extends FRGatherer {
   /** @type {LH.Gatherer.GathererMeta} */
   meta = {
@@ -18,8 +37,17 @@ class RobotsTxt extends FRGatherer {
    * @return {Promise<LH.Artifacts['RobotsTxt']>}
    */
   async getArtifact(passContext) {
-    const {finalUrl} = passContext.baseArtifacts.URL;
-    const robotsUrl = new URL('/robots.txt', finalUrl).href;
+    // Iframe fetcher still has issues with CSPs.
+    // Only use the fetcher if we are fetching over the protocol.
+    if (await passContext.driver.fetcher.shouldUseLegacyFetcher()) {
+      return passContext.driver.executionContext.evaluate(getRobotsTxtContent, {
+        args: [],
+        useIsolation: true,
+      });
+    }
+
+    const robotsUrl = new URL('/robots.txt', passContext.url).href;
+    await passContext.driver.fetcher.enable();
     return passContext.driver.fetcher.fetchResource(robotsUrl)
       .catch(err => ({status: null, content: null, errorMessage: err.message}));
   }