lighthouse 9.5.0-dev.20220319 → 9.5.0-dev.20220322

package/lighthouse-core/audits/byte-efficiency/duplicated-javascript.js

@@ -163,14 +163,13 @@ class DuplicatedJavascript extends ByteEfficiencyAudit {
         /** @type {number|undefined} */
         let transferRatio = transferRatioByUrl.get(url);
         if (transferRatio === undefined) {
-          const networkRecord = getRequestForScript(networkRecords, script);
-
           if (!script || script.length === undefined) {
             // This should never happen because we found the wasted bytes from bundles, which required contents in a ScriptElement.
             continue;
           }
 
           const contentLength = script.length;
+          const networkRecord = getRequestForScript(networkRecords, script);
           transferRatio = DuplicatedJavascript._estimateTransferRatio(networkRecord, contentLength);
           transferRatioByUrl.set(url, transferRatio);
         }

package/lighthouse-core/audits/byte-efficiency/legacy-javascript.js

@@ -373,12 +373,12 @@ class LegacyJavascript extends ByteEfficiencyAudit {
     if (transferRatio !== undefined) return transferRatio;
 
     const script = artifacts.Scripts.find(script => script.url === url);
-    const networkRecord = getRequestForScript(networkRecords, script);
 
     if (!script || script.content === null) {
       // Can't find content, so just use 1.
       transferRatio = 1;
     } else {
+      const networkRecord = getRequestForScript(networkRecords, script);
       const contentLength = script.length || 0;
       const transferSize =
         ByteEfficiencyAudit.estimateTransferSize(networkRecord, contentLength, 'Script');

package/lighthouse-core/audits/diagnostics.js

@@ -9,7 +9,7 @@ const Audit = require('./audit.js');
 const MainThreadTasksComputed = require('../computed/main-thread-tasks.js');
 const NetworkRecordsComputed = require('../computed/network-records.js');
 const NetworkAnalysisComputed = require('../computed/network-analysis.js');
-const NetworkAnalyzer = require('../lib/dependency-graph/simulator/network-analyzer.js');
+const MainResource = require('../computed/main-resource.js');
 
 class Diagnostics extends Audit {
   /**
@@ -22,7 +22,7 @@ class Diagnostics extends Audit {
       title: 'Diagnostics',
       description: 'Collection of useful page vitals.',
       supportedModes: ['navigation'],
-      requiredArtifacts: ['traces', 'devtoolsLogs'],
+      requiredArtifacts: ['URL', 'traces', 'devtoolsLogs'],
     };
   }
 
@@ -37,9 +37,10 @@ class Diagnostics extends Audit {
     const tasks = await MainThreadTasksComputed.request(trace, context);
     const records = await NetworkRecordsComputed.request(devtoolsLog, context);
     const analysis = await NetworkAnalysisComputed.request(devtoolsLog, context);
+    const mainResource = await MainResource.request({devtoolsLog, URL: artifacts.URL}, context);
 
     const toplevelTasks = tasks.filter(t => !t.parent);
-    const mainDocumentTransferSize = NetworkAnalyzer.findMainDocument(records).transferSize;
+    const mainDocumentTransferSize = mainResource.transferSize;
     const totalByteWeight = records.reduce((sum, r) => sum + (r.transferSize || 0), 0);
     const totalTaskTime = toplevelTasks.reduce((sum, t) => sum + (t.duration || 0), 0);
     const maxRtt = Math.max(...analysis.additionalRttByOrigin.values()) + analysis.rtt;

package/lighthouse-core/audits/seo/http-status-code.js

@@ -6,11 +6,10 @@
 'use strict';
 
 const Audit = require('../audit.js');
-const NetworkRecords = require('../../computed/network-records.js');
 const HTTP_UNSUCCESSFUL_CODE_LOW = 400;
 const HTTP_UNSUCCESSFUL_CODE_HIGH = 599;
 const i18n = require('../../lib/i18n/i18n.js');
-const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
+const MainResource = require('../../computed/main-resource.js');
 
 const UIStrings = {
   /** Title of a Lighthouse audit that provides detail on the HTTP status code a page responds with. This descriptive title is shown when the page has responded with a valid HTTP status code. */
@@ -47,8 +46,7 @@ class HTTPStatusCode extends Audit {
   static async audit(artifacts, context) {
     const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
     const URL = artifacts.URL;
-    const networkRecords = await NetworkRecords.request(devtoolsLog, context);
-    const mainResource = NetworkAnalyzer.findMainDocument(networkRecords, URL.finalUrl);
+    const mainResource = await MainResource.request({devtoolsLog, URL}, context);
 
     const statusCode = mainResource.statusCode;
 

package/lighthouse-core/audits/server-response-time.js

@@ -8,8 +8,6 @@
 const Audit = require('./audit.js');
 const i18n = require('../lib/i18n/i18n.js');
 const MainResource = require('../computed/main-resource.js');
-const NetworkRecords = require('../computed/network-records.js');
-const NetworkAnalyzer = require('../lib/dependency-graph/simulator/network-analyzer.js');
 
 const UIStrings = {
   /** Title of a diagnostic audit that provides detail on how long it took from starting a request to when the server started responding. This descriptive title is shown to users when the amount is acceptable and no user action is required. */
@@ -39,7 +37,7 @@ class ServerResponseTime extends Audit {
       title: str_(UIStrings.title),
       failureTitle: str_(UIStrings.failureTitle),
       description: str_(UIStrings.description),
-      supportedModes: ['timespan', 'navigation'],
+      supportedModes: ['navigation'],
       requiredArtifacts: ['devtoolsLogs', 'URL', 'GatherContext'],
     };
   }
@@ -61,20 +59,7 @@ class ServerResponseTime extends Audit {
     const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
 
     /** @type {LH.Artifacts.NetworkRequest} */
-    let mainResource;
-    if (artifacts.GatherContext.gatherMode === 'timespan') {
-      const networkRecords = await NetworkRecords.request(devtoolsLog, context);
-      const optionalMainResource = NetworkAnalyzer.findOptionalMainDocument(
-        networkRecords,
-        artifacts.URL.finalUrl
-      );
-      if (!optionalMainResource) {
-        return {score: null, notApplicable: true};
-      }
-      mainResource = optionalMainResource;
-    } else {
-      mainResource = await MainResource.request({devtoolsLog, URL: artifacts.URL}, context);
-    }
+    const mainResource = await MainResource.request({devtoolsLog, URL: artifacts.URL}, context);
 
     const responseTime = ServerResponseTime.calculateResponseTime(mainResource);
     const passed = responseTime < TOO_SLOW_THRESHOLD_MS;

package/lighthouse-core/computed/page-dependency-graph.js

@@ -394,6 +394,9 @@ class PageDependencyGraph {
     const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
     const cpuNodes = PageDependencyGraph.getCPUNodes(processedTrace);
 
+    // TODO: Remove this usage of `findMainDocument` and create a new function to get the first document request.
+    // https://github.com/GoogleChrome/lighthouse/issues/8984
+    //
     // The main document request is the earliest network request *of type document*.
     // This will be different from the root request when there are server redirects.
     const mainDocumentRequest = NetworkAnalyzer.findMainDocument(networkRecords);

package/lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -224,6 +224,7 @@ async function _navigation(navigationContext) {
   await collectPhaseArtifacts({phase: 'startInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'startSensitiveInstrumentation', ...phaseState});
   const navigateResult = await _navigate(navigationContext);
+  phaseState.baseArtifacts.URL.finalUrl = navigateResult.finalUrl;
   phaseState.url = navigateResult.finalUrl;
   await collectPhaseArtifacts({phase: 'stopSensitiveInstrumentation', ...phaseState});
   await collectPhaseArtifacts({phase: 'stopInstrumentation', ...phaseState});

package/lighthouse-core/gather/gather-runner.js

@@ -80,6 +80,7 @@ class GatherRunner {
         ...passContext.passConfig,
       });
       passContext.url = finalUrl;
+      passContext.baseArtifacts.URL.finalUrl = finalUrl;
       if (passContext.passConfig.loadFailureMode === 'fatal') {
         passContext.LighthouseRunWarnings.push(...warnings);
       }

package/lighthouse-core/gather/gatherers/link-elements.js

@@ -8,10 +8,9 @@
 const LinkHeader = require('http-link-header');
 const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
 const {URL} = require('../../lib/url-shim.js');
-const NetworkRecords = require('../../computed/network-records.js');
-const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
 const pageFunctions = require('../../lib/page-functions.js');
 const DevtoolsLog = require('./devtools-log.js');
+const MainResource = require('../../computed/main-resource.js');
 
 /* globals HTMLLinkElement getNodeDetails */
 
@@ -114,12 +113,12 @@ class LinkElements extends FRGatherer {
 
   /**
    * @param {LH.Gatherer.FRTransitionalContext} context
-   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
-   * @return {LH.Artifacts['LinkElements']}
+   * @param {LH.Artifacts['DevtoolsLog']} devtoolsLog
+   * @return {Promise<LH.Artifacts['LinkElements']>}
    */
-  static getLinkElementsInHeaders(context, networkRecords) {
-    const finalUrl = context.url;
-    const mainDocument = NetworkAnalyzer.findMainDocument(networkRecords, finalUrl);
+  static async getLinkElementsInHeaders(context, devtoolsLog) {
+    const mainDocument =
+      await MainResource.request({devtoolsLog, URL: context.baseArtifacts.URL}, context);
 
     /** @type {LH.Artifacts['LinkElements']} */
     const linkElements = [];
@@ -130,7 +129,7 @@ class LinkElements extends FRGatherer {
       for (const link of LinkHeader.parse(header.value).refs) {
         linkElements.push({
           rel: link.rel || '',
-          href: normalizeUrlOrNull(link.uri, finalUrl),
+          href: normalizeUrlOrNull(link.uri, context.baseArtifacts.URL.finalUrl),
           hrefRaw: link.uri || '',
           hreflang: link.hreflang || '',
           as: link.as || '',
@@ -146,12 +145,12 @@ class LinkElements extends FRGatherer {
 
   /**
    * @param {LH.Gatherer.FRTransitionalContext} context
-   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
+   * @param {LH.Artifacts['DevtoolsLog']} devtoolsLog
    * @return {Promise<LH.Artifacts['LinkElements']>}
    */
-  async _getArtifact(context, networkRecords) {
+  async _getArtifact(context, devtoolsLog) {
     const fromDOM = await LinkElements.getLinkElementsInDOM(context);
-    const fromHeaders = LinkElements.getLinkElementsInHeaders(context, networkRecords);
+    const fromHeaders = await LinkElements.getLinkElementsInHeaders(context, devtoolsLog);
     const linkElements = fromDOM.concat(fromHeaders);
 
     for (const link of linkElements) {
@@ -168,7 +167,7 @@ class LinkElements extends FRGatherer {
    * @return {Promise<LH.Artifacts['LinkElements']>}
    */
   async afterPass(context, loadData) {
-    return this._getArtifact({...context, dependencies: {}}, loadData.networkRecords);
+    return this._getArtifact({...context, dependencies: {}}, loadData.devtoolsLog);
   }
 
   /**
@@ -176,8 +175,7 @@ class LinkElements extends FRGatherer {
    * @return {Promise<LH.Artifacts['LinkElements']>}
    */
   async getArtifact(context) {
-    const records = await NetworkRecords.request(context.dependencies.DevtoolsLog, context);
-    return this._getArtifact(context, records);
+    return this._getArtifact(context, context.dependencies.DevtoolsLog);
   }
 }
 

package/lighthouse-core/gather/gatherers/main-document-content.js

@@ -6,10 +6,9 @@
 'use strict';
 
 const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
-const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
-const NetworkRecords = require('../../computed/network-records.js');
 const DevtoolsLog = require('./devtools-log.js');
 const {fetchResponseBodyFromCache} = require('../driver/network.js');
+const MainResource = require('../../computed/main-resource.js');
 
 /**
  * Collects the content of the main html document.
@@ -24,14 +23,16 @@ class MainDocumentContent extends FRGatherer {
   /**
    *
    * @param {LH.Gatherer.FRTransitionalContext} context
-   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
+   * @param {LH.Artifacts['DevtoolsLog']} devtoolsLog
    * @return {Promise<LH.Artifacts['MainDocumentContent']>}
    */
-  async _getArtifact(context, networkRecords) {
-    const mainResource = NetworkAnalyzer.findMainDocument(networkRecords, context.url);
+  async _getArtifact(context, devtoolsLog) {
+    const mainResource =
+      await MainResource.request({devtoolsLog, URL: context.baseArtifacts.URL}, context);
     const session = context.driver.defaultSession;
     return fetchResponseBodyFromCache(session, mainResource.requestId);
   }
+
   /**
    *
    * @param {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} context
@@ -39,8 +40,7 @@ class MainDocumentContent extends FRGatherer {
    */
   async getArtifact(context) {
     const devtoolsLog = context.dependencies.DevtoolsLog;
-    const networkRecords = await NetworkRecords.request(devtoolsLog, context);
-    return this._getArtifact(context, networkRecords);
+    return this._getArtifact(context, devtoolsLog);
   }
 
   /**
@@ -49,7 +49,7 @@ class MainDocumentContent extends FRGatherer {
    * @return {Promise<LH.Artifacts['MainDocumentContent']>}
    */
   async afterPass(passContext, loadData) {
-    return this._getArtifact({...passContext, dependencies: {}}, loadData.networkRecords);
+    return this._getArtifact({...passContext, dependencies: {}}, loadData.devtoolsLog);
   }
 }
 

package/lighthouse-core/lib/script-helpers.js

@@ -7,11 +7,10 @@
 
 /**
  * @param {LH.Artifacts.NetworkRequest[]} networkRecords
- * @param {LH.Artifacts.Script|undefined} script
+ * @param {LH.Artifacts.Script} script
  * @return {LH.Artifacts.NetworkRequest|undefined}
  */
 function getRequestForScript(networkRecords, script) {
-  if (!script) return;
   let networkRequest = networkRecords.find(request => request.url === script.url);
   while (networkRequest?.redirectDestination) {
     networkRequest = networkRequest.redirectDestination;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse",
-  "version": "9.5.0-dev.20220319",
+  "version": "9.5.0-dev.20220322",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./lighthouse-core/index.js",
   "bin": {

package/report/generator/report-generator.js

@@ -93,29 +93,54 @@ class ReportGenerator {
     const separator = ',';
     /** @param {string} value @return {string} */
     const escape = value => `"${value.replace(/"/g, '""')}"`;
-    /** @param {Array<string | number>} row @return {string[]} */
-    const rowFormatter = row => row.map(value => value.toString()).map(escape);
-
-    // Possible TODO: tightly couple headers and row values
-    const header = ['requestedUrl', 'finalUrl', 'category', 'name', 'title', 'type', 'score'];
-    const table = Object.keys(lhr.categories).map(categoryId => {
-      const rows = [];
-      const category = lhr.categories[categoryId];
-      const overallCategoryScore = category.score === null ? -1 : category.score;
-      rows.push(rowFormatter([lhr.requestedUrl, lhr.finalUrl, category.title,
-        `${categoryId}-score`, `Overall ${category.title} Category Score`, 'numeric',
-        overallCategoryScore]));
-      return rows.concat(category.auditRefs.map(auditRef => {
+    /** @param {ReadonlyArray<string | number | null>} row @return {string[]} */
+    const rowFormatter = row => row.map(value => {
+      if (value === null) return 'null';
+      return value.toString();
+    }).map(escape);
+
+    const rows = [];
+    const topLevelKeys = /** @type {const} */(
+      ['requestedUrl', 'finalUrl', 'fetchTime', 'gatherMode']);
+
+    // First we have metadata about the LHR.
+    rows.push(rowFormatter(topLevelKeys));
+    rows.push(rowFormatter(topLevelKeys.map(key => lhr[key])));
+
+    // Some spacing.
+    rows.push([]);
+
+    // Categories.
+    rows.push(['category', 'score']);
+    for (const category of Object.values(lhr.categories)) {
+      rows.push(rowFormatter([
+        category.id,
+        category.score,
+      ]));
+    }
+
+    rows.push([]);
+
+    // Audits.
+    rows.push(['category', 'audit', 'score', 'displayValue', 'description']);
+    for (const category of Object.values(lhr.categories)) {
+      for (const auditRef of category.auditRefs) {
         const audit = lhr.audits[auditRef.id];
-        // CSV validator wants all scores to be numeric, use -1 for now
-        const numericScore = audit.score === null ? -1 : audit.score;
-        return rowFormatter([lhr.requestedUrl, lhr.finalUrl, category.title, audit.id, audit.title,
-          audit.scoreDisplayMode, numericScore]);
-      }));
-    });
+        if (!audit) continue;
+
+        rows.push(rowFormatter([
+          category.id,
+          auditRef.id,
+          audit.score,
+          audit.displayValue || '',
+          audit.description,
+        ]));
+      }
+    }
 
-    return [header].concat(...table)
-      .map(row => row.join(separator)).join(CRLF);
+    return rows
+      .map(row => row.join(separator))
+      .join(CRLF);
   }
 
   /**

package/report/test/generator/report-generator-test.js

@@ -5,6 +5,8 @@
  */
 'use strict';
 
+/* eslint-disable no-irregular-whitespace */
+
 const assert = require('assert').strict;
 const fs = require('fs');
 
@@ -81,28 +83,33 @@ describe('ReportGenerator', () => {
 
     it('creates CSV for results', async () => {
       const path = './.results-as-csv.csv';
-      const headers = {
-        category: '',
-        name: '',
-        title: '',
-        type: '',
-        score: 42,
-      };
 
       const csvOutput = ReportGenerator.generateReport(sampleResults, 'csv');
       fs.writeFileSync(path, csvOutput);
 
       const lines = csvOutput.split('\n');
       expect(lines.length).toBeGreaterThan(100);
-      expect(lines.slice(0, 3).join('\n')).toMatchInlineSnapshot(`
-"requestedUrl,finalUrl,category,name,title,type,score
-\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"performance-score\\",\\"Overall Performance Category Score\\",\\"numeric\\",\\"0.26\\"
-\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"first-contentful-paint\\",\\"First Contentful Paint\\",\\"numeric\\",\\"0.01\\"
+      expect(lines.slice(0, 15).join('\n')).toMatchInlineSnapshot(`
+"\\"requestedUrl\\",\\"finalUrl\\",\\"fetchTime\\",\\"gatherMode\\"
+\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"2021-09-07T20:11:11.853Z\\",\\"navigation\\"
+
+category,score
+\\"performance\\",\\"0.26\\"
+\\"accessibility\\",\\"0.78\\"
+\\"best-practices\\",\\"0.25\\"
+\\"seo\\",\\"0.67\\"
+\\"pwa\\",\\"0.3\\"
+
+category,audit,score,displayValue,description
+\\"performance\\",\\"first-contentful-paint\\",\\"0.01\\",\\"6.8 s\\",\\"First Contentful Paint marks the time at which the first text or image is painted. [Learn more](https://web.dev/first-contentful-paint/).\\"
+\\"performance\\",\\"interactive\\",\\"0.41\\",\\"8.2 s\\",\\"Time to interactive is the amount of time it takes for the page to become fully interactive. [Learn more](https://web.dev/interactive/).\\"
+\\"performance\\",\\"speed-index\\",\\"0.21\\",\\"8.1 s\\",\\"Speed Index shows how quickly the contents of a page are visibly populated. [Learn more](https://web.dev/speed-index/).\\"
+\\"performance\\",\\"total-blocking-time\\",\\"0.2\\",\\"1,220 ms\\",\\"Sum of all time periods between FCP and Time to Interactive, when task length exceeded 50ms, expressed in milliseconds. [Learn more](https://web.dev/lighthouse-total-blocking-time/).\\"
 "
 `);
 
       try {
-        await csvValidator(path, headers);
+        await csvValidator(path);
       } catch (err) {
         assert.fail('CSV parser error:\n' + err.join('\n'));
       } finally {
@@ -110,13 +117,13 @@ describe('ReportGenerator', () => {
       }
     });
 
-    it('creates CSV for results including overall category scores', () => {
+    it('creates CSV for results including categories', () => {
       const csvOutput = ReportGenerator.generateReport(sampleResults, 'csv');
-      expect(csvOutput).toContain('performance-score');
-      expect(csvOutput).toContain('accessibility-score');
-      expect(csvOutput).toContain('best-practices-score');
-      expect(csvOutput).toContain('seo-score');
-      expect(csvOutput).toContain('pwa-score');
+      expect(csvOutput).toContain('performance');
+      expect(csvOutput).toContain('accessibility');
+      expect(csvOutput).toContain('best-practices');
+      expect(csvOutput).toContain('seo');
+      expect(csvOutput).toContain('pwa');
     });
 
     it('writes extended info', () => {