lighthouse 8.6.0 → 9.0.0-dev.20211118

package/lighthouse-core/gather/gatherers/accessibility.js

@@ -87,10 +87,37 @@ function createAxeRuleResultArtifact(result) {
     // @ts-expect-error - getNodeDetails put into scope via stringification
     const nodeDetails = getNodeDetails(/** @type {HTMLElement} */ (element));
 
+    /** @type {Set<HTMLElement>} */
+    const relatedNodeElements = new Set();
+    /** @param {import('axe-core/axe').ImpactValue} impact */
+    const impactToNumber =
+      (impact) => [null, 'minor', 'moderate', 'serious', 'critical'].indexOf(impact);
+    const checkResults = [...node.any, ...node.all, ...node.none]
+      // @ts-expect-error CheckResult.impact is a string, even though ImpactValue is a thing.
+      .sort((a, b) => impactToNumber(b.impact) - impactToNumber(a.impact));
+    for (const checkResult of checkResults) {
+      for (const relatedNode of checkResult.relatedNodes || []) {
+        /** @type {HTMLElement} */
+        // @ts-expect-error - should always exist, just being cautious.
+        const relatedElement = relatedNode.element;
+
+        // Prevent overloading the report with way too many nodes.
+        if (relatedNodeElements.size >= 3) break;
+        // Should always exist, just being cautious.
+        if (!relatedElement) continue;
+        if (element === relatedElement) continue;
+
+        relatedNodeElements.add(relatedElement);
+      }
+    }
+    // @ts-expect-error - getNodeDetails put into scope via stringification
+    const relatedNodeDetails = [...relatedNodeElements].map(getNodeDetails);
+
     return {
       target,
       failureSummary,
       node: nodeDetails,
+      relatedNodes: relatedNodeDetails,
     };
   });
 

package/lighthouse-core/gather/gatherers/dobetterweb/response-compression.js

@@ -10,6 +10,7 @@
   */
 'use strict';
 
+const {Buffer} = require('buffer');
 const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js');
 const URL = require('../../../lib/url-shim.js');
 const Sentry = require('../../../lib/sentry.js');

package/lighthouse-core/gather/gatherers/image-elements.js

@@ -12,7 +12,6 @@
 const log = require('lighthouse-logger');
 const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
 const pageFunctions = require('../../lib/page-functions.js');
-const URL = require('../../lib/url-shim.js');
 const FontSize = require('./seo/font-size.js');
 
 /* global window, getElementsInDocument, Image, getNodeDetails, ShadowRoot */
@@ -310,8 +309,6 @@ class ImageElements extends FRGatherer {
     let skippedCount = 0;
 
     for (const element of elements) {
-      element.mimeType = URL.guessMimeType(element.src);
-
       if (reachedGatheringBudget) {
         skippedCount++;
         continue;

package/lighthouse-core/gather/gatherers/inspector-issues.js

@@ -69,6 +69,8 @@ class InspectorIssues extends FRGatherer {
       heavyAds: [],
       /** @type {Array<LH.Crdp.Audits.ContentSecurityPolicyIssueDetails>} */
       contentSecurityPolicy: [],
+      /** @type {Array<LH.Crdp.Audits.DeprecationIssueDetails>} */
+      deprecations: [],
     };
 
     for (const issue of this._issues) {
@@ -105,6 +107,9 @@ class InspectorIssues extends FRGatherer {
       if (issue.details.contentSecurityPolicyIssueDetails) {
         artifact.contentSecurityPolicy.push(issue.details.contentSecurityPolicyIssueDetails);
       }
+      if (issue.details.deprecationIssueDetails) {
+        artifact.deprecations.push(issue.details.deprecationIssueDetails);
+      }
     }
 
     return artifact;

package/lighthouse-core/lib/asset-saver.js

@@ -16,7 +16,10 @@ const Metrics = require('./traces/pwmetrics-events.js');
 const NetworkAnalysisComputed = require('../computed/network-analysis.js');
 const LoadSimulatorComputed = require('../computed/load-simulator.js');
 const LHError = require('../lib/lh-error.js');
-const pipeline = promisify(stream.pipeline);
+// TODO(esmodules): Rollup does not support `promisfy` or `stream.pipeline`. Bundled files
+// don't need anything in this file except for `stringifyReplacer`, so a check for
+// truthiness before using is enough.
+const pipeline = promisify && promisify(stream.pipeline);
 
 const artifactsFilename = 'artifacts.json';
 const traceSuffix = '.trace.json';
@@ -235,9 +238,9 @@ async function saveTrace(traceData, traceFilename) {
   const traceIter = traceJsonGenerator(traceData);
   const writeStream = fs.createWriteStream(traceFilename);
 
-  // TODO: Can remove Readable.from() in Node 13, promisify(pipeline) in Node 15.
+  // TODO: Can remove promisify(pipeline) in Node 15.
   // https://nodejs.org/api/stream.html#stream_stream_pipeline_streams_callback
-  return pipeline(stream.Readable.from(traceIter), writeStream);
+  return pipeline(traceIter, writeStream);
 }
 
 /**
@@ -250,7 +253,7 @@ function saveDevtoolsLog(devtoolsLog, devtoolLogFilename) {
   const logIter = arrayOfObjectsJsonGenerator(devtoolsLog);
   const writeStream = fs.createWriteStream(devtoolLogFilename);
 
-  return pipeline(stream.Readable.from(logIter), writeStream);
+  return pipeline(logIter, writeStream);
 }
 
 /**

package/lighthouse-core/lib/emulation.js

@@ -5,6 +5,8 @@
  */
 'use strict';
 
+const {version: lighthouseVersion} = require('../../package.json');
+
 const NO_THROTTLING_METRICS = {
   latency: 0,
   downloadThroughput: 0,
@@ -16,6 +18,46 @@ const NO_CPU_THROTTLE_METRICS = {
   rate: 1,
 };
 
+/**
+ * @param {string} userAgent
+ * @param {LH.Config.Settings['formFactor']} formFactor
+ * @return {LH.Crdp.Emulation.SetUserAgentOverrideRequest['userAgentMetadata']}
+ */
+function parseUseragentIntoMetadata(userAgent, formFactor) {
+  const match = userAgent.match(/Chrome\/([\d.]+)/); // eg 'Chrome/(71.0.3577.0)'
+  const fullVersion = (match && match[1]) || '99.0.1234.0';
+  const [version] = fullVersion.split('.', 1);
+  const brands = [
+    {brand: 'Chromium', version},
+    {brand: 'Google Chrome', version},
+    {brand: 'Lighthouse', version: lighthouseVersion},
+  ];
+
+  const motoG4Details = {
+    platform: 'Android',
+    platformVersion: '6.0',
+    architecture: '',
+    model: 'Moto G4',
+  };
+  const macDesktopDetails = {
+    platform: 'macOS',
+    platformVersion: '10.15.7',
+    architecture: 'x86',
+    model: '',
+  };
+  const mobile = formFactor === 'mobile';
+
+  return {
+    brands,
+    fullVersion,
+    // Since config users can supply a custom useragent, they likely are emulating something
+    // other than Moto G4 and MacOS Desktop.
+    // TODO: Determine how to thoughtfully expose this metadata/client-hints configurability.
+    ...(mobile ? motoG4Details : macDesktopDetails),
+    mobile,
+  };
+}
+
 /**
  * @param {LH.Gatherer.FRProtocolSession} session
  * @param {LH.Config.Settings} settings
@@ -23,8 +65,10 @@ const NO_CPU_THROTTLE_METRICS = {
  */
 async function emulate(session, settings) {
   if (settings.emulatedUserAgent !== false) {
+    const userAgent = /** @type {string} */ (settings.emulatedUserAgent);
     await session.sendCommand('Network.setUserAgentOverride', {
-      userAgent: /** @type {string} */ (settings.emulatedUserAgent),
+      userAgent,
+      userAgentMetadata: parseUseragentIntoMetadata(userAgent, settings.formFactor),
     });
   }
   // See devtools-entry for one usecase for disabling screenEmulation

package/lighthouse-core/lib/i18n/i18n.js

@@ -12,9 +12,11 @@ const lookupClosestLocale = require('lookup-closest-locale');
 const {getAvailableLocales} = require('../../../shared/localization/format.js');
 const log = require('lighthouse-logger');
 const {LH_ROOT} = require('../../../root.js');
-const {isIcuMessage, _formatMessage} = require('../../../shared/localization/format.js');
-
-const DEFAULT_LOCALE = 'en';
+const {
+  isIcuMessage,
+  formatMessage,
+  DEFAULT_LOCALE,
+} = require('../../../shared/localization/format.js');
 
 const UIStrings = {
   /** Used to show the duration in milliseconds that something lasted. The `{timeInMs}` placeholder will be replaced with the time duration, shown in milliseconds (e.g. 63 ms) */
@@ -118,23 +120,32 @@ const UIStrings = {
  * - supported locales in Intl formatters
  *
  * If `locale` isn't provided or one could not be found, DEFAULT_LOCALE is returned.
+ *
+ * By default any of the locales Lighthouse has strings for can be returned, but this
+ * can be overriden with `possibleLocales`, useful e.g. when Lighthouse is bundled and
+ * only DEFAULT_LOCALE is available, but `possibleLocales` can be used to select a
+ * locale available to be downloaded on demand.
  * @param {string|string[]=} locales
+ * @param {Array<string>=} possibleLocales
  * @return {LH.Locale}
  */
-function lookupLocale(locales) {
-  // If Node was built with `--with-intl=none`, `Intl` won't exist.
+function lookupLocale(locales, possibleLocales) {
+  // TODO: lookupLocale may need to be split into two functions, one that canonicalizes
+  // locales and one that looks up the best locale filename for a given locale.
+  // e.g. `en-IE` is canonical, but uses `en-GB.json`. See TODO in locales.js
+
   if (typeof Intl !== 'object') {
+    // If Node was built with `--with-intl=none`, `Intl` won't exist.
     throw new Error('Lighthouse must be run in Node with `Intl` support. See https://nodejs.org/api/intl.html for help');
   }
 
-  // TODO: could do more work to sniff out the user's locale
   const canonicalLocales = Intl.getCanonicalLocales(locales);
 
   // Filter by what's available in this runtime.
   const availableLocales = Intl.NumberFormat.supportedLocalesOf(canonicalLocales);
 
   // Get available locales and transform into object to match `lookupClosestLocale`'s API.
-  const localesWithMessages = getAvailableLocales();
+  const localesWithMessages = possibleLocales || getAvailableLocales();
   const localesWithmessagesObj = /** @type {Record<LH.Locale, LhlMessages>} */ (
     Object.fromEntries(localesWithMessages.map(l => [l, {}])));
 
@@ -182,7 +193,7 @@ function createIcuMessageFn(filename, fileStrings) {
     return {
       i18nId,
       values,
-      formattedDefault: _formatMessage(message, values, DEFAULT_LOCALE),
+      formattedDefault: formatMessage(message, values, DEFAULT_LOCALE),
     };
   };
 

package/lighthouse-core/lib/lh-env.js

@@ -5,6 +5,8 @@
  */
 'use strict';
 
+const process = require('process');
+
 module.exports = {
   // NODE_ENV is set to test by jest and by smokehouse CLI runner
   // CI as a catchall for everything we do in GitHub Actions

package/lighthouse-core/lib/page-functions.js

@@ -208,7 +208,8 @@ function computeBenchmarkIndex() {
 
     while (Date.now() - start < 500) {
       let s = '';
-      for (let j = 0; j < 10000; j++) s += 'a'; // eslint-disable-line no-unused-vars
+      for (let j = 0; j < 10000; j++) s += 'a';
+      if (s.length === 1) throw new Error('will never happen, but prevents compiler optimizations');
 
       iterations++;
     }

package/lighthouse-core/lib/proto-preprocessor.js

@@ -29,10 +29,22 @@ function processForProto(lhr) {
   // 'ignore unknown fields' in the language of conversion.
   if (reportJson.configSettings) {
     // The settings that are in both proto and LHR
-    const {formFactor, locale, onlyCategories, channel} = reportJson.configSettings;
+    const {
+      formFactor,
+      locale,
+      onlyCategories,
+      channel,
+      throttling,
+      throttlingMethod} = reportJson.configSettings;
 
     // @ts-expect-error - intentionally only a subset of settings.
-    reportJson.configSettings = {formFactor, locale, onlyCategories, channel};
+    reportJson.configSettings = {
+      formFactor,
+      locale,
+      onlyCategories,
+      channel,
+      throttling,
+      throttlingMethod};
   }
 
   // Remove runtimeError if it is NO_ERROR

package/lighthouse-core/lib/statistics.js

@@ -5,6 +5,12 @@
  */
 'use strict';
 
+// The exact double values for the max and min scores possible in each range.
+const MIN_PASSING_SCORE = 0.90000000000000002220446049250313080847263336181640625;
+const MAX_AVERAGE_SCORE = 0.899999999999999911182158029987476766109466552734375;
+const MIN_AVERAGE_SCORE = 0.5;
+const MAX_FAILING_SCORE = 0.499999999999999944488848768742172978818416595458984375;
+
 /**
  * Approximates the Gauss error function, the probability that a random variable
  * from the standard normal distribution lies within [-x, x]. Moved from
@@ -28,37 +34,6 @@ function erf(x) {
   return sign * (1 - y * Math.exp(-x * x));
 }
 
-/**
- * Creates a log-normal distribution à la traceviewer's statistics package.
- * Specified by providing the median value, at which the score will be 0.5,
- * and the falloff, the initial point of diminishing returns where any
- * improvement in value will yield increasingly smaller gains in score. Both
- * values should be in the same units (e.g. milliseconds). See
- *   https://www.desmos.com/calculator/tx1wcjk8ch
- * for an interactive view of the relationship between these parameters and
- * the typical parameterization (location and shape) of the log-normal
- * distribution.
- * @param {number} median
- * @param {number} falloff
- * @return {{computeComplementaryPercentile: function(number): number}}
- */
-function getLogNormalDistribution(median, falloff) {
-  const location = Math.log(median);
-
-  // The "falloff" value specified the location of the smaller of the positive
-  // roots of the third derivative of the log-normal CDF. Calculate the shape
-  // parameter in terms of that value and the median.
-  const logRatio = Math.log(falloff / median);
-  const shape = Math.sqrt(1 - 3 * logRatio - Math.sqrt((logRatio - 3) * (logRatio - 3) - 8)) / 2;
-
-  return {
-    computeComplementaryPercentile(x) {
-      const standardizedX = (Math.log(x) - location) / (Math.SQRT2 * shape);
-      return (1 - erf(standardizedX)) / 2;
-    },
-  };
-}
-
 /**
  * Returns the score (1 - percentile) of `value` in a log-normal distribution
  * specified by the `median` value, at which the score will be 0.5, and a 10th
@@ -76,24 +51,37 @@ function getLogNormalScore({median, p10}, value) {
   // Required for the log-normal distribution.
   if (median <= 0) throw new Error('median must be greater than zero');
   if (p10 <= 0) throw new Error('p10 must be greater than zero');
-  // Not required, but if p10 > median, it flips around and becomes the p90 point.
+  // Not strictly required, but if p10 > median, it flips around and becomes the p90 point.
   if (p10 >= median) throw new Error('p10 must be less than the median');
 
   // Non-positive values aren't in the distribution, so always 1.
   if (value <= 0) return 1;
 
-  // Closest double to `erfc-1(2 * 1/10)`.
+  // Closest double to `erfc-1(1/5)`.
   const INVERSE_ERFC_ONE_FIFTH = 0.9061938024368232;
 
-  // Shape (σ) is `log(p10/median) / (sqrt(2)*erfc^-1(2 * 1/10))` and
+  // Shape (σ) is `|log(p10/median) / (sqrt(2)*erfc^-1(1/5))|` and
   // standardizedX is `1/2 erfc(log(value/median) / (sqrt(2)*σ))`, so simplify a bit.
-  const xLogRatio = Math.log(value / median);
-  const p10LogRatio = -Math.log(p10 / median); // negate to keep σ positive.
+  const xRatio = Math.max(Number.MIN_VALUE, value / median); // value and median are > 0, so is ratio.
+  const xLogRatio = Math.log(xRatio);
+  const p10Ratio = Math.max(Number.MIN_VALUE, p10 / median); // p10 and median are > 0, so is ratio.
+  const p10LogRatio = -Math.log(p10Ratio); // negate to keep σ positive.
   const standardizedX = xLogRatio * INVERSE_ERFC_ONE_FIFTH / p10LogRatio;
   const complementaryPercentile = (1 - erf(standardizedX)) / 2;
 
-  // Clamp to [0, 1] to avoid any floating-point out-of-bounds issues.
-  return Math.min(1, Math.max(0, complementaryPercentile));
+  // Clamp to avoid floating-point out-of-bounds issues and keep score in expected range.
+  let score;
+  if (value <= p10) {
+    // Passing. Clamp to [0.9, 1].
+    score = Math.max(MIN_PASSING_SCORE, Math.min(1, complementaryPercentile));
+  } else if (value <= median) {
+    // Average. Clamp to [0.5, 0.9).
+    score = Math.max(MIN_AVERAGE_SCORE, Math.min(MAX_AVERAGE_SCORE, complementaryPercentile));
+  } else {
+    // Failing. Clamp to [0, 0.5).
+    score = Math.max(0, Math.min(MAX_FAILING_SCORE, complementaryPercentile));
+  }
+  return score;
 }
 
 /**
@@ -112,6 +100,5 @@ function linearInterpolation(x0, y0, x1, y1, x) {
 
 module.exports = {
   linearInterpolation,
-  getLogNormalDistribution,
   getLogNormalScore,
 };

package/lighthouse-core/lib/tappable-rects.js

@@ -5,14 +5,7 @@
  */
 'use strict';
 
-const {
-  filterOutRectsContainedByOthers,
-  filterOutTinyRects,
-  rectsTouchOrOverlap,
-  rectContainsPoint,
-  getBoundingRect,
-  getRectCenterPoint,
-} = require('./rect-helpers.js');
+const RectHelpers = require('./rect-helpers.js');
 
 /**
  * Merge client rects together and remove small ones. This may result in a larger overall
@@ -23,8 +16,8 @@ const {
 function getTappableRectsFromClientRects(clientRects) {
   // 1x1px rect shouldn't be reason to treat the rect as something the user should tap on.
   // Often they're made invisble in some obscure way anyway, and only exist for e.g. accessibiliity.
-  clientRects = filterOutTinyRects(clientRects);
-  clientRects = filterOutRectsContainedByOthers(clientRects);
+  clientRects = RectHelpers.filterOutTinyRects(clientRects);
+  clientRects = RectHelpers.filterOutRectsContainedByOthers(clientRects);
   clientRects = mergeTouchingClientRects(clientRects);
   return clientRects;
 }
@@ -68,17 +61,17 @@ function mergeTouchingClientRects(clientRects) {
       const rectsLineUpVertically =
         almostEqual(crA.left, crB.left) || almostEqual(crA.right, crB.right);
       const canMerge =
-        rectsTouchOrOverlap(crA, crB) &&
+        RectHelpers.rectsTouchOrOverlap(crA, crB) &&
         (rectsLineUpHorizontally || rectsLineUpVertically);
 
       if (canMerge) {
-        const replacementClientRect = getBoundingRect([crA, crB]);
-        const mergedRectCenter = getRectCenterPoint(replacementClientRect);
+        const replacementClientRect = RectHelpers.getBoundingRect([crA, crB]);
+        const mergedRectCenter = RectHelpers.getRectCenterPoint(replacementClientRect);
 
         if (
           !(
-            rectContainsPoint(crA, mergedRectCenter) ||
-            rectContainsPoint(crB, mergedRectCenter)
+            RectHelpers.rectContainsPoint(crA, mergedRectCenter) ||
+            RectHelpers.rectContainsPoint(crB, mergedRectCenter)
           )
         ) {
           // Don't merge because the new shape is too different from the

package/lighthouse-core/lib/url-shim.js

@@ -9,7 +9,7 @@
  * URL shim so we keep our code DRY
  */
 
-const Util = require('../util-commonjs.js');
+const {Util} = require('../util-commonjs.js');
 
 /** @typedef {import('./network-request.js')} NetworkRequest */
 

package/lighthouse-core/runner.js

@@ -20,6 +20,7 @@ const URL = require('./lib/url-shim.js');
 const Sentry = require('./lib/sentry.js');
 const generateReport = require('../report/generator/report-generator.js').generateReport;
 const LHError = require('./lib/lh-error.js');
+const {version: lighthouseVersion} = require('../package.json');
 
 /** @typedef {import('./gather/connections/connection.js')} Connection */
 /** @typedef {import('./lib/arbitrary-equality-map.js')} ArbitraryEqualityMap */
@@ -112,7 +113,6 @@ class Runner {
       }
 
       // Entering: conclusion of the lighthouse result object
-      const lighthouseVersion = require('../package.json').version;
 
       // Use version from gathering stage.
       // If accessibility gatherer didn't run or errored, it won't be in credits.