lighthouse 9.5.0-dev.20221128 → 9.5.0-dev.20221129

package/cli/test/smokehouse/lighthouse-runners/devtools.js

@@ -47,7 +47,7 @@ async function setup() {
  */
 async function runLighthouse(url, configJson, testRunnerOptions = {}) {
   const chromeFlags = [
-    `--custom-devtools-frontend=file://${devtoolsDir}/out/Default/gen/front_end`,
+    `--custom-devtools-frontend=file://${devtoolsDir}/out/LighthouseIntegration/gen/front_end`,
   ];
   const {lhr, artifacts, logs} = await testUrlFromDevtools(url, {
     config: configJson,

package/core/audits/byte-efficiency/offscreen-images.js

@@ -151,7 +151,7 @@ class OffscreenImages extends ByteEfficiencyAudit {
     return images.filter(image => {
       if (image.wastedBytes < IGNORE_THRESHOLD_IN_BYTES) return false;
       if (image.wastedPercent < IGNORE_THRESHOLD_IN_PERCENT) return false;
-      return image.requestStartTime < interactiveTimestamp / 1e6 - IGNORE_THRESHOLD_IN_MS / 1000;
+      return image.requestStartTime < interactiveTimestamp / 1000 - IGNORE_THRESHOLD_IN_MS;
     });
   }
 

package/core/audits/byte-efficiency/render-blocking-resources.js

@@ -158,7 +158,7 @@ class RenderBlockingResources extends Audit {
     const deferredNodeIds = new Set();
     for (const resource of artifacts.TagsBlockingFirstPaint) {
       // Ignore any resources that finished after observed FCP (they're clearly not render-blocking)
-      if (resource.endTime * 1000 > fcpTsInMs) continue;
+      if (resource.endTime > fcpTsInMs) continue;
       // TODO: beacon to Sentry, https://github.com/GoogleChrome/lighthouse/issues/7041
       if (!nodesByUrl[resource.tag.url]) continue;
 

package/core/audits/critical-request-chains.js

@@ -70,8 +70,8 @@ class CriticalRequestChains extends Audit {
           depth,
           id,
           node: child,
-          chainDuration: (child.request.endTime - startTime) * 1000,
-          chainTransferSize: (transferSize + child.request.transferSize),
+          chainDuration: child.request.endTime - startTime,
+          chainTransferSize: transferSize + child.request.transferSize,
         });
 
         // Carry on walking.
@@ -96,7 +96,7 @@ class CriticalRequestChains extends Audit {
       transferSize: 0,
     };
     CriticalRequestChains._traverse(tree, opts => {
-      const duration = opts.chainDuration;
+      const duration = opts.chainDuration * 1000;
       if (duration > longest.duration) {
         longest.duration = duration;
         longest.transferSize = opts.chainTransferSize;
@@ -123,9 +123,9 @@ class CriticalRequestChains extends Audit {
       const request = opts.node.request;
       const simpleRequest = {
         url: request.url,
-        startTime: request.startTime,
-        endTime: request.endTime,
-        responseReceivedTime: request.responseReceivedTime,
+        startTime: request.startTime / 1000,
+        endTime: request.endTime / 1000,
+        responseReceivedTime: request.responseReceivedTime / 1000,
         transferSize: request.transferSize,
       };
 

package/core/audits/font-display.js

@@ -165,7 +165,7 @@ class FontDisplay extends Audit {
       .map(record => {
         // In reality the end time should be calculated with paint time included
         // all browsers wait 3000ms to block text so we make sure 3000 is our max wasted time
-        const wastedMs = Math.min((record.endTime - record.startTime) * 1000, 3000);
+        const wastedMs = Math.min(record.endTime - record.startTime, 3000);
 
         return {
           url: record.url,

package/core/audits/network-requests.js

@@ -47,8 +47,8 @@ class NetworkRequests extends Audit {
     }
 
     /** @param {number} time */
-    const timeToMs = time => time < earliestStartTime || !Number.isFinite(time) ?
-      undefined : (time - earliestStartTime) * 1000;
+    const normalizeTime = time => time < earliestStartTime || !Number.isFinite(time) ?
+      undefined : (time - earliestStartTime);
 
     const results = records.map(record => {
       const endTimeDeltaMs = record.lrStatistics?.endTimeDeltaMs;
@@ -64,8 +64,8 @@ class NetworkRequests extends Audit {
       return {
         url: UrlUtils.elideDataURI(record.url),
         protocol: record.protocol,
-        startTime: timeToMs(record.startTime),
-        endTime: timeToMs(record.endTime),
+        startTime: normalizeTime(record.startTime),
+        endTime: normalizeTime(record.endTime),
         finished: record.finished,
         transferSize: record.transferSize,
         resourceSize: record.resourceSize,
@@ -112,7 +112,7 @@ class NetworkRequests extends Audit {
 
     // Include starting timestamp to allow syncing requests with navStart/metric timestamps.
     const networkStartTimeTs = Number.isFinite(earliestStartTime) ?
-        earliestStartTime * 1_000_000 : undefined;
+        earliestStartTime * 1000 : undefined;
     tableDetails.debugData = {
       type: 'debugdata',
       networkStartTimeTs,

package/core/audits/redirects.js

@@ -125,9 +125,9 @@ class Redirects extends Audit {
       }
 
       const lanternTimingDeltaMs = redirectedTiming.startTime - initialTiming.startTime;
-      const observedTimingDeltaS = redirectedRequest.startTime - initialRequest.startTime;
+      const observedTimingDeltaMs = redirectedRequest.startTime - initialRequest.startTime;
       const wastedMs = settings.throttlingMethod === 'simulate' ?
-        lanternTimingDeltaMs : observedTimingDeltaS * 1000;
+        lanternTimingDeltaMs : observedTimingDeltaMs;
       totalWastedMs += wastedMs;
 
       tableRows.push({

package/core/audits/uses-rel-preconnect.js

@@ -20,7 +20,7 @@ import {LanternLargestContentfulPaint} from '../computed/metrics/lantern-largest
 // around for 10s. Meaning, the time delta between processing preconnect a request should be <10s,
 // otherwise it's wasted. We add a 5s margin so we are sure to capture all key requests.
 // @see https://github.com/GoogleChrome/lighthouse/issues/3106#issuecomment-333653747
-const PRECONNECT_SOCKET_MAX_IDLE = 15;
+const PRECONNECT_SOCKET_MAX_IDLE_IN_MS = 15_000;
 
 const IGNORE_THRESHOLD_IN_MS = 50;
 
@@ -96,7 +96,7 @@ class UsesRelPreconnectAudit extends Audit {
    * @return {boolean}
    */
   static socketStartTimeIsBelowThreshold(record, mainResource) {
-    return Math.max(0, record.startTime - mainResource.endTime) < PRECONNECT_SOCKET_MAX_IDLE;
+    return Math.max(0, record.startTime - mainResource.endTime) < PRECONNECT_SOCKET_MAX_IDLE_IN_MS;
   }
 
   /**
@@ -150,7 +150,7 @@ class UsesRelPreconnectAudit extends Audit {
           !lcpGraphURLs.has(record.url) ||
           // Filter out all resources where origins are already resolved.
           UsesRelPreconnectAudit.hasAlreadyConnectedToOrigin(record) ||
-          // Make sure the requests are below the PRECONNECT_SOCKET_MAX_IDLE (15s) mark.
+          // Make sure the requests are below the PRECONNECT_SOCKET_MAX_IDLE_IN_MS (15s) mark.
           !UsesRelPreconnectAudit.socketStartTimeIsBelowThreshold(record, mainResource)
         ) {
           return;
@@ -189,8 +189,8 @@ class UsesRelPreconnectAudit extends Audit {
       if (firstRecordOfOrigin.parsedURL.scheme === 'https') connectionTime = connectionTime * 2;
 
       const timeBetweenMainResourceAndDnsStart =
-        firstRecordOfOrigin.startTime * 1000 -
-        mainResource.endTime * 1000 +
+        firstRecordOfOrigin.startTime -
+        mainResource.endTime +
         firstRecordOfOrigin.timing.dnsStart;
 
       const wastedMs = Math.min(connectionTime, timeBetweenMainResourceAndDnsStart);

package/core/config/config-helpers.js

@@ -6,6 +6,7 @@
 
 import path from 'path';
 import {createRequire} from 'module';
+import url from 'url';
 
 import isDeepEqual from 'lodash/isEqual.js';
 
@@ -217,6 +218,11 @@ const bundledModules = new Map(/* BUILD_REPLACE_BUNDLED_MODULES */);
  * @param {string} requirePath
  */
 async function requireWrapper(requirePath) {
+  // For windows.
+  if (path.isAbsolute(requirePath)) {
+    requirePath = url.pathToFileURL(requirePath).href;
+  }
+
   /** @type {any} */
   let module;
   if (bundledModules.has(requirePath)) {
@@ -231,7 +237,6 @@ async function requireWrapper(requirePath) {
   if (module.default) return module.default;
 
   // Find a valid named export.
-  // TODO(esmodules): actually make all the audits/gatherers use named exports
   const methods = new Set(['meta']);
   const possibleNamedExports = Object.keys(module).filter(key => {
     if (!(module[key] && module[key] instanceof Object)) return false;
@@ -287,8 +292,12 @@ function requireAudit(auditPath, coreAuditList, configDir) {
     } else {
       // Otherwise, attempt to find it elsewhere. This throws if not found.
       const absolutePath = resolveModulePath(auditPath, configDir, 'audit');
-      // Use a relative path so bundler can easily expose it.
-      requirePath = path.relative(getModuleDirectory(import.meta), absolutePath);
+      if (isBundledEnvironment()) {
+        // Use a relative path so bundler can easily expose it.
+        requirePath = path.relative(getModuleDirectory(import.meta), absolutePath);
+      } else {
+        requirePath = absolutePath;
+      }
     }
   }
 

package/core/config/default-config.js

@@ -128,6 +128,7 @@ const artifacts = {
   Trace: '',
   Accessibility: '',
   AnchorElements: '',
+  BFCacheFailures: '',
   CacheContents: '',
   ConsoleMessages: '',
   CSSUsage: '',

package/core/gather/base-gatherer.js

@@ -103,5 +103,4 @@ class FRGatherer {
   }
 }
 
-// TODO(esmodules): do not use default export
 export default FRGatherer;

package/core/gather/driver/target-manager.js

@@ -58,15 +58,22 @@ class TargetManager extends ProtocolEventEmitter {
   async _onFrameNavigated(frameNavigatedEvent) {
     // Child frames are handled in `_onSessionAttached`.
     if (frameNavigatedEvent.frame.parentId) return;
+    if (!this._enabled) return;
 
     // It's not entirely clear when this is necessary, but when the page switches processes on
     // navigating from about:blank to the `requestedUrl`, resetting `setAutoAttach` has been
     // necessary in the past.
-    await this._rootCdpSession.send('Target.setAutoAttach', {
-      autoAttach: true,
-      flatten: true,
-      waitForDebuggerOnStart: true,
-    });
+    try {
+      await this._rootCdpSession.send('Target.setAutoAttach', {
+        autoAttach: true,
+        flatten: true,
+        waitForDebuggerOnStart: true,
+      });
+    } catch (err) {
+      // The page can be closed at the end of the run before this CDP function returns.
+      // In these cases, just ignore the error since we won't need the page anyway.
+      if (this._enabled) throw err;
+    }
   }
 
   /**

package/core/gather/driver.js

@@ -84,7 +84,7 @@ class Driver {
   /** @return {Promise<void>} */
   async disconnect() {
     if (this.defaultSession === throwingSession) return;
-    this._targetManager?.disable();
+    await this._targetManager?.disable();
     await this.defaultSession.dispose();
   }
 }

package/core/gather/gatherers/bf-cache-failures.js

@@ -0,0 +1,158 @@
+/**
+ * @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import FRGatherer from '../base-gatherer.js';
+import {waitForFrameNavigated, waitForLoadEvent} from '../driver/wait-for-condition.js';
+import DevtoolsLog from './devtools-log.js';
+
+class BFCacheFailures extends FRGatherer {
+  /** @type {LH.Gatherer.GathererMeta<'DevtoolsLog'>} */
+  meta = {
+    supportedModes: ['navigation', 'timespan'],
+    dependencies: {DevtoolsLog: DevtoolsLog.symbol},
+  };
+
+  /**
+   * @param {LH.Crdp.Page.BackForwardCacheNotRestoredExplanation[]} errorList
+   * @return {LH.Artifacts.BFCacheFailure}
+   */
+  static processBFCacheEventList(errorList) {
+    /** @type {LH.Artifacts.BFCacheNotRestoredReasonsTree} */
+    const notRestoredReasonsTree = {
+      Circumstantial: {},
+      PageSupportNeeded: {},
+      SupportPending: {},
+    };
+
+    for (const err of errorList) {
+      const bfCacheErrorsMap = notRestoredReasonsTree[err.type];
+      bfCacheErrorsMap[err.reason] = [];
+    }
+
+    return {notRestoredReasonsTree};
+  }
+
+  /**
+   * @param {LH.Crdp.Page.BackForwardCacheNotRestoredExplanationTree} errorTree
+   * @return {LH.Artifacts.BFCacheFailure}
+   */
+  static processBFCacheEventTree(errorTree) {
+    /** @type {LH.Artifacts.BFCacheNotRestoredReasonsTree} */
+    const notRestoredReasonsTree = {
+      Circumstantial: {},
+      PageSupportNeeded: {},
+      SupportPending: {},
+    };
+
+    /**
+     * @param {LH.Crdp.Page.BackForwardCacheNotRestoredExplanationTree} node
+     */
+    function traverse(node) {
+      for (const error of node.explanations) {
+        const bfCacheErrorsMap = notRestoredReasonsTree[error.type];
+        const frameUrls = bfCacheErrorsMap[error.reason] || [];
+        frameUrls.push(node.url);
+        bfCacheErrorsMap[error.reason] = frameUrls;
+      }
+
+      for (const child of node.children) {
+        traverse(child);
+      }
+    }
+
+    traverse(errorTree);
+
+    return {notRestoredReasonsTree};
+  }
+
+  /**
+   * @param {LH.Crdp.Page.BackForwardCacheNotUsedEvent|undefined} event
+   * @return {LH.Artifacts.BFCacheFailure}
+   */
+  static processBFCacheEvent(event) {
+    if (event?.notRestoredExplanationsTree) {
+      return BFCacheFailures.processBFCacheEventTree(event.notRestoredExplanationsTree);
+    }
+    return BFCacheFailures.processBFCacheEventList(event?.notRestoredExplanations || []);
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   * @return {Promise<LH.Crdp.Page.BackForwardCacheNotUsedEvent|undefined>}
+   */
+  async activelyCollectBFCacheEvent(context) {
+    const session = context.driver.defaultSession;
+
+    /** @type {LH.Crdp.Page.BackForwardCacheNotUsedEvent|undefined} */
+    let bfCacheEvent = undefined;
+
+    /**
+     * @param {LH.Crdp.Page.BackForwardCacheNotUsedEvent} event
+     */
+    function onBfCacheNotUsed(event) {
+      bfCacheEvent = event;
+    }
+
+    session.on('Page.backForwardCacheNotUsed', onBfCacheNotUsed);
+
+    const history = await session.sendCommand('Page.getNavigationHistory');
+    const entry = history.entries[history.currentIndex];
+
+    await Promise.all([
+      session.sendCommand('Page.navigate', {url: 'about:blank'}),
+      waitForLoadEvent(session, 0).promise,
+    ]);
+
+    await Promise.all([
+      session.sendCommand('Page.navigateToHistoryEntry', {entryId: entry.id}),
+      waitForFrameNavigated(session).promise,
+    ]);
+
+    session.off('Page.backForwardCacheNotUsed', onBfCacheNotUsed);
+
+    return bfCacheEvent;
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} context
+   * @return {LH.Crdp.Page.BackForwardCacheNotUsedEvent[]}
+   */
+  passivelyCollectBFCacheEvents(context) {
+    const events = [];
+    for (const event of context.dependencies.DevtoolsLog) {
+      if (event.method === 'Page.backForwardCacheNotUsed') {
+        events.push(event.params);
+      }
+    }
+    return events;
+  }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} context
+   * @return {Promise<LH.Artifacts['BFCacheFailures']>}
+   */
+  async getArtifact(context) {
+    const events = this.passivelyCollectBFCacheEvents(context);
+    if (context.gatherMode === 'navigation') {
+      const activelyCollectedEvent = await this.activelyCollectBFCacheEvent(context);
+      if (activelyCollectedEvent) events.push(activelyCollectedEvent);
+    }
+
+    return events.map(BFCacheFailures.processBFCacheEvent);
+  }
+
+  /**
+   * @param {LH.Gatherer.PassContext} passContext
+   * @param {LH.Gatherer.LoadData} loadData
+   * @return {Promise<LH.Artifacts['BFCacheFailures']>}
+   */
+  async afterPass(passContext, loadData) {
+    return this.getArtifact({...passContext, dependencies: {DevtoolsLog: loadData.devtoolsLog}});
+  }
+}
+
+export default BFCacheFailures;
+

package/core/gather/gatherers/seo/font-size.js

@@ -34,38 +34,6 @@ function hasFontSizeDeclaration(style) {
   return !!style && !!style.cssProperties.find(({name}) => name === FONT_SIZE_PROPERTY_NAME);
 }
 
-/**
- * Computes the CSS specificity of a given selector, i.e. #id > .class > div
- * TODO: Handle pseudo selectors (:not(), :where, :nth-child) and attribute selectors
- * LIMITATION: !important is not respected
- *
- * @see https://developer.mozilla.org/en-US/docs/Web/CSS/Specificity
- * @see https://www.smashingmagazine.com/2010/04/css-specificity-and-inheritance/
- * @see https://drafts.csswg.org/selectors-4/#specificity-rules
- *
- * @param {string} selector
- * @return {number}
- */
-function computeSelectorSpecificity(selector) {
-  // Remove universal selector and separator characters, then split.
-  const tokens = selector.replace(/[*\s+>~]/g, ' ').split(' ');
-
-  let numIDs = 0;
-  let numClasses = 0;
-  let numTypes = 0;
-
-  for (const token of tokens) {
-    const ids = token.match(/(\b|^)#[a-z0-9_-]+/gi) || [];
-    const classes = token.match(/(\b|^)\.[a-z0-9_-]+/gi) || [];
-    const types = token.match(/^[a-z]+/i) ? [1] : [];
-    numIDs += ids.length;
-    numClasses += classes.length;
-    numTypes += types.length;
-  }
-
-  return Math.min(9, numIDs) * 100 + Math.min(9, numClasses) * 10 + Math.min(9, numTypes);
-}
-
 /**
  * Finds the most specific directly matched CSS font-size rule from the list.
  *
@@ -74,31 +42,21 @@ function computeSelectorSpecificity(selector) {
  * @return {NodeFontData['cssRule']|undefined}
  */
 function findMostSpecificMatchedCSSRule(matchedCSSRules = [], isDeclarationOfInterest) {
-  let maxSpecificity = -Infinity;
-  /** @type {LH.Crdp.CSS.CSSRule|undefined} */
-  let maxSpecificityRule;
-
-  for (const {rule, matchingSelectors} of matchedCSSRules) {
-    if (isDeclarationOfInterest(rule.style)) {
-      const specificities = matchingSelectors.map(idx =>
-        computeSelectorSpecificity(rule.selectorList.selectors[idx].text)
-      );
-      const specificity = Math.max(...specificities);
-      // Use greater OR EQUAL so that the last rule wins in the event of a tie
-      if (specificity >= maxSpecificity) {
-        maxSpecificity = specificity;
-        maxSpecificityRule = rule;
-      }
+  let mostSpecificRule;
+  for (let i = matchedCSSRules.length - 1; i >= 0; i--) {
+    if (isDeclarationOfInterest(matchedCSSRules[i].rule.style)) {
+      mostSpecificRule = matchedCSSRules[i].rule;
+      break;
     }
   }
 
-  if (maxSpecificityRule) {
+  if (mostSpecificRule) {
     return {
       type: 'Regular',
-      ...maxSpecificityRule.style,
+      ...mostSpecificRule.style,
       parentRule: {
-        origin: maxSpecificityRule.origin,
-        selectors: maxSpecificityRule.selectorList.selectors,
+        origin: mostSpecificRule.origin,
+        selectors: mostSpecificRule.selectorList.selectors,
       },
     };
   }
@@ -376,7 +334,6 @@ class FontSize extends FRGatherer {
 
 export default FontSize;
 export {
-  computeSelectorSpecificity,
   getEffectiveFontRule,
   findMostSpecificMatchedCSSRule,
 };

package/core/legacy/config/legacy-default-config.js

@@ -71,6 +71,7 @@ legacyDefaultConfig.passes = [{
     'inspector-issues',
     'source-maps',
     'full-page-screenshot',
+    'bf-cache-failures',
   ],
 },
 {

package/core/lib/asset-saver.js

@@ -20,7 +20,7 @@ import {LighthouseError} from '../lib/lh-error.js';
 
 const {promisify} = util;
 
-// TODO(esmodules): Rollup does not support `promisfy` or `stream.pipeline`. Bundled files
+// Rollup does not support `promisfy` or `stream.pipeline`. Bundled files
 // don't need anything in this file except for `stringifyReplacer`, so a check for
 // truthiness before using is enough.
 // TODO: Can remove promisify(pipeline) in Node 15.

package/core/lib/dependency-graph/base-node.js

@@ -50,6 +50,7 @@ class BaseNode {
   }
 
   /**
+   * In microseconds
    * @return {number}
    */
   get startTime() {
@@ -57,6 +58,7 @@ class BaseNode {
   }
 
   /**
+   * In microseconds
    * @return {number}
    */
   get endTime() {

package/core/lib/dependency-graph/network-node.js

@@ -25,14 +25,14 @@ class NetworkNode extends BaseNode {
    * @return {number}
    */
   get startTime() {
-    return this._record.startTime * 1000 * 1000;
+    return this._record.startTime * 1000;
   }
 
   /**
    * @return {number}
    */
   get endTime() {
-    return this._record.endTime * 1000 * 1000;
+    return this._record.endTime * 1000;
   }
 
   /**

package/core/lib/dependency-graph/simulator/network-analyzer.js

@@ -150,7 +150,7 @@ class NetworkAnalyzer {
       if (!Number.isFinite(timing.receiveHeadersEnd) || timing.receiveHeadersEnd < 0) return;
 
       // Compute the amount of time downloading everything after the first congestion window took
-      const totalTime = (record.endTime - record.startTime) * 1000;
+      const totalTime = record.endTime - record.startTime;
       const downloadTimeAfterFirstByte = totalTime - timing.receiveHeadersEnd;
       const numberOfRoundTrips = Math.log2(record.transferSize / INITIAL_CWD);
 
@@ -399,8 +399,8 @@ class NetworkAnalyzer {
 
       // If we've made it this far, all the times we need should be valid (i.e. not undefined/-1).
       totalBytes += record.transferSize;
-      boundaries.push({time: record.responseReceivedTime, isStart: true});
-      boundaries.push({time: record.endTime, isStart: false});
+      boundaries.push({time: record.responseReceivedTime / 1000, isStart: true});
+      boundaries.push({time: record.endTime / 1000, isStart: false});
       return boundaries;
     }, /** @type {Array<{time: number, isStart: boolean}>} */([])).sort((a, b) => a.time - b.time);
 

package/core/lib/dependency-graph/simulator/simulator.js

@@ -236,8 +236,11 @@ class Simulator {
    * currently in flight.
    */
   _updateNetworkCapacity() {
+    const inFlight = this._numberInProgress(BaseNode.TYPES.NETWORK);
+    if (inFlight === 0) return;
+
     for (const connection of this._connectionPool.connectionsInUse()) {
-      connection.setThroughput(this._throughput / this._nodes[NodeState.InProgress].size);
+      connection.setThroughput(this._throughput / inFlight);
     }
   }