lighthouse 9.3.1-dev.20220202 → 9.3.1-dev.20220203

package/lighthouse-cli/test/smokehouse/core-tests.js

@@ -60,6 +60,7 @@ import seoPassing from './test-definitions/seo-passing.js';
 import seoStatus403 from './test-definitions/seo-status-403.js';
 import seoTapTargets from './test-definitions/seo-tap-targets.js';
 import sourceMaps from './test-definitions/source-maps.js';
+import timing from './test-definitions/timing.js';
 
 /** @type {ReadonlyArray<Smokehouse.TestDfn>} */
 const smokeTests = [
@@ -119,6 +120,7 @@ const smokeTests = [
   seoStatus403,
   seoTapTargets,
   sourceMaps,
+  timing,
 ];
 
 export default smokeTests;

package/lighthouse-cli/test/smokehouse/readme.md

@@ -64,11 +64,15 @@ Individual elements of an array can be asserted by using numeric properties in a
 
 However, if an array literal is used as the expectation, an extra condition is enforced that the actual array _must_ have the same length as the provided expected array.
 
+Arrays can be checked against a subset of elements using the special `_includes` property. The value of `_includes` _must_ be an array. Each assertion in `_includes` will remove the matching item from consideration for the rest.
+
 **Examples**:
 | Actual | Expected | Result |
 | -- | -- | -- |
 | `[{url: 'http://badssl.com'}, {url: 'http://example.com'}]` | `{1: {url: 'http://example.com'}}` | ✅ PASS |
 | `[{timeInMs: 5}, {timeInMs: 15}]` | `{length: 2}` | ✅ PASS |
+| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}]}` | ✅ PASS |
+| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}, {timeInMs: 5}]}` | ❌ FAIL |
 | `[{timeInMs: 5}, {timeInMs: 15}]` | `[{timeInMs: 5}]` | ❌ FAIL |
 
 ### Special environment checks

package/lighthouse-cli/test/smokehouse/report-assert.js

@@ -111,6 +111,35 @@ function findDifference(path, actual, expected) {
     const keyPath = path + keyAccessor;
     const expectedValue = expected[key];
 
+    if (key === '_includes') {
+      if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array');
+      if (!Array.isArray(actual)) {
+        return {
+          path,
+          actual: 'Actual value is not an array',
+          expected,
+        };
+      }
+
+      const actualCopy = [...actual];
+      for (const expectedEntry of expectedValue) {
+        const matchingIndex =
+          actualCopy.findIndex(actualEntry => !findDifference(keyPath, actualEntry, expectedEntry));
+        if (matchingIndex !== -1) {
+          actualCopy.splice(matchingIndex, 1);
+          continue;
+        }
+
+        return {
+          path,
+          actual: 'Item not found in array',
+          expected: expectedEntry,
+        };
+      }
+
+      continue;
+    }
+
     const actualValue = actual[key];
     const subDifference = findDifference(keyPath, actualValue, expectedValue);
 
@@ -305,6 +334,12 @@ function collateResults(localConsole, actual, expected) {
     return makeComparison(auditName + ' audit', actualResult, expectedResult);
   });
 
+  const timingAssertions = [];
+  if (expected.lhr.timing) {
+    const comparison = makeComparison('timing', actual.lhr.timing, expected.lhr.timing);
+    timingAssertions.push(comparison);
+  }
+
   /** @type {Comparison[]} */
   const requestCountAssertion = [];
   if (expected.networkRequests) {
@@ -322,6 +357,7 @@ function collateResults(localConsole, actual, expected) {
     ...requestCountAssertion,
     ...artifactAssertions,
     ...auditAssertions,
+    ...timingAssertions,
   ];
 }
 

package/lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -287,7 +287,8 @@ async function navigation(options) {
     skipAboutBlank: configContext.skipAboutBlank,
   };
 
-  return Runner.run(
+  const runnerOptions = {config, computedCache};
+  const artifacts = await Runner.gather(
     async () => {
       const driver = new Driver(page);
       const requestedUrl = URL.normalizeUrl(url);
@@ -298,11 +299,9 @@ async function navigation(options) {
 
       return finalizeArtifacts(baseArtifacts, artifacts);
     },
-    {
-      config,
-      computedCache: new Map(),
-    }
+    runnerOptions
   );
+  return Runner.audit(artifacts, runnerOptions);
 }
 
 module.exports = {

package/lighthouse-core/fraggle-rock/gather/snapshot-runner.js

@@ -26,7 +26,8 @@ async function snapshot(options) {
   const computedCache = new Map();
   const url = await options.page.url();
 
-  return Runner.run(
+  const runnerOptions = {config, computedCache};
+  const artifacts = await Runner.gather(
     async () => {
       const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'snapshot'});
       baseArtifacts.URL.requestedUrl = url;
@@ -50,11 +51,9 @@ async function snapshot(options) {
       const artifacts = await awaitArtifacts(artifactState);
       return finalizeArtifacts(baseArtifacts, artifacts);
     },
-    {
-      config,
-      computedCache,
-    }
+    runnerOptions
   );
+  return Runner.audit(artifacts, runnerOptions);
 }
 
 module.exports = {

package/lighthouse-core/fraggle-rock/gather/timespan-runner.js

@@ -50,7 +50,8 @@ async function startTimespan(options) {
   return {
     async endTimespan() {
       const finalUrl = await options.page.url();
-      return Runner.run(
+      const runnerOptions = {config, computedCache};
+      const artifacts = await Runner.gather(
         async () => {
           baseArtifacts.URL.requestedUrl = requestedUrl;
           baseArtifacts.URL.finalUrl = finalUrl;
@@ -63,11 +64,9 @@ async function startTimespan(options) {
           const artifacts = await awaitArtifacts(artifactState);
           return finalizeArtifacts(baseArtifacts, artifacts);
         },
-        {
-          config,
-          computedCache,
-        }
+        runnerOptions
       );
+      return Runner.audit(artifacts, runnerOptions);
     },
   };
 }

package/lighthouse-core/index.js

@@ -47,11 +47,11 @@ async function lighthouse(url, flags = {}, configJSON, userConnection) {
   const connection = userConnection || new ChromeProtocol(flags.port, flags.hostname);
 
   // kick off a lighthouse run
-  const gatherFn = () => {
+  const artifacts = await Runner.gather(() => {
     const requestedUrl = URL.normalizeUrl(url);
     return Runner._gatherArtifactsFromBrowser(requestedUrl, options, connection);
-  };
-  return Runner.run(gatherFn, options);
+  }, options);
+  return Runner.audit(artifacts, options);
 }
 
 /**

package/lighthouse-core/runner.js

@@ -28,14 +28,16 @@ const {version: lighthouseVersion} = require('../package.json');
 class Runner {
   /**
    * @template {LH.Config.Config | LH.Config.FRConfig} TConfig
-   * @param {(runnerData: {config: TConfig, driverMock?: Driver}) => Promise<LH.Artifacts>} gatherFn
-   * @param {{config: TConfig, computedCache: Map<string, ArbitraryEqualityMap>, driverMock?: Driver}} runOpts
+   * @param {LH.Artifacts} artifacts
+   * @param {{config: TConfig, driverMock?: Driver, computedCache: Map<string, ArbitraryEqualityMap>}} options
    * @return {Promise<LH.RunnerResult|undefined>}
    */
-  static async run(gatherFn, runOpts) {
-    const settings = runOpts.config.settings;
+  static async audit(artifacts, options) {
+    const {config, computedCache} = options;
+    const settings = config.settings;
+
     try {
-      const runnerStatus = {msg: 'Runner setup', id: 'lh:runner:run'};
+      const runnerStatus = {msg: 'Audit phase', id: 'lh:runner:audit'};
       log.time(runnerStatus, 'verbose');
 
       /**
@@ -44,24 +46,14 @@ class Runner {
        */
       const lighthouseRunWarnings = [];
 
-      const sentryContext = Sentry.getContext();
-      Sentry.captureBreadcrumb({
-        message: 'Run started',
-        category: 'lifecycle',
-        data: sentryContext?.extra,
-      });
-
-      const artifacts = await this.gatherAndManageArtifacts(gatherFn, runOpts);
-
       // Potentially quit early
       if (settings.gatherMode && !settings.auditMode) return;
 
-      // Audit phase
-      if (!runOpts.config.audits) {
+      if (!config.audits) {
         throw new Error('No audits to evaluate.');
       }
-      const auditResultsById = await Runner._runAudits(settings, runOpts.config.audits, artifacts,
-          lighthouseRunWarnings, runOpts.computedCache);
+      const auditResultsById = await Runner._runAudits(settings, config.audits, artifacts,
+          lighthouseRunWarnings, computedCache);
 
       // LHR construction phase
       const resultsStatus = {msg: 'Generating results...', id: 'lh:runner:generate'};
@@ -82,8 +74,8 @@ class Runner {
 
       /** @type {Record<string, LH.RawIcu<LH.Result.Category>>} */
       let categories = {};
-      if (runOpts.config.categories) {
-        categories = ReportScoring.scoreAllCategories(runOpts.config.categories, auditResultsById);
+      if (config.categories) {
+        categories = ReportScoring.scoreAllCategories(config.categories, auditResultsById);
       }
 
       log.timeEnd(resultsStatus);
@@ -108,7 +100,7 @@ class Runner {
         audits: auditResultsById,
         configSettings: settings,
         categories,
-        categoryGroups: runOpts.config.groups || undefined,
+        categoryGroups: config.groups || undefined,
         stackPacks: stackPacks.getStackPacks(artifacts.Stacks),
         timing: this._getTiming(artifacts),
         i18n: {
@@ -134,12 +126,7 @@ class Runner {
 
       return {lhr, artifacts, report};
     } catch (err) {
-      // i18n LighthouseError strings.
-      if (err.friendlyMessage) {
-        err.friendlyMessage = format.getFormatted(err.friendlyMessage, settings.locale);
-      }
-      await Sentry.captureException(err, {level: 'fatal'});
-      throw err;
+      throw Runner.createRunnerError(err, settings);
     }
   }
 
@@ -150,38 +137,72 @@ class Runner {
    *
    * @template {LH.Config.Config | LH.Config.FRConfig} TConfig
    * @param {(runnerData: {config: TConfig, driverMock?: Driver}) => Promise<LH.Artifacts>} gatherFn
-   * @param {{config: TConfig, driverMock?: Driver}} options
+   * @param {{config: TConfig, driverMock?: Driver, computedCache: Map<string, ArbitraryEqualityMap>}} options
    * @return {Promise<LH.Artifacts>}
    */
-  static async gatherAndManageArtifacts(gatherFn, options) {
+  static async gather(gatherFn, options) {
     const settings = options.config.settings;
 
-    // Gather phase
     // Either load saved artifacts from disk or from the browser.
-    let artifacts;
-    if (settings.auditMode && !settings.gatherMode) {
-      // No browser required, just load the artifacts from disk.
-      const path = this._getDataSavePath(settings);
-      artifacts = assetSaver.loadArtifacts(path);
-      const requestedUrl = artifacts.URL.requestedUrl;
-
-      if (!requestedUrl) {
-        throw new Error('Cannot run audit mode on empty URL');
-      }
-    } else {
-      artifacts = await gatherFn({
-        config: options.config,
-        driverMock: options.driverMock,
+    try {
+      const sentryContext = Sentry.getContext();
+      Sentry.captureBreadcrumb({
+        message: 'Run started',
+        category: 'lifecycle',
+        data: sentryContext?.extra,
       });
 
-      // -G means save these to disk (e.g. ./latest-run).
-      if (settings.gatherMode) {
+      /** @type {LH.Artifacts} */
+      let artifacts;
+      if (settings.auditMode && !settings.gatherMode) {
+        // No browser required, just load the artifacts from disk.
         const path = this._getDataSavePath(settings);
-        await assetSaver.saveArtifacts(artifacts, path);
+        artifacts = assetSaver.loadArtifacts(path);
+        const requestedUrl = artifacts.URL.requestedUrl;
+
+        if (!requestedUrl) {
+          throw new Error('Cannot run audit mode on empty URL');
+        }
+      } else {
+        const runnerStatus = {msg: 'Gather phase', id: 'lh:runner:gather'};
+        log.time(runnerStatus, 'verbose');
+
+        artifacts = await gatherFn({
+          config: options.config,
+          driverMock: options.driverMock,
+        });
+
+        log.timeEnd(runnerStatus);
+
+        // If `gather` is run multiple times before `audit`, the timing entries for each `gather` can pollute one another.
+        // We need to clear the timing entries at the end of gathering.
+        // Set artifacts.Timing again to ensure lh:runner:gather is included.
+        artifacts.Timing = log.takeTimeEntries();
+
+        // -G means save these to disk (e.g. ./latest-run).
+        if (settings.gatherMode) {
+          const path = this._getDataSavePath(settings);
+          await assetSaver.saveArtifacts(artifacts, path);
+        }
       }
+
+      return artifacts;
+    } catch (err) {
+      throw Runner.createRunnerError(err, settings);
     }
+  }
 
-    return artifacts;
+  /**
+   * @param {any} err
+   * @param {LH.Config.Settings} settings
+   */
+  static createRunnerError(err, settings) {
+    // i18n LighthouseError strings.
+    if (err.friendlyMessage) {
+      err.friendlyMessage = format.getFormatted(err.friendlyMessage, settings.locale);
+    }
+    Sentry.captureException(err, {level: 'fatal'});
+    return err;
   }
 
   /**
@@ -197,8 +218,11 @@ class Runner {
     const timingEntriesKeyValues = [
       ...timingEntriesFromArtifacts,
       ...timingEntriesFromRunner,
-      // As entries can share a name, dedupe based on the startTime timestamp
-    ].map(entry => /** @type {[number, PerformanceEntry]} */ ([entry.startTime, entry]));
+    ].map(entry => /** @type {[string, PerformanceEntry]} */ ([
+      // As entries can share a name and start time, dedupe based on the name, startTime and duration
+      `${entry.startTime}-${entry.name}-${entry.duration}`,
+      entry,
+    ]));
     const timingEntries = Array.from(new Map(timingEntriesKeyValues).values())
     // Truncate timestamps to hundredths of a millisecond saves ~4KB. No need for microsecond
     // resolution.
@@ -212,8 +236,11 @@ class Runner {
         entryType: entry.entryType,
       };
     }).sort((a, b) => a.startTime - b.startTime);
-    const runnerEntry = timingEntries.find(e => e.name === 'lh:runner:run');
-    return {entries: timingEntries, total: runnerEntry?.duration || 0};
+    const gatherEntry = timingEntries.find(e => e.name === 'lh:runner:gather');
+    const auditEntry = timingEntries.find(e => e.name === 'lh:runner:audit');
+    const gatherTiming = gatherEntry?.duration || 0;
+    const auditTiming = auditEntry?.duration || 0;
+    return {entries: timingEntries, total: gatherTiming + auditTiming};
   }
 
   /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse",
-  "version": "9.3.1-dev.20220202",
+  "version": "9.3.1-dev.20220203",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./lighthouse-core/index.js",
   "bin": {

package/types/smokehouse.d.ts

@@ -20,6 +20,9 @@ declare global {
         code?: any;
         message?: any;
       };
+      timing?: {
+        entries?: any
+      }
     }
 
     export type ExpectedRunnerResult = {