lighthouse 9.5.0-dev.20220504 → 9.5.0-dev.20220505

package/lighthouse-cli/test/smokehouse/lighthouse-runners/bundle.js

@@ -8,47 +8,76 @@
  * @fileoverview  A runner that launches Chrome and executes Lighthouse via a
  * bundle to test that bundling has produced correct and runnable code.
  * Currently uses `lighthouse-dt-bundle.js`.
+ * Runs in a worker to avoid messing up marky's global state.
  */
 
 import fs from 'fs';
+import os from 'os';
+import {Worker, isMainThread, parentPort, workerData} from 'worker_threads';
+import {once} from 'events';
 
 import puppeteer from 'puppeteer-core';
 import ChromeLauncher from 'chrome-launcher';
 
 import ChromeProtocol from '../../../../lighthouse-core/gather/connections/cri.js';
 import {LH_ROOT} from '../../../../root.js';
+import {loadArtifacts, saveArtifacts} from '../../../../lighthouse-core/lib/asset-saver.js';
 
-const originalBuffer = global.Buffer;
-const originalRequire = global.require;
-if (typeof globalThis === 'undefined') {
-  // @ts-expect-error - exposing for loading of dt-bundle.
-  global.globalThis = global;
+// This runs only in the worker. The rest runs on the main thread.
+if (!isMainThread && parentPort) {
+  (async () => {
+    const {url, configJson, testRunnerOptions} = workerData;
+    try {
+      const result = await runBundledLighthouse(url, configJson, testRunnerOptions);
+      // Save to assets directory because LighthouseError won't survive postMessage.
+      const assetsDir = fs.mkdtempSync(os.tmpdir() + '/smoke-bundle-assets-');
+      await saveArtifacts(result.artifacts, assetsDir);
+      const value = {
+        lhr: result.lhr,
+        assetsDir,
+      };
+      parentPort?.postMessage({type: 'result', value});
+    } catch (err) {
+      console.error(err);
+      parentPort?.postMessage({type: 'error', value: err});
+    }
+  })();
 }
 
-// Load bundle, which creates a `global.runBundledLighthouse`.
-eval(fs.readFileSync(LH_ROOT + '/dist/lighthouse-dt-bundle.js', 'utf-8'));
-
-global.require = originalRequire;
-global.Buffer = originalBuffer;
-
-/** @type {import('../../../../lighthouse-core/index.js')} */
-// @ts-expect-error - not worth giving test global an actual type.
-const lighthouse = global.runBundledLighthouse;
-
 /**
- * Launch Chrome and do a full Lighthouse run via the Lighthouse DevTools bundle.
  * @param {string} url
- * @param {LH.Config.Json=} configJson
- * @param {{isDebug?: boolean, useFraggleRock?: boolean}=} testRunnerOptions
- * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
+ * @param {LH.Config.Json|undefined} configJson
+ * @param {{isDebug?: boolean, useFraggleRock?: boolean}} testRunnerOptions
+ * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts}>}
  */
-async function runLighthouse(url, configJson, testRunnerOptions = {}) {
+async function runBundledLighthouse(url, configJson, testRunnerOptions) {
+  if (isMainThread || !parentPort) {
+    throw new Error('must be called in worker');
+  }
+
+  const originalBuffer = global.Buffer;
+  const originalRequire = global.require;
+  if (typeof globalThis === 'undefined') {
+    // @ts-expect-error - exposing for loading of dt-bundle.
+    global.globalThis = global;
+  }
+
+  // Load bundle, which creates a `global.runBundledLighthouse`.
+  eval(fs.readFileSync(LH_ROOT + '/dist/lighthouse-dt-bundle.js', 'utf-8'));
+
+  global.require = originalRequire;
+  global.Buffer = originalBuffer;
+
+  /** @type {import('../../../../lighthouse-core/index.js')} */
+  // @ts-expect-error - not worth giving test global an actual type.
+  const lighthouse = global.runBundledLighthouse;
+
   // Launch and connect to Chrome.
   const launchedChrome = await ChromeLauncher.launch();
   const port = launchedChrome.port;
 
+  // Run Lighthouse.
   try {
-    // Run Lighthouse.
     const logLevel = testRunnerOptions.isDebug ? 'info' : undefined;
     let runnerResult;
     if (testRunnerOptions.useFraggleRock) {
@@ -66,7 +95,6 @@ async function runLighthouse(url, configJson, testRunnerOptions = {}) {
     return {
       lhr: runnerResult.lhr,
       artifacts: runnerResult.artifacts,
-      log: '', // TODO: if want to run in parallel, need to capture lighthouse-logger output.
     };
   } finally {
     // Clean up and return results.
@@ -74,6 +102,53 @@ async function runLighthouse(url, configJson, testRunnerOptions = {}) {
   }
 }
 
+/**
+ * Launch Chrome and do a full Lighthouse run via the Lighthouse DevTools bundle.
+ * @param {string} url
+ * @param {LH.Config.Json=} configJson
+ * @param {{isDebug?: boolean, useFraggleRock?: boolean}=} testRunnerOptions
+ * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
+ */
+async function runLighthouse(url, configJson, testRunnerOptions = {}) {
+  /** @type {string[]} */
+  const logs = [];
+  const worker = new Worker(new URL(import.meta.url), {
+    stdout: true,
+    stderr: true,
+    workerData: {url, configJson, testRunnerOptions},
+  });
+  worker.stdout.setEncoding('utf8');
+  worker.stderr.setEncoding('utf8');
+  worker.stdout.addListener('data', (data) => {
+    process.stdout.write(data);
+    logs.push(`STDOUT: ${data}`);
+  });
+  worker.stderr.addListener('data', (data) => {
+    process.stderr.write(data);
+    logs.push(`STDERR: ${data}`);
+  });
+  const [workerResponse] = await once(worker, 'message');
+  const log = logs.join('') + '\n';
+
+  if (workerResponse.type === 'error') {
+    new Error(`Worker returned an error: ${workerResponse.value}\nLog:\n${log}`);
+  }
+
+  const result = workerResponse.value;
+  if (!result.lhr || !result.assetsDir) {
+    throw new Error(`invalid response from worker:\n${JSON.stringify(result, null, 2)}`);
+  }
+
+  const artifacts = loadArtifacts(result.assetsDir);
+  fs.rmSync(result.assetsDir, {recursive: true});
+
+  return {
+    lhr: result.lhr,
+    artifacts,
+    log,
+  };
+}
+
 export {
   runLighthouse,
 };

package/lighthouse-core/fraggle-rock/gather/driver.js

@@ -74,7 +74,7 @@ class Driver {
     const session = await this._page.target().createCDPSession();
     this._session = this.defaultSession = new ProtocolSession(session);
     this._executionContext = new ExecutionContext(this._session);
-    this._fetcher = new Fetcher(this._session, this._executionContext);
+    this._fetcher = new Fetcher(this._session);
     log.timeEnd(status);
   }
 

package/lighthouse-core/gather/driver.js

@@ -64,7 +64,7 @@ class Driver {
   defaultSession = this;
 
   // eslint-disable-next-line no-invalid-this
-  fetcher = new Fetcher(this.defaultSession, this.executionContext);
+  fetcher = new Fetcher(this.defaultSession);
 
   /**
    * @param {Connection} connection

package/lighthouse-core/gather/fetcher.js

@@ -6,124 +6,55 @@
 'use strict';
 
 /**
- * @fileoverview Fetcher is a utility for making requests within the context of the page.
- * Requests can circumvent CORS, and so are good for fetching source maps that may be hosted
- * on a different origin.
+ * @fileoverview Fetcher is a utility for making requests to any arbitrary resource,
+ * ignoring normal browser constraints such as CORS.
  */
 
-/* global document */
+/* global fetch */
 
 /** @typedef {{content: string|null, status: number|null}} FetchResponse */
 
-const log = require('lighthouse-logger');
-const {getBrowserVersion} = require('./driver/environment.js');
-
 class Fetcher {
   /**
    * @param {LH.Gatherer.FRProtocolSession} session
-   * @param {import('./driver/execution-context.js')} executionContext
    */
-  constructor(session, executionContext) {
+  constructor(session) {
     this.session = session;
-    this.executionContext = executionContext;
-    /** @type {Map<string, (event: LH.Crdp.Fetch.RequestPausedEvent) => void>} */
-    this._onRequestPausedHandlers = new Map();
-    this._onRequestPaused = this._onRequestPaused.bind(this);
-    this._enabled = false;
   }
 
   /**
-   * Chrome M92 and above:
-   * We use `Network.loadNetworkResource` to fetch each resource.
-   *
-   * Chrome <M92:
-   * The Fetch domain accepts patterns for controlling what requests are intercepted, but we
-   * enable the domain for all patterns and filter events at a lower level to support multiple
-   * concurrent usages. Reasons for this:
-   *
-   * 1) only one set of patterns may be applied for the entire domain.
-   * 2) every request that matches the patterns are paused and only resumes when certain Fetch
-   *    commands are sent. So a listener of the `Fetch.requestPaused` event must either handle
-   *    the requests it cares about, or explicitly allow them to continue.
-   * 3) if multiple commands to continue the same request are sent, protocol errors occur.
+   * Fetches any resource using the network directly.
    *
-   * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific
-   * urls to be intercepted via `fetcher._setOnRequestPausedHandler`.
-   */
-  async enable() {
-    if (this._enabled) return;
-
-    this._enabled = true;
-    await this.session.sendCommand('Fetch.enable', {
-      patterns: [{requestStage: 'Request'}, {requestStage: 'Response'}],
-    });
-    await this.session.on('Fetch.requestPaused', this._onRequestPaused);
-  }
-
-  async disable() {
-    if (!this._enabled) return;
-
-    this._enabled = false;
-    await this.session.off('Fetch.requestPaused', this._onRequestPaused);
-    await this.session.sendCommand('Fetch.disable');
-    this._onRequestPausedHandlers.clear();
-  }
-
-  /**
    * @param {string} url
-   * @param {(event: LH.Crdp.Fetch.RequestPausedEvent) => void} handler
-   */
-  async _setOnRequestPausedHandler(url, handler) {
-    this._onRequestPausedHandlers.set(url, handler);
-  }
-
-  /**
-   * @param {LH.Crdp.Fetch.RequestPausedEvent} event
+   * @param {{timeout: number}=} options timeout is in ms
+   * @return {Promise<FetchResponse>}
    */
-  _onRequestPaused(event) {
-    const handler = this._onRequestPausedHandlers.get(event.request.url);
-    if (handler) {
-      handler(event);
-    } else {
-      // Nothing cares about this URL, so continue.
-      this.session.sendCommand('Fetch.continueRequest', {requestId: event.requestId}).catch(err => {
-        log.error('Fetcher', `Failed to continueRequest: ${err.message}`);
-      });
+  async fetchResource(url, options = {timeout: 2_000}) {
+    // In Lightrider, `Network.loadNetworkResource` is not implemented, but fetch
+    // is configured to work for any resource.
+    if (global.isLightrider) {
+      return this._wrapWithTimeout(this._fetchWithFetchApi(url), options.timeout);
     }
-  }
 
-  /**
-   * `Network.loadNetworkResource` was introduced in M88.
-   * The long timeout bug with `IO.read` was fixed in M92:
-   * https://bugs.chromium.org/p/chromium/issues/detail?id=1191757
-   * Lightrider has a bug forcing us to use the old version for now:
-   * https://docs.google.com/document/d/1V-DxgsOFMPxUuFrdGPQpyiCqSljvgNlOqXCtqDtd0b8/edit?usp=sharing&resourcekey=0-aIaIqcHFKG-0dX4MAudBEw
-   * @return {Promise<boolean>}
-   */
-  async shouldUseLegacyFetcher() {
-    const {milestone} = await getBrowserVersion(this.session);
-    return milestone < 92 || Boolean(global.isLightrider);
+    return this._fetchResourceOverProtocol(url, options);
   }
 
   /**
-   * Requires that `fetcher.enable` has been called.
-   *
-   * Fetches any resource in a way that circumvents CORS.
-   *
    * @param {string} url
-   * @param {{timeout: number}=} options timeout is in ms
    * @return {Promise<FetchResponse>}
    */
-  async fetchResource(url, options = {timeout: 2_000}) {
-    if (!this._enabled) {
-      throw new Error('Must call `enable` before using fetchResource');
-    }
+  async _fetchWithFetchApi(url) {
+    const response = await fetch(url);
 
-    if (await this.shouldUseLegacyFetcher()) {
-      return this._fetchResourceIframe(url, options);
-    }
+    let content = null;
+    try {
+      content = await response.text();
+    } catch {}
 
-    return this._fetchResourceOverProtocol(url, options);
+    return {
+      content,
+      status: response.status,
+    };
   }
 
   /**
@@ -172,18 +103,6 @@ class Fetcher {
     };
   }
 
-  /**
-   * @param {string} requestId
-   * @return {Promise<string>}
-   */
-  async _resolveResponseBody(requestId) {
-    const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId});
-    if (responseBody.base64Encoded) {
-      return Buffer.from(responseBody.body, 'base64').toString();
-    }
-    return responseBody.body;
-  }
-
   /**
    * @param {string} url
    * @param {{timeout: number}} options timeout is in ms
@@ -191,18 +110,7 @@ class Fetcher {
    */
   async _fetchResourceOverProtocol(url, options) {
     const startTime = Date.now();
-
-    /** @type {NodeJS.Timeout} */
-    let timeoutHandle;
-    const timeoutPromise = new Promise((_, reject) => {
-      timeoutHandle = setTimeout(reject, options.timeout, new Error('Timed out fetching resource'));
-    });
-
-    const responsePromise = this._loadNetworkResource(url);
-
-    /** @type {{stream: LH.Crdp.IO.StreamHandle|null, status: number|null}} */
-    const response = await Promise.race([responsePromise, timeoutPromise])
-      .finally(() => clearTimeout(timeoutHandle));
+    const response = await this._wrapWithTimeout(this._loadNetworkResource(url), options.timeout);
 
     const isOk = response.status && response.status >= 200 && response.status <= 299;
     if (!response.stream || !isOk) return {status: response.status, content: null};
@@ -213,105 +121,21 @@ class Fetcher {
   }
 
   /**
-   * Fetches resource by injecting an iframe into the page.
-   * @param {string} url
-   * @param {{timeout: number}} options timeout is in ms
-   * @return {Promise<FetchResponse>}
+   * @template T
+   * @param {Promise<T>} promise
+   * @param {number} ms
    */
-  async _fetchResourceIframe(url, options) {
-    /** @type {Promise<FetchResponse>} */
-    const requestInterceptionPromise = new Promise((resolve, reject) => {
-      /** @param {LH.Crdp.Fetch.RequestPausedEvent} event */
-      const handlerAsync = async event => {
-        const {requestId, responseStatusCode} = event;
-
-        // The first requestPaused event is for the request stage. Continue it.
-        if (!responseStatusCode) {
-          // Remove cookies so we aren't buying stuff on Amazon.
-          const headers = Object.entries(event.request.headers)
-            .filter(([name]) => name !== 'Cookie')
-            .map(([name, value]) => {
-              return {name, value};
-            });
-
-          await this.session.sendCommand('Fetch.continueRequest', {
-            requestId,
-            headers,
-          });
-          return;
-        }
-
-        if (responseStatusCode >= 200 && responseStatusCode <= 299) {
-          resolve({
-            status: responseStatusCode,
-            content: await this._resolveResponseBody(requestId),
-          });
-        } else {
-          resolve({status: responseStatusCode, content: null});
-        }
-
-        // Fail the request (from the page's perspective) so that the iframe never loads.
-        await this.session.sendCommand('Fetch.failRequest', {requestId, errorReason: 'Aborted'});
-      };
-      this._setOnRequestPausedHandler(url, event => handlerAsync(event).catch(reject));
-    });
-
-    /**
-     * @param {string} src
-     */
-    /* c8 ignore start */
-    function injectIframe(src) {
-      const iframe = document.createElement('iframe');
-      // Try really hard not to affect the page.
-      iframe.style.display = 'none';
-      iframe.style.visibility = 'hidden';
-      iframe.style.position = 'absolute';
-      iframe.style.top = '-1000px';
-      iframe.style.left = '-1000px';
-      iframe.style.width = '1px';
-      iframe.style.height = '1px';
-      iframe.src = src;
-      iframe.onload = iframe.onerror = () => {
-        iframe.remove();
-        iframe.onload = null;
-        iframe.onerror = null;
-      };
-      document.body.appendChild(iframe);
-    }
-    /* c8 ignore stop */
-
+  async _wrapWithTimeout(promise, ms) {
     /** @type {NodeJS.Timeout} */
-    let asyncTimeout;
-    /** @type {Promise<never>} */
+    let timeoutHandle;
     const timeoutPromise = new Promise((_, reject) => {
-      asyncTimeout = setTimeout(reject, options.timeout, new Error('Timed out fetching resource.'));
-    });
-
-    const racePromise = Promise.race([
-      timeoutPromise,
-      requestInterceptionPromise,
-    ]).finally(() => clearTimeout(asyncTimeout));
-
-    // Temporarily disable auto-attaching for this iframe.
-    await this.session.sendCommand('Target.setAutoAttach', {
-      autoAttach: false,
-      waitForDebuggerOnStart: false,
-    });
-
-    const injectionPromise = this.executionContext.evaluate(injectIframe, {
-      args: [url],
-      useIsolation: true,
+      timeoutHandle = setTimeout(reject, ms, new Error('Timed out fetching resource'));
     });
 
-    const [fetchResult] = await Promise.all([racePromise, injectionPromise]);
-
-    await this.session.sendCommand('Target.setAutoAttach', {
-      flatten: true,
-      autoAttach: true,
-      waitForDebuggerOnStart: true,
-    });
-
-    return fetchResult;
+    /** @type {Promise<T>} */
+    const wrappedPromise = await Promise.race([promise, timeoutPromise])
+      .finally(() => clearTimeout(timeoutHandle));
+    return wrappedPromise;
   }
 }
 

package/lighthouse-core/gather/gather-runner.js

@@ -183,11 +183,6 @@ class GatherRunner {
       const resetStorage = !options.settings.disableStorageReset;
       if (resetStorage) await storage.clearDataForOrigin(session, options.requestedUrl);
 
-      // Disable fetcher, in case a gatherer enabled it.
-      // This cleanup should be removed once the only usage of
-      // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
-      await driver.fetcher.disable();
-
       await driver.disconnect();
     } catch (err) {
       // Ignore disconnecting error if browser was already closed.
@@ -524,12 +519,6 @@ class GatherRunner {
           await GatherRunner.populateBaseArtifacts(passContext);
           isFirstPass = false;
         }
-
-        // Disable fetcher for every pass, in case a gatherer enabled it.
-        // Noop if fetcher was never enabled.
-        // This cleanup should be removed once the only usage of
-        // fetcher (fetching arbitrary URLs) is replaced by new protocol support.
-        await driver.fetcher.disable();
       }
 
       await GatherRunner.disposeDriver(driver, options);

package/lighthouse-core/gather/gatherers/seo/robots-txt.js

@@ -7,25 +7,6 @@
 
 const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js');
 
-/* global fetch, location */
-
-/** @return {Promise<LH.Artifacts['RobotsTxt']>} */
-/* c8 ignore start */
-async function getRobotsTxtContent() {
-  try {
-    const response = await fetch(new URL('/robots.txt', location.href).href);
-    if (!response.ok) {
-      return {status: response.status, content: null};
-    }
-
-    const content = await response.text();
-    return {status: response.status, content};
-  } catch (err) {
-    return {status: null, content: null, errorMessage: err.message};
-  }
-}
-/* c8 ignore stop */
-
 class RobotsTxt extends FRGatherer {
   /** @type {LH.Gatherer.GathererMeta} */
   meta = {
@@ -37,18 +18,8 @@ class RobotsTxt extends FRGatherer {
    * @return {Promise<LH.Artifacts['RobotsTxt']>}
    */
   async getArtifact(passContext) {
-    // Iframe fetcher still has issues with CSPs.
-    // Only use the fetcher if we are fetching over the protocol.
-    if (await passContext.driver.fetcher.shouldUseLegacyFetcher()) {
-      return passContext.driver.executionContext.evaluate(getRobotsTxtContent, {
-        args: [],
-        useIsolation: true,
-      });
-    }
-
     const {finalUrl} = passContext.baseArtifacts.URL;
     const robotsUrl = new URL('/robots.txt', finalUrl).href;
-    await passContext.driver.fetcher.enable();
     return passContext.driver.fetcher.fetchResource(robotsUrl)
       .catch(err => ({status: null, content: null, errorMessage: err.message}));
   }

package/lighthouse-core/gather/gatherers/source-maps.js

@@ -143,7 +143,6 @@ class SourceMaps extends FRGatherer {
    * @return {Promise<LH.Artifacts['SourceMaps']>}
    */
   async getArtifact(context) {
-    await context.driver.fetcher.enable();
     const eventProcessPromises = this._scriptParsedEvents
       .map((event) => this._retrieveMapFromScriptParsedEvent(context.driver, event));
     return Promise.all(eventProcessPromises);

package/lighthouse-core/lib/stack-packs.js

@@ -19,11 +19,10 @@ const stackPacksToInclude = [
     packId: 'wordpress',
     requiredStacks: ['js:wordpress'],
   },
-  // waiting for https://github.com/johnmichel/Library-Detector-for-Chrome/pull/193
-  // {
-  //   packId: 'ezoic',
-  //   requiredStacks: ['js:ezoic'],
-  // },
+  {
+    packId: 'ezoic',
+    requiredStacks: ['js:ezoic'],
+  },
   {
     packId: 'drupal',
     requiredStacks: ['js:drupal'],

package/lighthouse-core/lib/tracehouse/task-groups.js

@@ -50,6 +50,7 @@ const taskGroups = {
       'UpdateLayer',
       'UpdateLayerTree',
       'CompositeLayers',
+      'PrePaint', // New name for UpdateLayerTree: https://crrev.com/c/3519012
     ],
   },
   scriptParseCompile: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse",
-  "version": "9.5.0-dev.20220504",
+  "version": "9.5.0-dev.20220505",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./lighthouse-core/index.js",
   "bin": {
@@ -37,7 +37,7 @@
     "start": "yarn build-report --standalone && node ./lighthouse-cli/index.js",
     "jest": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js",
     "test": "yarn diff:sample-json && yarn lint --quiet && yarn unit && yarn type-check",
-    "test-bundle": "yarn smoke --runner bundle -j=1 --retries=2",
+    "test-bundle": "yarn smoke --runner bundle --retries=2",
     "test-clients": "yarn jest \"$PWD/clients/\" && yarn jest --testMatch=\"**/clients/test/**/*-test-pptr.js\"",
     "test-viewer": "yarn unit-viewer && yarn jest --testMatch=\"**/viewer/**/*-test-pptr.js\"",
     "test-treemap": "yarn unit-treemap && yarn jest --testMatch=\"**/treemap/**/*-test-pptr.js\"",
@@ -190,7 +190,7 @@
     "http-link-header": "^0.8.0",
     "intl-messageformat": "^4.4.0",
     "jpeg-js": "^0.4.3",
-    "js-library-detector": "^6.4.0",
+    "js-library-detector": "^6.5.0",
     "lighthouse-logger": "^1.3.0",
     "lighthouse-stack-packs": "^1.8.1",
     "lodash": "^4.17.21",