lighthouse 9.6.8 → 10.0.0

package/core/gather/driver/storage.d.ts

@@ -0,0 +1,24 @@
+/**
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {string} url
+ * @return {Promise<LH.IcuMessage[]>}
+ */
+export function clearDataForOrigin(session: LH.Gatherer.FRProtocolSession, url: string): Promise<LH.IcuMessage[]>;
+/**
+ * Clear the network cache on disk and in memory.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @return {Promise<LH.IcuMessage[]>}
+ */
+export function clearBrowserCaches(session: LH.Gatherer.FRProtocolSession): Promise<LH.IcuMessage[]>;
+/**
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {string} url
+ * @return {Promise<LH.IcuMessage | undefined>}
+ */
+export function getImportantStorageWarning(session: LH.Gatherer.FRProtocolSession, url: string): Promise<LH.IcuMessage | undefined>;
+export namespace UIStrings {
+    const warningData: string;
+    const warningCacheTimeout: string;
+    const warningOriginDataTimeout: string;
+}
+//# sourceMappingURL=storage.d.ts.map

package/core/gather/driver/storage.js

@@ -0,0 +1,149 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import log from 'lighthouse-logger';
+
+import * as i18n from '../../lib/i18n/i18n.js';
+
+/* eslint-disable max-len */
+const UIStrings = {
+  /**
+   * @description A warning that previously-saved data may have affected the measured performance and instructions on how to avoid the problem. "locations" will be a list of possible types of data storage locations, e.g. "IndexedDB",  "Local Storage", or "Web SQL".
+   * @example {IndexedDB, Local Storage} locations
+   */
+  warningData: `{locationCount, plural,
+    =1 {There may be stored data affecting loading performance in this location: {locations}. ` +
+      `Audit this page in an incognito window to prevent those resources ` +
+      `from affecting your scores.}
+    other {There may be stored data affecting loading ` +
+      `performance in these locations: {locations}. ` +
+      `Audit this page in an incognito window to prevent those resources ` +
+      `from affecting your scores.}
+  }`,
+  /** A warning that the data in the browser cache may have affected the measured performance because the operation to clear the browser cache timed out. */
+  warningCacheTimeout: 'Clearing the browser cache timed out. Try auditing this page again and file a bug if the issue persists.',
+  /** A warning that the data on the page's origin may have affected the measured performance because the operation to clear the origin data timed out. */
+  warningOriginDataTimeout: 'Clearing the origin data timed out. Try auditing this page again and file a bug if the issue persists.',
+};
+/* eslint-enable max-len */
+
+const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings);
+
+
+/**
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {string} url
+ * @return {Promise<LH.IcuMessage[]>}
+ */
+async function clearDataForOrigin(session, url) {
+  const status = {msg: 'Cleaning origin data', id: 'lh:storage:clearDataForOrigin'};
+  log.time(status);
+
+  const warnings = [];
+
+  const origin = new URL(url).origin;
+
+  // Clear some types of storage.
+  // Cookies are not cleared, so the user isn't logged out.
+  // indexeddb, websql, and localstorage are not cleared to prevent loss of potentially important data.
+  //   https://chromedevtools.github.io/debugger-protocol-viewer/tot/Storage/#type-StorageType
+  const typesToClear = [
+    // 'cookies',
+    'file_systems',
+    'shader_cache',
+    'service_workers',
+    'cache_storage',
+  ].join(',');
+
+  // `Storage.clearDataForOrigin` is one of our PROTOCOL_TIMEOUT culprits and this command is also
+  // run in the context of PAGE_HUNG to cleanup. We'll keep the timeout low and just warn if it fails.
+  session.setNextProtocolTimeout(5000);
+
+  try {
+    await session.sendCommand('Storage.clearDataForOrigin', {
+      origin: origin,
+      storageTypes: typesToClear,
+    });
+  } catch (err) {
+    if (/** @type {LH.LighthouseError} */ (err).code === 'PROTOCOL_TIMEOUT') {
+      log.warn('Driver', 'clearDataForOrigin timed out');
+      warnings.push(str_(UIStrings.warningOriginDataTimeout));
+    } else {
+      throw err;
+    }
+  } finally {
+    log.timeEnd(status);
+  }
+
+  return warnings;
+}
+
+/**
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {string} url
+ * @return {Promise<LH.IcuMessage | undefined>}
+ */
+async function getImportantStorageWarning(session, url) {
+  const usageData = await session.sendCommand('Storage.getUsageAndQuota', {
+    origin: url,
+  });
+  /** @type {Record<string, string>} */
+  const storageTypeNames = {
+    local_storage: 'Local Storage',
+    indexeddb: 'IndexedDB',
+    websql: 'Web SQL',
+  };
+  const locations = usageData.usageBreakdown
+    .filter(usage => usage.usage)
+    .map(usage => storageTypeNames[usage.storageType] || '')
+    .filter(Boolean);
+  if (locations.length) {
+    // TODO(#11495): Use Intl.ListFormat with Node 12
+    return str_(UIStrings.warningData, {
+      locations: locations.join(', '),
+      locationCount: locations.length,
+    });
+  }
+}
+
+
+/**
+ * Clear the network cache on disk and in memory.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @return {Promise<LH.IcuMessage[]>}
+ */
+async function clearBrowserCaches(session) {
+  const status = {msg: 'Cleaning browser cache', id: 'lh:storage:clearBrowserCaches'};
+  log.time(status);
+
+  const warnings = [];
+
+  try {
+    // Wipe entire disk cache
+    await session.sendCommand('Network.clearBrowserCache');
+    // Toggle 'Disable Cache' to evict the memory cache
+    await session.sendCommand('Network.setCacheDisabled', {cacheDisabled: true});
+    await session.sendCommand('Network.setCacheDisabled', {cacheDisabled: false});
+  } catch (err) {
+    if (/** @type {LH.LighthouseError} */ (err).code === 'PROTOCOL_TIMEOUT') {
+      log.warn('Driver', 'clearBrowserCaches timed out');
+      warnings.push(str_(UIStrings.warningCacheTimeout));
+    } else {
+      throw err;
+    }
+  } finally {
+    log.timeEnd(status);
+  }
+
+  return warnings;
+}
+
+export {
+  clearDataForOrigin,
+  clearBrowserCaches,
+  getImportantStorageWarning,
+  UIStrings,
+};

package/core/gather/driver/target-manager.d.ts

@@ -0,0 +1,61 @@
+export type TargetWithSession = {
+    target: LH.Crdp.Target.TargetInfo;
+    cdpSession: LH.Puppeteer.CDPSession;
+    session: LH.Gatherer.FRProtocolSession;
+    protocolListener: (event: unknown) => void;
+};
+export type ProtocolEventMap = {
+    'protocolevent': [LH.Protocol.RawEventMessage];
+};
+export type ProtocolEventMessageEmitter = LH.Protocol.StrictEventEmitterClass<ProtocolEventMap>;
+declare const TargetManager_base: ProtocolEventMessageEmitter;
+/**
+ * Tracks targets (the page itself, its iframes, their iframes, etc) as they
+ * appear and allows listeners to the flattened protocol events from all targets.
+ */
+export class TargetManager extends TargetManager_base {
+    /** @param {LH.Puppeteer.CDPSession} cdpSession */
+    constructor(cdpSession: LH.Puppeteer.CDPSession);
+    _enabled: boolean;
+    _rootCdpSession: import("../../../types/puppeteer.js").default.CDPSession;
+    /**
+     * A map of target id to target/session information. Used to ensure unique
+     * attached targets.
+     * @type {Map<string, TargetWithSession>}
+     */
+    _targetIdToTargets: Map<string, TargetWithSession>;
+    /**
+     * @param {LH.Puppeteer.CDPSession} cdpSession
+     */
+    _onSessionAttached(cdpSession: LH.Puppeteer.CDPSession): Promise<void>;
+    /**
+     * @param {LH.Crdp.Page.FrameNavigatedEvent} frameNavigatedEvent
+     */
+    _onFrameNavigated(frameNavigatedEvent: LH.Crdp.Page.FrameNavigatedEvent): Promise<void>;
+    /**
+     * @param {string} sessionId
+     * @return {LH.Gatherer.FRProtocolSession}
+     */
+    _findSession(sessionId: string): LH.Gatherer.FRProtocolSession;
+    /**
+     * Returns the root session.
+     * @return {LH.Gatherer.FRProtocolSession}
+     */
+    rootSession(): LH.Gatherer.FRProtocolSession;
+    /**
+     * Returns a listener for all protocol events from session, and augments the
+     * event with the sessionId.
+     * @param {string} sessionId
+     */
+    _getProtocolEventListener(sessionId: string): <EventName extends keyof import("puppeteer-core").ProtocolMapping.Events>(method: EventName, params: import("../../../types/protocol.js").default.RawEventMessageRecord[EventName]["params"]) => void;
+    /**
+     * @return {Promise<void>}
+     */
+    enable(): Promise<void>;
+    /**
+     * @return {Promise<void>}
+     */
+    disable(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=target-manager.d.ts.map

package/core/gather/driver/target-manager.js

@@ -0,0 +1,209 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+/**
+ * @fileoverview This class tracks multiple targets (the page itself and its OOPIFs) and allows consumers to
+ * listen for protocol events before each target is resumed.
+ */
+
+import EventEmitter from 'events';
+
+import log from 'lighthouse-logger';
+
+import {ProtocolSession} from '../session.js';
+
+/**
+ * @typedef {{
+ *   target: LH.Crdp.Target.TargetInfo,
+ *   cdpSession: LH.Puppeteer.CDPSession,
+ *   session: LH.Gatherer.FRProtocolSession,
+ *   protocolListener: (event: unknown) => void,
+ * }} TargetWithSession
+ */
+
+// Add protocol event types to EventEmitter.
+/** @typedef {{'protocolevent': [LH.Protocol.RawEventMessage]}} ProtocolEventMap */
+/** @typedef {LH.Protocol.StrictEventEmitterClass<ProtocolEventMap>} ProtocolEventMessageEmitter */
+const ProtocolEventEmitter = /** @type {ProtocolEventMessageEmitter} */ (EventEmitter);
+
+/**
+ * Tracks targets (the page itself, its iframes, their iframes, etc) as they
+ * appear and allows listeners to the flattened protocol events from all targets.
+ */
+class TargetManager extends ProtocolEventEmitter {
+  /** @param {LH.Puppeteer.CDPSession} cdpSession */
+  constructor(cdpSession) {
+    super();
+
+    this._enabled = false;
+    this._rootCdpSession = cdpSession;
+
+    /**
+     * A map of target id to target/session information. Used to ensure unique
+     * attached targets.
+     * @type {Map<string, TargetWithSession>}
+     */
+    this._targetIdToTargets = new Map();
+
+    this._onSessionAttached = this._onSessionAttached.bind(this);
+    this._onFrameNavigated = this._onFrameNavigated.bind(this);
+  }
+
+  /**
+   * @param {LH.Crdp.Page.FrameNavigatedEvent} frameNavigatedEvent
+   */
+  async _onFrameNavigated(frameNavigatedEvent) {
+    // Child frames are handled in `_onSessionAttached`.
+    if (frameNavigatedEvent.frame.parentId) return;
+    if (!this._enabled) return;
+
+    // It's not entirely clear when this is necessary, but when the page switches processes on
+    // navigating from about:blank to the `requestedUrl`, resetting `setAutoAttach` has been
+    // necessary in the past.
+    try {
+      await this._rootCdpSession.send('Target.setAutoAttach', {
+        autoAttach: true,
+        flatten: true,
+        waitForDebuggerOnStart: true,
+      });
+    } catch (err) {
+      // The page can be closed at the end of the run before this CDP function returns.
+      // In these cases, just ignore the error since we won't need the page anyway.
+      if (this._enabled) throw err;
+    }
+  }
+
+  /**
+   * @param {string} sessionId
+   * @return {LH.Gatherer.FRProtocolSession}
+   */
+  _findSession(sessionId) {
+    for (const {session, cdpSession} of this._targetIdToTargets.values()) {
+      if (cdpSession.id() === sessionId) return session;
+    }
+
+    throw new Error(`session ${sessionId} not found`);
+  }
+
+  /**
+   * Returns the root session.
+   * @return {LH.Gatherer.FRProtocolSession}
+   */
+  rootSession() {
+    const rootSessionId = this._rootCdpSession.id();
+    return this._findSession(rootSessionId);
+  }
+
+  /**
+   * @param {LH.Puppeteer.CDPSession} cdpSession
+   */
+  async _onSessionAttached(cdpSession) {
+    const newSession = new ProtocolSession(cdpSession);
+
+    try {
+      const target = await newSession.sendCommand('Target.getTargetInfo').catch(() => null);
+      const targetType = target?.targetInfo?.type;
+      const hasValidTargetType = targetType === 'page' || targetType === 'iframe';
+      // TODO: should detach from target in this case?
+      // See pptr: https://github.com/puppeteer/puppeteer/blob/733cbecf487c71483bee8350e37030edb24bc021/src/common/Page.ts#L495-L526
+      if (!target || !hasValidTargetType) return;
+
+      // No need to continue if target has already been seen.
+      const targetId = target.targetInfo.targetId;
+      if (this._targetIdToTargets.has(targetId)) return;
+
+      newSession.setTargetInfo(target.targetInfo);
+      const targetName = target.targetInfo.url || target.targetInfo.targetId;
+      log.verbose('target-manager', `target ${targetName} attached`);
+
+      const trueProtocolListener = this._getProtocolEventListener(newSession.id());
+      /** @type {(event: unknown) => void} */
+      // @ts-expect-error - pptr currently typed only for single arg emits.
+      const protocolListener = trueProtocolListener;
+      cdpSession.on('*', protocolListener);
+      cdpSession.on('sessionattached', this._onSessionAttached);
+
+      const targetWithSession = {
+        target: target.targetInfo,
+        cdpSession,
+        session: newSession,
+        protocolListener,
+      };
+      this._targetIdToTargets.set(targetId, targetWithSession);
+
+      // We want to receive information about network requests from iframes, so enable the Network domain.
+      await newSession.sendCommand('Network.enable');
+      // We also want to receive information about subtargets of subtargets, so make sure we autoattach recursively.
+      await newSession.sendCommand('Target.setAutoAttach', {
+        autoAttach: true,
+        flatten: true,
+        waitForDebuggerOnStart: true,
+      });
+    } catch (err) {
+      // Sometimes targets can be closed before we even have a chance to listen to their network activity.
+      if (/Target closed/.test(err.message)) return;
+
+      throw err;
+    } finally {
+      // Resume the target if it was paused, but if it's unnecessary, we don't care about the error.
+      await newSession.sendCommand('Runtime.runIfWaitingForDebugger').catch(() => {});
+    }
+  }
+
+  /**
+   * Returns a listener for all protocol events from session, and augments the
+   * event with the sessionId.
+   * @param {string} sessionId
+   */
+  _getProtocolEventListener(sessionId) {
+    /**
+     * @template {keyof LH.Protocol.RawEventMessageRecord} EventName
+     * @param {EventName} method
+     * @param {LH.Protocol.RawEventMessageRecord[EventName]['params']} params
+     */
+    const onProtocolEvent = (method, params) => {
+      // Cast because tsc 4.7 still can't quite track the dependent parameters.
+      const payload = /** @type {LH.Protocol.RawEventMessage} */ ({method, params, sessionId});
+      this.emit('protocolevent', payload);
+    };
+
+    return onProtocolEvent;
+  }
+
+  /**
+   * @return {Promise<void>}
+   */
+  async enable() {
+    if (this._enabled) return;
+
+    this._enabled = true;
+    this._targetIdToTargets = new Map();
+
+    this._rootCdpSession.on('Page.frameNavigated', this._onFrameNavigated);
+
+    await this._rootCdpSession.send('Page.enable');
+
+    // Start with the already attached root session.
+    await this._onSessionAttached(this._rootCdpSession);
+  }
+
+  /**
+   * @return {Promise<void>}
+   */
+  async disable() {
+    this._rootCdpSession.off('Page.frameNavigated', this._onFrameNavigated);
+
+    for (const {cdpSession, protocolListener} of this._targetIdToTargets.values()) {
+      cdpSession.off('*', protocolListener);
+      cdpSession.off('sessionattached', this._onSessionAttached);
+    }
+
+    this._enabled = false;
+    this._targetIdToTargets = new Map();
+  }
+}
+
+export {TargetManager};

package/core/gather/driver/wait-for-condition.d.ts

@@ -0,0 +1,116 @@
+export type NetworkMonitor = InstanceType<typeof import("./network-monitor.js")['NetworkMonitor']>;
+export type NetworkMonitorEvent = import('./network-monitor.js').NetworkMonitorEvent;
+export type CancellableWait<T = void> = {
+    promise: Promise<T>;
+    cancel: () => void;
+};
+export type WaitOptions = {
+    pauseAfterFcpMs: number;
+    pauseAfterLoadMs: number;
+    networkQuietThresholdMs: number;
+    cpuQuietThresholdMs: number;
+    maxWaitForLoadedMs: number;
+    maxWaitForFcpMs: number | undefined;
+    _waitForTestOverrides?: {
+        waitForFcp: typeof waitForFcp;
+        waitForLoadEvent: typeof waitForLoadEvent;
+        waitForNetworkIdle: typeof waitForNetworkIdle;
+        waitForCPUIdle: typeof waitForCPUIdle;
+    } | undefined;
+};
+/** @typedef {InstanceType<import('./network-monitor.js')['NetworkMonitor']>} NetworkMonitor */
+/** @typedef {import('./network-monitor.js').NetworkMonitorEvent} NetworkMonitorEvent */
+/**
+ * @template [T=void]
+ * @typedef CancellableWait
+ * @prop {Promise<T>} promise
+ * @prop {() => void} cancel
+ */
+/**
+ * @typedef WaitOptions
+ * @prop {number} pauseAfterFcpMs
+ * @prop {number} pauseAfterLoadMs
+ * @prop {number} networkQuietThresholdMs
+ * @prop {number} cpuQuietThresholdMs
+ * @prop {number} maxWaitForLoadedMs
+ * @prop {number|undefined} maxWaitForFcpMs
+ * @prop {{waitForFcp: typeof waitForFcp, waitForLoadEvent: typeof waitForLoadEvent, waitForNetworkIdle: typeof waitForNetworkIdle, waitForCPUIdle: typeof waitForCPUIdle}} [_waitForTestOverrides]
+ */
+/**
+ * Returns a promise that resolves immediately.
+ * Used for placeholder conditions that we don't want to start waiting for just yet, but still want
+ * to satisfy the same interface.
+ * @return {{promise: Promise<void>, cancel: function(): void}}
+ */
+export function waitForNothing(): {
+    promise: Promise<void>;
+    cancel: () => void;
+};
+/**
+ * Returns a promise that resolve when a frame has been navigated.
+ * Used for detecting that our about:blank reset has been completed.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @return {CancellableWait<LH.Crdp.Page.FrameNavigatedEvent>}
+ */
+export function waitForFrameNavigated(session: LH.Gatherer.FRProtocolSession): CancellableWait<LH.Crdp.Page.FrameNavigatedEvent>;
+/**
+ * Returns a promise that resolve when a frame has a FCP.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {number} pauseAfterFcpMs
+ * @param {number} maxWaitForFcpMs
+ * @return {CancellableWait}
+ */
+export function waitForFcp(session: LH.Gatherer.FRProtocolSession, pauseAfterFcpMs: number, maxWaitForFcpMs: number): CancellableWait;
+/**
+ * Return a promise that resolves `pauseAfterLoadMs` after the load event
+ * fires and a method to cancel internal listeners and timeout.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {number} pauseAfterLoadMs
+ * @return {CancellableWait}
+ */
+export function waitForLoadEvent(session: LH.Gatherer.FRProtocolSession, pauseAfterLoadMs: number): CancellableWait;
+/**
+ * Returns a promise that resolves when the network has been idle (after DCL) for
+ * `networkQuietThresholdMs` ms and a method to cancel internal network listeners/timeout.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {NetworkMonitor} networkMonitor
+ * @param {{networkQuietThresholdMs: number, busyEvent: NetworkMonitorEvent, idleEvent: NetworkMonitorEvent, isIdle(recorder: NetworkMonitor): boolean, pretendDCLAlreadyFired?: boolean}} networkQuietOptions
+ * @return {CancellableWait}
+ */
+export function waitForNetworkIdle(session: LH.Gatherer.FRProtocolSession, networkMonitor: NetworkMonitor, networkQuietOptions: {
+    networkQuietThresholdMs: number;
+    busyEvent: NetworkMonitorEvent;
+    idleEvent: NetworkMonitorEvent;
+    isIdle(recorder: NetworkMonitor): boolean;
+    pretendDCLAlreadyFired?: boolean | undefined;
+}): CancellableWait;
+/**
+ * Resolves when there have been no long tasks for at least waitForCPUQuiet ms.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {number} waitForCPUQuiet
+ * @return {CancellableWait}
+ */
+export function waitForCPUIdle(session: LH.Gatherer.FRProtocolSession, waitForCPUQuiet: number): CancellableWait;
+/**
+ * Returns a promise that resolves when:
+ * - All of the following conditions have been met:
+ *    - page has no security issues
+ *    - pauseAfterLoadMs milliseconds have passed since the load event.
+ *    - networkQuietThresholdMs milliseconds have passed since the last network request that exceeded
+ *      2 inflight requests (network-2-quiet has been reached).
+ *    - cpuQuietThresholdMs have passed since the last long task after network-2-quiet.
+ * - maxWaitForLoadedMs milliseconds have passed.
+ * See https://github.com/GoogleChrome/lighthouse/issues/627 for more.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {NetworkMonitor} networkMonitor
+ * @param {WaitOptions} options
+ * @return {Promise<{timedOut: boolean}>}
+ */
+export function waitForFullyLoaded(session: LH.Gatherer.FRProtocolSession, networkMonitor: NetworkMonitor, options: WaitOptions): Promise<{
+    timedOut: boolean;
+}>;
+/**
+ * @param {LH.Gatherer.FRTransitionalDriver} driver
+ */
+export function waitForUserToContinue(driver: LH.Gatherer.FRTransitionalDriver): Promise<void>;
+//# sourceMappingURL=wait-for-condition.d.ts.map