lighthouse 13.5.0 → 13.6.0-dev.20261027

package/core/gather/gatherers/scripts.js

@@ -5,19 +5,31 @@
  */
 
 import BaseGatherer from '../base-gatherer.js';
+import Trace from './trace.js';
+
+/**
+ * @typedef {Object} TraceScript
+ * @property {string} scriptId
+ * @property {string} [sourceText]
+ * @property {string[]} [chunks]
+ * @property {string} [url]
+ * @property {string} [sourceUrl]
+ * @property {number} [executionContextId]
+ * @property {string} [frame]
+ */
 
 /**
  * @template T, U
  * @param {Array<T>} values
- * @param {(value: T) => Promise<U>} promiseMapper
+ * @param {(value: T, index: number) => Promise<U>} promiseMapper
  * @param {boolean} runInSeries
  * @return {Promise<Array<U>>}
  */
 async function runInSeriesOrParallel(values, promiseMapper, runInSeries) {
   if (runInSeries) {
     const results = [];
-    for (const value of values) {
-      const result = await promiseMapper(value);
+    for (let i = 0; i < values.length; i++) {
+      const result = await promiseMapper(values[i], i);
       results.push(result);
     }
     return results;
@@ -27,6 +39,121 @@ async function runInSeriesOrParallel(values, promiseMapper, runInSeries) {
   }
 }
 
+/**
+ * Extracts script source text and disambiguating metadata from trace rundown events,
+ * grouped by `scriptId`.
+ * @param {LH.TraceEvent[]} traceEvents
+ * @return {Map<string, TraceScript[]>}
+ */
+function collectTraceScripts(traceEvents) {
+  /** @type {Map<string, TraceScript>} */
+  const byIsolateAndScriptId = new Map();
+
+  /**
+   * @param {NonNullable<LH.TraceEvent['args']['data']>} data
+   * @return {TraceScript}
+   */
+  const getOrCreateRecord = (data) => {
+    const scriptId = String(data.scriptId);
+    const key = `${data.isolate ?? ''}.${scriptId}`;
+    let record = byIsolateAndScriptId.get(key);
+    if (!record) {
+      record = {scriptId};
+      byIsolateAndScriptId.set(key, record);
+    }
+    return record;
+  };
+
+  for (const event of traceEvents) {
+    const data = event.args?.data;
+    if (!data || data.scriptId === undefined) continue;
+
+    if (event.cat === 'disabled-by-default-devtools.target-rundown') {
+      if (typeof data.frame === 'string') {
+        getOrCreateRecord(data).frame = data.frame;
+      }
+    } else if (event.cat === 'disabled-by-default-devtools.v8-source-rundown' &&
+        event.name === 'ScriptCatchup') {
+      const record = getOrCreateRecord(data);
+      if (typeof data.url === 'string') record.url = data.url;
+      if (typeof data.sourceUrl === 'string') record.sourceUrl = data.sourceUrl;
+      if (typeof data.executionContextId === 'number') {
+        record.executionContextId = data.executionContextId;
+      }
+    } else if (event.cat === 'disabled-by-default-devtools.v8-source-rundown-sources' &&
+        typeof data.sourceText === 'string') {
+      const record = getOrCreateRecord(data);
+      if (event.name === 'ScriptCatchup') {
+        record.sourceText = data.sourceText;
+      } else if (event.name === 'LargeScriptCatchup') {
+        if (!record.chunks || data.splitIndex === 0) record.chunks = [];
+        record.chunks.push(data.sourceText);
+      }
+    }
+  }
+
+  /** @type {Map<string, TraceScript[]>} */
+  const byScriptId = new Map();
+  for (const record of byIsolateAndScriptId.values()) {
+    if (record.chunks) record.sourceText = record.chunks.join('');
+    if (record.sourceText === undefined) continue;
+
+    let list = byScriptId.get(record.scriptId);
+    if (!list) {
+      list = [];
+      byScriptId.set(record.scriptId, list);
+    }
+    list.push(record);
+  }
+
+  return byScriptId;
+}
+
+/**
+ * @param {TraceScript} traceScript
+ * @param {LH.Crdp.Debugger.ScriptParsedEvent} script
+ * @return {boolean}
+ */
+function matchesScriptParsedEvent(traceScript, script) {
+  const frameId = script.executionContextAuxData?.frameId;
+  if (frameId && traceScript.frame !== undefined && traceScript.frame !== frameId) {
+    return false;
+  }
+  const scriptUrl = script.embedderName || script.url;
+  if (traceScript.url !== undefined &&
+      traceScript.url !== scriptUrl &&
+      traceScript.sourceUrl !== script.url) {
+    return false;
+  }
+  if (script.executionContextId !== undefined &&
+      traceScript.executionContextId !== undefined &&
+      traceScript.executionContextId !== script.executionContextId) {
+    return false;
+  }
+  return true;
+}
+
+/**
+ * @param {LH.Crdp.Debugger.ScriptParsedEvent} script
+ * @param {Map<string, TraceScript[]>} traceScriptsByScriptId
+ * @return {string|undefined}
+ */
+function findTraceScriptSource(script, traceScriptsByScriptId) {
+  const candidates = traceScriptsByScriptId.get(script.scriptId);
+  if (!candidates) return undefined;
+
+  const matches = candidates.filter((candidate) => matchesScriptParsedEvent(candidate, script));
+  if (matches.length === 1) return matches[0].sourceText;
+
+  const frameId = script.executionContextAuxData?.frameId;
+  if (matches.length > 1 && frameId) {
+    const frameMatches = matches.filter((candidate) => candidate.frame === frameId);
+    if (frameMatches.length === 1) return frameMatches[0].sourceText;
+  }
+
+  return undefined;
+}
+
 /**
  * Returns true if the script was created via our own calls
  * to Runtime.evaluate.
@@ -50,18 +177,16 @@ function isLighthouseRuntimeEvaluateScript(script) {
 class Scripts extends BaseGatherer {
   static symbol = Symbol('Scripts');
 
-  /** @type {LH.Gatherer.GathererMeta} */
+  /** @type {LH.Gatherer.GathererMeta<'Trace'>} */
   meta = {
     symbol: Scripts.symbol,
     supportedModes: ['timespan', 'navigation'],
+    dependencies: {Trace: Trace.symbol},
   };
 
   /** @type {LH.Crdp.Debugger.ScriptParsedEvent[]} */
   _scriptParsedEvents = [];
 
-  /** @type {Array<string | undefined>} */
-  _scriptContents = [];
-
   constructor() {
     super();
     this.onScriptParsed = this.onScriptParsed.bind(this);
@@ -89,26 +214,42 @@ class Scripts extends BaseGatherer {
    * @param {LH.Gatherer.Context} context
    */
   async stopInstrumentation(context) {
+    const session = context.driver.defaultSession;
+    session.off('Debugger.scriptParsed', this.onScriptParsed);
+  }
+
+  /**
+   * @param {LH.Gatherer.Context<'Trace'>} context
+   * @return {Promise<LH.Artifacts['Scripts']>}
+   */
+  async getArtifact(context) {
     const session = context.driver.defaultSession;
     const formFactor = context.baseArtifacts.HostFormFactor;
+    const traceScriptsByScriptId = collectTraceScripts(context.dependencies.Trace.traceEvents);
+    const traceSources = this._scriptParsedEvents.map(
+      (script) => findTraceScriptSource(script, traceScriptsByScriptId)
+    );
 
-    session.off('Debugger.scriptParsed', this.onScriptParsed);
+    if (traceSources.some((source) => source === undefined)) {
+      // `disableAsyncStacks()` in `prepare.js` runs after `stopInstrumentation` and calls
+      // `Debugger.disable`, so re-enable `Debugger` if we need to call `Debugger.getScriptSource`.
+      await session.sendCommand('Debugger.enable');
+    }
 
     // If run on a mobile device, be sensitive to memory limitations and only
     // request one at a time.
-    this._scriptContents = await runInSeriesOrParallel(
+    const scriptContents = await runInSeriesOrParallel(
       this._scriptParsedEvents,
-      ({scriptId}) => {
-        return session.sendCommand('Debugger.getScriptSource', {scriptId})
+      (script, i) => {
+        if (traceSources[i] !== undefined) return Promise.resolve(traceSources[i]);
+        return session.sendCommand('Debugger.getScriptSource', {scriptId: script.scriptId})
           .then((resp) => resp.scriptSource)
           .catch(() => undefined);
       },
       formFactor === 'mobile' /* runInSeries */
     );
     await session.sendCommand('Debugger.disable');
-  }
 
-  async getArtifact() {
     /** @type {LH.Artifacts['Scripts']} */
     const scripts = this._scriptParsedEvents.map((event, i) => {
       // 'embedderName' and 'url' are confusingly named, so we rewrite them here.
@@ -123,10 +264,12 @@ class Scripts extends BaseGatherer {
         // embedderName is optional on the protocol because backends like Node may not set it.
         // For our purposes, it is always set. But just in case it isn't... fallback to the url.
         url: event.embedderName || event.url,
-        content: this._scriptContents[i],
+        content: scriptContents[i],
       };
     });
 
+    this._scriptParsedEvents = [];
+
     return scripts;
   }
 }

package/core/gather/gatherers/stylesheets.js

@@ -82,6 +82,8 @@ class Stylesheets extends BaseGatherer {
 
     // Ensure we finish fetching all stylesheet contents before disabling the CSS domain
     const sheets = await Promise.all(this._sheetPromises.values());
+    this._sheetPromises.clear();
+    this._session = undefined;
 
     await session.sendCommand('CSS.disable');
     await session.sendCommand('DOM.disable');

package/core/gather/gatherers/trace-elements.js

@@ -21,6 +21,7 @@ import {ProcessedTrace} from '../../computed/processed-trace.js';
 import {Responsiveness} from '../../computed/metrics/responsiveness.js';
 import {CumulativeLayoutShift} from '../../computed/metrics/cumulative-layout-shift.js';
 import {TraceEngineResult} from '../../computed/trace-engine-result.js';
+import {Helpers} from '../../lib/trace-engine.js';
 import SourceMaps from './source-maps.js';
 
 /** @typedef {{nodeId: number, animations?: {name?: string, failureReasonsMask?: number, unsupportedProperties?: string[]}[], type?: string}} TraceElementData */
@@ -80,7 +81,7 @@ class TraceElements extends BaseGatherer {
     }
 
     if (navigationId) {
-      if (insightSet.navigation?.args.data?.navigationId !== navigationId) {
+      if (Helpers.Trace.navigationIDForEvent(insightSet.navigation) !== navigationId) {
         return [];
       }
     } else {

package/core/gather/gatherers/trace.d.ts

@@ -1,6 +1,12 @@
 export default Trace;
 declare class Trace extends BaseGatherer {
     static getDefaultTraceCategories(): string[];
+    /**
+     * @param {LH.Gatherer.ProtocolSession} session
+     * @param {string} handle
+     * @return {Promise<LH.Trace>}
+     */
+    static _readTraceFromStream(session: LH.Gatherer.ProtocolSession, handle: string): Promise<LH.Trace>;
     /**
      * @param {LH.Gatherer.ProtocolSession} session
      * @return {Promise<LH.Trace>}

package/core/gather/gatherers/trace.js

@@ -73,29 +73,68 @@ class Trace extends BaseGatherer {
 
   /**
    * @param {LH.Gatherer.ProtocolSession} session
+   * @param {string} handle
    * @return {Promise<LH.Trace>}
    */
-  static async endTraceAndCollectEvents(session) {
+  static async _readTraceFromStream(session, handle) {
     /** @type {Array<LH.TraceEvent>} */
     const traceEvents = [];
+    const decoder = new TextDecoder();
+    let isEof = false;
+    let remainder = '';
+
+    try {
+      while (!isEof) {
+        const response = await session.sendCommand('IO.read', {handle});
+        isEof = Boolean(response.eof);
+        remainder += response.base64Encoded ?
+          decoder.decode(Buffer.from(response.data, 'base64'), {stream: !isEof}) :
+          response.data;
+
+        // Perfetto's `ExportJson` serializes each trace event and trailing `metadata`
+        // as compact single-line JSON (with string newlines escaped as `\\n`), emitting
+        // raw `,\n` only between top-level elements of `traceEvents`.
+        // Maintain the invariant that `remainder` always starts with `{"traceEvents":[`.
+        const lastEventSep = remainder.lastIndexOf('},\n');
+        if (lastEventSep !== -1) {
+          const {traceEvents: batch} = JSON.parse(`${remainder.slice(0, lastEventSep + 1)}]}`);
+          for (const event of batch) {
+            traceEvents.push(event);
+          }
+          remainder = `{"traceEvents":[${remainder.slice(lastEventSep + 3)}`;
+        }
+      }
+    } finally {
+      await session.sendCommand('IO.close', {handle});
+    }
 
-    /**
-     * Listener for when dataCollected events fire for each trace chunk
-     * @param {LH.Crdp.Tracing.DataCollectedEvent} data
-     */
-    const dataListener = function(data) {
-      traceEvents.push(...data.value);
-    };
-    session.on('Tracing.dataCollected', dataListener);
-
-    return new Promise((resolve, reject) => {
-      session.once('Tracing.tracingComplete', _ => {
-        session.off('Tracing.dataCollected', dataListener);
-        resolve({traceEvents});
-      });
+    const traceJson = JSON.parse(remainder);
+    for (const event of traceJson.traceEvents) {
+      traceEvents.push(event);
+    }
+    traceJson.traceEvents = traceEvents;
+    return traceJson;
+  }
 
-      session.sendCommand('Tracing.end').catch(reject);
+  /**
+   * @param {LH.Gatherer.ProtocolSession} session
+   * @return {Promise<LH.Trace>}
+   */
+  static async endTraceAndCollectEvents(session) {
+    /** @type {LH.Crdp.Tracing.TracingCompleteEvent} */
+    const completeEvent = await new Promise((resolve, reject) => {
+      session.once('Tracing.tracingComplete', resolve);
+      session.sendCommand('Tracing.end').catch(err => {
+        session.off('Tracing.tracingComplete', resolve);
+        reject(err);
+      });
     });
+
+    if (!completeEvent.stream) {
+      throw new Error('Tracing.tracingComplete did not return a stream handle');
+    }
+
+    return Trace._readTraceFromStream(session, completeEvent.stream);
   }
 
   static symbol = Symbol('Trace');
@@ -115,6 +154,7 @@ class Trace extends BaseGatherer {
     await driver.defaultSession.sendCommand('Page.enable');
     await driver.defaultSession.sendCommand('Tracing.start', {
       categories: traceCategories.join(','),
+      transferMode: 'ReturnAsStream',
       options: 'sampling-frequency=10000', // 1000 is default and too slow.
     });
 

package/core/lib/arbitrary-equality-map.d.ts

@@ -5,9 +5,7 @@
  */
 export class ArbitraryEqualityMap {
     /**
-     * Determines whether two objects are deeply equal. Defers to lodash isEqual, but is kept here for
-     * easy usage by consumers.
-     * See https://lodash.com/docs/4.17.5#isEqual.
+     * Determines whether two objects are deeply equal.
      * @param {*} objA
      * @param {*} objB
      * @return {boolean}
@@ -44,4 +42,17 @@ export class ArbitraryEqualityMap {
      */
     _findIndexOf(key: any): number;
 }
+/**
+ * @license
+ * Copyright 2018 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @param {*} a
+ * @param {*} b
+ * @param {WeakMap<object, WeakSet<object>>=} seen
+ * @return {boolean}
+ */
+declare function deepEquals(a: any, b: any, seen?: WeakMap<object, WeakSet<object>> | undefined): boolean;
+export { deepEquals as isEqual };
 //# sourceMappingURL=arbitrary-equality-map.d.ts.map

package/core/lib/arbitrary-equality-map.js

@@ -4,7 +4,116 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import {isEqual} from 'lodash-es';
+/**
+ * @param {*} a
+ * @param {*} b
+ * @param {WeakMap<object, WeakSet<object>>=} seen
+ * @return {boolean}
+ */
+function deepEquals(a, b, seen) {
+  if (a === b || (Number.isNaN(a) && Number.isNaN(b))) return true;
+  if (typeof a !== 'object' || a === null || typeof b !== 'object' || b === null) {
+    return false;
+  }
+  if (a.constructor !== b.constructor) return false;
+
+  if (a instanceof Date && b instanceof Date) {
+    return a.getTime() === b.getTime() || (Number.isNaN(a.getTime()) && Number.isNaN(b.getTime()));
+  }
+  if (a instanceof RegExp && b instanceof RegExp) {
+    return a.source === b.source && a.flags === b.flags;
+  }
+  if (ArrayBuffer.isView(a) && ArrayBuffer.isView(b)) {
+    if (a.byteLength !== b.byteLength) return false;
+    const viewA = new Uint8Array(a.buffer, a.byteOffset, a.byteLength);
+    const viewB = new Uint8Array(b.buffer, b.byteOffset, b.byteLength);
+    for (let i = 0; i < viewA.length; i++) {
+      if (viewA[i] !== viewB[i]) return false;
+    }
+    return true;
+  }
+  if (a instanceof ArrayBuffer && b instanceof ArrayBuffer) {
+    if (a.byteLength !== b.byteLength) return false;
+    const viewA = new Uint8Array(a);
+    const viewB = new Uint8Array(b);
+    for (let i = 0; i < viewA.length; i++) {
+      if (viewA[i] !== viewB[i]) return false;
+    }
+    return true;
+  }
+
+  if (!seen) {
+    seen = new WeakMap();
+  }
+  let seenB = seen.get(a);
+  if (seenB) {
+    if (seenB.has(b)) return true;
+  } else {
+    seenB = new WeakSet();
+    seen.set(a, seenB);
+  }
+  seenB.add(b);
+
+  if (Array.isArray(a)) {
+    const arrB = /** @type {unknown[]} */ (b);
+    if (a.length !== arrB.length) return false;
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEquals(a[i], arrB[i], seen)) return false;
+    }
+    return true;
+  }
+
+  if (a instanceof Map && b instanceof Map) {
+    if (a.size !== b.size) return false;
+    for (const [k, v] of a) {
+      if (!b.has(k) || !deepEquals(v, b.get(k), seen)) return false;
+    }
+    return true;
+  }
+
+  if (a instanceof Set && b instanceof Set) {
+    if (a.size !== b.size) return false;
+    /** @type {unknown[]} */
+    const unmatchedA = [];
+    for (const v of a) {
+      if (!b.has(v)) {
+        if (typeof v !== 'object' || v === null) return false;
+        unmatchedA.push(v);
+      }
+    }
+    if (unmatchedA.length === 0) return true;
+    /** @type {unknown[]} */
+    const unmatchedB = [];
+    for (const v of b) {
+      if (!a.has(v)) {
+        if (typeof v !== 'object' || v === null) return false;
+        unmatchedB.push(v);
+      }
+    }
+    if (unmatchedA.length !== unmatchedB.length) return false;
+    for (const itemA of unmatchedA) {
+      const matchIndex = unmatchedB.findIndex(itemB => deepEquals(itemA, itemB, seen));
+      if (matchIndex === -1) return false;
+      unmatchedB.splice(matchIndex, 1);
+    }
+    return true;
+  }
+
+  if (a instanceof Error && b instanceof Error) {
+    if (a.name !== b.name || a.message !== b.message || !deepEquals(a.cause, b.cause, seen)) {
+      return false;
+    }
+  }
+
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) return false;
+  for (let i = 0; i < keysA.length; i++) {
+    const key = keysA[i];
+    if (!Object.hasOwn(b, key) || !deepEquals(a[key], b[key], seen)) return false;
+  }
+  return true;
+}
 
 /**
  * @fileoverview This class is designed to allow maps with arbitrary equality functions.
@@ -65,16 +174,14 @@ class ArbitraryEqualityMap {
   }
 
   /**
-   * Determines whether two objects are deeply equal. Defers to lodash isEqual, but is kept here for
-   * easy usage by consumers.
-   * See https://lodash.com/docs/4.17.5#isEqual.
+   * Determines whether two objects are deeply equal.
    * @param {*} objA
    * @param {*} objB
    * @return {boolean}
    */
   static deepEquals(objA, objB) {
-    return isEqual(objA, objB);
+    return deepEquals(objA, objB);
   }
 }
 
-export {ArbitraryEqualityMap};
+export {ArbitraryEqualityMap, deepEquals as isEqual};