lighthouse 9.5.0-dev.20230123 → 9.5.0-dev.20230124

package/shared/util.d.ts

@@ -0,0 +1,122 @@
+export type SnippetValue = import('../types/lhr/audit-details').default.SnippetValue;
+export class Util {
+    static get RATINGS(): {
+        PASS: {
+            label: string;
+            minScore: number;
+        };
+        AVERAGE: {
+            label: string;
+            minScore: number;
+        };
+        FAIL: {
+            label: string;
+        };
+        ERROR: {
+            label: string;
+        };
+    };
+    static get PASS_THRESHOLD(): number;
+    static get MS_DISPLAY_VALUE(): string;
+    /**
+     * If LHR is older than 10.0 it will not have the `finalDisplayedUrl` property.
+     * Old LHRs should have the `finalUrl` property which will work fine for the report.
+     *
+     * @param {LH.Result} lhr
+     */
+    static getFinalDisplayedUrl(lhr: LH.Result): string;
+    /**
+     * If LHR is older than 10.0 it will not have the `mainDocumentUrl` property.
+     * Old LHRs should have the `finalUrl` property which is the same as `mainDocumentUrl`.
+     *
+     * @param {LH.Result} lhr
+     */
+    static getMainDocumentUrl(lhr: LH.Result): string | undefined;
+    /**
+     * @param {LH.Result} lhr
+     * @return {LH.Result.FullPageScreenshot=}
+     */
+    static getFullPageScreenshot(lhr: LH.Result): LH.Result.FullPageScreenshot | undefined;
+    /**
+     * Split a string by markdown code spans (enclosed in `backticks`), splitting
+     * into segments that were enclosed in backticks (marked as `isCode === true`)
+     * and those that outside the backticks (`isCode === false`).
+     * @param {string} text
+     * @return {Array<{isCode: true, text: string}|{isCode: false, text: string}>}
+     */
+    static splitMarkdownCodeSpans(text: string): Array<{
+        isCode: true;
+        text: string;
+    } | {
+        isCode: false;
+        text: string;
+    }>;
+    /**
+     * Split a string on markdown links (e.g. [some link](https://...)) into
+     * segments of plain text that weren't part of a link (marked as
+     * `isLink === false`), and segments with text content and a URL that did make
+     * up a link (marked as `isLink === true`).
+     * @param {string} text
+     * @return {Array<{isLink: true, text: string, linkHref: string}|{isLink: false, text: string}>}
+     */
+    static splitMarkdownLink(text: string): Array<{
+        isLink: true;
+        text: string;
+        linkHref: string;
+    } | {
+        isLink: false;
+        text: string;
+    }>;
+    /**
+     * @param {URL} parsedUrl
+     * @param {{numPathParts?: number, preserveQuery?: boolean, preserveHost?: boolean}=} options
+     * @return {string}
+     */
+    static getURLDisplayName(parsedUrl: URL, options?: {
+        numPathParts?: number | undefined;
+        preserveQuery?: boolean | undefined;
+        preserveHost?: boolean | undefined;
+    } | undefined): string;
+    /**
+     * Split a URL into a file, hostname and origin for easy display.
+     * @param {string} url
+     * @return {{file: string, hostname: string, origin: string}}
+     */
+    static parseURL(url: string): {
+        file: string;
+        hostname: string;
+        origin: string;
+    };
+    /**
+     * @param {string|URL} value
+     * @return {!URL}
+     */
+    static createOrReturnURL(value: string | URL): URL;
+    /**
+     * Gets the tld of a domain
+     *
+     * @param {string} hostname
+     * @return {string} tld
+     */
+    static getTld(hostname: string): string;
+    /**
+     * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+     * @param {string|URL} url hostname or URL object
+     * @return {string}
+     */
+    static getRootDomain(url: string | URL): string;
+    /**
+     * Returns only lines that are near a message, or the first few lines if there are
+     * no line messages.
+     * @param {SnippetValue['lines']} lines
+     * @param {SnippetValue['lineMessages']} lineMessages
+     * @param {number} surroundingLineCount Number of lines to include before and after
+     * the message. If this is e.g. 2 this function might return 5 lines.
+     */
+    static filterRelevantLines(lines: SnippetValue['lines'], lineMessages: SnippetValue['lineMessages'], surroundingLineCount: number): {
+        content: string;
+        lineNumber: number;
+        truncated?: boolean | undefined;
+    }[];
+}
+//# sourceMappingURL=util.d.ts.map

package/shared/util.js

@@ -0,0 +1,332 @@
+/**
+ * @license
+ * Copyright 2017 The Lighthouse Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** @typedef {import('../types/lhr/audit-details').default.SnippetValue} SnippetValue */
+
+const ELLIPSIS = '\u2026';
+const NBSP = '\xa0';
+const PASS_THRESHOLD = 0.9;
+
+const RATINGS = {
+  PASS: {label: 'pass', minScore: PASS_THRESHOLD},
+  AVERAGE: {label: 'average', minScore: 0.5},
+  FAIL: {label: 'fail'},
+  ERROR: {label: 'error'},
+};
+
+// 25 most used tld plus one domains (aka public suffixes) from http archive.
+// @see https://github.com/GoogleChrome/lighthouse/pull/5065#discussion_r191926212
+// The canonical list is https://publicsuffix.org/learn/ but we're only using subset to conserve bytes
+const listOfTlds = [
+  'com', 'co', 'gov', 'edu', 'ac', 'org', 'go', 'gob', 'or', 'net', 'in', 'ne', 'nic', 'gouv',
+  'web', 'spb', 'blog', 'jus', 'kiev', 'mil', 'wi', 'qc', 'ca', 'bel', 'on',
+];
+
+class Util {
+  static get RATINGS() {
+    return RATINGS;
+  }
+
+  static get PASS_THRESHOLD() {
+    return PASS_THRESHOLD;
+  }
+
+  static get MS_DISPLAY_VALUE() {
+    return `%10d${NBSP}ms`;
+  }
+
+  /**
+   * If LHR is older than 10.0 it will not have the `finalDisplayedUrl` property.
+   * Old LHRs should have the `finalUrl` property which will work fine for the report.
+   *
+   * @param {LH.Result} lhr
+   */
+  static getFinalDisplayedUrl(lhr) {
+    if (lhr.finalDisplayedUrl) return lhr.finalDisplayedUrl;
+    if (lhr.finalUrl) return lhr.finalUrl;
+    throw new Error('Could not determine final displayed URL');
+  }
+
+  /**
+   * If LHR is older than 10.0 it will not have the `mainDocumentUrl` property.
+   * Old LHRs should have the `finalUrl` property which is the same as `mainDocumentUrl`.
+   *
+   * @param {LH.Result} lhr
+   */
+  static getMainDocumentUrl(lhr) {
+    return lhr.mainDocumentUrl || lhr.finalUrl;
+  }
+
+  /**
+   * @param {LH.Result} lhr
+   * @return {LH.Result.FullPageScreenshot=}
+   */
+  static getFullPageScreenshot(lhr) {
+    if (lhr.fullPageScreenshot) {
+      return lhr.fullPageScreenshot;
+    }
+
+    // Prior to 10.0.
+    const details = /** @type {LH.Result.FullPageScreenshot=} */ (
+      lhr.audits['full-page-screenshot']?.details);
+    return details;
+  }
+
+  /**
+   * Split a string by markdown code spans (enclosed in `backticks`), splitting
+   * into segments that were enclosed in backticks (marked as `isCode === true`)
+   * and those that outside the backticks (`isCode === false`).
+   * @param {string} text
+   * @return {Array<{isCode: true, text: string}|{isCode: false, text: string}>}
+   */
+  static splitMarkdownCodeSpans(text) {
+    /** @type {Array<{isCode: true, text: string}|{isCode: false, text: string}>} */
+    const segments = [];
+
+    // Split on backticked code spans.
+    const parts = text.split(/`(.*?)`/g);
+    for (let i = 0; i < parts.length; i ++) {
+      const text = parts[i];
+
+      // Empty strings are an artifact of splitting, not meaningful.
+      if (!text) continue;
+
+      // Alternates between plain text and code segments.
+      const isCode = i % 2 !== 0;
+      segments.push({
+        isCode,
+        text,
+      });
+    }
+
+    return segments;
+  }
+
+  /**
+   * Split a string on markdown links (e.g. [some link](https://...)) into
+   * segments of plain text that weren't part of a link (marked as
+   * `isLink === false`), and segments with text content and a URL that did make
+   * up a link (marked as `isLink === true`).
+   * @param {string} text
+   * @return {Array<{isLink: true, text: string, linkHref: string}|{isLink: false, text: string}>}
+   */
+  static splitMarkdownLink(text) {
+    /** @type {Array<{isLink: true, text: string, linkHref: string}|{isLink: false, text: string}>} */
+    const segments = [];
+
+    const parts = text.split(/\[([^\]]+?)\]\((https?:\/\/.*?)\)/g);
+    while (parts.length) {
+      // Shift off the same number of elements as the pre-split and capture groups.
+      const [preambleText, linkText, linkHref] = parts.splice(0, 3);
+
+      if (preambleText) { // Skip empty text as it's an artifact of splitting, not meaningful.
+        segments.push({
+          isLink: false,
+          text: preambleText,
+        });
+      }
+
+      // Append link if there are any.
+      if (linkText && linkHref) {
+        segments.push({
+          isLink: true,
+          text: linkText,
+          linkHref,
+        });
+      }
+    }
+
+    return segments;
+  }
+
+  /**
+   * @param {URL} parsedUrl
+   * @param {{numPathParts?: number, preserveQuery?: boolean, preserveHost?: boolean}=} options
+   * @return {string}
+   */
+  static getURLDisplayName(parsedUrl, options) {
+    // Closure optional properties aren't optional in tsc, so fallback needs undefined  values.
+    options = options || {numPathParts: undefined, preserveQuery: undefined,
+      preserveHost: undefined};
+    const numPathParts = options.numPathParts !== undefined ? options.numPathParts : 2;
+    const preserveQuery = options.preserveQuery !== undefined ? options.preserveQuery : true;
+    const preserveHost = options.preserveHost || false;
+
+    let name;
+
+    if (parsedUrl.protocol === 'about:' || parsedUrl.protocol === 'data:') {
+      // Handle 'about:*' and 'data:*' URLs specially since they have no path.
+      name = parsedUrl.href;
+    } else {
+      name = parsedUrl.pathname;
+      const parts = name.split('/').filter(part => part.length);
+      if (numPathParts && parts.length > numPathParts) {
+        name = ELLIPSIS + parts.slice(-1 * numPathParts).join('/');
+      }
+
+      if (preserveHost) {
+        name = `${parsedUrl.host}/${name.replace(/^\//, '')}`;
+      }
+      if (preserveQuery) {
+        name = `${name}${parsedUrl.search}`;
+      }
+    }
+
+    const MAX_LENGTH = 64;
+    if (parsedUrl.protocol !== 'data:') {
+      // Always elide hexadecimal hash
+      name = name.replace(/([a-f0-9]{7})[a-f0-9]{13}[a-f0-9]*/g, `$1${ELLIPSIS}`);
+      // Also elide other hash-like mixed-case strings
+      name = name.replace(/([a-zA-Z0-9-_]{9})(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])[a-zA-Z0-9-_]{10,}/g,
+        `$1${ELLIPSIS}`);
+      // Also elide long number sequences
+      name = name.replace(/(\d{3})\d{6,}/g, `$1${ELLIPSIS}`);
+      // Merge any adjacent ellipses
+      name = name.replace(/\u2026+/g, ELLIPSIS);
+
+      // Elide query params first
+      if (name.length > MAX_LENGTH && name.includes('?')) {
+        // Try to leave the first query parameter intact
+        name = name.replace(/\?([^=]*)(=)?.*/, `?$1$2${ELLIPSIS}`);
+
+        // Remove it all if it's still too long
+        if (name.length > MAX_LENGTH) {
+          name = name.replace(/\?.*/, `?${ELLIPSIS}`);
+        }
+      }
+    }
+
+    // Elide too long names next
+    if (name.length > MAX_LENGTH) {
+      const dotIndex = name.lastIndexOf('.');
+      if (dotIndex >= 0) {
+        name = name.slice(0, MAX_LENGTH - 1 - (name.length - dotIndex)) +
+          // Show file extension
+          `${ELLIPSIS}${name.slice(dotIndex)}`;
+      } else {
+        name = name.slice(0, MAX_LENGTH - 1) + ELLIPSIS;
+      }
+    }
+
+    return name;
+  }
+
+  /**
+   * Split a URL into a file, hostname and origin for easy display.
+   * @param {string} url
+   * @return {{file: string, hostname: string, origin: string}}
+   */
+  static parseURL(url) {
+    const parsedUrl = new URL(url);
+    return {
+      file: Util.getURLDisplayName(parsedUrl),
+      hostname: parsedUrl.hostname,
+      origin: parsedUrl.origin,
+    };
+  }
+
+  /**
+   * @param {string|URL} value
+   * @return {!URL}
+   */
+  static createOrReturnURL(value) {
+    if (value instanceof URL) {
+      return value;
+    }
+
+    return new URL(value);
+  }
+
+  /**
+   * Gets the tld of a domain
+   *
+   * @param {string} hostname
+   * @return {string} tld
+   */
+  static getTld(hostname) {
+    const tlds = hostname.split('.').slice(-2);
+
+    if (!listOfTlds.includes(tlds[0])) {
+      return `.${tlds[tlds.length - 1]}`;
+    }
+
+    return `.${tlds.join('.')}`;
+  }
+
+  /**
+   * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+   * @param {string|URL} url hostname or URL object
+   * @return {string}
+   */
+  static getRootDomain(url) {
+    const hostname = Util.createOrReturnURL(url).hostname;
+    const tld = Util.getTld(hostname);
+
+    // tld is .com or .co.uk which means we means that length is 1 to big
+    // .com => 2 & .co.uk => 3
+    const splitTld = tld.split('.');
+
+    // get TLD + root domain
+    return hostname.split('.').slice(-splitTld.length).join('.');
+  }
+
+  /**
+   * Returns only lines that are near a message, or the first few lines if there are
+   * no line messages.
+   * @param {SnippetValue['lines']} lines
+   * @param {SnippetValue['lineMessages']} lineMessages
+   * @param {number} surroundingLineCount Number of lines to include before and after
+   * the message. If this is e.g. 2 this function might return 5 lines.
+   */
+  static filterRelevantLines(lines, lineMessages, surroundingLineCount) {
+    if (lineMessages.length === 0) {
+      // no lines with messages, just return the first bunch of lines
+      return lines.slice(0, surroundingLineCount * 2 + 1);
+    }
+
+    const minGapSize = 3;
+    const lineNumbersToKeep = new Set();
+    // Sort messages so we can check lineNumbersToKeep to see how big the gap to
+    // the previous line is.
+    lineMessages = lineMessages.sort((a, b) => (a.lineNumber || 0) - (b.lineNumber || 0));
+    lineMessages.forEach(({lineNumber}) => {
+      let firstSurroundingLineNumber = lineNumber - surroundingLineCount;
+      let lastSurroundingLineNumber = lineNumber + surroundingLineCount;
+
+      while (firstSurroundingLineNumber < 1) {
+        // make sure we still show (surroundingLineCount * 2 + 1) lines in total
+        firstSurroundingLineNumber++;
+        lastSurroundingLineNumber++;
+      }
+      // If only a few lines would be omitted normally then we prefer to include
+      // extra lines to avoid the tiny gap
+      if (lineNumbersToKeep.has(firstSurroundingLineNumber - minGapSize - 1)) {
+        firstSurroundingLineNumber -= minGapSize;
+      }
+      for (let i = firstSurroundingLineNumber; i <= lastSurroundingLineNumber; i++) {
+        const surroundingLineNumber = i;
+        lineNumbersToKeep.add(surroundingLineNumber);
+      }
+    });
+
+    return lines.filter(line => lineNumbersToKeep.has(line.lineNumber));
+  }
+}
+
+export {
+  Util,
+};

package/types/artifacts.d.ts

@@ -17,6 +17,7 @@ import type { TaskNode as _TaskNode } from '../core/lib/tracehouse/main-thread-t
 import AuditDetails from './lhr/audit-details.js'
 import Config from './config.js';
 import Gatherer from './gatherer.js';
+import {IEntity} from 'third-party-web';
 import {IcuMessage} from './lhr/i18n.js';
 import LHResult from './lhr/lhr.js'
 import Protocol from './protocol.js';
@@ -925,6 +926,19 @@ declare module Artifacts {
     /** The MIME type of the underlying image file. */
     mimeType?: string;
   }
+
+  interface Entity extends IEntity {
+    isUnrecognized?: boolean;
+  }
+
+  interface EntityClassification {
+    urlsByEntity: Map<Entity, Set<string>>;
+    entityByUrl: Map<string, Entity>;
+    firstParty?: Entity;
+
+    // Convenience methods.
+    isFirstParty: (url: string) => boolean;
+  }
 }
 
 export interface Trace {

package/types/lhr/lhr.d.ts

@@ -56,6 +56,8 @@ interface Result {
   };
   /** An array containing the result of all stack packs. */
   stackPacks?: Result.StackPack[];
+  /** All the origins encountered during this Lighthouse run, and information about what web property (aka "entity") they belong to. Won't be present for snapshot mode. */
+  entities?: Result.Entities;
   /** Screenshot taken of the full page, with node rects referencing audit results. If there was an error with collection, this is null. If disabled via the disableFullPageScreenshot setting, this is undefined. */
   fullPageScreenshot?: Result.FullPageScreenshot | null;
 }
@@ -151,6 +153,37 @@ declare module Result {
     nodes: Record<string, AuditDetails.Rect>;
   }
 
+  /**
+   * Entity classification for the run, for resolving URLs/items to entities in report.
+   * The two lookup tables (LUT) below provide space-optimized, O(1) index lookup into entities.list.
+   */
+  interface Entities {
+    /** All entities (1st and 3rd party) discovered during the run */
+    list: Array<LhrEntity>;
+    /** Name of the first-party entity */
+    firstParty?: string;
+    /** Entity-name to entity index lookup table  */
+    entityIndexByName: Record<string, number>;
+    /** URL origin to entity index lookup table */
+    entityIndexByOrigin: Record<string, number>;
+  }
+
+  /**
+   * An entity that's either recognized by third-party-web or made up by Lighthouse.
+   */
+  interface LhrEntity {
+    /** Name of the entity. Maps to third-party-web unique name for recognized entities and a root domain name for the unrecognized. */
+    name: string;
+    /** Homepage URL for a recognized entity, if available in third-party-web. */
+    homepage?: string;
+    /** Category name that the entity belongs to, if available. */
+    category?: string;
+    /** Is this entity the first party? */
+    isFirstParty?: boolean;
+    /** Is this entity recognized by third-party-web? */
+    isUnrecognized?: boolean;
+  }
+
   /**
    * Info about an `LH.IcuMessage` value that was localized to a string when
    * included in the LHR. Value is either a