lighthouse 9.6.8 → 10.0.0-dev.20230210

package/lighthouse-core/audits/seo/robots-txt.js

@@ -1,256 +0,0 @@
-/**
- * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-/**
- * @fileoverview Validates robots.txt file according to the official standard and its various
- * extensions respected by the popular web crawlers.
- * Validator rules, and the resources backing these rules, can be found here:
- * https://github.com/GoogleChrome/lighthouse/issues/4356#issuecomment-375489925
- */
-
-const Audit = require('../audit.js');
-const URL = require('../../lib/url-shim.js');
-
-const HTTP_CLIENT_ERROR_CODE_LOW = 400;
-const HTTP_SERVER_ERROR_CODE_LOW = 500;
-
-const DIRECTIVE_SITEMAP = 'sitemap';
-const DIRECTIVE_USER_AGENT = 'user-agent';
-const DIRECTIVE_ALLOW = 'allow';
-const DIRECTIVE_DISALLOW = 'disallow';
-const DIRECTIVES_GROUP_MEMBERS = new Set([DIRECTIVE_ALLOW, DIRECTIVE_DISALLOW]);
-const DIRECTIVE_SAFELIST = new Set([
-  DIRECTIVE_USER_AGENT, DIRECTIVE_DISALLOW, // standard
-  DIRECTIVE_ALLOW, DIRECTIVE_SITEMAP, // universally supported
-  'crawl-delay', // yahoo, bing, yandex
-  'clean-param', 'host', // yandex
-  'request-rate', 'visit-time', 'noindex', // not officially supported, but used in the wild
-]);
-const SITEMAP_VALID_PROTOCOLS = new Set(['https:', 'http:', 'ftp:']);
-const i18n = require('../../lib/i18n/i18n.js');
-
-const UIStrings = {
-  /** Title of a Lighthouse audit that provides detail on the site's robots.txt file. Note: "robots.txt" is a canonical filename and should not be translated. This descriptive title is shown when the robots.txt file is present and configured correctly. */
-  title: 'robots.txt is valid',
-  /** Title of a Lighthouse audit that provides detail on the site's robots.txt file. Note: "robots.txt" is a canonical filename and should not be translated. This descriptive title is shown when the robots.txt file is misconfigured, which makes the page hard or impossible to scan via web crawler. */
-  failureTitle: 'robots.txt is not valid',
-  /** Description of a Lighthouse audit that tells the user *why* they need to have a valid robots.txt file. Note: "robots.txt" is a canonical filename and should not be translated. This is displayed after a user expands the section to see more. No character length limits. */
-  description: 'If your robots.txt file is malformed, crawlers may not be able to understand ' +
-  'how you want your website to be crawled or indexed. [Learn more](https://web.dev/robots-txt/).',
-  /**
-   * @description Label for the audit identifying that the robots.txt request has returned a specific HTTP status code. Note: "robots.txt" is a canonical filename and should not be translated.
-   * @example {500} statusCode
-   * */
-  displayValueHttpBadCode: 'Request for robots.txt returned HTTP status: {statusCode}',
-  /** [ICU Syntax] Label for the audit identifying the number of errors that occured while validating the robots.txt file. "itemCount" will be replaced by the integer count of errors encountered. */
-  displayValueValidationError: `{itemCount, plural,
-    =1 {1 error found}
-    other {# errors found}
-    }`,
-  /** Explanatory message stating that there was a failure in an audit caused by Lighthouse not being able to download the robots.txt file for the site.  Note: "robots.txt" is a canonical filename and should not be translated. */
-  explanation: 'Lighthouse was unable to download a robots.txt file',
-};
-
-const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings);
-
-/**
- * @param {string} directiveName
- * @param {string} directiveValue
- * @throws will throw an exception if given directive is invalid
- */
-function verifyDirective(directiveName, directiveValue) {
-  if (!DIRECTIVE_SAFELIST.has(directiveName)) {
-    throw new Error('Unknown directive');
-  }
-
-  if (directiveName === DIRECTIVE_SITEMAP) {
-    let sitemapUrl;
-
-    try {
-      sitemapUrl = new URL(directiveValue);
-    } catch (e) {
-      throw new Error('Invalid sitemap URL');
-    }
-
-    if (!SITEMAP_VALID_PROTOCOLS.has(sitemapUrl.protocol)) {
-      throw new Error('Invalid sitemap URL protocol');
-    }
-  }
-
-  if (directiveName === DIRECTIVE_USER_AGENT && !directiveValue) {
-    throw new Error('No user-agent specified');
-  }
-
-  if (directiveName === DIRECTIVE_ALLOW || directiveName === DIRECTIVE_DISALLOW) {
-    if (directiveValue !== '' && directiveValue[0] !== '/' && directiveValue[0] !== '*') {
-      throw new Error('Pattern should either be empty, start with "/" or "*"');
-    }
-
-    const dollarIndex = directiveValue.indexOf('$');
-
-    if (dollarIndex !== -1 && dollarIndex !== directiveValue.length - 1) {
-      throw new Error('"$" should only be used at the end of the pattern');
-    }
-  }
-}
-
-/**
- * @param {string} line single line from a robots.txt file
- * @throws will throw an exception if given line has errors
- * @return {{directive: string, value: string}|null}
- */
-function parseLine(line) {
-  const hashIndex = line.indexOf('#');
-
-  if (hashIndex !== -1) {
-    line = line.substr(0, hashIndex);
-  }
-
-  line = line.trim();
-
-  if (line.length === 0) {
-    return null;
-  }
-
-  const colonIndex = line.indexOf(':');
-
-  if (colonIndex === -1) {
-    throw new Error('Syntax not understood');
-  }
-
-  const directiveName = line.slice(0, colonIndex).trim().toLowerCase();
-  const directiveValue = line.slice(colonIndex + 1).trim();
-
-  verifyDirective(directiveName, directiveValue);
-
-  return {
-    directive: directiveName,
-    value: directiveValue,
-  };
-}
-
-/**
- * @param {string} content
- * @return {Array<{index: string, line: string, message: string}>}
- */
-function validateRobots(content) {
-  /**
-   * @type Array<{index: string, line: string, message: string}>
-   */
-  const errors = [];
-  let inGroup = false;
-
-  content
-    .split(/\r\n|\r|\n/)
-    .forEach((line, index) => {
-      let parsedLine;
-
-      try {
-        parsedLine = parseLine(line);
-      } catch (e) {
-        errors.push({
-          index: (index + 1).toString(),
-          line: line,
-          message: e.message.toString(),
-        });
-      }
-
-      if (!parsedLine) {
-        return;
-      }
-
-      // group-member records (allow, disallow) have to be precided with a start-of-group record (user-agent)
-      // see: https://developers.google.com/search/reference/robots_txt#grouping-of-records
-      if (parsedLine.directive === DIRECTIVE_USER_AGENT) {
-        inGroup = true;
-      } else if (!inGroup && DIRECTIVES_GROUP_MEMBERS.has(parsedLine.directive)) {
-        errors.push({
-          index: (index + 1).toString(),
-          line: line,
-          message: 'No user-agent specified',
-        });
-      }
-    });
-
-  return errors;
-}
-
-class RobotsTxt extends Audit {
-  /**
-   * @return {LH.Audit.Meta}
-   */
-  static get meta() {
-    return {
-      id: 'robots-txt',
-      title: str_(UIStrings.title),
-      failureTitle: str_(UIStrings.failureTitle),
-      description: str_(UIStrings.description),
-      requiredArtifacts: ['RobotsTxt'],
-    };
-  }
-
-  /**
-   * @param {LH.Artifacts} artifacts
-   * @return {LH.Audit.Product}
-   */
-  static audit(artifacts) {
-    const {
-      status,
-      content,
-    } = artifacts.RobotsTxt;
-
-    if (!status) {
-      return {
-        score: 0,
-        explanation: str_(UIStrings.explanation),
-      };
-    }
-
-    if (status >= HTTP_SERVER_ERROR_CODE_LOW) {
-      return {
-        score: 0,
-        displayValue: str_(UIStrings.displayValueHttpBadCode, {statusCode: status}),
-      };
-    } else if (status >= HTTP_CLIENT_ERROR_CODE_LOW || content === '') {
-      return {
-        score: 1,
-        notApplicable: true,
-      };
-    }
-
-    // If status is good, content must be not null.
-    if (content === null) {
-      throw new Error(`Status ${status} was valid, but content was null`);
-    }
-
-    const validationErrors = validateRobots(content);
-
-    /** @type {LH.Audit.Details.Table['headings']} */
-    const headings = [
-      {key: 'index', itemType: 'text', text: 'Line #'},
-      {key: 'line', itemType: 'code', text: 'Content'},
-      {key: 'message', itemType: 'code', text: 'Error'},
-    ];
-
-    const details = Audit.makeTableDetails(headings, validationErrors, {});
-    let displayValue;
-
-    if (validationErrors.length) {
-      displayValue =
-        str_(UIStrings.displayValueValidationError, {itemCount: validationErrors.length});
-    }
-
-    return {
-      score: Number(validationErrors.length === 0),
-      details,
-      displayValue,
-    };
-  }
-}
-
-module.exports = RobotsTxt;
-module.exports.UIStrings = UIStrings;

package/lighthouse-core/audits/seo/tap-targets.js

@@ -1,352 +0,0 @@
-/**
- * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-/**
- * @fileoverview Checks that links, buttons, etc. are sufficiently large and that there's
- * no other tap target that's too close so that the user might accidentally tap on.
- */
-const Audit = require('../audit.js');
-const ComputedViewportMeta = require('../../computed/viewport-meta.js');
-const {
-  rectsTouchOrOverlap,
-  getRectOverlapArea,
-  getRectAtCenter,
-  allRectsContainedWithinEachOther,
-  getLargestRect,
-  getBoundingRectWithPadding,
-} = require('../../lib/rect-helpers.js');
-const {getTappableRectsFromClientRects} = require('../../lib/tappable-rects.js');
-const i18n = require('../../lib/i18n/i18n.js');
-
-const UIStrings = {
-  /** Title of a Lighthouse audit that provides detail on whether tap targets (like buttons and links) on a page are big enough so they can easily be tapped on a mobile device. This descriptive title is shown when tap targets are easy to tap on. */
-  title: 'Tap targets are sized appropriately',
-  /** Descriptive title of a Lighthouse audit that provides detail on whether tap targets (like buttons and links) on a page are big enough so they can easily be tapped on a mobile device. This descriptive title is shown when tap targets are not easy to tap on. */
-  failureTitle: 'Tap targets are not sized appropriately',
-  /** Description of a Lighthouse audit that tells the user why buttons and links need to be big enough and what 'big enough' means. This is displayed after a user expands the section to see more. No character length limits. 'Learn More' becomes link text to additional documentation. */
-  description: 'Interactive elements like buttons and links should be large enough (48x48px), and have enough space around them, to be easy enough to tap without overlapping onto other elements. [Learn more](https://web.dev/tap-targets/).',
-  /** Label of a table column that identifies tap targets (like buttons and links) that have failed the audit and aren't easy to tap on. */
-  tapTargetHeader: 'Tap Target',
-  /** Label of a table column that identifies a tap target (like a link or button) that overlaps with another tap target. */
-  overlappingTargetHeader: 'Overlapping Target',
-  /** Explanatory message stating that there was a failure in an audit caused by the viewport meta tag not being optimized for mobile screens, which caused tap targets like buttons and links to be too small to tap on. */
-  /* eslint-disable-next-line max-len */
-  explanationViewportMetaNotOptimized: 'Tap targets are too small because there\'s no viewport meta tag optimized for mobile screens',
-  /** Explanatory message stating that a certain percentage of the tap targets (like buttons and links) on the page are of an appropriately large size. */
-  displayValue: '{decimalProportion, number, percent} appropriately sized tap targets',
-};
-
-const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings);
-
-const FINGER_SIZE_PX = 48;
-// Ratio of the finger area tapping on an unintended element
-// to the finger area tapping on the intended element
-const MAX_ACCEPTABLE_OVERLAP_SCORE_RATIO = 0.25;
-
-/**
- * Returns a tap target augmented with a bounding rect for quick overlapping
- * rejections. Rect contains all the client rects, padded to half FINGER_SIZE_PX.
- * @param {LH.Artifacts.TapTarget[]} targets
- * @return {BoundedTapTarget[]}
- */
-function getBoundedTapTargets(targets) {
-  return targets.map(tapTarget => {
-    return {
-      tapTarget,
-      paddedBoundsRect: getBoundingRectWithPadding(tapTarget.clientRects, FINGER_SIZE_PX),
-    };
-  });
-}
-
-/**
- * @param {LH.Artifacts.Rect} cr
- */
-function clientRectBelowMinimumSize(cr) {
-  return cr.width < FINGER_SIZE_PX || cr.height < FINGER_SIZE_PX;
-}
-
-/**
- * A target is "too small" if none of its clientRects are at least the size of a finger.
- * @param {BoundedTapTarget[]} targets
- * @return {BoundedTapTarget[]}
- */
-function getTooSmallTargets(targets) {
-  return targets.filter(target => {
-    return target.tapTarget.clientRects.every(clientRectBelowMinimumSize);
-  });
-}
-
-/**
- * @param {BoundedTapTarget[]} tooSmallTargets
- * @param {BoundedTapTarget[]} allTargets
- * @return {TapTargetOverlapFailure[]}
- */
-function getAllOverlapFailures(tooSmallTargets, allTargets) {
-  /** @type {TapTargetOverlapFailure[]} */
-  const failures = [];
-
-  tooSmallTargets.forEach(target => {
-    // Convert client rects to unique tappable areas from a user's perspective
-    const tappableRects = getTappableRectsFromClientRects(target.tapTarget.clientRects);
-
-    for (const maybeOverlappingTarget of allTargets) {
-      if (maybeOverlappingTarget === target) {
-        // Checking the same target with itself, skip.
-        continue;
-      }
-
-      if (!rectsTouchOrOverlap(target.paddedBoundsRect, maybeOverlappingTarget.paddedBoundsRect)) {
-        // Bounding boxes (padded with half FINGER_SIZE_PX) don't overlap, skip.
-        continue;
-      }
-
-      if (target.tapTarget.href === maybeOverlappingTarget.tapTarget.href) {
-        const isHttpOrHttpsLink = /https?:\/\//.test(target.tapTarget.href);
-        if (isHttpOrHttpsLink) {
-          // No overlap because same target action, skip.
-          continue;
-        }
-      }
-
-      const maybeOverlappingRects = maybeOverlappingTarget.tapTarget.clientRects;
-      if (allRectsContainedWithinEachOther(tappableRects, maybeOverlappingRects)) {
-        // If one tap target is fully contained within the other that's
-        // probably intentional (e.g. an item with a delete button inside).
-        // We'll miss some problems because of this, but that's better
-        // than falsely reporting a failure.
-        continue;
-      }
-
-      const rectFailure = getOverlapFailureForTargetPair(tappableRects, maybeOverlappingRects);
-      if (rectFailure) {
-        failures.push({
-          ...rectFailure,
-          tapTarget: target.tapTarget,
-          overlappingTarget: maybeOverlappingTarget.tapTarget,
-        });
-      }
-    }
-  });
-
-  return failures;
-}
-
-/**
- * @param {LH.Artifacts.Rect[]} tappableRects
- * @param {LH.Artifacts.Rect[]} maybeOverlappingRects
- * @return {ClientRectOverlapFailure | null}
- */
-function getOverlapFailureForTargetPair(tappableRects, maybeOverlappingRects) {
-  /** @type ClientRectOverlapFailure | null */
-  let greatestFailure = null;
-
-  for (const targetCR of tappableRects) {
-    const fingerRect = getRectAtCenter(targetCR, FINGER_SIZE_PX);
-    // Score indicates how much of the finger area overlaps each target when the user
-    // taps on the center of targetCR
-    const tapTargetScore = getRectOverlapArea(fingerRect, targetCR);
-
-    for (const maybeOverlappingCR of maybeOverlappingRects) {
-      const overlappingTargetScore = getRectOverlapArea(fingerRect, maybeOverlappingCR);
-
-      const overlapScoreRatio = overlappingTargetScore / tapTargetScore;
-      if (overlapScoreRatio < MAX_ACCEPTABLE_OVERLAP_SCORE_RATIO) {
-        // low score means it's clear that the user tried to tap on the targetCR,
-        // rather than the other tap target client rect
-        continue;
-      }
-
-      // only update our state if this was the biggest failure we've seen for this pair
-      if (!greatestFailure || overlapScoreRatio > greatestFailure.overlapScoreRatio) {
-        greatestFailure = {
-          overlapScoreRatio,
-          tapTargetScore,
-          overlappingTargetScore,
-        };
-      }
-    }
-  }
-  return greatestFailure;
-}
-
-/**
- * Only report one failure if two targets overlap each other
- * @param {TapTargetOverlapFailure[]} overlapFailures
- * @return {TapTargetOverlapFailure[]}
- */
-function mergeSymmetricFailures(overlapFailures) {
-  /** @type TapTargetOverlapFailure[] */
-  const failuresAfterMerging = [];
-
-  overlapFailures.forEach((failure, overlapFailureIndex) => {
-    const symmetricFailure = overlapFailures.find(f =>
-      f.tapTarget === failure.overlappingTarget &&
-      f.overlappingTarget === failure.tapTarget
-    );
-
-    if (!symmetricFailure) {
-      failuresAfterMerging.push(failure);
-      return;
-    }
-
-    const {overlapScoreRatio: failureOSR} = failure;
-    const {overlapScoreRatio: symmetricOSR} = symmetricFailure;
-    // Push if:
-    // - the current failure has a higher OSR
-    // - OSRs are the same, and the current failure comes before its symmetric partner in the list
-    // Otherwise do nothing and let the symmetric partner be pushed later.
-    if (failureOSR > symmetricOSR || (
-      failureOSR === symmetricOSR &&
-      overlapFailureIndex < overlapFailures.indexOf(symmetricFailure)
-    )) {
-      failuresAfterMerging.push(failure);
-    }
-  });
-
-  return failuresAfterMerging;
-}
-
-/**
- * @param {TapTargetOverlapFailure[]} overlapFailures
- * @return {TapTargetTableItem[]}
- */
-function getTableItems(overlapFailures) {
-  const tableItems = overlapFailures.map(failure => {
-    const largestCR = getLargestRect(failure.tapTarget.clientRects);
-    const width = Math.floor(largestCR.width);
-    const height = Math.floor(largestCR.height);
-    const size = width + 'x' + height;
-    return {
-      tapTarget: Audit.makeNodeItem(failure.tapTarget.node),
-      overlappingTarget: Audit.makeNodeItem(failure.overlappingTarget.node),
-      tapTargetScore: failure.tapTargetScore,
-      overlappingTargetScore: failure.overlappingTargetScore,
-      overlapScoreRatio: failure.overlapScoreRatio,
-      size,
-      width,
-      height,
-    };
-  });
-
-  tableItems.sort((a, b) => {
-    return b.overlapScoreRatio - a.overlapScoreRatio;
-  });
-
-  return tableItems;
-}
-
-class TapTargets extends Audit {
-  /**
-   * @return {LH.Audit.Meta}
-   */
-  static get meta() {
-    return {
-      id: 'tap-targets',
-      title: str_(UIStrings.title),
-      failureTitle: str_(UIStrings.failureTitle),
-      description: str_(UIStrings.description),
-      requiredArtifacts: ['MetaElements', 'TapTargets'],
-    };
-  }
-
-  /**
-   * @param {LH.Artifacts} artifacts
-   * @param {LH.Audit.Context} context
-   * @return {Promise<LH.Audit.Product>}
-   */
-  static async audit(artifacts, context) {
-    if (context.settings.formFactor === 'desktop') {
-      // Tap target sizes aren't important for desktop SEO, so disable the audit there.
-      // On desktop people also tend to have more precise pointing devices than fingers.
-      return {
-        score: 1,
-        notApplicable: true,
-      };
-    }
-
-    const viewportMeta = await ComputedViewportMeta.request(artifacts.MetaElements, context);
-    if (!viewportMeta.isMobileOptimized) {
-      return {
-        score: 0,
-        explanation: str_(UIStrings.explanationViewportMetaNotOptimized),
-      };
-    }
-
-    // Augment the targets with padded bounding rects for quick intersection testing.
-    const boundedTapTargets = getBoundedTapTargets(artifacts.TapTargets);
-
-    const tooSmallTargets = getTooSmallTargets(boundedTapTargets);
-    const overlapFailures = getAllOverlapFailures(tooSmallTargets, boundedTapTargets);
-    const overlapFailuresForDisplay = mergeSymmetricFailures(overlapFailures);
-    const tableItems = getTableItems(overlapFailuresForDisplay);
-
-    /** @type {LH.Audit.Details.Table['headings']} */
-    const headings = [
-      {key: 'tapTarget', itemType: 'node', text: str_(UIStrings.tapTargetHeader)},
-      {key: 'size', itemType: 'text', text: str_(i18n.UIStrings.columnSize)},
-      {key: 'overlappingTarget', itemType: 'node', text: str_(UIStrings.overlappingTargetHeader)},
-    ];
-
-    const details = Audit.makeTableDetails(headings, tableItems);
-
-    const tapTargetCount = artifacts.TapTargets.length;
-    const failingTapTargetCount = new Set(overlapFailures.map(f => f.tapTarget)).size;
-    const passingTapTargetCount = tapTargetCount - failingTapTargetCount;
-
-    let score = 1;
-    let passingTapTargetRatio = 1;
-    if (failingTapTargetCount > 0) {
-      passingTapTargetRatio = (passingTapTargetCount / tapTargetCount);
-      // If there are any failures then we don't want the audit to pass,
-      // so keep the score below 90.
-      score = passingTapTargetRatio * 0.89;
-    }
-    const displayValue = str_(UIStrings.displayValue, {decimalProportion: passingTapTargetRatio});
-
-    return {
-      score,
-      details,
-      displayValue,
-    };
-  }
-}
-
-TapTargets.FINGER_SIZE_PX = FINGER_SIZE_PX;
-
-module.exports = TapTargets;
-module.exports.UIStrings = UIStrings;
-
-
-/** @typedef {{
-  overlapScoreRatio: number;
-  tapTargetScore: number;
-  overlappingTargetScore: number;
-}} ClientRectOverlapFailure */
-
-/** @typedef {{
-  overlapScoreRatio: number;
-  tapTargetScore: number;
-  overlappingTargetScore: number;
-  tapTarget: LH.Artifacts.TapTarget;
-  overlappingTarget: LH.Artifacts.TapTarget;
-}} TapTargetOverlapFailure */
-
-/** @typedef {{
-  paddedBoundsRect: LH.Artifacts.Rect;
-  tapTarget: LH.Artifacts.TapTarget;
-}} BoundedTapTarget */
-
-/** @typedef {{
-  tapTarget: LH.Audit.Details.NodeValue;
-  overlappingTarget: LH.Audit.Details.NodeValue;
-  size: string;
-  overlapScoreRatio: number;
-  height: number;
-  width: number;
-  tapTargetScore: number;
-  overlappingTargetScore: number;
-}} TapTargetTableItem */