lighthouse 9.6.7 → 10.0.0

package/core/audits/seo/is-crawlable.js

@@ -0,0 +1,223 @@
+/**
+ * @license Copyright 2017 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import robotsParser from 'robots-parser';
+
+import {Audit} from '../audit.js';
+import {MainResource} from '../../computed/main-resource.js';
+import * as i18n from '../../lib/i18n/i18n.js';
+
+const BOT_USER_AGENTS = new Set([
+  undefined,
+  'Googlebot',
+  'bingbot',
+  'DuckDuckBot',
+  'archive.org_bot',
+]);
+
+const BLOCKLIST = new Set([
+  'noindex',
+  'none',
+]);
+const ROBOTS_HEADER = 'x-robots-tag';
+const UNAVAILABLE_AFTER = 'unavailable_after';
+
+const UIStrings = {
+  /** Title of a Lighthouse audit that provides detail on if search-engine crawlers are blocked from indexing the page. This title is shown when the page is not blocked from indexing and can be crawled. */
+  title: 'Page isn’t blocked from indexing',
+  /** Title of a Lighthouse audit that provides detail on if search-engine crawlers are blocked from indexing the page. This title is shown when the page has been configured to block indexing and therefore cannot be indexed by search engines. */
+  failureTitle: 'Page is blocked from indexing',
+  /** Description of a Lighthouse audit that tells the user *why* allowing search-engine crawling of their page is beneficial. This is displayed after a user expands the section to see more. No character length limits. The last sentence starting with 'Learn' becomes link text to additional documentation. */
+  description: 'Search engines are unable to include your pages in search results ' +
+      'if they don\'t have permission to crawl them. [Learn more about crawler directives](https://developer.chrome.com/docs/lighthouse/seo/is-crawlable/).',
+};
+
+const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings);
+
+/**
+ * Checks if given directive is a valid unavailable_after directive with a date in the past
+ * @param {string} directive
+ * @return {boolean}
+ */
+function isUnavailable(directive) {
+  const parts = directive.split(':');
+
+  if (parts.length <= 1 || parts[0] !== UNAVAILABLE_AFTER) {
+    return false;
+  }
+
+  const date = Date.parse(parts.slice(1).join(':'));
+
+  return !isNaN(date) && date < Date.now();
+}
+
+/**
+ * Returns true if any of provided directives blocks page from being indexed
+ * @param {string} directives assumes no user-agent prefix
+ * @return {boolean}
+ */
+function hasBlockingDirective(directives) {
+  return directives.split(',')
+    .map(d => d.toLowerCase().trim())
+    .some(d => BLOCKLIST.has(d) || isUnavailable(d));
+}
+
+/**
+ * Returns user agent if specified in robots header (e.g. `googlebot: noindex`)
+ * @param {string} directives
+ * @return {string|undefined}
+ */
+function getUserAgentFromHeaderDirectives(directives) {
+  const parts = directives.match(/^([^,:]+):/);
+
+  // Check if directives are prefixed with `googlebot:`, `googlebot-news:`, `otherbot:`, etc.
+  // but ignore `unavailable_after:` which is a valid directive
+  if (!!parts && parts[1].toLowerCase() !== UNAVAILABLE_AFTER) {
+    return parts[1];
+  }
+}
+
+class IsCrawlable extends Audit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      id: 'is-crawlable',
+      title: str_(UIStrings.title),
+      failureTitle: str_(UIStrings.failureTitle),
+      description: str_(UIStrings.description),
+      supportedModes: ['navigation'],
+      requiredArtifacts: ['MetaElements', 'RobotsTxt', 'URL', 'devtoolsLogs'],
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts.MetaElement} metaElement
+   */
+  static handleMetaElement(metaElement) {
+    const content = metaElement.content || '';
+    if (hasBlockingDirective(content)) {
+      return {
+        source: {
+          ...Audit.makeNodeItem(metaElement.node),
+          snippet: `<meta name="${metaElement.name}" content="${content}" />`,
+        },
+      };
+    }
+  }
+
+  /**
+   * @param {string|undefined} userAgent
+   * @param {LH.Artifacts.NetworkRequest} mainResource
+   * @param {LH.Artifacts.MetaElement[]} metaElements
+   * @param {import('robots-parser').Robot|undefined} parsedRobotsTxt
+   * @param {URL} robotsTxtUrl
+   */
+  static determineIfCrawlableForUserAgent(userAgent, mainResource, metaElements,
+      parsedRobotsTxt, robotsTxtUrl) {
+    /** @type {LH.Audit.Details.Table['items']} */
+    const blockingDirectives = [];
+
+    // Prefer a meta element specific to a user agent, fallback to generic 'robots' if not present.
+    // https://developers.google.com/search/blog/2007/03/using-robots-meta-tag#directing-a-robots-meta-tag-specifically-at-googlebot
+    let meta;
+    if (userAgent) meta = metaElements.find(meta => meta.name === userAgent.toLowerCase());
+    if (!meta) meta = metaElements.find(meta => meta.name === 'robots');
+    if (meta) {
+      const blockingDirective = IsCrawlable.handleMetaElement(meta);
+      if (blockingDirective) blockingDirectives.push(blockingDirective);
+    }
+
+    for (const header of mainResource.responseHeaders || []) {
+      if (header.name.toLowerCase() !== ROBOTS_HEADER) continue;
+
+      const directiveUserAgent = getUserAgentFromHeaderDirectives(header.value);
+      if (directiveUserAgent !== userAgent && directiveUserAgent !== undefined) continue;
+
+      let directiveWithoutUserAgentPrefix = header.value.trim();
+      if (userAgent && header.value.startsWith(`${userAgent}:`)) {
+        directiveWithoutUserAgentPrefix = header.value.replace(`${userAgent}:`, '');
+      }
+      if (!hasBlockingDirective(directiveWithoutUserAgentPrefix)) continue;
+
+      blockingDirectives.push({source: `${header.name}: ${header.value}`});
+    }
+
+    if (parsedRobotsTxt && !parsedRobotsTxt.isAllowed(mainResource.url, userAgent)) {
+      const line = parsedRobotsTxt.getMatchingLineNumber(mainResource.url) || 1;
+      blockingDirectives.push({
+        source: {
+          type: /** @type {const} */ ('source-location'),
+          url: robotsTxtUrl.href,
+          urlProvider: /** @type {const} */ ('network'),
+          line: line - 1,
+          column: 0,
+        },
+      });
+    }
+
+    return blockingDirectives;
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @param {LH.Audit.Context} context
+   * @return {Promise<LH.Audit.Product>}
+   */
+  static async audit(artifacts, context) {
+    const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
+    const mainResource = await MainResource.request({devtoolsLog, URL: artifacts.URL}, context);
+
+    const robotsTxtUrl = new URL('/robots.txt', mainResource.url);
+    const parsedRobotsTxt = artifacts.RobotsTxt.content ?
+      robotsParser(robotsTxtUrl.href, artifacts.RobotsTxt.content) :
+      undefined;
+
+    // Only fail if all known bots and generic bots (UserAgent '*' or 'robots' directive)
+    // are blocked from crawling.
+    // If at least one bot is allowed, we pass the audit. Any known bots that are not allowed
+    // will be listed in a warning.
+
+    /** @type {Array<string|undefined>} */
+    const blockedUserAgents = [];
+    const genericBlockingDirectives = [];
+
+    for (const userAgent of BOT_USER_AGENTS) {
+      const blockingDirectives = IsCrawlable.determineIfCrawlableForUserAgent(
+        userAgent, mainResource, artifacts.MetaElements, parsedRobotsTxt, robotsTxtUrl);
+      if (blockingDirectives.length > 0) {
+        blockedUserAgents.push(userAgent);
+      }
+      if (userAgent === undefined) {
+        genericBlockingDirectives.push(...blockingDirectives);
+      }
+    }
+
+    const score = blockedUserAgents.length === BOT_USER_AGENTS.size ? 0 : 1;
+    const warnings = [];
+    if (score && blockedUserAgents.length > 0) {
+      const list = blockedUserAgents.filter(Boolean).join(', ');
+      // eslint-disable-next-line max-len
+      warnings.push(`The following bot user agents are blocked from crawling: ${list}. The audit is otherwise passing, because at least one bot was explicitly allowed.`);
+    }
+
+    /** @type {LH.Audit.Details.Table['headings']} */
+    const headings = [
+      {key: 'source', valueType: 'code', label: 'Blocking Directive Source'},
+    ];
+    const details = Audit.makeTableDetails(headings, score === 0 ? genericBlockingDirectives : []);
+
+    return {
+      score,
+      details,
+      warnings,
+    };
+  }
+}
+
+export default IsCrawlable;
+export {UIStrings};

package/core/audits/seo/link-text.d.ts

@@ -0,0 +1,16 @@
+export default LinkText;
+declare class LinkText extends Audit {
+    /**
+     * @param {LH.Artifacts} artifacts
+     * @return {LH.Audit.Product}
+     */
+    static audit(artifacts: LH.Artifacts): LH.Audit.Product;
+}
+export namespace UIStrings {
+    const title: string;
+    const failureTitle: string;
+    const description: string;
+    const displayValue: string;
+}
+import { Audit } from "../audit.js";
+//# sourceMappingURL=link-text.d.ts.map

package/core/audits/seo/link-text.js

@@ -0,0 +1,160 @@
+/**
+ * @license Copyright 2017 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import {Audit} from '../audit.js';
+import UrlUtils from '../../lib/url-utils.js';
+import * as i18n from '../../lib/i18n/i18n.js';
+
+const BLOCKLIST = new Set([
+  // English
+  'click here',
+  'click this',
+  'go',
+  'here',
+  'information',
+  'learn more',
+  'more',
+  'more info',
+  'more information',
+  'right here',
+  'read more',
+  'see more',
+  'start',
+  'this',
+  // Japanese
+  'ここをクリック',
+  'こちらをクリック',
+  'リンク',
+  '続きを読む',
+  '続く',
+  '全文表示',
+  // Spanish
+  'click aquí',
+  'click aqui',
+  'clicka aquí',
+  'clicka aqui',
+  'pincha aquí',
+  'pincha aqui',
+  'aquí',
+  'aqui',
+  'más',
+  'mas',
+  'más información',
+  'más informacion',
+  'mas información',
+  'mas informacion',
+  'este',
+  'enlace',
+  'este enlace',
+  'empezar',
+  // Portuguese
+  'clique aqui',
+  'ir',
+  'mais informação',
+  'mais informações',
+  'mais',
+  'veja mais',
+  // Korean
+  '여기',
+  '여기를 클릭',
+  '클릭',
+  '링크',
+  '자세히',
+  '자세히 보기',
+  '계속',
+  '이동',
+  '전체 보기',
+  // Swedish
+  'här',
+  'klicka här',
+  'läs mer',
+  'mer',
+  'mer info',
+  'mer information',
+]);
+
+const UIStrings = {
+  /** Title of a Lighthouse audit that tests if each link on a page contains a sufficient description of what a user will find when they click it. Generic, non-descriptive text like "click here" doesn't give an indication of what the link leads to. This descriptive title is shown when all links on the page have sufficient textual descriptions. */
+  title: 'Links have descriptive text',
+  /** Title of a Lighthouse audit that tests if each link on a page contains a sufficient description of what a user will find when they click it. Generic, non-descriptive text like "click here" doesn't give an indication of what the link leads to. This descriptive title is shown when one or more links on the page contain generic, non-descriptive text. */
+  failureTitle: 'Links do not have descriptive text',
+  /** Description of a Lighthouse audit that tells the user *why* they need to have descriptive text on the links in their page. This is displayed after a user expands the section to see more. No character length limits. The last sentence starting with 'Learn' becomes link text to additional documentation. */
+  description: 'Descriptive link text helps search engines understand your content. ' +
+  '[Learn how to make links more accessible](https://developer.chrome.com/docs/lighthouse/seo/link-text/).',
+  /** [ICU Syntax] Label for the audit identifying the number of links found. "link" here refers to the links in a web page to other web pages. */
+  displayValue: `{itemCount, plural,
+    =1 {1 link found}
+    other {# links found}
+    }`,
+};
+
+const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings);
+
+class LinkText extends Audit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      id: 'link-text',
+      title: str_(UIStrings.title),
+      failureTitle: str_(UIStrings.failureTitle),
+      description: str_(UIStrings.description),
+      requiredArtifacts: ['URL', 'AnchorElements'],
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @return {LH.Audit.Product}
+   */
+  static audit(artifacts) {
+    const failingLinks = artifacts.AnchorElements
+      .filter(link => link.href && !link.rel.includes('nofollow'))
+      .filter(link => {
+        const href = link.href.toLowerCase();
+        if (
+          href.startsWith('javascript:') ||
+          href.startsWith('mailto:') ||
+          // This line prevents the audit from flagging anchor links.
+          // In this case it is better to use `finalDisplayedUrl` than `mainDocumentUrl`.
+          UrlUtils.equalWithExcludedFragments(link.href, artifacts.URL.finalDisplayedUrl)
+        ) {
+          return false;
+        }
+
+        return BLOCKLIST.has(link.text.trim().toLowerCase());
+      })
+      .map(link => {
+        return {
+          href: link.href,
+          text: link.text.trim(),
+        };
+      });
+
+    /** @type {LH.Audit.Details.Table['headings']} */
+    const headings = [
+      {key: 'href', valueType: 'url', label: 'Link destination'},
+      {key: 'text', valueType: 'text', label: 'Link Text'},
+    ];
+
+    const details = Audit.makeTableDetails(headings, failingLinks);
+    let displayValue;
+
+    if (failingLinks.length) {
+      displayValue = str_(UIStrings.displayValue, {itemCount: failingLinks.length});
+    }
+
+    return {
+      score: Number(failingLinks.length === 0),
+      details,
+      displayValue,
+    };
+  }
+}
+
+export default LinkText;
+export {UIStrings};

package/core/audits/seo/manual/structured-data.d.ts

@@ -0,0 +1,12 @@
+export default StructuredData;
+/**
+ * @fileoverview Manual SEO audit to check if structured data on page is valid.
+ */
+declare class StructuredData extends ManualAudit {
+}
+export namespace UIStrings {
+    const description: string;
+    const title: string;
+}
+import ManualAudit from "../../manual/manual-audit.js";
+//# sourceMappingURL=structured-data.d.ts.map

package/core/audits/seo/manual/structured-data.js

@@ -0,0 +1,37 @@
+/**
+ * @license Copyright 2017 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import ManualAudit from '../../manual/manual-audit.js';
+import * as i18n from '../../../lib/i18n/i18n.js';
+
+const UIStrings = {
+  /** Description of a Lighthouse audit that provides detail on the structured data in a page. "Structured data" is a standardized data format on a page that helps a search engine categorize and understand its contents. This description is displayed after a user expands the section to see more. No character length limits. The last sentence starting with 'Learn' becomes link text to additional documentation. */
+  description: 'Run the [Structured Data Testing Tool](https://search.google.com/structured-data/testing-tool/) and the [Structured Data Linter](http://linter.structured-data.org/) to validate structured data. [Learn more about Structured Data](https://developer.chrome.com/docs/lighthouse/seo/structured-data/).',
+  /** Title of a Lighthouse audit that prompts users to manually check their page for valid structured data. "Structured data" is a standardized data format on a page that helps a search engine categorize and understand its contents. */
+  title: 'Structured data is valid',
+};
+
+const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings);
+
+/**
+ * @fileoverview Manual SEO audit to check if structured data on page is valid.
+ */
+
+class StructuredData extends ManualAudit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return Object.assign({
+      id: 'structured-data',
+      description: str_(UIStrings.description),
+      title: str_(UIStrings.title),
+    }, super.partialMeta);
+  }
+}
+
+export default StructuredData;
+export {UIStrings};

package/core/audits/seo/meta-description.d.ts

@@ -0,0 +1,16 @@
+export default Description;
+declare class Description extends Audit {
+    /**
+     * @param {LH.Artifacts} artifacts
+     * @return {LH.Audit.Product}
+     */
+    static audit(artifacts: LH.Artifacts): LH.Audit.Product;
+}
+export namespace UIStrings {
+    const title: string;
+    const failureTitle: string;
+    const description: string;
+    const explanation: string;
+}
+import { Audit } from "../audit.js";
+//# sourceMappingURL=meta-description.d.ts.map

package/core/audits/seo/meta-description.js

@@ -0,0 +1,66 @@
+/**
+ * @license Copyright 2017 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import {Audit} from '../audit.js';
+import * as i18n from '../../lib/i18n/i18n.js';
+
+const UIStrings = {
+  /** Title of a Lighthouse audit that provides detail on the web page's document meta description. This descriptive title is shown when the document has a meta description. "meta" should be left untranslated because it refers to an HTML element. */
+  title: 'Document has a meta description',
+  /** Title of a Lighthouse audit that provides detail on the web page's document meta description. This descriptive title is shown when the document does not have a meta description. "meta" should be left untranslated because it refers to an HTML element. */
+  failureTitle: 'Document does not have a meta description',
+  /** Description of a Lighthouse audit that tells the user *why* they need to have meta descriptions on their page. This is displayed after a user expands the section to see more. No character length limits. The last sentence starting with 'Learn' becomes link text to additional documentation. */
+  description: 'Meta descriptions may be included in search results to concisely summarize ' +
+      'page content. ' +
+      '[Learn more about the meta description](https://developer.chrome.com/docs/lighthouse/seo/meta-description/).',
+  /** Explanatory message stating that there was a failure in an audit caused by the page's meta description text being empty. */
+  explanation: 'Description text is empty.',
+};
+
+const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings);
+
+class Description extends Audit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      id: 'meta-description',
+      title: str_(UIStrings.title),
+      failureTitle: str_(UIStrings.failureTitle),
+      description: str_(UIStrings.description),
+      requiredArtifacts: ['MetaElements'],
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @return {LH.Audit.Product}
+   */
+  static audit(artifacts) {
+    const metaDescription = artifacts.MetaElements.find(meta => meta.name === 'description');
+    if (!metaDescription) {
+      return {
+        score: 0,
+      };
+    }
+
+    const description = metaDescription.content || '';
+    if (description.trim().length === 0) {
+      return {
+        score: 0,
+        explanation: str_(UIStrings.explanation),
+      };
+    }
+
+    return {
+      score: 1,
+    };
+  }
+}
+
+export default Description;
+export {UIStrings};

package/core/audits/seo/plugins.d.ts

@@ -0,0 +1,15 @@
+export default Plugins;
+declare class Plugins extends Audit {
+    /**
+     * @param {LH.Artifacts} artifacts
+     * @return {LH.Audit.Product}
+     */
+    static audit(artifacts: LH.Artifacts): LH.Audit.Product;
+}
+export namespace UIStrings {
+    const title: string;
+    const failureTitle: string;
+    const description: string;
+}
+import { Audit } from "../audit.js";
+//# sourceMappingURL=plugins.d.ts.map