lighthouse 9.2.0-dev.20220109 → 9.2.0-dev.20220113

package/lighthouse-cli/test/smokehouse/smokehouse.js

@@ -275,7 +275,6 @@ function getShardedDefinitions(testDefns, shardArg) {
   return shardDefns;
 }
 
-
 export {
   runSmokehouse,
   getShardedDefinitions,

package/lighthouse-core/audits/audit.js

@@ -73,13 +73,28 @@ class Audit {
    * considering a log-normal distribution governed by two control points (the 10th
    * percentile value and the median value) and represents the percentage of sites that are
    * greater than `value`.
+   *
+   * Score characteristics:
+   * - within [0, 1]
+   * - rounded to two digits
+   * - value must meet or beat a controlPoint value to meet or exceed its percentile score:
+   *   - value > median will give a score < 0.5; value ≤ median will give a score ≥ 0.5.
+   *   - value > p10 will give a score < 0.9; value ≤ p10 will give a score ≥ 0.9.
+   * - values < p10 will get a slight boost so a score of 1 is achievable by a
+   *   `value` other than those close to 0. Scores of > ~0.99524 end up rounded to 1.
    * @param {{median: number, p10: number}} controlPoints
    * @param {number} value
    * @return {number}
    */
   static computeLogNormalScore(controlPoints, value) {
-    const percentile = statistics.getLogNormalScore(controlPoints, value);
-    return clampTo2Decimals(percentile);
+    let percentile = statistics.getLogNormalScore(controlPoints, value);
+    // Add a boost to scores of 90+, linearly ramping from 0 at 0.9 to half a
+    // point (0.005) at 1. Expands scores in (0.9, 1] to (0.9, 1.005], so more top
+    // scores will be a perfect 1 after the two-digit `Math.floor()` rounding below.
+    if (percentile > 0.9) { // getLogNormalScore ensures `percentile` can't exceed 1.
+      percentile += 0.05 * (percentile - 0.9);
+    }
+    return Math.floor(percentile * 100) / 100;
   }
 
   /**

package/lighthouse-core/audits/deprecations.js

@@ -62,8 +62,8 @@ class Deprecations extends Audit {
     const bundles = await JsBundles.request(artifacts, context);
 
     let deprecations;
-    if (artifacts.InspectorIssues.deprecations.length) {
-      deprecations = artifacts.InspectorIssues.deprecations
+    if (artifacts.InspectorIssues.deprecationIssue.length) {
+      deprecations = artifacts.InspectorIssues.deprecationIssue
         .map(deprecation => {
           const {url, lineNumber, columnNumber} = deprecation.sourceCodeLocation;
           const bundle = bundles.find(bundle => bundle.script.src === url);

package/lighthouse-core/audits/dobetterweb/inspector-issues.js

@@ -161,19 +161,19 @@ class IssuesPanelEntries extends Audit {
     /** @type LH.Audit.Details.TableItem[] */
     const items = [];
 
-    if (issues.mixedContent.length) {
-      items.push(this.getMixedContentRow(issues.mixedContent));
+    if (issues.mixedContentIssue.length) {
+      items.push(this.getMixedContentRow(issues.mixedContentIssue));
     }
-    if (issues.sameSiteCookies.length) {
-      items.push(this.getSameSiteCookieRow(issues.sameSiteCookies));
+    if (issues.sameSiteCookieIssue.length) {
+      items.push(this.getSameSiteCookieRow(issues.sameSiteCookieIssue));
     }
-    if (issues.blockedByResponse.length) {
-      items.push(this.getBlockedByResponseRow(issues.blockedByResponse));
+    if (issues.blockedByResponseIssue.length) {
+      items.push(this.getBlockedByResponseRow(issues.blockedByResponseIssue));
     }
-    if (issues.heavyAds.length) {
+    if (issues.heavyAdIssue.length) {
       items.push({issueType: str_(UIStrings.issueTypeHeavyAds)});
     }
-    const cspIssues = issues.contentSecurityPolicy.filter(issue => {
+    const cspIssues = issues.contentSecurityPolicyIssue.filter(issue => {
       // kTrustedTypesSinkViolation and kTrustedTypesPolicyViolation aren't currently supported by the Issues panel
       return issue.contentSecurityPolicyViolationType !== 'kTrustedTypesSinkViolation' &&
         issue.contentSecurityPolicyViolationType !== 'kTrustedTypesPolicyViolation';

package/lighthouse-core/audits/is-on-https.js

@@ -87,7 +87,7 @@ class HTTPS extends Audit {
         {key: 'resolution', itemType: 'text', text: str_(UIStrings.columnResolution)},
       ];
 
-      for (const details of artifacts.InspectorIssues.mixedContent) {
+      for (const details of artifacts.InspectorIssues.mixedContentIssue) {
         let item = items.find(item => item.url === details.insecureURL);
         if (!item) {
           item = {url: details.insecureURL};

package/lighthouse-core/audits/seo/is-crawlable.js

@@ -6,7 +6,8 @@
 'use strict';
 
 const Audit = require('../audit.js');
-const robotsParser = require('robots-parser');
+// TODO(esmodules): cast can be removed when this switches to import.
+const robotsParser = /** @type {typeof import('robots-parser').default} */ (/** @type {unknown} */(require('robots-parser'))); // eslint-disable-line max-len
 const URL = require('../../lib/url-shim.js');
 const MainResource = require('../../computed/main-resource.js');
 const BLOCKLIST = new Set([

package/lighthouse-core/gather/gatherers/inspector-issues.js

@@ -58,58 +58,48 @@ class InspectorIssues extends FRGatherer {
    * @return {Promise<LH.Artifacts['InspectorIssues']>}
    */
   async _getArtifact(networkRecords) {
+    /** @type {LH.Artifacts.InspectorIssues} */
     const artifact = {
-      /** @type {Array<LH.Crdp.Audits.MixedContentIssueDetails>} */
-      mixedContent: [],
-      /** @type {Array<LH.Crdp.Audits.SameSiteCookieIssueDetails>} */
-      sameSiteCookies: [],
-      /** @type {Array<LH.Crdp.Audits.BlockedByResponseIssueDetails>} */
-      blockedByResponse: [],
-      /** @type {Array<LH.Crdp.Audits.HeavyAdIssueDetails>} */
-      heavyAds: [],
-      /** @type {Array<LH.Crdp.Audits.ContentSecurityPolicyIssueDetails>} */
-      contentSecurityPolicy: [],
-      /** @type {Array<LH.Crdp.Audits.DeprecationIssueDetails>} */
-      deprecations: [],
+      attributionReportingIssue: [],
+      blockedByResponseIssue: [],
+      clientHintIssue: [],
+      contentSecurityPolicyIssue: [],
+      corsIssue: [],
+      deprecationIssue: [],
+      genericIssue: [],
+      heavyAdIssue: [],
+      lowTextContrastIssue: [],
+      mixedContentIssue: [],
+      navigatorUserAgentIssue: [],
+      quirksModeIssue: [],
+      sameSiteCookieIssue: [],
+      sharedArrayBufferIssue: [],
+      twaQualityEnforcement: [],
+      wasmCrossOriginModuleSharingIssue: [],
     };
-
-    for (const issue of this._issues) {
-      if (issue.details.mixedContentIssueDetails) {
-        const issueDetails = issue.details.mixedContentIssueDetails;
-        const issueReqId = issueDetails.request?.requestId;
-        // Duplicate issues can occur for the same request; only use the one with a matching networkRequest.
-        if (issueReqId &&
-          networkRecords.find(req => req.requestId === issueReqId)) {
-          artifact.mixedContent.push(issueDetails);
+    const keys = /** @type {Array<keyof LH.Artifacts['InspectorIssues']>} */(Object.keys(artifact));
+    for (const key of keys) {
+      // The wasmCrossOriginModuleSharingIssue key doesn't follow the pattern of the rest. See
+      // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/public/devtools_protocol/browser_protocol.pdl;l=811;drc=9c10bf258928b5d169ba6a449f8f33958f7947ea
+      /** @type {`${key}Details` | `wasmCrossOriginModuleSharingIssue`} */
+      const detailsKey = (key === 'wasmCrossOriginModuleSharingIssue' ? `${key}` : `${key}Details`);
+      const allDetails = this._issues.map(issue => issue.details[detailsKey]);
+      for (const detail of allDetails) {
+        if (!detail) {
+          continue;
         }
-      }
-      if (issue.details.sameSiteCookieIssueDetails) {
-        const issueDetails = issue.details.sameSiteCookieIssueDetails;
-        const issueReqId = issueDetails.request?.requestId;
         // Duplicate issues can occur for the same request; only use the one with a matching networkRequest.
-        if (issueReqId &&
-          networkRecords.find(req => req.requestId === issueReqId)) {
-          artifact.sameSiteCookies.push(issueDetails);
+        const requestId = 'request' in detail && detail.request && detail.request.requestId;
+        if (requestId) {
+          if (networkRecords.find(req => req.requestId === requestId)) {
+            // @ts-expect-error - detail types are not all compatible
+            artifact[key].push(detail);
+          }
+        } else {
+          // @ts-expect-error - detail types are not all compatible
+          artifact[key].push(detail);
         }
       }
-      if (issue.details.blockedByResponseIssueDetails) {
-        const issueDetails = issue.details.blockedByResponseIssueDetails;
-        const issueReqId = issueDetails.request?.requestId;
-        // Duplicate issues can occur for the same request; only use the one with a matching networkRequest.
-        if (issueReqId &&
-          networkRecords.find(req => req.requestId === issueReqId)) {
-          artifact.blockedByResponse.push(issueDetails);
-        }
-      }
-      if (issue.details.heavyAdIssueDetails) {
-        artifact.heavyAds.push(issue.details.heavyAdIssueDetails);
-      }
-      if (issue.details.contentSecurityPolicyIssueDetails) {
-        artifact.contentSecurityPolicy.push(issue.details.contentSecurityPolicyIssueDetails);
-      }
-      if (issue.details.deprecationIssueDetails) {
-        artifact.deprecations.push(issue.details.deprecationIssueDetails);
-      }
     }
 
     return artifact;

package/lighthouse-core/lib/asset-saver.js

@@ -19,6 +19,8 @@ const LHError = require('../lib/lh-error.js');
 // TODO(esmodules): Rollup does not support `promisfy` or `stream.pipeline`. Bundled files
 // don't need anything in this file except for `stringifyReplacer`, so a check for
 // truthiness before using is enough.
+// TODO: Can remove promisify(pipeline) in Node 15.
+// https://nodejs.org/api/stream.html#streams-promises-api
 const pipeline = promisify && promisify(stream.pipeline);
 
 const artifactsFilename = 'artifacts.json';
@@ -238,8 +240,6 @@ async function saveTrace(traceData, traceFilename) {
   const traceIter = traceJsonGenerator(traceData);
   const writeStream = fs.createWriteStream(traceFilename);
 
-  // TODO: Can remove promisify(pipeline) in Node 15.
-  // https://nodejs.org/api/stream.html#stream_stream_pipeline_streams_callback
   return pipeline(traceIter, writeStream);
 }
 
@@ -250,10 +250,12 @@ async function saveTrace(traceData, traceFilename) {
  * @return {Promise<void>}
  */
 function saveDevtoolsLog(devtoolsLog, devtoolLogFilename) {
-  const logIter = arrayOfObjectsJsonGenerator(devtoolsLog);
   const writeStream = fs.createWriteStream(devtoolLogFilename);
 
-  return pipeline(logIter, writeStream);
+  return pipeline(function* () {
+    yield* arrayOfObjectsJsonGenerator(devtoolsLog);
+    yield '\n';
+  }, writeStream);
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse",
-  "version": "9.2.0-dev.20220109",
+  "version": "9.2.0-dev.20220113",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./lighthouse-core/index.js",
   "bin": {
@@ -14,7 +14,7 @@
   "scripts": {
     "prepack": "yarn build-report --standalone --flow --esm",
     "build-all": "npm-run-posix-or-windows build-all:task",
-    "build-all:task": "yarn build-report && yarn build-cdt-lib && yarn build-devtools && (yarn build-extension & yarn build-lr & yarn build-viewer & yarn build-treemap & yarn build-smokehouse-bundle & wait) && yarn build-pack",
+    "build-all:task": "yarn build-report && yarn build-cdt-lib && yarn build-devtools && concurrently 'yarn build-extension' 'yarn build-lr' 'yarn build-viewer' 'yarn build-treemap' 'yarn build-smokehouse-bundle' && yarn build-pack",
     "build-all:task:windows": "yarn build-report && yarn build-cdt-lib && yarn build-extension && yarn build-devtools && yarn build-lr && yarn build-viewer && yarn build-treemap && yarn build-smokehouse-bundle",
     "build-cdt-lib": "node ./build/build-cdt-lib.js",
     "build-extension": "yarn build-extension-chrome && yarn build-extension-firefox",
@@ -137,11 +137,12 @@
     "c8": "^7.4.0",
     "chalk": "^2.4.1",
     "chrome-devtools-frontend": "1.0.922924",
+    "concurrently": "^6.4.0",
     "conventional-changelog-cli": "^2.1.1",
     "cpy": "^8.1.2",
     "cross-env": "^7.0.2",
     "csv-validator": "^0.0.3",
-    "devtools-protocol": "0.0.939404",
+    "devtools-protocol": "0.0.957544",
     "es-main": "^1.0.2",
     "eslint": "^8.4.1",
     "eslint-config-google": "^0.14.0",
@@ -205,7 +206,7 @@
     "parse-cache-control": "1.0.1",
     "ps-list": "^8.0.0",
     "raven": "^2.2.1",
-    "robots-parser": "^2.3.0",
+    "robots-parser": "^3.0.0",
     "semver": "^5.3.0",
     "speedline-core": "^1.4.3",
     "third-party-web": "^0.12.7",
@@ -215,7 +216,7 @@
     "yargs-parser": "^20.2.4"
   },
   "resolutions": {
-    "puppeteer/**/devtools-protocol": "0.0.939404"
+    "puppeteer/**/devtools-protocol": "0.0.957544"
   },
   "repository": "GoogleChrome/lighthouse",
   "keywords": [

package/report/test/generator/report-generator-test.js

@@ -97,7 +97,7 @@ describe('ReportGenerator', () => {
       expect(lines.slice(0, 3).join('\n')).toMatchInlineSnapshot(`
 "requestedUrl,finalUrl,category,name,title,type,score
 \\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"performance-score\\",\\"Overall Performance Category Score\\",\\"numeric\\",\\"0.26\\"
-\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"first-contentful-paint\\",\\"First Contentful Paint\\",\\"numeric\\",\\"0.02\\"
+\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"first-contentful-paint\\",\\"First Contentful Paint\\",\\"numeric\\",\\"0.01\\"
 "
 `);
 

package/types/artifacts.d.ts

@@ -553,12 +553,22 @@ declare module Artifacts {
   }
 
   interface InspectorIssues {
-    mixedContent: LH.Crdp.Audits.MixedContentIssueDetails[];
-    sameSiteCookies: LH.Crdp.Audits.SameSiteCookieIssueDetails[];
-    blockedByResponse: LH.Crdp.Audits.BlockedByResponseIssueDetails[];
-    heavyAds: LH.Crdp.Audits.HeavyAdIssueDetails[];
-    contentSecurityPolicy: LH.Crdp.Audits.ContentSecurityPolicyIssueDetails[];
-    deprecations: LH.Crdp.Audits.DeprecationIssueDetails[];
+    attributionReportingIssue: LH.Crdp.Audits.AttributionReportingIssueDetails[];
+    blockedByResponseIssue: LH.Crdp.Audits.BlockedByResponseIssueDetails[];
+    clientHintIssue: LH.Crdp.Audits.ClientHintIssueDetails[];
+    contentSecurityPolicyIssue: LH.Crdp.Audits.ContentSecurityPolicyIssueDetails[];
+    corsIssue: LH.Crdp.Audits.CorsIssueDetails[];
+    deprecationIssue: LH.Crdp.Audits.DeprecationIssueDetails[];
+    genericIssue: LH.Crdp.Audits.GenericIssueDetails[];
+    heavyAdIssue: LH.Crdp.Audits.HeavyAdIssueDetails[];
+    lowTextContrastIssue: LH.Crdp.Audits.LowTextContrastIssueDetails[];
+    mixedContentIssue: LH.Crdp.Audits.MixedContentIssueDetails[];
+    navigatorUserAgentIssue: LH.Crdp.Audits.NavigatorUserAgentIssueDetails[];
+    quirksModeIssue: LH.Crdp.Audits.QuirksModeIssueDetails[];
+    sameSiteCookieIssue: LH.Crdp.Audits.SameSiteCookieIssueDetails[];
+    sharedArrayBufferIssue: LH.Crdp.Audits.SharedArrayBufferIssueDetails[];
+    twaQualityEnforcement: LH.Crdp.Audits.TrustedWebActivityIssueDetails[];
+    wasmCrossOriginModuleSharingIssue: LH.Crdp.Audits.WasmCrossOriginModuleSharingIssueDetails[];
   }
 
   // Computed artifact types below.

package/types/robots-parser/index.d.ts

@@ -1,20 +0,0 @@
-/**
- * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-
-declare module 'robots-parser' {
-  interface Robots {
-    isAllowed(url: string, ua?: string): boolean | undefined;
-    isDisallowed(url: string, ua?: string): boolean | undefined;
-    getMatchingLineNumber(url: string, ua?: string): number | undefined;
-    getCrawlDelay(ua?: string): number | undefined;
-    getSitemaps(): Array<string>;
-    getPreferredHost(): string | null;
-  }
-
-  function RobotsParser(url: string, robotsTxt: string): Robots;
-
-  export = RobotsParser;
-}