lighthouse 9.4.0-dev.20220227 → 9.4.0-dev.20220228

package/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -186,6 +186,7 @@ async function begin() {
     console.log('\n✨ Be sure to have recently run this: yarn build-all');
   }
   const {runLighthouse} = await import(runnerPath);
+  runLighthouse.runnerName = argv.runner;
 
   // Find test definition file and filter by requestedTestIds.
   let testDefnPath = argv.testsPath || coreTestDefnsPath;

package/lighthouse-cli/test/smokehouse/readme.md

@@ -66,6 +66,8 @@ However, if an array literal is used as the expectation, an extra condition is e
 
 Arrays can be checked against a subset of elements using the special `_includes` property. The value of `_includes` _must_ be an array. Each assertion in `_includes` will remove the matching item from consideration for the rest.
 
+Arrays can be asserted to not match any elements using the special `_excludes` property. The value of `_excludes` _must_ be an array. If an `_includes` check is defined before an `_excludes` check, only the element not matched under the previous will be considered.
+
 **Examples**:
 | Actual | Expected | Result |
 | -- | -- | -- |
@@ -73,6 +75,9 @@ Arrays can be checked against a subset of elements using the special `_includes`
 | `[{timeInMs: 5}, {timeInMs: 15}]` | `{length: 2}` | ✅ PASS |
 | `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}]}` | ✅ PASS |
 | `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}, {timeInMs: 5}]}` | ❌ FAIL |
+| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{timeInMs: 5}]}` | ✅ PASS |
+| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{timeInMs: 15}]}` | ❌ FAIL |
+| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{}]}` | ❌ FAIL |
 | `[{timeInMs: 5}, {timeInMs: 15}]` | `[{timeInMs: 5}]` | ❌ FAIL |
 
 ### Special environment checks
@@ -104,6 +109,15 @@ If an expectation requires a minimum version of Chromium, use `_minChromiumMiles
   },
 ```
 
+All pruning checks:
+
+- `_minChromiumMilestone`
+- `_maxChromiumMilestone`
+- `_legacyOnly`
+- `_fraggleRockOnly`
+- `_skipInBundled`
+- `_runner` (set to same value provided to CLI --runner flag, ex: `'devtools'`)
+
 ## Pipeline
 
 The different frontends launch smokehouse with a set of tests to run. Smokehouse then coordinates the tests using a particular method of running Lighthouse (CLI, as a bundle, etc).

package/lighthouse-cli/test/smokehouse/report-assert.js

@@ -103,6 +103,8 @@ function findDifference(path, actual, expected) {
     };
   }
 
+  let inclExclCopy;
+
   // We only care that all expected's own properties are on actual (and not the other way around).
   // Note an expected `undefined` can match an actual that is either `undefined` or not defined.
   for (const key of Object.keys(expected)) {
@@ -112,6 +114,8 @@ function findDifference(path, actual, expected) {
     const expectedValue = expected[key];
 
     if (key === '_includes') {
+      inclExclCopy = [...actual];
+
       if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array');
       if (!Array.isArray(actual)) {
         return {
@@ -121,12 +125,12 @@ function findDifference(path, actual, expected) {
         };
       }
 
-      const actualCopy = [...actual];
       for (const expectedEntry of expectedValue) {
         const matchingIndex =
-          actualCopy.findIndex(actualEntry => !findDifference(keyPath, actualEntry, expectedEntry));
+          inclExclCopy.findIndex(actualEntry =>
+            !findDifference(keyPath, actualEntry, expectedEntry));
         if (matchingIndex !== -1) {
-          actualCopy.splice(matchingIndex, 1);
+          inclExclCopy.splice(matchingIndex, 1);
           continue;
         }
 
@@ -140,6 +144,33 @@ function findDifference(path, actual, expected) {
       continue;
     }
 
+    if (key === '_excludes') {
+      // Re-use state from `_includes` check, if there was one.
+      /** @type {any[]} */
+      const arrToCheckAgainst = inclExclCopy || actual;
+
+      if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array');
+      if (!Array.isArray(actual)) continue;
+
+      const expectedExclusions = expectedValue;
+      for (const expectedExclusion of expectedExclusions) {
+        const matchingIndex = arrToCheckAgainst.findIndex(actualEntry =>
+            !findDifference(keyPath, actualEntry, expectedExclusion));
+        if (matchingIndex !== -1) {
+          return {
+            path,
+            actual: arrToCheckAgainst[matchingIndex],
+            expected: {
+              message: 'Expected to not find matching entry via _excludes',
+              expectedExclusion,
+            },
+          };
+        }
+      }
+
+      continue;
+    }
+
     const actualValue = actual[key];
     const subDifference = findDifference(keyPath, actualValue, expectedValue);
 
@@ -187,7 +218,7 @@ function makeComparison(name, actualResult, expectedResult) {
  * @param {LocalConsole} localConsole
  * @param {LH.Result} lhr
  * @param {Smokehouse.ExpectedRunnerResult} expected
- * @param {{isBundled?: boolean}=} reportOptions
+ * @param {{runner?: string, isBundled?: boolean}=} reportOptions
  */
 function pruneExpectations(localConsole, lhr, expected, reportOptions) {
   const isFraggleRock = lhr.configSettings.channel === 'fraggle-rock-cli';
@@ -217,8 +248,20 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
    * @param {*} obj
    */
   function pruneRecursively(obj) {
-    for (const key of Object.keys(obj)) {
-      const value = obj[key];
+    /**
+     * @param {string} key
+     */
+    const remove = (key) => {
+      if (Array.isArray(obj)) {
+        obj.splice(Number(key), 1);
+      } else {
+        delete obj[key];
+      }
+    };
+
+    // Because we may be deleting keys, we should iterate the keys backwards
+    // otherwise arrays with multiple pruning checks will skip elements.
+    for (const [key, value] of Object.entries(obj).reverse()) {
       if (!value || typeof value !== 'object') {
         continue;
       }
@@ -229,42 +272,32 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
           JSON.stringify(value, null, 2),
           `Actual Chromium version: ${getChromeVersion()}`,
         ].join(' '));
-        if (Array.isArray(obj)) {
-          obj.splice(Number(key), 1);
-        } else {
-          delete obj[key];
-        }
+        remove(key);
       } else if (value._legacyOnly && isFraggleRock) {
         localConsole.log([
           `[${key}] marked legacy only but run is Fraggle Rock, pruning expectation:`,
           JSON.stringify(value, null, 2),
         ].join(' '));
-        if (Array.isArray(obj)) {
-          obj.splice(Number(key), 1);
-        } else {
-          delete obj[key];
-        }
+        remove(key);
       } else if (value._fraggleRockOnly && !isFraggleRock) {
         localConsole.log([
           `[${key}] marked Fraggle Rock only but run is legacy, pruning expectation:`,
           JSON.stringify(value, null, 2),
           `Actual channel: ${lhr.configSettings.channel}`,
         ].join(' '));
-        if (Array.isArray(obj)) {
-          obj.splice(Number(key), 1);
-        } else {
-          delete obj[key];
-        }
+        remove(key);
       } else if (value._skipInBundled && !isBundled) {
         localConsole.log([
           `[${key}] marked as skip in bundled and runner is bundled, pruning expectation:`,
           JSON.stringify(value, null, 2),
         ].join(' '));
-        if (Array.isArray(obj)) {
-          obj.splice(Number(key), 1);
-        } else {
-          delete obj[key];
-        }
+        remove(key);
+      } else if (value._runner && reportOptions?.runner !== value._runner) {
+        localConsole.log([
+          `[${key}] is only for runner ${value._runner}, pruning expectation:`,
+          JSON.stringify(value, null, 2),
+        ].join(' '));
+        remove(key);
       } else {
         pruneRecursively(value);
       }
@@ -275,6 +308,7 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
     delete obj._skipInBundled;
     delete obj._minChromiumMilestone;
     delete obj._maxChromiumMilestone;
+    delete obj._runner;
   }
 
   const cloned = cloneDeep(expected);
@@ -420,7 +454,7 @@ function reportAssertion(localConsole, assertion) {
  * summary. Returns count of passed and failed tests.
  * @param {{lhr: LH.Result, artifacts: LH.Artifacts, networkRequests?: string[]}} actual
  * @param {Smokehouse.ExpectedRunnerResult} expected
- * @param {{isDebug?: boolean, isBundled?: boolean}=} reportOptions
+ * @param {{runner?: string, isDebug?: boolean, isBundled?: boolean}=} reportOptions
  * @return {{passed: number, failed: number, log: string}}
  */
 function getAssertionReport(actual, expected, reportOptions = {}) {

package/lighthouse-cli/test/smokehouse/smokehouse.js

@@ -57,7 +57,7 @@ async function runSmokehouse(smokeTestDefns, smokehouseOptions) {
     useFraggleRock,
     jobs = DEFAULT_CONCURRENT_RUNS,
     retries = DEFAULT_RETRIES,
-    lighthouseRunner = cliLighthouseRunner,
+    lighthouseRunner = Object.assign(cliLighthouseRunner, {runnerName: 'cli'}),
     takeNetworkRequestUrls,
   } = smokehouseOptions;
   assertPositiveInteger('jobs', jobs);
@@ -159,7 +159,10 @@ async function runSmokeTest(smokeTestDefn, testOptions) {
     }
 
     // Assert result.
-    report = getAssertionReport(result, expectations, {isDebug});
+    report = getAssertionReport(result, expectations, {
+      runner: lighthouseRunner.runnerName,
+      isDebug,
+    });
 
     runs.push({
       ...result,

package/lighthouse-core/audits/dobetterweb/inspector-issues.js

@@ -71,19 +71,19 @@ class IssuesPanelEntries extends Audit {
   }
 
   /**
-   * @param {Array<LH.Crdp.Audits.SameSiteCookieIssueDetails>} sameSiteCookieIssues
+   * @param {Array<LH.Crdp.Audits.CookieIssueDetails>} CookieIssues
    * @return {LH.Audit.Details.TableItem}
    */
-  static getSameSiteCookieRow(sameSiteCookieIssues) {
+  static getCookieRow(CookieIssues) {
     const requestUrls = new Set();
-    for (const issue of sameSiteCookieIssues) {
+    for (const issue of CookieIssues) {
       const requestUrl = (issue.request?.url) || issue.cookieUrl;
       if (requestUrl) {
         requestUrls.add(requestUrl);
       }
     }
     return {
-      issueType: 'SameSite cookie',
+      issueType: 'Cookie',
       subItems: {
         type: 'subitems',
         items: Array.from(requestUrls).map(url => {
@@ -164,8 +164,8 @@ class IssuesPanelEntries extends Audit {
     if (issues.mixedContentIssue.length) {
       items.push(this.getMixedContentRow(issues.mixedContentIssue));
     }
-    if (issues.sameSiteCookieIssue.length) {
-      items.push(this.getSameSiteCookieRow(issues.sameSiteCookieIssue));
+    if (issues.cookieIssue.length) {
+      items.push(this.getCookieRow(issues.cookieIssue));
     }
     if (issues.blockedByResponseIssue.length) {
       items.push(this.getBlockedByResponseRow(issues.blockedByResponseIssue));

package/lighthouse-core/gather/gatherers/inspector-issues.js

@@ -73,7 +73,7 @@ class InspectorIssues extends FRGatherer {
       mixedContentIssue: [],
       navigatorUserAgentIssue: [],
       quirksModeIssue: [],
-      sameSiteCookieIssue: [],
+      cookieIssue: [],
       sharedArrayBufferIssue: [],
       twaQualityEnforcement: [],
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse",
-  "version": "9.4.0-dev.20220227",
+  "version": "9.4.0-dev.20220228",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./lighthouse-core/index.js",
   "bin": {
@@ -140,7 +140,7 @@
     "cpy": "^8.1.2",
     "cross-env": "^7.0.2",
     "csv-validator": "^0.0.3",
-    "devtools-protocol": "0.0.964215",
+    "devtools-protocol": "0.0.975298",
     "es-main": "^1.0.2",
     "eslint": "^8.4.1",
     "eslint-config-google": "^0.14.0",
@@ -213,7 +213,7 @@
     "yargs-parser": "^21.0.0"
   },
   "resolutions": {
-    "puppeteer/**/devtools-protocol": "0.0.964215"
+    "puppeteer/**/devtools-protocol": "0.0.975298"
   },
   "repository": "GoogleChrome/lighthouse",
   "keywords": [

package/third-party/chromium-synchronization/inspector-issueAdded-types-test.js

@@ -38,6 +38,7 @@ Array [
   "blockedByResponseIssueDetails",
   "clientHintIssueDetails",
   "contentSecurityPolicyIssueDetails",
+  "cookieIssueDetails",
   "corsIssueDetails",
   "deprecationIssueDetails",
   "federatedAuthRequestIssueDetails",
@@ -47,7 +48,6 @@ Array [
   "mixedContentIssueDetails",
   "navigatorUserAgentIssueDetails",
   "quirksModeIssueDetails",
-  "sameSiteCookieIssueDetails",
   "sharedArrayBufferIssueDetails",
   "twaQualityEnforcementDetails",
 ]

package/types/artifacts.d.ts

@@ -566,7 +566,7 @@ declare module Artifacts {
     mixedContentIssue: LH.Crdp.Audits.MixedContentIssueDetails[];
     navigatorUserAgentIssue: LH.Crdp.Audits.NavigatorUserAgentIssueDetails[];
     quirksModeIssue: LH.Crdp.Audits.QuirksModeIssueDetails[];
-    sameSiteCookieIssue: LH.Crdp.Audits.SameSiteCookieIssueDetails[];
+    cookieIssue: LH.Crdp.Audits.CookieIssueDetails[];
     sharedArrayBufferIssue: LH.Crdp.Audits.SharedArrayBufferIssueDetails[];
     twaQualityEnforcement: LH.Crdp.Audits.TrustedWebActivityIssueDetails[];
   }

package/types/smokehouse.d.ts

@@ -52,7 +52,7 @@ declare global {
       {expectations: Smokehouse.ExpectedRunnerResult | Array<Smokehouse.ExpectedRunnerResult>}
 
     export type LighthouseRunner =
-      (url: string, configJson?: Config.Json, runnerOptions?: {isDebug?: boolean; useFraggleRock?: boolean}) => Promise<{lhr: LHResult, artifacts: Artifacts, log: string}>;
+      {runnerName?: string} & ((url: string, configJson?: Config.Json, runnerOptions?: {isDebug?: boolean; useFraggleRock?: boolean}) => Promise<{lhr: LHResult, artifacts: Artifacts, log: string}>);
 
     export interface SmokehouseOptions {
       /** If true, performs extra logging from the test runs. */