lighthouse 9.6.8 → 10.0.0

package/lighthouse-cli/test/smokehouse/lighthouse-runners/cli.js

@@ -1,151 +0,0 @@
-/**
- * @license Copyright 2019 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-/**
- * @fileoverview  A runner that executes Lighthouse via the Lighthouse CLI to
- * test the full pipeline, from parsing arguments on the command line to writing
- * results to disk. When complete, reads back the artifacts and LHR and returns
- * them.
- */
-
-import {promises as fs} from 'fs';
-import {promisify} from 'util';
-import {execFile} from 'child_process';
-
-import log from 'lighthouse-logger';
-
-import assetSaver from '../../../../lighthouse-core/lib/asset-saver.js';
-import {LocalConsole} from '../lib/local-console.js';
-import {ChildProcessError} from '../lib/child-process-error.js';
-import {LH_ROOT} from '../../../../root.js';
-
-const execFileAsync = promisify(execFile);
-
-/**
- * Launch Chrome and do a full Lighthouse run via the Lighthouse CLI.
- * @param {string} url
- * @param {LH.Config.Json=} configJson
- * @param {{isDebug?: boolean, useFraggleRock?: boolean}=} testRunnerOptions
- * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
- */
-async function runLighthouse(url, configJson, testRunnerOptions = {}) {
-  const {isDebug} = testRunnerOptions;
-  const tmpDir = `${LH_ROOT}/.tmp/smokehouse`;
-  await fs.mkdir(tmpDir, {recursive: true});
-  const tmpPath = await fs.mkdtemp(`${tmpDir}/smokehouse-`);
-  return internalRun(url, tmpPath, configJson, testRunnerOptions)
-    // Wait for internalRun() before removing scratch directory.
-    .finally(() => !isDebug && fs.rm(tmpPath, {recursive: true, force: true}));
-}
-
-/**
- * @param {LH.Config.Json=} configJson
- * @return {LH.Config.Json|undefined}
- */
-function convertToFraggleRockConfig(configJson) {
-  if (!configJson) return configJson;
-  if (!configJson.passes) return configJson;
-
-  return {
-    ...configJson,
-    navigations: configJson.passes.map(pass => ({...pass, id: pass.passName})),
-  };
-}
-
-/**
- * Internal runner.
- * @param {string} url
- * @param {string} tmpPath
- * @param {LH.Config.Json=} configJson
- * @param {{isDebug?: boolean, useFraggleRock?: boolean}=} options
- * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
- */
-async function internalRun(url, tmpPath, configJson, options) {
-  const {isDebug = false, useFraggleRock = false} = options || {};
-  const localConsole = new LocalConsole();
-
-  const outputPath = `${tmpPath}/smokehouse.report.json`;
-  const artifactsDirectory = `${tmpPath}/artifacts/`;
-
-  const args = [
-    `${LH_ROOT}/lighthouse-cli/index.js`,
-    `${url}`,
-    `--output-path=${outputPath}`,
-    '--output=json',
-    `-G=${artifactsDirectory}`,
-    `-A=${artifactsDirectory}`,
-    '--quiet',
-    '--port=0',
-    '--quiet',
-  ];
-
-  if (useFraggleRock) {
-    args.push('--fraggle-rock');
-    configJson = convertToFraggleRockConfig(configJson);
-  }
-
-  // Config can be optionally provided.
-  if (configJson) {
-    const configPath = `${tmpPath}/config.json`;
-    await fs.writeFile(configPath, JSON.stringify(configJson));
-    args.push(`--config-path=${configPath}`);
-  }
-
-  const command = 'node';
-  const env = {...process.env, NODE_ENV: 'test'};
-  localConsole.log(`${log.dim}$ ${command} ${args.join(' ')} ${log.reset}`);
-
-  /** @type {{stdout: string, stderr: string, code?: number}} */
-  let execResult;
-  try {
-    execResult = await execFileAsync(command, args, {env});
-  } catch (e) {
-    // exec-thrown errors have stdout, stderr, and exit code from child process.
-    execResult = e;
-  }
-
-  const exitCode = execResult.code || 0;
-  if (isDebug) {
-    localConsole.log(`exit code ${exitCode}`);
-    localConsole.log(`STDOUT: ${execResult.stdout}`);
-    localConsole.log(`STDERR: ${execResult.stderr}`);
-  }
-
-  try {
-    await fs.access(outputPath);
-  } catch (e) {
-    throw new ChildProcessError(`Lighthouse run failed to produce a report and exited with ${exitCode}.`, // eslint-disable-line max-len
-        localConsole.getLog());
-  }
-
-  /** @type {LH.Result} */
-  const lhr = JSON.parse(await fs.readFile(outputPath, 'utf8'));
-  const artifacts = assetSaver.loadArtifacts(artifactsDirectory);
-
-  // Output has been established as existing, so can log for debug.
-  if (isDebug) {
-    localConsole.log(`LHR output available at: ${outputPath}`);
-    localConsole.log(`Artifacts avaiable in: ${artifactsDirectory}`);
-  }
-
-  // There should either be both an error exitCode and a lhr.runtimeError or neither.
-  if (Boolean(exitCode) !== Boolean(lhr.runtimeError)) {
-    const runtimeErrorCode = lhr.runtimeError?.code;
-    throw new ChildProcessError(`Lighthouse did not exit with an error correctly, exiting with ${exitCode} but with runtimeError '${runtimeErrorCode}'`, // eslint-disable-line max-len
-        localConsole.getLog());
-  }
-
-  return {
-    lhr,
-    artifacts,
-    log: localConsole.getLog(),
-  };
-}
-
-export {
-  runLighthouse,
-};

package/lighthouse-cli/test/smokehouse/lighthouse-runners/devtools.js

@@ -1,106 +0,0 @@
-/**
- * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-/**
- * @fileoverview A runner that launches Chrome and executes Lighthouse via DevTools.
- */
-
-import fs from 'fs';
-import os from 'os';
-import {spawn} from 'child_process';
-
-import {LH_ROOT} from '../../../../root.js';
-
-const devtoolsDir =
-  process.env.DEVTOOLS_PATH || `${LH_ROOT}/.tmp/chromium-web-tests/devtools/devtools-frontend`;
-
-/**
- * @param {string} command
- * @param {string[]} args
- */
-async function spawnAndLog(command, args) {
-  let log = '';
-
-  /** @type {Promise<void>} */
-  const promise = new Promise((resolve, reject) => {
-    const spawnHandle = spawn(command, args);
-    spawnHandle.on('close', code => {
-      if (code === 0) resolve();
-      else reject(new Error(`Command exited with code ${code}`));
-    });
-    spawnHandle.on('error', reject);
-    spawnHandle.stdout.on('data', data => {
-      console.log(data.toString());
-      log += `STDOUT: ${data.toString()}`;
-    });
-    spawnHandle.stderr.on('data', data => {
-      console.log(data.toString());
-      log += `STDERR: ${data.toString()}`;
-    });
-  });
-  await promise;
-
-  return log;
-}
-
-/** @type {Promise<void>} */
-let buildDevtoolsPromise;
-/**
- * Download/pull latest DevTools, build Lighthouse for DevTools, roll to DevTools, and build DevTools.
- */
-async function buildDevtools() {
-  if (process.env.CI) return;
-
-  process.env.DEVTOOLS_PATH = devtoolsDir;
-  await spawnAndLog('bash', ['lighthouse-core/test/chromium-web-tests/download-devtools.sh']);
-  await spawnAndLog('bash', ['lighthouse-core/test/chromium-web-tests/roll-devtools.sh']);
-}
-
-/**
- * Launch Chrome and do a full Lighthouse run via DevTools.
- * By default, the latest DevTools frontend is used (.tmp/chromium-web-tests/devtools/devtools-frontend)
- * unless DEVTOOLS_PATH is set.
- * CHROME_PATH determines which Chrome is used–otherwise the default is puppeteer's chrome binary.
- * @param {string} url
- * @param {LH.Config.Json=} configJson
- * @param {{isDebug?: boolean}=} testRunnerOptions
- * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
- */
-async function runLighthouse(url, configJson, testRunnerOptions = {}) {
-  if (!buildDevtoolsPromise) buildDevtoolsPromise = buildDevtools();
-  await buildDevtoolsPromise;
-
-  const outputDir = fs.mkdtempSync(os.tmpdir() + '/lh-smoke-cdt-runner-');
-  const chromeFlags = [
-    `--custom-devtools-frontend=file://${devtoolsDir}/out/Default/gen/front_end`,
-  ];
-  const args = [
-    'run-devtools',
-    url,
-    `--chrome-flags=${chromeFlags.join(' ')}`,
-    '--output-dir', outputDir,
-  ];
-  if (configJson) {
-    args.push('--config', JSON.stringify(configJson));
-  }
-
-  const log = await spawnAndLog('yarn', args);
-  const lhr = JSON.parse(fs.readFileSync(`${outputDir}/lhr-0.json`, 'utf-8'));
-  const artifacts = JSON.parse(fs.readFileSync(`${outputDir}/artifacts-0.json`, 'utf-8'));
-
-  if (testRunnerOptions.isDebug) {
-    console.log(`${url} results saved at ${outputDir}`);
-  } else {
-    fs.rmSync(outputDir, {recursive: true, force: true});
-  }
-
-  return {lhr, artifacts, log};
-}
-
-export {
-  runLighthouse,
-};

package/lighthouse-cli/test/smokehouse/readme.md

@@ -1,204 +0,0 @@
-# Smokehouse
-
-Smokehouse is the Lighthouse end-to-end/smoke test runner. It takes in a set of URLs (usually pointing to custom-built test sites), runs Lighthouse on them, and compares the results against a set of expectations.
-
-By default this is done using the Lighthouse CLI (to exercise the full pipeline) with the tests listed in [`smokehouse/core-tests.js`](./core-tests.js).
-
-## Options
-
-See [`SmokehouseOptions`](https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-cli/test/smokehouse/smokehouse.js#L23).
-
-## Test definitions
-
-| Name           | Type                               | Description                                                                                                                      |
-| -------------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
-| `id`           | `string`                           | The string identifier of the test.                                                                                               |
-| `expectations` | `{lhr: Object, artifacts: Object}` | See below.                                                                                                                       |
-| `config`       | `LH.Config.Json` (optional)        | An optional Lighthouse config. If not specified, the default config is used.                                                     |
-| `runSerially`  | `boolean` (optional)               | An optional flag. If set to true, the test won't be run in parallel to other tests. Useful if the test is performance sensitive. |
-
-### Expectations
-
-The smoke test expectations can assert the values of the Lighthouse result (the `lhr`) and gathered `artifacts` for multiple URLs. The URL to be tested is specified in the expectations's `requestedUrl` field.
-
-The expectations are asserted as a subset of the actual results: all values in the expectations must be in the actual results, but not all actual results must be asserted.
-
-Examples can be found in the [core tests](./test-definitions/).
-
-### Special numeric expectations
-
-If checking a number somewhere in the Lighthouse results, numeric comparisons can be used in place of a raw expected number. This allows asserting ranges or categories of numbers where the exact value isn't necessarily important, or to allow for expected variability in a test.
-
-The comparator is specified with a string, and the actual value being tested must be a number. Whitespace may be included in the string for readability.
-
-The following operators are supported:
-
-| Operator | Example      |
-| -------- | ------------ |
-| `>`      | `'>0'`       |
-| `>=`     | `'>=5'`      |
-| `<`      | `'<1'`       |
-| `<=`     | `'<=10'`     |
-| `+/-`    | `'100+/-10'` |
-| `±`      | `'100±10'`   |
-
-**Examples**:
-| Actual | Expected | Result |
-| -- | -- | -- |
-| `{timeInMs: 50}` | `{timeInMs: '>0'}` | ✅ PASS |
-| `{numericValue: 3969.135}` | `{numericValue: '1000±100'}` | ❌ FAIL |
-
-### Special string expectations
-
-If checking a string somewhere in the Lighthouse results, a regular expression can be used in place of a string literal.
-
-**Examples**:
-| Actual | Expected | Result |
-| -- | -- | -- |
-| `{displayValue: '4.0 s'}` | `{displayValue: /^\d+\.\d+/}` | ✅ PASS |
-| `{url: 'http://example.com'}` | `{url: /^https/}` | ❌ FAIL |
-
-### Special array expectations
-
-Individual elements of an array can be asserted by using numeric properties in an object, e.g. asserting the third element in an array is 5: `{2: 5}`.
-
-However, if an array literal is used as the expectation, an extra condition is enforced that the actual array _must_ have the same length as the provided expected array.
-
-Arrays can be checked against a subset of elements using the special `_includes` property. The value of `_includes` _must_ be an array. Each assertion in `_includes` will remove the matching item from consideration for the rest.
-
-Arrays can be asserted to not match any elements using the special `_excludes` property. The value of `_excludes` _must_ be an array. If an `_includes` check is defined before an `_excludes` check, only the element not matched under the previous will be considered.
-
-**Examples**:
-| Actual | Expected | Result |
-| -- | -- | -- |
-| `[{url: 'http://badssl.com'}, {url: 'http://example.com'}]` | `{1: {url: 'http://example.com'}}` | ✅ PASS |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `{length: 2}` | ✅ PASS |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}]}` | ✅ PASS |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}, {timeInMs: 5}]}` | ❌ FAIL |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{timeInMs: 5}]}` | ✅ PASS |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{timeInMs: 15}]}` | ❌ FAIL |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{}]}` | ❌ FAIL |
-| `[{timeInMs: 5}, {timeInMs: 15}]` | `[{timeInMs: 5}]` | ❌ FAIL |
-
-### Special environment checks
-
-If an expectation requires a minimum version of Chromium, use `_minChromiumVersion: xx.x.x.x` to conditionally ignore that entire object in the expectation.
-Can be as specific as you like (`_minChromiumVersion: xx` works too).
-
-**Examples**:
-```js
-{
-  artifacts: {
-    InspectorIssues: {
-      // Mixed Content issues weren't added to the protocol until M84.
-      _minChromiumVersion: '84', // The entire `InspectorIssues` is ignored for older Chrome.
-      mixedContent: [
-        {
-          resourceType: 'Image',
-          resolutionStatus: 'MixedContentWarning',
-          insecureURL: 'http://www.mixedcontentexamples.com/Content/Test/steveholt.jpg',
-          mainResourceURL: 'https://www.mixedcontentexamples.com/Test/NonSecureImage',
-          request: {
-            url: 'http://www.mixedcontentexamples.com/Content/Test/steveholt.jpg',
-          },
-        },
-      ],
-    },
-    TraceElements: {
-      // ... anything here won't be ignored
-    }
-  },
-```
-
-All pruning checks:
-
-- `_minChromiumVersion`
-- `_maxChromiumVersion`
-- `_legacyOnly`
-- `_fraggleRockOnly`
-- `_skipInBundled`
-- `_runner` (set to same value provided to CLI --runner flag, ex: `'devtools'`)
-
-## Pipeline
-
-The different frontends launch smokehouse with a set of tests to run. Smokehouse then coordinates the tests using a particular method of running Lighthouse (CLI, as a bundle, etc).
-
-```
-Smokehouse Frontends                                        Lighthouse Runners
-+------------+
-|            |
-|   bin.js   +----+                                           +--------------+
-|            |    |                                           |              |
-+------------+    |                                       +-->+    cli.js    |
-                  |                                       |   |              |
-+------------+    |            +---------------+          |   +--------------+
-|            |    | testDefns> |               |  config> |
-|   node.js  +---------------->+ smokehouse.js +<---------+
-|            |    |            |               |   <lhr   |   +--------------+
-+------------+    |            +-------+-------+          |   |              |
-                  |                    ^                  +-->+   bundle.js  |
-+------------+    |                    |                  |   |              |
-|            |    |                    |                  |   +--------------+
-|   lib.js   +----+                    v                  |
-|            |                +--------+--------+         |
-+------------+                |                 |         |   +--------------+
-                              |  report/assert  |         |   |              |
-                              |                 |         +-->+  devtools.js |
-                              +-----------------+             |              |
-                                                              +--------------+
-```
-
-### Smokehouse frontends
-
-- `frontends/smokehouse-bin.js` - runs smokehouse from the command line
-- `lib` - configurable entrypoint to smokehouse, can be bundled to run in a browser environment
-- `node.js` - run smokehouse from a node process
-
-### Smokehouse
-
-- `smokehouse.js` - takes a set of smoke-test definitions and runs them via a passed-in runner. Smokehouse is bundleable and can run in a browser as long as runner used is bundleable as well.
-
-### Lighthouse runners
-
-- `lighthouse-runners/cli.js` - the original test runner, exercising the Lighthouse CLI from command-line argument parsing to the results written to disk on completion.
-- `lighthouse-runners/bundle.js` - a smoke test runner that operates on an already-bundled version of Lighthouse for end-to-end testing of that version.
-- `lighthouse-runners/devtools.js` - a smoke test runner that operates on Lighthouse running from inside DevTools.
-
-## Custom smoke tests (for plugins et al.)
-
-Smokehouse comes with a core set of test definitions, but it can run any set of tests. Custom extensions of Lighthouse (like plugins) can provide their own tests and run them via the same infrastructure. For example:
-
-- have a test site on a public URL or via a local server (e.g. `https://localhost:8080`)
-- create a test definition (e.g. in `plugin-tests.js`)
-  ```js
-  const smokeTests = [{
-    id: 'pluginTest',
-    expectations: require('./expectations.js'),
-    // config: ..., // If left out, uses default LH config
-    // runSerially: true, // If test is perf-sensitive
-  };
-  module.exports = smokeTests;
-  ```
-- create a test expectations file (e.g. `expectations.js`)
-  ```js
-  const expectations = [{
-    lhr: {
-      requestedUrl: 'http://localhost:8080/index.html',
-      finalUrl: 'http://localhost:8080/index.html',
-      audits: {
-        'preload-as': {
-          score: 1,
-          displayValue: /^Found 0 preload requests/,
-        },
-      },
-    },
-  };
-  module.exports = expectations;
-  ```
-- with `lighthouse` installed as a dependency/peer dependency, run
-
-  `yarn smokehouse --tests-path plugin-tests.js`
-
-  or
-
-  `npx --no-install smokehouse --tests-path plugin-tests.js`