lighthouse 13.5.0 → 13.6.0-dev.20261027

package/cli/run.js

@@ -6,6 +6,7 @@
 
 /* eslint-disable no-console */
 
+import fs from 'fs';
 import path from 'path';
 import os from 'os';
 
@@ -131,7 +132,7 @@ function printErrorAndExit(err) {
 /**
  * @param {LH.RunnerResult} runnerResult
  * @param {LH.CliFlags} flags
- * @return {Promise<void>}
+ * @return {Promise<string|undefined>}
  */
 async function saveResults(runnerResult, flags) {
   const cwd = process.cwd();
@@ -145,43 +146,55 @@ async function saveResults(runnerResult, flags) {
   if (!shouldSaveResults) return;
   const {lhr, artifacts, report} = runnerResult;
 
+  const isDirectoryOutput = Boolean(
+    flags.outputPath &&
+    fs.existsSync(flags.outputPath) &&
+    fs.statSync(flags.outputPath).isDirectory()
+  );
+
   // Use the output path as the prefix for all generated files.
   // If no output path is set, generate a file prefix using the URL and date.
-  const configuredPath = !flags.outputPath || flags.outputPath === 'stdout' ?
-      getLhrFilenamePrefix(lhr) :
-      flags.outputPath.replace(/\.\w{2,4}$/, '');
+  let configuredPath;
+  if (!flags.outputPath || flags.outputPath === 'stdout') {
+    configuredPath = getLhrFilenamePrefix(lhr);
+  } else if (isDirectoryOutput) {
+    configuredPath = path.join(flags.outputPath, getLhrFilenamePrefix(lhr));
+  } else {
+    configuredPath = flags.outputPath.replace(/\.\w{2,4}$/, '');
+  }
   const resolvedPath = path.resolve(cwd, configuredPath);
 
   if (flags.saveAssets) {
     await assetSaver.saveAssets(artifacts, lhr.audits, resolvedPath);
   }
 
+  /** @type {string|undefined} */
+  let htmlOutputPath;
   for (const outputType of flags.output) {
     const extension = outputType;
     const output = report[flags.output.indexOf(outputType)];
     let outputPath = `${resolvedPath}.report.${extension}`;
-    // If there was only a single output and the user specified an outputPath, force usage of it.
-    if (flags.outputPath && flags.output.length === 1) outputPath = flags.outputPath;
+    // If there was only a single output and the user specified a file outputPath, force usage of it.
+    if (flags.outputPath && !isDirectoryOutput && flags.output.length === 1) {
+      outputPath = flags.outputPath;
+    }
     await Printer.write(output, outputType, outputPath);
 
     if (outputType === Printer.OutputMode[Printer.OutputMode.html]) {
-      if (flags.view) {
-        open(outputPath, {wait: false});
-      } else {
-        // eslint-disable-next-line max-len
-        log.log('CLI', 'Protip: Run lighthouse with `--view` to immediately open the HTML report in your browser');
-      }
+      htmlOutputPath = outputPath;
     }
   }
+
+  return htmlOutputPath;
 }
 
 /**
- * @param {string} url
+ * @param {string|string[]|undefined} urlOrUrls
  * @param {LH.CliFlags} flags
  * @param {LH.Config|undefined} config
- * @return {Promise<LH.RunnerResult|undefined>}
+ * @return {Promise<LH.RunnerResult|LH.RunnerResult[]|undefined>}
  */
-async function runLighthouse(url, flags, config) {
+async function runLighthouse(urlOrUrls, flags, config) {
   /** @param {any} reason */
   function handleTheUnhandled(reason) {
     process.stderr.write(`Unhandled Rejection. Reason: ${reason}\n`);
@@ -193,48 +206,85 @@ async function runLighthouse(url, flags, config) {
   /** @type {ChromeLauncher.LaunchedChrome|undefined} */
   let launchedChrome;
 
-  try {
-    if (url && flags.auditMode && !flags.gatherMode) {
-      log.warn('CLI', 'URL parameter is ignored if -A flag is used without -G flag');
-    }
+  const urls = [urlOrUrls].flat();
+  if (urls.length === 0) urls.push(undefined);
+  const isBatch = urls.length > 1;
 
-    const shouldGather = flags.gatherMode || flags.gatherMode === flags.auditMode;
-    const shouldUseLocalChrome = UrlUtils.isLikeLocalhost(flags.hostname);
-    if (shouldGather && shouldUseLocalChrome) {
-      launchedChrome = await getDebuggableChrome(flags);
-      flags.port = launchedChrome.port;
-    }
+  const shouldGather = flags.gatherMode || flags.gatherMode === flags.auditMode;
+  const shouldUseLocalChrome = UrlUtils.isLikeLocalhost(flags.hostname);
+  const initialPort = flags.port;
+  flags.channel = 'cli';
 
-    flags.channel = 'cli';
+  /** @type {Array<LH.RunnerResult>} */
+  const results = [];
+  /** @type {Array<string>} */
+  const htmlReportPaths = [];
+  /** @type {Array<ExitError>} */
+  const errors = [];
 
-    const runnerResult = await lighthouse(url, flags, config);
+  try {
+    for (const url of urls) {
+      if (url && flags.auditMode && !flags.gatherMode) {
+        log.warn('CLI', 'URL parameter is ignored if -A flag is used without -G flag');
+      }
 
-    // If in gatherMode only, there will be no runnerResult.
-    if (runnerResult) {
-      await saveResults(runnerResult, flags);
+      try {
+        if (shouldGather && shouldUseLocalChrome && !launchedChrome) {
+          launchedChrome = await getDebuggableChrome({...flags, port: initialPort});
+          flags.port = launchedChrome.port;
+        }
+
+        const runnerResult = await lighthouse(url, flags, config);
+
+        // If in gatherMode only, there will be no runnerResult.
+        if (runnerResult) {
+          const htmlPath = await saveResults(runnerResult, flags);
+          if (htmlPath) htmlReportPaths.push(htmlPath);
+          results.push(runnerResult);
+        }
+
+        // Runtime errors indicate something was *very* wrong with the page result.
+        // We don't want the user to have to parse the report to figure it out, so we'll still exit
+        // with an error code after we saved the results.
+        if (runnerResult?.lhr.runtimeError) {
+          const {runtimeError} = runnerResult.lhr;
+          throw Object.assign(new Error(runtimeError.message), {
+            name: 'LighthouseError',
+            friendlyMessage: runtimeError.message,
+            code: runtimeError.code,
+            stack: undefined,
+          });
+        }
+      } catch (err) {
+        errors.push(err);
+        await launchedChrome?.kill();
+        launchedChrome = undefined;
+        if (!isBatch || flags.failFast) break;
+        const errMsg = err.friendlyMessage || err.message;
+        log.error('CLI', `Runtime error encountered for ${url}: ${errMsg}`);
+      }
     }
-
-    launchedChrome?.kill();
+  } finally {
+    await launchedChrome?.kill();
     process.removeListener('unhandledRejection', handleTheUnhandled);
+  }
 
-    // Runtime errors indicate something was *very* wrong with the page result.
-    // We don't want the user to have to parse the report to figure it out, so we'll still exit
-    // with an error code after we saved the results.
-    if (runnerResult?.lhr.runtimeError) {
-      const {runtimeError} = runnerResult.lhr;
-      return printErrorAndExit({
-        name: 'LighthouseError',
-        friendlyMessage: runtimeError.message,
-        code: runtimeError.code,
-        message: runtimeError.message,
-      });
+  if (htmlReportPaths.length > 0) {
+    if (flags.view) {
+      for (const htmlPath of htmlReportPaths) {
+        open(htmlPath, {wait: false});
+      }
+    } else {
+      // eslint-disable-next-line max-len
+      log.log('CLI', 'Protip: Run lighthouse with `--view` to immediately open the HTML report in your browser');
     }
+  }
 
-    return runnerResult;
-  } catch (err) {
-    launchedChrome?.kill();
-    return printErrorAndExit(err);
+  if (errors.length > 0) {
+    return printErrorAndExit(errors[0]);
   }
+
+  return isBatch ? results : results[0];
 }
 
 export {

package/cli/sentry-prompt.js

@@ -60,16 +60,15 @@ function prompt() {
 function askPermission() {
   return Promise.resolve().then(_ => {
     const configstore = new Configstore('lighthouse');
-    let isErrorReportingEnabled = configstore.get('isErrorReportingEnabled');
+    const isErrorReportingEnabled = configstore.get('isErrorReportingEnabled');
     if (typeof isErrorReportingEnabled === 'boolean') {
       return Promise.resolve(isErrorReportingEnabled);
     }
 
     return prompt()
       .then(response => {
-        isErrorReportingEnabled = response;
-        configstore.set('isErrorReportingEnabled', isErrorReportingEnabled);
-        return isErrorReportingEnabled;
+        configstore.set('isErrorReportingEnabled', response);
+        return response;
       });
   // Error accessing configstore; default to false.
   }).catch(_ => false);

package/cli/test/smokehouse/__snapshots__/report-assert-test.js.snap

@@ -1,4 +1,4 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
+// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
 
 exports[`getAssertionReport works (multiple failing) 1`] = `
 "X difference at cumulative-layout-shift audit.details.items.length

package/cli/test/smokehouse/config/exclusions.js

@@ -23,7 +23,7 @@ const exclusions = {
     'redirects-http',
     // Disabled because these tests use settings that cannot be fully configured in
     // DevTools (e.g. throttling method "provided").
-    'metrics-tricky-tti', 'metrics-tricky-tti-late-fcp', 'screenshot',
+    'metrics-delayed-lcp', 'metrics-tricky-tti', 'metrics-tricky-tti-late-fcp', 'screenshot',
     // Disabled because of differences that need further investigation
     'byte-efficiency', 'byte-gzip', 'perf-preload',
     // Disabled because a renderer crash also breaks devtools frontend
@@ -78,6 +78,8 @@ const exclusions = {
 };
 
 for (const array of Object.values(exclusions)) {
+  // https://github.com/GoogleChrome/lighthouse/issues/17277
+  array.push('webmcp');
   // https://github.com/GoogleChrome/lighthouse/issues/14271
   array.push('lantern-idle-callback-short');
   // glitch is gone.

package/cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -216,7 +216,9 @@ async function begin() {
     if (testDefnPath === coreTestDefnsPath) {
       const {createServers} = await import('../../fixtures/static-server.js');
       servers = await createServers();
-      takeNetworkRequestUrls = servers[0].takeRequestUrls.bind(servers[0]);
+      if (servers.length) {
+        takeNetworkRequestUrls = servers[0].takeRequestUrls.bind(servers[0]);
+      }
     }
 
     const prunedTestDefns = pruneExpectedNetworkRequests(testDefns, takeNetworkRequestUrls);
@@ -234,7 +236,7 @@ async function begin() {
 
     smokehouseResult = (await runSmokehouse(prunedTestDefns, options));
   } finally {
-    servers?.forEach(s => s.close());
+    if (servers) await Promise.all(servers.map(s => s.close()));
   }
 
   let smokehouseOutputDir;
@@ -244,7 +246,7 @@ async function begin() {
     smokehouseOutputDir = `${LH_ROOT}/.tmp/smokehouse-failures`;
     testResultsToOutput = smokehouseResult.testResults.filter(r => r.failed);
   } else if (!process.env.CI) {
-    // Otherwise, only write to disk in debug mode.
+    // Otherwise, write all results to disk when running locally.
     smokehouseOutputDir = `${LH_ROOT}/.tmp/smokehouse-output`;
     testResultsToOutput = smokehouseResult.testResults;
   }

package/cli/test/smokehouse/report-assert-test.js

@@ -73,6 +73,31 @@ describe('findDiffersences', () => {
       expected: {duration: '<100'},
       diffs: [{path: '.duration', actual: 100, expected: '<100'}],
     },
+    'range (negative 1)': {
+      actual: {duration: -3},
+      expected: {duration: '>-5'},
+      diffs: null,
+    },
+    'range (negative 2)': {
+      actual: {duration: -9},
+      expected: {duration: '-10 +/- 2'},
+      diffs: null,
+    },
+    'range (invalid numerical expectation without prefix)': {
+      actual: {duration: 3},
+      expected: {duration: '+/- 5'},
+      diffs: [{path: '.duration', actual: 3, expected: '+/- 5'}],
+    },
+    'range (invalid numerical expectation with prefix)': {
+      actual: {duration: 15},
+      expected: {duration: '10 < 20'},
+      diffs: [{path: '.duration', actual: 15, expected: '10 < 20'}],
+    },
+    'range (numeric actual vs array expected)': {
+      actual: {duration: 10},
+      expected: {duration: ['>0']},
+      diffs: [{path: '.duration', actual: 10, expected: ['>0']}],
+    },
 
     'array (1)': {
       actual: {prices: [0, 1, 2, 3, 4, 5]},
@@ -112,6 +137,11 @@ describe('findDiffersences', () => {
       expected: {prices: {'3': {nested: '>3'}}},
       diffs: [{path: '.prices[3].nested', actual: 3, expected: '>3'}],
     },
+    'array (8)': {
+      actual: {prices: {0: 1, length: 1}},
+      expected: {prices: [1]},
+      diffs: [{path: '.prices.length', actual: {0: 1, length: 1}, expected: [1]}],
+    },
 
     '_includes (1)': {
       actual: {prices: [0, 1, 2, 3, 4, 5]},
@@ -295,4 +325,39 @@ describe('getAssertionReport', () => {
     expect(report).toMatchObject({passed: 3, failed: 1});
     expect(clean(report.log)).toMatchSnapshot();
   });
+
+  it('asserts expected.lhr.userAgent', () => {
+    const passingReport = getAssertionReport({lhr, artifacts}, {
+      lhr: {
+        audits: {},
+        requestedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
+        finalDisplayedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
+        userAgent: /Chrome\//,
+      },
+    });
+    expect(passingReport).toMatchObject({passed: 4, failed: 0, log: ''});
+
+    const failingReport = getAssertionReport({lhr, artifacts}, {
+      lhr: {
+        audits: {},
+        requestedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
+        finalDisplayedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
+        userAgent: 'NotTheUserAgent',
+      },
+    });
+    expect(failingReport).toMatchObject({passed: 3, failed: 1});
+  });
+
+  it('lazily evaluates hostUserAgent only when chromium version check is used', () => {
+    const customLhr = structuredClone(lhr);
+    customLhr.environment.hostUserAgent = 'InvalidUserAgentWithoutChromeVersion';
+    const report = getAssertionReport({lhr: customLhr, artifacts}, {
+      lhr: {
+        audits: {},
+        requestedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
+        finalDisplayedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
+      },
+    });
+    expect(report).toMatchObject({passed: 3, failed: 0, log: ''});
+  });
 });

package/cli/test/smokehouse/report-assert.js

@@ -30,7 +30,7 @@ import {chromiumVersionCheck} from './version-check.js';
  * @property {Difference[]|null} diffs
  */
 
-const NUMBER_REGEXP = /(?:\d|\.)+/.source;
+const NUMBER_REGEXP = /-?(?:\d|\.)+/.source;
 const OPS_REGEXP = /<=?|>=?|\+\/-|±/.source;
 // An optional number, optional whitespace, an operator, optional whitespace, a number.
 const NUMERICAL_EXPECTATION_REGEXP =
@@ -48,21 +48,27 @@ const NUMERICAL_EXPECTATION_REGEXP =
  * @return {boolean}
  */
 function matchesExpectation(actual, expected) {
-  if (typeof actual === 'number' && NUMERICAL_EXPECTATION_REGEXP.test(expected)) {
+  if (
+    typeof actual === 'number' &&
+    typeof expected === 'string' &&
+    NUMERICAL_EXPECTATION_REGEXP.test(expected)
+  ) {
     const parts = expected.match(NUMERICAL_EXPECTATION_REGEXP);
+    if (!parts) return false;
     const [, prefixNumber, operator, postfixNumber] = parts;
     switch (operator) {
       case '>':
-        return actual > postfixNumber;
+        return prefixNumber === undefined && actual > Number(postfixNumber);
       case '>=':
-        return actual >= postfixNumber;
+        return prefixNumber === undefined && actual >= Number(postfixNumber);
       case '<':
-        return actual < postfixNumber;
+        return prefixNumber === undefined && actual < Number(postfixNumber);
       case '<=':
-        return actual <= postfixNumber;
+        return prefixNumber === undefined && actual <= Number(postfixNumber);
       case '+/-':
       case '±':
-        return Math.abs(actual - prefixNumber) <= postfixNumber;
+        return prefixNumber !== undefined &&
+          Math.abs(actual - Number(prefixNumber)) <= Number(postfixNumber);
       default:
         throw new Error(`unexpected operator ${operator}`);
     }
@@ -117,21 +123,8 @@ function findDifferences(path, actual, expected) {
     const expectedValue = expected[key];
 
     if (key === '_includes') {
-      if (Array.isArray(actual)) {
-        inclExclCopy = [...actual];
-      } else if (typeof actual === 'object') {
-        inclExclCopy = Object.entries(actual);
-      }
-
       if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array');
-      if (!inclExclCopy) {
-        diffs.push({
-          path,
-          actual: 'Actual value is not an array or object',
-          expected,
-        });
-        continue;
-      }
+      inclExclCopy = Array.isArray(actual) ? [...actual] : Object.entries(actual);
 
       for (const expectedEntry of expectedValue) {
         const matchingIndex =
@@ -153,24 +146,11 @@ function findDifferences(path, actual, expected) {
     }
 
     if (key === '_excludes') {
-      // Re-use state from `_includes` check, if there was one.
-      if (!inclExclCopy) {
-        if (Array.isArray(actual)) {
-          // We won't be removing items, so we can just copy the reference.
-          inclExclCopy = actual;
-        } else if (typeof actual === 'object') {
-          inclExclCopy = Object.entries(actual);
-        }
-      }
-
       if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array');
+      // Re-use state from `_includes` check, if there was one.
       if (!inclExclCopy) {
-        diffs.push({
-          path,
-          actual: 'Actual value is not an array or object',
-          expected,
-        });
-        continue;
+        // We won't be removing items, so we can just copy the reference.
+        inclExclCopy = Array.isArray(actual) ? actual : Object.entries(actual);
       }
 
       const expectedExclusions = expectedValue;
@@ -201,7 +181,7 @@ function findDifferences(path, actual, expected) {
   // This still allows for asserting that the first n elements of an array are specified elements,
   // but requires using an object literal (ex: {0: x, 1: y, 2: z} matches [x, y, z, q, w, e] and
   // {0: x, 1: y, 2: z, length: 5} does not match [x, y, z].
-  if (Array.isArray(expected) && actual.length !== expected.length) {
+  if (Array.isArray(expected) && (!Array.isArray(actual) || actual.length !== expected.length)) {
     diffs.push({
       path: `${path}.length`,
       actual,
@@ -256,6 +236,7 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
    * @param {*} obj
    */
   function failsChromeVersionCheck(obj) {
+    if (!obj._minChromiumVersion && !obj._maxChromiumVersion) return false;
     return !chromiumVersionCheck({
       version: getChromeVersionString(),
       min: obj._minChromiumVersion,
@@ -376,6 +357,10 @@ function collateResults(localConsole, actual, expected) {
   /** @type {Comparison[]} */
   const extraAssertions = [];
 
+  if (expected.lhr.userAgent) {
+    extraAssertions.push(makeComparison('userAgent', actual.lhr.userAgent, expected.lhr.userAgent));
+  }
+
   if (expected.lhr.timing) {
     const comparison = makeComparison('timing', actual.lhr.timing, expected.lhr.timing);
     extraAssertions.push(comparison);
@@ -425,7 +410,7 @@ function reportAssertion(localConsole, assertion) {
   RegExp.prototype.toJSON = RegExp.prototype.toString;
 
   if (assertion.equal) {
-    if (isPlainObject(assertion.actual)) {
+    if (isPlainObject(assertion.actual) || Array.isArray(assertion.actual)) {
       localConsole.log(`  ${log.greenify(log.tick)} ${assertion.name}`);
     } else {
       localConsole.log(`  ${log.greenify(log.tick)} ${assertion.name}: ` +

package/cli/test/smokehouse/smokehouse-test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=smokehouse-test.d.ts.map

package/cli/test/smokehouse/smokehouse-test.js

@@ -0,0 +1,120 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as td from 'testdouble';
+
+import {runSmokehouse} from './smokehouse.js';
+import {ChildProcessError} from './lib/child-process-error.js';
+
+/** @type {Smokehouse.TestDfn} */
+const smokeTestDefn = {
+  id: 'test-smoke',
+  expectations: {
+    lhr: {
+      requestedUrl: 'http://localhost:10200/simple-page.html',
+      finalDisplayedUrl: 'http://localhost:10200/simple-page.html',
+      audits: {},
+    },
+  },
+};
+
+describe('runSmokehouse error logging', () => {
+  /** @type {string[]} */
+  let loggedMessages;
+
+  beforeEach(() => {
+    loggedMessages = [];
+    td.replace(console, 'log', (/** @type {unknown[]} */ ...args) => {
+      loggedMessages.push(args.map(String).join(' '));
+    });
+  });
+
+  afterEach(() => {
+    td.reset();
+  });
+
+  it('logs runner logger output when lighthouseRunner returns a malformed result', async () => {
+    /** @type {Smokehouse.LighthouseRunner} */
+    // @ts-expect-error - testing malformed runner result
+    const lighthouseRunner = async (_url, _config, logger) => {
+      logger?.log('runner progress before malformed result');
+      return {lhr: {}, artifacts: undefined};
+    };
+
+    const {success, testResults} = await runSmokehouse([smokeTestDefn], {
+      lighthouseRunner,
+      retries: 0,
+    });
+
+    expect(success).toBe(false);
+    expect(testResults).toHaveLength(1);
+    expect(testResults[0].failed).toBe(1);
+
+    const combinedOutput = loggedMessages.join('\n');
+    expect(combinedOutput).toContain('lighthouse runner returned a bad result');
+    expect(combinedOutput).toContain('runner progress before malformed result');
+  });
+
+  it('logs runner logger output when lighthouseRunner throws a regular Error', async () => {
+    /** @type {Smokehouse.LighthouseRunner} */
+    const lighthouseRunner = async (_url, _config, logger) => {
+      logger?.log('runner progress before throwing');
+      throw new Error('unexpected runner failure');
+    };
+
+    const {success, testResults} = await runSmokehouse([smokeTestDefn], {
+      lighthouseRunner,
+      retries: 0,
+    });
+
+    expect(success).toBe(false);
+    expect(testResults).toHaveLength(1);
+    expect(testResults[0].failed).toBe(1);
+
+    const combinedOutput = loggedMessages.join('\n');
+    expect(combinedOutput).toContain('unexpected runner failure');
+    expect(combinedOutput).toContain('runner progress before throwing');
+  });
+
+  it('does not duplicate runner logger output for ChildProcessError', async () => {
+    /** @type {Smokehouse.LighthouseRunner} */
+    const lighthouseRunner = async (_url, _config, logger) => {
+      logger?.log('unique-cli-runner-log-line');
+      throw new ChildProcessError('Lighthouse run failed', logger?.getLog());
+    };
+
+    const {success} = await runSmokehouse([smokeTestDefn], {
+      lighthouseRunner,
+      retries: 0,
+    });
+
+    expect(success).toBe(false);
+
+    const combinedOutput = loggedMessages.join('\n');
+    const occurrences = combinedOutput.split('unique-cli-runner-log-line').length - 1;
+    expect(occurrences).toBe(1);
+  });
+
+  it('does not duplicate runner logger output when Error message already includes it', async () => {
+    /** @type {Smokehouse.LighthouseRunner} */
+    const lighthouseRunner = async (_url, _config, logger) => {
+      logger?.log('unique-worker-runner-log-line');
+      const log = logger?.getLog() || '';
+      throw new Error(`Worker returned an error: boom\nLog:\n${log}\n`);
+    };
+
+    const {success} = await runSmokehouse([smokeTestDefn], {
+      lighthouseRunner,
+      retries: 0,
+    });
+
+    expect(success).toBe(false);
+
+    const combinedOutput = loggedMessages.join('\n');
+    const occurrences = combinedOutput.split('unique-worker-runner-log-line').length - 1;
+    expect(occurrences).toBe(1);
+  });
+});