lighthouse 9.6.8 → 10.0.0-dev.20230209

package/lighthouse-core/lib/minification-estimator.js

@@ -1,193 +0,0 @@
-/**
- * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-// https://www.ecma-international.org/ecma-262/9.0/index.html#sec-punctuators
-// eslint-disable-next-line max-len
-const PUNCTUATOR_REGEX = /(return|case|{|\(|\[|\.\.\.|;|,|<|>|<=|>=|==|!=|===|!==|\+|-|\*|%|\*\*|\+\+|--|<<|>>|>>>|&|\||\^|!|~|&&|\|\||\?|:|=|\+=|-=|\*=|%=|\*\*=|<<=|>>=|>>>=|&=|\|=|\^=|=>|\/|\/=|\})$/;
-const WHITESPACE_REGEX = /( |\n|\t)+$/;
-
-/**
- * Look backwards from `startPosition` in `content` for an ECMAScript punctuator.
- * This is used to differentiate a RegExp from a divide statement.
- * If a punctuator immediately precedes a lone `/`, the `/` must be the start of a RegExp.
- *
- * @param {string} content
- * @param {number} startPosition
- */
-function hasPunctuatorBefore(content, startPosition) {
-  for (let i = startPosition; i > 0; i--) {
-    // Try to grab at least 6 characters so we can check for `return`
-    const sliceStart = Math.max(0, i - 6);
-    const precedingCharacters = content.slice(sliceStart, i);
-    // Skip over any ending whitespace
-    if (WHITESPACE_REGEX.test(precedingCharacters)) continue;
-    // Check if it's a punctuator
-    return PUNCTUATOR_REGEX.test(precedingCharacters);
-  }
-
-  // The beginning of the content counts too for our purposes.
-  // i.e. a script can't start with a divide symbol
-  return true;
-}
-
-
-/**
- *
- * @param {string} content
- * @param {{singlelineComments: boolean, regex: boolean}} features
- */
-function computeTokenLength(content, features) {
-  let totalTokenLength = 0;
-  let isInSinglelineComment = false;
-  let isInMultilineComment = false;
-  let isInLicenseComment = false;
-  let isInString = false;
-  let isInRegex = false;
-  let isInRegexCharacterClass = false;
-  let stringOpenChar = null;
-
-  /**
-   * Acts as stack for brace tracking.
-   * @type {('templateBrace'|'normalBrace')[]}
-   */
-  const templateLiteralDepth = [];
-
-  for (let i = 0; i < content.length; i++) {
-    const twoChars = content.substr(i, 2);
-    const char = twoChars.charAt(0);
-
-    const isWhitespace = char === ' ' || char === '\n' || char === '\t';
-    const isAStringOpenChar = char === `'` || char === '"' || char === '`';
-
-    if (isInSinglelineComment) {
-      if (char === '\n') {
-        // End the comment when you hit a newline
-        isInSinglelineComment = false;
-      }
-    } else if (isInMultilineComment) {
-      // License comments count
-      if (isInLicenseComment) totalTokenLength++;
-
-      if (twoChars === '*/') {
-        // License comments count, account for the '/' character we're skipping over
-        if (isInLicenseComment) totalTokenLength++;
-        // End the comment when we hit the closing sequence
-        isInMultilineComment = false;
-        // Skip over the '/' character since we've already processed it
-        i++;
-      }
-    } else if (isInString) {
-      // String characters count
-      totalTokenLength++;
-
-      if (stringOpenChar === '`' && twoChars === '${') {
-        // Start new template literal
-        templateLiteralDepth.push('templateBrace');
-        isInString = false;
-        totalTokenLength++;
-        i++;
-      } else if (char === '\\') {
-        // Skip over any escaped characters
-        totalTokenLength++;
-        i++;
-      } else if (char === stringOpenChar) {
-        // End the string when we hit the same stringOpenCharacter
-        isInString = false;
-        // console.log(i, 'exiting string', stringOpenChar)
-      }
-    } else if (isInRegex) {
-      // Regex characters count
-      totalTokenLength++;
-
-      if (char === '\\') {
-        // Skip over any escaped characters
-        totalTokenLength++;
-        i++;
-      } else if (char === '[') {
-        // Register that we're entering a character class so we don't leave the regex prematurely
-        isInRegexCharacterClass = true;
-      } else if (char === ']' && isInRegexCharacterClass) {
-        // Register that we're exiting the character class
-        isInRegexCharacterClass = false;
-      } else if (char === '/' && !isInRegexCharacterClass) {
-        // End the string when we hit the regex close character
-        isInRegex = false;
-        // console.log(i, 'leaving regex', char)
-      }
-    } else {
-      // We're not in any particular token mode, look for the start of different
-      if (twoChars === '/*') {
-        // Start the multi-line comment
-        isInMultilineComment = true;
-        // Check if it's a license comment so we know whether to count it
-        isInLicenseComment = content.charAt(i + 2) === '!';
-        // += 2 because we are processing 2 characters, not just 1
-        if (isInLicenseComment) totalTokenLength += 2;
-        // Skip over the '*' character since we've already processed it
-        i++;
-      } else if (twoChars === '//' && features.singlelineComments) {
-        // Start the single-line comment
-        isInSinglelineComment = true;
-        isInMultilineComment = false;
-        isInLicenseComment = false;
-        // Skip over the second '/' character since we've already processed it
-        i++;
-      } else if (char === '/' && features.regex && hasPunctuatorBefore(content, i)) {
-        // Start the regex
-        isInRegex = true;
-        // Regex characters count
-        totalTokenLength++;
-      } else if (char === '{' && templateLiteralDepth.length) {
-        // Start normal code brace if inside a template literal
-        templateLiteralDepth.push('normalBrace');
-        totalTokenLength++;
-      } else if (char === '}' && templateLiteralDepth.length) {
-        // End one template literal if closing brace is for a template literal
-        if (templateLiteralDepth[templateLiteralDepth.length - 1] === 'templateBrace') {
-          isInString = true;
-          stringOpenChar = '`';
-        }
-        templateLiteralDepth.pop();
-        totalTokenLength++;
-      } else if (isAStringOpenChar) {
-        // Start the string
-        isInString = true;
-        // Save the open character for later so we know when to close it
-        stringOpenChar = char;
-        // String characters count
-        totalTokenLength++;
-      } else if (!isWhitespace) {
-        // All non-whitespace characters count
-        totalTokenLength++;
-      }
-    }
-  }
-
-  // If the content contained unbalanced comments, it's either invalid or we had a parsing error.
-  // Report the token length as the entire string so it will be ignored.
-  if (isInMultilineComment || isInString) {
-    return content.length;
-  }
-
-  return totalTokenLength;
-}
-
-/**
- * @param {string} content
- */
-function computeJSTokenLength(content) {
-  return computeTokenLength(content, {singlelineComments: true, regex: true});
-}
-
-/**
- * @param {string} content
- */
-function computeCSSTokenLength(content) {
-  return computeTokenLength(content, {singlelineComments: false, regex: false});
-}
-
-module.exports = {computeJSTokenLength, computeCSSTokenLength};

package/lighthouse-core/lib/minify-devtoolslog.js

@@ -1,87 +0,0 @@
-/**
- * @license Copyright 2019 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-/* eslint-disable no-console */
-
-/**
- * @fileoverview Minifies a devtools log by removing noisy header values, eliminating data URIs, etc.
- */
-
-const headersToKeep = new Set([
-  // Request headers
-  'accept',
-  'accept-encoding',
-  'accept-ranges',
-  // Response headers
-  'status',
-  'content-length',
-  'content-type',
-  'content-encoding',
-  'content-range',
-  'etag',
-  'cache-control',
-  'last-modified',
-  'link',
-  'x-robots-tag',
-]);
-
-/** @param {LH.Crdp.Network.Headers} [headers] */
-function cleanHeaders(headers) {
-  if (!headers) return;
-
-  for (const key of Object.keys(headers)) {
-    if (!headersToKeep.has(key.toLowerCase())) delete headers[key];
-  }
-}
-
-/** @param {{url: string}} obj */
-function cleanDataURI(obj) {
-  obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER');
-}
-
-/** @param {LH.Crdp.Network.Response} [response] */
-function cleanResponse(response) {
-  if (!response) return;
-  cleanDataURI(response);
-  cleanHeaders(response.requestHeaders);
-  cleanHeaders(response.headers);
-  response.securityDetails = undefined;
-  response.headersText = undefined;
-  response.requestHeadersText = undefined;
-
-  /** @type {any} */
-  const timing = response.timing || {};
-  for (const [k, v] of Object.entries(timing)) {
-    if (v === -1) timing[k] = undefined;
-  }
-}
-
-/**
- * @param {LH.DevtoolsLog} log
- * @return {LH.DevtoolsLog}
- */
-function minifyDevtoolsLog(log) {
-  return log.map(original => {
-    /** @type {LH.Protocol.RawEventMessage} */
-    const entry = JSON.parse(JSON.stringify(original));
-
-    switch (entry.method) {
-      case 'Network.requestWillBeSent':
-        cleanDataURI(entry.params.request);
-        cleanHeaders(entry.params.request.headers);
-        cleanResponse(entry.params.redirectResponse);
-        break;
-      case 'Network.responseReceived':
-        cleanResponse(entry.params.response);
-        break;
-    }
-
-    return entry;
-  });
-}
-
-module.exports = {minifyDevtoolsLog};

package/lighthouse-core/lib/minify-trace.js

@@ -1,173 +0,0 @@
-/**
- * @license Copyright 2019 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-/* eslint-disable no-console */
-
-/**
- * @fileoverview Minifies a trace by removing unnecessary events, throttling screenshots, etc.
- *  See the following files for necessary events:
- *    - lighthouse-core/computed/page-dependency-graph.js
- *    - lighthouse-core/lib/dependency-graph/cpu-node.js
- *    - lighthouse-core/lib/tracehouse/trace-processor.js
- */
-
-const TracingProcessor = require('./tracehouse/trace-processor.js');
-
-const toplevelTaskNames = new Set([
-  'RunTask', // m71+
-  'ThreadControllerImpl::RunTask', // m69-70
-  'ThreadControllerImpl::DoWork', // m66-68
-  'TaskQueueManager::ProcessTaskFromWorkQueue', // m65 and below
-]);
-
-const traceCategoriesToAlwaysKeep = new Set([
-  // Labels the threads correctly in DevTools Performance panel
-  '__metadata',
-]);
-
-const traceEventsToAlwaysKeep = new Set([
-  'Screenshot',
-  'TracingStartedInBrowser',
-  'TracingStartedInPage',
-  'navigationStart',
-  'ParseAuthorStyleSheet',
-  'ParseHTML',
-  'PlatformResourceSendRequest',
-  'ResourceSendRequest',
-  'ResourceReceiveResponse',
-  'ResourceFinish',
-  'ResourceReceivedData',
-  'EventDispatch',
-  'LayoutShift',
-  'FrameCommittedInBrowser',
-  'EventTiming',
-  'Responsiveness.Renderer.UserInteraction',
-  // Not currently used by Lighthouse but might be used in the future for cross-frame LCP
-  'NavStartToLargestContentfulPaint::Invalidate::AllFrames::UKM',
-  'NavStartToLargestContentfulPaint::Candidate::AllFrames::UKM',
-  // Needed for CPU profiler task attribution
-  'Profile',
-  'ProfileChunk',
-]);
-
-const traceEventsToKeepInToplevelTask = new Set([
-  // Needed for CPU node timing simulations
-  'Layout',
-  'RunMicrotasks',
-  // All of these are needed to create graph relationships
-  'TimerInstall',
-  'TimerFire',
-  'InvalidateLayout',
-  'ScheduleStyleRecalculation',
-  'EvaluateScript',
-  'XHRReadyStateChange',
-  'FunctionCall',
-  'v8.compile',
-  'ParseAuthorStyleSheet',
-  'ResourceSendRequest',
-]);
-
-const traceEventsToKeepInProcess = new Set([
-  ...toplevelTaskNames,
-  ...traceEventsToKeepInToplevelTask,
-
-  // See the DevTools marker events
-  // https://source.chromium.org/chromium/chromium/src/+/main:third_party/devtools-frontend/src/front_end/timeline_model/TimelineModel.js?q=-f:out%20f:TimelineModel%20%22MarkLoad:%20%27%22&ss=chromium
-  'firstPaint',
-  'firstContentfulPaint',
-  'firstMeaningfulPaint',
-  'firstMeaningfulPaintCandidate',
-  'loadEventEnd',
-  'MarkLoad',
-  'domContentLoadedEventEnd',
-  'MarkDOMContent',
-  'largestContentfulPaint::Invalidate',
-  'largestContentfulPaint::Candidate',
-  'Animation',
-]);
-
-/**
- * @param {LH.TraceEvent[]} events
- */
-function filterOutUnnecessaryTasksByNameAndDuration(events) {
-  const {pid} = TracingProcessor.findMainFrameIds(events);
-
-  return events.filter(evt => {
-    if (toplevelTaskNames.has(evt.name) && evt.dur < 1000) return false;
-    if (evt.pid === pid && traceEventsToKeepInProcess.has(evt.name)) return true;
-    if (traceCategoriesToAlwaysKeep.has(evt.cat)) return true;
-    return traceEventsToAlwaysKeep.has(evt.name);
-  });
-}
-
-/**
- * Filters out tasks that are not within a toplevel task.
- * @param {LH.TraceEvent[]} events
- */
-function filterOutOrphanedTasks(events) {
-  const toplevelRanges = events
-    .filter(evt => toplevelTaskNames.has(evt.name))
-    .map(evt => [evt.ts, evt.ts + evt.dur]);
-
-  /** @param {LH.TraceEvent} e */
-  const isInToplevelTask = e => toplevelRanges.some(([start, end]) => e.ts >= start && e.ts <= end);
-
-  return events.filter((evt, index) => {
-    if (!traceEventsToKeepInToplevelTask.has(evt.name)) return true;
-    if (!isInToplevelTask(evt)) return false;
-
-    if (evt.ph === 'B') {
-      const endEvent = events.slice(index).find(e => e.name === evt.name && e.ph === 'E');
-      return endEvent && isInToplevelTask(endEvent);
-    } else {
-      return true;
-    }
-  });
-}
-
-/**
- * Throttles screenshot events in the trace to 2fps.
- * @param {LH.TraceEvent[]} events
- */
-function filterOutExcessiveScreenshots(events) {
-  const screenshotTimestamps = events.filter(evt => evt.name === 'Screenshot').map(evt => evt.ts);
-
-  let lastScreenshotTs = -Infinity;
-  return events.filter(evt => {
-    if (evt.name !== 'Screenshot') return true;
-    const timeSinceLastScreenshot = evt.ts - lastScreenshotTs;
-    const nextScreenshotTs = screenshotTimestamps.find(ts => ts > evt.ts);
-    const timeUntilNextScreenshot = nextScreenshotTs ? nextScreenshotTs - evt.ts : Infinity;
-    const threshold = 500 * 1000; // Throttle to ~2fps
-    // Keep the frame if it's been more than 500ms since the last frame we kept or the next frame won't happen for at least 500ms
-    const shouldKeep = timeUntilNextScreenshot > threshold || timeSinceLastScreenshot > threshold;
-    if (shouldKeep) lastScreenshotTs = evt.ts;
-    return shouldKeep;
-  });
-}
-
-/**
- * @param {LH.TraceEvent[]} events
- */
-function filterTraceEvents(events) {
-  // Filter out event names we don't care about and tasks <1ms
-  let filtered = filterOutUnnecessaryTasksByNameAndDuration(events);
-  // Filter out events not inside a toplevel task
-  filtered = filterOutOrphanedTasks(filtered);
-  // Filter down the screenshots to key moments + 2fps animations
-  return filterOutExcessiveScreenshots(filtered);
-}
-
-/**
- * @param {LH.Trace} inputTrace
- * @return {LH.Trace}
- */
-function minifyTrace(inputTrace) {
-  return {traceEvents: filterTraceEvents(inputTrace.traceEvents)};
-}
-
-module.exports = {minifyTrace};

package/lighthouse-core/lib/navigation-error.js

@@ -1,145 +0,0 @@
-/**
- * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-'use strict';
-
-const LHError = require('./lh-error.js');
-const NetworkAnalyzer = require('./dependency-graph/simulator/network-analyzer.js');
-
-/**
- * Returns an error if the original network request failed or wasn't found.
- * @param {LH.Artifacts.NetworkRequest|undefined} mainRecord
- * @return {LH.LighthouseError|undefined}
- */
-function getNetworkError(mainRecord) {
-  if (!mainRecord) {
-    return new LHError(LHError.errors.NO_DOCUMENT_REQUEST);
-  } else if (mainRecord.failed) {
-    const netErr = mainRecord.localizedFailDescription;
-    // Match all resolution and DNS failures
-    // https://cs.chromium.org/chromium/src/net/base/net_error_list.h?rcl=cd62979b
-    if (
-      netErr === 'net::ERR_NAME_NOT_RESOLVED' ||
-      netErr === 'net::ERR_NAME_RESOLUTION_FAILED' ||
-      netErr.startsWith('net::ERR_DNS_')
-    ) {
-      return new LHError(LHError.errors.DNS_FAILURE);
-    } else {
-      return new LHError(LHError.errors.FAILED_DOCUMENT_REQUEST, {errorDetails: netErr});
-    }
-  } else if (mainRecord.hasErrorStatusCode()) {
-    return new LHError(LHError.errors.ERRORED_DOCUMENT_REQUEST, {
-      statusCode: `${mainRecord.statusCode}`,
-    });
-  }
-}
-
-/**
- * Returns an error if we ended up on the `chrome-error` page and all other requests failed.
- * @param {LH.Artifacts.NetworkRequest|undefined} mainRecord
- * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
- * @return {LH.LighthouseError|undefined}
- */
-function getInterstitialError(mainRecord, networkRecords) {
-  // If we never requested a document, there's no interstitial error, let other cases handle it.
-  if (!mainRecord) return undefined;
-
-  const interstitialRequest = networkRecords.find(record =>
-    record.documentURL.startsWith('chrome-error://')
-  );
-  // If the page didn't end up on a chrome interstitial, there's no error here.
-  if (!interstitialRequest) return undefined;
-
-  // If the main document didn't fail, we didn't end up on an interstitial.
-  // FIXME: This doesn't handle client-side redirects.
-  // None of our error-handling deals with this case either because passContext.url doesn't handle non-network redirects.
-  if (!mainRecord.failed) return undefined;
-
-  // If a request failed with the `net::ERR_CERT_*` collection of errors, then it's a security issue.
-  if (mainRecord.localizedFailDescription.startsWith('net::ERR_CERT')) {
-    return new LHError(LHError.errors.INSECURE_DOCUMENT_REQUEST, {
-      securityMessages: mainRecord.localizedFailDescription,
-    });
-  }
-
-  // If we made it this far, it's a generic Chrome interstitial error.
-  return new LHError(LHError.errors.CHROME_INTERSTITIAL_ERROR);
-}
-
-/**
- * Returns an error if we try to load a non-HTML page.
- * Expects a network request with all redirects resolved, otherwise the MIME type may be incorrect.
- * @param {LH.Artifacts.NetworkRequest|undefined} finalRecord
- * @return {LH.LighthouseError|undefined}
- */
-function getNonHtmlError(finalRecord) {
-  // MIME types are case-insenstive but Chrome normalizes MIME types to be lowercase.
-  const HTML_MIME_TYPE = 'text/html';
-
-  // If we never requested a document, there's no doctype error, let other cases handle it.
-  if (!finalRecord) return undefined;
-
-  // mimeType is determined by the browser, we assume Chrome is determining mimeType correctly,
-  // independently of 'Content-Type' response headers, and always sending mimeType if well-formed.
-  if (HTML_MIME_TYPE !== finalRecord.mimeType) {
-    return new LHError(LHError.errors.NOT_HTML, {mimeType: finalRecord.mimeType});
-  }
-
-  return undefined;
-}
-
-/**
- * Returns an error if the page load should be considered failed, e.g. from a
- * main document request failure, a security issue, etc.
- * @param {LH.LighthouseError|undefined} navigationError
- * @param {{url: string, loadFailureMode: LH.Gatherer.PassContext['passConfig']['loadFailureMode'], networkRecords: Array<LH.Artifacts.NetworkRequest>}} context
- * @return {LH.LighthouseError|undefined}
- */
-function getPageLoadError(navigationError, context) {
-  const {url, loadFailureMode, networkRecords} = context;
-  /** @type {LH.Artifacts.NetworkRequest|undefined} */
-  let mainRecord;
-  try {
-    mainRecord = NetworkAnalyzer.findMainDocument(networkRecords, url);
-  } catch (_) {}
-
-  // MIME Type is only set on the final redirected document request. Use this for the HTML check instead of root.
-  let finalRecord;
-  if (mainRecord) {
-    finalRecord = NetworkAnalyzer.resolveRedirects(mainRecord);
-  }
-
-  const networkError = getNetworkError(mainRecord);
-  const interstitialError = getInterstitialError(mainRecord, networkRecords);
-  const nonHtmlError = getNonHtmlError(finalRecord);
-
-  // Check to see if we need to ignore the page load failure.
-  // e.g. When the driver is offline, the load will fail without page offline support.
-  if (loadFailureMode === 'ignore') return;
-
-  // We want to special-case the interstitial beyond FAILED_DOCUMENT_REQUEST. See https://github.com/GoogleChrome/lighthouse/pull/8865#issuecomment-497507618
-  if (interstitialError) return interstitialError;
-
-  // Network errors are usually the most specific and provide the best reason for why the page failed to load.
-  // Prefer networkError over navigationError.
-  // Example: `DNS_FAILURE` is better than `NO_FCP`.
-  if (networkError) return networkError;
-
-  // Error if page is not HTML.
-  if (nonHtmlError) return nonHtmlError;
-
-  // Navigation errors are rather generic and express some failure of the page to render properly.
-  // Use `navigationError` as the last resort.
-  // Example: `NO_FCP`, the page never painted content for some unknown reason.
-  return navigationError;
-}
-
-
-module.exports = {
-  getNetworkError,
-  getInterstitialError,
-  getPageLoadError,
-  getNonHtmlError,
-};