lighthouse 8.5.1-dev.20211012 → 8.5.1

package/lighthouse-core/lib/i18n/README.md

@@ -1,7 +1,7 @@
 # Terminology
 
 * **CTC format**: The [Chrome extension & Chrome app i18n format](https://developer.chrome.com/extensions/i18n-messages) with some minor changes. JSON with their specified model for declaring placeholders, examples, etc. Used as an interchange data format.
-* **LHL syntax** (Lighthouse Localizable syntax): The ICU-friendly string syntax that is used to author `UIStrings` and is seen in the locale files in `shared/localization/locales/*.json`. Lighthouse has a custom syntax these strings combines many ICU message features along with some markdown.
+* **LHL syntax** (Lighthouse Localizable syntax): The ICU-friendly string syntax that is used to author `UIStrings` and is seen in the locale files in `i18n/locales/*.json`. Lighthouse has a custom syntax these strings combines many ICU message features along with some markdown.
 * **ICU**: ICU (International Components for Unicode) is a localization project and standard defined by the Unicode consortium. In general, we refer to "ICU" as the [ICU message formatting](http://userguide.icu-project.org/formatparse/messages) syntax.
 
 # The Lighthouse i18n pipeline
@@ -12,11 +12,11 @@ The collection and translation pipeline:
 ```
  Source files:                                         Locale files:
 +---------------------------+                         +----------------------------------------------
-|                           ++                        | shared/localization/locales/en-US.json |
-| const UIStrings = { ... };|-+                 +---> | shared/localization/locales/en-XL.json |
+|                           ++                        | lighthouse-core/lib/i18n/locales/en-US.json |
+| const UIStrings = { ... };|-+                 +---> | lighthouse-core/lib/i18n/locales/en-XL.json |
 |                           |-|                 |     +----------------------------------------------+
 +-----------------------------|                 |     |                                             ||
- +----------------------------|                 |     | shared/localization/locales/*.json     |-<+
+ +----------------------------|                 |     | lighthouse-core/lib/i18n/locales/*.json     |-<+
   +---------------------------+                 |     |                                             || |
                            |                    |     +----------------------------------------------| |
   $ yarn                   |                    |      +---------------------------------------------+ |
@@ -218,7 +218,7 @@ CTC is a name that is distinct and identifies this as the Chrome translation for
 1.  String called in `.js` file, converted to `LH.IcuMessage` object.
 
 1.  Message object is replaced with the localized string via
-    `format.replaceIcuMessages` and `format.getFormatted`.
+    `i18n.replaceIcuMessages` and `i18n.getFormatted`.
 
 #### Example:
 
@@ -251,7 +251,7 @@ CTC is a name that is distinct and identifies this as the Chrome translation for
     }
     ```
 
-3.  Lookup in `format.replaceIcuMessages` and `format.getFormatted` will attempt to find the message in this order:
+3.  Lookup in `i18n.replaceIcuMessages` and `i18n.getFormatted` will attempt to find the message in this order:
 
     1.  `locales/{locale}.json` The best result. `icuMessage.i18nId` is found in the target locale and the resulting string should appear correct.
 

package/lighthouse-core/lib/i18n/i18n.js

@@ -5,17 +5,23 @@
  */
 'use strict';
 
-/** @typedef {import('../../../shared/localization/locales').LhlMessages} LhlMessages */
+/** @typedef {import('../../lib/i18n/locales').LhlMessages} LhlMessages */
 
 const path = require('path');
+const MessageFormat = require('intl-messageformat').default;
 const lookupClosestLocale = require('lookup-closest-locale');
-const {getAvailableLocales} = require('../../../shared/localization/format.js');
+const LOCALES = require('./locales.js');
+const {isObjectOfUnknownValues, isObjectOrArrayOfUnknownValues} = require('../type-verifiers.js');
 const log = require('lighthouse-logger');
 const {LH_ROOT} = require('../../../root.js');
-const {isIcuMessage, _formatMessage} = require('../../../shared/localization/format.js');
 
 const DEFAULT_LOCALE = 'en';
 
+/** @typedef {import('intl-messageformat-parser').Element} MessageElement */
+/** @typedef {import('intl-messageformat-parser').ArgumentElement} ArgumentElement */
+
+const MESSAGE_I18N_ID_REGEX = / | [^\s]+$/;
+
 const UIStrings = {
   /** Used to show the duration in milliseconds that something lasted. The `{timeInMs}` placeholder will be replaced with the time duration, shown in milliseconds (e.g. 63 ms) */
   ms: '{timeInMs, number, milliseconds}\xa0ms',
@@ -111,6 +117,27 @@ const UIStrings = {
   itemSeverityHigh: 'High',
 };
 
+const formats = {
+  number: {
+    bytes: {
+      maximumFractionDigits: 0,
+    },
+    milliseconds: {
+      maximumFractionDigits: 0,
+    },
+    seconds: {
+      // Force the seconds to the tenths place for limited output and ease of scanning
+      minimumFractionDigits: 1,
+      maximumFractionDigits: 1,
+    },
+    extendedPercent: {
+      // Force allow up to two digits after decimal place in percentages. (Intl.NumberFormat options)
+      maximumFractionDigits: 2,
+      style: 'percent',
+    },
+  },
+};
+
 /**
  * Look up the best available locale for the requested language through these fall backs:
  * - exact match
@@ -133,12 +160,7 @@ function lookupLocale(locales) {
   // Filter by what's available in this runtime.
   const availableLocales = Intl.NumberFormat.supportedLocalesOf(canonicalLocales);
 
-  // Get available locales and transform into object to match `lookupClosestLocale`'s API.
-  const localesWithMessages = getAvailableLocales();
-  const localesWithmessagesObj = /** @type {Record<LH.Locale, LhlMessages>} */ (
-    Object.fromEntries(localesWithMessages.map(l => [l, {}])));
-
-  const closestLocale = lookupClosestLocale(availableLocales, localesWithmessagesObj);
+  const closestLocale = lookupClosestLocale(availableLocales, LOCALES);
 
   if (!closestLocale) {
     // Log extra info if we're pretty sure this version of Node was built with `--with-intl=small-icu`.
@@ -152,6 +174,191 @@ function lookupLocale(locales) {
   return closestLocale || DEFAULT_LOCALE;
 }
 
+/**
+ * Function to retrieve all 'argumentElement's from an ICU message. An argumentElement
+ * is an ICU element with an argument in it, like '{varName}' or '{varName, number, bytes}'. This
+ * differs from 'messageElement's which are just arbitrary text in a message.
+ *
+ * Notes:
+ *  This function will recursively inspect plural elements for nested argumentElements.
+ *
+ *  We need to find all the elements from the plural format sections, but
+ *  they need to be deduplicated. I.e. "=1{hello {icu}} =other{hello {icu}}"
+ *  the variable "icu" would appear twice if it wasn't de duplicated. And they cannot
+ *  be stored in a set because they are not equal since their locations are different,
+ *  thus they are stored via a Map keyed on the "id" which is the ICU varName.
+ *
+ * @param {Array<MessageElement>} icuElements
+ * @param {Map<string, ArgumentElement>} [seenElementsById]
+ * @return {Map<string, ArgumentElement>}
+ */
+function collectAllCustomElementsFromICU(icuElements, seenElementsById = new Map()) {
+  for (const el of icuElements) {
+    // We are only interested in elements that need ICU formatting (argumentElements)
+    if (el.type !== 'argumentElement') continue;
+
+    seenElementsById.set(el.id, el);
+
+    // Plurals need to be inspected recursively
+    if (!el.format || el.format.type !== 'pluralFormat') continue;
+    // Look at all options of the plural (=1{} =other{}...)
+    for (const option of el.format.options) {
+      // Run collections on each option's elements
+      collectAllCustomElementsFromICU(option.value.elements, seenElementsById);
+    }
+  }
+
+  return seenElementsById;
+}
+
+/**
+ * Returns a copy of the `values` object, with the values formatted based on how
+ * they will be used in their icuMessage, e.g. KB or milliseconds. The original
+ * object is unchanged.
+ * @param {MessageFormat} messageFormatter
+ * @param {Readonly<Record<string, string | number>>} values
+ * @param {string} lhlMessage Used for clear error logging.
+ * @return {Record<string, string | number>}
+ */
+function _preformatValues(messageFormatter, values, lhlMessage) {
+  const elementMap = collectAllCustomElementsFromICU(messageFormatter.getAst().elements);
+  const argumentElements = [...elementMap.values()];
+
+  /** @type {Record<string, string | number>} */
+  const formattedValues = {};
+
+  for (const {id, format} of argumentElements) {
+    // Throw an error if a message's value isn't provided
+    if (id && (id in values) === false) {
+      throw new Error(`ICU Message "${lhlMessage}" contains a value reference ("${id}") ` +
+        `that wasn't provided`);
+    }
+
+    const value = values[id];
+
+    // Direct `{id}` replacement and non-numeric values need no formatting.
+    if (!format || format.type !== 'numberFormat') {
+      formattedValues[id] = value;
+      continue;
+    }
+
+    if (typeof value !== 'number') {
+      throw new Error(`ICU Message "${lhlMessage}" contains a numeric reference ("${id}") ` +
+        'but provided value was not a number');
+    }
+
+    // Format values for known styles.
+    if (format.style === 'milliseconds') {
+      // Round all milliseconds to the nearest 10.
+      formattedValues[id] = Math.round(value / 10) * 10;
+    } else if (format.style === 'seconds' && id === 'timeInMs') {
+      // Convert all seconds to the correct unit (currently only for `timeInMs`).
+      formattedValues[id] = Math.round(value / 100) / 10;
+    } else if (format.style === 'bytes') {
+      // Replace all the bytes with KB.
+      formattedValues[id] = value / 1024;
+    } else {
+      // For all other number styles, the value isn't changed.
+      formattedValues[id] = value;
+    }
+  }
+
+  // Throw an error if a value is provided but has no placeholder in the message.
+  for (const valueId of Object.keys(values)) {
+    if (valueId in formattedValues) continue;
+
+    // errorCode is a special case always allowed to help LHError ease-of-use.
+    if (valueId === 'errorCode') {
+      formattedValues.errorCode = values.errorCode;
+      continue;
+    }
+
+    throw new Error(`Provided value "${valueId}" does not match any placeholder in ` +
+      `ICU message "${lhlMessage}"`);
+  }
+
+  return formattedValues;
+}
+
+/**
+ * Format string `message` by localizing `values` and inserting them.
+ * @param {string} message
+ * @param {Record<string, string | number>} values
+ * @param {LH.Locale} locale
+ * @return {string}
+ */
+function _formatMessage(message, values = {}, locale) {
+  // When using accented english, force the use of a different locale for number formatting.
+  const localeForMessageFormat = (locale === 'en-XA' || locale === 'en-XL') ? 'de-DE' : locale;
+
+  const formatter = new MessageFormat(message, localeForMessageFormat, formats);
+
+  // Preformat values for the message format like KB and milliseconds.
+  const valuesForMessageFormat = _preformatValues(formatter, values, message);
+
+  return formatter.format(valuesForMessageFormat);
+}
+
+/**
+ * Retrieves the localized version of `icuMessage` and formats with any given
+ * value replacements.
+ * @param {LH.IcuMessage} icuMessage
+ * @param {LH.Locale} locale
+ * @return {string}
+ */
+function _localizeIcuMessage(icuMessage, locale) {
+  const localeMessages = LOCALES[locale];
+  if (!localeMessages) throw new Error(`Unsupported locale '${locale}'`);
+  const localeMessage = localeMessages[icuMessage.i18nId];
+
+  // Fall back to the default (usually the original english message) if we couldn't find a
+  // message in the specified locale. This could be because of string drift between
+  // Lighthouse versions or because new strings haven't been updated yet. Better to have
+  // an english message than no message at all; in some cases it won't even matter.
+  if (!localeMessage) {
+    return icuMessage.formattedDefault;
+  }
+
+  return _formatMessage(localeMessage.message, icuMessage.values, locale);
+}
+
+/** @param {string[]} pathInLHR */
+function _formatPathAsString(pathInLHR) {
+  let pathAsString = '';
+  for (const property of pathInLHR) {
+    if (/^[a-z]+$/i.test(property)) {
+      if (pathAsString.length) pathAsString += '.';
+      pathAsString += property;
+    } else {
+      if (/]|"|'|\s/.test(property)) throw new Error(`Cannot handle "${property}" in i18n`);
+      pathAsString += `[${property}]`;
+    }
+  }
+
+  return pathAsString;
+}
+
+/**
+ * @param {LH.Locale} locale
+ * @return {Record<string, string>}
+ */
+function getRendererFormattedStrings(locale) {
+  const localeMessages = LOCALES[locale];
+  if (!localeMessages) throw new Error(`Unsupported locale '${locale}'`);
+
+  const icuMessageIds = Object.keys(localeMessages).filter(f => f.startsWith('report/'));
+  /** @type {Record<string, string>} */
+  const strings = {};
+  for (const icuMessageId of icuMessageIds) {
+    const [filename, key] = icuMessageId.split(' | ');
+    if (!filename.endsWith('util.js')) throw new Error(`Unexpected message: ${icuMessageId}`);
+
+    strings[key] = localeMessages[icuMessageId].message;
+  }
+
+  return strings;
+}
+
 /**
  * Returns a function that generates `LH.IcuMessage` objects to localize the
  * messages in `fileStrings` and the shared `i18n.UIStrings`.
@@ -189,6 +396,125 @@ function createIcuMessageFn(filename, fileStrings) {
   return getIcuMessageFn;
 }
 
+/**
+ * Returns whether `icuMessageOrNot`` is an `LH.IcuMessage` instance.
+ * @param {unknown} icuMessageOrNot
+ * @return {icuMessageOrNot is LH.IcuMessage}
+ */
+function isIcuMessage(icuMessageOrNot) {
+  if (!isObjectOfUnknownValues(icuMessageOrNot)) {
+    return false;
+  }
+
+  const {i18nId, values, formattedDefault} = icuMessageOrNot;
+  if (typeof i18nId !== 'string') {
+    return false;
+  }
+
+  // formattedDefault is required.
+  if (typeof formattedDefault !== 'string') {
+    return false;
+  }
+
+  // Values is optional.
+  if (values !== undefined) {
+    if (!isObjectOfUnknownValues(values)) {
+      return false;
+    }
+    for (const value of Object.values(values)) {
+      if (typeof value !== 'string' && typeof value !== 'number') {
+        return false;
+      }
+    }
+  }
+
+  // Finally return true if i18nId seems correct.
+  return MESSAGE_I18N_ID_REGEX.test(i18nId);
+}
+
+/**
+ * Get the localized and formatted form of `icuMessageOrRawString` if it's an
+ * LH.IcuMessage, or get it back directly if it's already a string.
+ * Warning: this function throws if `icuMessageOrRawString` is not the expected
+ * type (use function from `createIcuMessageFn` to create a valid LH.IcuMessage)
+ * or `locale` isn't supported (use `lookupLocale` to find a valid locale).
+ * @param {LH.IcuMessage | string} icuMessageOrRawString
+ * @param {LH.Locale} locale
+ * @return {string}
+ */
+function getFormatted(icuMessageOrRawString, locale) {
+  if (isIcuMessage(icuMessageOrRawString)) {
+    return _localizeIcuMessage(icuMessageOrRawString, locale);
+  }
+
+  if (typeof icuMessageOrRawString === 'string') {
+    return icuMessageOrRawString;
+  }
+
+  // Should be impossible from types, but do a strict check in case malformed JSON makes it this far.
+  throw new Error('Attempted to format invalid icuMessage type');
+}
+
+/**
+ * Recursively walk the input object, looking for property values that are
+ * `LH.IcuMessage`s and replace them with their localized values. Primarily
+ * used with the full LHR or a Config as input.
+ * Returns a map of locations that were replaced to the `IcuMessage` that was at
+ * that location.
+ * @param {unknown} inputObject
+ * @param {LH.Locale} locale
+ * @return {LH.Result.IcuMessagePaths}
+ */
+function replaceIcuMessages(inputObject, locale) {
+  /**
+   * @param {unknown} subObject
+   * @param {LH.Result.IcuMessagePaths} icuMessagePaths
+   * @param {string[]} pathInLHR
+   */
+  function replaceInObject(subObject, icuMessagePaths, pathInLHR = []) {
+    if (!isObjectOrArrayOfUnknownValues(subObject)) return;
+
+    for (const [property, possibleIcuMessage] of Object.entries(subObject)) {
+      const currentPathInLHR = pathInLHR.concat([property]);
+
+      // Replace any IcuMessages with a localized string.
+      if (isIcuMessage(possibleIcuMessage)) {
+        const formattedString = _localizeIcuMessage(possibleIcuMessage, locale);
+        const messageInstancesInLHR = icuMessagePaths[possibleIcuMessage.i18nId] || [];
+        const currentPathAsString = _formatPathAsString(currentPathInLHR);
+
+        messageInstancesInLHR.push(
+          possibleIcuMessage.values ?
+            {values: possibleIcuMessage.values, path: currentPathAsString} :
+            currentPathAsString
+        );
+
+        // @ts-ignore - tsc doesn't like that `property` can be either string key or array index.
+        subObject[property] = formattedString;
+        icuMessagePaths[possibleIcuMessage.i18nId] = messageInstancesInLHR;
+      } else {
+        replaceInObject(possibleIcuMessage, icuMessagePaths, currentPathInLHR);
+      }
+    }
+  }
+
+  /** @type {LH.Result.IcuMessagePaths} */
+  const icuMessagePaths = {};
+  replaceInObject(inputObject, icuMessagePaths);
+  return icuMessagePaths;
+}
+
+/**
+ * Populate the i18n string lookup dict with locale data
+ * Used when the host environment selects the locale and serves lighthouse the intended locale file
+ * @see https://docs.google.com/document/d/1jnt3BqKB-4q3AE94UWFA0Gqspx8Sd_jivlB7gQMlmfk/edit
+ * @param {LH.Locale} locale
+ * @param {LhlMessages} lhlMessages
+ */
+function registerLocaleData(locale, lhlMessages) {
+  LOCALES[locale] = lhlMessages;
+}
+
 /**
  * Returns true if the given value is a string or an LH.IcuMessage.
  * @param {unknown} value
@@ -199,9 +525,16 @@ function isStringOrIcuMessage(value) {
 }
 
 module.exports = {
+  _formatPathAsString,
   UIStrings,
   lookupLocale,
+  getRendererFormattedStrings,
   createIcuMessageFn,
+  getFormatted,
+  replaceIcuMessages,
+  isIcuMessage,
+  collectAllCustomElementsFromICU,
+  registerLocaleData,
   isStringOrIcuMessage,
   // TODO: exported for backwards compatibility. Consider removing on future breaking change.
   createMessageInstanceIdFn: createIcuMessageFn,

package/{shared/localization → lighthouse-core/lib/i18n}/locales/en-US.json

@@ -1,115 +1,4 @@
 {
-  "flow-report/src/i18n/ui-strings.js | allReports": {
-    "message": "All Reports"
-  },
-  "flow-report/src/i18n/ui-strings.js | categories": {
-    "message": "Categories"
-  },
-  "flow-report/src/i18n/ui-strings.js | categoryAccessibility": {
-    "message": "Accessibility"
-  },
-  "flow-report/src/i18n/ui-strings.js | categoryBestPractices": {
-    "message": "Best Practices"
-  },
-  "flow-report/src/i18n/ui-strings.js | categoryPerformance": {
-    "message": "Performance"
-  },
-  "flow-report/src/i18n/ui-strings.js | categoryProgressiveWebApp": {
-    "message": "Progressive Web App"
-  },
-  "flow-report/src/i18n/ui-strings.js | categorySeo": {
-    "message": "SEO"
-  },
-  "flow-report/src/i18n/ui-strings.js | desktop": {
-    "message": "Desktop"
-  },
-  "flow-report/src/i18n/ui-strings.js | helpDialogTitle": {
-    "message": "Understanding the Lighthouse Flow Report"
-  },
-  "flow-report/src/i18n/ui-strings.js | helpLabel": {
-    "message": "Understanding Flows"
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseInstructionNavigation": {
-    "message": "Use Navigation reports to..."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseInstructionSnapshot": {
-    "message": "Use Snapshot reports to..."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseInstructionTimespan": {
-    "message": "Use Timespan reports to..."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseNavigation1": {
-    "message": "Obtain a Lighthouse Performance score."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseNavigation2": {
-    "message": "Measure page load Performance metrics such as Largest Contentful Paint and Speed Index."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseNavigation3": {
-    "message": "Assess Progressive Web App capabilities."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseSnapshot1": {
-    "message": "Find accessibility issues in single page applications or complex forms."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseSnapshot2": {
-    "message": "Evaluate best practices of menus and UI elements hidden behind interaction."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseTimespan1": {
-    "message": "Measure layout shifts and JavaScript execution time on a series of interactions."
-  },
-  "flow-report/src/i18n/ui-strings.js | helpUseCaseTimespan2": {
-    "message": "Discover performance opportunities to improve the experience for long-lived pages and single-page applications."
-  },
-  "flow-report/src/i18n/ui-strings.js | mobile": {
-    "message": "Mobile"
-  },
-  "flow-report/src/i18n/ui-strings.js | navigationDescription": {
-    "message": "Page load"
-  },
-  "flow-report/src/i18n/ui-strings.js | navigationLongDescription": {
-    "message": "Navigation reports analyze a single page load, exactly like the original Lighthouse reports."
-  },
-  "flow-report/src/i18n/ui-strings.js | navigationReport": {
-    "message": "Navigation report"
-  },
-  "flow-report/src/i18n/ui-strings.js | ratingAverage": {
-    "message": "Average"
-  },
-  "flow-report/src/i18n/ui-strings.js | ratingError": {
-    "message": "Error"
-  },
-  "flow-report/src/i18n/ui-strings.js | ratingFail": {
-    "message": "Poor"
-  },
-  "flow-report/src/i18n/ui-strings.js | ratingPass": {
-    "message": "Good"
-  },
-  "flow-report/src/i18n/ui-strings.js | save": {
-    "message": "Save"
-  },
-  "flow-report/src/i18n/ui-strings.js | snapshotDescription": {
-    "message": "Captured state of page"
-  },
-  "flow-report/src/i18n/ui-strings.js | snapshotLongDescription": {
-    "message": "Snapshot reports analyze the page in a particular state, typically after user interactions."
-  },
-  "flow-report/src/i18n/ui-strings.js | snapshotReport": {
-    "message": "Snapshot report"
-  },
-  "flow-report/src/i18n/ui-strings.js | summary": {
-    "message": "Summary"
-  },
-  "flow-report/src/i18n/ui-strings.js | timespanDescription": {
-    "message": "User interactions"
-  },
-  "flow-report/src/i18n/ui-strings.js | timespanLongDescription": {
-    "message": "Timespan reports analyze an arbitrary period of time, typically containing user interactions."
-  },
-  "flow-report/src/i18n/ui-strings.js | timespanReport": {
-    "message": "Timespan report"
-  },
-  "flow-report/src/i18n/ui-strings.js | title": {
-    "message": "Lighthouse User Flow Report"
-  },
   "lighthouse-core/audits/accessibility/accesskeys.js | description": {
     "message": "Access keys let users quickly focus a part of the page. For proper navigation, each access key must be unique. [Learn more](https://web.dev/accesskeys/)."
   },