lighthouse 8.5.1 → 8.6.0-dev.20211016

package/shared/localization/format.js

@@ -0,0 +1,455 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const fs = require('fs');
+
+const MessageFormat = require('intl-messageformat').default;
+const {isObjectOfUnknownValues, isObjectOrArrayOfUnknownValues} = require('../type-verifiers.js');
+
+/** Contains available locales with messages. May be an empty object if bundled. */
+const LOCALE_MESSAGES = require('./locales.js');
+
+const DEFAULT_LOCALE = 'en-US';
+
+/**
+ * The locale tags for the localized messages available to Lighthouse on disk.
+ * When bundled, these will be inlined by brfs.
+ * These locales are considered the "canonical" locales. We support other locales which
+ * are simply aliases to one of these. ex: es-AR (alias) -> es-419 (canonical)
+ */
+let CANONICAL_LOCALES = ['__availableLocales__'];
+// TODO: need brfs in gh-pages-app. For now, above is replaced, see build-i18n.module.js
+if (fs.readdirSync) {
+  CANONICAL_LOCALES = fs.readdirSync(__dirname + '/locales/')
+    .filter(basename => basename.endsWith('.json') && !basename.endsWith('.ctc.json'))
+    .map(locale => locale.replace('.json', ''))
+    .sort();
+}
+
+/** @typedef {import('intl-messageformat-parser').Element} MessageElement */
+/** @typedef {import('intl-messageformat-parser').ArgumentElement} ArgumentElement */
+
+const MESSAGE_I18N_ID_REGEX = / | [^\s]+$/;
+
+const formats = {
+  number: {
+    bytes: {
+      maximumFractionDigits: 0,
+    },
+    milliseconds: {
+      maximumFractionDigits: 0,
+    },
+    seconds: {
+      // Force the seconds to the tenths place for limited output and ease of scanning
+      minimumFractionDigits: 1,
+      maximumFractionDigits: 1,
+    },
+    extendedPercent: {
+      // Force allow up to two digits after decimal place in percentages. (Intl.NumberFormat options)
+      maximumFractionDigits: 2,
+      style: 'percent',
+    },
+  },
+};
+
+/**
+ * Function to retrieve all 'argumentElement's from an ICU message. An argumentElement
+ * is an ICU element with an argument in it, like '{varName}' or '{varName, number, bytes}'. This
+ * differs from 'messageElement's which are just arbitrary text in a message.
+ *
+ * Notes:
+ *  This function will recursively inspect plural elements for nested argumentElements.
+ *
+ *  We need to find all the elements from the plural format sections, but
+ *  they need to be deduplicated. I.e. "=1{hello {icu}} =other{hello {icu}}"
+ *  the variable "icu" would appear twice if it wasn't de duplicated. And they cannot
+ *  be stored in a set because they are not equal since their locations are different,
+ *  thus they are stored via a Map keyed on the "id" which is the ICU varName.
+ *
+ * @param {Array<MessageElement>} icuElements
+ * @param {Map<string, ArgumentElement>} [seenElementsById]
+ * @return {Map<string, ArgumentElement>}
+ */
+function collectAllCustomElementsFromICU(icuElements, seenElementsById = new Map()) {
+  for (const el of icuElements) {
+    // We are only interested in elements that need ICU formatting (argumentElements)
+    if (el.type !== 'argumentElement') continue;
+
+    seenElementsById.set(el.id, el);
+
+    // Plurals need to be inspected recursively
+    if (!el.format || el.format.type !== 'pluralFormat') continue;
+    // Look at all options of the plural (=1{} =other{}...)
+    for (const option of el.format.options) {
+      // Run collections on each option's elements
+      collectAllCustomElementsFromICU(option.value.elements, seenElementsById);
+    }
+  }
+
+  return seenElementsById;
+}
+
+/**
+ * Returns a copy of the `values` object, with the values formatted based on how
+ * they will be used in their icuMessage, e.g. KB or milliseconds. The original
+ * object is unchanged.
+ * @param {MessageFormat} messageFormatter
+ * @param {Readonly<Record<string, string | number>>} values
+ * @param {string} lhlMessage Used for clear error logging.
+ * @return {Record<string, string | number>}
+ */
+function _preformatValues(messageFormatter, values, lhlMessage) {
+  const elementMap = collectAllCustomElementsFromICU(messageFormatter.getAst().elements);
+  const argumentElements = [...elementMap.values()];
+
+  /** @type {Record<string, string | number>} */
+  const formattedValues = {};
+
+  for (const {id, format} of argumentElements) {
+    // Throw an error if a message's value isn't provided
+    if (id && (id in values) === false) {
+      throw new Error(`ICU Message "${lhlMessage}" contains a value reference ("${id}") ` +
+        `that wasn't provided`);
+    }
+
+    const value = values[id];
+
+    // Direct `{id}` replacement and non-numeric values need no formatting.
+    if (!format || format.type !== 'numberFormat') {
+      formattedValues[id] = value;
+      continue;
+    }
+
+    if (typeof value !== 'number') {
+      throw new Error(`ICU Message "${lhlMessage}" contains a numeric reference ("${id}") ` +
+        'but provided value was not a number');
+    }
+
+    // Format values for known styles.
+    if (format.style === 'milliseconds') {
+      // Round all milliseconds to the nearest 10.
+      formattedValues[id] = Math.round(value / 10) * 10;
+    } else if (format.style === 'seconds' && id === 'timeInMs') {
+      // Convert all seconds to the correct unit (currently only for `timeInMs`).
+      formattedValues[id] = Math.round(value / 100) / 10;
+    } else if (format.style === 'bytes') {
+      // Replace all the bytes with KB.
+      formattedValues[id] = value / 1024;
+    } else {
+      // For all other number styles, the value isn't changed.
+      formattedValues[id] = value;
+    }
+  }
+
+  // Throw an error if a value is provided but has no placeholder in the message.
+  for (const valueId of Object.keys(values)) {
+    if (valueId in formattedValues) continue;
+
+    // errorCode is a special case always allowed to help LHError ease-of-use.
+    if (valueId === 'errorCode') {
+      formattedValues.errorCode = values.errorCode;
+      continue;
+    }
+
+    throw new Error(`Provided value "${valueId}" does not match any placeholder in ` +
+      `ICU message "${lhlMessage}"`);
+  }
+
+  return formattedValues;
+}
+
+/**
+ * Format string `message` by localizing `values` and inserting them. `message`
+ * is assumed to already be in the given locale.
+ * If you need to localize a messagem `getFormatted` is probably what you want.
+ * @param {string} message
+ * @param {Record<string, string | number>} values
+ * @param {LH.Locale} locale
+ * @return {string}
+ */
+function formatMessage(message, values = {}, locale) {
+  // When using accented english, force the use of a different locale for number formatting.
+  const localeForMessageFormat = (locale === 'en-XA' || locale === 'en-XL') ? 'de-DE' : locale;
+
+  const formatter = new MessageFormat(message, localeForMessageFormat, formats);
+
+  // Preformat values for the message format like KB and milliseconds.
+  const valuesForMessageFormat = _preformatValues(formatter, values, message);
+
+  return formatter.format(valuesForMessageFormat);
+}
+
+/**
+ * Retrieves the localized version of `icuMessage` and formats with any given
+ * value replacements.
+ * @param {LH.IcuMessage} icuMessage
+ * @param {LH.Locale} locale
+ * @return {string}
+ */
+function _localizeIcuMessage(icuMessage, locale) {
+  const localeMessages = _getLocaleMessages(locale);
+  const localeMessage = localeMessages[icuMessage.i18nId];
+
+  // Use the DEFAULT_LOCALE fallback (usually the original english message) if we couldn't
+  // find a message in the specified locale. Possible reasons:
+  //  - string drift between Lighthouse versions
+  //  - in a bundle stripped of locale files but running in the DEFAULT_LOCALE
+  //  - new strings haven't been updated yet in a local dev run
+  // Better to have an english message than no message at all; in some cases it
+  // won't even matter.
+  if (!localeMessage) {
+    return icuMessage.formattedDefault;
+  }
+
+  return formatMessage(localeMessage.message, icuMessage.values, locale);
+}
+
+/**
+ * @param {LH.Locale} locale
+ * @return {Record<string, string>}
+ */
+function getRendererFormattedStrings(locale) {
+  const localeMessages = _getLocaleMessages(locale);
+
+  // If `localeMessages` is empty in the bundled and DEFAULT_LOCALE case, this
+  // will be empty and the report will fall back to the util UIStrings for these.
+  const icuMessageIds = Object.keys(localeMessages).filter(f => f.startsWith('report/'));
+  /** @type {Record<string, string>} */
+  const strings = {};
+  for (const icuMessageId of icuMessageIds) {
+    const {filename, key} = getIcuMessageIdParts(icuMessageId);
+    if (!filename.endsWith('util.js')) throw new Error(`Unexpected message: ${icuMessageId}`);
+
+    strings[key] = localeMessages[icuMessageId].message;
+  }
+
+  return strings;
+}
+
+/**
+ * Returns whether `icuMessageOrNot`` is an `LH.IcuMessage` instance.
+ * @param {unknown} icuMessageOrNot
+ * @return {icuMessageOrNot is LH.IcuMessage}
+ */
+function isIcuMessage(icuMessageOrNot) {
+  if (!isObjectOfUnknownValues(icuMessageOrNot)) {
+    return false;
+  }
+
+  const {i18nId, values, formattedDefault} = icuMessageOrNot;
+  if (typeof i18nId !== 'string') {
+    return false;
+  }
+
+  // formattedDefault is required.
+  if (typeof formattedDefault !== 'string') {
+    return false;
+  }
+
+  // Values is optional.
+  if (values !== undefined) {
+    if (!isObjectOfUnknownValues(values)) {
+      return false;
+    }
+    for (const value of Object.values(values)) {
+      if (typeof value !== 'string' && typeof value !== 'number') {
+        return false;
+      }
+    }
+  }
+
+  // Finally return true if i18nId seems correct.
+  return MESSAGE_I18N_ID_REGEX.test(i18nId);
+}
+
+/**
+ * Get the localized and formatted form of `icuMessageOrRawString` if it's an
+ * LH.IcuMessage, or get it back directly if it's already a string.
+ * Warning: this function throws if `icuMessageOrRawString` is not the expected
+ * type (use function from `createIcuMessageFn` to create a valid LH.IcuMessage)
+ * or `locale` isn't supported (use `lookupLocale` to find a valid locale).
+ * @param {LH.IcuMessage | string} icuMessageOrRawString
+ * @param {LH.Locale} locale
+ * @return {string}
+ */
+function getFormatted(icuMessageOrRawString, locale) {
+  if (isIcuMessage(icuMessageOrRawString)) {
+    return _localizeIcuMessage(icuMessageOrRawString, locale);
+  }
+
+  if (typeof icuMessageOrRawString === 'string') {
+    return icuMessageOrRawString;
+  }
+
+  // Should be impossible from types, but do a strict check in case malformed JSON makes it this far.
+  throw new Error('Attempted to format invalid icuMessage type');
+}
+
+/** @param {string[]} pathInLHR */
+function _formatPathAsString(pathInLHR) {
+  let pathAsString = '';
+  for (const property of pathInLHR) {
+    if (/^[a-z]+$/i.test(property)) {
+      if (pathAsString.length) pathAsString += '.';
+      pathAsString += property;
+    } else {
+      if (/]|"|'|\s/.test(property)) throw new Error(`Cannot handle "${property}" in i18n`);
+      pathAsString += `[${property}]`;
+    }
+  }
+
+  return pathAsString;
+}
+
+/**
+ * Recursively walk the input object, looking for property values that are
+ * `LH.IcuMessage`s and replace them with their localized values. Primarily
+ * used with the full LHR or a Config as input.
+ * Returns a map of locations that were replaced to the `IcuMessage` that was at
+ * that location.
+ * @param {unknown} inputObject
+ * @param {LH.Locale} locale
+ * @return {LH.Result.IcuMessagePaths}
+ */
+function replaceIcuMessages(inputObject, locale) {
+  /**
+   * @param {unknown} subObject
+   * @param {LH.Result.IcuMessagePaths} icuMessagePaths
+   * @param {string[]} pathInLHR
+   */
+  function replaceInObject(subObject, icuMessagePaths, pathInLHR = []) {
+    if (!isObjectOrArrayOfUnknownValues(subObject)) return;
+
+    for (const [property, possibleIcuMessage] of Object.entries(subObject)) {
+      const currentPathInLHR = pathInLHR.concat([property]);
+
+      // Replace any IcuMessages with a localized string.
+      if (isIcuMessage(possibleIcuMessage)) {
+        const formattedString = getFormatted(possibleIcuMessage, locale);
+        const messageInstancesInLHR = icuMessagePaths[possibleIcuMessage.i18nId] || [];
+        const currentPathAsString = _formatPathAsString(currentPathInLHR);
+
+        messageInstancesInLHR.push(
+          possibleIcuMessage.values ?
+            {values: possibleIcuMessage.values, path: currentPathAsString} :
+            currentPathAsString
+        );
+
+        // @ts-ignore - tsc doesn't like that `property` can be either string key or array index.
+        subObject[property] = formattedString;
+        icuMessagePaths[possibleIcuMessage.i18nId] = messageInstancesInLHR;
+      } else {
+        replaceInObject(possibleIcuMessage, icuMessagePaths, currentPathInLHR);
+      }
+    }
+  }
+
+  /** @type {LH.Result.IcuMessagePaths} */
+  const icuMessagePaths = {};
+  replaceInObject(inputObject, icuMessagePaths);
+  return icuMessagePaths;
+}
+
+/**
+ * Returns the locale messages for the given `locale`, if they exist.
+ * Throws if an unsupported locale.
+ *
+ * NOTE: If DEFAULT_LOCALE is requested and this is inside a bundle with locale
+ * messages stripped, an empty object will be returned. Default fallbacks will need to handle that case.
+ * @param {LH.Locale} locale
+ * @return {import('./locales').LhlMessages}
+ */
+function _getLocaleMessages(locale) {
+  const localeMessages = LOCALE_MESSAGES[locale];
+  if (!localeMessages) {
+    if (locale === DEFAULT_LOCALE) {
+      // If the default locale isn't in LOCALE_MESSAGES, this is likely executing
+      // in a bundle. Let the caller use the fallbacks available.
+      return {};
+    }
+    throw new Error(`Unsupported locale '${locale}'`);
+  }
+
+  return localeMessages;
+}
+
+/**
+ * Returns whether the `requestedLocale` can be used.
+ * @param {LH.Locale} requestedLocale
+ * @return {boolean}
+ */
+function hasLocale(requestedLocale) {
+  // The default locale is always supported through `IcuMessage.formattedDefault`.
+  if (requestedLocale === DEFAULT_LOCALE) return true;
+
+  const hasIntlSupport = Intl.NumberFormat.supportedLocalesOf([requestedLocale]).length > 0;
+  const hasMessages = Boolean(LOCALE_MESSAGES[requestedLocale]);
+
+  return hasIntlSupport && hasMessages;
+}
+
+/**
+ * Returns a list of canonical locales, as defined by the existent message files.
+ * In practice, each of these may have aliases in the full list returned by
+ * `getAvailableLocales()`.
+ * TODO: create a CanonicalLocale type
+ * @return {Array<string>}
+ */
+function getCanonicalLocales() {
+  return CANONICAL_LOCALES;
+}
+
+/**
+ * Returns a list of available locales.
+ *  - if full build, this includes all canonical locales, aliases, and any locale added
+ *      via `registerLocaleData`.
+ *  - if bundled and locale messages have been stripped (locales.js shimmed), this includes
+ *      only DEFAULT_LOCALE and any locales from `registerLocaleData`.
+ * @return {Array<LH.Locale>}
+ */
+function getAvailableLocales() {
+  const localesWithMessages = new Set([...Object.keys(LOCALE_MESSAGES), DEFAULT_LOCALE]);
+  return /** @type {Array<LH.Locale>} */ ([...localesWithMessages].sort());
+}
+
+/**
+ * Populate the i18n string lookup dict with locale data
+ * Used when the host environment selects the locale and serves lighthouse the intended locale file
+ * @see https://docs.google.com/document/d/1jnt3BqKB-4q3AE94UWFA0Gqspx8Sd_jivlB7gQMlmfk/edit
+ * @param {LH.Locale} locale
+ * @param {import('./locales').LhlMessages} lhlMessages
+ */
+function registerLocaleData(locale, lhlMessages) {
+  LOCALE_MESSAGES[locale] = lhlMessages;
+}
+
+/**
+ * @param {string} i18nMessageId
+ */
+function getIcuMessageIdParts(i18nMessageId) {
+  if (!MESSAGE_I18N_ID_REGEX.test(i18nMessageId)) {
+    throw Error(`"${i18nMessageId}" does not appear to be a valid ICU message id`);
+  }
+  const [filename, key] = i18nMessageId.split(' | ');
+  return {filename, key};
+}
+
+module.exports = {
+  DEFAULT_LOCALE,
+  _formatPathAsString,
+  collectAllCustomElementsFromICU,
+  isIcuMessage,
+  getFormatted,
+  getRendererFormattedStrings,
+  replaceIcuMessages,
+  hasLocale,
+  registerLocaleData,
+  formatMessage,
+  getIcuMessageIdParts,
+  getAvailableLocales,
+  getCanonicalLocales,
+};

package/{lighthouse-core/lib/cdt/Common.js → shared/localization/i18n-module.js}

@@ -1,14 +1,9 @@
 /**
- * @license Copyright 2020 The Lighthouse Authors. All Rights Reserved.
+ * @license Copyright 2021 Google Inc. All Rights Reserved.
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 'use strict';
 
-/** @fileoverview Needed for the generated/SourceMap.js file */
-
-const Common = {
-  console,
-};
-
-module.exports = Common;
+module.exports.swapLocale = require('./swap-locale.js');
+module.exports.format = require('./format.js');